From 93d25e74868d62ddbdaec2e692f6b9b00b9f0915 Mon Sep 17 00:00:00 2001 From: od Date: Wed, 28 Aug 2024 21:02:26 -0600 Subject: [PATCH] updated exe name (compiler/dbg/rel), flags, install; better checks in scenario tests; cleanup --- CMakeLists.txt | 94 ++++++----- test/check.py | 197 ----------------------- test/spcheck.py | 349 +++++++++++++++++++++++++++++++++++++++++ test/swat_io_ndiff.py | 139 ----------------- test/swat_io_udiff.py | 356 ------------------------------------------ 5 files changed, 401 insertions(+), 734 deletions(-) delete mode 100644 test/check.py create mode 100755 test/spcheck.py delete mode 100644 test/swat_io_ndiff.py delete mode 100644 test/swat_io_udiff.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 745fb18..d3468e8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -27,69 +27,81 @@ if(NOT TAG) endif() endif() -string(TOLOWER ${CMAKE_HOST_SYSTEM_PROCESSOR} ARCH) -if (CMAKE_SYSTEM_NAME STREQUAL "Linux") - set(SWAT_OS "lin-${ARCH}") -elseif(CMAKE_SYSTEM_NAME STREQUAL "Windows") - set(SWAT_OS "win-${ARCH}") -elseif(CMAKE_SYSTEM_NAME STREQUAL "Darwin") - set(SWAT_OS "mac-${ARCH}") -else() - set(SWAT_OS "unknown") -endif () - -# SWAT Version number -set(SWAT_VERSION ${TAG}) -set(SWATPLUS_EXE "swatplus-${SWAT_VERSION}-${SWAT_OS}") - -# Enable this to 'TRUE' to see the fortran command on compile -# set(CMAKE_VERBOSE_MAKEFILE FALSE) if (UNIX) if(CMAKE_Fortran_COMPILER_ID STREQUAL Intel) - # set(fdialect "-fpe0 -free -diag-disable=10448") - set(fdialect "-fpe0 -free -traceback -warn all") - set(fdebug "-traceback -warn all") - set(frelease "-O3") + set(fdialect "-free -fpe0 -traceback -diag-disable=10448") + set(fdebug "-warn all") + set(frelease "-O") + set(FFC "ifo") link_libraries("-static") elseif(CMAKE_Fortran_COMPILER_ID STREQUAL IntelLLVM) - set(fdialect "-free") - set(fdebug "-fpe0 -traceback -warn all") - set(frelease "-O3") + set(fdialect "-free -fpe0 -traceback") + set(fdebug "-warn all -O0") + set(frelease "-O") + set(FFC "ifx") link_libraries("-static") elseif(CMAKE_Fortran_COMPILER_ID MATCHES GNU) - set(fdialect "-fcheck=all -Wall -ffpe-trap=invalid,zero,overflow,underflow -fimplicit-none -ffree-line-length-none -fbacktrace -finit-local-zero -fno-unsafe-math-optimizations -frounding-math -fsignaling-nans") - set(fdebug "") + set(fdialect "-fcheck=all -ffpe-trap=invalid,zero,overflow,underflow -fimplicit-none -ffree-line-length-none -fbacktrace -finit-local-zero -fno-unsafe-math-optimizations -frounding-math -fsignaling-nans") + set(fdebug "-Wall") set(frelease "-O") + set(FFC "gcc") if(NOT APPLE) link_libraries("-static") endif() endif() elseif(WIN32) if(CMAKE_Fortran_COMPILER_ID STREQUAL Intel) - set(fdialect "/free /Qdiag-disable:all /Qdiag-disable:remarks") - set(fdebug "/traceback /warn:all") - set(frelease "/O3") + set(fdialect "/free /fpe0 /traceback /Qdiag-disable:all /Qdiag-disable:remarks") + set(fdebug "/warn:all") + set(frelease "/O") + set(FFC "ifo") link_libraries("-static") elseif(CMAKE_Fortran_COMPILER_ID STREQUAL IntelLLVM) - set(fdialect "/free") - set(fdebug "/traceback /warn:all") - set(frelease "/O3") + set(fdialect "/free /fpe0 /traceback") + set(fdebug "/warn:all") + set(frelease "/O") + set(FFC "ifx") link_libraries("-static") elseif(CMAKE_Fortran_COMPILER_ID MATCHES GNU) - set(fdialect "-fcheck=all -Wall -ffpe-trap=invalid,zero,overflow,underflow -fimplicit-none -ffree-line-length-none -fbacktrace -finit-local-zero -fno-unsafe-math-optimizations -frounding-math -fsignaling-nans") - set(fdebug "") - set(frelease "-O") + set(fdialect "-fcheck=all -ffpe-trap=invalid,zero,overflow,underflow -fimplicit-none -ffree-line-length-none -fbacktrace -finit-local-zero -fno-unsafe-math-optimizations -frounding-math -fsignaling-nans") + set(fdebug "-Wall ") + set(FFC "gcc") + set(frelease "-O") endif() endif() +string(TOLOWER ${CMAKE_HOST_SYSTEM_PROCESSOR} ARCH) +if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + set(AR "${FFC}-lin_${ARCH}") +elseif(CMAKE_SYSTEM_NAME STREQUAL "Windows") + set(AR "${FFC}-win_${ARCH}") +elseif(CMAKE_SYSTEM_NAME STREQUAL "Darwin") + set(AR "${FFC}-mac_${ARCH}") +else() + set(AR "unknown") +endif() + +if(CMAKE_BUILD_TYPE STREQUAL "Release") + set(TY "-Rel") +elseif(CMAKE_BUILD_TYPE STREQUAL "Debug") + set(TY "-Dbg") +else() + set(TY "") +endif() + +# SWAT Version number +set(SWAT_VERSION ${TAG}) +set(SWATPLUS_EXE "swatplus-${SWAT_VERSION}-${AR}${TY}") + +# Enable this to 'TRUE' to see the fortran command on compile +set(CMAKE_VERBOSE_MAKEFILE FALSE) set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} ${fdialect}") set(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG} ${fdebug}") # set(CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE} ${frelease}") set(CMAKE_Fortran_FLAGS_RELEASE "${frelease}") - ############################################################################# # Build # list (SORT _variableNames) @@ -120,28 +132,26 @@ endif() file(GLOB sources src/*.f90) add_executable(${SWATPLUS_EXE} ${sources}) -# set_target_properties(${PROJECT} PROPERTIES VS_DEBUGGER_WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}\\Data") - ############################################################################# # Install -install(TARGETS ${SWATPLUS_EXE} DESTINATION RUNTIME DESTINATION .) +install(TARGETS ${SWATPLUS_EXE} DESTINATION RUNTIME DESTINATION ${PROJECT_SOURCE_DIR}) ############################################################################# # Testing find_package(Python3 REQUIRED) -set(check_py "${PROJECT_SOURCE_DIR}/test/check.py") +set(spcheck "${PROJECT_SOURCE_DIR}/test/spcheck.py") set(exe_path "${PROJECT_BINARY_DIR}/${SWATPLUS_EXE}") set(test_dir "${PROJECT_BINARY_DIR}/data") -set(ref_dir "${PROJECT_SOURCE_DIR}/data") +set(ref_dir "${PROJECT_SOURCE_DIR}/data") # error tolerances set(rerr "0.01") set(aerr "1e-8") +add_test(Ames_sub1 ${Python3_EXECUTABLE} ${spcheck} ctest ${exe_path} ${ref_dir}/Ames_sub1 ${test_dir} --abserr ${aerr} --relerr ${rerr}) # add_test(Ithaca_sub6 python3 ${check_py} ${exe_path} ${ref_dir}/Ithaca_sub6 ${test_dir} ${aerr} ${rerr}) -add_test(Ames_sub1 ${Python3_EXECUTABLE} ${check_py} ${exe_path} ${ref_dir}/Ames_sub1 ${test_dir} ${aerr} ${rerr}) ############################################################################# diff --git a/test/check.py b/test/check.py deleted file mode 100644 index 7be9062..0000000 --- a/test/check.py +++ /dev/null @@ -1,197 +0,0 @@ -# Author, fgeter@colostate.edu -# This is generic swat_plus test script. -# arg 1: absolute path to the swat executable. -# arg 2: absolute path to the reference scenario (golden) -# arg 3: absolute path to the test scenario base for swat+ execution -# -# Example: -# python check.py /sp1/build/swatplus.exe /sp1/data/Ames_sub1 /sp1/build/data -# -# swat outputs will be found in the test data folder + scenario, e.g. /sp1/build/data/Ames_sub1. -import os -import pathlib -import shutil -import sys -from subprocess import Popen, PIPE - - -# replace all non-numeric characters (not: 1-9, E, +, -, .) with spaces -def sanitize(a: str) -> str: - s = list(a) - for i in range(0, len(s)): - if (s[i].isdigit() - or (s[i] == '+' and s[i + 1].isdigit()) - or (s[i] == '-' and s[i + 1].isdigit()) - or s[i] == '.' - or (s[i].lower() == 'e' and s[i - 1].isdigit()) - or s[i] == '\n'): - continue - s[i] = ' ' - return "".join(s).strip() - - -# check if s is a float -def is_float(s: str) -> bool: - if s is None: - return False - try: - int(s) # skip comparisons if ints - return False - except ValueError: - try: - float(s) - return True # only compare floats - except ValueError: - return False - - -def pos_line(line1: str, linetok: [str]) -> str: - pos: int = 0 - a: [int] = [] - for t in linetok: - pos = line1.find(t, pos) - a.append(pos) - pos += len(t) - s: [str] = list(' ' * len(line1)) - for i, v in enumerate(a): - for j, d in enumerate(str(i)): - s[v + j] = d - return "".join(s) - - -# compare all corresponding floats in two lines, use an absolute and relative error -def compare_line(lineno: int, line1: str, line2: str, aerr: float, rerr: float) -> tuple: - l1 = sanitize(line1) - l2 = sanitize(line2) - - if len(l1) == 0 and len(l2) == 0: - return 0, 0.0, 0.0 - - l1arr = l1.split() - l2arr = l2.split() - - # only keep floats on the list - l1arr = [i for i in l1arr if is_float(i)] - l2arr = [i for i in l2arr if is_float(i)] - - if len(l1arr) != len(l2arr): - return -1, 0.0, 0.0 # danger - - err: int = 0 - first: bool = True - max_re: float = 0.0 - max_ae: float = 0.0 - for i, (t1, t2) in enumerate(zip(l1arr, l2arr)): - # print(t1, t2) - if is_float(t1) and is_float(t2): - f1 = float(t1) - f2 = float(t2) - if abs(f1 - f2) >= aerr + rerr * abs(f2): - if first: - print(f"\n {' ' * len(str(lineno))}Field # {pos_line(line1, l1arr)}") - print(f"Line {lineno}: (1) '{line1.rstrip()}'") - print(f" {' ' * len(str(lineno))} (2) '{line2.rstrip()}'") - first = False - if f2 != 0: - re = round((abs(f1 - f2) / f2), 5) - max_re = max(max_re, re) - else: - re = '' - ae = round(abs(f1 - f2), 5) - max_ae = max(max_ae, ae) - print(f" Field #{i}: {f1} (1) <-> {f2} (2) aerr: {ae}, rerr: {re}") - err += 1 - return err, max_ae, max_re - - -# fdiff: compare two files line by line, field by field, only process fields that are floats, ignore the rest -# assumptions: -# the files are ascii -# the files have the same overall structure -# the files have the same number of lines -# the corresponding lines have the same number of fields -# the corresponding lines might have numerical differences in their fields -# -# corresponding float fields of two lines are compared and produce an error if: -# abs(field1 - field2) >= aerr + rerr * abs(field2) -# aerr : absolute error, default 1e-8 -# rerr : relative error, default 1e-5 (.001 percent) -# -# args: the files to compare, absolute and relative error -# return: tuple (# of errors, max abs error, max rel error) -def fdiff(file1: any, file2: any, aerr: float = 1e-8, rerr: float = 1e-5) -> tuple: - errors: int = 0 - max_aerr: float = 0.0 - max_rerr: float = 0.0 - with open(file1, 'r') as f1, open(file2, 'r') as f2: - for lineno, (l1, l2) in enumerate(zip(f1, f2)): - if ("MODULAR" in l1) and ("MODULAR" in l2): - continue - err, max_a, max_r = compare_line(lineno, l1, l2, aerr, rerr) - errors += err - max_aerr = max(max_aerr, max_a) - max_rerr = max(max_rerr, max_r) - return errors, max_aerr, max_rerr - - -# --------- - -def copy_data(from_dir: str, to_dir: str) -> None: - shutil.copytree(from_dir, to_dir, dirs_exist_ok=True) - - -def run_swat(swat_model: str, wdir: str) -> int: - p = Popen([swat_model], cwd=wdir, stdin=PIPE, stdout=PIPE, stderr=PIPE) - stdout, stderr = p.communicate() - print(stdout.decode()) - print(stderr.decode()) - if p.returncode != 0: - print(f"SWAT+ exited with code: {p.returncode}") - exit(1) - return p.returncode - - -def check(dir1: str, dir2: str, aerr: float, rerr: float, *files: list[str]) -> int: - total_err: int = 0 - for file in files: - print(f"Processing '{file}' in {dir1} (1) and {dir2} (2):") - err, max_ae, max_re = fdiff(os.path.join(dir1, file), os.path.join(dir2, file), aerr=aerr, rerr=rerr) - total_err += err - print(f"\nResults for '{file}': {dir1} <-> {dir2}, #err = {err}, max aerr: {max_ae}, max rerr: {max_re}") - return total_err - - -def test(swat_model: str, test_data_dir: str, tmp_dir: str, aerr: float, rerr: float) -> int: - scenario_dir: str = os.path.join(tmp_dir, pathlib.PurePath(test_data_dir).name) - pathlib.Path(tmp_dir).mkdir(parents=True, exist_ok=True) - - # 1. copy data from data dir to scenario dir - copy_data(test_data_dir, scenario_dir) - - # 2. run swat+ in the scenario dir - run_swat(swat_model, scenario_dir) - - # 3. compare the selected output files - # files = ['hru_ls_aa.txt','mgt_out.txt', 'hru_totc.txt', 'basin_totc.txt', 'basin_wb_aa.txt'] - test_files: str = os.path.join(scenario_dir, '.testfiles.txt') - errors: int = 0 - if os.path.isfile(test_files): - with open(test_files) as f: - files = [line.strip() for line in f if line.strip() != '' and not line.strip().startswith('#')] - errors = check(test_data_dir, scenario_dir, aerr, rerr, *files) - - return errors - - -if __name__ == "__main__": - if len(sys.argv) != 6: - print(f"Usage: {sys.argv[0]} ") - exit(1) - - aerr: float = float(sys.argv[4]) # aerr: float = 1e-8 # absolute error. - rerr: float = float(sys.argv[5]) # rerr: float = 0.05 # 5 % relative error threshold. - - err: int = test(sys.argv[1], sys.argv[2], sys.argv[3], aerr, rerr) - if err > 0: - print(f'\nTotal: {err} differences with rerr of >= {rerr} and aerr >= {aerr}') - exit(1) diff --git a/test/spcheck.py b/test/spcheck.py new file mode 100755 index 0000000..c277ac5 --- /dev/null +++ b/test/spcheck.py @@ -0,0 +1,349 @@ +#!/usr/bin/env python3 + +# Generic SWAT+ checker, O. David, CSU, 2024 +# - runs through ctest with known scenarios +# - Allows to execute various executables against a scenario +# - Allows to compare scenario outputs for output deltas + +import os +import pathlib +import shutil +import sys +import argparse +import glob +from subprocess import Popen +from datetime import datetime + + + + +# replace all non-numeric characters (not: 1-9, E, +, -, .) with spaces +def sanitize(a: str) -> str: + s = list(a) + for i in range(0, len(s)): + if (s[i].isdigit() + or (s[i] == '+' and s[i + 1].isdigit()) + or (s[i] == '-' and s[i + 1].isdigit()) + or s[i] == '.' + or (s[i].lower() == 'e' and s[i - 1].isdigit()) + or s[i] == '\n'): + continue + s[i] = ' ' + return "".join(s).strip() + + +# check if s is a float +def is_float(s: str) -> bool: + if s is None: + return False + try: + int(s) # skip comparisons if ints + return False + except ValueError: + try: + float(s) + return True # only compare floats + except ValueError: + return False + + +def pos_line(line1: str, linetok: [str]) -> str: + pos: int = 0 + a: [int] = [] + for t in linetok: + pos = line1.find(t, pos) + a.append(pos) + pos += len(t) + s: [str] = list(' ' * len(line1)) + for i, v in enumerate(a): + for j, d in enumerate(str(i)): + s[v + j] = d + return "".join(s) + + +# compare all corresponding floats in two lines, use an absolute and relative error +def comp_line(lineno: int, line1: str, line2: str, aerr: float, rerr: float, status: str) -> tuple: + global print_status_line + + l1 = sanitize(line1) + l2 = sanitize(line2) + + if len(l1) == 0 and len(l2) == 0: + return 0, 0.0, 0.0 + + l1arr = l1.split() + l2arr = l2.split() + + # only keep floats on the list + l1arr = [i for i in l1arr if is_float(i)] + l2arr = [i for i in l2arr if is_float(i)] + + if len(l1arr) != len(l2arr): + return -1, 0.0, 0.0 # danger + + err: int = 0 + first: bool = True + max_re: float = 0.0 + max_ae: float = 0.0 + for i, (t1, t2) in enumerate(zip(l1arr, l2arr)): + # print(t1, t2) + if is_float(t1) and is_float(t2): + f1 = float(t1) + f2 = float(t2) + if abs(f1 - f2) >= aerr + rerr * abs(f2): + if first: + if print_status_line: + print(status) + print_status_line = False + print(f"\n {' ' * len(str(lineno))}Field # {pos_line(line1, l1arr)}") + print(f"Line {lineno}: (1) '{line1.rstrip()}'") + print(f" {' ' * len(str(lineno))} (2) '{line2.rstrip()}'") + first = False + if f2 != 0: + re = round((abs(f1 - f2) / f2), 5) + max_re = max(max_re, re) + else: + re = '' + ae = round(abs(f1 - f2), 5) + max_ae = max(max_ae, ae) + print(f" Field #{i}: {f1} (1) <-> {f2} (2) abserr: {ae}, relerr: {re}") + err += 1 + return err, max_ae, max_re + + +# fdiff: compare two files line by line, field by field, only process fields that are floats, ignore the rest +# assumptions: +# the files are ascii +# the files have the same overall structure +# the files have the same number of lines +# the corresponding lines have the same number of fields +# the corresponding lines might have numerical differences in their fields +# +# corresponding float fields of two lines are compared and produce an error if: +# abs(field1 - field2) >= aerr + rerr * abs(field2) +# aerr : absolute error, default 1e-8 +# rerr : relative error, default 1e-5 (.001 percent) +# +# args: the files to compare, absolute and relative error +# return: tuple (# of errors, max abs error, max rel error) +def cmp_file(file1: any, file2: any, aerr: float = 1e-8, rerr: float = 1e-5, status: str = None, nlines: int = -1, nerrorlines: int = -1) -> tuple: + errors: int = 0 + error_lines: int = 0 + max_aerr: float = 0.0 + max_rerr: float = 0.0 + if nlines == -1: + nlines = sys.maxsize + if nerrorlines == -1: + nerrorlines = sys.maxsize + with open(file1, 'r') as f1, open(file2, 'r') as f2: + for lineno, (l1, l2) in enumerate(zip(f1, f2)): + if ("MODULAR" in l1) and ("MODULAR" in l2): + continue + err, max_a, max_r = comp_line(lineno, l1, l2, aerr, rerr, status) + errors += err + max_aerr = max(max_aerr, max_a) + max_rerr = max(max_rerr, max_r) + + if err > 0: + error_lines += 1 + if lineno >= nlines: + break + if error_lines >= nerrorlines: + break + + return errors, max_aerr, max_rerr + + +def copy_data(from_dir: str, to_dir: str) -> None: + shutil.copytree(from_dir, to_dir, dirs_exist_ok=True) + + +def run_swat(swat_model: str, wdir: str) -> int: + if not os.path.exists(swat_model): + raise Exception(f'Model not found: {swat_model}') + p = Popen(executable=swat_model, args=[], cwd=wdir, stdout=sys.stdout, stderr=sys.stdout) + p.wait() + if p.returncode != 0: + print(f"\nSWAT+ exited with code: {p.returncode}") + else: + print(f"\nCreated new output in {wdir}") + return p.returncode + + +def comp_scenario(dir1: str, dir2: str, aerr: float, rerr: float, files: list[str], nlines: int, nerrorlines: int) -> int: + total_err: int = 0 + global print_status_line + for file in files: + status = f"Processing '{file}' in \n (1) {dir1} and \n (2) {dir2}" + print_status_line = True + err, max_ae, max_re = cmp_file(os.path.join(dir1, file), os.path.join(dir2, file), + aerr=aerr, rerr=rerr, status=status, nlines=nlines, nerrorlines=nerrorlines) + total_err += err + if err>0: + print(f"\nResults for '{file}': #err = {err}, max aerr: {max_ae}, max rerr: {max_re}\n\n####") + return total_err + + +def ctest_all(swat_model: str, test_data_dir: str, tmp_dir: str, aerr: float, rerr: float, nlines:int, nerrorlines:int) -> int: + scenario_dir: str = os.path.join(tmp_dir, pathlib.PurePath(test_data_dir).name) + pathlib.Path(tmp_dir).mkdir(parents=True, exist_ok=True) + + # 1. copy data from data dir to scenario dir + copy_data(test_data_dir, scenario_dir) + + # 2. run swat+ in the scenario dir + run_swat(swat_model, scenario_dir) + + # 3. compare the selected output files + # files = ['hru_ls_aa.txt','mgt_out.txt', 'hru_totc.txt', 'basin_totc.txt', 'basin_wb_aa.txt'] + test_files: str = os.path.join(scenario_dir, '.testfiles.txt') + errors: int = 0 + if os.path.isfile(test_files): + with open(test_files) as f: + files = [line.strip() for line in f if line.strip() != '' and not line.strip().startswith('#')] + errors = comp_scenario(test_data_dir, scenario_dir, aerr, rerr, files, nlines=nlines, nerrorlines=nerrorlines) + else: + raise Exception(f'Not found: \'{test_files}\', cannot compare output files.') + return errors + + +def get_next_run_number(dir: str) -> str: + files = os.listdir(dir) + if not files: + return '01' + else: + files = sorted(files) + last = files[-1] + n = last.split('#') + no = int(n[0]) + 1 + return f'{no:02d}' + + +def find_run(dir: str, no: int) -> str | None: + pref = f'{no:02d}' + f = glob.glob(f'{dir}/{pref}#*') + if f: + if f[0].endswith("-FAILED"): + raise Exception(f'Cannot compare a failed run: \'{f[0]}\'') + return f[0] + return None + + + +def run(args): + cwd = os.getcwd() + sce_src_path = os.path.join(cwd, data_dir, args.scenario) + if not os.path.exists(sce_src_path): + raise Exception(f'path not found: {sce_src_path}') + + sce_run_path = os.path.join(cwd, runs_dir, args.scenario) + exe_path = os.path.join(cwd, build_dir, args.executable) + + time = datetime.now().strftime('%Y-%m-%dT%H:%M:%S') + os.makedirs(sce_run_path, exist_ok=True) + next_no = get_next_run_number(sce_run_path) + + new_run = f'{next_no}#{time}#{args.executable}' + + run_path = os.path.join(sce_run_path, new_run) + print(f"new run: '{run_path}'") + + # create folder + os.makedirs(run_path, exist_ok=True) + + # copy scenario + copy_data(sce_src_path, run_path) + + # run swat + ret = run_swat(exe_path, run_path) + + if ret != 0: + os.rename(run_path, run_path + '-FAILED') + + +def cmp(args): + cwd = os.getcwd() + sce_src_path = os.path.join(cwd, data_dir, args.scenario) + sce_run_path = os.path.join(cwd, runs_dir, args.scenario) + + if not os.path.exists(sce_src_path): + raise Exception(f'path not found: {sce_src_path}') + if not os.path.exists(sce_run_path): + raise Exception(f'path not found: {sce_run_path}') + + a = find_run(sce_run_path, args.a) + b = find_run(sce_run_path, args.b) + + if a is None: + raise Exception(f'Invalid run number {args.a} for scenario {args.scenario}') + if b is None: + raise Exception(f'Invalid run number {args.b} for scenario {args.scenario}') + + test_files: str = os.path.join(sce_src_path, '.testfiles.txt') + if os.path.isfile(test_files): + with open(test_files) as f: + files = [line.strip() for line in f if line.strip() != '' and not line.strip().startswith('#')] + err = comp_scenario(a, b, args.abserr, args.relerr, files, args.nlines, args.nerrorlines) + if err > 0: + print(f'\n\nTotal: {err} differences with relerr of >= {args.relerr} and abserr >= {args.abserr}') + + +def ctest(args): + if not os.path.exists(args.model_path): + raise Exception(f'model not found: {args.model_path}') + if not os.path.isdir(args.data_dir): + raise Exception(f'data dir not found: {args.data_dir}') + + err: int = ctest_all(args.model_path, args.data_dir, args.tmp_dir, args.abserr, args.relerr, args.nlines, args.nerrorlines) + if err > 0: + print(f'\nTotal: {err} differences with relerr of >= {args.relerr} and abserr >= {args.abserr}') + sys.exit(1) + + +# 'run' with (scenario, exe) name 1_executablename_date +# 'compare' (scenario, no1, no2, aerr, rerr) + +build_dir = "build" +data_dir = "data" +runs_dir = "runs" + +# Not the best implementation, not thread safe. flag to print the comparison header +# when the first diff is recognized. +print_status_line = True + +if __name__ == "__main__": + parser = argparse.ArgumentParser("spcheck", description = 'Utilities for running SWAT+ scenarios') + subparsers = parser.add_subparsers() + + run_parser = subparsers.add_parser('run', help='run a scenario') + run_parser.add_argument("executable", type=str, help="name of the executable") + run_parser.add_argument("scenario", type=str, help="scenario name") + run_parser.set_defaults(func=run) + + cmp_parser = subparsers.add_parser('compare', help='compares two runs of the same scenarios') + cmp_parser.add_argument("scenario", type=str, help="scenario name") + cmp_parser.add_argument("a", type=int, help="first scenario run") + cmp_parser.add_argument("b", type=int, help="second scenario run") + cmp_parser.add_argument("--abserr", type=float, default=1e-8, help="absolute error, default: %(default)s") + cmp_parser.add_argument("--relerr", type=float, default=0.05, help="relative error, default: %(default)s") + cmp_parser.add_argument("--nlines", type=int, default=-1, help="max # of lines checked per file, default: %(default)s (=unlimited)") + cmp_parser.add_argument("--nerrorlines", type=int, default=-1, help="max # of error lines reported per file, default: %(default)s (=unlimited)") + cmp_parser.set_defaults(func=cmp) + + ctest_parser = subparsers.add_parser('ctest', help='compare new scenario against golden one') + ctest_parser.add_argument("model_path", type=str, help="scenario name") + ctest_parser.add_argument("data_dir", type=str, help="golden scenario") + ctest_parser.add_argument("tmp_dir", type=str, help="new scenario") + ctest_parser.add_argument("--abserr", type=float, default=1e-8, help="absolute error, default: %(default)s") + ctest_parser.add_argument("--relerr", type=float, default=0.05, help="relative error, default: %(default)s") + ctest_parser.add_argument("--nlines", type=int, default=-1, help="max # of lines checked per file, default: %(default)s (=unlimited)") + ctest_parser.add_argument("--nerrorlines", type=int, default=-1, help="max # of error lines reported per file, default: %(default)s (=unlimited)") + ctest_parser.set_defaults(func=ctest) + + if len(sys.argv) < 2: + parser.print_help() + sys.exit(0) + + args = parser.parse_args() + args.func(args) + diff --git a/test/swat_io_ndiff.py b/test/swat_io_ndiff.py deleted file mode 100644 index 4139c18..0000000 --- a/test/swat_io_ndiff.py +++ /dev/null @@ -1,139 +0,0 @@ -# Author fgeter@colostate.edu -# reference swat input/output folder and a current -# swat run input/output folder. -# This script outputs a diff file for each file where -# there is difference between the reference file and -# the current run file. If there is no difference, -# no diff output will be generated for that file. -# The diff files will be output to a swat root folder in -# a tmp directory. -# This script called as follows: -# python swat_io_diff.py full_path_to_tmp_folder_for_diff_output full_path_to_reference_folder_location full_path_to_current_run_folder_location -# -# IMPORTANT: The tmp folder contents will be removed automatically to remove any previous diff files. - -import difflib -import filecmp -import sys -import os -import glob - - -executable = os.path.basename(os.path.normpath(sys.argv[0])) -# if len(sys.argv) != 3: -# print(f"{executable} requires three arguments on the command line:") -# print("\tfull_path_to_tmp_folder_for_diff_output") -# print("\tfull_path_to_reference_folder_location") -# print("\tfull_path_to_current_run_folder_location") -# exit(1) - -# tmp_dir_location = os.path.normpath(sys.argv[1]) -# rf_location = os.path.normpath(sys.argv[2]) -# cf_location = os.path.normpath(sys.argv[3]) - -tmp_dir_location = os.path.normpath("/home/fgeter/csu-scripts/sp_cmake/sp/sp/tmp/") -rf_location = os.path.normpath("/home/fgeter/csu-scripts/sp_cmake/sp/sp/ref_data/Ithaca_sub6/") -cf_location = os.path.normpath("/home/fgeter/csu-scripts/sp_cmake/sp/sp/test_data/Ithaca_sub6/") - -diff_folder = os.path.join(tmp_dir_location, "diff_files") -sig_diff_folder = os.path.join(tmp_dir_location, "sig_diff_files") - -try: - os.makedirs(tmp_dir_location, mode=0o766, exist_ok=True) -except Exception as e: - print(f"Could create or access temporary file location {tmp_dir_location}. Python error is:") - print(e) - exit(1) - -if not os.path.exists(rf_location): - print(f"The full path to the reference folder {rf_location} either does not exist or the path has incorrect permissions.") - exit(1) - -if not os.path.exists(cf_location): - print(f"The full path to the current run folder {cf_location} either does not exist or has incorrect permissions.") - exit(1) - -try: - os.makedirs(diff_folder, mode=0o766, exist_ok=True) -except Exception as e: - print(f"Could create or access diff file location {diff_folder}. Python error is:") - print(e) - exit(1) - -try: - os.makedirs(sig_diff_folder, mode=0o766, exist_ok=True) -except Exception as e: - print(f"Could create or access significant diff file location {sig_diff_folder}. Python error is:") - print(e) - exit(1) - -df_folder_files = glob.glob(os.path.join(diff_folder, "*")) -for file in df_folder_files: - try: - os.remove(file) - except Exception as e: - print(f"Warning: Could not remove old diff file {file} in {diff_folder}. Python error is:") - print(e) - continue - -sig_diff_folder_files = glob.glob(os.path.join(sig_diff_folder, "*")) -for file in sig_diff_folder_files: - try: - os.remove(file) - except Exception as e: - print(f"Warning: Could not remove old signifcant diff file {file} in {sig_diff_folder}. Python error is:") - print(e) - continue - -filecmp.clear_cache() -dir_list = os.listdir(cf_location) -cmp_result = filecmp.cmpfiles(rf_location, cf_location, dir_list, shallow=True)[1] -for file in cmp_result: - - full_path1 = os.path.join(rf_location, file) - if os.path.isdir(full_path1): - continue - f1_object = open(full_path1, "r") - file1_contents = f1_object.readlines() - - full_path2 = os.path.join(cf_location, file) - f2_object = open(full_path2, "r") - file2_contents = f2_object.readlines() - print(full_path1, full_path2) - - # diff = difflib.ndiff(file1_contents, file2_contents) - diff = difflib.unified_diff(file1_contents, file2_contents, n=0) - output_filename = file + "_diff" - output_full_path = os.path.join(diff_folder, output_filename) - print(output_full_path) - output_file_object = open(output_full_path, "w") - for line in diff: - output_file_object.write(line) - output_file_object.close() - - - - - # print(l) - # print() - -# fname1 = "reference_file.txt" -# fname2 = "new_file.txt" -# if not filecmp.cmp(fname1, fname2): -# f1 = open(fname1, "r") -# file1 = f1.readlines() -# f2 = open(fname2, "r") -# file2 = f2.readlines() -# diff = difflib.ndiff(file1, file2) -# output_filename = fname2.split(".")[0] + "_diff.txt" -# output_file_object = open(output_filename, "w") -# for line in diff: -# # line = line.replace("\n", "") -# # print(line) -# output_file_object.write(line) -# f1.close() -# f2.close() -# output_file_object.close() -# print(f"Output file {fname2} has changed. See {output_filename} for the changes.") - - diff --git a/test/swat_io_udiff.py b/test/swat_io_udiff.py deleted file mode 100644 index d4adec6..0000000 --- a/test/swat_io_udiff.py +++ /dev/null @@ -1,356 +0,0 @@ -# Author fgeter@colostate.edu -# This script reads swat input and output files and outputs -# two diff files for each file where the difference is between -# a swat output reference file and a current corresponding swat output file. -# If there is no difference between files, no diff outputs will -# be generated for that file. -# -# If there is a difference between the files, then only lines -# where the difference occured will be output in the diff files. -# The diff files will be output to an abosulte path given as the first -# argument on the command line. If this path does not exist, then ths -# script will attempt to create it. In addition, a subdirectory called -# "diff_files" will be created under this absolute path to store the diff files. -# There are two types of diff files that are output. The first is -# is a the raw output from the python function difflib.unified_diff. -# The second is generated by processing the difflib.unified_diff output -# to show on each line the line number in the original files where there is a -# difference along with the actual line itself followed by a line showing -# where in the line changes have occurred. The change is indicated be the -# the character "^". If this line begins with "dc", then there appears to be -# only changes in data in the swat output columns. -# If it begins with "fc", it appears there is a swat -# output format change between the current and reference file. The "fc" is -# output if the number of swat columns for the line in question appears to have -# changed. -# -# This script is called as follows: -# python swat_io_udiff.py [full_path_to_tmp_folder_for_diff_output] [full_path_to_reference_folder_location] [full_path_to_current_run_folder_location] -# If the reference path and current path are a file and not a folder, -# then only that file will be processed. -# -# IMPORTANT: If a file to be compared is specified on the command line, the corresponding diff -# file will be deleted before a new one is created of the same name. -# If a folder to be compared is specified on the command line, -# all the diff files in the corresponding diff folder will be -# deleted before new ones are created. - - -import difflib -import filecmp -import sys -import os -import glob -import subprocess -import shutil - -from datetime import datetime - - -def create_test_io_folders(): - test_build_folder = os.path.normpath(os.path.abspath(sys.argv[1])) - swat_exe = sys.argv[2] - reference_data_folder = os.path.normpath(os.path.abspath(sys.argv[3])) - # test_build_folder = os.path.normpath("/home/fgeter/csu-scripts/sp1/build/") - # swat_exe = "swatplus_60_5_7.exe" - # reference_data_folder = os.path.normpath("/home/fgeter/csu-scripts/sp1/data/Ithaca_sub6/") - if not os.path.exists(test_build_folder): - print(f"The full path to build folder {test_build_folder} either") - print("does not exist or the path has incorrect permissions.") - exit(1) - - if not os.path.exists(reference_data_folder): - print(f"The full path to reference data folder {reference_data_folder} either") - print("does not exist or the path has incorrect permissions.") - exit(1) - - if os.path.isdir(reference_data_folder) == False: - print(f"The full path to reference data folder {reference_data_folder} is ") - print("is a file and not folder. It must be folder.") - exit(1) - - # Create the test data folder if it does not exist - test_data_folder = os.path.join(test_build_folder, "data") - try: - os.makedirs(test_data_folder, mode=0o766, exist_ok=True) - except Exception as e: - print(f"Could not create test data folder {test_data_folder}.") - print(f"Python error is:\n{e}") - exit(1) - - # Open log file in test data folder - log_filename = os.path.basename(reference_data_folder) + "_diff.log" - log_path_name = os.path.join(test_data_folder, log_filename) - try: - log = open(log_path_name, "w") - except Exception as e: - print(f"Could not create log file. Python error is\n{e}") - exit(1) - now = datetime.now() - log.write(f"Start of test processing at {now}\n\n") - print(f"See log file at {log_filename} for test processing info.") - return test_build_folder, swat_exe, reference_data_folder, test_data_folder, log - -def copy_data_folder(reference_data_folder, test_data_folder, log): - log.write("Copying reference data to test data folder.\n") - foldername = os.path.basename(reference_data_folder) - swat_data_folder = os.path.join(test_data_folder, foldername) - if os.path.exists(swat_data_folder): - log.write(f"Removing previous test run folder {swat_data_folder}.\n") - try: - shutil.rmtree(swat_data_folder) - except Exception as e: - message = f"Could not remove old swat test data folder {test_data_folder}"\ - + f"Python errer is :\n{e}" - print(message) - log.write(f"{message}\n") - try: - shutil.copytree(reference_data_folder, swat_data_folder) - except Exception as e: - message = f"Could not copy test data folder {reference_data_folder} to {test_data_folder}"\ - + f"Python errer is :\n{e}" - print(message) - log.write(f"{message}\n") - exit(1) - return swat_data_folder - - -def swat_run_check(test_build_folder, swat_exe, swat_data_folder, log): - log.write("Executing swatplus.\n") - swatplus_executable = os.path.join(test_build_folder, swat_exe) - os.chdir(swat_data_folder) - result = subprocess.check_call([swatplus_executable]) - if result == 0: - message = f"swatplus execution succeded." - log.write(f"{message}\n") - else: - message = f"swatplus execution failed." - log.write(f"{message}\n") - exit(1) - return - - -def create_diff_folder(swat_data_folder, log): - diff_folder_name = os.path.basename(swat_data_folder) + "_diff_files" - diff_folder_path = os.path.dirname(swat_data_folder) - diff_folder_path = os.path.join(diff_folder_path, diff_folder_name) - if os.path.exists(diff_folder_path): - log.write(f"Attempting to removing previous diff folder {diff_folder_path}.\n") - try: - shutil.rmtree(diff_folder_path) - except Exception as e: - message = f"Could not remove previous diff folder {diff_folder_path}." +\ - + f"Python errer is :\n{e}" - print(message) - log.write(f"{message}\n") - exit(1) - log.write(f"Removed previous diff folder {diff_folder_path}.\n") - log.write("Creating diff folder.\n") - try: - os.makedirs(diff_folder_path, mode=0o766, exist_ok=True) - except Exception as e: - message = f"Could not create test data folder {diff_folder_path}." + \ - f"Python error is:\n{e}" - log.write(f"{message}\n") - exit(1) - log.write(f"Created diff folder {diff_folder_path}.\n") - return diff_folder_path - - - -def file_comparison_list(reference_data_folder, swat_data_folder, diff_folder_path, log): - # Make a list tuples where each tuple has four items: - # 1. The full path to the reference swat output file to be compared to. - # 2. The full path to the current swat output file to be compared with the reference file. - # 3. The full path to the location to store the output of of the unified_diff. - # 4. The full path to the location to store the post process output of unified_diff output. - - log.write("Start of creating a list of files to compare and the names of their diff output files\n") - cf_isfile = False - cf_isdir = False - compare_files = [] - - # 1st case, if a file (not a folder) is specified on the command line to compared. - if os.path.isfile(swat_data_folder): - cf_file = os.path.basename(swat_data_folder) - rf_file = os.path.basename(reference_data_folder) - diff_file1 = cf_file + "_diff1" - diff_file2 = rf_file + "_diff2" - diff_path1 = os.path.join(diff_folder_path, diff_file1) - diff_path2 = os.path.join(diff_folder_path, diff_file2) - cf_path =swat_data_folder - rf_path =reference_data_folder - item = (rf_path, cf_path, diff_path1, diff_path2) - log.write(f"Adding {os.path.basename(item[1])}.\n") - compare_files = compare_files + [item] - # 2nd case, if a folder is specified on the command line to compared. - else: - for dirName, subdirList, fileList in os.walk(swat_data_folder): - for fname in fileList: - # if the folder is not subdirectory of the folder - # specified on the commandline. - if dirName == swat_data_folder: - cf_path = os.path.join(dirName, fname) - rf_path = os.path.join(dirName.replace(dirName, reference_data_folder), fname) - diff_path1 = os.path.join(diff_folder_path, fname+"_diff1") - diff_path2 = os.path.join(diff_folder_path, fname+"_diff2") - item = (rf_path, cf_path, diff_path1, diff_path2) - log.write(f"Adding {os.path.basename(item[1])}.\n") - compare_files = compare_files + [item] - # if the folder is a subdirectory of the folder - # specified on the commandline. - else: - subdir = os.path.basename(dirName.replace(swat_data_folder, "")) - cf_path = os.path.join(swat_data_folder, subdir, fname) - rf_path = os.path.join(reference_data_folder, subdir, fname) - diff_path1 = os.path.join(diff_folder_path, subdir, fname+"_diff1") - diff_path2 = os.path.join(diff_folder_path, subdir, fname+"_diff2") - item = (rf_path, cf_path, diff_path1, diff_path2) - log.write(f"Adding {os.path.basename(item[1])}.\n") - compare_files = compare_files + [item] - log.write("End of creating a list of files to compare and there diff output files\n\n") - return(compare_files) - - -def is_binary(file_name): - # This is a function to check and see if the file is a binary file. - # Returns False if it is a binary file and True if it is not a binary file. - try: - with open(file_name, 'tr') as check_file: # try open file in text mode - check_file.read() - return False - except: # if fail then file is non-text (binary) - return True - - -def create_diff_files(reference_data_folder, swat_data_folder, compare_files, log): - log.write("Start of creating diff files\n") - # Create the diff files - files_with_changed_format = [] - for item in compare_files: - rf_file_path = item[0] - cf_file_path = item[1] - if is_binary(cf_file_path) or is_binary(rf_file_path): - log.write(f"\nEither or both of the files:\n") - log.write(f"\t{rf_file_path}\n") - log.write(f"\t{cf_file_path}\n") - log.write("appear to be binary files. Skipping this file comparison.\n\n") - continue - if filecmp.cmp(rf_file_path, cf_file_path) == True: - log.write(f"No difference between files {rf_file_path} and {cf_file_path}\n") - else: - rf_file_object = open(rf_file_path, "r") - rf_contents = rf_file_object.readlines() - cf_file_object = open(cf_file_path, "r") - cf_contents = cf_file_object.readlines() - log.write(f"Doing a unified diff between: {rf_file_path} and {cf_file_path}\n") - udiff = difflib.unified_diff(rf_contents, cf_contents, n=0, fromfile=reference_data_folder, tofile=swat_data_folder) - diff1_filename = item[2] - diff2_filename = item[3] - if os.path.exists(os.path.dirname(diff1_filename)) == False: - os.makedirs(os.path.dirname(diff1_filename)) - diff1_output_file_object = open(diff1_filename, "w") - - for line in udiff: - diff1_output_file_object.write(line) - diff1_output_file_object.close() - - diff1_output_file_object = open(diff1_filename, "r") - diff1_contents = diff1_output_file_object.readlines() - diff2_output_file_object = open(diff2_filename, "w") - log.write(f"Processing unified diff output for : {diff1_filename} and {diff2_filename}\n") - line_dict = {} - file_format_change = False - for line in diff1_contents: - if line[0:3] in ["---", "+++"]: - diff2_output_file_object.write(line) - continue - if line[0:2] == "@@": - line_list= line.split() - delete_info = line_list[1].split(",") - delete_line = -int(delete_info[0]) - if len(delete_info) == 1: - delete_num = 1 - else: - delete_num = int(delete_info[1]) - add_info = line_list[2].split(",") - add_line = int(add_info[0]) - if len(add_info) == 1: - add_num = 1 - else: - add_num = int(add_info[1]) - elif line[0] == "-": - line_out = str(delete_line) + " " + line - line_out_key = str(delete_line).zfill(12) + "_0" - line_dict[line_out_key] = line_out - delete_line = delete_line + 1 - delete_num = delete_num -1 - if delete_num == 0: - continue - elif line[0] == "+": - line_out = str(add_line) + " " + line - line_out_key = str(add_line).zfill(12) + "_1" - line_dict[line_out_key] = line_out - add_line = add_line + 1 - add_num = add_num - 1 - if add_num == 0: - continue - output_keys = list(line_dict.keys()) - same_line_list = [] - for x in sorted(output_keys): - cur_line = line_dict[x] - diff2_output_file_object.write(cur_line) - if len(same_line_list) < 2: - same_line_list.extend([line_dict[x]]) - if len(same_line_list) == 2: - l1 = same_line_list[0] - l2 = same_line_list[1] - format_change = False - if len(l1.split()) != len(l2.split()): - format_change = True - file_format_change = True - len1 = len(l1) - len2 = len(l2) - maxl = max(len1, len2) - change_line = "" - for c in range(0,maxl): - if c < len1 and c < len2: - if l1[c] == l2[c] or (l1[c] == "-" and l2[c] == "+"): - change_line = change_line + " " - else: - change_line = change_line + "^" - else: - change_line = change_line + "^" - if format_change: - change_line = change_line.replace(" ", "fc", 1) - else: - change_line = change_line.replace(" ", "dc", 1) - diff2_output_file_object.write(change_line + "\n") - same_line_list = [] - if file_format_change: - files_with_changed_format.extend([cf_file_path]) - if len(files_with_changed_format) > 0: - log.write("\nWarning: There are files that appear to have changed output format:\n") - for f in files_with_changed_format: - log.write(f"\t{f}\n") - log.write("End of creating diff files\n\n") - return(files_with_changed_format) - - -def run(): - test_build_folder, swat_exe, reference_data_folder, test_data_folder, log = create_test_io_folders() - swat_data_folder = copy_data_folder(reference_data_folder, test_data_folder, log) - swat_run_check(test_build_folder, swat_exe, swat_data_folder, log) - diff_folder_path = create_diff_folder(swat_data_folder, log) - # reference_data_folder, swat_data_folder, diff_folder, log = file_folder_prep() - comp_files = file_comparison_list(reference_data_folder, swat_data_folder, diff_folder_path, log) - # delete_old_files(reference_data_folder, comp_files, log) - files_with_changed_format = create_diff_files(reference_data_folder, swat_data_folder, comp_files, log) - now = datetime.now() - log.write(f"End of diff processing at {now}\n\n") - log.close() - return files_with_changed_format - - -if __name__ == "__main__": - files_with_changed_format = run()