diff --git a/Buildings/Resources/Scripts/travis/templates/BoilerPlant.py b/Buildings/Resources/Scripts/travis/templates/BoilerPlant.py index 177881df49d..c50385de3ea 100755 --- a/Buildings/Resources/Scripts/travis/templates/BoilerPlant.py +++ b/Buildings/Resources/Scripts/travis/templates/BoilerPlant.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # coding: utf-8 +# Requires Python >= 3.9 + # This script shall be run from the directory `modelica-buildings/Buildings`, # i.e., where the top-level `package.mo` file can be found. # The script takes as an optional positional argument the Modelica tool to use @@ -20,129 +22,118 @@ # - 0 if all simulations succeed, # - 1 otherwise. -import inspect -import itertools -import os -import re -import sys -# For CPU- and I/O-heavy jobs, we prefer multiprocessing.Pool because it provides better process isolation. -from multiprocessing import Pool - -import pandas as pd +from core import * try: SIMULATOR = sys.argv[1] except IndexError: SIMULATOR = 'Dymola' -MODELS = [ +MODELS: list[str] = [ 'Buildings.Templates.HeatingPlants.HotWater.Validation.BoilerPlant', ] -CRED = '\033[91m' -CGREEN = '\033[92m' -CEND = '\033[0m' - -MODIF_GRID = { - 'Buildings.Templates.HeatingPlants.HotWater.Validation.BoilerPlant': - dict( - BOI__typ=[ - 'Buildings.Templates.HeatingPlants.HotWater.Types.Boiler.Condensing', - 'Buildings.Templates.HeatingPlants.HotWater.Types.Boiler.NonCondensing', - 'Buildings.Templates.HeatingPlants.HotWater.Types.Boiler.Hybrid', - ], - BOI__nBoiCon_select=[ - '2', - ], - BOI__nBoiNon_select=[ - '2', - ], - BOI__typPumHeaWatPriCon=[ - 'Buildings.Templates.HeatingPlants.HotWater.Types.PumpsPrimary.Variable', - 'Buildings.Templates.HeatingPlants.HotWater.Types.PumpsPrimary.Constant', - 'Buildings.Templates.HeatingPlants.HotWater.Types.PumpsPrimary.FactoryVariable', - 'Buildings.Templates.HeatingPlants.HotWater.Types.PumpsPrimary.FactoryConstant', - ], - BOI__typPumHeaWatPriNon=[ - 'Buildings.Templates.HeatingPlants.HotWater.Types.PumpsPrimary.Variable', - 'Buildings.Templates.HeatingPlants.HotWater.Types.PumpsPrimary.Constant', - 'Buildings.Templates.HeatingPlants.HotWater.Types.PumpsPrimary.FactoryVariable', - 'Buildings.Templates.HeatingPlants.HotWater.Types.PumpsPrimary.FactoryConstant', - ], - BOI__typArrPumHeaWatPriCon_select=[ - 'Buildings.Templates.Components.Types.PumpArrangement.Dedicated', - 'Buildings.Templates.Components.Types.PumpArrangement.Headered', - ], - BOI__typArrPumHeaWatPriNon_select=[ - 'Buildings.Templates.Components.Types.PumpArrangement.Dedicated', - 'Buildings.Templates.Components.Types.PumpArrangement.Headered', - ], - BOI__typPumHeaWatSec1_select=[ - 'Buildings.Templates.HeatingPlants.HotWater.Types.PumpsSecondary.Centralized', - 'Buildings.Templates.HeatingPlants.HotWater.Types.PumpsSecondary.None', - ], - BOI__typPumHeaWatSec2_select=[ - 'Buildings.Templates.HeatingPlants.HotWater.Types.PumpsSecondary.Centralized', - ], - BOI__ctl__typMeaCtlHeaWatPri=[ - 'Buildings.Templates.HeatingPlants.HotWater.Types.PrimaryOverflowMeasurement.FlowDecoupler', - 'Buildings.Templates.HeatingPlants.HotWater.Types.PrimaryOverflowMeasurement.FlowDifference', - 'Buildings.Templates.HeatingPlants.HotWater.Types.PrimaryOverflowMeasurement.TemperatureSupplySensor', - 'Buildings.Templates.HeatingPlants.HotWater.Types.PrimaryOverflowMeasurement.TemperatureBoilerSensor', - ], - BOI__ctl__have_senDpHeaWatLoc=[ - 'true', - 'false', - ], - BOI__ctl__locSenVHeaWatPri=[ - 'Buildings.Templates.HeatingPlants.HotWater.Types.SensorLocation.Supply', - 'Buildings.Templates.HeatingPlants.HotWater.Types.SensorLocation.Return', - ], - BOI__ctl__locSenVHeaWatSec=[ - 'Buildings.Templates.HeatingPlants.HotWater.Types.SensorLocation.Supply', - 'Buildings.Templates.HeatingPlants.HotWater.Types.SensorLocation.Return', - ], - ), +MODIF_GRID: dict[str, dict[str, list[str]]] = { + 'Buildings.Templates.HeatingPlants.HotWater.Validation.BoilerPlant': dict( + BOI__typ=[ + 'Buildings.Templates.HeatingPlants.HotWater.Types.Boiler.Condensing', + 'Buildings.Templates.HeatingPlants.HotWater.Types.Boiler.NonCondensing', + 'Buildings.Templates.HeatingPlants.HotWater.Types.Boiler.Hybrid', + ], + BOI__nBoiCon_select=[ + '2', + ], + BOI__nBoiNon_select=[ + '2', + ], + BOI__typPumHeaWatPriCon=[ + 'Buildings.Templates.HeatingPlants.HotWater.Types.PumpsPrimary.Variable', + 'Buildings.Templates.HeatingPlants.HotWater.Types.PumpsPrimary.Constant', + 'Buildings.Templates.HeatingPlants.HotWater.Types.PumpsPrimary.FactoryVariable', + 'Buildings.Templates.HeatingPlants.HotWater.Types.PumpsPrimary.FactoryConstant', + ], + BOI__typPumHeaWatPriNon=[ + 'Buildings.Templates.HeatingPlants.HotWater.Types.PumpsPrimary.Variable', + 'Buildings.Templates.HeatingPlants.HotWater.Types.PumpsPrimary.Constant', + 'Buildings.Templates.HeatingPlants.HotWater.Types.PumpsPrimary.FactoryVariable', + 'Buildings.Templates.HeatingPlants.HotWater.Types.PumpsPrimary.FactoryConstant', + ], + BOI__typArrPumHeaWatPriCon_select=[ + 'Buildings.Templates.Components.Types.PumpArrangement.Dedicated', + 'Buildings.Templates.Components.Types.PumpArrangement.Headered', + ], + BOI__typArrPumHeaWatPriNon_select=[ + 'Buildings.Templates.Components.Types.PumpArrangement.Dedicated', + 'Buildings.Templates.Components.Types.PumpArrangement.Headered', + ], + BOI__typPumHeaWatSec1_select=[ + 'Buildings.Templates.HeatingPlants.HotWater.Types.PumpsSecondary.Centralized', + 'Buildings.Templates.HeatingPlants.HotWater.Types.PumpsSecondary.None', + ], + BOI__typPumHeaWatSec2_select=[ + 'Buildings.Templates.HeatingPlants.HotWater.Types.PumpsSecondary.Centralized', + ], + BOI__ctl__typMeaCtlHeaWatPri=[ + 'Buildings.Templates.HeatingPlants.HotWater.Types.PrimaryOverflowMeasurement.FlowDecoupler', + 'Buildings.Templates.HeatingPlants.HotWater.Types.PrimaryOverflowMeasurement.FlowDifference', + 'Buildings.Templates.HeatingPlants.HotWater.Types.PrimaryOverflowMeasurement.TemperatureSupplySensor', + 'Buildings.Templates.HeatingPlants.HotWater.Types.PrimaryOverflowMeasurement.TemperatureBoilerSensor', + ], + BOI__ctl__have_senDpHeaWatLoc=[ + 'true', + 'false', + ], + BOI__ctl__locSenVHeaWatPri=[ + 'Buildings.Templates.HeatingPlants.HotWater.Types.SensorLocation.Supply', + 'Buildings.Templates.HeatingPlants.HotWater.Types.SensorLocation.Return', + ], + BOI__ctl__locSenVHeaWatSec=[ + 'Buildings.Templates.HeatingPlants.HotWater.Types.SensorLocation.Supply', + 'Buildings.Templates.HeatingPlants.HotWater.Types.SensorLocation.Return', + ], + ), } -# A case is excluded if the "exclusion test" returns true for any of the items from EXCLUDE. +# A case is excluded if the "exclusion test" returns true for any of the item value from EXCLUDE. # Exclusion test: # - concatenate all class modifications, -# - return true if all strings from the item of EXCLUDE are found in the concatenation product. +# - return true if all strings from the item value of EXCLUDE are found in the concatenation product. # - (re patterns are supported: for instance negative lookahead assertion using (?!pattern).) -EXCLUDE = { - 'Buildings.Templates.HeatingPlants.HotWater.Validation.BoilerPlant': [[ - 'Buildings.Templates.HeatingPlants.HotWater.Types.Boiler.Hybrid', - 'Buildings.Templates.HeatingPlants.HotWater.Types.PumpsSecondary.None', - ],], +EXCLUDE: dict[str, list[list[str]]] = { + 'Buildings.Templates.HeatingPlants.HotWater.Validation.BoilerPlant': [ + [ + 'Buildings.Templates.HeatingPlants.HotWater.Types.Boiler.Hybrid', + 'Buildings.Templates.HeatingPlants.HotWater.Types.PumpsSecondary.None', + ], + ], } # Class modifications are removed for each model to be simulated according to the following rules. -# For each item of the list provided for each model in REMOVE_MODIF: +# For each item (2-tuple) of the list provided (as value) for each model (key) in REMOVE_MODIF (dict): # - if all patterns of item[0] are found in the original class modifications, and # - if a class modification contains any item within item[1], then it is removed. # - (re patterns are supported: for instance negative lookahead assertion using (?!pattern).) -# Removing the class modifications this way yields duplicate sets of class modifications. +# Removing the class modifications this way yields many duplicate sets of class modifications. # Those are pruned afterwards. -REMOVE_MODIF = { +REMOVE_MODIF: dict[str, list[tuple[list[str], list[str]]]] = { 'Buildings.Templates.HeatingPlants.HotWater.Validation.BoilerPlant': [ - [ + ( [ 'typ=Buildings.Templates.HeatingPlants.HotWater.Types.Boiler.Hybrid', ], [ 'typPumHeaWatSec1_select', ], - ], - [ + ), + ( [ 'typ=Buildings.Templates.HeatingPlants.HotWater.Types.Boiler.(?!Hybrid)', ], [ 'typPumHeaWatSec2_select', ], - ], - [ + ), + ( [ 'typ=Buildings.Templates.HeatingPlants.HotWater.Types.Boiler.(?!Hybrid)', 'typPumHeaWatSec1_select=Buildings.Templates.HeatingPlants.HotWater.Types.PumpsSecondary.None', @@ -151,8 +142,8 @@ 'typMeaCtlHeaWatPri', 'locSenVHeaWatSec', ], - ], - [ + ), + ( [ 'typPumHeaWatSec(1|2)_select=Buildings.Templates.HeatingPlants.HotWater.Types.PumpsSecondary.(?!None)', 'typMeaCtlHeaWatPri=Buildings.Templates.HeatingPlants.HotWater.Types.PrimaryOverflowMeasurement.(?!FlowDifference)', @@ -160,8 +151,8 @@ [ 'locSenVHeaWatSec', ], - ], - [ + ), + ( [ 'typ=Buildings.Templates.HeatingPlants.HotWater.Types.Boiler.Condensing', 'typPumHeaWatSec1_select=Buildings.Templates.HeatingPlants.HotWater.Types.PumpsSecondary.(?!None)', @@ -171,8 +162,8 @@ 'typMeaCtlHeaWatPri', 'locSenVHeaWatPri', ], - ], - [ + ), + ( [ 'typ=Buildings.Templates.HeatingPlants.HotWater.Types.Boiler.NonCondensing', 'typPumHeaWatSec1_select=Buildings.Templates.HeatingPlants.HotWater.Types.PumpsSecondary.(?!None)', @@ -182,8 +173,8 @@ 'typMeaCtlHeaWatPri', 'locSenVHeaWatPri', ], - ], - [ + ), + ( [ 'typ=Buildings.Templates.HeatingPlants.HotWater.Types.Boiler.Condensing', 'typPumHeaWatSec1_select=Buildings.Templates.HeatingPlants.HotWater.Types.PumpsSecondary.(?!None)', @@ -193,8 +184,8 @@ [ 'locSenVHeaWatPri', ], - ], - [ + ), + ( [ 'typ=Buildings.Templates.HeatingPlants.HotWater.Types.Boiler.NonCondensing', 'typPumHeaWatSec1_select=Buildings.Templates.HeatingPlants.HotWater.Types.PumpsSecondary.(?!None)', @@ -204,8 +195,8 @@ [ 'locSenVHeaWatPri', ], - ], - [ + ), + ( [ 'typ=Buildings.Templates.HeatingPlants.HotWater.Types.Boiler.Hybrid', 'typPumHeaWatSec2_select=Buildings.Templates.HeatingPlants.HotWater.Types.PumpsSecondary.(?!None)', @@ -215,8 +206,8 @@ [ 'locSenVHeaWatPri', ], - ], - [ + ), + ( [ 'typ=Buildings.Templates.HeatingPlants.HotWater.Types.Boiler.Condensing', ], @@ -225,8 +216,8 @@ 'nBoiNon_select', 'typPumHeaWatPriNon', ], - ], - [ + ), + ( [ 'typ=Buildings.Templates.HeatingPlants.HotWater.Types.Boiler.NonCondensing', ], @@ -235,260 +226,48 @@ 'nBoiCon_select', 'typPumHeaWatPriCon', ], - ], - [ + ), + ( [ 'typPumHeaWatPriCon=Buildings.Templates.HeatingPlants.HotWater.Types.PumpsPrimary.Factory' ], [ 'typArrPumHeaWatPriCon_select', ], - ], - [ + ), + ( [ 'typPumHeaWatPriNon=Buildings.Templates.HeatingPlants.HotWater.Types.PumpsPrimary.Factory' ], [ 'typArrPumHeaWatPriNon_select', ], - ], + ), ], } -def simulateCase(arg, simulator): - """Set common parameters and run simulation with buildingspy. - - Args: - arg: A list of 3 elements: - [model name, list of class modifications, suffix for mat file name] - simulator: A string indicating the Modelica tool for simulating the model. - - Returns: - Error code: 0 if success. - """ - - # Local imports required for multiprocessing in Jupyter notebook. - import glob - import json - import os - import re - import shutil - import tempfile - - if simulator == 'Dymola': - from buildingspy.simulate.Dymola import Simulator - elif simulator == 'Optimica': - from buildingspy.simulate.Optimica import Simulator - else: - return 4, f'Unsupported simulation tool: {simulator}.' - - mat_root = re.split(r"\.", arg[0])[-1] - mat_suffix = re.sub(r"\.", "_", str(arg[2])) - output_dir_prefix = f"{mat_root}_{mat_suffix}" - cwd = os.getcwd() - # We need to create temp directories at the same level as Buildings because of - # the way volumes are mounted in docker run, see Buildings/Resources/Scripts/travis/dymola/dymola. - output_dir_path = tempfile.mkdtemp(prefix=output_dir_prefix, dir=os.path.abspath(os.pardir)) - - # The following make Dymola worker cd into outputDirectory. - s = Simulator(arg[0], outputDirectory=output_dir_path) - - if simulator == 'Dymola': - s.addPreProcessingStatement(r'Advanced.TranslationInCommandLog:=true;') - s.addPreProcessingStatement(r'openModel("../Buildings/package.mo", changeDirectory=false);') - if simulator == 'Optimica': - # Set MODELICAPATH (only in child process, so this won't affect main process). - os.environ['MODELICAPATH'] = os.path.abspath(os.pardir) - - for modif in arg[1]: - s.addModelModifier(modif) - - s.setSolver("CVode") - s.setTolerance(1e-6) - s.printModelAndTime() - - try: - s.simulate() - except Exception as e: - toreturn = 2 - print(e) - finally: - os.chdir(cwd) - - # Test if simulation succeeded. - try: - if simulator == 'Dymola': - with open(os.path.join(output_dir_path, 'simulator.log')) as fh: - log = fh.read() - if re.search('\n = false', log): - toreturn = 1 - else: - toreturn = 0 - elif simulator == 'Optimica': - with open(glob.glob(os.path.join(fr'{output_dir_path}', '*buildingspy.json'))[0], 'r') as f: - log = json.load(f) - if log['simulation']['success']: - toreturn = 0 - else: - toreturn = 1 - except (FileNotFoundError, IndexError) as e: - toreturn = 3 - log = e - finally: - if toreturn == 0: - shutil.rmtree(output_dir_path, ignore_errors=True) - else: - print(f'Simulation failed in {output_dir_path} with the following class modifications:\n' +\ - ',\n'.join(arg[1]) + '\n') - - return toreturn, log - - -def simulate_cases(args, simulator=SIMULATOR, asy=True): - """Main method that configures and runs all simulations.""" - - # Workaround for multiprocessing that isn't strictly supported on Windows Jupyter Notebook. - with open('tmp_func.py', 'w') as file: - file.write(inspect.getsource(simulateCase).replace(simulateCase.__name__, "task")) - sys.path.append('.') - from tmp_func import task - - args_with_fixed = [(el, simulator) for el in args] - results = [] - if __name__ == '__main__': - func = task - pool = Pool(os.cpu_count()) - if asy: - results = pool.starmap_async(func, args_with_fixed) - else: - results = pool.starmap(func, args_with_fixed) - pool.close() - pool.join() - - return results - - -def generate_modif_list(dic): - """Generates a list of class modifications. - - Args: - dic: A dictionary where the keys are the component or variable - to be modified, and the values are the modifications to be applied. - """ - to_return = [] - for param, val in dic.items(): - if 'redeclare' in param: - modif = re.sub('(.*)redeclare__(.*)', fr'\g<1>redeclare {val} \g<2>', param) - else: - modif = param + '=' + val - modif = re.sub('__', '(', modif) - modif = modif + ')' * modif.count('(') - to_return.append(modif) - return to_return - - -def generate_tag(dic): - tag = '' - for param, val in dic.items(): - tag = tag + '_' + re.split(r'\.', val)[-1] - return re.sub('^_', '', tag) - - -def remove_items_by_indices(lst, indices): - """Removes items from list (in place) based on their indices.""" - for idx in sorted(list(dict.fromkeys(indices)), reverse=True): - if idx < len(lst): - lst.pop(idx) - - if __name__ == '__main__': # Generate combinations. - # combinations_dicts is a dictionary where - # each key is a model to be simulated, - # each value is a list of dictionaries where the keys are the component or variable - # to be modified, and the values are the modifications to be applied. - combinations_dicts = dict() - for model in MODELS: - keys, values = zip(*MODIF_GRID[model].items()) - combinations_dicts[model] = [dict(zip(keys, v)) for v in itertools.product(*values)] - # args is a list of lists where - # the first item is a model to be simulated, - # the second item is the list of class modifications. - args = [] - for model, modif_dict in combinations_dicts.items(): - for el in modif_dict: - args.append([model, generate_modif_list(el)]) + combinations: list[tuple[str, list[str], str]] = generate_combinations(models=MODELS, modif_grid=MODIF_GRID) - # Remove class modifications. - indices_to_pop = [] - for i, arg in enumerate(args): - tmp = [] - if arg[0] in REMOVE_MODIF: - modif_concat = ''.join(arg[1]) - for item in REMOVE_MODIF[arg[0]]: - if all(re.search(el, modif_concat) for el in item[0]): - for pattern_to_remove in item[1]: - for j, modif in enumerate(arg[1]): - if re.search(pattern_to_remove, modif): - tmp.append(j) - indices_to_pop.append(tmp) - for i in range(len(args)): - remove_items_by_indices(args[i][1], indices_to_pop[i]) + # Prune class modifications. + prune_modifications(combinations=combinations, remove_modif=REMOVE_MODIF, exclude=EXCLUDE) - # Remove duplicates. - indices_to_pop = [] - for i, arg in enumerate(args): - for j in range(i+1, len(args)): - if arg == args[j]: - indices_to_pop.append(j) - remove_items_by_indices(args, indices_to_pop) - - # Exclude cases. - ## (We iterate over a copy of the `args` list to allow removing items of `args` during iteration.) - for arg in args.copy(): - if arg[0] in EXCLUDE: - modif_concat = ''.join(arg[1]) - if any(all(re.search(el, modif_concat) for el in ell) for ell in EXCLUDE[arg[0]]): - args.remove(arg) - - print(f'Number of cases to be simulated with {SIMULATOR}: {len(args)}.\n') - - # Change tag for shorter length (simply use list index). - for i, arg in enumerate(args): - args[i].append(str(i)) - - # FIXME - args = args[:2] + # FIXME(AntoineGautier PR#3364): Temporarily limit the number of simulations to be run (for testing purposes only). + combinations = combinations[:2] # Simulate cases. - results = simulate_cases(args, asy=False) - - try: - os.unlink('tmp_func.py') - os.unlink('unitTestsTemplates.log') - except FileNotFoundError: - pass - - df = pd.DataFrame( - dict( - model=[el[0] for el in args], - tag=[el[2] for el in args], - modif=[el[1] for el in args], - errorcode=[r[0] for r in results], - errorlog=[r[1] for r in results], - )) + results = simulate_cases(combinations, simulator=SIMULATOR, asy=False) - with open('unitTestsTemplates.log', 'w') as FH: - for idx in df[df.errorcode != 0].index: - FH.write( - f'*** Simulation failed for {df.iloc[idx].model} with the error code {df.iloc[idx].errorcode} ' +\ - 'and the following class modifications and error log.\n\n' +\ - ',\n'.join(df.iloc[idx].modif) + f'\n\n{df.iloc[idx].errorlog}\n\n' - ) + # Report and clean. + df = report_clean(combinations, results) + # Log and exit. if df.errorcode.abs().sum() != 0: - print(CRED + 'Some simulations failed: ' + CEND + 'see the file `unitTestsTemplates.log`.\n') + print( + CRED + 'Some simulations failed: ' + CEND + 'see the file `unitTestsTemplates.log`.\n' + ) sys.exit(1) else: print(CGREEN + 'All simulations succeeded.\n' + CEND) diff --git a/Buildings/Resources/Scripts/travis/templates/checkandrun.sh b/Buildings/Resources/Scripts/travis/templates/checkandrun.sh index a5199e05764..f268eee04fc 100755 --- a/Buildings/Resources/Scripts/travis/templates/checkandrun.sh +++ b/Buildings/Resources/Scripts/travis/templates/checkandrun.sh @@ -37,7 +37,6 @@ echo $CHECKSUM > ./Resources/Scripts/travis/templates/checksum # Diff / HEAD if $TRAVISRUN; then DIFFCHECKSUM="$(git diff --name-only HEAD | grep 'Resources/Scripts/travis/templates/checksum')" - if [[ $? == 0 ]]; then echo "Computed checksum does not match checksum on HEAD: please commit updated checksum for Templates." echo "Checksum on HEAD:" @@ -50,21 +49,21 @@ fi # Diff / master DIFFCHECKSUM="$(git diff --name-only origin/master | grep 'Resources/Scripts/travis/templates/checksum')" - -if [[ $? == 0 ]]; then +if [[ $? == 0 ]]; then echo "Computed checksum does not match checksum on master." echo "Running simulations for models in Templates with $SIMULATOR." # Launch simulations (typically several thousands). r= ./Resources/Scripts/travis/templates/BoilerPlant.py $SIMULATOR r=$r$? - (($r==0)) - if [[ $? == 0 ]]; then + if [ "$r" = 0 ]; then exit 0 else - printf "Below is the error log.\n\n" - cat unitTestsTemplates.log - exit 1 + if [[ -s unitTestsTemplates.log ]]; then + printf "Below is the error log.\n\n" + cat unitTestsTemplates.log + exit 1 + fi fi else echo "Computed checksum matches checksum on master: no further check performed." diff --git a/Buildings/Resources/Scripts/travis/templates/core.py b/Buildings/Resources/Scripts/travis/templates/core.py new file mode 100644 index 00000000000..3562039234a --- /dev/null +++ b/Buildings/Resources/Scripts/travis/templates/core.py @@ -0,0 +1,296 @@ +#!/usr/bin/env python +# coding: utf-8 + +# Requires Python >= 3.9 + +# This file contains imports and functions used by other Python scripts in the same directory. + +import inspect +import itertools +import os +import re +import sys + +# For CPU- and I/O-heavy jobs, we prefer multiprocessing.Pool because it provides better process isolation. +from multiprocessing import Pool + +import pandas as pd + +CRED = '\033[91m' +CGREEN = '\033[92m' +CEND = '\033[0m' + + +def simulate_case( + arg: tuple[str, list[str], str], + simulator: str, +) -> tuple[int, str]: + """Set common parameters and run simulation with buildingspy. + + Args: + arg: tuple with (model name, list of class modifications, suffix for mat file name) + simulator: Modelica tool for simulating the model + + Returns: + (error code, log) tuple: error code is 0 for success + """ + # Local imports required for multiprocessing in Jupyter notebook. + import glob + import json + import os + import re + import shutil + import tempfile + + if simulator == 'Dymola': + from buildingspy.simulate.Dymola import Simulator + elif simulator == 'Optimica': + from buildingspy.simulate.Optimica import Simulator + else: + return 4, f'Unsupported simulation tool: {simulator}.' + + mat_root = re.split(r"\.", arg[0])[-1] + mat_suffix = re.sub(r"\.", "_", str(arg[2])) + output_dir_prefix = f"{mat_root}_{mat_suffix}" + cwd = os.getcwd() + # We need to create temporary directories at the same level as Buildings because of + # the way volumes are mounted in docker run, see Buildings/Resources/Scripts/travis/dymola/dymola. + output_dir_path = tempfile.mkdtemp(prefix=output_dir_prefix, dir=os.path.abspath(os.pardir)) + + # The following make Dymola worker cd into outputDirectory. + s = Simulator(arg[0], outputDirectory=output_dir_path) + + if simulator == 'Dymola': + s.addPreProcessingStatement(r'Advanced.TranslationInCommandLog:=true;') + s.addPreProcessingStatement(r'openModel("../Buildings/package.mo", changeDirectory=false);') + if simulator == 'Optimica': + # Set MODELICAPATH (only in child process, so this won't affect main process). + os.environ['MODELICAPATH'] = os.path.abspath(os.pardir) + + for modif in arg[1]: + s.addModelModifier(modif) + + s.setSolver("CVode") + s.setTolerance(1e-6) + s.printModelAndTime() + + try: + s.simulate() + except Exception as e: + toreturn = 2 + print(e) + finally: + os.chdir(cwd) + + # Test if simulation succeeded. + try: + if simulator == 'Dymola': + with open(os.path.join(output_dir_path, 'simulator.log')) as fh: + log = fh.read() + if re.search('\n = false', log): + toreturn = 1 + else: + toreturn = 0 + elif simulator == 'Optimica': + with open( + glob.glob(os.path.join(fr'{output_dir_path}', '*buildingspy.json'))[0], 'r' + ) as f: + log = json.load(f) + if log['simulation']['success']: + toreturn = 0 + else: + toreturn = 1 + except (FileNotFoundError, IndexError) as e: + toreturn = 3 + log = e + finally: + if toreturn == 0: + shutil.rmtree(output_dir_path, ignore_errors=True) + else: + print( + f'Simulation failed in {output_dir_path} with the following class modifications:\n' + + ',\n'.join(arg[1]) + + '\n' + ) + + return toreturn, log + + +def simulate_cases( + args: list[tuple[str, list[str]]], + simulator: str, + asy: bool = True, +) -> list[tuple[int, str]]: + """Main method that configures and runs all simulations. + + Returns: + List of (error code, log) tuples + """ + # Workaround for multiprocessing that isn't strictly supported on Windows Jupyter Notebook. + with open('tmp_func.py', 'w') as file: + file.write(inspect.getsource(simulate_case).replace(simulate_case.__name__, "task")) + sys.path.append('.') + from tmp_func import task + + args_with_fixed = [(el, simulator) for el in args] + results = [] + func = task + pool = Pool(os.cpu_count()) + if asy: + results = pool.starmap_async(func, args_with_fixed) + else: + results = pool.starmap(func, args_with_fixed) + pool.close() + pool.join() + + return results + + +def generate_modif_list(dic: dict[str, list[str]]) -> list[str]: + """Generates a list of class modifications. + + Args: + dic: A dictionary where each key is the component or variable to be modified, + and the corresponding value is a list of modifications to be applied. + """ + to_return = [] + for param, val in dic.items(): + if 'redeclare' in param: + modif = re.sub('(.*)redeclare__(.*)', fr'\g<1>redeclare {val} \g<2>', param) + else: + modif = param + '=' + val + modif = re.sub('__', '(', modif) + modif = modif + ')' * modif.count('(') + to_return.append(modif) + return to_return + + +# The function below is kept for reference, but it is not used as it yields tags that are too long. +# +# def generate_tag(dic): +# tag = '' +# for param, val in dic.items(): +# tag = tag + '_' + re.split(r'\.', val)[-1] +# return re.sub('^_', '', tag) + + +def remove_items_by_indices(lst: list, indices: list[int]) -> None: + """Removes items from list (in place) based on their indices.""" + for idx in sorted(list(dict.fromkeys(indices)), reverse=True): + if idx < len(lst): + lst.pop(idx) + + +def generate_combinations( + models: list[str], + modif_grid: dict[str, dict[str, list[str]]], +) -> list[tuple[str, list[str], str]]: + """Generate all possible combinations. + + Returns: a list of 3-tuples where + the first item of the tuple is a model to be simulated, + the second item of the tuple is the list of class modifications, + the third item of the tuple is a tag. + """ + + # Generate combinations. + # combinations_dicts is a dictionary where + # each key is a model to be simulated, + # each value is a list of dictionaries where each key is the component or variable + # to be modified, and the corresponding value is the modification to be applied. + combinations_dicts = dict() + for model in models: + keys, values = zip(*modif_grid[model].items()) + combinations_dicts[model] = [dict(zip(keys, v)) for v in itertools.product(*values)] + + combinations = [] + tag = 0 # Simply tag each element with str(index). + for model, modif_dict_list in combinations_dicts.items(): + for el in modif_dict_list: + combinations.append((model, generate_modif_list(el), str(tag))) + tag = tag + 1 + + return combinations + + +def prune_modifications( + combinations: list[tuple[str, list[str], str]], + remove_modif: dict[str, list[tuple[list[str], list[str]]]], + exclude: dict[str, list[list[str]]], +) -> None: + """Remove class modifications and update tags. + + Returns: + None (modifies inplace) + """ + indices_to_pop = [] + for i, arg in enumerate(combinations): + tmp = [] + if arg[0] in remove_modif: + modif_concat = ''.join(arg[1]) + for item in remove_modif[arg[0]]: + if all(re.search(el, modif_concat) for el in item[0]): + for pattern_to_remove in item[1]: + for j, modif in enumerate(arg[1]): + if re.search(pattern_to_remove, modif): + tmp.append(j) + indices_to_pop.append(tmp) + + for i in range(len(combinations)): + remove_items_by_indices(combinations[i][1], indices_to_pop[i]) + + # Remove duplicates. + indices_to_pop = [] + for i, arg in enumerate(combinations): + for j in range(i + 1, len(combinations)): + if arg[:2] == combinations[j][:2]: # Compare w/o tag at index 3. + indices_to_pop.append(j) + remove_items_by_indices(combinations, indices_to_pop) + + # Exclude cases. + ## We iterate over a copy of the `combinations` list to allow removing items of `combinations` during iteration. + for arg in combinations.copy(): + if arg[0] in exclude: + modif_concat = ''.join(arg[1]) + if any(all(re.search(el, modif_concat) for el in ell) for ell in exclude[arg[0]]): + combinations.remove(arg) + + # Update tags. (Because pruning resulted in a sparse list of indices.) + for i, arg in enumerate(combinations): + combinations[i] = (*combinations[i][:2], str(i)) + + print(f'Number of cases to be simulated: {len(combinations)}.\n') + + +def report_clean( + combinations: list[tuple[str, list[str], str]], + results: list[tuple[int, str]], +) -> pd.DataFrame: + """Report and clean after simulations.""" + + try: + os.unlink('tmp_func.py') + os.unlink('unitTestsTemplates.log') + except FileNotFoundError: + pass + + df = pd.DataFrame( + dict( + model=[el[0] for el in combinations], + tag=[el[2] for el in combinations], + modif=[el[1] for el in combinations], + errorcode=[r[0] for r in results], + errorlog=[r[1] for r in results], + ) + ) + + with open('unitTestsTemplates.log', 'w') as FH: + for idx in df[df.errorcode != 0].index: + FH.write( + f'*** Simulation failed for {df.iloc[idx].model} with the error code {df.iloc[idx].errorcode} ' + + 'and the following class modifications and error log.\n\n' + + ',\n'.join(df.iloc[idx].modif) + + f'\n\n{df.iloc[idx].errorlog}\n\n' + ) + + return df