From 8afd1f3a11fb76d66f46ce4a27e57e89fb2f2961 Mon Sep 17 00:00:00 2001 From: Harald Taxt Walnum Date: Tue, 27 Feb 2024 12:44:38 +0100 Subject: [PATCH 01/11] first implementation --- testcase.py | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/testcase.py b/testcase.py index 20ee4f183..8e08ed7f6 100644 --- a/testcase.py +++ b/testcase.py @@ -22,6 +22,7 @@ import os import json import array as a +import pandas as pd class TestCase(object): '''Class that implements the test case. @@ -359,6 +360,8 @@ def advance(self, u): # Check if scenario is over if self.start_time >= self.end_time: self.scenario_end = True + # store results + self.store_results() # Log and return logging.info(message) return status, message, payload @@ -1327,6 +1330,41 @@ def _get_full_current_state(self): return z + def store_results(self): + '''Stores results from scenario in working directory. + When ran with service, the result will be packed in the result tarball and + be retrieveable with the test_id + + + Returns + ------- + None. + + ''' + + name = "results" + # get results_json + results_json = { + "dateRun": str(datetime.now(tz=pytz.UTC)), + "boptestVersion": self.version, + "emulatorName": self.get_name()[2]['name'], + "controlStep": str(self.get_step()[2]), + "forecastParameters": {}, # for future use to store used parameters? + "measurementParameters": {}, # for future use to store used parameters? + "kpis": self.get_kpis()[2], + "scenario": self.add_forecast_uncertainty(self.keys_to_camel_case(self.get_scenario()[2])), + } + + # store results_json + with open(name + ".json", "w") as outfile: + json.dump(results_json, outfile) + + # get results trajectories + results = self.get_results(self, self.input_names + self.output_names, self.initial_time, self.end_time)[2] + results_df = pd.DataFrame.from_dict(results) + # store + results_df.to_csv(name + ".csv") + def to_camel_case(self, snake_str): components = snake_str.split('_') # We capitalize the first letter of each component except the first one From 5780811d28c85afbd1a243b0274b7b9dd1864533 Mon Sep 17 00:00:00 2001 From: Harald Taxt Walnum Date: Tue, 27 Feb 2024 15:27:56 +0100 Subject: [PATCH 02/11] fixed get_results call --- testcase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testcase.py b/testcase.py index 8e08ed7f6..6941b417f 100644 --- a/testcase.py +++ b/testcase.py @@ -1360,7 +1360,7 @@ def store_results(self): json.dump(results_json, outfile) # get results trajectories - results = self.get_results(self, self.input_names + self.output_names, self.initial_time, self.end_time)[2] + results = self.get_results(self.input_names + self.output_names, self.initial_time, self.end_time)[2] results_df = pd.DataFrame.from_dict(results) # store results_df.to_csv(name + ".csv") From ee92c0648b6030da0720460144f2d7451a2b486d Mon Sep 17 00:00:00 2001 From: Harald Taxt Walnum Date: Tue, 27 Feb 2024 21:59:30 +0100 Subject: [PATCH 03/11] fixed results quiery as some outputs are removed from output_names --- testcase.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/testcase.py b/testcase.py index 6941b417f..9c4a624ff 100644 --- a/testcase.py +++ b/testcase.py @@ -1358,9 +1358,12 @@ def store_results(self): # store results_json with open(name + ".json", "w") as outfile: json.dump(results_json, outfile) - + + # get list of results, need to use output metadata because duplicate inputs are removed + result_list = self.input_names + list(self.outputs_metadata.keys()) # get results trajectories - results = self.get_results(self.input_names + self.output_names, self.initial_time, self.end_time)[2] + results = self.get_results(result_list, self.initial_time, self.end_time)[2] + # convert to dataframe results_df = pd.DataFrame.from_dict(results) # store results_df.to_csv(name + ".csv") From 3d4099d506760403e23f8a5bdb05051a0225aee6 Mon Sep 17 00:00:00 2001 From: Harald Taxt Walnum Date: Fri, 8 Mar 2024 11:53:20 +0100 Subject: [PATCH 04/11] created _get_test_results() to avoid duplicate code --- testcase.py | 66 +++++++++++++++++++++++++++-------------------------- 1 file changed, 34 insertions(+), 32 deletions(-) diff --git a/testcase.py b/testcase.py index 9c4a624ff..710968871 100644 --- a/testcase.py +++ b/testcase.py @@ -1145,27 +1145,16 @@ def post_results_to_dashboard(self, api_key, tags, unit_test=False): dash_server = os.environ['BOPTEST_DASHBOARD_SERVER'] # Create payload uid = str(uuid.uuid4()) - payload = { - "results": [ - { - "uid": uid, - "dateRun": str(datetime.now(tz=pytz.UTC)), - "boptestVersion": self.version, - "isShared": True, - "controlStep": str(self.get_step()[2]), - "account": { + test_results = self._get_test_results() + api_parameters = { + "uid": uid, + "isShared": True, + "account": { "apiKey": api_key - }, - "forecastParameters":{}, - "tags": tags, - "kpis": self.get_kpis()[2], - "scenario": self.add_forecast_uncertainty(self.keys_to_camel_case(self.get_scenario()[2])), - "buildingType": { - "uid": self.get_name()[2]['name'] - } - } - ] + }, + "tags": tags, } + payload = {"results":[{**test_results, **api_parameters}]} dash_url = "%s/api/results" % dash_server # Post to dashboard if not unit_test: @@ -1329,6 +1318,28 @@ def _get_full_current_state(self): z.update(self.u) return z + + def _get_test_results(self): + '''Collect test results. + + Returns + ------- + results: dict + Dictionary of test specific results. + ''' + results = { + "dateRun": str(datetime.now(tz=pytz.UTC)), + "boptestVersion": self.version, + "controlStep": str(self.get_step()[2]), + "forecastParameters": {}, # for future use to store used parameters? + "measurementParameters": {}, # for future use to store used parameters? + "kpis": self.get_kpis()[2], + "scenario": self.add_forecast_uncertainty(self.keys_to_camel_case(self.get_scenario()[2])), + "buildingType": { + "uid": self.get_name()[2]['name'], + } + } + return results def store_results(self): '''Stores results from scenario in working directory. @@ -1342,21 +1353,12 @@ def store_results(self): ''' - name = "results" + file_name = "results" # get results_json - results_json = { - "dateRun": str(datetime.now(tz=pytz.UTC)), - "boptestVersion": self.version, - "emulatorName": self.get_name()[2]['name'], - "controlStep": str(self.get_step()[2]), - "forecastParameters": {}, # for future use to store used parameters? - "measurementParameters": {}, # for future use to store used parameters? - "kpis": self.get_kpis()[2], - "scenario": self.add_forecast_uncertainty(self.keys_to_camel_case(self.get_scenario()[2])), - } + results_json = self._get_test_results() # store results_json - with open(name + ".json", "w") as outfile: + with open(file_name + ".json", "w") as outfile: json.dump(results_json, outfile) # get list of results, need to use output metadata because duplicate inputs are removed @@ -1366,7 +1368,7 @@ def store_results(self): # convert to dataframe results_df = pd.DataFrame.from_dict(results) # store - results_df.to_csv(name + ".csv") + results_df.to_csv(file_name + ".csv") def to_camel_case(self, snake_str): components = snake_str.split('_') From f50b7b4a3065e0c0a9800b3fdb4f2d97e9b4bfef Mon Sep 17 00:00:00 2001 From: Harald Taxt Walnum Date: Fri, 8 Mar 2024 12:34:35 +0100 Subject: [PATCH 05/11] updated releasenotes.md --- releasenotes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/releasenotes.md b/releasenotes.md index d5c50d8c8..5c05fe2bd 100644 --- a/releasenotes.md +++ b/releasenotes.md @@ -16,7 +16,7 @@ Released on xx/xx/xxxx. - Correct typo in documentation for ``multizone_office_simple_air``, cooling setback temperature changed from 12 to 30. This is for [#605](https://github.com/ibpsa/project1-boptest/issues/605). - Modify unit tests for test case scenarios to only simulate two days after warmup instead of the whole two-week scenario. This is for [#576](https://github.com/ibpsa/project1-boptest/issues/576). - Fix unit tests for possible false passes in certain test cases. This is for [#620](https://github.com/ibpsa/project1-boptest/issues/620). - +- Add storing of scenario results to simulation directory. This is for [#626](https://github.com/ibpsa/project1-boptest/issues/626). ## BOPTEST v0.5.0 From 9987e2d2accbb2bab805c11a12319a2a1adbdbf3 Mon Sep 17 00:00:00 2001 From: David Blum Date: Mon, 11 Mar 2024 13:01:48 -0700 Subject: [PATCH 06/11] Make time as index for csv --- testcase.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/testcase.py b/testcase.py index 710968871..941935f10 100644 --- a/testcase.py +++ b/testcase.py @@ -1318,7 +1318,7 @@ def _get_full_current_state(self): z.update(self.u) return z - + def _get_test_results(self): '''Collect test results. @@ -1360,13 +1360,14 @@ def store_results(self): # store results_json with open(file_name + ".json", "w") as outfile: json.dump(results_json, outfile) - + # get list of results, need to use output metadata because duplicate inputs are removed result_list = self.input_names + list(self.outputs_metadata.keys()) # get results trajectories results = self.get_results(result_list, self.initial_time, self.end_time)[2] - # convert to dataframe + # convert to dataframe with time as index results_df = pd.DataFrame.from_dict(results) + results_df.index = results_df['time'] # store results_df.to_csv(file_name + ".csv") From 74314ea7fe659f0baa2194ef032fb5347a3c0972 Mon Sep 17 00:00:00 2001 From: David Blum Date: Mon, 11 Mar 2024 13:03:16 -0700 Subject: [PATCH 07/11] Use points instead of parameters --- testcase.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/testcase.py b/testcase.py index 941935f10..b0743271e 100644 --- a/testcase.py +++ b/testcase.py @@ -1331,8 +1331,9 @@ def _get_test_results(self): "dateRun": str(datetime.now(tz=pytz.UTC)), "boptestVersion": self.version, "controlStep": str(self.get_step()[2]), - "forecastParameters": {}, # for future use to store used parameters? - "measurementParameters": {}, # for future use to store used parameters? + "forecastPoints": {}, # for future use to store used forecast points + "measurementPoints": {}, # for future use to store used measurement points + "inputPoints": {}, # for future use to store used input points "kpis": self.get_kpis()[2], "scenario": self.add_forecast_uncertainty(self.keys_to_camel_case(self.get_scenario()[2])), "buildingType": { From 485301c1bb57c891703ec0adf276ebb868e3be46 Mon Sep 17 00:00:00 2001 From: David Blum Date: Mon, 11 Mar 2024 13:07:27 -0700 Subject: [PATCH 08/11] Doc formatting and edits --- testcase.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/testcase.py b/testcase.py index b0743271e..30279468c 100644 --- a/testcase.py +++ b/testcase.py @@ -1320,13 +1320,15 @@ def _get_full_current_state(self): return z def _get_test_results(self): - '''Collect test results. + '''Collect test results and information into a dictionary. Returns ------- results: dict - Dictionary of test specific results. + Dictionary of test specific results and information. + ''' + results = { "dateRun": str(datetime.now(tz=pytz.UTC)), "boptestVersion": self.version, @@ -1340,36 +1342,35 @@ def _get_test_results(self): "uid": self.get_name()[2]['name'], } } + return results def store_results(self): - '''Stores results from scenario in working directory. - When ran with service, the result will be packed in the result tarball and - be retrieveable with the test_id + '''Stores results from scenario in working directory as json and csv. + When run with Service, the result will be packed in the result tarball and + be retrieveable with the test_id. Returns ------- - None. + None ''' file_name = "results" # get results_json results_json = self._get_test_results() - # store results_json with open(file_name + ".json", "w") as outfile: json.dump(results_json, outfile) - - # get list of results, need to use output metadata because duplicate inputs are removed + # get list of results, need to use output metadata so duplicate inputs are removed result_list = self.input_names + list(self.outputs_metadata.keys()) # get results trajectories results = self.get_results(result_list, self.initial_time, self.end_time)[2] # convert to dataframe with time as index results_df = pd.DataFrame.from_dict(results) results_df.index = results_df['time'] - # store + # store results csv results_df.to_csv(file_name + ".csv") def to_camel_case(self, snake_str): From ed371c0c0dab6dffadf09683754478552e83acf6 Mon Sep 17 00:00:00 2001 From: David Blum Date: Wed, 13 Mar 2024 08:58:18 -0700 Subject: [PATCH 09/11] Revert back to forecastParameters without other points --- testcase.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/testcase.py b/testcase.py index 30279468c..326d54fb3 100644 --- a/testcase.py +++ b/testcase.py @@ -1333,9 +1333,7 @@ def _get_test_results(self): "dateRun": str(datetime.now(tz=pytz.UTC)), "boptestVersion": self.version, "controlStep": str(self.get_step()[2]), - "forecastPoints": {}, # for future use to store used forecast points - "measurementPoints": {}, # for future use to store used measurement points - "inputPoints": {}, # for future use to store used input points + "forecastParameters":{}, "kpis": self.get_kpis()[2], "scenario": self.add_forecast_uncertainty(self.keys_to_camel_case(self.get_scenario()[2])), "buildingType": { From edc0c3c9df1e4026c165297e633f6e91cd51dd67 Mon Sep 17 00:00:00 2001 From: David Blum Date: Wed, 13 Mar 2024 13:54:06 -0700 Subject: [PATCH 10/11] Update dict in python2,3 compatible way for unit tests to pass --- testcase.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/testcase.py b/testcase.py index 326d54fb3..1c23c6457 100644 --- a/testcase.py +++ b/testcase.py @@ -1154,7 +1154,8 @@ def post_results_to_dashboard(self, api_key, tags, unit_test=False): }, "tags": tags, } - payload = {"results":[{**test_results, **api_parameters}]} + test_results.update(api_parameters) + payload = {"results":[test_results]} dash_url = "%s/api/results" % dash_server # Post to dashboard if not unit_test: From 4b810917b86309d773338a93d68b6286120c4d80 Mon Sep 17 00:00:00 2001 From: David Blum Date: Thu, 14 Mar 2024 08:58:08 -0700 Subject: [PATCH 11/11] Update releasenotes.md [ci skip] --- releasenotes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/releasenotes.md b/releasenotes.md index 9673b2ae5..9aaaba341 100644 --- a/releasenotes.md +++ b/releasenotes.md @@ -17,7 +17,7 @@ Released on xx/xx/xxxx. - Modify unit tests for test case scenarios to only simulate two days after warmup instead of the whole two-week scenario. This is for [#576](https://github.com/ibpsa/project1-boptest/issues/576). - Fix unit tests for possible false passes in certain test cases. This is for [#620](https://github.com/ibpsa/project1-boptest/issues/620). - Add ``activate`` control inputs to all test case documentation and update ``get_html_IO.py`` to print one file with all inputs, outputs, and forecasts. This is for [#555](https://github.com/ibpsa/project1-boptest/issues/555). -- Add storing of scenario results to simulation directory. This is for [#626](https://github.com/ibpsa/project1-boptest/issues/626). +- Add storing of scenario result trajectories, kpis, and test information to simulation directory within test case docker container. This is for [#626](https://github.com/ibpsa/project1-boptest/issues/626). ## BOPTEST v0.5.0