diff --git a/update_eplus_compare/AutomateVersionTranslation.ipynb b/update_eplus_compare/AutomateVersionTranslation.ipynb
index d25849d3..9ea82573 100644
--- a/update_eplus_compare/AutomateVersionTranslation.ipynb
+++ b/update_eplus_compare/AutomateVersionTranslation.ipynb
@@ -5,7 +5,7 @@
"metadata": {},
"source": [
"\n",
- " Transition from E+ 23.2.0 to E+ 24.1.0
\n",
+ " Transition from E+ 24.1.0 to E+ 24.2.0\n",
"\n",
"
\n",
"\n",
@@ -64,11 +64,13 @@
"import matplotlib.pyplot as plt\n",
"import seaborn as sns\n",
"import re\n",
+ "import tempfile\n",
"\n",
"#import csv\n",
"import glob as gb\n",
"\n",
- "#import pathlib\n",
+ "from pathlib import Path\n",
+ "from typing import Optional\n",
"\n",
"import datetime\n",
"import sqlite3\n",
@@ -79,7 +81,6 @@
"\n",
"#import tqdm\n",
"from tqdm.notebook import trange, tqdm\n",
- "import pathlib\n",
"\n",
"from ipywidgets import HTML\n",
"from IPython.display import display\n",
@@ -107,7 +108,7 @@
"metadata": {},
"outputs": [],
"source": [
- "IDF_DIR = os.getcwd()\n",
+ "IDF_DIR = Path('.').absolute()\n",
"IDF_DIR"
]
},
@@ -118,39 +119,36 @@
"outputs": [],
"source": [
"# Path to the OpenStudio-resources/testruns/ directory\n",
- "# TESTRUNS_DIR = '/home/julien/Software/Others/OpenStudio-resources/testruns/'\n",
- "TESTRUNS_DIR = os.path.abspath('../testruns')\n",
- "OS_RES_DIR = os.path.abspath('..')\n",
+ "TESTRUNS_DIR = Path('../testruns').resolve()\n",
+ "OS_RES_DIR = Path('..').resolve()\n",
"\n",
- "EPLUS_OLD_VERSION = '23.2.0'\n",
- "OS_OLD_VERSION = '3.7.0'\n",
+ "EPLUS_OLD_VERSION = '24.1.0'\n",
+ "OS_OLD_VERSION = '3.8.0'\n",
"\n",
- "EPLUS_NEW_VERSION = '24.1.0'\n",
- "OS_NEW_VERSION = '3.8.0'\n",
+ "EPLUS_NEW_VERSION = '24.2.0'\n",
+ "OS_NEW_VERSION = '3.9.0' # This is really an alpha, with 24.2.0 upgrade...\n",
"\n",
"# Careful: We'll cd into the TRANSITION_CLI_DIR to run it, so it must be a writtable location...\n",
- "TRANSITION_CLI_DIR = '/home/julien/Software/Others/EnergyPlus-build-release-develop/Products'\n",
- "# For some reason this one doesn't work\n",
- "# TRANSITION_CLI_DIR = '/home/julien/Software/Others/OS-build2/EnergyPlus-8.9.0-1c5ba897d1-Linux-x86_64/EnergyPlus-8-9-0/PreProcess/IDFVersionUpdater/'\n",
- "# TRANSITION_CLI_DIR = '/home/julien/Software/Others/OS-build-release/EnergyPlus-24.1.0-9d7789a3ac-Linux-Ubuntu20.04-x86_64/PreProcess/IDFVersionUpdater'\n",
+ "TRANSITION_CLI_DIR = Path('~/Software/Others/EnergyPlus-build-release-develop/Products').expanduser()\n",
+ "# TRANSITION_CLI_DIR = Path('~/Software/Others/OS-build-release/EnergyPlus-24.1.0-9d7789a3ac-Linux-Ubuntu20.04-x86_64/PreProcess/IDFVersionUpdater').expanduser()\n",
"\n",
"# Force a given number of parallel process \n",
"# (defaults to nproc - 2, leaving one physical core free if you have hyperthreading)\n",
"N = None\n",
- "WEATHER_FILE= os.path.abspath(os.path.join(IDF_DIR, '../weatherdata/USA_IL_Chicago-OHare.Intl.AP.725300_TMY3.epw'))\n",
+ "WEATHER_FILE= (IDF_DIR / '../weatherdata/USA_IL_Chicago-OHare.Intl.AP.725300_TMY3.epw').resolve()\n",
"\n",
"# Path to EnergyPlus application & idd \n",
- "OLD_EPLUS_EXE = '/usr/local/EnergyPlus-23-2-0/energyplus'\n",
+ "OLD_EPLUS_EXE = Path('/usr/local/EnergyPlus-24-1-0/energyplus')\n",
+ "\n",
"# NEW_EPLUS_EXE = os.path.join(TRANSITION_CLI_DIR, 'energyplus-9.2.0')\n",
- "NEW_EPLUS_EXE = '/usr/local/EnergyPlus-24-1-0/energyplus'\n",
+ "NEW_EPLUS_EXE = Path('/usr/local/EnergyPlus-24-2-0/energyplus')\n",
"# NEW_EPLUS_EXE = '/home/julien/Software/Others/EnergyPlus-build-release-develop/Products/energyplus'\n",
"# NEW_EPLUS_EXE = '/home/julien/Software/Others/OS-build2/EnergyPlus-8.9.0-1c5ba897d1-Linux-x86_64/EnergyPlus-8-9-0/energyplus-8.9.0'\n",
"# NEW_EPLUS_EXE = '/home/julien/Downloads/Temp/EnergyPlus-23.1.0-ff86a13c18-Linux-Ubuntu20.04-x86_64/energyplus'\n",
"\n",
"# Path to OpenStudio CLIs\n",
- "OLD_OS_CLI = '/usr/local/openstudio-3.7.0/bin/openstudio'\n",
- "NEW_OS_CLI = '/usr/local/openstudio-3.8.0/bin/openstudio'\n",
- "# NEW_OS_CLI = '/home/julien/Software/Others/OS-build-release/Products/openstudio'\n",
+ "OLD_OS_CLI = Path('/usr/local/openstudio-3.8.0/bin/openstudio')\n",
+ "NEW_OS_CLI = Path('~/Software/Others/OS-build-release/Products/openstudio').expanduser()\n",
"# OLD_IDD_FILE = '/usr/local/EnergyPlus-8-8-0/Energy+.idd'\n",
"# NEW_IDD_FILE = os.path.join(TRANSITION_DIR, 'Energy+.idd')\n",
"\n",
@@ -159,6 +157,21 @@
"REGRESSION_TEST_FILTER = None"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "assert TRANSITION_CLI_DIR.is_dir()\n",
+ "assert WEATHER_FILE.is_file()\n",
+ "\n",
+ "assert OLD_EPLUS_EXE.exists()\n",
+ "assert OLD_EPLUS_EXE.exists()\n",
+ "assert OLD_OS_CLI.exists()\n",
+ "assert NEW_OS_CLI.exists()"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
@@ -166,6 +179,16 @@
"## Setup"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "ep_old_v_dashed = EPLUS_OLD_VERSION.replace('.', '-')\n",
+ "ep_new_v_dashed = EPLUS_NEW_VERSION.replace('.', '-')"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -174,21 +197,33 @@
"source": [
"# For the translation to work,\n",
"# you'll have to chdir to the Transition CLI's folder\n",
- "TRANSITION_CLI = os.path.abspath(os.path.join(TRANSITION_CLI_DIR,\n",
- " 'Transition-V{}-to-V{}'.format(EPLUS_OLD_VERSION.replace('.', '-'),\n",
- " EPLUS_NEW_VERSION.replace('.', '-'))))\n",
+ "TRANSITION_CLI = TRANSITION_CLI_DIR / f\"Transition-V{ep_old_v_dashed}-to-V{ep_new_v_dashed}\"\n",
+ "assert TRANSITION_CLI.is_file()\n",
+ "\n",
+ "OLD_IDD_FILE = TRANSITION_CLI_DIR / f\"V{ep_old_v_dashed}-Energy+.idd\"\n",
+ "NEW_IDD_FILE = TRANSITION_CLI_DIR / f\"V{ep_new_v_dashed}-Energy+.idd\"\n",
+ "assert OLD_IDD_FILE.is_file()\n",
+ "assert NEW_IDD_FILE.is_file()\n",
+ "\n",
+ "TRANSITION_REPORT_VAR_PATH = TRANSITION_CLI_DIR / f\"Report Variables {ep_old_v_dashed} to {ep_new_v_dashed}.csv\"\n",
+ "if TRANSITION_REPORT_VAR_PATH.is_file():\n",
+ " print(f\"Found {TRANSITION_REPORT_VAR_PATH}\")\n",
+ "\n",
"# TODO: Temp override\n",
- "# TRANSITION_CLI = '/home/julien/Software/Others/EnergyPlus-build-release-develop/Products/Transition-V9-5-0-to-V9-6-0'\n",
+ "# TRANSITION_CLI = Path('/home/julien/Software/Others/EnergyPlus-build-release-develop/Products/Transition-V9-5-0-to-V9-6-0')\n",
"TRANSITION_CLI"
]
},
{
- "cell_type": "raw",
+ "cell_type": "code",
+ "execution_count": null,
"metadata": {},
+ "outputs": [],
"source": [
- "import psutil\n",
- "physical_cpus = psutil.cpu_count(logical=False)\n",
- "multiprocessing.cpu_count() * (physical_cpus - 1) / physical_cpus"
+ "if False:\n",
+ " import psutil\n",
+ " physical_cpus = psutil.cpu_count(logical=False)\n",
+ " multiprocessing.cpu_count() * (physical_cpus - 1) / physical_cpus"
]
},
{
@@ -203,6 +238,13 @@
" print(\"Defaulting number of processes to {}\".format(N))"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -210,19 +252,16 @@
"outputs": [],
"source": [
"# Create directories\n",
- "OLD_OS_DIR = \"{o}-{e}\".format(e=EPLUS_OLD_VERSION, o=OS_OLD_VERSION)\n",
- "OLD_OS_DIR = os.path.join(IDF_DIR, OLD_OS_DIR)\n",
+ "OLD_OS_DIR = IDF_DIR / f\"{OS_OLD_VERSION}-{EPLUS_OLD_VERSION}\"\n",
"\n",
- "NEW_OS_DIR = \"{o}-{e}\".format(e=EPLUS_NEW_VERSION, o=OS_NEW_VERSION)\n",
- "NEW_OS_DIR = os.path.join(IDF_DIR, NEW_OS_DIR)\n",
+ "NEW_OS_DIR = IDF_DIR / f\"{OS_NEW_VERSION}-{EPLUS_NEW_VERSION}\"\n",
"\n",
- "TRANSITION_DIR = \"Transition-{e}\".format(e=EPLUS_NEW_VERSION)\n",
- "TRANSITION_DIR = os.path.join(IDF_DIR, TRANSITION_DIR)\n",
+ "TRANSITION_DIR = IDF_DIR / f\"Transition-{EPLUS_NEW_VERSION}\"\n",
"\n",
"for p in [OLD_OS_DIR, NEW_OS_DIR, TRANSITION_DIR]:\n",
- " if not os.path.exists(p):\n",
- " os.makedirs(p)\n",
- " print('Creating directory: {}'.format(p))"
+ " if not p.is_dir():\n",
+ " p.mkdir(parents=True)\n",
+ " print(f\"Creating directory: {p}\")"
]
},
{
@@ -245,6 +284,19 @@
" 'DIR': TRANSITION_DIR}"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def stringify_cmd(cmd) -> list[str]:\n",
+ " \"\"\"Takes a list of arguments that may be Path or str or int and turns into a List[str].\n",
+ " So that is it suitable to pass to subprocess.\n",
+ " \"\"\"\n",
+ " return list(map(str, cmd))"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
@@ -281,7 +333,7 @@
"outputs": [],
"source": [
"# Delete all testruns to ensure we don't end up grabbing the idf and sql from another version\n",
- "if os.path.exists(TESTRUNS_DIR):\n",
+ "if TESTRUNS_DIR.is_dir():\n",
" shutil.rmtree(TESTRUNS_DIR)"
]
},
@@ -312,10 +364,8 @@
]
},
{
- "cell_type": "code",
- "execution_count": null,
+ "cell_type": "raw",
"metadata": {},
- "outputs": [],
"source": [
"c_args = shlex.split(command)\n",
"\n",
@@ -380,15 +430,16 @@
"outputs": [],
"source": [
"found_idfs = []\n",
- "for f in gb.iglob(os.path.join(TESTRUNS_DIR, '**/*/in.idf')):\n",
- " f2 = os.path.relpath(f, TESTRUNS_DIR)\n",
+ "for f in TESTRUNS_DIR.glob('**/*/in.idf'):\n",
+ " f2 = f.relative_to(TESTRUNS_DIR)\n",
" \n",
- " test_name = os.path.split(os.path.split(f2)[0])[0]\n",
+ " test_name = f2.parts[0]\n",
" #print(test_name)\n",
- " dst_path = os.path.join(IDF_DIR, \"{}.idf\".format(test_name))\n",
+ " dst_path = IDF_DIR / f\"{test_name}.idf\"\n",
" shutil.copyfile(f, dst_path)\n",
" found_idfs.append(test_name)\n",
- "found_idfs = set(found_idfs)"
+ "found_idfs = set(found_idfs)\n",
+ "len(found_idfs)"
]
},
{
@@ -404,16 +455,16 @@
"metadata": {},
"outputs": [],
"source": [
- "found_sqls = []\n",
- "\n",
- "for f in gb.iglob(os.path.join(TESTRUNS_DIR, '**/*/*.sql')):\n",
- " f2 = os.path.relpath(f, TESTRUNS_DIR)\n",
- " test_name = os.path.split(os.path.split(f2)[0])[0]\n",
+ "for f in TESTRUNS_DIR.glob(\"**/*/*.sql\"):\n",
+ " f2 = f.relative_to(TESTRUNS_DIR)\n",
+ " \n",
+ " test_name = f2.parts[0]\n",
" # print(test_name)\n",
- " dst_folder = os.path.join(OLD_OS_DIR, test_name)\n",
- " if not os.path.exists(dst_folder):\n",
- " os.makedirs(dst_folder)\n",
- " dst_path = os.path.join(dst_folder, \"eplusout.sql\")\n",
+ " dst_folder = OLD_OS_DIR / test_name\n",
+ " if not dst_folder.is_dir():\n",
+ " dst_folder.mkdir(parents=False, exist_ok=False)\n",
+ " \n",
+ " dst_path = dst_folder / \"eplusout.sql\"\n",
" # print(dst_path)\n",
" shutil.copyfile(f, dst_path)\n",
" found_sqls.append(test_name)\n",
@@ -462,7 +513,57 @@
"metadata": {},
"outputs": [],
"source": [
- "def translate_file(path):\n",
+ "def _prepare_temp_dir_for_transition() -> Path:\n",
+ " temp_dir = Path(tempfile.mkdtemp(prefix='transition-'))\n",
+ " shutil.copy(OLD_IDD_FILE, temp_dir)\n",
+ " shutil.copy(NEW_IDD_FILE, temp_dir)\n",
+ " if TRANSITION_REPORT_VAR_PATH.is_file():\n",
+ " shutil.copy(TRANSITION_REPORT_VAR_PATH, temp_dir)\n",
+ " return temp_dir\n",
+ "\n",
+ "def translate_file_parallelizable(idf_path: Path):\n",
+ " assert idf_path.is_file(), f\"Could not find {idf_path}\"\n",
+ " \n",
+ " temp_dir = _prepare_temp_dir_for_transition()\n",
+ " cmd = stringify_cmd([TRANSITION_CLI, idf_path])\n",
+ " r = subprocess.run(\n",
+ " cmd,\n",
+ " capture_output=True,\n",
+ " shell=False,\n",
+ " encoding='utf-8',\n",
+ " cwd=temp_dir,\n",
+ " )\n",
+ " \n",
+ " if r.returncode == 0: \n",
+ " # Move the resulting IDF into the new dir\n",
+ " new_file = idf_path.with_suffix('.idfnew')\n",
+ " assert new_file.is_file(), f\"Could not find {new_file}\"\n",
+ " new_dest = TRANSITION_DIR / idf_path.name\n",
+ " shutil.move(new_file, new_dest)\n",
+ "\n",
+ " # Move the old version into its directory\n",
+ " old_file = idf_path.with_suffix('.idfold')\n",
+ " assert old_file.is_file(), f\"Could not find {old_file}\"\n",
+ " old_dest = OLD_OS_DIR / idf_path.name\n",
+ " shutil.move(old_file, old_dest)\n",
+ "\n",
+ " # Delete original file\n",
+ " idf_path.unlink()\n",
+ "\n",
+ " # print('Done for {}.idf - {}'.format(eplus_file, path))\n",
+ " else:\n",
+ " print(f\"Error for {idf_path}:\")\n",
+ " print(r.stdout)\n",
+ " print(r.stderr)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def translate_file_regular_serially(path):\n",
" \"\"\"\n",
" Runs the file throught the transition utility and save in the right folder\n",
" Will move the ori file to the subdirectory OLD_DIR (eg `./8.8.0/`)\n",
@@ -471,7 +572,8 @@
" \n",
" eplus_file, ext = os.path.splitext(os.path.split(path)[1])\n",
" \n",
- " process = subprocess.Popen([TRANSITION_CLI, path],\n",
+ " cmd = stringify_cmd([TRANSITION_CLI, path])\n",
+ " process = subprocess.Popen(cmd,\n",
" shell=False,\n",
" stdout=subprocess.PIPE,\n",
" stderr=subprocess.PIPE)\n",
@@ -506,7 +608,7 @@
"metadata": {},
"outputs": [],
"source": [
- "files = [os.path.abspath(file) for file in gb.glob(os.path.join(IDF_DIR, '*.idf'))]\n",
+ "files = list(IDF_DIR.glob('*.idf'))\n",
"len(files)"
]
},
@@ -516,14 +618,17 @@
"metadata": {},
"outputs": [],
"source": [
- "files"
+ "# for x in problems.index[problems[next(x for x in problems.columns if x[1] == 'Transition')].isna()]:\n",
+ "# shutil.copy(f\"{OLD_OS_DIR}/{x}.idf\", f\"{IDF_DIR}/{x}.idf\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
- "**Transitionning in parallel seems to fail, so do it serially...**"
+ "**Transitionning in parallel seems to fail, so do it serially...**\n",
+ "\n",
+ "**Edit**: I made it work by using a temporary directory"
]
},
{
@@ -532,21 +637,15 @@
"metadata": {},
"outputs": [],
"source": [
- "if False:\n",
- " # You must cd to the Transition CLI's folder for it to work\n",
- " os.chdir(TRANSITION_CLI_DIR)\n",
- "\n",
+ "if True:\n",
" # Takes about 10minutes on my machine with 12 threads allocated\n",
- " pool = multiprocessing.Pool(processes=12)\n",
+ " pool = multiprocessing.Pool(processes=N)\n",
"\n",
- " desc = 'Translation from {} to {}
'.format(EPLUS_OLD_VERSION,\n",
- " EPLUS_NEW_VERSION)\n",
+ " desc = f'Translation from {EPLUS_OLD_VERSION} to {EPLUS_NEW_VERSION}
'\n",
" label = HTML(desc)\n",
" display(label)\n",
- " for _ in tqdm(pool.imap_unordered(translate_file, files), total=len(files)):\n",
- " pass\n",
- "\n",
- " os.chdir(IDF_DIR)"
+ " for _ in tqdm(pool.imap_unordered(translate_file_parallelizable, files), total=len(files)):\n",
+ " pass"
]
},
{
@@ -555,15 +654,16 @@
"metadata": {},
"outputs": [],
"source": [
- "os.chdir(TRANSITION_CLI_DIR)\n",
+ "if False:\n",
+ " os.chdir(TRANSITION_CLI_DIR)\n",
"\n",
- "desc = 'Translation from {} to {}
'.format(EPLUS_OLD_VERSION,\n",
- " EPLUS_NEW_VERSION)\n",
- "label = HTML(desc)\n",
- "for file in tqdm(files):\n",
- " translate_file(file)\n",
- " \n",
- "os.chdir(IDF_DIR)"
+ " desc = 'Translation from {} to {}
'.format(EPLUS_OLD_VERSION,\n",
+ " EPLUS_NEW_VERSION)\n",
+ " label = HTML(desc)\n",
+ " for file in tqdm(files):\n",
+ " translate_file_regular_serially(file)\n",
+ "\n",
+ " os.chdir(IDF_DIR)"
]
},
{
@@ -574,9 +674,9 @@
"source": [
"# At this point, you shouldn't have any .idf files in the IDF_DIR directory\n",
"# If you do, means that the transition failed\n",
- "all_files = gb.glob(os.path.join(IDF_DIR, '*.idf'))\n",
- "all_files += gb.glob(os.path.join(IDF_DIR, '*.idfnew'))\n",
- "all_files += gb.glob(os.path.join(IDF_DIR, '*.idfold'))\n",
+ "all_files = []\n",
+ "for e in ['idf', 'idfnew', 'idfold']:\n",
+ " all_files += list(IDF_DIR.glob(f'*.{e}'))\n",
"\n",
"all_files"
]
@@ -793,8 +893,8 @@
"SQL_QUERY_SIM_INFO = 'SELECT EnergyPlusVersion FROM Simulations'\n",
"\n",
"VERSION_REGEX = re.compile(r'Version (?P\\d+)\\.(?P\\d+)\\.'\n",
- " '(?P\\d+)-(?P\\w+),\\s+'\n",
- " 'YMD=(?P[0-9\\.: ]+)')\n",
+ " r'(?P\\d+)-(?P\\w+),\\s+'\n",
+ " r'YMD=(?P[0-9\\.: ]+)')\n",
"\n",
"\n",
"# Remove all files in the output directory except these\n",
@@ -807,7 +907,7 @@
"metadata": {},
"outputs": [],
"source": [
- "def parse_sql_version_and_sitekbtu(output_directory):\n",
+ "def parse_sql_version_and_sitekbtu(output_directory: Path) -> Optional[pd.Series]:\n",
" \"\"\"\n",
" This function grabs the EnergyPlusVersion and the total site energy\n",
" from the SQL file.\n",
@@ -823,15 +923,14 @@
" (or None if it didn't run), which name is the test_name\n",
" (gotten from the name of the output_directory)\n",
" \"\"\"\n",
- " sql_files = gb.glob(os.path.join(output_directory, \"*.sql\"))\n",
+ " sql_files = list(Path(output_directory).absolute().glob(\"*.sql\"))\n",
"\n",
" version_with_sha = None\n",
" site_kbtu = None\n",
"\n",
" if len(sql_files) == 1:\n",
" sql_path = sql_files[0]\n",
- " abs_sql_path = os.path.abspath(sql_path)\n",
- " sql_uri = '{}?mode=ro'.format(pathlib.Path(abs_sql_path).as_uri())\n",
+ " sql_uri = f\"{sql_path.as_uri()}?mode=ro\"\n",
" with sqlite3.connect(sql_uri, uri=True) as con:\n",
" cursor = con.cursor()\n",
" r = cursor.execute(SQL_QUERY_SIM_INFO).fetchone()\n",
@@ -841,9 +940,9 @@
" if m:\n",
" gpdict = m.groupdict()\n",
" version_with_sha = \"{}.{}.{}-{}\".format(gpdict['Major'],\n",
- " gpdict['Minor'],\n",
- " gpdict['Patch'],\n",
- " gpdict['SHA'])\n",
+ " gpdict['Minor'],\n",
+ " gpdict['Patch'],\n",
+ " gpdict['SHA'])\n",
" else:\n",
" msg = (\"Cannot find the EnergyPlusVersion in the SQL file. \"\n",
" \"For:\\n{}\".format(output_directory))\n",
@@ -868,58 +967,24 @@
"metadata": {},
"outputs": [],
"source": [
- "def run_OLD_eplus_sim(eplus_file):\n",
- " \"\"\"\n",
- " Runs the simulation with OLD_EPLUS_EXE and calls parse_sql\n",
- " \"\"\"\n",
- " base, file_with_ext = os.path.split(os.path.abspath(eplus_file))\n",
- " output_directory = os.path.join(base, os.path.splitext(file_with_ext)[0])\n",
- " # If directory exists, delete it\n",
- " if os.path.exists(output_directory):\n",
- " shutil.rmtree(output_directory)\n",
- " # Recreate it\n",
- " os.makedirs(output_directory)\n",
+ "def run_eplus_sim(\n",
+ " eplus_file: Path, ep_cli: Path,\n",
+ " expand_objects: bool = False,\n",
+ " verbose: bool = False\n",
+ ") -> pd.Series:\n",
+ " eplus_file = Path(eplus_file).absolute()\n",
+ " assert eplus_file.is_file(), f\"Could not find {eplus_file}\"\n",
" \n",
- " process = subprocess.Popen([OLD_EPLUS_EXE,\n",
- " # '-i', OLD_IDD_FILE\n",
- " '-w', WEATHER_FILE,\n",
- " '-d', output_directory,\n",
- " eplus_file],\n",
- " stdout=subprocess.PIPE,\n",
- " stderr=subprocess.PIPE,\n",
- " universal_newlines=True, \n",
- " shell=False)\n",
- " \n",
- " # wait for the process to terminate\n",
- " out, err = process.communicate()\n",
- " errcode = process.returncode\n",
- " if errcode == 0:\n",
- " # Clean up output directory\n",
- " [os.remove(x) for x in gb.glob(os.path.join(output_directory, '*'))\n",
- " if os.path.splitext(x)[1] not in KEEP_EXT]\n",
- " return parse_sql_version_and_sitekbtu(output_directory)\n",
- " else:\n",
- " print(\"ERROR: {}\".format(eplus_file))\n",
- " print(out)\n",
- " print(err)\n",
- " return pd.Series([None, None],\n",
- " index=['E+', 'SiteKBTU'],\n",
- " name = os.path.split(output_directory)[1])\n",
- " \n",
- "def run_NEW_eplus_sim(eplus_file, expand_objects=False):\n",
- " \"\"\"\n",
- " Runs the simulation with NEW_EPLUS_EXE and calls parse_sql\n",
- " \"\"\"\n",
- " base, file_with_ext = os.path.split(os.path.abspath(eplus_file))\n",
- " output_directory = os.path.join(base, os.path.splitext(file_with_ext)[0])\n",
+ " output_directory = eplus_file.parent / eplus_file.stem\n",
" # If directory exists, delete it\n",
- " if os.path.exists(output_directory):\n",
+ " if output_directory.is_dir():\n",
" shutil.rmtree(output_directory)\n",
+ " \n",
" # Recreate it\n",
- " os.makedirs(output_directory)\n",
+ " output_directory.mkdir(parents=True, exist_ok=False)\n",
" \n",
" cmd = [\n",
- " NEW_EPLUS_EXE\n",
+ " ep_cli\n",
" ]\n",
" if expand_objects:\n",
" cmd.append('-x')\n",
@@ -929,40 +994,57 @@
" '-d', output_directory,\n",
" eplus_file\n",
" ]\n",
- " # print(f\"cmd={' '.join(cmd)}\\n\")\n",
- " process = subprocess.Popen(cmd,\n",
- " stdout=subprocess.PIPE,\n",
- " stderr=subprocess.PIPE,\n",
- " universal_newlines=True, \n",
- " shell=False)\n",
- " \n",
- " # wait for the process to terminate\n",
- " out, err = process.communicate()\n",
- " errcode = process.returncode\n",
- " if errcode == 0:\n",
+ " cmd = stringify_cmd(cmd)\n",
+ " if verbose:\n",
+ " print(\" \".join(cmd))\n",
+ " r = subprocess.run(\n",
+ " cmd,\n",
+ " capture_output=True,\n",
+ " shell=False,\n",
+ " encoding='utf-8',\n",
+ " )\n",
+ " \n",
+ " if r.returncode == 0: \n",
" # Clean up output directory\n",
- " [os.remove(x) for x in gb.glob(os.path.join(output_directory, '*'))\n",
- " if os.path.splitext(x)[1] not in KEEP_EXT]\n",
+ " [\n",
+ " x.unlink() for x in output_directory.glob('*') \n",
+ " if x.suffix not in KEEP_EXT\n",
+ " ]\n",
" return parse_sql_version_and_sitekbtu(output_directory)\n",
" else:\n",
- " print(\"ERROR: {}\".format(eplus_file))\n",
- " # print(out)\n",
- " # print(err)\n",
+ " print(f\"ERROR with code {r.returncode}: {eplus_file}\")\n",
+ " # print(r.stdout)\n",
+ " # print(r.stderr)\n",
" return pd.Series([None, None],\n",
" index=['E+', 'SiteKBTU'],\n",
- " name = os.path.split(output_directory)[1])"
+ " name = os.path.split(output_directory)[1])\n",
+ "\n",
+ "def run_OLD_eplus_sim(eplus_file: Path, expand_objects: bool = False, verbose: bool = False):\n",
+ " \"\"\"\n",
+ " Runs the simulation with OLD_EPLUS_EXE and calls parse_sql\n",
+ " \"\"\"\n",
+ " return run_eplus_sim(eplus_file=eplus_file, ep_cli=OLD_EPLUS_EXE,\n",
+ " expand_objects=expand_objects, verbose=verbose)\n",
+ "\n",
+ " \n",
+ "def run_NEW_eplus_sim(eplus_file: Path, expand_objects: bool = False, verbose: bool = False) -> pd.Series:\n",
+ " \"\"\"\n",
+ " Runs the simulation with NEW_EPLUS_EXE and calls parse_sql\n",
+ " \"\"\"\n",
+ " return run_eplus_sim(eplus_file=eplus_file, ep_cli=NEW_EPLUS_EXE,\n",
+ " expand_objects=expand_objects, verbose=verbose)"
]
},
{
- "cell_type": "markdown",
+ "cell_type": "code",
+ "execution_count": null,
"metadata": {},
+ "outputs": [],
"source": [
- "```\n",
"# Test one sim\n",
- "files = gb.glob(os.path.join(TRANSITION_DIR, '*.idf'))\n",
- "eplus_file = files[0]\n",
- "run_NEW_eplus_sim(eplus_file)\n",
- "```"
+ "if False: \n",
+ " eplus_file = next(TRANSITION_DIR.glob('*.idf'))\n",
+ " run_NEW_eplus_sim(eplus_file)"
]
},
{
@@ -985,7 +1067,7 @@
"outputs": [],
"source": [
"if False:\n",
- " files = gb.glob(os.path.join(OLD_DIR, '*.idf'))\n",
+ " files = list(OLD_DIR.glob(\"*.idf\"))\n",
"\n",
" pool = multiprocessing.Pool(processes=N)\n",
"\n",
@@ -1137,7 +1219,7 @@
]
},
{
- "cell_type": "markdown",
+ "cell_type": "raw",
"metadata": {},
"source": [
"```\n",
@@ -1166,8 +1248,10 @@
]
},
{
- "cell_type": "raw",
+ "cell_type": "code",
+ "execution_count": null,
"metadata": {},
+ "outputs": [],
"source": [
"# If you ever send a Keyboard Interrupt signal to kill the above cell\n",
"# (via Kernel > Interrupt), you need to run this too\n",
@@ -1253,7 +1337,7 @@
"# Delete all testruns to ensure we don't end up grabbing the idf and sql from another version\n",
"# model_tests.rb only cleans out testruns/testXXX directories for tests we do request\n",
"# So if you use a regression test filter, you could have left overs\n",
- "if os.path.exists(TESTRUNS_DIR):\n",
+ "if TESTRUNS_DIR.is_dir():\n",
" shutil.rmtree(TESTRUNS_DIR)"
]
},
@@ -1332,15 +1416,6 @@
"TESTRUNS_DIR, NEW_OS_DIR"
]
},
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "len(sorted(list(gb.iglob(os.path.join(TESTRUNS_DIR, '**/*/in.idf')))))"
- ]
- },
{
"cell_type": "code",
"execution_count": null,
@@ -1348,12 +1423,12 @@
"outputs": [],
"source": [
"found_idfs = []\n",
- "for f in gb.iglob(os.path.join(TESTRUNS_DIR, '**/*/in.idf')):\n",
- " f2 = os.path.relpath(f, TESTRUNS_DIR)\n",
+ "for f in TESTRUNS_DIR.glob('**/*/in.idf'):\n",
+ " f2 = f.relative_to(TESTRUNS_DIR)\n",
" \n",
- " test_name = os.path.split(os.path.split(f2)[0])[0]\n",
+ " test_name = f2.parts[0]\n",
" #print(test_name)\n",
- " dst_path = os.path.join(NEW_OS_DIR, \"{}.idf\".format(test_name))\n",
+ " dst_path = NEW_OS_DIR / f\"{test_name}.idf\"\n",
" shutil.copyfile(f, dst_path)\n",
" found_idfs.append(test_name)\n",
"found_idfs = set(found_idfs)\n",
@@ -1384,15 +1459,16 @@
"source": [
"found_sqls = []\n",
"\n",
- "for f in gb.iglob(os.path.join(TESTRUNS_DIR, '**/*/*.sql')):\n",
- " f2 = os.path.relpath(f, TESTRUNS_DIR)\n",
+ "for f in TESTRUNS_DIR.glob(\"**/*/*.sql\"):\n",
+ " f2 = f.relative_to(TESTRUNS_DIR)\n",
" \n",
- " test_name = os.path.split(os.path.split(f2)[0])[0]\n",
+ " test_name = f2.parts[0]\n",
" # print(test_name)\n",
- " dst_folder = os.path.join(NEW_OS_DIR, test_name)\n",
- " if not os.path.exists(dst_folder):\n",
- " os.makedirs(dst_folder)\n",
- " dst_path = os.path.join(dst_folder, \"eplusout.sql\")\n",
+ " dst_folder = NEW_OS_DIR / test_name\n",
+ " if not dst_folder.is_dir():\n",
+ " dst_folder.mkdir(parents=False, exist_ok=False)\n",
+ " \n",
+ " dst_path = dst_folder / \"eplusout.sql\"\n",
" # print(dst_path)\n",
" shutil.copyfile(f, dst_path)\n",
" found_sqls.append(test_name)\n",
@@ -1477,9 +1553,9 @@
"\"\"\"\n",
"\n",
"TIME_REGEX = re.compile(r'.* Elapsed Time=(?P\\d+)hr *(?P\\d+)min'\n",
- " ' *(?P[\\d\\.]+)sec')\n",
+ " r' *(?P[\\d\\.]+)sec')\n",
"\n",
- "def parse_sql_runtime(output_directory):\n",
+ "def parse_sql_runtime(output_directory: Path):\n",
" \"\"\"\n",
" This function grabs the E+ runtime from the SQL file.\n",
" \n",
@@ -1492,14 +1568,13 @@
" ---------\n",
" * runtime (datetime.time)\n",
" \"\"\"\n",
+ " output_directory = Path(output_directory).absolute()\n",
+ " sql_files = list(output_directory.glob(\"*.sql\"))\n",
" \n",
- " sql_files = gb.glob(os.path.join(output_directory, \"*.sql\"))\n",
- "\n",
" runtime = None\n",
" if len(sql_files) == 1:\n",
" sql_path = sql_files[0]\n",
- " abs_sql_path = os.path.abspath(sql_path)\n",
- " sql_uri = '{}?mode=ro'.format(pathlib.Path(abs_sql_path).as_uri())\n",
+ " sql_uri = f'{sql_path.as_uri()}?mode=ro'\n",
" with sqlite3.connect(sql_uri, uri=True) as con:\n",
" cursor = con.cursor()\n",
" r = cursor.execute(SQL_QUERY_RUNTIME).fetchone()\n",
@@ -1524,7 +1599,7 @@
" #raise ValueError(msg)\n",
" print(msg)\n",
" \n",
- " return [os.path.split(output_directory)[1], runtime]"
+ " return [output_directory.name, runtime]"
]
},
{
@@ -1808,7 +1883,9 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {},
+ "metadata": {
+ "scrolled": false
+ },
"outputs": [],
"source": [
"def background_colors(val):\n",
@@ -1818,7 +1895,7 @@
" fmt = s.format('#F4C7C3')\n",
" return fmt\n",
"print(\"These are the files were we have some (but not all) failures\")\n",
- "problems.style.applymap(background_colors)"
+ "problems.style.map(background_colors)"
]
},
{
@@ -1834,8 +1911,28 @@
"metadata": {},
"outputs": [],
"source": [
- "# files = [f'{TRANSITION_DIR}/{x}.idf' for x in problems.index]\n",
- "# for f in files:\n",
+ "files = [f'{TRANSITION_DIR}/{x}.idf' for x\n",
+ " in problems.index[problems[next(x for x in problems.columns if x[1] == 'Transition')].isna()]]\n",
+ "len(files)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# ERROR: Could not find input data dictionary: /path/to/OpenStudio-resources/update_eplus_compare/Energy+.idd.\n",
+ "shutil.copy(OLD_EPLUS_EXE.parent / 'Energy+.idd', IDF_DIR)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# for f in tqdm(files):\n",
"# run_NEW_eplus_sim(f, expand_objects=True)\n",
" \n",
"# Reparse the SQLs above again"
@@ -1890,7 +1987,7 @@
},
"outputs": [],
"source": [
- "pct_threshold = 0.0001\n",
+ "pct_threshold = 0.001\n",
"print(\"Setting % diff threshold to {:.3%}\".format(pct_threshold))"
]
},
@@ -1987,6 +2084,88 @@
"plt.show()"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "s_eplus = df_diff.iloc[:, 0]\n",
+ "s_eplus.index = pd.MultiIndex.from_tuples(s_eplus[s_eplus.abs() >= pct_threshold].index.str.split('.').tolist())"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "s_eplus = s_eplus.swaplevel(0, 1).loc['osm']\n",
+ "s_eplus.name = 'Diff from 24.1.0 to 24.2.0'"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "s_eplus.to_frame().style.format(\"{:.2%}\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "large_eplus_diffs = s_eplus.index.tolist()\n",
+ "large_eplus_diffs"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "temp_dir = IDF_DIR / 'large_eplus_diffs'\n",
+ "temp_dir.mkdir(parents=True, exist_ok=True)\n",
+ "for f in large_eplus_diffs:\n",
+ " old_idf = OLD_OS_DIR / f\"{f}.osm.idf\"\n",
+ " assert old_idf.is_file()\n",
+ " trans_idf = TRANSITION_DIR / f\"{f}.osm.idf\"\n",
+ " assert trans_idf.is_file()\n",
+ " shutil.copyfile(old_idf, temp_dir / f\"{f}.osm_{EPLUS_OLD_VERSION.replace('.', '_')}.idf\")\n",
+ " copied_new_idf = temp_dir / f\"{f}.osm_{EPLUS_NEW_VERSION.replace('.', '_')}.idf\"\n",
+ " shutil.copyfile(trans_idf, copied_new_idf)\n",
+ " \n",
+ " subprocess.check_output([\n",
+ " '/home/julien/Software/Others/OS-build-release/Products/openstudio',\n",
+ " '-e',\n",
+ " f\"w = OpenStudio::Workspace.load('{copied_new_idf}').get(); w.save('{copied_new_idf}', true)\"\n",
+ " ])\n",
+ " "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "EPLUS_NEW_VERSION.replace('.', '_')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "temp_dir"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
@@ -2299,8 +2478,8 @@
" columns = ['FuelType','Units']\n",
" \"\"\"\n",
"\n",
- " abs_sql_path = os.path.abspath(sql_path)\n",
- " sql_uri = '{}?mode=ro'.format(pathlib.Path(abs_sql_path).as_uri())\n",
+ " abs_sql_path = Path(sql_path).absolute()\n",
+ " sql_uri = '{}?mode=ro'.format(abs_sql_path.as_uri())\n",
" \n",
" # RowName = '#{end_use}'\n",
" # ColumnName='#{fuel_type}'\n",
@@ -3585,8 +3764,8 @@
" (gotten from the name of the output_directory)\n",
" \"\"\"\n",
"\n",
- " abs_sql_path = os.path.abspath(sql_path)\n",
- " sql_uri = '{}?mode=ro'.format(pathlib.Path(abs_sql_path).as_uri())\n",
+ " abs_sql_path = Path(sql_path).absolute()\n",
+ " sql_uri = '{}?mode=ro'.format(abs_sql_path.as_uri())\n",
" with sqlite3.connect(sql_uri, uri=True) as con:\n",
" cursor = con.cursor()\n",
" r = cursor.execute(SQL_QUERY_SIM_INFO).fetchone()\n",
@@ -3712,7 +3891,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.9.7"
+ "version": "3.12.2"
},
"toc": {
"base_numbering": 1,