From 4e342011002c08819b5f0992366f9e5b1d01028d Mon Sep 17 00:00:00 2001
From: Callow <timjcallow@gmail.com>
Date: Tue, 7 Nov 2023 16:21:57 +0100
Subject: [PATCH 1/9] Add files for development test generation

---
 tests/dev_tests/benchmarking.py         | 276 ++++++++++++++++++++++++
 tests/dev_tests/pressure_benchmarks.csv | 101 +++++++++
 tests/dev_tests/run_benchmark_tests.py  |  35 +++
 tests/dev_tests/submit.slurm            |  12 ++
 tests/dev_tests/test.py                 |  66 ++++++
 5 files changed, 490 insertions(+)
 create mode 100644 tests/dev_tests/benchmarking.py
 create mode 100644 tests/dev_tests/pressure_benchmarks.csv
 create mode 100644 tests/dev_tests/run_benchmark_tests.py
 create mode 100644 tests/dev_tests/submit.slurm
 create mode 100644 tests/dev_tests/test.py

diff --git a/tests/dev_tests/benchmarking.py b/tests/dev_tests/benchmarking.py
new file mode 100644
index 0000000..8af584c
--- /dev/null
+++ b/tests/dev_tests/benchmarking.py
@@ -0,0 +1,276 @@
+"""
+Sets up and runs advanced benchmarking tests for developers.
+
+It is designed to submit many jobs simultaneously on an HPC system.
+Please see the README.md file for more information.
+"""
+
+import os
+import pandas as pd
+import shutil
+import json
+import subprocess
+import numpy as np
+
+
+def set_up_calcs(basedir, env, testfile="test.py"):
+    """
+    Set up calculations by creating directories and preparing files.
+
+    Parameters
+    ----------
+    basedir : str
+        The base directory where calculations will be set up.
+    env : str
+        The environment variable to be used in the calculations.
+    testfile : str, optional
+        The name of the test file to use for calculations.
+
+    Returns
+    -------
+    None
+    """
+    if os.path.exists(basedir):
+        user_input = (
+            input(
+                f"The directory {basedir} already exists."
+                + "Are you sure you want to continue? (yes/no): "
+            )
+            .strip()
+            .lower()
+        )
+        if user_input != "yes":
+            print("Operation cancelled.")
+            return
+
+    # Read the CSV file
+    df = pd.read_csv("pressure_benchmarks.csv")
+
+    # Loop through the rows of the dataframe
+    for index, row in df.iterrows():
+        species = row["species"]
+        rho = row["rho"]
+        rho_round = round(rho, 3)
+        temp = row["temp"]
+        temp_round = round(temp, 3)
+
+        # Create the sub-folder
+        subdir = os.path.join(basedir, f"{species}/rho_{rho_round}/T_{temp_round}")
+        os.makedirs(subdir, exist_ok=True)
+
+        # Read the original submit.slurm file
+        with open("submit.slurm", "r") as file:
+            filedata = file.read()
+
+        # Replace the target string
+        filedata = filedata.replace("env={ENV}", f"env={env}")
+
+        # Write the modified submit.slurm file to the new location
+        with open(subdir + "/submit.slurm", "w") as file:
+            file.write(filedata)
+
+        # same for python script
+        shutil.copy(testfile, subdir + "/test.py")
+
+        # Make a JSON file containing all the information from the row
+        row_dict = row.to_dict()
+        with open(os.path.join(subdir, "input.json"), "w") as f:
+            json.dump(row_dict, f, indent=4)
+
+    print("Setup complete.")
+
+
+def run_calcs(basedir):
+    """
+    Run calculations by submitting jobs to an HPC system.
+
+    Parameters
+    ----------
+    basedir : str
+        The base directory where calculations are set up.
+
+    Returns
+    -------
+    None
+    """
+    # Read the CSV file
+    df = pd.read_csv("pressure_benchmarks.csv")
+
+    # Loop through the rows of the dataframe
+    for index, row in df.iterrows():
+        species = row["species"]
+        rho = row["rho"]
+        rho_round = round(rho, 3)
+        temp = row["temp"]
+        temp_round = round(temp, 3)
+
+        # Define the sub-folder
+        subdir = os.path.join(basedir, f"{species}/rho_{rho_round}/T_{temp_round}")
+
+        # Check if the directory exists
+        if not os.path.exists(subdir):
+            print(f"Directory does not exist: {subdir}")
+            continue
+
+        # Check if submit.slurm exists in the directory
+        if not os.path.exists(os.path.join(subdir, "submit.slurm")):
+            print(f"submit.slurm does not exist in: {subdir}")
+            continue
+
+        # Execute sbatch submit.slurm
+        try:
+            subprocess.run(["sbatch", "submit.slurm"], check=True, cwd=subdir)
+            print(f"Job submitted for {species}, rho_{rho}, T_{temp}")
+        except subprocess.CalledProcessError as e:
+            print(f"Failed to submit job for {species}, rho_{rho}, T_{temp}: {e}")
+
+
+# Function to check if the values are close
+def check_close(input_val, output_val, atol=1e-9, rtol=1e-2):
+    """
+    Check if two values are close to each other within a tolerance.
+
+    By default it checks for a relative tolerance of 1%.
+    Absolute tolerance is not considered.
+
+    Parameters
+    ----------
+    input_val : float
+        The first value to compare.
+    output_val : float
+        The second value to compare.
+    atol : float, optional
+        The absolute tolerance parameter.
+    rtol : float, optional
+        The relative tolerance parameter.
+
+    Returns
+    -------
+    bool
+        True if the values are close within the specified tolerances, False otherwise.
+    """
+    return np.isclose(input_val, output_val, atol=atol, rtol=rtol)
+
+
+# Function to calculate the percentage error
+def calculate_percentage_error(value1, value2):
+    """
+    Calculate the percentage error between two values.
+
+    Parameters
+    ----------
+    value1 : float
+        The first value (reference value).
+    value2 : float
+        The second value (value to compare).
+
+    Returns
+    -------
+    float
+        The percentage error between the two values.
+    """
+    if value1 == 0 and value2 == 0:
+        return 0  # Avoid division by zero if both values are zero
+    try:
+        return round(abs((value1 - value2) / value1) * 100, 3)
+    except ZeroDivisionError:
+        return np.inf  # Return infinity if value1 is zero
+
+
+# Function to gather benchmark results and save to a new file
+def gather_benchmark_results(basedir, new_filename):
+    """
+    Gather benchmark results from JSON files and save them to a new CSV file.
+
+    Parameters
+    ----------
+    basedir : str
+        The base directory where the benchmark results are stored.
+    new_filename : str
+        The name of the new CSV file to save the results.
+
+    Returns
+    -------
+    None
+    """
+    # Read the existing dataframe
+    df = pd.read_csv("pressure_benchmarks.csv")
+
+    # Prepare a list to hold the new data
+    new_data = []
+
+    # Loop through the rows of the dataframe
+    for index, row in df.iterrows():
+        species = row["species"]
+        rho = row["rho"]
+        temp = row["temp"]
+
+        # Define the sub-folder
+        subdir = os.path.join(
+            basedir, f"{species}/rho_{round(rho, 3)}/T_{round(temp, 3)}"
+        )
+
+        # Initialize the outcome and percentage errors
+        outcome = "fail"
+        pc_err_st = pc_err_vir = pc_err_id = time_s = None
+
+        # Read input.json and output.json files
+        try:
+            with open(os.path.join(subdir, "input.json"), "r") as f:
+                input_data = json.load(f)
+
+            with open(os.path.join(subdir, "output.json"), "r") as f:
+                output_data = json.load(f)
+
+            # Extract the relevant values
+            P_st_rr_input = input_data["P_st_rr"]
+            P_vir_nocorr_input = input_data["P_vir_nocorr"]
+            P_id_input = input_data["P_id"]
+
+            P_st_rr_output = output_data["P_st_rr"]
+            P_vir_nocorr_output = output_data["P_vir_nocorr"]
+            P_id_output = output_data["P_id"]
+            time_s = round(output_data["time"], 2)
+
+            # Calculate percentage errors
+            pc_err_st = calculate_percentage_error(P_st_rr_input, P_st_rr_output)
+            pc_err_vir = calculate_percentage_error(
+                P_vir_nocorr_input, P_vir_nocorr_output
+            )
+            pc_err_id = calculate_percentage_error(P_id_input, P_id_output)
+
+            # Check if the values are close within the specified relative tolerance
+            # The original calculations were converged to 1%
+            rtol = 1e-2  # 1%
+            if (
+                np.isclose(P_st_rr_input, P_st_rr_output, rtol=rtol)
+                and np.isclose(P_vir_nocorr_input, P_vir_nocorr_output, rtol=rtol)
+                and np.isclose(P_id_input, P_id_output, rtol=rtol)
+            ):
+                outcome = "pass"
+
+        except Exception as e:
+            print(f"Error reading files for {species}, rho_{rho}, T_{temp}: {e}")
+            outcome = "fail"
+
+        # Append the new row to the data list
+        new_data.append(
+            [species, rho, temp, outcome, pc_err_st, pc_err_vir, pc_err_id, time_s]
+        )
+
+    # Create the new dataframe with additional columns for percentage errors
+    new_df_columns = [
+        "species",
+        "rho",
+        "temp",
+        "outcome",
+        "pc_err_st",
+        "pc_err_vir",
+        "pc_err_id",
+        "time_s",
+    ]
+    new_df = pd.DataFrame(new_data, columns=new_df_columns)
+
+    # Save the new dataframe to the specified CSV file
+    new_df.to_csv(new_filename, index=False)
+    print(f"Results saved to {new_filename}")
diff --git a/tests/dev_tests/pressure_benchmarks.csv b/tests/dev_tests/pressure_benchmarks.csv
new file mode 100644
index 0000000..1c90957
--- /dev/null
+++ b/tests/dev_tests/pressure_benchmarks.csv
@@ -0,0 +1,101 @@
+species,rho,temp,nconv,nmax,lmax,ngrid,nkpts,P_id,P_st_rr,P_vir_nocorr_A
+C,15.499297,2786.4411157095856,1e-05,24,64,500,20,2072044.7230353267,2017313.5879749784,2039377.1269299167
+C,19.374129,0.861732831109502,1e-05,13,13,500,20,16103.546265615396,21595.25568343568,16021.731183437241
+Mg,17.222007,1393.2206009414344,1e-05,24,60,500,20,1095119.300216119,1068912.6205698776,1075873.3758255616
+H,0.578952,15.66845720164852,1e-05,13,17,500,20,742.5591618533304,742.3922037993696,585.6867926766153
+C,19.374129,696.6102573840757,1e-05,20,36,500,20,622269.3306860247,598344.1075766252,606683.5332387362
+Si,18.631996,696.6102573840757,1e-05,22,44,500,20,531446.1352081429,518812.6632073754,519672.61316564155
+Ne,7.895891,0.430866415554751,1.0000000000000002e-07,19,19,750,225,1620.4179778840714,240.08536105528836,-140.74839213597792
+O,3.630449,348.3055164718118,1e-05,22,44,500,20,54565.730665016374,53604.23695665938,53304.06291678641
+He,5.353887,0.430866415554751,1e-05,13,13,500,100,1517.2580597614203,1730.0200558186996,1522.4201583723757
+Mg,64.582573,116.10169520179876,1e-05,16,20,500,20,266905.8253467549,248319.91539609712,234440.24739279025
+O,14.863006,8574.040885789896,1e-05,36,120,1000,20,6156025.076233147,6035788.265357383,6071568.360390914
+Na,9.666274,21.76900860620513,1e-05,13,17,500,20,3946.783439708051,3955.4136520148368,3034.0137871641264
+He,0.502807,0.430866415554751,1.0000000000000002e-06,16,16,1125,337,6.133380717138889,-7.725225617444218,-11.096506480945479
+Al,9.445146,4.30866415554751,1.0000000000000002e-06,16,16,500,20,2628.5089477983142,3194.662500983706,1900.2567801261862
+Mg,17.222007,1.723465662219004,1.0000000000000002e-06,16,16,500,20,11961.893152232244,7819.237715107365,4269.42263784376
+Al,1.349307,1393.2206009414344,1e-05,40,128,1000,20,86013.76469098694,84806.08443695665,84763.93445120311
+Al,13.493065,0.861732831109502,1.0000000000000002e-06,16,16,500,20,5851.968973077679,6128.275963275882,3547.59116977093
+He,5.353887,0.0861732831109502,1e-05,13,13,500,67,1674.0879962460224,1734.475940857292,1523.9729286474012
+Ne,7.895891,21.54332077773755,1e-05,13,17,500,20,3229.1298428058712,3126.162104865358,2339.265358263317
+B,24.651668,2786.4411157095856,1e-05,22,56,500,20,3057303.207049976,2961155.2552372273,3006617.147725778
+Mg,8.611005,64.62996233321265,1e-05,18,24,500,20,12243.014832745544,12507.159891425832,11353.825157241165
+O,14.863006,174.1527582359059,1e-05,18,28,500,20,88504.34430701211,85850.39449506745,84007.74072717172
+H,12.483579,1378.772529775203,1e-05,18,28,500,20,1694684.8823776576,1570453.6428157066,1646972.544658832
+Na,4.833143,174.1525858893397,1e-05,20,36,500,20,26075.469177663486,25899.663020546384,25195.219285855874
+Mg,19.374763,2.585198493328506,1.0000000000000002e-06,16,16,500,20,15201.58028765471,9466.931256192258,5263.265766508805
+He,0.502807,0.0430866415554751,1.0000000000000002e-06,16,16,1125,337,18.061188707619593,-7.725199018052239,-11.098148387291118
+C,12.635119,13932.185241653116,1e-05,40,140,1000,20,8516607.379886763,8369004.500871842,8401826.80450733
+N,2.52736,0.1723465662219004,1.0000000000000002e-07,19,19,500,30,2.075276973001873,69.5351896626629,-124.94136850303559
+H,0.099869,344.6931324438008,1e-05,20,52,500,20,3301.9030178867306,3219.086812714775,3242.609496543868
+B,36.977477,87.07629294466985,1e-05,13,17,500,20,129929.55355816505,121284.47865108424,117064.5434231308
+H,5.266511,344.6931324438008,1e-05,16,24,500,20,181204.16552210183,164418.37171375376,172667.09473191603
+N,2.52736,64.62996233321265,1.0000000000000002e-06,21,31,500,20,4483.745861729148,4540.798453493657,4226.014189057272
+C,19.374129,253.31282870087145,1e-05,18,28,500,20,201073.28043859446,192306.01322537693,192143.49567784954
+Mg,51.666042,2786.4411157095856,1e-05,24,60,500,20,6697284.531336954,6506423.612025182,6584255.1290801
+H,0.337057,1378.772529775203,1e-05,22,64,500,20,44736.807116474716,43603.10389107079,44053.15686159396
+Mg,1.076376,116.10169520179876,1e-05,22,44,500,20,3766.365140142541,3800.697648459035,3650.0034362991623
+B,24.651668,116.10169520179876,1e-05,16,20,500,20,103203.29138933303,97557.21120025766,95285.38254171523
+Mg,11.624855,43.0866415554751,1e-05,18,24,500,67,10057.94088660828,10702.47710831082,9249.96432616323
+H,0.153417,10.771660388868774,1e-05,13,17,500,20,114.29669542755276,122.32074721750365,91.50819109751524
+H,0.012484,86.1732831109502,1e-05,20,52,500,20,102.1212968405153,101.17078321965263,99.94848586909782
+B,12.325826,0.861732831109502,1.0000000000000002e-06,16,16,500,20,7331.516977271224,9906.389563880288,6389.985486625057
+Ne,15.025693,0.2154332077773755,1.0000000000000002e-06,16,16,500,45,6847.908318250356,2822.3253804670408,1447.9742275631058
+H,0.41904,2.6929150972171936,1e-05,13,13,500,45,55.774132075681464,123.84122617979442,15.392067534998734
+H,0.195057,21.54332077773755,1e-05,16,20,500,20,352.80701165497095,357.5125169917259,315.3183416031003
+Mg,43.055026,11145.764549011628,1e-05,36,112,500,20,22843759.670317728,22538815.6110869,22519877.799240515
+H,0.021572,10.771660388868774,1e-05,16,24,500,20,17.724820762671545,19.49486101825701,16.300197454979777
+C,8.500494,0.430866415554751,1.0000000000000002e-06,16,16,500,20,2100.346957570165,4348.434970767885,2275.8219841213017
+Al,8.095839,8.61732831109502,1.0000000000000002e-06,16,20,500,20,1844.917209555024,2434.0457559681968,1600.6565215364083
+Si,6.987,87.07629294466985,1e-05,18,28,500,20,13594.509951774591,13749.20748217314,12779.112003476848
+Ne,3.728316,64.62996233321265,1e-05,18,28,500,20,5810.035083449752,5942.947612569971,5459.732274366159
+N,2.0,3.446931324438008,1.0000000000000002e-07,19,19,500,100,22.957932598350343,46.6371236967057,-41.14470694851638
+Na,3.86651,696.6102573840757,1e-05,26,68,500,20,117184.52237752012,115256.15571256407,115098.31066130062
+C,0.100006,0.861732831109502,1.0000000000000002e-08,19,19,7593,30,0.0017044666999578198,-0.03543056474796335,-0.04573489324137927
+H,0.153417,172.3465662219004,1e-05,18,36,500,20,2524.5598518494394,2449.7039213881426,2462.601397838032
+He,2.74119,21.54332077773755,1e-05,13,17,500,20,2072.7990362862133,1925.2027888894665,1553.9319743658525
+B,17.256161,0.861732831109502,1.0000000000000002e-06,16,16,500,20,14885.885189890676,17776.144693373008,12627.508200846942
+Al,9.445146,69.66101712107925,1e-05,18,24,500,20,13800.187996344275,14060.228071850977,12813.518491120296
+C,25.832156,5572.882317592455,1e-05,26,76,500,20,6940555.130916195,6771865.5271821935,6837936.951340649
+N,1.75,3.446931324438008,1.0000000000000002e-07,19,19,500,100,18.106748136788806,27.7768530034472,-36.09982786914078
+C,4.174268,435.3814647233492,1e-05,22,44,500,20,82877.85420219765,80903.44557553809,81026.51254810768
+H,0.982675,8.208005216318007,1e-05,13,13,500,20,749.8024726092902,856.1409450479246,566.339027717984
+B,9.860662,21.76900860620513,1.0000000000000002e-06,16,20,500,20,7399.216392922991,8737.71839262863,7059.388342108884
+Mg,51.666042,696.6102573840757,1e-05,20,36,500,20,1462507.6174028104,1407932.5945653832,1420750.3592689722
+Na,7.733027,87.07629294466985,1e-05,18,28,500,20,16770.689241152682,16896.262015579578,15781.789069926801
+Mg,25.833021,2786.4411157095856,1e-05,26,72,500,20,3365633.013756881,3287110.163200919,3312641.872531811
+N,1.75,0.1723465662219004,1.0000000000000002e-06,16,16,750,67,0.00015626913177607007,3.5537687896254395,-83.86969868379147
+Al,16.191678,87.07629294466985,1e-05,18,24,500,20,33575.624850874716,33704.12636102551,31233.941059454155
+Mg,43.055026,3.446931324438008,1.0000000000000002e-06,16,16,500,20,70520.96381561867,40024.958972449334,27155.583353105714
+O,7.261642,86.1732831109502,1e-05,18,24,500,20,18159.014850425745,18034.475615439045,16934.09581004218
+Al,0.674653,696.6102573840757,1e-05,36,116,1000,20,21100.567885432876,20849.884504597376,20775.189380687298
+N,2.75,3.446931324438008,1.0000000000000002e-07,19,19,750,67,45.28898213823985,142.29804259068027,-36.8036903679005
+Si,2.329,174.1525858893397,1e-05,22,44,500,20,12901.774162791582,12907.999910164795,12532.701517759473
+O,2.486325,8574.040885789896,1e-05,56,200,1000,20,1031651.4565232176,1023374.9291797916,1017488.4658179588
+He,0.669236,2757.545059550406,1e-05,34,112,1000,20,89348.6872097471,87635.9315089785,88104.20108905066
+He,2.74119,1.723465662219004,1.0000000000000002e-06,16,16,500,100,257.0643706330844,297.20574127470303,168.50438931780909
+N,3.705859,0.861732831109502,1.0000000000000002e-07,19,19,500,67,37.05440774029069,310.1767632827016,-88.95095787518439
+Al,16.191678,18576.27424835271,1e-05,54,192,1000,20,14017816.344198935,13846420.025505768,13829834.983796185
+N,2.0,0.5170396986657012,1.0000000000000002e-07,19,19,500,30,0.24752000903241236,21.201548398848367,-93.27351974150228
+Al,9.445146,5572.882317592455,1e-05,40,132,1000,20,2438477.186149089,2396723.5566470046,2404261.118093777
+B,24.651668,58.050890687540935,1e-05,13,17,500,20,54822.25071717421,53514.59970763821,49802.817804532126
+B,0.246517,87.07629294466985,1e-05,22,48,500,20,791.91747017267,797.563696885084,770.2670206289662
+N,2.52736,0.6462996233321265,1.0000000000000002e-07,19,19,500,30,3.251339158002116,74.01425602715815,-114.41743304446022
+Ne,7.895891,4.30866415554751,1.0000000000000002e-07,19,19,750,150,916.4214368677691,276.9479038440781,-120.56117740932196
+B,11.093243,116.10169520179876,1e-05,18,24,500,20,43657.7500610936,42168.84351212618,40940.173207310414
+B,12.325826,10.884461216461007,1.0000000000000002e-06,16,16,500,20,8455.229603605061,10660.352836897882,7821.516574773115
+N,2.424,1.723465662219004,1.0000000000000002e-07,19,19,500,45,8.871859592182247,77.29472711009203,-75.55783075600385
+B,19.721324,21.76900860620513,1.0000000000000002e-06,16,20,500,20,24356.011107891016,26116.684084952467,21808.51105041802
+C,8.500494,13932.185241653116,1e-05,44,156,1000,20,5730660.080595019,5643676.005689563,5653126.29938754
+C,12.635119,21.54332077773755,1.0000000000000002e-06,16,20,500,20,10273.347515179765,12612.286732659137,10085.38828684373
+C,25.832156,253.31282870087145,1e-05,18,24,500,20,268613.3784107917,255196.37863070218,255580.8245776044
+Mg,12.916506,1393.2206009414344,1e-05,24,64,500,20,824963.0171843449,806800.1357665508,810924.7730660931
+H,0.982675,86.1732831109502,1e-05,16,24,500,20,8144.758006680799,7606.654133757894,7614.509904700588
+Si,11.645,21.54332077773755,1.0000000000000002e-06,16,20,500,20,4891.991070187644,6021.293349262431,4705.302480247869
+Al,21.588905,2.585198493328506,1.0000000000000002e-06,16,16,500,20,15902.235796147248,13607.953442528125,8313.764690889153
+N,7.829576,4.30866415554751,1.0000000000000002e-06,16,16,500,20,986.8234534579067,2461.583979828196,1219.042096490022
+H,0.00156,21.54332077773755,1e-05,22,52,500,20,3.1169687051281896,3.1936813649872935,3.0279557694947297
+Na,7.733027,17.415189650307482,1e-05,13,17,500,20,2276.3231648714886,2330.1155539733827,1725.48946617373
+O,7.261642,696.6118946764548,1e-05,22,52,500,20,232103.35655636596,226485.58313443142,227473.4649333156
+Al,18.890291,0.861732831109502,1.0000000000000002e-06,16,16,500,20,11850.441039502504,10927.095624507278,6543.079675754301
+B,2.465165,2786.4411157095856,1e-05,34,108,1000,20,306607.69808648265,300637.8484254644,302270.2302787752
diff --git a/tests/dev_tests/run_benchmark_tests.py b/tests/dev_tests/run_benchmark_tests.py
new file mode 100644
index 0000000..9a8d3e0
--- /dev/null
+++ b/tests/dev_tests/run_benchmark_tests.py
@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+
+"""Example of how to run the benchmark tests."""
+
+import sys
+from datetime import datetime
+import benchmarking
+
+
+# user inputs - MUST BE EDITED
+
+env = "atoMEC312"  # name of conda environment
+
+# location for calculations - we set up a new folder based on the date and environment
+savedir = datetime.now().strftime("%d-%m-%y") + "_" + env
+
+# type of calculation: either "setup", "run", "setup_and_run", or "evaluate"
+calc_type = "evaluate"
+
+# end of user inputs
+
+if "setup" in calc_type:
+    benchmarking.set_up_calcs(savedir, env, testfile="test.py")
+
+if "run" in calc_type:
+    benchmarking.run_calcs(savedir)
+
+if calc_type == "evaluate":
+    benchmarking.gather_benchmark_results(savedir, savedir + "_benchmark_results.csv")
+
+if calc_type not in ["setup", "setup_and_run", "run", "evaluate"]:
+    sys.exit(
+        "Calculation type not recognised. Must be one of "
+        + "'setup', 'setup_and_run', 'run', or 'evaluate'"
+    )
diff --git a/tests/dev_tests/submit.slurm b/tests/dev_tests/submit.slurm
new file mode 100644
index 0000000..29a9857
--- /dev/null
+++ b/tests/dev_tests/submit.slurm
@@ -0,0 +1,12 @@
+#!/bin/bash -l
+#SBATCH --partition=rome
+#SBATCH --ntasks=10
+#SBATCH --nodes=1
+#SBATCH --mem=80G
+
+env={ENV}
+module purge
+conda activate $env
+export OMP_NUM_THREADS=1
+
+python -u test.py > test.log
diff --git a/tests/dev_tests/test.py b/tests/dev_tests/test.py
new file mode 100644
index 0000000..fd352d0
--- /dev/null
+++ b/tests/dev_tests/test.py
@@ -0,0 +1,66 @@
+"""Test template file for atoMEC."""
+
+from atoMEC import Atom, models, config, mathtools, unitconv
+from atoMEC.postprocess import pressure
+import json
+import time
+
+st_time = time.time()
+
+config.numcores = -1
+
+# load input params and set up the atom and model objects
+with open("input.json", "r") as f:
+    input = json.load(f)
+
+atom = Atom(input["species"], input["temp"], density=input["rho"], units_temp="eV")
+model = models.ISModel(atom, bc="bands", unbound="quantum")
+
+# the scf calculation
+out = model.CalcEnergy(
+    input["nmax"],
+    input["lmax"],
+    grid_params={"ngrid": input["ngrid"], "s0": 1e-4},
+    conv_params={"nconv": input["nconv"], "vconv": 1e-1, "econv": 1e-2},
+    band_params={"nkpts": input["nkpts"]},
+    scf_params={"maxscf": 30},
+    write_info=True,
+    grid_type="sqrt",
+)
+
+# stress tensor pressure
+P_st_rr = pressure.stress_tensor(
+    atom, model, out["orbitals"], out["potential"], only_rr=True
+)
+
+# virial pressure
+P_vir_nocorr = pressure.virial(
+    atom,
+    model,
+    out["energy"],
+    out["density"],
+    out["orbitals"],
+    out["potential"],
+    use_correction=False,
+    method="B",
+)
+
+# compute ideal pressure
+chem_pot = mathtools.chem_pot(out["orbitals"])
+P_elec_id = pressure.ideal_electron(atom, chem_pot)
+
+end_time = time.time()
+tdiff = end_time - st_time
+
+output_dict = {
+    "species": input["species"],
+    "temp": input["temp"],
+    "rho": input["rho"],
+    "P_st_rr": unitconv.ha_to_gpa * P_st_rr,
+    "P_vir_nocorr": unitconv.ha_to_gpa * P_vir_nocorr,
+    "P_id": unitconv.ha_to_gpa * P_elec_id,
+    "time": tdiff,
+}
+
+with open("output.json", "w") as f:
+    json.dump(output_dict, f)

From d000bbc4c665c3199ecc918201f08fa8a5bf9266 Mon Sep 17 00:00:00 2001
From: Callow <timjcallow@gmail.com>
Date: Tue, 7 Nov 2023 16:49:26 +0100
Subject: [PATCH 2/9] Add atoMEC 1.4.0 benchmark timings

---
 tests/dev_tests/atoMEC_v1.4.0_py312.csv | 101 ++++++++++++++++++++++++
 1 file changed, 101 insertions(+)
 create mode 100644 tests/dev_tests/atoMEC_v1.4.0_py312.csv

diff --git a/tests/dev_tests/atoMEC_v1.4.0_py312.csv b/tests/dev_tests/atoMEC_v1.4.0_py312.csv
new file mode 100644
index 0000000..375904c
--- /dev/null
+++ b/tests/dev_tests/atoMEC_v1.4.0_py312.csv
@@ -0,0 +1,101 @@
+species,rho,temp,outcome,pc_err_st,pc_err_vir,pc_err_id,time_s
+C,15.499297,2786.4411157095856,pass,0.001,0.001,0.0,15.7
+C,19.374129,0.861732831109502,pass,0.056,0.175,0.088,7.54
+Mg,17.222007,1393.2206009414344,pass,0.0,0.0,0.0,12.76
+H,0.578952,15.66845720164852,pass,0.0,0.0,0.0,4.43
+C,19.374129,696.6102573840757,pass,0.0,0.0,0.0,7.8
+Si,18.631996,696.6102573840757,pass,0.007,0.009,0.007,19.1
+Ne,7.895891,0.430866415554751,pass,0.004,0.038,0.001,150.79
+O,3.630449,348.3055164718118,pass,0.007,0.007,0.007,14.78
+He,5.353887,0.430866415554751,pass,0.065,0.099,0.031,15.78
+Mg,64.582573,116.10169520179876,pass,0.014,0.027,0.013,10.72
+O,14.863006,8574.040885789896,pass,0.0,0.002,0.0,45.38
+Na,9.666274,21.76900860620513,pass,0.048,0.183,0.05,7.91
+He,0.502807,0.430866415554751,pass,0.003,0.023,0.004,214.72
+Al,9.445146,4.30866415554751,pass,0.009,0.072,0.014,11.2
+Mg,17.222007,1.723465662219004,pass,0.008,0.043,0.008,10.19
+Al,1.349307,1393.2206009414344,pass,0.001,0.001,0.0,93.36
+Al,13.493065,0.861732831109502,pass,0.007,0.049,0.011,11.31
+He,5.353887,0.0861732831109502,pass,0.062,0.098,0.024,11.09
+Ne,7.895891,21.54332077773755,pass,0.042,0.15,0.041,7.85
+B,24.651668,2786.4411157095856,pass,0.0,0.0,0.0,11.03
+Mg,8.611005,64.62996233321265,pass,0.016,0.041,0.014,11.78
+O,14.863006,174.1527582359059,pass,0.008,0.016,0.008,12.52
+H,12.483579,1378.772529775203,pass,0.0,0.0,0.0,5.02
+Na,4.833143,174.1525858893397,pass,0.009,0.023,0.009,17.99
+Mg,19.374763,2.585198493328506,pass,0.007,0.033,0.006,11.07
+He,0.502807,0.0430866415554751,pass,0.003,0.023,0.002,211.46
+C,12.635119,13932.185241653116,pass,0.005,0.006,0.001,58.24
+N,2.52736,0.1723465662219004,pass,0.007,0.015,0.021,20.25
+H,0.099869,344.6931324438008,pass,0.001,0.001,0.0,8.6
+B,36.977477,87.07629294466985,pass,0.011,0.016,0.012,6.53
+H,5.266511,344.6931324438008,pass,0.0,0.0,0.0,4.42
+N,2.52736,64.62996233321265,pass,0.001,0.004,0.001,20.22
+C,19.374129,253.31282870087145,pass,0.009,0.01,0.008,10.89
+Mg,51.666042,2786.4411157095856,pass,0.001,0.001,0.0,12.48
+H,0.337057,1378.772529775203,pass,0.006,0.006,0.001,11.11
+Mg,1.076376,116.10169520179876,pass,0.008,0.026,0.008,21.93
+B,24.651668,116.10169520179876,pass,0.01,0.014,0.01,8.51
+Mg,11.624855,43.0866415554751,pass,0.103,0.01,0.058,27.43
+H,0.153417,10.771660388868774,pass,0.002,0.003,0.002,3.8
+H,0.012484,86.1732831109502,pass,0.001,0.001,0.0,9.51
+B,12.325826,0.861732831109502,pass,0.003,0.014,0.006,8.67
+Ne,15.025693,0.2154332077773755,pass,0.01,0.062,0.005,16.2
+H,0.41904,2.6929150972171936,pass,0.0,0.003,0.0,3.95
+H,0.195057,21.54332077773755,pass,0.0,0.0,0.0,3.99
+Mg,43.055026,11145.764549011628,pass,0.016,0.003,0.003,132.75
+H,0.021572,10.771660388868774,pass,0.006,0.005,0.005,6.13
+C,8.500494,0.430866415554751,pass,0.007,0.033,0.011,9.64
+Al,8.095839,8.61732831109502,pass,0.01,0.078,0.015,12.73
+Si,6.987,87.07629294466985,pass,0.01,0.029,0.01,13.81
+Ne,3.728316,64.62996233321265,pass,0.012,0.034,0.011,13.81
+N,2.0,3.446931324438008,pass,0.004,0.021,0.003,49.75
+Na,3.86651,696.6102573840757,pass,0.0,0.0,0.0,14.57
+C,0.100006,0.861732831109502,pass,0.0,0.0,0.0,238.96
+H,0.153417,172.3465662219004,pass,0.0,0.0,0.0,6.27
+He,2.74119,21.54332077773755,pass,0.02,0.024,0.017,7.17
+B,17.256161,0.861732831109502,pass,0.003,0.009,0.005,9.38
+Al,9.445146,69.66101712107925,pass,0.015,0.041,0.014,12.73
+C,25.832156,5572.882317592455,pass,0.001,0.002,0.0,14.21
+N,1.75,3.446931324438008,pass,0.007,0.025,0.004,50.62
+C,4.174268,435.3814647233492,pass,0.0,0.0,0.0,8.13
+H,0.982675,8.208005216318007,pass,0.0,0.0,0.0,3.3
+B,9.860662,21.76900860620513,pass,0.004,0.014,0.005,9.95
+Mg,51.666042,696.6102573840757,pass,0.008,0.009,0.008,13.73
+Na,7.733027,87.07629294466985,pass,0.011,0.028,0.01,13.62
+Mg,25.833021,2786.4411157095856,pass,0.0,0.0,0.0,14.95
+N,1.75,0.1723465662219004,pass,0.486,0.097,0.789,33.7
+Al,16.191678,87.07629294466985,pass,0.013,0.032,0.012,11.84
+Mg,43.055026,3.446931324438008,pass,0.003,0.01,0.002,9.99
+O,7.261642,86.1732831109502,pass,0.012,0.033,0.012,11.88
+Al,0.674653,696.6102573840757,pass,0.001,0.001,0.0,50.67
+N,2.75,3.446931324438008,pass,0.003,0.048,0.004,51.43
+Si,2.329,174.1525858893397,pass,0.006,0.017,0.006,21.87
+O,2.486325,8574.040885789896,pass,0.195,0.006,0.002,659.87
+He,0.669236,2757.545059550406,pass,0.008,0.007,0.001,37.36
+He,2.74119,1.723465662219004,pass,0.007,0.017,0.004,31.03
+N,3.705859,0.861732831109502,pass,0.003,0.031,0.006,35.16
+Al,16.191678,18576.27424835271,pass,0.431,0.007,0.011,590.86
+N,2.0,0.5170396986657012,pass,0.016,0.015,0.031,20.93
+Al,9.445146,5572.882317592455,pass,0.013,0.015,0.007,163.42
+B,24.651668,58.050890687540935,pass,0.014,0.026,0.016,9.44
+B,0.246517,87.07629294466985,pass,0.007,0.011,0.007,21.29
+N,2.52736,0.6462996233321265,pass,0.007,0.017,0.016,22.3
+Ne,7.895891,4.30866415554751,pass,0.004,0.046,0.002,104.5
+B,11.093243,116.10169520179876,pass,0.011,0.015,0.01,14.7
+B,12.325826,10.884461216461007,pass,0.004,0.013,0.006,9.79
+N,2.424,1.723465662219004,pass,0.006,0.026,0.01,27.94
+B,19.721324,21.76900860620513,pass,0.003,0.007,0.004,12.75
+C,8.500494,13932.185241653116,pass,0.216,0.002,0.005,71.59
+C,12.635119,21.54332077773755,pass,0.003,0.009,0.003,11.83
+C,25.832156,253.31282870087145,pass,0.01,0.011,0.01,9.25
+Mg,12.916506,1393.2206009414344,pass,0.0,0.0,0.0,13.09
+H,0.982675,86.1732831109502,pass,0.0,0.0,0.0,4.82
+Si,11.645,21.54332077773755,pass,0.005,0.025,0.006,12.85
+Al,21.588905,2.585198493328506,pass,0.007,0.035,0.009,10.68
+N,7.829576,4.30866415554751,pass,0.006,0.034,0.01,9.99
+H,0.00156,21.54332077773755,pass,0.001,0.001,0.0,9.58
+Na,7.733027,17.415189650307482,pass,0.049,0.228,0.052,8.68
+O,7.261642,696.6118946764548,pass,0.0,0.0,0.0,10.05
+Al,18.890291,0.861732831109502,pass,0.006,0.031,0.008,10.77
+B,2.465165,2786.4411157095856,pass,0.002,0.002,0.0,34.35

From 8d0505b849e124de3691b2cbcc2e2d732c08eb19 Mon Sep 17 00:00:00 2001
From: Callow <timjcallow@gmail.com>
Date: Tue, 7 Nov 2023 16:54:28 +0100
Subject: [PATCH 3/9] Add README draft

---
 tests/dev_tests/README.md | 39 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)
 create mode 100644 tests/dev_tests/README.md

diff --git a/tests/dev_tests/README.md b/tests/dev_tests/README.md
new file mode 100644
index 0000000..5733429
--- /dev/null
+++ b/tests/dev_tests/README.md
@@ -0,0 +1,39 @@
+# Development Test Suite for atoMEC
+
+## Overview
+
+This directory is equipped with the necessary code and datasets to facilitate the generation and execution of development tests for the atoMEC project. These tests are tailored to evaluate the performance of the codebase, with a focus on the `CalcEnergy` function, its related components, and behavior under extreme edge cases. While the `tests/` folder contains CI tests that verify the code's functionality, the development tests here are aimed at benchmarking performance. They are not mandatory but are recommended for developers making significant changes to performance-critical parts of the code, especially when modifications impact the execution time observed in CI tests.
+
+## Development Testing Tools
+
+The development tests themselves are not directly included. Instead, the repository provides the necessary tools to generate and run these tests:
+
+- `benchmarking.py`: The core module containing functions to set up the benchmarking environment.
+- `pressure_benchmarks.csv`: The dataset containing parameters for generating test cases.
+- `test.py`: The template for creating individual test scripts.
+- `submit.slurm`: A sample SLURM submission script for use on HPC systems.
+- `run_benchmark_tests.py`: A script that demonstrates how to orchestrate the entire testing workflow using the provided tools.
+
+## Environment Assumption
+
+The execution of these scripts presupposes that atoMEC is operated within a Conda environment, ensuring a consistent and controlled runtime for the tests.
+
+## Benchmarking Protocol
+
+Benchmarking should be conducted against the results from the most recent iteration of the development branch, currently documented as `atoMEC_v1.4.0_py312.csv`. This comparison is critical for tracking performance changes introduced by recent code updates.
+
+## Execution Instructions
+
+To initiate the benchmarking process, ensure that your Conda environment is active and that you have the necessary HPC system access.
+
+Follow these steps to prepare and conduct the benchmarks:
+
+1. Utilize the functions within `benchmarking.py` to prepare the calculation directories as dictated by `pressure_benchmarks.csv`.
+2. Distribute the `test.py` script template and the `submit.slurm` file to their designated locations.
+3. Launch the `run_benchmark_tests.py` script to automate the submission of benchmarking jobs to the HPC system.
+
+Refer to the documentation within each file for more detailed instructions on their usage.
+
+## Contributing
+
+Developers are invited to enhance the test suite by updating the templates, source code, and datasets to align with the evolving requirements of the atoMEC project. Your contributions are vital to maintaining the project's standards of robustness and efficiency.

From 7d11d433ecd10cb078cefa2b4e88e652197c0bef Mon Sep 17 00:00:00 2001
From: Callow <timjcallow@gmail.com>
Date: Tue, 7 Nov 2023 17:22:52 +0100
Subject: [PATCH 4/9] Delete timings benchmark file

---
 tests/dev_tests/atoMEC_v1.4.0_py312.csv | 101 ------------------------
 1 file changed, 101 deletions(-)
 delete mode 100644 tests/dev_tests/atoMEC_v1.4.0_py312.csv

diff --git a/tests/dev_tests/atoMEC_v1.4.0_py312.csv b/tests/dev_tests/atoMEC_v1.4.0_py312.csv
deleted file mode 100644
index 375904c..0000000
--- a/tests/dev_tests/atoMEC_v1.4.0_py312.csv
+++ /dev/null
@@ -1,101 +0,0 @@
-species,rho,temp,outcome,pc_err_st,pc_err_vir,pc_err_id,time_s
-C,15.499297,2786.4411157095856,pass,0.001,0.001,0.0,15.7
-C,19.374129,0.861732831109502,pass,0.056,0.175,0.088,7.54
-Mg,17.222007,1393.2206009414344,pass,0.0,0.0,0.0,12.76
-H,0.578952,15.66845720164852,pass,0.0,0.0,0.0,4.43
-C,19.374129,696.6102573840757,pass,0.0,0.0,0.0,7.8
-Si,18.631996,696.6102573840757,pass,0.007,0.009,0.007,19.1
-Ne,7.895891,0.430866415554751,pass,0.004,0.038,0.001,150.79
-O,3.630449,348.3055164718118,pass,0.007,0.007,0.007,14.78
-He,5.353887,0.430866415554751,pass,0.065,0.099,0.031,15.78
-Mg,64.582573,116.10169520179876,pass,0.014,0.027,0.013,10.72
-O,14.863006,8574.040885789896,pass,0.0,0.002,0.0,45.38
-Na,9.666274,21.76900860620513,pass,0.048,0.183,0.05,7.91
-He,0.502807,0.430866415554751,pass,0.003,0.023,0.004,214.72
-Al,9.445146,4.30866415554751,pass,0.009,0.072,0.014,11.2
-Mg,17.222007,1.723465662219004,pass,0.008,0.043,0.008,10.19
-Al,1.349307,1393.2206009414344,pass,0.001,0.001,0.0,93.36
-Al,13.493065,0.861732831109502,pass,0.007,0.049,0.011,11.31
-He,5.353887,0.0861732831109502,pass,0.062,0.098,0.024,11.09
-Ne,7.895891,21.54332077773755,pass,0.042,0.15,0.041,7.85
-B,24.651668,2786.4411157095856,pass,0.0,0.0,0.0,11.03
-Mg,8.611005,64.62996233321265,pass,0.016,0.041,0.014,11.78
-O,14.863006,174.1527582359059,pass,0.008,0.016,0.008,12.52
-H,12.483579,1378.772529775203,pass,0.0,0.0,0.0,5.02
-Na,4.833143,174.1525858893397,pass,0.009,0.023,0.009,17.99
-Mg,19.374763,2.585198493328506,pass,0.007,0.033,0.006,11.07
-He,0.502807,0.0430866415554751,pass,0.003,0.023,0.002,211.46
-C,12.635119,13932.185241653116,pass,0.005,0.006,0.001,58.24
-N,2.52736,0.1723465662219004,pass,0.007,0.015,0.021,20.25
-H,0.099869,344.6931324438008,pass,0.001,0.001,0.0,8.6
-B,36.977477,87.07629294466985,pass,0.011,0.016,0.012,6.53
-H,5.266511,344.6931324438008,pass,0.0,0.0,0.0,4.42
-N,2.52736,64.62996233321265,pass,0.001,0.004,0.001,20.22
-C,19.374129,253.31282870087145,pass,0.009,0.01,0.008,10.89
-Mg,51.666042,2786.4411157095856,pass,0.001,0.001,0.0,12.48
-H,0.337057,1378.772529775203,pass,0.006,0.006,0.001,11.11
-Mg,1.076376,116.10169520179876,pass,0.008,0.026,0.008,21.93
-B,24.651668,116.10169520179876,pass,0.01,0.014,0.01,8.51
-Mg,11.624855,43.0866415554751,pass,0.103,0.01,0.058,27.43
-H,0.153417,10.771660388868774,pass,0.002,0.003,0.002,3.8
-H,0.012484,86.1732831109502,pass,0.001,0.001,0.0,9.51
-B,12.325826,0.861732831109502,pass,0.003,0.014,0.006,8.67
-Ne,15.025693,0.2154332077773755,pass,0.01,0.062,0.005,16.2
-H,0.41904,2.6929150972171936,pass,0.0,0.003,0.0,3.95
-H,0.195057,21.54332077773755,pass,0.0,0.0,0.0,3.99
-Mg,43.055026,11145.764549011628,pass,0.016,0.003,0.003,132.75
-H,0.021572,10.771660388868774,pass,0.006,0.005,0.005,6.13
-C,8.500494,0.430866415554751,pass,0.007,0.033,0.011,9.64
-Al,8.095839,8.61732831109502,pass,0.01,0.078,0.015,12.73
-Si,6.987,87.07629294466985,pass,0.01,0.029,0.01,13.81
-Ne,3.728316,64.62996233321265,pass,0.012,0.034,0.011,13.81
-N,2.0,3.446931324438008,pass,0.004,0.021,0.003,49.75
-Na,3.86651,696.6102573840757,pass,0.0,0.0,0.0,14.57
-C,0.100006,0.861732831109502,pass,0.0,0.0,0.0,238.96
-H,0.153417,172.3465662219004,pass,0.0,0.0,0.0,6.27
-He,2.74119,21.54332077773755,pass,0.02,0.024,0.017,7.17
-B,17.256161,0.861732831109502,pass,0.003,0.009,0.005,9.38
-Al,9.445146,69.66101712107925,pass,0.015,0.041,0.014,12.73
-C,25.832156,5572.882317592455,pass,0.001,0.002,0.0,14.21
-N,1.75,3.446931324438008,pass,0.007,0.025,0.004,50.62
-C,4.174268,435.3814647233492,pass,0.0,0.0,0.0,8.13
-H,0.982675,8.208005216318007,pass,0.0,0.0,0.0,3.3
-B,9.860662,21.76900860620513,pass,0.004,0.014,0.005,9.95
-Mg,51.666042,696.6102573840757,pass,0.008,0.009,0.008,13.73
-Na,7.733027,87.07629294466985,pass,0.011,0.028,0.01,13.62
-Mg,25.833021,2786.4411157095856,pass,0.0,0.0,0.0,14.95
-N,1.75,0.1723465662219004,pass,0.486,0.097,0.789,33.7
-Al,16.191678,87.07629294466985,pass,0.013,0.032,0.012,11.84
-Mg,43.055026,3.446931324438008,pass,0.003,0.01,0.002,9.99
-O,7.261642,86.1732831109502,pass,0.012,0.033,0.012,11.88
-Al,0.674653,696.6102573840757,pass,0.001,0.001,0.0,50.67
-N,2.75,3.446931324438008,pass,0.003,0.048,0.004,51.43
-Si,2.329,174.1525858893397,pass,0.006,0.017,0.006,21.87
-O,2.486325,8574.040885789896,pass,0.195,0.006,0.002,659.87
-He,0.669236,2757.545059550406,pass,0.008,0.007,0.001,37.36
-He,2.74119,1.723465662219004,pass,0.007,0.017,0.004,31.03
-N,3.705859,0.861732831109502,pass,0.003,0.031,0.006,35.16
-Al,16.191678,18576.27424835271,pass,0.431,0.007,0.011,590.86
-N,2.0,0.5170396986657012,pass,0.016,0.015,0.031,20.93
-Al,9.445146,5572.882317592455,pass,0.013,0.015,0.007,163.42
-B,24.651668,58.050890687540935,pass,0.014,0.026,0.016,9.44
-B,0.246517,87.07629294466985,pass,0.007,0.011,0.007,21.29
-N,2.52736,0.6462996233321265,pass,0.007,0.017,0.016,22.3
-Ne,7.895891,4.30866415554751,pass,0.004,0.046,0.002,104.5
-B,11.093243,116.10169520179876,pass,0.011,0.015,0.01,14.7
-B,12.325826,10.884461216461007,pass,0.004,0.013,0.006,9.79
-N,2.424,1.723465662219004,pass,0.006,0.026,0.01,27.94
-B,19.721324,21.76900860620513,pass,0.003,0.007,0.004,12.75
-C,8.500494,13932.185241653116,pass,0.216,0.002,0.005,71.59
-C,12.635119,21.54332077773755,pass,0.003,0.009,0.003,11.83
-C,25.832156,253.31282870087145,pass,0.01,0.011,0.01,9.25
-Mg,12.916506,1393.2206009414344,pass,0.0,0.0,0.0,13.09
-H,0.982675,86.1732831109502,pass,0.0,0.0,0.0,4.82
-Si,11.645,21.54332077773755,pass,0.005,0.025,0.006,12.85
-Al,21.588905,2.585198493328506,pass,0.007,0.035,0.009,10.68
-N,7.829576,4.30866415554751,pass,0.006,0.034,0.01,9.99
-H,0.00156,21.54332077773755,pass,0.001,0.001,0.0,9.58
-Na,7.733027,17.415189650307482,pass,0.049,0.228,0.052,8.68
-O,7.261642,696.6118946764548,pass,0.0,0.0,0.0,10.05
-Al,18.890291,0.861732831109502,pass,0.006,0.031,0.008,10.77
-B,2.465165,2786.4411157095856,pass,0.002,0.002,0.0,34.35

From d874cc899c033c1081605e205b5c83302091456a Mon Sep 17 00:00:00 2001
From: Tim Callow <timjcallow@gmail.com>
Date: Tue, 7 Nov 2023 17:43:43 +0100
Subject: [PATCH 5/9] Make README.md simpler

---
 tests/dev_tests/README.md | 38 +++++++++++++-------------------------
 1 file changed, 13 insertions(+), 25 deletions(-)

diff --git a/tests/dev_tests/README.md b/tests/dev_tests/README.md
index 5733429..c57595d 100644
--- a/tests/dev_tests/README.md
+++ b/tests/dev_tests/README.md
@@ -1,39 +1,27 @@
-# Development Test Suite for atoMEC
+# Development test suite
 
 ## Overview
 
-This directory is equipped with the necessary code and datasets to facilitate the generation and execution of development tests for the atoMEC project. These tests are tailored to evaluate the performance of the codebase, with a focus on the `CalcEnergy` function, its related components, and behavior under extreme edge cases. While the `tests/` folder contains CI tests that verify the code's functionality, the development tests here are aimed at benchmarking performance. They are not mandatory but are recommended for developers making significant changes to performance-critical parts of the code, especially when modifications impact the execution time observed in CI tests.
+This directory contains the necessary code and data to enable the generation and execution of development tests for the atoMEC project. These tests are dsigned to evaluate the _performance_ of the code, with a focus on the `CalcEnergy` function, its related components, and behavior under extreme edge cases. They are distinct from the CI tests, which are designed to check the _correctness_ of the code across the full codebase. They are not mandatory but are recommended for developers making significant changes to performance-critical parts of the code, especially when modifications impact the execution time observed in CI tests.
 
-## Development Testing Tools
+## Development testing tools
 
 The development tests themselves are not directly included. Instead, the repository provides the necessary tools to generate and run these tests:
 
-- `benchmarking.py`: The core module containing functions to set up the benchmarking environment.
-- `pressure_benchmarks.csv`: The dataset containing parameters for generating test cases.
-- `test.py`: The template for creating individual test scripts.
-- `submit.slurm`: A sample SLURM submission script for use on HPC systems.
-- `run_benchmark_tests.py`: A script that demonstrates how to orchestrate the entire testing workflow using the provided tools.
+- `benchmarking.py`: The core module containing functions to set up the benchmarking environment
+- `pressure_benchmarks.csv`: The dataset containing parameters for generating test cases
+- `test.py`: The template for creating individual test scripts
+- `submit.slurm`: A sample SLURM submission script for use on HPC systems
+- `run_benchmark_tests.py`: A script that demonstrates how to run the entire testing workflow using the provided tools
 
-## Environment Assumption
+## Environment assumption
 
-The execution of these scripts presupposes that atoMEC is operated within a Conda environment, ensuring a consistent and controlled runtime for the tests.
+The testing workflow currently assumes that atoMEC is operated within a Conda virtual environment.
 
-## Benchmarking Protocol
+## Evaluation and benchmarking protocol
 
-Benchmarking should be conducted against the results from the most recent iteration of the development branch, currently documented as `atoMEC_v1.4.0_py312.csv`. This comparison is critical for tracking performance changes introduced by recent code updates.
+Benchmarking should be conducted against the results from the most recent iteration of the development branch. This means that *two* testing workflows should be set-up, one for the branch being submitted as a PR, and one for atoMEC's development branch. Performance improvements could be justified using various statistical metrics.
 
 ## Execution Instructions
 
-To initiate the benchmarking process, ensure that your Conda environment is active and that you have the necessary HPC system access.
-
-Follow these steps to prepare and conduct the benchmarks:
-
-1. Utilize the functions within `benchmarking.py` to prepare the calculation directories as dictated by `pressure_benchmarks.csv`.
-2. Distribute the `test.py` script template and the `submit.slurm` file to their designated locations.
-3. Launch the `run_benchmark_tests.py` script to automate the submission of benchmarking jobs to the HPC system.
-
-Refer to the documentation within each file for more detailed instructions on their usage.
-
-## Contributing
-
-Developers are invited to enhance the test suite by updating the templates, source code, and datasets to align with the evolving requirements of the atoMEC project. Your contributions are vital to maintaining the project's standards of robustness and efficiency.
+The full testing workflow can be run on a slurm-based HPC system with the `run_benchmark_tests.py` script. The script needs to be first run in "setup_and_run" mode, which sets up the calculations and submits them to the slurm system (these steps can also be run separately if preferred). Then it should be run in "evaluate" mode, to collect and summarize the results.

From 84798d03f5b5e6c20489c4541f8662709d09b5b1 Mon Sep 17 00:00:00 2001
From: Callow <timjcallow@gmail.com>
Date: Tue, 7 Nov 2023 17:53:38 +0100
Subject: [PATCH 6/9] Add analyze_results function

---
 tests/dev_tests/benchmarking.py        | 66 ++++++++++++++++++++++++++
 tests/dev_tests/run_benchmark_tests.py |  1 +
 2 files changed, 67 insertions(+)

diff --git a/tests/dev_tests/benchmarking.py b/tests/dev_tests/benchmarking.py
index 8af584c..820f41d 100644
--- a/tests/dev_tests/benchmarking.py
+++ b/tests/dev_tests/benchmarking.py
@@ -274,3 +274,69 @@ def gather_benchmark_results(basedir, new_filename):
     # Save the new dataframe to the specified CSV file
     new_df.to_csv(new_filename, index=False)
     print(f"Results saved to {new_filename}")
+
+def analyze_benchmark_results(csv_file):
+    """
+    Reads benchmark results from a CSV file and analyzes the data.
+
+    Parameters
+    ----------
+    csv_file : str
+        The path to the CSV file containing benchmark results with columns:
+        'species', 'rho', 'temp', 'outcome', 'pc_err_st', 'pc_err_vir', 'pc_err_id', 'time_s'
+
+    Returns
+    -------
+    None
+    """
+    # Read the CSV file into a DataFrame
+    df = pd.read_csv(csv_file)
+
+    # Calculate the number of tests passed
+    total_tests = len(df)
+    passed_tests = len(df[df['outcome'] == 'pass'])
+    pass_fraction = passed_tests / total_tests
+
+    # Calculate median and max for error columns
+    pc_err_st_stats = {'median': df['pc_err_st'].median(), 'max': df['pc_err_st'].max()}
+    pc_err_vir_stats = {'median': df['pc_err_vir'].median(), 'max': df['pc_err_vir'].max()}
+    pc_err_id_stats = {'median': df['pc_err_id'].median(), 'max': df['pc_err_id'].max()}
+
+    # Calculate mean, median, quartiles, and max for time column
+    time_stats = {
+        'mean': df['time_s'].mean(),
+        'median': df['time_s'].median(),
+        'q1': df['time_s'].quantile(0.25),
+        'q3': df['time_s'].quantile(0.75),
+        'max': df['time_s'].max()
+    }
+
+    # Format the table
+    table = f"""
+    Benchmarking Results Analysis
+    -----------------------------
+    Tests passed: {passed_tests} / {total_tests} ({pass_fraction:.2%})
+
+    Error Statistics:
+
+    | Error Type   | Median    | Max       |
+    |--------------|-----------|-----------|
+    | pc_err_st    | {pc_err_st_stats['median']:.2f}      | {pc_err_st_stats['max']:.2f}      |
+    | pc_err_vir   | {pc_err_vir_stats['median']:.2f}      | {pc_err_vir_stats['max']:.2f}      |
+    | pc_err_id    | {pc_err_id_stats['median']:.2f}      | {pc_err_id_stats['max']:.2f}      |
+
+    Time Statistics (s):
+    | Statistic |   Value  |
+    |-----------|----------|
+    | Mean      | {time_stats['mean']:7.2f} |
+    | Median    | {time_stats['median']:7.2f} |
+    | Q1        | {time_stats['q1']:7.2f} |
+    | Q3        | {time_stats['q3']:7.2f} |
+    | Max       | {time_stats['max']:7.2f} |
+    """
+    print(table)
+
+
+
+
+
diff --git a/tests/dev_tests/run_benchmark_tests.py b/tests/dev_tests/run_benchmark_tests.py
index 9a8d3e0..487d56e 100644
--- a/tests/dev_tests/run_benchmark_tests.py
+++ b/tests/dev_tests/run_benchmark_tests.py
@@ -27,6 +27,7 @@
 
 if calc_type == "evaluate":
     benchmarking.gather_benchmark_results(savedir, savedir + "_benchmark_results.csv")
+    benchmarking.analyze_benchmark_results(savedir + "_benchmark_results.csv")
 
 if calc_type not in ["setup", "setup_and_run", "run", "evaluate"]:
     sys.exit(

From 3d9e1a9e6c2e30d6e0c46e283003a7f82e7936c3 Mon Sep 17 00:00:00 2001
From: Callow <timjcallow@gmail.com>
Date: Wed, 8 Nov 2023 11:12:25 +0100
Subject: [PATCH 7/9] Improve formatting, add function and script to compare
 two csv files

---
 tests/dev_tests/benchmarking.py         | 119 +++++++++++++++++++-----
 tests/dev_tests/comp_benchmark_tests.py |  17 ++++
 2 files changed, 111 insertions(+), 25 deletions(-)
 create mode 100644 tests/dev_tests/comp_benchmark_tests.py

diff --git a/tests/dev_tests/benchmarking.py b/tests/dev_tests/benchmarking.py
index 820f41d..f110a5e 100644
--- a/tests/dev_tests/benchmarking.py
+++ b/tests/dev_tests/benchmarking.py
@@ -275,15 +275,15 @@ def gather_benchmark_results(basedir, new_filename):
     new_df.to_csv(new_filename, index=False)
     print(f"Results saved to {new_filename}")
 
+
 def analyze_benchmark_results(csv_file):
     """
-    Reads benchmark results from a CSV file and analyzes the data.
+    Read benchmark results from a CSV file and analyzes the data.
 
     Parameters
     ----------
     csv_file : str
-        The path to the CSV file containing benchmark results with columns:
-        'species', 'rho', 'temp', 'outcome', 'pc_err_st', 'pc_err_vir', 'pc_err_id', 'time_s'
+        The path to the CSV file containing benchmark results
 
     Returns
     -------
@@ -294,21 +294,24 @@ def analyze_benchmark_results(csv_file):
 
     # Calculate the number of tests passed
     total_tests = len(df)
-    passed_tests = len(df[df['outcome'] == 'pass'])
+    passed_tests = len(df[df["outcome"] == "pass"])
     pass_fraction = passed_tests / total_tests
 
     # Calculate median and max for error columns
-    pc_err_st_stats = {'median': df['pc_err_st'].median(), 'max': df['pc_err_st'].max()}
-    pc_err_vir_stats = {'median': df['pc_err_vir'].median(), 'max': df['pc_err_vir'].max()}
-    pc_err_id_stats = {'median': df['pc_err_id'].median(), 'max': df['pc_err_id'].max()}
+    err_st_stats = {"median": df["pc_err_st"].median(), "max": df["pc_err_st"].max()}
+    err_vir_stats = {
+        "median": df["pc_err_vir"].median(),
+        "max": df["pc_err_vir"].max(),
+    }
+    err_id_stats = {"median": df["pc_err_id"].median(), "max": df["pc_err_id"].max()}
 
     # Calculate mean, median, quartiles, and max for time column
     time_stats = {
-        'mean': df['time_s'].mean(),
-        'median': df['time_s'].median(),
-        'q1': df['time_s'].quantile(0.25),
-        'q3': df['time_s'].quantile(0.75),
-        'max': df['time_s'].max()
+        "mean": df["time_s"].mean(),
+        "median": df["time_s"].median(),
+        "q1": df["time_s"].quantile(0.25),
+        "q3": df["time_s"].quantile(0.75),
+        "max": df["time_s"].max(),
     }
 
     # Format the table
@@ -317,26 +320,92 @@ def analyze_benchmark_results(csv_file):
     -----------------------------
     Tests passed: {passed_tests} / {total_tests} ({pass_fraction:.2%})
 
-    Error Statistics:
-
-    | Error Type   | Median    | Max       |
-    |--------------|-----------|-----------|
-    | pc_err_st    | {pc_err_st_stats['median']:.2f}      | {pc_err_st_stats['max']:.2f}      |
-    | pc_err_vir   | {pc_err_vir_stats['median']:.2f}      | {pc_err_vir_stats['max']:.2f}      |
-    | pc_err_id    | {pc_err_id_stats['median']:.2f}      | {pc_err_id_stats['max']:.2f}      |
+    Error Statistics
+    -----------------------------------------
+    | Error Type  | Median     | Max        |
+    |-------------|------------|------------|
+    | pc_err_st   | {err_st_stats['median']:10.2f} | {err_st_stats['max']:10.2f} |
+    | pc_err_vir  | {err_vir_stats['median']:10.2f} | {err_vir_stats['max']:10.2f} |
+    | pc_err_id   | {err_id_stats['median']:10.2f} | {err_id_stats['max']:10.2f} |
 
-    Time Statistics (s):
+    Time Statistics (s)
+    ------------------------
     | Statistic |   Value  |
     |-----------|----------|
-    | Mean      | {time_stats['mean']:7.2f} |
-    | Median    | {time_stats['median']:7.2f} |
-    | Q1        | {time_stats['q1']:7.2f} |
-    | Q3        | {time_stats['q3']:7.2f} |
-    | Max       | {time_stats['max']:7.2f} |
+    | Mean      | {time_stats['mean']:8.2f} |
+    | Median    | {time_stats['median']:8.2f} |
+    | Q1        | {time_stats['q1']:8.2f} |
+    | Q3        | {time_stats['q3']:8.2f} |
+    | Max       | {time_stats['max']:8.2f} |
     """
     print(table)
 
 
+def calc_time_diff(csv_ref, csv_new):
+    """
+    Calculate the average percentage difference in timings between two atoMEC versions.
+
+    Parameters
+    ----------
+    csv_ref : str
+        Filepath to the reference CSV file.
+    csv_new : str
+        Filepath to the new CSV file to compare against the reference.
+
+    Returns
+    -------
+    float
+        The average absolute percentage difference of the 'time_s' column
+        between the reference and new datasets.
+
+    Notes
+    -----
+    The percentage difference is calculated using the formula:
+    100 * abs(time_new - time_ref) / time_ref
+    This formula gives the average of the absolute relative differences
+    from the reference to the new values.
+    """
+    # Read the time_s column from each CSV into a DataFrame
+    df_ref = pd.read_csv(csv_ref)["time_s"]
+    df_new = pd.read_csv(csv_new)["time_s"]
 
+    # Calculate the absolute percentage difference
+    time_diff_pc = 100 * (df_ref - df_new) / df_ref
 
+    # Return the mean of these percentage differences, rounded to 2 decimal places
+    return round(time_diff_pc.mean(), 2)
 
+
+def comp_benchmark_results(csv_ref, csv_new):
+    """
+    Compare benchmark results between a reference CSV file and a new CSV file.
+
+    Prints individual test and timings results, and finally does a row-by-row
+    percentage difference of timings.
+
+    Parameters
+    ----------
+    csv_ref : str
+        Filepath to the reference CSV file containing benchmark results.
+    csv_new : str
+        Filepath to the new CSV file containing benchmark results to be compared.
+
+    Returns
+    -------
+    None
+    """
+    # Print results from the reference CSV
+    print("\nResults from reference csv")
+    print("--------------------------")
+    analyze_benchmark_results(csv_ref)
+
+    # Print results from the new CSV
+    print("\nResults from new csv")
+    print("--------------------------")
+    analyze_benchmark_results(csv_new)
+
+    # Calculate and print the average time percentage difference
+    avg_time_diff = calc_time_diff(csv_ref, csv_new)
+    print("\n-------------------------------------")
+    print(f" Average time % difference = {avg_time_diff}% ")
+    print("-------------------------------------")
diff --git a/tests/dev_tests/comp_benchmark_tests.py b/tests/dev_tests/comp_benchmark_tests.py
new file mode 100644
index 0000000..5864eac
--- /dev/null
+++ b/tests/dev_tests/comp_benchmark_tests.py
@@ -0,0 +1,17 @@
+#!/usr/bin/env python3
+
+"""Compare results from two versions of atoMEC."""
+
+import benchmarking
+
+# user inputs - MUST BE EDITED
+
+# reference csv file (generated from latest atoMEC dev branch)
+csv_ref_file = "03-11-23_atoMEC38_benchmark_results.csv"
+
+# new csv file (generated from branch submitted as PR)
+csv_new_file = "08-11-23_atoMEC312_benchmark_results.csv"
+
+# end of user inputs
+
+benchmarking.comp_benchmark_results(csv_ref_file, csv_new_file)

From 38d3587f7ee7c403ae14db2f6b9fa78dd84bf2c2 Mon Sep 17 00:00:00 2001
From: Callow <timjcallow@gmail.com>
Date: Wed, 8 Nov 2023 11:12:47 +0100
Subject: [PATCH 8/9] Rename P_vir_nocorr_A to P_vir_nocorr

---
 tests/dev_tests/pressure_benchmarks.csv | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/dev_tests/pressure_benchmarks.csv b/tests/dev_tests/pressure_benchmarks.csv
index 1c90957..036cda6 100644
--- a/tests/dev_tests/pressure_benchmarks.csv
+++ b/tests/dev_tests/pressure_benchmarks.csv
@@ -1,4 +1,4 @@
-species,rho,temp,nconv,nmax,lmax,ngrid,nkpts,P_id,P_st_rr,P_vir_nocorr_A
+species,rho,temp,nconv,nmax,lmax,ngrid,nkpts,P_id,P_st_rr,P_vir_nocorr
 C,15.499297,2786.4411157095856,1e-05,24,64,500,20,2072044.7230353267,2017313.5879749784,2039377.1269299167
 C,19.374129,0.861732831109502,1e-05,13,13,500,20,16103.546265615396,21595.25568343568,16021.731183437241
 Mg,17.222007,1393.2206009414344,1e-05,24,60,500,20,1095119.300216119,1068912.6205698776,1075873.3758255616

From 9536fa84acaaf524780a31cbd4667da7d6d5c3f1 Mon Sep 17 00:00:00 2001
From: Tim Callow <timjcallow@gmail.com>
Date: Wed, 8 Nov 2023 11:24:32 +0100
Subject: [PATCH 9/9] Improve README.md

---
 tests/dev_tests/README.md | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/tests/dev_tests/README.md b/tests/dev_tests/README.md
index c57595d..5682b67 100644
--- a/tests/dev_tests/README.md
+++ b/tests/dev_tests/README.md
@@ -13,15 +13,16 @@ The development tests themselves are not directly included. Instead, the reposit
 - `test.py`: The template for creating individual test scripts
 - `submit.slurm`: A sample SLURM submission script for use on HPC systems
 - `run_benchmark_tests.py`: A script that demonstrates how to run the entire testing workflow using the provided tools
+- `comp_benchmark_tests.py`: A script that compares the results from two csv files generated from `run_benchmark_tests.py`
 
 ## Environment assumption
 
 The testing workflow currently assumes that atoMEC is operated within a Conda virtual environment.
 
-## Evaluation and benchmarking protocol
-
-Benchmarking should be conducted against the results from the most recent iteration of the development branch. This means that *two* testing workflows should be set-up, one for the branch being submitted as a PR, and one for atoMEC's development branch. Performance improvements could be justified using various statistical metrics.
-
 ## Execution Instructions
 
 The full testing workflow can be run on a slurm-based HPC system with the `run_benchmark_tests.py` script. The script needs to be first run in "setup_and_run" mode, which sets up the calculations and submits them to the slurm system (these steps can also be run separately if preferred). Then it should be run in "evaluate" mode, to collect and summarize the results.
+
+## Evaluation and benchmarking protocol
+
+Benchmarking should be conducted against the results from the most recent iteration of the development branch. This means that *two* testing workflows should be set-up, one for the branch being submitted as a PR, and one for atoMEC's development branch. After generating the results, performance can be compared by running the `comp_benchmark_tests.py` script. The most important benchmark is considered to be the "Average time % difference", an average of the row-by-row percentage difference between the times taken.