From 11b8202afbba7c8ddd84cc212dc1758a789bfe64 Mon Sep 17 00:00:00 2001
From: XiangweiW <517395458@qq.com>
Date: Mon, 23 Oct 2023 21:03:20 +0800
Subject: [PATCH 01/28] Implementing parallel learning on BOPTEST-gym

---
 README.md                   |  21 ++++++++
 generateDockerComposeYml.py |  79 +++++++++++++++++++++++++++
 parallel_BOPTESTgymTrain.py | 103 ++++++++++++++++++++++++++++++++++++
 3 files changed, 203 insertions(+)
 create mode 100644 generateDockerComposeYml.py
 create mode 100644 parallel_BOPTESTgymTrain.py

diff --git a/README.md b/README.md
index 40911f3..3650ece 100644
--- a/README.md
+++ b/README.md
@@ -75,6 +75,27 @@ Running BOPTEST locally is substantially faster
 2) Run a BOPTEST case with the building emulator model to be controlled (instructions [here](https://github.com/ibpsa/project1-boptest/blob/master/README.md)).  
 3) Check out the `master` branch of this repository and run the example above replacing the url to be `url = 'http://127.0.0.1:5000'` and avoiding the `testcase` argument to the `BoptestGymEnv` class. 
 
+## Parallel-Start (running BOPTEST locally)
+
+To facilitate the training and testing process, we provide scripts that automates the deployment of multiple BOPTEST instances using Docker Compose. The deployment dynamically checks for available ports, generates a Docker Compose YAML file, and takes care of naming conflicts to ensure smooth deployment.
+
+### Usage
+
+1. Specify the BOPTEST root directory either by passing it as a command-line argument or by defining the `boptest_root` variable at the beginning of the script `generateDockerComposeYml.py`. The script prioritizes the command-line argument if provided.
+
+Example using command-line argument:
+
+```bash
+python generateDockerComposeYml.py absolute_boptest_root_dir
+```
+
+2. Train the BOPTEST-gym environment in parallel either by passing it as a command-line argument or by defining the `boptest_root` variable at the beginning of the script `parallel_BOPTESTgymTrain.py`. The script prioritizes the command-line argument if provided.
+
+Example using command-line argument:
+
+```bash
+python parallel_BOPTESTgymTrain.py absolute_boptest_root_dir
+```
 
 ## Citing the project
 
diff --git a/generateDockerComposeYml.py b/generateDockerComposeYml.py
new file mode 100644
index 0000000..07f0b33
--- /dev/null
+++ b/generateDockerComposeYml.py
@@ -0,0 +1,79 @@
+import os
+import socket
+import shutil
+import yaml
+import sys
+
+
+boptest_root = "./"  # You can define boptest_root_dir here when use IDLE
+
+# Get the argument from command line when use Linux
+if len(sys.argv) >= 2:
+    boptest_root_dir = sys.argv[1]
+else:
+    boptest_root_dir = boptest_root
+
+num_services = 15  # Total Services needed
+base_port = 5000  # Start Port number
+
+
+# Function to check if a port is available
+def is_port_available(port):
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
+        return sock.connect_ex(('localhost', port)) != 0
+
+
+services = {}
+last_assigned_port = base_port - 1  # Initial value set to one less than the first port to be checked
+for _ in range(num_services):
+    port = last_assigned_port + 1  # Start checking from the next port after the last assigned one
+
+    # If the port is unavailable, continue to check the next one
+    while not is_port_available(port):
+        print(f"Port {port} is occupied.")
+        port += 1
+        if port > base_port + num_services:
+            raise Exception("Too many ports are occupied.")
+
+    last_assigned_port = port  # Update the last assigned port
+
+    service_name = f"boptest{port}"
+    service_config = {
+        "image": "boptest_base",
+        "build": {"context": "."},
+        "volumes": [
+            "./testcases/${TESTCASE}/models/wrapped.fmu:${APP_PATH}/models/wrapped.fmu",
+            "./testcases/${TESTCASE}/doc/:${APP_PATH}/doc/",
+            "./restapi.py:${APP_PATH}/restapi.py",
+            "./testcase.py:${APP_PATH}/testcase.py",
+            "./version.txt:${APP_PATH}/version.txt",
+            "./data:${APP_PATH}/data/",
+            "./forecast:${APP_PATH}/forecast/",
+            "./kpis:${APP_PATH}/kpis/",
+        ],
+        "ports": [f"127.0.0.1:{port}:5000"],
+        "networks": ["boptest-net"],
+        "restart": "on-failure"  # restart on-failure
+    }
+    services[service_name] = service_config
+
+docker_compose_content = {
+    "version": "3.7",
+    "services": services,
+    "networks": {
+        "boptest-net": {
+            "name": "boptest-net",
+            "attachable": True,
+        }
+    },
+}
+
+# Check whether the docker-compose.yml file exists in the BOPTEST root directory
+docker_compose_path = os.path.join(boptest_root_dir, 'docker-compose.yml')
+if os.path.exists(docker_compose_path):
+    # If it exists, rename to docker-compose_origin.yml
+    shutil.move(docker_compose_path, os.path.join(boptest_root_dir, 'docker-compose_origin.yml'))
+
+# Create a new docker-compose.yml file in the BOPTEST root directory
+with open(docker_compose_path, "w") as file:
+    yaml.dump(docker_compose_content, file, default_flow_style=False)
diff --git a/parallel_BOPTESTgymTrain.py b/parallel_BOPTESTgymTrain.py
new file mode 100644
index 0000000..fd6ea68
--- /dev/null
+++ b/parallel_BOPTESTgymTrain.py
@@ -0,0 +1,103 @@
+import os
+import sys
+import yaml
+import torch
+
+from stable_baselines3 import DQN, PPO, DDPG, SAC, TD3, A2C
+from stable_baselines3.common.vec_env import SubprocVecEnv
+from stable_baselines3.common.callbacks import EvalCallback
+from boptestGymEnv import BoptestGymEnv, NormalizedObservationWrapper, DiscretizedActionWrapper
+
+
+boptest_root = "./"  # You can define boptest_root_dir here when use IDLE
+
+# Get the argument from command line when use Linux
+if len(sys.argv) >= 2:
+    boptest_root_dir = sys.argv[1]
+else:
+    boptest_root_dir = boptest_root
+
+docker_compose_loc = os.path.join(boptest_root_dir, "docker-compose.yml")
+
+# Read the docker-compose.yml file
+with open(docker_compose_loc, 'r') as stream:
+    try:
+        docker_compose_data = yaml.safe_load(stream)
+        services = docker_compose_data.get('services', {})
+
+        # Extract the port and URL of the service
+        urls = []
+        for service, config in services.items():
+            ports = config.get('ports', [])
+            for port in ports:
+                # Extract host port
+                host_port = port.split(':')[1]
+                urls.append(f'http://127.0.0.1:{host_port}')
+
+        print(urls)  # Print URLs
+
+    except yaml.YAMLError as exc:
+        print(exc)
+
+
+# Create a function to initialize the environment
+def make_env(url):
+    def _init():
+        env = BoptestGymEnv(
+            url=url,
+            actions=['oveHeaPumY_u'],
+            observations={
+                'time': (0, 604800),
+                'reaTZon_y': (280., 310.),
+                'TDryBul': (265, 303),
+                'HDirNor': (0, 862),
+                'InternalGainsRad[1]': (0, 219),
+                'PriceElectricPowerHighlyDynamic': (-0.4, 0.4),
+                'LowerSetp[1]': (280., 310.),
+                'UpperSetp[1]': (280., 310.)
+            },
+            scenario={'electricity_price': 'dynamic'},
+            predictive_period=24 * 3600,
+            regressive_period=6 * 3600,
+            random_start_time=True,
+            excluding_periods=[(16 * 24 * 3600, 30 * 24 * 3600), (108 * 24 * 3600, 122 * 24 * 3600)],
+            max_episode_length=14 * 24 * 3600,
+            warmup_period=24 * 3600,
+            step_period=15 * 60
+        )
+        env = NormalizedObservationWrapper(env)  # Add observation normalization if needed
+        env = DiscretizedActionWrapper(env, n_bins_act=10)  # Add action discretization if needed
+
+        return env
+
+    return _init
+
+
+if __name__ == '__main__':
+    # Use URLs obtained from docker-compose.yml
+    if urls:  # Make sure the urls list is not empty
+        envs = [make_env(url) for url in urls]
+
+        # Create a parallel environment using SubprocVecEnv
+        vec_env = SubprocVecEnv(envs)
+
+        # Example: Create a DQN model
+        log_dir = "./vec_dqn_log/"
+        eval_callback = EvalCallback(vec_env, best_model_save_path=log_dir, log_path=log_dir, eval_freq=5000)
+
+        device = 'cuda' if torch.cuda.is_available() else 'cpu'
+
+        # Instantiate an RL agent with DQN
+        model = DQN('MlpPolicy', vec_env, verbose=1, gamma=0.99, learning_rate=5e-4,
+                    batch_size=24, seed=123456, buffer_size=365 * 24,
+                    learning_starts=24, train_freq=1, exploration_initial_eps=1.0,
+                    exploration_final_eps=0.01, exploration_fraction=0.1, tensorboard_log=log_dir, device=device)
+        # Main training loop
+        model.learn(total_timesteps=500000, callback=eval_callback)
+    else:
+        print("No URLs found. Please check your docker-compose.yml file.")
+
+
+
+
+

From dbd440d32929e63a0f742bc43fa645b07fc0936d Mon Sep 17 00:00:00 2001
From: XiangweiW <86037302+XiangweiW@users.noreply.github.com>
Date: Mon, 23 Oct 2023 21:13:04 +0800
Subject: [PATCH 02/28] Update README.md

---
 README.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 3650ece..65620cc 100644
--- a/README.md
+++ b/README.md
@@ -77,11 +77,11 @@ Running BOPTEST locally is substantially faster
 
 ## Parallel-Start (running BOPTEST locally)
 
-To facilitate the training and testing process, we provide scripts that automates the deployment of multiple BOPTEST instances using Docker Compose. The deployment dynamically checks for available ports, generates a Docker Compose YAML file, and takes care of naming conflicts to ensure smooth deployment.
+To facilitate the training and testing process, we provide scripts that automate the deployment of multiple BOPTEST instances using Docker Compose and then train the BOPTEST-gym environment in parallel. The deployment dynamically checks for available ports, generates a Docker Compose YAML file, and takes care of naming conflicts to ensure smooth deployment.
 
 ### Usage
 
-1. Specify the BOPTEST root directory either by passing it as a command-line argument or by defining the `boptest_root` variable at the beginning of the script `generateDockerComposeYml.py`. The script prioritizes the command-line argument if provided.
+1. Specify the BOPTEST root directory either by passing it as a command-line argument or by defining the `boptest_root` variable at the beginning of the script `generateDockerComposeYml.py`. The script prioritizes the command-line argument if provided. Users are allowed to change the Start Port number and Total Services as needed.
 
 Example using command-line argument:
 
@@ -89,7 +89,7 @@ Example using command-line argument:
 python generateDockerComposeYml.py absolute_boptest_root_dir
 ```
 
-2. Train the BOPTEST-gym environment in parallel either by passing it as a command-line argument or by defining the `boptest_root` variable at the beginning of the script `parallel_BOPTESTgymTrain.py`. The script prioritizes the command-line argument if provided.
+2. Train the BOPTEST-gym environment in parallel either by passing it as a command-line argument or by defining the `boptest_root` variable at the beginning of the script `parallel_BOPTESTgymTrain.py`. The script prioritizes the command-line argument if provided. Users are allowed to change the initial conditions, algorithms and hyperparameters.
 
 Example using command-line argument:
 

From aacb4786fcc03d60b509aacb20d9991ec983ce25 Mon Sep 17 00:00:00 2001
From: Javier Arroyo <javier.arroyo@kuleuven.be>
Date: Tue, 14 Nov 2023 09:59:15 +0100
Subject: [PATCH 03/28] Remove tensorboard dependency.

---
 parallel_BOPTESTgymTrain.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/parallel_BOPTESTgymTrain.py b/parallel_BOPTESTgymTrain.py
index fd6ea68..5235301 100644
--- a/parallel_BOPTESTgymTrain.py
+++ b/parallel_BOPTESTgymTrain.py
@@ -91,7 +91,7 @@ def _init():
         model = DQN('MlpPolicy', vec_env, verbose=1, gamma=0.99, learning_rate=5e-4,
                     batch_size=24, seed=123456, buffer_size=365 * 24,
                     learning_starts=24, train_freq=1, exploration_initial_eps=1.0,
-                    exploration_final_eps=0.01, exploration_fraction=0.1, tensorboard_log=log_dir, device=device)
+                    exploration_final_eps=0.01, exploration_fraction=0.1, device=device)
         # Main training loop
         model.learn(total_timesteps=500000, callback=eval_callback)
     else:

From f52585e4385933a447c1826b8b001c93f146b929 Mon Sep 17 00:00:00 2001
From: Javier Arroyo <javier.arroyo@kuleuven.be>
Date: Tue, 14 Nov 2023 10:39:19 +0100
Subject: [PATCH 04/28] Do not import agents not used.

---
 parallel_BOPTESTgymTrain.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/parallel_BOPTESTgymTrain.py b/parallel_BOPTESTgymTrain.py
index 5235301..43ba3a2 100644
--- a/parallel_BOPTESTgymTrain.py
+++ b/parallel_BOPTESTgymTrain.py
@@ -3,7 +3,7 @@
 import yaml
 import torch
 
-from stable_baselines3 import DQN, PPO, DDPG, SAC, TD3, A2C
+from stable_baselines3 import DQN
 from stable_baselines3.common.vec_env import SubprocVecEnv
 from stable_baselines3.common.callbacks import EvalCallback
 from boptestGymEnv import BoptestGymEnv, NormalizedObservationWrapper, DiscretizedActionWrapper

From b240c556012e50fb1da0e87f0e5cbdce814a616e Mon Sep 17 00:00:00 2001
From: Javier Arroyo <javier.arroyo@kuleuven.be>
Date: Tue, 14 Nov 2023 10:42:13 +0100
Subject: [PATCH 05/28] Use new logging functionality.

---
 parallel_BOPTESTgymTrain.py | 28 +++++++++++++++++++++-------
 1 file changed, 21 insertions(+), 7 deletions(-)

diff --git a/parallel_BOPTESTgymTrain.py b/parallel_BOPTESTgymTrain.py
index 43ba3a2..142723c 100644
--- a/parallel_BOPTESTgymTrain.py
+++ b/parallel_BOPTESTgymTrain.py
@@ -3,10 +3,13 @@
 import yaml
 import torch
 
+from testing import utilities
 from stable_baselines3 import DQN
 from stable_baselines3.common.vec_env import SubprocVecEnv
 from stable_baselines3.common.callbacks import EvalCallback
-from boptestGymEnv import BoptestGymEnv, NormalizedObservationWrapper, DiscretizedActionWrapper
+from stable_baselines3.common.vec_env.vec_monitor import VecMonitor
+from stable_baselines3.common.logger import configure
+from boptestGymEnv import BoptestGymEnv, NormalizedObservationWrapper, DiscretizedActionWrapper, SaveAndTestCallback
 
 
 boptest_root = "./"  # You can define boptest_root_dir here when use IDLE
@@ -39,7 +42,6 @@
     except yaml.YAMLError as exc:
         print(exc)
 
-
 # Create a function to initialize the environment
 def make_env(url):
     def _init():
@@ -81,10 +83,17 @@ def _init():
         # Create a parallel environment using SubprocVecEnv
         vec_env = SubprocVecEnv(envs)
 
-        # Example: Create a DQN model
-        log_dir = "./vec_dqn_log/"
-        eval_callback = EvalCallback(vec_env, best_model_save_path=log_dir, log_path=log_dir, eval_freq=5000)
-
+        # Define logging directory. Monitoring data and agent model will be stored here
+        log_dir = os.path.join(utilities.get_root_path(), 'examples', 'agents', 'DQN_parallel')
+        os.makedirs(log_dir, exist_ok=True)
+    
+        # Modify the environment to include the callback
+        vec_env = VecMonitor(venv=vec_env, filename=os.path.join(log_dir,'monitor.csv'))
+                
+        # Create the callback: check every 100 steps. We keep it very short for testing 
+        eval_callback = EvalCallback(vec_env, best_model_save_path=log_dir, log_path=log_dir, eval_freq=100)
+
+        # Try to find CUDA core since it's optimized for parallel computing tasks
         device = 'cuda' if torch.cuda.is_available() else 'cpu'
 
         # Instantiate an RL agent with DQN
@@ -92,8 +101,13 @@ def _init():
                     batch_size=24, seed=123456, buffer_size=365 * 24,
                     learning_starts=24, train_freq=1, exploration_initial_eps=1.0,
                     exploration_final_eps=0.01, exploration_fraction=0.1, device=device)
+        
+        # set up logger
+        new_logger = configure(log_dir, ['csv'])
+        model.set_logger(new_logger)
+
         # Main training loop
-        model.learn(total_timesteps=500000, callback=eval_callback)
+        model.learn(total_timesteps=500, callback=eval_callback)
     else:
         print("No URLs found. Please check your docker-compose.yml file.")
 

From bd5ddb204b2763ccdb652f3957d90a4c900704e8 Mon Sep 17 00:00:00 2001
From: Javier Arroyo <javier.arroyo@kuleuven.be>
Date: Tue, 14 Nov 2023 11:53:43 +0100
Subject: [PATCH 06/28] Reduce default number of services.

---
 generateDockerComposeYml.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/generateDockerComposeYml.py b/generateDockerComposeYml.py
index 07f0b33..02bdef4 100644
--- a/generateDockerComposeYml.py
+++ b/generateDockerComposeYml.py
@@ -13,7 +13,7 @@
 else:
     boptest_root_dir = boptest_root
 
-num_services = 15  # Total Services needed
+num_services = 2  # Total Services needed
 base_port = 5000  # Start Port number
 
 

From 178d6b84744e7f2af0c0458a881c17d34f35e193 Mon Sep 17 00:00:00 2001
From: Javier Arroyo <javier.arroyo@kuleuven.be>
Date: Tue, 14 Nov 2023 11:59:12 +0100
Subject: [PATCH 07/28] Rename header in README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index ae1e184..33e31cb 100644
--- a/README.md
+++ b/README.md
@@ -75,7 +75,7 @@ Running BOPTEST locally is substantially faster
 2) Run a BOPTEST case with the building emulator model to be controlled (instructions [here](https://github.com/ibpsa/project1-boptest/blob/master/README.md)).  
 3) Check out the `master` branch of this repository and run the example above replacing the url to be `url = 'http://127.0.0.1:5000'` and avoiding the `testcase` argument to the `BoptestGymEnv` class. 
 
-## Parallel-Start (running BOPTEST locally)
+## Quick-Start (running BOPTEST locally with parallel training)
 
 To facilitate the training and testing process, we provide scripts that automate the deployment of multiple BOPTEST instances using Docker Compose and then train the BOPTEST-gym environment in parallel. The deployment dynamically checks for available ports, generates a Docker Compose YAML file, and takes care of naming conflicts to ensure smooth deployment.
 

From d1985eba4d136845da163ad9b63447a03ffbc057 Mon Sep 17 00:00:00 2001
From: Javier Arroyo <javier.arroyo@kuleuven.be>
Date: Tue, 14 Nov 2023 12:01:04 +0100
Subject: [PATCH 08/28] Move parallel script to examples.

---
 .../parallel_BOPTESTgymTrain.py                                   | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename parallel_BOPTESTgymTrain.py => examples/parallel_BOPTESTgymTrain.py (100%)

diff --git a/parallel_BOPTESTgymTrain.py b/examples/parallel_BOPTESTgymTrain.py
similarity index 100%
rename from parallel_BOPTESTgymTrain.py
rename to examples/parallel_BOPTESTgymTrain.py

From 757cb610ec5d68aa6043a917f529bea900e4a965 Mon Sep 17 00:00:00 2001
From: Javier Arroyo <javier.arroyo@kuleuven.be>
Date: Tue, 14 Nov 2023 12:01:30 +0100
Subject: [PATCH 09/28] Rename script.

---
 examples/{parallel_BOPTESTgymTrain.py => run_parallel.py} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename examples/{parallel_BOPTESTgymTrain.py => run_parallel.py} (100%)

diff --git a/examples/parallel_BOPTESTgymTrain.py b/examples/run_parallel.py
similarity index 100%
rename from examples/parallel_BOPTESTgymTrain.py
rename to examples/run_parallel.py

From b0d11bd8bc236daf58471368a96c64883a765cc1 Mon Sep 17 00:00:00 2001
From: Javier Arroyo <javier.arroyo@kuleuven.be>
Date: Tue, 14 Nov 2023 12:32:37 +0100
Subject: [PATCH 10/28] Fix evaluation period and add clarifying note.

---
 examples/run_parallel.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/examples/run_parallel.py b/examples/run_parallel.py
index 142723c..dedbc0b 100644
--- a/examples/run_parallel.py
+++ b/examples/run_parallel.py
@@ -90,8 +90,12 @@ def _init():
         # Modify the environment to include the callback
         vec_env = VecMonitor(venv=vec_env, filename=os.path.join(log_dir,'monitor.csv'))
                 
-        # Create the callback: check every 100 steps. We keep it very short for testing 
-        eval_callback = EvalCallback(vec_env, best_model_save_path=log_dir, log_path=log_dir, eval_freq=100)
+        # Create the callback: evaluate with one episode after 100 steps for training. We keep it very short for testing.
+        # When using multiple environments, each call to ``env.step()`` will effectively correspond to ``n_envs`` steps. 
+        # To account for that, you can use ``eval_freq = eval_freq/len(envs)``
+        eval_freq = 100
+        eval_callback = EvalCallback(vec_env, best_model_save_path=log_dir, log_path=log_dir, 
+                                     eval_freq=int(eval_freq/len(envs)), n_eval_episodes=1, deterministic=True)
 
         # Try to find CUDA core since it's optimized for parallel computing tasks
         device = 'cuda' if torch.cuda.is_available() else 'cpu'
@@ -107,7 +111,7 @@ def _init():
         model.set_logger(new_logger)
 
         # Main training loop
-        model.learn(total_timesteps=500, callback=eval_callback)
+        model.learn(total_timesteps=100, callback=eval_callback)
     else:
         print("No URLs found. Please check your docker-compose.yml file.")
 

From 991d571fb0cb580d4316371eaa4c6360e246410c Mon Sep 17 00:00:00 2001
From: Javier Arroyo <javier.arroyo@kuleuven.be>
Date: Tue, 14 Nov 2023 12:41:10 +0100
Subject: [PATCH 11/28] Set random seed in script.

---
 examples/run_parallel.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/examples/run_parallel.py b/examples/run_parallel.py
index dedbc0b..7568927 100644
--- a/examples/run_parallel.py
+++ b/examples/run_parallel.py
@@ -2,6 +2,7 @@
 import sys
 import yaml
 import torch
+import random
 
 from testing import utilities
 from stable_baselines3 import DQN
@@ -11,6 +12,10 @@
 from stable_baselines3.common.logger import configure
 from boptestGymEnv import BoptestGymEnv, NormalizedObservationWrapper, DiscretizedActionWrapper, SaveAndTestCallback
 
+seed = 123456
+
+# Seed for random starting times of episodes
+random.seed(seed)
 
 boptest_root = "./"  # You can define boptest_root_dir here when use IDLE
 

From 8c67c6bfa71d09c26c8de27afb141104408bd445 Mon Sep 17 00:00:00 2001
From: Javier Arroyo <javier.arroyo@kuleuven.be>
Date: Tue, 14 Nov 2023 12:41:58 +0100
Subject: [PATCH 12/28] Reduce max_episode_length of example for shorter
 evaluation time during testing.

---
 examples/run_parallel.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/run_parallel.py b/examples/run_parallel.py
index 7568927..4555f63 100644
--- a/examples/run_parallel.py
+++ b/examples/run_parallel.py
@@ -68,7 +68,7 @@ def _init():
             regressive_period=6 * 3600,
             random_start_time=True,
             excluding_periods=[(16 * 24 * 3600, 30 * 24 * 3600), (108 * 24 * 3600, 122 * 24 * 3600)],
-            max_episode_length=14 * 24 * 3600,
+            max_episode_length=24 * 3600,
             warmup_period=24 * 3600,
             step_period=15 * 60
         )

From 963085f73deaf44e4b6e904e59e6de7316147c9b Mon Sep 17 00:00:00 2001
From: Javier Arroyo <javier.arroyo@kuleuven.be>
Date: Tue, 14 Nov 2023 14:41:00 +0100
Subject: [PATCH 13/28] Remove not used callback.

---
 examples/run_parallel.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/run_parallel.py b/examples/run_parallel.py
index 4555f63..adaaa26 100644
--- a/examples/run_parallel.py
+++ b/examples/run_parallel.py
@@ -10,7 +10,7 @@
 from stable_baselines3.common.callbacks import EvalCallback
 from stable_baselines3.common.vec_env.vec_monitor import VecMonitor
 from stable_baselines3.common.logger import configure
-from boptestGymEnv import BoptestGymEnv, NormalizedObservationWrapper, DiscretizedActionWrapper, SaveAndTestCallback
+from boptestGymEnv import BoptestGymEnv, NormalizedObservationWrapper, DiscretizedActionWrapper
 
 seed = 123456
 

From d577d83e8a39e5d1670f864df1f2bd91f1a3228c Mon Sep 17 00:00:00 2001
From: Javier Arroyo <javier.arroyo@kuleuven.be>
Date: Tue, 14 Nov 2023 14:45:09 +0100
Subject: [PATCH 14/28] Insert functionality in generate_urls_from_yml method.

---
 examples/run_parallel.py | 63 ++++++++++++++++++++++++++--------------
 1 file changed, 42 insertions(+), 21 deletions(-)

diff --git a/examples/run_parallel.py b/examples/run_parallel.py
index adaaa26..5a37f24 100644
--- a/examples/run_parallel.py
+++ b/examples/run_parallel.py
@@ -25,27 +25,47 @@
 else:
     boptest_root_dir = boptest_root
 
-docker_compose_loc = os.path.join(boptest_root_dir, "docker-compose.yml")
 
-# Read the docker-compose.yml file
-with open(docker_compose_loc, 'r') as stream:
-    try:
-        docker_compose_data = yaml.safe_load(stream)
-        services = docker_compose_data.get('services', {})
-
-        # Extract the port and URL of the service
-        urls = []
-        for service, config in services.items():
-            ports = config.get('ports', [])
-            for port in ports:
-                # Extract host port
-                host_port = port.split(':')[1]
-                urls.append(f'http://127.0.0.1:{host_port}')
-
-        print(urls)  # Print URLs
-
-    except yaml.YAMLError as exc:
-        print(exc)
+def generate_urls_from_yml(boptest_root_dir=boptest_root_dir):
+    '''Method that returns as many urls for BOPTEST-Gym environments 
+    as those specified at the BOPTEST `docker-compose.yml` file. 
+    It assumes that `generateDockerComposeYml.py` has been called first. 
+
+    Parameters
+    ----------
+    boptest_root_dir: str
+        String with directory to BOPTEST where the `docker-compose.yml` 
+        file should be located. 
+
+    Returns
+    -------
+    urls: list
+        List of urls where BOPTEST test cases will be allocated. 
+
+    '''
+    docker_compose_loc = os.path.join(boptest_root_dir, "docker-compose.yml")
+
+    # Read the docker-compose.yml file
+    with open(docker_compose_loc, 'r') as stream:
+        try:
+            docker_compose_data = yaml.safe_load(stream)
+            services = docker_compose_data.get('services', {})
+
+            # Extract the port and URL of the service
+            urls = []
+            for service, config in services.items():
+                ports = config.get('ports', [])
+                for port in ports:
+                    # Extract host port
+                    host_port = port.split(':')[1]
+                    urls.append(f'http://127.0.0.1:{host_port}')
+
+            print(urls)  # Print URLs
+
+        except yaml.YAMLError as exc:
+            print(exc)
+    
+    return urls
 
 # Create a function to initialize the environment
 def make_env(url):
@@ -82,6 +102,7 @@ def _init():
 
 if __name__ == '__main__':
     # Use URLs obtained from docker-compose.yml
+    urls = generate_urls_from_yml(boptest_root_dir)
     if urls:  # Make sure the urls list is not empty
         envs = [make_env(url) for url in urls]
 
@@ -89,7 +110,7 @@ def _init():
         vec_env = SubprocVecEnv(envs)
 
         # Define logging directory. Monitoring data and agent model will be stored here
-        log_dir = os.path.join(utilities.get_root_path(), 'examples', 'agents', 'DQN_parallel')
+        log_dir = os.path.join(utilities.get_root_path(), 'examples', 'agents', 'DQN_vectorized')
         os.makedirs(log_dir, exist_ok=True)
     
         # Modify the environment to include the callback

From c7b08e14d545fd9a9abf1ff57ccef82e61c4415d Mon Sep 17 00:00:00 2001
From: Javier Arroyo <javier.arroyo@kuleuven.be>
Date: Tue, 14 Nov 2023 14:46:09 +0100
Subject: [PATCH 15/28] Rename to vectorized.

---
 examples/{run_parallel.py => run_vectorized.py} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename examples/{run_parallel.py => run_vectorized.py} (100%)

diff --git a/examples/run_parallel.py b/examples/run_vectorized.py
similarity index 100%
rename from examples/run_parallel.py
rename to examples/run_vectorized.py

From cff2e097cf0fe94116daacfb4443685df107a2f1 Mon Sep 17 00:00:00 2001
From: Javier Arroyo <javier.arroyo@kuleuven.be>
Date: Tue, 14 Nov 2023 14:48:57 +0100
Subject: [PATCH 16/28] Do not check for empty list.

---
 examples/run_vectorized.py | 69 +++++++++++++++++++-------------------
 1 file changed, 34 insertions(+), 35 deletions(-)

diff --git a/examples/run_vectorized.py b/examples/run_vectorized.py
index 5a37f24..2f1a291 100644
--- a/examples/run_vectorized.py
+++ b/examples/run_vectorized.py
@@ -103,43 +103,42 @@ def _init():
 if __name__ == '__main__':
     # Use URLs obtained from docker-compose.yml
     urls = generate_urls_from_yml(boptest_root_dir)
-    if urls:  # Make sure the urls list is not empty
-        envs = [make_env(url) for url in urls]
 
-        # Create a parallel environment using SubprocVecEnv
-        vec_env = SubprocVecEnv(envs)
-
-        # Define logging directory. Monitoring data and agent model will be stored here
-        log_dir = os.path.join(utilities.get_root_path(), 'examples', 'agents', 'DQN_vectorized')
-        os.makedirs(log_dir, exist_ok=True)
+    envs = [make_env(url) for url in urls]
+
+    # Create a parallel environment using SubprocVecEnv
+    vec_env = SubprocVecEnv(envs)
+
+    # Define logging directory. Monitoring data and agent model will be stored here
+    log_dir = os.path.join(utilities.get_root_path(), 'examples', 'agents', 'DQN_vectorized')
+    os.makedirs(log_dir, exist_ok=True)
+
+    # Modify the environment to include the callback
+    vec_env = VecMonitor(venv=vec_env, filename=os.path.join(log_dir,'monitor.csv'))
+            
+    # Create the callback: evaluate with one episode after 100 steps for training. We keep it very short for testing.
+    # When using multiple environments, each call to ``env.step()`` will effectively correspond to ``n_envs`` steps. 
+    # To account for that, you can use ``eval_freq = eval_freq/len(envs)``
+    eval_freq = 100
+    eval_callback = EvalCallback(vec_env, best_model_save_path=log_dir, log_path=log_dir, 
+                                    eval_freq=int(eval_freq/len(envs)), n_eval_episodes=1, deterministic=True)
+
+    # Try to find CUDA core since it's optimized for parallel computing tasks
+    device = 'cuda' if torch.cuda.is_available() else 'cpu'
+
+    # Instantiate an RL agent with DQN
+    model = DQN('MlpPolicy', vec_env, verbose=1, gamma=0.99, learning_rate=5e-4,
+                batch_size=24, seed=123456, buffer_size=365 * 24,
+                learning_starts=24, train_freq=1, exploration_initial_eps=1.0,
+                exploration_final_eps=0.01, exploration_fraction=0.1, device=device)
     
-        # Modify the environment to include the callback
-        vec_env = VecMonitor(venv=vec_env, filename=os.path.join(log_dir,'monitor.csv'))
-                
-        # Create the callback: evaluate with one episode after 100 steps for training. We keep it very short for testing.
-        # When using multiple environments, each call to ``env.step()`` will effectively correspond to ``n_envs`` steps. 
-        # To account for that, you can use ``eval_freq = eval_freq/len(envs)``
-        eval_freq = 100
-        eval_callback = EvalCallback(vec_env, best_model_save_path=log_dir, log_path=log_dir, 
-                                     eval_freq=int(eval_freq/len(envs)), n_eval_episodes=1, deterministic=True)
-
-        # Try to find CUDA core since it's optimized for parallel computing tasks
-        device = 'cuda' if torch.cuda.is_available() else 'cpu'
-
-        # Instantiate an RL agent with DQN
-        model = DQN('MlpPolicy', vec_env, verbose=1, gamma=0.99, learning_rate=5e-4,
-                    batch_size=24, seed=123456, buffer_size=365 * 24,
-                    learning_starts=24, train_freq=1, exploration_initial_eps=1.0,
-                    exploration_final_eps=0.01, exploration_fraction=0.1, device=device)
-        
-        # set up logger
-        new_logger = configure(log_dir, ['csv'])
-        model.set_logger(new_logger)
-
-        # Main training loop
-        model.learn(total_timesteps=100, callback=eval_callback)
-    else:
-        print("No URLs found. Please check your docker-compose.yml file.")
+    # set up logger
+    new_logger = configure(log_dir, ['csv'])
+    model.set_logger(new_logger)
+
+    # Main training loop
+    model.learn(total_timesteps=100, callback=eval_callback)
+
 
 
 

From 0c4a63b6032e56059831bc78394dfa2c2ba76f2b Mon Sep 17 00:00:00 2001
From: Javier Arroyo <javier.arroyo@kuleuven.be>
Date: Tue, 14 Nov 2023 16:59:49 +0100
Subject: [PATCH 17/28] Move functionality inside method.

---
 examples/run_vectorized.py | 55 +++++++++++++++++++++-----------------
 1 file changed, 30 insertions(+), 25 deletions(-)

diff --git a/examples/run_vectorized.py b/examples/run_vectorized.py
index 2f1a291..8509be5 100644
--- a/examples/run_vectorized.py
+++ b/examples/run_vectorized.py
@@ -17,16 +17,7 @@
 # Seed for random starting times of episodes
 random.seed(seed)
 
-boptest_root = "./"  # You can define boptest_root_dir here when use IDLE
-
-# Get the argument from command line when use Linux
-if len(sys.argv) >= 2:
-    boptest_root_dir = sys.argv[1]
-else:
-    boptest_root_dir = boptest_root
-
-
-def generate_urls_from_yml(boptest_root_dir=boptest_root_dir):
+def generate_urls_from_yml(boptest_root_dir):
     '''Method that returns as many urls for BOPTEST-Gym environments 
     as those specified at the BOPTEST `docker-compose.yml` file. 
     It assumes that `generateDockerComposeYml.py` has been called first. 
@@ -67,8 +58,9 @@ def generate_urls_from_yml(boptest_root_dir=boptest_root_dir):
     
     return urls
 
-# Create a function to initialize the environment
 def make_env(url):
+    ''' Function that instantiates the environment. 
+    '''
     def _init():
         env = BoptestGymEnv(
             url=url,
@@ -99,35 +91,26 @@ def _init():
 
     return _init
 
-
-if __name__ == '__main__':
-    # Use URLs obtained from docker-compose.yml
-    urls = generate_urls_from_yml(boptest_root_dir)
-
-    envs = [make_env(url) for url in urls]
-
-    # Create a parallel environment using SubprocVecEnv
-    vec_env = SubprocVecEnv(envs)
-
+def train_DQN_vectorized(venv):
     # Define logging directory. Monitoring data and agent model will be stored here
     log_dir = os.path.join(utilities.get_root_path(), 'examples', 'agents', 'DQN_vectorized')
     os.makedirs(log_dir, exist_ok=True)
 
     # Modify the environment to include the callback
-    vec_env = VecMonitor(venv=vec_env, filename=os.path.join(log_dir,'monitor.csv'))
+    venv = VecMonitor(venv=venv, filename=os.path.join(log_dir,'monitor.csv'))
             
     # Create the callback: evaluate with one episode after 100 steps for training. We keep it very short for testing.
     # When using multiple environments, each call to ``env.step()`` will effectively correspond to ``n_envs`` steps. 
     # To account for that, you can use ``eval_freq = eval_freq/len(envs)``
     eval_freq = 100
-    eval_callback = EvalCallback(vec_env, best_model_save_path=log_dir, log_path=log_dir, 
-                                    eval_freq=int(eval_freq/len(envs)), n_eval_episodes=1, deterministic=True)
+    eval_callback = EvalCallback(venv, best_model_save_path=log_dir, log_path=log_dir, 
+                                 eval_freq=int(eval_freq/len(envs)), n_eval_episodes=1, deterministic=True)
 
     # Try to find CUDA core since it's optimized for parallel computing tasks
     device = 'cuda' if torch.cuda.is_available() else 'cpu'
 
     # Instantiate an RL agent with DQN
-    model = DQN('MlpPolicy', vec_env, verbose=1, gamma=0.99, learning_rate=5e-4,
+    model = DQN('MlpPolicy', venv, verbose=1, gamma=0.99, learning_rate=5e-4,
                 batch_size=24, seed=123456, buffer_size=365 * 24,
                 learning_starts=24, train_freq=1, exploration_initial_eps=1.0,
                 exploration_final_eps=0.01, exploration_fraction=0.1, device=device)
@@ -139,6 +122,28 @@ def _init():
     # Main training loop
     model.learn(total_timesteps=100, callback=eval_callback)
 
+if __name__ == '__main__':
+
+    boptest_root = "./"
+
+    # Get the argument from command line when use Linux
+    if len(sys.argv) >= 2:
+        boptest_root_dir = sys.argv[1]
+    else:
+        boptest_root_dir = boptest_root
+
+    # Use URLs obtained from docker-compose.yml
+    urls = generate_urls_from_yml(boptest_root_dir=boptest_root_dir)
+
+    # Create BOPTEST-Gym environment replicas
+    envs = [make_env(url) for url in urls]
+    
+    # Create a vectorized environment using SubprocVecEnv
+    venv = SubprocVecEnv(envs)
+    
+    # Train vectorized environment
+    train_DQN_vectorized(venv)
+
 
 
 

From bda440514187991731175364091a23fd0afbb24a Mon Sep 17 00:00:00 2001
From: Javier Arroyo <javier.arroyo@kuleuven.be>
Date: Tue, 14 Nov 2023 17:36:02 +0100
Subject: [PATCH 18/28] Create test for vectorized environment.

---
 examples/run_vectorized.py    | 19 ++++++++++----
 testing/test_boptestGymEnv.py | 47 +++++++++++++++++++++++++++++++++--
 2 files changed, 59 insertions(+), 7 deletions(-)

diff --git a/examples/run_vectorized.py b/examples/run_vectorized.py
index 8509be5..aa33988 100644
--- a/examples/run_vectorized.py
+++ b/examples/run_vectorized.py
@@ -91,9 +91,18 @@ def _init():
 
     return _init
 
-def train_DQN_vectorized(venv):
-    # Define logging directory. Monitoring data and agent model will be stored here
-    log_dir = os.path.join(utilities.get_root_path(), 'examples', 'agents', 'DQN_vectorized')
+def train_DQN_vectorized(venv, 
+                         log_dir=os.path.join(utilities.get_root_path(), 'examples', 'agents', 'DQN_vectorized')):
+    '''Method to train DQN agent using vectorized environment. 
+
+    Parameters
+    ----------
+    venv: stable_baselines3.common.vec_env.SubprocVecEnv
+        vectorized environment to be learned. 
+
+    '''
+
+    # Create logging directory if not exists. Monitoring data and agent model will be stored here
     os.makedirs(log_dir, exist_ok=True)
 
     # Modify the environment to include the callback
@@ -101,10 +110,10 @@ def train_DQN_vectorized(venv):
             
     # Create the callback: evaluate with one episode after 100 steps for training. We keep it very short for testing.
     # When using multiple environments, each call to ``env.step()`` will effectively correspond to ``n_envs`` steps. 
-    # To account for that, you can use ``eval_freq = eval_freq/len(envs)``
+    # To account for that, you can use ``eval_freq = eval_freq/venv.num_envs``
     eval_freq = 100
     eval_callback = EvalCallback(venv, best_model_save_path=log_dir, log_path=log_dir, 
-                                 eval_freq=int(eval_freq/len(envs)), n_eval_episodes=1, deterministic=True)
+                                 eval_freq=int(eval_freq/venv.num_envs), n_eval_episodes=1, deterministic=True)
 
     # Try to find CUDA core since it's optimized for parallel computing tasks
     device = 'cuda' if torch.cuda.is_available() else 'cpu'
diff --git a/testing/test_boptestGymEnv.py b/testing/test_boptestGymEnv.py
index b57b0ac..8aedf36 100644
--- a/testing/test_boptestGymEnv.py
+++ b/testing/test_boptestGymEnv.py
@@ -12,11 +12,12 @@
 import shutil
 from testing import utilities
 from examples import run_baseline, run_sample, run_save_callback,\
-    run_variable_episode, train_RL
+    run_variable_episode, run_vectorized, train_RL
 from collections import OrderedDict
 from boptestGymEnv import BoptestGymEnv
 from stable_baselines3.common.env_checker import check_env
-from stable_baselines3 import A2C
+from stable_baselines3.common.vec_env import SubprocVecEnv
+from stable_baselines3 import A2C, DQN
 
 url = 'http://127.0.0.1:5000'
 
@@ -382,6 +383,48 @@ def test_variable_episode(self):
         
         # Remove model to prove further testing
         shutil.rmtree(log_dir, ignore_errors=True)
+    
+    def test_vectorized(self, boptest_root = "./"):
+        '''
+        Instantiates a vectorized environment with two BOPTEST-Gym environment replicas
+        and learns from them when running in parallel using DQN for 100 timesteps.
+        It assumes that `generateDockerComposeYml.py` is called first using
+        `num_services=2` and `TESTCASE=bestest_hydronic_heat_pump docker-compose up` 
+        is invoked after to initialize the two BOPTEST test cases. 
+        Note that this test is also using the `EvalCallback` class from 
+        `stable_baselines3.common.callbacks` instead of the
+        `boptestGymEnv.SaveAndTestCallback` that we typically use because 
+        the former was more convenient for use with vectorized environments. 
+
+        '''
+        # Define logging directory. Monitoring data and agent model will be stored here
+        log_dir = os.path.join(utilities.get_root_path(), 'examples', 'agents', 'DQN_vectorized')
+
+        # Use URLs obtained from docker-compose.yml
+        urls = run_vectorized.generate_urls_from_yml(boptest_root_dir=boptest_root)
+
+        # Create BOPTEST-Gym environment replicas
+        envs = [run_vectorized.make_env(url) for url in urls]
+        
+        # Create a vectorized environment using SubprocVecEnv
+        venv = SubprocVecEnv(envs)
+        
+        # Perform a short training example with parallel learning
+        run_vectorized.train_DQN_vectorized(venv, log_dir=log_dir)  
+        
+        # Load the trained agent
+        model = DQN.load(os.path.join(log_dir, 'best_model'))
+        
+        # Test one step with the trained model
+        obs = venv.reset()[0]
+        df = pd.DataFrame([model.predict(obs)[0]], columns=['value'])
+        df.index.name = 'keys'
+        ref_filepath    = os.path.join(utilities.get_root_path(), 
+                            'testing', 'references', 'vectorized_training.csv')
+        self.compare_ref_values_df(df, ref_filepath)
+        
+        # Remove model to prove further testing
+        shutil.rmtree(log_dir, ignore_errors=True)     
         
     def check_obs_act_rew_kpi(self, obs=None, act=None, rew=None, kpi=None,
                               label='default'):

From e533cea6ea2f7845b4417dd75acdd4eed40d11a0 Mon Sep 17 00:00:00 2001
From: Javier Arroyo <javier.arroyo@kuleuven.be>
Date: Wed, 15 Nov 2023 10:43:47 +0100
Subject: [PATCH 19/28] Move seed inside environment constructor to get
 consistent results. Ideally we shouldn't need to pass the seed to the
 boptestGymEnv and we would inherit the seed using the SubprocVecEnv.seed()
 method from stable baselines.

---
 examples/run_vectorized.py    | 15 +++++++++------
 testing/test_boptestGymEnv.py |  6 ++++--
 2 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/examples/run_vectorized.py b/examples/run_vectorized.py
index aa33988..1b1c9da 100644
--- a/examples/run_vectorized.py
+++ b/examples/run_vectorized.py
@@ -12,11 +12,6 @@
 from stable_baselines3.common.logger import configure
 from boptestGymEnv import BoptestGymEnv, NormalizedObservationWrapper, DiscretizedActionWrapper
 
-seed = 123456
-
-# Seed for random starting times of episodes
-random.seed(seed)
-
 def generate_urls_from_yml(boptest_root_dir):
     '''Method that returns as many urls for BOPTEST-Gym environments 
     as those specified at the BOPTEST `docker-compose.yml` file. 
@@ -58,10 +53,18 @@ def generate_urls_from_yml(boptest_root_dir):
     
     return urls
 
-def make_env(url):
+def make_env(url, seed):
     ''' Function that instantiates the environment. 
+    Parameters
+    ----------
+    url: string
+        Rest API url for communication with this environment. 
+    seed: integer
+        Seed for random starting times of episodes in this environment.
     '''
+
     def _init():
+        random.seed(seed)
         env = BoptestGymEnv(
             url=url,
             actions=['oveHeaPumY_u'],
diff --git a/testing/test_boptestGymEnv.py b/testing/test_boptestGymEnv.py
index 8aedf36..d4e181a 100644
--- a/testing/test_boptestGymEnv.py
+++ b/testing/test_boptestGymEnv.py
@@ -397,14 +397,16 @@ def test_vectorized(self, boptest_root = "./"):
         the former was more convenient for use with vectorized environments. 
 
         '''
+
         # Define logging directory. Monitoring data and agent model will be stored here
         log_dir = os.path.join(utilities.get_root_path(), 'examples', 'agents', 'DQN_vectorized')
 
         # Use URLs obtained from docker-compose.yml
         urls = run_vectorized.generate_urls_from_yml(boptest_root_dir=boptest_root)
 
-        # Create BOPTEST-Gym environment replicas
-        envs = [run_vectorized.make_env(url) for url in urls]
+        # Create BOPTEST-Gym environment replicas, each with its own random seed
+        seed = 123456
+        envs = [run_vectorized.make_env(url,seed+idx) for idx,url in enumerate(urls)]
         
         # Create a vectorized environment using SubprocVecEnv
         venv = SubprocVecEnv(envs)

From 2722e3005745d0611df82c68a7b12ebdb4fa0a6f Mon Sep 17 00:00:00 2001
From: Javier Arroyo <javier.arroyo@kuleuven.be>
Date: Wed, 15 Nov 2023 10:57:02 +0100
Subject: [PATCH 20/28] Add reference.

---
 testing/references/vectorized_training.csv | 2 ++
 1 file changed, 2 insertions(+)
 create mode 100644 testing/references/vectorized_training.csv

diff --git a/testing/references/vectorized_training.csv b/testing/references/vectorized_training.csv
new file mode 100644
index 0000000..59144a1
--- /dev/null
+++ b/testing/references/vectorized_training.csv
@@ -0,0 +1,2 @@
+keys,value
+0,0

From a272f5a79455601860166df105ec2b1ae5bca438 Mon Sep 17 00:00:00 2001
From: Javier Arroyo <javier.arroyo@kuleuven.be>
Date: Wed, 15 Nov 2023 12:36:47 +0100
Subject: [PATCH 21/28] Have independent test for vectorized environment.

---
 testing/test_boptestGymEnv.py | 105 ++++++++++++++++++++--------------
 1 file changed, 62 insertions(+), 43 deletions(-)

diff --git a/testing/test_boptestGymEnv.py b/testing/test_boptestGymEnv.py
index d4e181a..f10d2dd 100644
--- a/testing/test_boptestGymEnv.py
+++ b/testing/test_boptestGymEnv.py
@@ -7,6 +7,7 @@
 
 import unittest
 import os
+import sys
 import pandas as pd
 import random
 import shutil
@@ -383,8 +384,66 @@ def test_variable_episode(self):
         
         # Remove model to prove further testing
         shutil.rmtree(log_dir, ignore_errors=True)
-    
-    def test_vectorized(self, boptest_root = "./"):
+        
+    def check_obs_act_rew_kpi(self, obs=None, act=None, rew=None, kpi=None,
+                              label='default'):
+        '''Auxiliary method to check for observations, actions, rewards, 
+        and/or kpis of a particular test case run. 
+        
+        '''
+        
+        # Check observation values
+        if obs is not None:
+            df = pd.DataFrame(obs)
+            df.index.name = 'time' # utilities package requires 'time' as name for index
+            ref_filepath = os.path.join(utilities.get_root_path(), 'testing', 'references', 'observations_{}.csv'.format(label))
+            self.compare_ref_timeseries_df(df, ref_filepath) 
+        
+        # Check actions values
+        if act is not None:
+            df = pd.DataFrame(act)
+            df.index.name = 'time' # utilities package requires 'time' as name for index
+            ref_filepath = os.path.join(utilities.get_root_path(), 'testing', 'references', 'actions_{}.csv'.format(label))
+            self.compare_ref_timeseries_df(df, ref_filepath) 
+        
+        # Check reward values
+        if rew is not None:
+            df = pd.DataFrame(rew)
+            df.index.name = 'time' # utilities package requires 'time' as name for index
+            ref_filepath = os.path.join(utilities.get_root_path(), 'testing', 'references', 'rewards_{}.csv'.format(label))
+            self.compare_ref_timeseries_df(df, ref_filepath) 
+            
+        if kpi is not None:
+            df = pd.DataFrame(data=[kpi]).T
+            df.columns = ['value']
+            df.index.name = 'keys'
+            # Time ratio is not checked since depends on the machine where tests are run
+            df.drop('time_rat', inplace=True)
+            # Drop rows with non-calculated KPIs
+            df.dropna(inplace=True)
+            ref_filepath = os.path.join(utilities.get_root_path(), 'testing', 'references', 'kpis_{}.csv'.format(label))
+            self.compare_ref_values_df(df, ref_filepath)
+
+
+class BoptestGymVecTest(unittest.TestCase, utilities.partialChecks):
+    '''Tests learning with a vectorized environment of BOPTEST-Gym.
+         
+    '''
+
+    def setUp(self):
+        '''Reads the location of boptest root directory which should be passed from 
+        the terminal when invoking this test. 
+        '''
+        
+        boptest_root = "./"
+
+        # Get the argument from command line when use Linux
+        if len(sys.argv) >= 2:
+            self.boptest_root_dir = sys.argv[1:][1]
+        else:
+            self.boptest_root_dir = boptest_root
+
+    def test_vectorized(self):
         '''
         Instantiates a vectorized environment with two BOPTEST-Gym environment replicas
         and learns from them when running in parallel using DQN for 100 timesteps.
@@ -402,7 +461,7 @@ def test_vectorized(self, boptest_root = "./"):
         log_dir = os.path.join(utilities.get_root_path(), 'examples', 'agents', 'DQN_vectorized')
 
         # Use URLs obtained from docker-compose.yml
-        urls = run_vectorized.generate_urls_from_yml(boptest_root_dir=boptest_root)
+        urls = run_vectorized.generate_urls_from_yml(boptest_root_dir=self.boptest_root_dir)
 
         # Create BOPTEST-Gym environment replicas, each with its own random seed
         seed = 123456
@@ -427,46 +486,6 @@ def test_vectorized(self, boptest_root = "./"):
         
         # Remove model to prove further testing
         shutil.rmtree(log_dir, ignore_errors=True)     
-        
-    def check_obs_act_rew_kpi(self, obs=None, act=None, rew=None, kpi=None,
-                              label='default'):
-        '''Auxiliary method to check for observations, actions, rewards, 
-        and/or kpis of a particular test case run. 
-        
-        '''
-        
-        # Check observation values
-        if obs is not None:
-            df = pd.DataFrame(obs)
-            df.index.name = 'time' # utilities package requires 'time' as name for index
-            ref_filepath = os.path.join(utilities.get_root_path(), 'testing', 'references', 'observations_{}.csv'.format(label))
-            self.compare_ref_timeseries_df(df, ref_filepath) 
-        
-        # Check actions values
-        if act is not None:
-            df = pd.DataFrame(act)
-            df.index.name = 'time' # utilities package requires 'time' as name for index
-            ref_filepath = os.path.join(utilities.get_root_path(), 'testing', 'references', 'actions_{}.csv'.format(label))
-            self.compare_ref_timeseries_df(df, ref_filepath) 
-        
-        # Check reward values
-        if rew is not None:
-            df = pd.DataFrame(rew)
-            df.index.name = 'time' # utilities package requires 'time' as name for index
-            ref_filepath = os.path.join(utilities.get_root_path(), 'testing', 'references', 'rewards_{}.csv'.format(label))
-            self.compare_ref_timeseries_df(df, ref_filepath) 
-            
-        if kpi is not None:
-            df = pd.DataFrame(data=[kpi]).T
-            df.columns = ['value']
-            df.index.name = 'keys'
-            # Time ratio is not checked since depends on the machine where tests are run
-            df.drop('time_rat', inplace=True)
-            # Drop rows with non-calculated KPIs
-            df.dropna(inplace=True)
-            ref_filepath = os.path.join(utilities.get_root_path(), 'testing', 'references', 'kpis_{}.csv'.format(label))
-            self.compare_ref_values_df(df, ref_filepath)
-
 
 class BoptestGymServiceTest(unittest.TestCase, utilities.partialChecks):
     '''Tests the BOPTEST-Gym service.

From edfcab98512ee8f7c6e73b020fe54b557f63f9fd Mon Sep 17 00:00:00 2001
From: Javier Arroyo <javier.arroyo@kuleuven.be>
Date: Wed, 15 Nov 2023 12:37:04 +0100
Subject: [PATCH 22/28] Fix doptest typo.

---
 testing/Makefile | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/testing/Makefile b/testing/Makefile
index 3e72400..a373fd5 100644
--- a/testing/Makefile
+++ b/testing/Makefile
@@ -65,17 +65,17 @@ pull-boptestgym:
 	docker pull ${IMG_REGI}:${VERSION}
 	docker tag ${IMG_REGI}:${VERSION} ${IMG_NAME}
 
-make download-doptest:
+make download-boptest:
 	curl -L -o boptest.zip https://github.com/ibpsa/project1-boptest/archive/${BOPTEST_COMMIT}.zip
 	unzip -o -q boptest.zip
 	
 run-boptest-case:
-	make download-doptest
+	make download-boptest
 	cd project1-boptest-${BOPTEST_COMMIT} && \
 	TESTCASE=bestest_hydronic_heat_pump docker-compose up -d --quiet-pull
 
 run-boptest-case-no-cache:
-	make download-doptest
+	make download-boptest
 	cd project1-boptest-${BOPTEST_COMMIT} && \
 	TESTCASE=bestest_hydronic_heat_pump docker-compose up -d --force-recreate --build
 

From 4180122dbf761a355ca681193699d255a77df179 Mon Sep 17 00:00:00 2001
From: Javier Arroyo <javier.arroyo@kuleuven.be>
Date: Wed, 15 Nov 2023 12:37:24 +0100
Subject: [PATCH 23/28] Set new tag for development version.

---
 testing/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/testing/Makefile b/testing/Makefile
index a373fd5..c6b7fab 100644
--- a/testing/Makefile
+++ b/testing/Makefile
@@ -13,7 +13,7 @@ IMG_REGI=javierarroyo/boptestgym
 BOPTEST_COMMIT=12ceafe42983d42e535385dee1daa1d25673e2aa
 
 # Define current BOPTEST-Gym version (should be even with BOPTEST version defined in commit above)
-VERSION = 0.5.0
+VERSION = 0.5.0-dev
 
 build-boptestgym:
 	docker build -f ${ROOT}/testing/Dockerfile \

From 6be17865cf529ea9d8e83f994aabb0beb23f774c Mon Sep 17 00:00:00 2001
From: Javier Arroyo <javier.arroyo@kuleuven.be>
Date: Wed, 15 Nov 2023 12:38:02 +0100
Subject: [PATCH 24/28] Install PyYAML in new environment version.

---
 testing/Dockerfile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/testing/Dockerfile b/testing/Dockerfile
index 64fa65b..e4ad354 100644
--- a/testing/Dockerfile
+++ b/testing/Dockerfile
@@ -29,7 +29,8 @@ RUN pip install \
     nbconvert==7.7.3 \
     gym \
     shimmy \
-    ipykernel==6.25.1
+    ipykernel==6.25.1 \
+    PyYAML
 
 # Add developer user with permissions in the working directory
 RUN useradd -ms /bin/bash developer

From 9985038662e0c2d02bbe45a5490902c3a235eb8d Mon Sep 17 00:00:00 2001
From: Javier Arroyo <javier.arroyo@kuleuven.be>
Date: Wed, 15 Nov 2023 12:38:34 +0100
Subject: [PATCH 25/28] Add new vectorized test to Makefile.

---
 testing/Makefile | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/testing/Makefile b/testing/Makefile
index c6b7fab..09ed149 100644
--- a/testing/Makefile
+++ b/testing/Makefile
@@ -79,6 +79,12 @@ run-boptest-case-no-cache:
 	cd project1-boptest-${BOPTEST_COMMIT} && \
 	TESTCASE=bestest_hydronic_heat_pump docker-compose up -d --force-recreate --build
 
+run-boptest-vectorized:
+	make download-boptest && \
+	cd .. && python3 generateDockerComposeYml.py testing/project1-boptest-${BOPTEST_COMMIT} && \
+	cd testing/project1-boptest-${BOPTEST_COMMIT} && \
+	TESTCASE=bestest_hydronic_heat_pump docker-compose up -d --quiet-pull
+
 stop-boptest-case:
 	cd project1-boptest-${BOPTEST_COMMIT} && docker-compose down
 
@@ -91,6 +97,10 @@ cleanup-boptest:
 test-local:
 	python3 -m unittest test_boptestGymEnv.BoptestGymEnvTest
 
+# Vectorized needs to run separate since modifies docker-compose.yml to have multiple boptest instances 
+test-vectorized:
+	python3 -m unittest test_boptestGymEnv.BoptestGymVecTest project1-boptest-${BOPTEST_COMMIT}
+
 # The tutorial is using boptest-gym-service and covers most of the functionality of boptest-gym 
 test-service:
 	python3 -m unittest test_boptestGymEnv.BoptestGymServiceTest
@@ -103,6 +113,14 @@ test-local-in-container:
 	make stop-boptest-case
 	make cleanup-boptest
 
+test-vectorized-in-container:
+	make run-boptest-vectorized
+	make run-boptestgym-detached
+	make exec-boptestgym ARGS="make test-vectorized"
+	make stop-boptestgym
+	make stop-boptest-case
+	make cleanup-boptest
+
 test-service-in-container:
 	make run-boptestgym-detached
 	make exec-boptestgym ARGS="make test-service"

From cb0936ce073a73323a1ae9917279ed9c0a30413a Mon Sep 17 00:00:00 2001
From: Javier Arroyo <javier.arroyo@kuleuven.be>
Date: Wed, 15 Nov 2023 12:38:51 +0100
Subject: [PATCH 26/28] Add new vectorized test to github-actions.yml.

---
 .github/workflows/github-actions.yml | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/.github/workflows/github-actions.yml b/.github/workflows/github-actions.yml
index 5ece15e..f815ce3 100644
--- a/.github/workflows/github-actions.yml
+++ b/.github/workflows/github-actions.yml
@@ -17,6 +17,22 @@ jobs:
          ls ${{ github.workspace }}
       - name: Test local version
         run: make test-local-in-container
+  test-vectorized:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: testing
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v3
+      - name: Pull boptestgym image from registry
+        run: make pull-boptestgym
+      - run: echo "The name of your branch is ${{ github.ref }} and your repository is ${{ github.repository }}."
+      - name: List of files in the repository
+        run: | 
+         ls ${{ github.workspace }}
+      - name: Test vectorized environment
+        run: make test-vectorized-in-container
   test-service:
     runs-on: ubuntu-latest
     defaults:

From 18e4e78c23024f8464cc2a80595930da45591099 Mon Sep 17 00:00:00 2001
From: Javier Arroyo <javier.arroyo@kuleuven.be>
Date: Wed, 15 Nov 2023 12:39:42 +0100
Subject: [PATCH 27/28] Polish doc in readme.

---
 README.md | 13 ++++---------
 1 file changed, 4 insertions(+), 9 deletions(-)

diff --git a/README.md b/README.md
index 33e31cb..25da481 100644
--- a/README.md
+++ b/README.md
@@ -75,9 +75,10 @@ Running BOPTEST locally is substantially faster
 2) Run a BOPTEST case with the building emulator model to be controlled (instructions [here](https://github.com/ibpsa/project1-boptest/blob/master/README.md)).  
 3) Check out the `master` branch of this repository and run the example above replacing the url to be `url = 'http://127.0.0.1:5000'` and avoiding the `testcase` argument to the `BoptestGymEnv` class. 
 
-## Quick-Start (running BOPTEST locally with parallel training)
+## Quick-Start (running BOPTEST locally in a vectorized environment)
 
-To facilitate the training and testing process, we provide scripts that automate the deployment of multiple BOPTEST instances using Docker Compose and then train the BOPTEST-gym environment in parallel. The deployment dynamically checks for available ports, generates a Docker Compose YAML file, and takes care of naming conflicts to ensure smooth deployment.
+To facilitate the training and testing process, we provide scripts that automate the deployment of multiple BOPTEST instances using Docker Compose and then train an RL agent with a vectorized BOPTEST-gym environment. The deployment dynamically checks for available ports, generates a Docker Compose YAML file, and takes care of naming conflicts to ensure smooth deployment.
+Running a vectorized environment allows you to deploy as many BoptestGymEnv instances as cores you have available for the agent to learn from all of them in parallel (see [here](https://stable-baselines3.readthedocs.io/en/master/guide/vec_envs.html) for more information, we specifically use [`SubprocVecEnv`](https://stable-baselines3.readthedocs.io/en/master/guide/vec_envs.html#subprocvecenv)). This substantially speeds up the training process. 
 
 ### Usage
 
@@ -89,13 +90,7 @@ Example using command-line argument:
 python generateDockerComposeYml.py absolute_boptest_root_dir
 ```
 
-2. Train the BOPTEST-gym environment in parallel either by passing it as a command-line argument or by defining the `boptest_root` variable at the beginning of the script `parallel_BOPTESTgymTrain.py`. The script prioritizes the command-line argument if provided. Users are allowed to change the initial conditions, algorithms and hyperparameters.
-
-Example using command-line argument:
-
-```bash
-python parallel_BOPTESTgymTrain.py absolute_boptest_root_dir
-```
+2. Train an RL agent with parallel learning with the vectorized BOPTEST-gym environment. See `/examples/run_vectorized.py` for an example on how to do so. 
 
 ## Versioning and main dependencies
 

From 9b48ab8d379d48e241172c8a1f3771bbc6abc983 Mon Sep 17 00:00:00 2001
From: Javier Arroyo <javier.arroyo@kuleuven.be>
Date: Wed, 15 Nov 2023 12:40:08 +0100
Subject: [PATCH 28/28] Add entry to releasenotes.

---
 releasenotes.md | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/releasenotes.md b/releasenotes.md
index 76553e6..ee738ff 100644
--- a/releasenotes.md
+++ b/releasenotes.md
@@ -2,6 +2,12 @@
 
 BOPTEST-Gym has two main dependencies: BOPTEST and Stable-Baselines3. For simplicity, the first two digits of the version number match the same two digits of the BOPTEST version of which BOPTEST-Gym is compatible with. For example, BOPTEST-Gym v0.3.x is compatible with BOPTEST v0.3.x. The last digit is reserved for other internal edits specific to this repository only. See [here](https://github.com/ibpsa/project1-boptest/blob/master/releasenotes.md) for BOPTEST release notes. 
 
+## BOPTEST-Gym v0.5.0-dev
+
+Released on xx/xx/xxxx.
+
+- Implement functionality and example with vectorized environment for parallel learning. This is for [#133](https://github.com/ibpsa/project1-boptest-gym/issues/133). 
+
 ## BOPTEST-Gym v0.5.0
 
 Released on 11/11/2023.