From 4d8fa80b07331d906eabb59bb062b882e0492433 Mon Sep 17 00:00:00 2001
From: Natalie Weires <nweires@google.com>
Date: Thu, 14 Mar 2024 15:40:24 +0000
Subject: [PATCH 1/2] Add option to run multiple jobs in a row

---
 buildstockbatch/gcp/gcp.py | 73 +++++++++++++++++++++++---------------
 1 file changed, 45 insertions(+), 28 deletions(-)

diff --git a/buildstockbatch/gcp/gcp.py b/buildstockbatch/gcp/gcp.py
index 0f7bcea6..e078aca2 100644
--- a/buildstockbatch/gcp/gcp.py
+++ b/buildstockbatch/gcp/gcp.py
@@ -1186,12 +1186,13 @@ def main():
         GcpBatch.run_combine_results_on_cloud(gcs_bucket, gcs_prefix, results_dir, do_timeseries)
     else:
         parser = argparse.ArgumentParser()
-        parser.add_argument("project_filename")
+        parser.add_argument("project_filenames", help="Comma-separated list of project YAML files to run.")
         parser.add_argument(
-            "job_identifier",
+            "job_identifiers",
             nargs="?",
             default=None,
-            help="Optional override of gcp.job_identifier in your project file. Max 48 characters.",
+            help="Comma-separated list of job IDs to use."
+            "Optional override of gcp.job_identifier in your project file. Max 48 characters.",
         )
         group = parser.add_mutually_exclusive_group()
         group.add_argument(
@@ -1232,34 +1233,50 @@ def main():
         else:
             logger.setLevel(logging.INFO)
 
-        # validate the project, and if --validateonly flag is set, return True if validation passes
-        GcpBatch.validate_project(os.path.abspath(args.project_filename))
-        if args.validateonly:
-            return True
+        project_filenames = args.project_filenames.split(",")
+        n_projects = len(project_filenames)
+        job_IDs = len(project_filenames) * [None]
+        if args.job_identifiers:
+            job_IDs = args.job_identifiers.split(",")
+            if len(job_IDs) != n_projects:
+                raise ValidationError(
+                    f"job_identifiers contains {len(args.job_identifiers.split(','))} IDs, "
+                    f"but project_filenames contains {n_projects} files"
+                )
 
-        batch = GcpBatch(args.project_filename, args.job_identifier, missing_only=args.missingonly)
-        if args.clean:
-            batch.clean()
-            return
-        if args.show_jobs:
-            batch.show_jobs()
+        for project_filename, job_ID in zip(project_filenames, job_IDs):
+            logger.info(f"----------Validating {project_filename}{f' ({job_ID})' if job_ID else ''}----------")
+            # validate the project, and if --validateonly flag is set, return True if validation passes
+            GcpBatch.validate_project(os.path.abspath(project_filename))
+
+        if args.validateonly:
             return
-        elif args.postprocessonly:
-            if batch.check_for_existing_jobs(pp_only=True):
-                return
-            batch.build_image()
-            batch.push_image()
-            batch.process_results()
-        else:
-            if batch.check_for_existing_jobs():
-                return
-            if not args.missingonly:
-                batch.check_output_dir()
 
-            batch.build_image()
-            batch.push_image()
-            batch.run_batch()
-            batch.process_results()
+        for project_filename, job_ID in zip(project_filenames, job_IDs):
+            logger.info(f"----------Starting {project_filename}{f' ({job_ID})' if job_ID else ''}----------")
+            batch = GcpBatch(project_filename, job_ID, missing_only=args.missingonly)
+            if args.clean:
+                batch.clean()
+                continue
+            if args.show_jobs:
+                batch.show_jobs()
+                continue
+            elif args.postprocessonly:
+                if batch.check_for_existing_jobs(pp_only=True):
+                    continue
+                batch.build_image()
+                batch.push_image()
+                batch.process_results()
+            else:
+                if batch.check_for_existing_jobs():
+                    continue
+                if not args.missingonly:
+                    batch.check_output_dir()
+
+                batch.build_image()
+                batch.push_image()
+                batch.run_batch()
+                batch.process_results()
 
 
 if __name__ == "__main__":

From 07ccfaea75ad94bb0d8acd5e76542dfb75953e8a Mon Sep 17 00:00:00 2001
From: Natalie Weires <nweires@google.com>
Date: Fri, 29 Mar 2024 16:15:22 +0000
Subject: [PATCH 2/2] Update docs

---
 buildstockbatch/gcp/gcp.py | 2 +-
 docs/run_sims.rst          | 4 ++++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/buildstockbatch/gcp/gcp.py b/buildstockbatch/gcp/gcp.py
index bf8fa552..1c4dae0f 100644
--- a/buildstockbatch/gcp/gcp.py
+++ b/buildstockbatch/gcp/gcp.py
@@ -1107,7 +1107,7 @@ def main():
             "job_identifiers",
             nargs="?",
             default=None,
-            help="Comma-separated list of job IDs to use."
+            help="Comma-separated list of job IDs to use. "
             "Optional override of gcp.job_identifier in your project file. Max 48 characters.",
         )
         group = parser.add_mutually_exclusive_group()
diff --git a/docs/run_sims.rst b/docs/run_sims.rst
index 16466630..ea925132 100644
--- a/docs/run_sims.rst
+++ b/docs/run_sims.rst
@@ -167,6 +167,10 @@ You can optionally override the ``job_identifier`` from the command line
 (unless you delete a previous job with the ``--clean`` option), so this option makes it easier to
 quickly assign a new ID with each run without updating the config file.
 
+You can also run multiple projects in a row by specifying multiple project files (and optionally multiple IDs):
+``buildstock_gcp project_file1,yaml,project_file2.yaml job_id1,job_id2``.
+This will validate all the projects upfront, then run them sequentially.
+
 
 Retry failed tasks
 ..................