Create batch_queue_flux as experimental module (#3978)

* Create batch_queue_flux stub * Mark batch_queue_flux as experimental * flux_{submit,wait,remove} * clang format * fix macos build errors? * small PR changes * add memset of buffer * add flux to makeflow documentation
cooperative-computing-lab · Jan 10, 2025 · ba4b5bb · ba4b5bb
1 parent 67db4e7
commit ba4b5bb
Show file tree

Hide file tree

Showing 5 changed files with 399 additions and 4 deletions.
diff --git a/batch_job/src/Makefile b/batch_job/src/Makefile
@@ -25,7 +25,8 @@ SOURCES = \
 	batch_queue_work_queue.c \
 	batch_queue_cluster.c \
 	batch_queue_k8s.c \
-	batch_queue_amazon.c
+	batch_queue_amazon.c \
+	batch_queue_flux.c
 
 PUBLIC_HEADERS = batch_queue.h batch_job.h batch_job_info.h batch_file.h batch_wrapper.h
 

diff --git a/batch_job/src/batch_queue.c b/batch_job/src/batch_queue.c
@@ -21,6 +21,7 @@ See the file COPYING for details.
 extern const struct batch_queue_module batch_queue_amazon;
 extern const struct batch_queue_module batch_queue_cluster;
 extern const struct batch_queue_module batch_queue_condor;
+extern const struct batch_queue_module batch_queue_flux;
 extern const struct batch_queue_module batch_queue_local;
 extern const struct batch_queue_module batch_queue_moab;
 extern const struct batch_queue_module batch_queue_uge;
@@ -45,13 +46,14 @@ static struct batch_queue_module batch_queue_unknown = {
 		NULL,
 };
 
-#define BATCH_JOB_SYSTEMS "local, vine, wq, condor, uge (sge), pbs, lsf, torque, moab, slurm, amazon, k8s, dryrun"
+#define BATCH_JOB_SYSTEMS "local, vine, wq, condor, uge (sge), pbs, lsf, torque, moab, slurm, amazon, k8s, flux, dryrun"
 
 const struct batch_queue_module *const batch_queue_modules[] = {
 		&batch_queue_amazon,
 		&batch_queue_cluster,
 		&batch_queue_condor,
 		&batch_queue_dryrun,
+		&batch_queue_flux,
 		&batch_queue_local,
 		&batch_queue_moab,
 		&batch_queue_uge,

diff --git a/batch_job/src/batch_queue.h b/batch_job/src/batch_queue.h
@@ -33,7 +33,7 @@ Basic use is as follows:
 
 <pre>
 // Create a queue for submitting to HTCondor.
-struct batch_queue *queue = batch_queue_create(BATCH_QUEUE_TYPE_CONDOR);
+struct batch_queue *queue = batch_queue_create(BATCH_QUEUE_TYPE_CONDOR,0,0);
 
 // Define a batch job consiting of a command with input and output files.
 struct batch_job *job = batch_job_create(queue);
@@ -43,7 +43,7 @@ batch_job_add_input_file(job,"/usr/share/dict/words","words.txt");
 batch_job_add_output_file(job,"output.txt","output.txt");
 
 // Submit the job to the queue, which returns a jobid 
-batch_job_id_t jobid = batch_queue_submit(queue,job);
+batch_queue_id_t jobid = batch_queue_submit(queue,job);
 printf("jobid %" PRIbjid" submitted\n",jobid);
 
 // Wait for a job to complete, which returns the jobid and info.
@@ -79,6 +79,7 @@ typedef enum {
 	BATCH_QUEUE_TYPE_TORQUE,              /**< Batch jobs will be send to the Torque scheduler. */
 	BATCH_QUEUE_TYPE_AMAZON,              /**< Batch jobs will be run inside Amazon EC2 Instances */
 	BATCH_QUEUE_TYPE_K8S,                 /**< Batch jobs will be run inside Kubernetes pods. */
+	BATCH_QUEUE_TYPE_FLUX,                /**< Batch jobs will be run on Flux. */
 	BATCH_QUEUE_TYPE_DRYRUN,              /**< Batch jobs will not actually run. */
 	BATCH_QUEUE_TYPE_UNKNOWN = -1         /**< An invalid batch queue type. */
 } batch_queue_type_t;