feat: added data structures in request manager to handle preempted re…

…quests
flexflow · Dec 6, 2024 · 4c1b2ce · 4c1b2ce
1 parent b0a5918
commit 4c1b2ce
Showing 1 changed file with 3 additions and 0 deletions.
diff --git a/include/flexflow/request_manager.h b/include/flexflow/request_manager.h
@@ -501,7 +501,9 @@ class RequestManager {
   int num_running_requests = 0;
   // Available requests in the batch config
   bool request_available[BatchConfig::MAX_NUM_REQUESTS];
+  bool request_preempted[BatchConfig::MAX_NUM_REQUESTS];
   int num_available_requests = 0;
+  int num_preempted_requests = 0;
   int ssm_completed = true;
   int ssm_tree_depth = 0;
 
@@ -529,6 +531,7 @@ class RequestManager {
   bool update_llm_decode_results(InferenceResult const &result);
   BatchConfig prepare_llm_prefilling_batch();
   BatchConfig prepare_decoding_batch();
+  BatchConfig prepare_decoding_batch_fcfs_slo();
   /* ---------- Incremental Decoding Helper Functions ---------- */
 
   /* ---------- Spec Decoding Helper Functions ---------- */