refresh request map for every process

nutanix · Feb 26, 2024 · 255f8ef · 255f8ef
1 parent 4c45d3c
commit 255f8ef
Showing 1 changed file with 5 additions and 5 deletions.
diff --git a/llm/handler.py b/llm/handler.py
@@ -73,11 +73,6 @@ class LLMHandler(BaseHandler, ABC):
  def __init__(self):
  super().__init__()
  self.initialized = False
- self.request = {
- "request_list": defaultdict(int),
- "request_ids": defaultdict(int),
- "request_type": defaultdict(int),
- }
  self.tokenizer = None
  self.map_location = None
  self.device = None
@@ -155,6 +150,11 @@ def preprocess(self, data: List) -> torch.Tensor:
  Tensor: Tokenized input data
  """
  input_list = []
+ self.request = {
+ "request_list": defaultdict(int),
+ "request_ids": defaultdict(int),
+ "request_type": defaultdict(int),
+ }
 
  for idx, input_data in enumerate(data):
  # Pre-process for Kserve v2 format