diff --git a/llm/handler.py b/llm/handler.py index 739e426..5e65338 100644 --- a/llm/handler.py +++ b/llm/handler.py @@ -73,11 +73,6 @@ class LLMHandler(BaseHandler, ABC): def __init__(self): super().__init__() self.initialized = False - self.request = { - "request_list": defaultdict(int), - "request_ids": defaultdict(int), - "request_type": defaultdict(int), - } self.tokenizer = None self.map_location = None self.device = None @@ -155,6 +150,11 @@ def preprocess(self, data: List) -> torch.Tensor: Tensor: Tokenized input data """ input_list = [] + self.request = { + "request_list": defaultdict(int), + "request_ids": defaultdict(int), + "request_type": defaultdict(int), + } for idx, input_data in enumerate(data): # Pre-process for Kserve v2 format