From 255f8efaae081e827a9e5fd10e4acc3eacf3cb0b Mon Sep 17 00:00:00 2001 From: Gavrish Prabhu Date: Mon, 26 Feb 2024 18:08:13 +0000 Subject: [PATCH 1/5] refresh request map for every process --- llm/handler.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/llm/handler.py b/llm/handler.py index 739e426..5e65338 100644 --- a/llm/handler.py +++ b/llm/handler.py @@ -73,11 +73,6 @@ class LLMHandler(BaseHandler, ABC): def __init__(self): super().__init__() self.initialized = False - self.request = { - "request_list": defaultdict(int), - "request_ids": defaultdict(int), - "request_type": defaultdict(int), - } self.tokenizer = None self.map_location = None self.device = None @@ -155,6 +150,11 @@ def preprocess(self, data: List) -> torch.Tensor: Tensor: Tokenized input data """ input_list = [] + self.request = { + "request_list": defaultdict(int), + "request_ids": defaultdict(int), + "request_type": defaultdict(int), + } for idx, input_data in enumerate(data): # Pre-process for Kserve v2 format From 1cb566fb670289471a25c117b630fea91aee5b38 Mon Sep 17 00:00:00 2001 From: Gavrish Prabhu Date: Mon, 26 Feb 2024 18:16:11 +0000 Subject: [PATCH 2/5] fix lint --- llm/handler.py | 1 + 1 file changed, 1 insertion(+) diff --git a/llm/handler.py b/llm/handler.py index 5e65338..3162dcb 100644 --- a/llm/handler.py +++ b/llm/handler.py @@ -78,6 +78,7 @@ def __init__(self): self.device = None self.device_map = None self.model = None + self.request = None def initialize(self, context: ts.context.Context): """ From 334eb150513653bef09f8c7189c25dd42fcda234 Mon Sep 17 00:00:00 2001 From: Gavrish Prabhu Date: Mon, 26 Feb 2024 18:25:39 +0000 Subject: [PATCH 3/5] update lint error from black --- llm/cleanup.py | 1 + llm/generate.py | 1 + llm/handler.py | 1 + llm/kubeflow_inference_run.py | 1 + llm/tests/test_generate.py | 1 + llm/utils/generate_data_model.py | 1 + llm/utils/hf_utils.py | 1 + llm/utils/marsgen.py | 1 + llm/utils/system_utils.py | 1 + llm/utils/tsutils.py | 1 + 10 files changed, 10 insertions(+) diff --git a/llm/cleanup.py b/llm/cleanup.py index 1d72691..27abeed 100644 --- a/llm/cleanup.py +++ b/llm/cleanup.py @@ -1,6 +1,7 @@ """ Clean up Kubernetes resources associated with a deployment. """ + import argparse import sys import requests diff --git a/llm/generate.py b/llm/generate.py index a73222f..11fccd0 100644 --- a/llm/generate.py +++ b/llm/generate.py @@ -2,6 +2,7 @@ Downloads model files, generates Model Archive (MAR) and config.properties file """ + import os import argparse import json diff --git a/llm/handler.py b/llm/handler.py index 3162dcb..cabf3cd 100644 --- a/llm/handler.py +++ b/llm/handler.py @@ -3,6 +3,7 @@ The handler provides functions to preprocess input data, make predictions using the model, and post-process the output for a particular use case. """ + import logging import os from abc import ABC diff --git a/llm/kubeflow_inference_run.py b/llm/kubeflow_inference_run.py index 43a0677..51dc500 100644 --- a/llm/kubeflow_inference_run.py +++ b/llm/kubeflow_inference_run.py @@ -3,6 +3,7 @@ in a Kubernetes cluster by performing tasks such as creating persistent storage, registering the model, and running inference. """ + import argparse import sys import os diff --git a/llm/tests/test_generate.py b/llm/tests/test_generate.py index 1a06bd2..9719d75 100644 --- a/llm/tests/test_generate.py +++ b/llm/tests/test_generate.py @@ -7,6 +7,7 @@ MODEL_CONFIG_PATH: Path to model_config.json file. MODEL_TEMP_CONFIG_PATH: Path to backup model_config.json file. """ + import os import argparse import json diff --git a/llm/utils/generate_data_model.py b/llm/utils/generate_data_model.py index 79d406b..54e4097 100644 --- a/llm/utils/generate_data_model.py +++ b/llm/utils/generate_data_model.py @@ -3,6 +3,7 @@ function set_values that sets the GenerateDataModel attributes and function set_model_files_and_mar that sets model path and mar output values. """ + import os import dataclasses import argparse diff --git a/llm/utils/hf_utils.py b/llm/utils/hf_utils.py index 4c34b0e..9def319 100644 --- a/llm/utils/hf_utils.py +++ b/llm/utils/hf_utils.py @@ -1,6 +1,7 @@ """ Utility functions for using HuggingFace Api """ + import sys from typing import List from huggingface_hub import HfApi diff --git a/llm/utils/marsgen.py b/llm/utils/marsgen.py index ac94b01..4c3e256 100644 --- a/llm/utils/marsgen.py +++ b/llm/utils/marsgen.py @@ -1,6 +1,7 @@ """ Generate a Model Archive (MAR) file for a specified LLM. """ + import json import os import sys diff --git a/llm/utils/system_utils.py b/llm/utils/system_utils.py index 8e81610..aa10c87 100644 --- a/llm/utils/system_utils.py +++ b/llm/utils/system_utils.py @@ -1,6 +1,7 @@ """ Utility functions to handle file and folder operations """ + import os import sys import shutil diff --git a/llm/utils/tsutils.py b/llm/utils/tsutils.py index 1017d54..b916bbe 100644 --- a/llm/utils/tsutils.py +++ b/llm/utils/tsutils.py @@ -1,6 +1,7 @@ """ Utility functions for running inference and getiing model parameters """ + import os import json import collections From 13c47d980121b7398724c30275b8070ccea52b0f Mon Sep 17 00:00:00 2001 From: Gavrish Prabhu Date: Tue, 27 Feb 2024 18:38:15 +0000 Subject: [PATCH 4/5] remove logs --- llm/handler.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/llm/handler.py b/llm/handler.py index cabf3cd..b84b97e 100644 --- a/llm/handler.py +++ b/llm/handler.py @@ -184,7 +184,6 @@ def preprocess(self, data: List) -> torch.Tensor: self.request["request_type"][idx] = "raw" input_list.append(row_input) - logger.info("Received text: %s", ", ".join(map(str, input_list))) encoded_input = self.tokenizer(input_list, padding=True, return_tensors="pt")[ "input_ids" ].to(self.device) @@ -227,7 +226,6 @@ def inference(self, data: torch.Tensor, *args, **kwargs) -> List: inference = [] inference = self.tokenizer.batch_decode(generated_ids, skip_special_tokens=True) - logger.info("Generated text is: %s", ", ".join(map(str, inference))) return inference def postprocess(self, data: List) -> List: From 2469eebccf1bd401b090287e70ce7fb6a9658aea Mon Sep 17 00:00:00 2001 From: Gavrish Prabhu Date: Wed, 28 Feb 2024 08:18:43 +0000 Subject: [PATCH 5/5] update transformer version --- llm/utils/model_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llm/utils/model_requirements.txt b/llm/utils/model_requirements.txt index fba6872..027d1a7 100644 --- a/llm/utils/model_requirements.txt +++ b/llm/utils/model_requirements.txt @@ -1,6 +1,6 @@ torch==2.0.1 tokenizers==0.15.0 -transformers==4.36.0 +transformers==4.38.1 accelerate==0.22.0 einops==0.6.1 bitsandbytes==0.41.1