updated smart_coding large model benchmark

Signed-off-by: boX <[email protected]>
kubeedge · Oct 25, 2024 · e1a0b3d · e1a0b3d
1 parent e94dad4
commit e1a0b3d
Show file tree

Hide file tree

Showing 6 changed files with 13 additions and 13,127 deletions.
diff --git a/examples/data/data_java.json b/examples/data/data_java.json
diff --git a/examples/data/data_python.json b/examples/data/data_python.json
diff --git a/examples/data/issue_comment.json b/examples/data/issue_comment.json
diff --git a/examples/data/request_issue.py b/examples/data/request_issue.py
diff --git a/examples/data/test_data.json b/examples/data/test_data.json
diff --git a/examples/smart_coding/smart_coding_learning_bench/issue/testalgorithms/gen/basemodel.py b/examples/smart_coding/smart_coding_learning_bench/issue/testalgorithms/gen/basemodel.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import absolute_import, division
+from __future__ import absolute_import, division, LOGGER.info_function
 
 import os
 import tempfile
@@ -44,11 +44,11 @@ class BaseModel:
 
     def __init__(self, **kwargs):
         self.model = AutoModelForCausalLM.from_pretrained(
-            "/home/xiebo/model/Qwen2.5-Coder-1.5B-Instruct",
+            "/root/autodl-tmp/Qwen2.5-Coder-1.5B-Instruct",
             torch_dtype="auto",
             device_map="auto"
         )
-        self.tokenizer = AutoTokenizer.from_pretrained("/home/xiebo/model/Qwen2.5-Coder-1.5B-Instruct")
+        self.tokenizer = AutoTokenizer.from_pretrained("/root/autodl-tmp/Qwen2.5-Coder-1.5B-Instruct")
 
     def train(self, train_data, valid_data=None, **kwargs):
         LOGGER.info("BaseModel train")
@@ -58,23 +58,26 @@ def save(self, model_path):
 
     def predict(self, data, input_shape=None, **kwargs):
         LOGGER.info("BaseModel predict")
-        LOGGER.info(f"Dataset: {data.dataset_name}")
-        LOGGER.info(f"Description: {data.description}")
-        LOGGER.info(f"Data Level 1 Dim: {data.level_1_dim}")
-        LOGGER.info(f"Data Level 2 Dim: {data.level_2_dim}")
+
+        if 'infer_system_prompt' in data.prompts:
+            infer_system_prompt = data.prompts['infer_system_prompt']
 
         answer_list = []
         for line in tqdm(data.x, desc="Processing", unit="question"):
             history = []
-            history.append({"role": "user", "content": line})
+            query = line.split('||')[0]
+            if infer_system_prompt:
+                history.append({"role": "system", "content": infer_system_prompt})
+            history.append({"role": "user", "content": query})
             response = self._infer(history)
             answer_list.append(response)
 
         judgement_list = []
 
         # evaluate by llm
         for index in tqdm(range(len(answer_list)), desc="Evaluating", ascii=False, ncols=75):
-            prompt = data.judge_prompts[index] + answer_list[index]
+            prompt = data.prompts['eval_user_template'].replace('{question}', data.x[index].split('||')[0]).replace(
+                '{reference}', data.x[index].split('||')[1]).replace('{answer}', answer_list[index])
             judgement = self._openai_generate(prompt)
             judgement_list.append(judgement)
 
@@ -109,8 +112,6 @@ def _infer(self, messages):
 
     def _openai_generate(self, user_question, system=None):
         key = os.getenv("DEEPSEEK_API_KEY")
-        if not key:
-            raise ValueError("You should set DEEPSEEK_API_KEY in your env.")
         client = OpenAI(api_key=key, base_url="https://api.deepseek.com")
 
         messages = []
@@ -126,4 +127,4 @@ def _openai_generate(self, user_question, system=None):
 
         res = response.choices[0].message.content
 
-        return res
+        return res