updated smart_coding large model benchmark

Signed-off-by: boX <[email protected]>

core/testcasecontroller/algorithm/paradigm/singletask_learning/singletask_learning.py

@@ -84,8 +84,5 @@ def _inference(self, job, trained_model):
         inference_output_dir = os.path.join(self.workspace, "output/inference/")
         os.environ["RESULT_SAVED_URL"] = inference_output_dir
         job.load(trained_model)
-        if hasattr(inference_dataset, 'need_other_info'):
-            infer_res = job.predict(inference_dataset)
-        else:
-            infer_res = job.predict(inference_dataset.x)
+        infer_res = job.predict(inference_dataset.x)
         return infer_res

core/testenvmanager/dataset/dataset.py

@@ -16,10 +16,9 @@
 
 import os
 import tempfile
-import json
 
 import pandas as pd
-from sedna.datasources import CSVDataParse, TxtDataParse, JSONDataParse, JsonlDataParse, JSONMetaDataParse
+from sedna.datasources import CSVDataParse, TxtDataParse, JSONDataParse
 
 from core.common import utils
 from core.common.constant import DatasetFormat
@@ -39,28 +38,12 @@ class Dataset:
     def __init__(self, config):
         self.train_url: str = ""
         self.test_url: str = ""
-        self.train_index: str = ""
-        self.test_index: str = ""
-        self.train_data: str = ""
-        self.test_data: str = ""
-        self.train_data_info: str = ""
-        self.test_data_info: str = ""
         self.label: str = ""
         self._parse_config(config)
 
     def _check_fields(self):
-        if self.train_index:
-            self._check_dataset_url(self.train_index)
-        if self.test_index:
-            self._check_dataset_url(self.test_index)
-        if self.train_data:
-            self._check_dataset_url(self.train_data)
-        if self.test_data:
-            self._check_dataset_url(self.test_data)
-        if self.train_data_info:
-            self._check_dataset_url(self.train_data_info)
-        if self.test_data_info:
-            self._check_dataset_url(self.test_data_info)
+        self._check_dataset_url(self.train_url)
+        self._check_dataset_url(self.test_url)
 
     def _parse_config(self, config):
         for attr, value in config.items():
@@ -120,20 +103,6 @@ def _process_index_file(self, file_url):
 
         return None
 
-    def _process_data_file(self, file_url):
-        file_format = utils.get_file_format(file_url)
-        if file_format == DatasetFormat.JSONL.value:
-            return file_url
-
-        return None
-
-    def _process_data_info_file(self, file_url):
-        file_format = utils.get_file_format(file_url)
-        if file_format == DatasetFormat.JSON.value:
-            return file_url
-
-        return None
-
     def process_dataset(self):
         """
         process dataset:
@@ -142,26 +111,9 @@ def process_dataset(self):
               in the index file(e.g.: txt index file).
 
         """
-        if self.train_index:
-            self.train_url = self._process_index_file(self.train_index)
-        elif self.train_data:
-            self.train_url = self._process_data_file(self.train_data)
-        elif self.train_data_info:
-            self.train_url = self._process_data_info_file(self.train_data_info)
-            # raise NotImplementedError('to be done')
-        else:
-            raise NotImplementedError('not one of train_index/train_data/train_data_info')
-
-        if self.test_index:
-            self.test_url = self._process_index_file(self.test_index)
-        elif self.test_data:
-            self.test_url = self._process_data_file(self.test_data)
-        elif self.test_data_info:
-            self.test_url = self._process_data_info_file(self.test_data_info)
-            # raise NotImplementedError('to be done')
-        else:
-            raise NotImplementedError('not one of test_index/test_data/test_data_info')
 
+        self.train_url = self._process_index_file(self.train_url)
+        self.test_url = self._process_index_file(self.test_url)
 
     # pylint: disable=too-many-arguments
     def split_dataset(self, dataset_url, dataset_format, ratio, method="default",
@@ -436,11 +388,6 @@ def load_data(cls, file: str, data_type: str, label=None, use_raw=False, feature
             e.g.: TxtDataParse, CSVDataParse.
 
         """
-        if file.split('/')[-1] == "metadata.json":
-            data = JSONMetaDataParse(data_type=data_type, func=feature_process)
-            data.parse(file)
-            return data
-
         data_format = utils.get_file_format(file)
 
         data = None
@@ -450,14 +397,11 @@ def load_data(cls, file: str, data_type: str, label=None, use_raw=False, feature
 
         if data_format == DatasetFormat.TXT.value:
             data = TxtDataParse(data_type=data_type, func=feature_process)
+            #print(file)
             data.parse(file, use_raw=use_raw)
 
         if data_format == DatasetFormat.JSON.value:
             data = JSONDataParse(data_type=data_type, func=feature_process)
             data.parse(file)
 
-        if data_format == DatasetFormat.JSONL.value:
-            data = JsonlDataParse(data_type=data_type, func=feature_process)
-            data.parse(file)
-
         return data

docs/proposals/scenarios/Smart_Coding/Smart Coding benchmark suite Proposal.md

@@ -75,13 +75,13 @@ The format of the issue test set is as follows:
   "body":"This is not possible right now afaik :/\r\n\r\nMaybe we could have something like this ? wdyt ?\r\n\r\n```python\r\nds = interleave_datasets(\r\n    [shuffled_dataset_a, dataset_b],\r\n    probabilities=probabilities,\r\n    stopping_strategy='all_exhausted',\r\n    reshuffle_each_iteration=True,\r\n)",
   "answer_1": {
     "user_login": "name",
-    "created_at":"time",
+    "created_at":"time"
     "updated_at": "time",
     "body":"This is not possible right now afaik :/\r\n\r\nMaybe we could have something like this ? wdyt ?\r\n\r\n```python\r\nds = interleave_datasets(\r\n    [shuffled_dataset_a, dataset_b],\r\n    probabilities=probabilities,\r\n    stopping_strategy='all_exhausted',\r\n    reshuffle_each_iteration=True,\r\n)",
   },
   "answer_2": {
     "user_login": "name",
-    "created_at":"time",
+    "created_at":"time"
     "updated_at": "time",
     "body":"XXXX"
   },

examples/smart_coding/smart_coding_learning_bench/issue/testalgorithms/gen/basemodel.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import absolute_import, division, LOGGER.info_function
+from __future__ import absolute_import, division
 
 import os
 import tempfile
@@ -44,11 +44,11 @@ class BaseModel:
 
     def __init__(self, **kwargs):
         self.model = AutoModelForCausalLM.from_pretrained(
-            "/root/autodl-tmp/Qwen2.5-Coder-1.5B-Instruct",
+            "/home/xiebo/model/Qwen2.5-Coder-1.5B-Instruct",
             torch_dtype="auto",
             device_map="auto"
         )
-        self.tokenizer = AutoTokenizer.from_pretrained("/root/autodl-tmp/Qwen2.5-Coder-1.5B-Instruct")
+        self.tokenizer = AutoTokenizer.from_pretrained("/home/xiebo/model/Qwen2.5-Coder-1.5B-Instruct")
 
     def train(self, train_data, valid_data=None, **kwargs):
         LOGGER.info("BaseModel train")
@@ -58,26 +58,23 @@ def save(self, model_path):
 
     def predict(self, data, input_shape=None, **kwargs):
         LOGGER.info("BaseModel predict")
-
-        if 'infer_system_prompt' in data.prompts:
-            infer_system_prompt = data.prompts['infer_system_prompt']
+        LOGGER.info(f"Dataset: {data.dataset_name}")
+        LOGGER.info(f"Description: {data.description}")
+        LOGGER.info(f"Data Level 1 Dim: {data.level_1_dim}")
+        LOGGER.info(f"Data Level 2 Dim: {data.level_2_dim}")
 
         answer_list = []
         for line in tqdm(data.x, desc="Processing", unit="question"):
             history = []
-            query = line.split('||')[0]
-            if infer_system_prompt:
-                history.append({"role": "system", "content": infer_system_prompt})
-            history.append({"role": "user", "content": query})
+            history.append({"role": "user", "content": line})
             response = self._infer(history)
             answer_list.append(response)
 
         judgement_list = []
 
         # evaluate by llm
         for index in tqdm(range(len(answer_list)), desc="Evaluating", ascii=False, ncols=75):
-            prompt = data.prompts['eval_user_template'].replace('{question}', data.x[index].split('||')[0]).replace(
-                '{reference}', data.x[index].split('||')[1]).replace('{answer}', answer_list[index])
+            prompt = data.judge_prompts[index] + answer_list[index]
             judgement = self._openai_generate(prompt)
             judgement_list.append(judgement)
 
@@ -112,6 +109,8 @@ def _infer(self, messages):
 
     def _openai_generate(self, user_question, system=None):
         key = os.getenv("DEEPSEEK_API_KEY")
+        if not key:
+            raise ValueError("You should set DEEPSEEK_API_KEY in your env.")
         client = OpenAI(api_key=key, base_url="https://api.deepseek.com")
 
         messages = []
@@ -127,4 +126,4 @@ def _openai_generate(self, user_question, system=None):
 
         res = response.choices[0].message.content
 
-        return res
+        return res