Converted Datadog_API.json to Datadog_API.jsonl

berkeley-function-call-leaderboard/bfcl/_apply_function_credential_config.py

@@ -1,6 +1,7 @@
 import glob
 import json
 import os
+import jsonlines
 
 from bfcl.constant import PROMPT_PATH
 from bfcl.eval_checker.executable_eval.custom_exception import NoAPIKeyError
@@ -34,24 +35,34 @@ def replace_placeholders(data):
 
 def process_file(input_file_path, output_file_path):
     modified_data = []
-    with open(input_file_path, "r") as f:
-        lines = f.readlines()
+    # with open(input_file_path, "r") as f:
+    #     lines = f.readlines()
+    with jsonlines.open(input_file_path) as reader:
+        lines = [obj for obj in reader]
+        reader.close()
         for line in lines:
             try:
                 data = json.loads(line)  # Parse each line as a JSON object
                 data = replace_placeholders(data)  # Replace placeholders
                 modified_data.append(json.dumps(data))  # Convert back to string and store
+
             except json.JSONDecodeError:
                 # Handle the case where a line is not a valid JSON object
                 print("Invalid JSON line skipped.")
                 continue
 
     # Write the modified data to the output file
-    with open(output_file_path, "w") as f:
-        for i, modified_line in enumerate(modified_data):
-            f.write(modified_line)
-            if i < len(modified_data) - 1:  # Check against the length of modified_data
-                f.write("\n")
+    # with open(output_file_path, "w") as f:
+    #     for i, modified_line in enumerate(modified_data):
+    #         f.write(modified_line)
+    #         if i < len(modified_data) - 1:  # Check against the length of modified_data
+    #             f.write("\n")
+    with jsonlines.open(output_file_path, "w") as writer:
+        for modified_line in modified_data:
+            writer.write(modified_line)
+            # if i < len(modified_data) - 1:  # Check against the length of modified_data
+            #     f.write("\n")
+            writer.close()
 
 
 def process_dir(input_dir, output_dir):
@@ -62,8 +73,8 @@ def process_dir(input_dir, output_dir):
     # Get a list of all entries in the folder
     entries = os.scandir(input_dir)
 
-    json_files_pattern = os.path.join(input_dir, "*.json")
-    for input_file_path in glob.glob(json_files_pattern):
+    jsonl_files_pattern = os.path.join(input_dir, "*.jsonl")
+    for input_file_path in glob.glob(jsonl_files_pattern):
         file_name = os.path.basename(input_file_path)
         output_file_path = os.path.join(output_dir, file_name)
         process_file(input_file_path, output_file_path)

berkeley-function-call-leaderboard/bfcl/eval_checker/eval_runner_helper.py

@@ -4,6 +4,7 @@
 import statistics
 from pathlib import Path
 from typing import Union
+import jsonlines
 
 import numpy as np
 from bfcl._apply_function_credential_config import apply_function_credential_config
@@ -17,7 +18,8 @@
 
 def extract_test_category(input_string: Union[str, Path]) -> str:
     input_string = str(input_string)
-    pattern = fr".*{VERSION_PREFIX}_(\w+?)(?:_score|_result)?\.json"
+    # pattern = fr".*{VERSION_PREFIX}_(\w+?)(?:_score|_result)?\.json"
+    pattern = fr".*{VERSION_PREFIX}_(\w+?)(?:_score|_result)?\.jsonl"
     match = re.search(pattern, input_string)
 
     # Check if there's a match and extract the captured group
@@ -28,10 +30,14 @@ def extract_test_category(input_string: Union[str, Path]) -> str:
 
 
 def find_file_with_suffix(folder_path: Path, suffix: str) -> Path:
-    for json_file in folder_path.glob("*.json"):
-        if extract_test_category(json_file) == suffix:
-            return json_file
-    raise FileNotFoundError(f"No JSON file found with suffix: {suffix}")
+    # for json_file in folder_path.glob("*.json"):
+    #     if extract_test_category(json_file) == suffix:
+    #         return json_file
+    # raise FileNotFoundError(f"No JSON file found with suffix: {suffix}")
+    for jsonl_file in folder_path.glob("*.jsonl"):
+        if extract_test_category(jsonl_file) == suffix:
+            return jsonl_file
+    raise FileNotFoundError(f"No JSONL file found with suffix: {suffix}")
 
 def is_multi_turn(test_category):
     return "multi_turn" in test_category
@@ -69,10 +75,14 @@ def is_sql(test_category):
 
 def load_file(file_path):
     result = []
-    with open(file_path) as f:
-        file = f.readlines()
-        for line in file:
-            result.append(json.loads(line))
+    # with open(file_path) as f:
+    #     file = f.readlines()
+    #     for line in file:
+    #         result.append(json.loads(line))
+    with jsonlines.open(file_path) as reader:
+        for obj in reader:
+            result.append(obj)
+        reader.close()
     return result
 
 
@@ -89,15 +99,20 @@ def write_list_of_dicts_to_file(filename, data, subdir=None):
         filename = os.path.join(subdir, filename)
 
     # Write the list of dictionaries to the file in JSON format
-    with open(filename, "w") as f:
-        for i, entry in enumerate(data):
-            # Go through each key-value pair in the dictionary to make sure the values are JSON serializable
+    # with open(filename, "w") as f:
+    #     for i, entry in enumerate(data):
+    #         # Go through each key-value pair in the dictionary to make sure the values are JSON serializable
+    #         entry = make_json_serializable(entry)
+    #         json_str = json.dumps(entry)
+    #         f.write(json_str)
+    #         if i < len(data) - 1:
+    #             f.write("\n")
+    with jsonlines.open(filename) as writer:
+        for entry in data:
             entry = make_json_serializable(entry)
             json_str = json.dumps(entry)
-            f.write(json_str)
-            if i < len(data) - 1:
-                f.write("\n")
-
+            writer.write(json_str)
+            writer.close()
 
 def make_json_serializable(value):
     if isinstance(value, dict):
@@ -730,7 +745,8 @@ def check_model_category_status(score_path):
         result_subdir = os.path.join(result_path, model_name)
         if os.path.exists(result_subdir):
             for result_file in os.listdir(result_subdir):
-                if result_file.endswith('.json'):
+                # if result_file.endswith('.json'):
+                if result_file.endswith('.jsonl'):
                     test_category = extract_test_category(result_file)
                     if test_category in category_status[model_name]:
                         category_status[model_name][test_category]["generated"] = True
@@ -820,11 +836,16 @@ def update_leaderboard_table_with_score_file(leaderboard_table, score_path: Path
     # Traverse each subdirectory
     for subdir in subdirs:
         model_name = subdir.relative_to(score_path).name
-        # Find and process all JSON files in the subdirectory
-        for model_score_json in subdir.glob("*.json"):
-            metadata = load_file(model_score_json)[0]
+        # # Find and process all JSON files in the subdirectory
+        # for model_score_json in subdir.glob("*.json"):
+        #     metadata = load_file(model_score_json)[0]
+        #     accuracy, total_count = metadata["accuracy"], metadata["total_count"]
+        #     test_category = extract_test_category(model_score_json)
+        # Find and process all JSONL files in the subdirectory
+        for model_score_jsonl in subdir.glob("*.jsonl"):
+            metadata = load_file(model_score_jsonl)[0]
             accuracy, total_count = metadata["accuracy"], metadata["total_count"]
-            test_category = extract_test_category(model_score_json)
+            test_category = extract_test_category(model_score_jsonl)
             if model_name not in leaderboard_table:
                 leaderboard_table[model_name] = {}
             if test_category not in leaderboard_table[model_name]:
@@ -834,10 +855,15 @@ def update_leaderboard_table_with_score_file(leaderboard_table, score_path: Path
                 }
 
 
-def collapse_json_objects(file_path):
-    with open(file_path, "r") as file:
-        content = file.read()
+# def collapse_json_objects(file_path):
+    # with open(file_path, "r") as file:
+    #     content = file.read()
 
+def collapse_jsonl_objects(file_path):
+    with jsonlines.open(file_path) as reader:
+        content = [obj for obj in reader]
+        reader.close()
+
     objects = []
     depth = 0
     obj_start = 0
@@ -852,8 +878,15 @@ def collapse_json_objects(file_path):
                 obj = content[obj_start : i + 1]
                 objects.append(obj)
 
-    with open(file_path, "w") as out_file:
+    # with open(file_path, "w") as out_file:
+    #     for obj in objects:
+    #         json_obj = json.loads(obj)
+    #         compact_json = json.dumps(json_obj, separators=(",", ":"))
+    #         out_file.write(compact_json + "\n")
+
+    with jsonlines.open(file_path) as writer:
         for obj in objects:
             json_obj = json.loads(obj)
             compact_json = json.dumps(json_obj, separators=(",", ":"))
-            out_file.write(compact_json + "\n")
+            writer.write(compact_json)
+            writer.close()

berkeley-function-call-leaderboard/bfcl/eval_checker/executable_eval/executable_checker.py

@@ -1,6 +1,7 @@
 import json
 import time
 from functools import lru_cache
+# import jsonlines
 
 import requests  # Do not remove this import even though it seems to be unused. It's used in the executable_checker_rest function.
 from bfcl.eval_checker.constant import (
@@ -14,6 +15,8 @@
 def load_eval_ground_truth():
     with open(REST_EVAL_GROUND_TRUTH_PATH, "r") as f:
         return f.readlines()
+    # with jsonlines.open(REST_EVAL_GROUND_TRUTH_PATH) as reader:
+    #     return reader.readlines()
 
 #### Main function ####
 def executable_checker_rest(func_call, idx):

berkeley-function-call-leaderboard/data/BFCL_v3_chatable.json → berkeley-function-call-leaderboard/data/BFCL_v3_chatable.jsonl

@@ -197,4 +197,4 @@
 {"question": "What is the air quality index in London 2022/08/16?", "function": ""}
 {"question": "Find the air quality index in San Diego at 12pm.", "function": ""}
 {"question": "Calculate the required water daily intake for a person with weight 70 kg.", "function": ""}
-{"question": "Find air quality index in San Jose for next three days.", "function": ""}
+{"question": "Find air quality index in San Jose for next three days.", "function": ""}

berkeley-function-call-leaderboard/data/BFCL_v3_exec_multiple.json → berkeley-function-call-leaderboard/data/BFCL_v3_exec_multiple.jsonl

@@ -47,4 +47,4 @@
 {"id": "exec_multiple_46", "question": [[{"role": "user", "content": "I'm planning to host a dinner party tonight and thought of serving some delicious dumplings and rice bowls. I need to order 101 dumplings at $0.1 each and 20 rice bowls at $10 per bowl. Can you calculate the total price for this order for me?"}]], "function": [{"name": "order_food", "description": "Orders food for a customer. Return the total price.", "parameters": {"type": "dict", "properties": {"item": {"type": "array", "items": {"type": "string"}, "description": "the name of the product."}, "quantity": {"type": "array", "items": {"type": "integer"}, "description": "the number of the product purchased."}, "price": {"type": "array", "items": {"type": "float"}, "description": "the price of the product."}}, "required": ["item", "quantity", "price"]}}, {"name": "calculate_total", "description": "Calculates the total price of an order given the quantities and prices.", "parameters": {"type": "dict", "properties": {"quantities": {"type": "array", "items": {"type": "integer"}, "description": "The quantities of each product."}, "prices": {"type": "array", "items": {"type": "float"}, "description": "The price of each product."}}, "required": ["quantities", "prices"]}}, {"name": "apply_discount", "description": "Applies a discount to the total price.", "parameters": {"type": "dict", "properties": {"total": {"type": "float", "description": "The original total price."}, "discount": {"type": "float", "description": "The discount percentage to apply."}}, "required": ["total", "discount"]}}], "execution_result_type": ["exact_match"], "ground_truth": ["order_food(item=['dumplings','rice bowl'], quantity=[101,20], price=[0.1,10])"]}
 {"id": "exec_multiple_47", "question": [[{"role": "user", "content": "I just rewatched \"Pulp Fiction,\" and I'm curious about the mastermind behind its direction. Could you find out who directed this iconic movie for me?"}]], "function": [{"name": "get_movie_director", "description": "Fetches the director of a movie from the OMDB API.", "parameters": {"type": "dict", "properties": {"movie_name": {"type": "string", "description": "The name of the movie."}}, "required": ["movie_name"]}}, {"name": "calculate_interest_rate", "description": "Calculates the interest rate for a given principal, rate, and time.", "parameters": {"type": "dict", "properties": {"principal": {"type": "float", "description": "The initial amount of money."}, "rate": {"type": "float", "description": "The interest rate per period."}, "time": {"type": "float", "description": "The time the money is invested or borrowed for."}}, "required": ["principal", "rate", "time"]}}, {"name": "convert_temperature", "description": "Converts temperature from Celsius to Fahrenheit or vice versa.", "parameters": {"type": "dict", "properties": {"temperature": {"type": "float", "description": "The temperature to convert."}, "unit_from": {"type": "string", "description": "The current unit of the temperature (Celsius or Fahrenheit)."}, "unit_to": {"type": "string", "description": "The unit to convert the temperature to (Celsius or Fahrenheit)."}}, "required": ["temperature", "unit_from", "unit_to"]}}, {"name": "generate_random_number", "description": "Generates a random number within a specified range.", "parameters": {"type": "dict", "properties": {"min": {"type": "integer", "description": "The minimum value of the range."}, "max": {"type": "integer", "description": "The maximum value of the range."}}, "required": ["min", "max"]}}], "execution_result_type": ["exact_match"], "ground_truth": ["get_movie_director(movie_name='Pulp Fiction')"]}
 {"id": "exec_multiple_48", "question": [[{"role": "user", "content": "I'm planning a movie night for my family this weekend, and I want to make sure the film is appropriate for all ages. We've settled on the idea of watching \"Avatar\", but I need to confirm its age rating before we proceed. Could you find out the age rating for the movie \"Avatar\"?"}]], "function": [{"name": "get_movie_rating", "description": "Fetches the age rating of a movie from the OMDB API.", "parameters": {"type": "dict", "properties": {"movie_name": {"type": "string", "description": "The name of the movie."}}, "required": ["movie_name"]}}, {"name": "get_movie_genre", "description": "Retrieves the genre of a movie from the OMDB API.", "parameters": {"type": "dict", "properties": {"movie_name": {"type": "string", "description": "The name of the movie to retrieve the genre for."}}, "required": ["movie_name"]}}, {"name": "get_director_by_movie_name", "description": "Gets the director of a movie.", "parameters": {"type": "dict", "properties": {"movie_name": {"type": "string", "description": "The movie to find the director of."}}, "required": ["movie_name"]}}], "execution_result_type": ["exact_match"], "ground_truth": ["get_movie_rating(movie_name='Avatar')"]}
-{"id": "exec_multiple_49", "question": [[{"role": "user", "content": "I have a set of vertices: [[1,2],[3,4],[1,4],[3,7]], and I'm curious about the area that these points, when connected in order, would enclose to form a polygon. Could you calculate the area of this polygon for me?"}]], "function": [{"name": "convert_coordinates", "description": "Converts a list of tuples into a list of lists.", "parameters": {"type": "dict", "properties": {"coordinates": {"type": "array", "items": {"type": "tuple", "items": {"type": "float"}, "description": "A single coordinate represented by a tuple (x, y)."}, "description": "The coordinates to be converted, where each coordinate is a tuple (x, y)."}}, "required": ["coordinates"]}}, {"name": "polygon_area", "description": "Calculate the area of a polygon given its vertices using the shoelace formula.", "parameters": {"type": "dict", "properties": {"vertices": {"type": "array", "items": {"type": "array", "items": {"type": "float"}, "minItems": 2, "maxItems": 2, "description": "A single vertex represented by a 2 element list [x, y]."}, "description": "The vertices of the polygon, where each vertex is a 2 element list [x, y]."}}, "required": ["vertices"]}}, {"name": "validate_polygon", "description": "Checks if the given vertices form a valid polygon.", "parameters": {"type": "dict", "properties": {"vertices": {"type": "array", "items": {"type": "array", "items": {"type": "float"}, "description": "A single vertex represented by a 2 element list [x, y]."}, "description": "The vertices of the polygon, where each vertex is a 2 element list [x, y]."}}, "required": ["vertices"]}}], "execution_result_type": ["exact_match"], "ground_truth": ["polygon_area(vertices=[[1,2],[3,4],[1,4],[3,7]])"]}
+{"id": "exec_multiple_49", "question": [[{"role": "user", "content": "I have a set of vertices: [[1,2],[3,4],[1,4],[3,7]], and I'm curious about the area that these points, when connected in order, would enclose to form a polygon. Could you calculate the area of this polygon for me?"}]], "function": [{"name": "convert_coordinates", "description": "Converts a list of tuples into a list of lists.", "parameters": {"type": "dict", "properties": {"coordinates": {"type": "array", "items": {"type": "tuple", "items": {"type": "float"}, "description": "A single coordinate represented by a tuple (x, y)."}, "description": "The coordinates to be converted, where each coordinate is a tuple (x, y)."}}, "required": ["coordinates"]}}, {"name": "polygon_area", "description": "Calculate the area of a polygon given its vertices using the shoelace formula.", "parameters": {"type": "dict", "properties": {"vertices": {"type": "array", "items": {"type": "array", "items": {"type": "float"}, "minItems": 2, "maxItems": 2, "description": "A single vertex represented by a 2 element list [x, y]."}, "description": "The vertices of the polygon, where each vertex is a 2 element list [x, y]."}}, "required": ["vertices"]}}, {"name": "validate_polygon", "description": "Checks if the given vertices form a valid polygon.", "parameters": {"type": "dict", "properties": {"vertices": {"type": "array", "items": {"type": "array", "items": {"type": "float"}, "description": "A single vertex represented by a 2 element list [x, y]."}, "description": "The vertices of the polygon, where each vertex is a 2 element list [x, y]."}}, "required": ["vertices"]}}], "execution_result_type": ["exact_match"], "ground_truth": ["polygon_area(vertices=[[1,2],[3,4],[1,4],[3,7]])"]}