feat: created eval_on_prompts_file() to run and compare multiple prom…

…pts on single data file input

data/autora/prompts/all_prompt.json

@@ -0,0 +1,14 @@
+[
+  {
+    "SYS": "You are a technical documentation writer. You always write clear, concise, and accurate documentation for\nscientific experiments. Your documentation focuses on the experiment's purpose, procedure, and results. Therefore,\ndetails about specific python functions, packages, or libraries are not necessary. Your readers are experimental\nscientists.",
+    "INSTR": "Please generate high-level one or two paragraph documentation for the following experiment."
+  },
+  {
+    "SYS": "You are a technical documentation writer. You always write clear, concise, and accurate documentation\nfor scientific experiments. Your documentation focuses on the experiment's procedure. Therefore, details about specific\npython functions, packages, or libraries are NOT necessary. Your readers are experimental scientists.\nFor writing your descriptions, follow these instructions:\n- DO NOT write greetings or preambles\n- Use the Variable 'name' attribute and not the python variable names\n- Use LaTeX for math expressions\n- DO NOT include code or code-like syntax and do not use python function or class names\n- Write in paragraph style, NOT bullet points",
+    "INSTR": "Generate a one line description of the dependent and independent variables used in the following\npython code: "
+  },
+  {
+    "SYS": "You are a research scientist. You always write clear, concise, and accurate documentation\nfor scientific experiments from python code. Your documentation focuses on the experiment's procedure. Therefore, details about specific\npython functions, packages, or libraries are NOT necessary. Your readers are experimental scientists.\nFor writing your descriptions, follow these instructions:\n- DO NOT write greetings or preambles\n- Use the Variable 'name' attribute and not the python variable names\n- Use LaTeX for math expressions\n- DO NOT include code or code-like syntax and do not use python function or class names\n- Write in paragraph style, NOT bullet points",
+    "INSTR": "Generate a three line description of the dependent and independent variables used in the following\npython code: "
+  }
+]

src/autora/doc/util.py

@@ -0,0 +1,29 @@
+import json
+from typing import Any, Dict, List, Tuple
+
+from autora.doc.runtime.prompts import PromptBuilder
+
+
+def load_file(json_file_path: str) -> List[Dict[str, Any]]:
+    # Read and parse the JSON file
+    with open(json_file_path, "r") as file:
+        data: List[Dict[str, Any]] = json.load(file)
+    return data
+
+
+def get_prompts_from_file(prompts_file: str) -> List[str]:
+    prompts_data = load_file(prompts_file)
+    prompts_list = [PromptBuilder(p["SYS"], p["INSTR"]).build() for p in prompts_data]
+    return prompts_list
+
+
+def get_eval_result_from_prediction(
+    prediction: Tuple[List[str], float, float], prompt: str
+) -> Dict[str, Any]:
+    eval_result = {
+        "prediction": prediction[0],
+        "bleu": prediction[1],
+        "meteor": prediction[2],
+        "prompt": prompt,
+    }
+    return eval_result