From 496f355d158e2a51d5d7658bc905c496ee3b8b0a Mon Sep 17 00:00:00 2001
From: Li Yin <li.yin.gravity@gmail.com>
Date: Fri, 23 Aug 2024 09:28:17 -0700
Subject: [PATCH 1/8] clean up code and comments

---
 README.md                                     |   5 +-
 adalflow/CHANGELOG.md                         |   3 +
 adalflow/adalflow/__init__.py                 |   6 +-
 .../output_parsers/dataclass_parser.py        |   1 -
 adalflow/adalflow/core/generator.py           |  15 +-
 adalflow/adalflow/optim/trainer/trainer.py    |  21 --
 docs/source/index.rst                         |  42 ++--
 docs/source/use_cases/question_answering.rst  |   2 +-
 use_cases/classification/train.py             |   8 +-
 .../classification/trec_task_string_output.py |   3 +-
 .../trec_task_structured_output.py            |   2 +-
 use_cases/classification/visualize.py         | 131 ++++++++---
 .../bhh_object_count/prepare_trainer.py       | 129 -----------
 .../bhh_object_count/task.py                  | 211 +++++++++---------
 .../bhh_object_count/train.py                 | 132 -----------
 .../bhh_object_count/train_new.py             |  16 +-
 16 files changed, 265 insertions(+), 462 deletions(-)
 delete mode 100644 use_cases/question_answering/bhh_object_count/prepare_trainer.py
 delete mode 100644 use_cases/question_answering/bhh_object_count/train.py

diff --git a/README.md b/README.md
index 0d9174f4..e206e61f 100644
--- a/README.md
+++ b/README.md
@@ -122,8 +122,7 @@ Because of this, no library can provide out-of-the-box solutions. Users must bui
 <!-- This is what AdalFlow is: light, modular, and robust, with a 100% readable codebase. -->
 
 
-Further reading: [How We Started](https://www.linkedin.com/posts/li-yin-ai_both-ai-research-and-engineering-use-pytorch-activity-7189366364694892544-Uk1U?utm_source=share&utm_medium=member_desktop),
-[Introduction](https://adalflow.sylph.ai/), [Design Philosophy](https://adalflow.sylph.ai/tutorials/lightrag_design_philosophy.html) and [Class hierarchy](https://adalflow.sylph.ai/tutorials/class_hierarchy.html).
+Further reading: [How We Started](https://www.linkedin.com/posts/li-yin-ai_both-ai-research-and-engineering-use-pytorch-activity-7189366364694892544-Uk1U?utm_source=share&utm_medium=member_desktop), <!-- [Introduction](https://adalflow.sylph.ai/),  -->[Design Philosophy](https://adalflow.sylph.ai/tutorials/lightrag_design_philosophy.html) and [Class hierarchy](https://adalflow.sylph.ai/tutorials/class_hierarchy.html).
 
 
 <!--
@@ -171,7 +170,7 @@ Just define it as a ``Parameter`` and pass it to our ``Generator``.
 
 ### **AdalComponent & Trainer**
 
-``AdalComponent`` acts as the `interpreter`  between task pipeline and the trainer, defining training and validation steps, optimizers, evaluators, loss functions, backward engine for textual gradients or tracing the demonstrations, the teacher generator.
+``AdalComponent`` acts as the 'interpreter'  between task pipeline and the trainer, defining training and validation steps, optimizers, evaluators, loss functions, backward engine for textual gradients or tracing the demonstrations, the teacher generator.
 
 <p align="center">
   <img src="https://raw.githubusercontent.com/SylphAI-Inc/LightRAG/main/docs/source/_static/images/trainer.png" alt="AdalFlow AdalComponent & Trainer">
diff --git a/adalflow/CHANGELOG.md b/adalflow/CHANGELOG.md
index 4d34b8b5..e509f624 100644
--- a/adalflow/CHANGELOG.md
+++ b/adalflow/CHANGELOG.md
@@ -1,3 +1,6 @@
+## [0.2.1] - 2024-09-01
+### Added
+- `get_cache_path`, instead of print out the cache path all the time, we add a ``get_cache_path`` to get the cache path.
 ## [0.2.0] - 2024-08-20
 ### Added
 - Qdrant retriever.
diff --git a/adalflow/adalflow/__init__.py b/adalflow/adalflow/__init__.py
index 3b19ff29..0ad0dbe6 100644
--- a/adalflow/adalflow/__init__.py
+++ b/adalflow/adalflow/__init__.py
@@ -2,7 +2,7 @@
 
 from adalflow.core.component import Component, fun_to_component
 from adalflow.core.container import Sequential
-from adalflow.core.base_data_class import DataClass
+from adalflow.core.base_data_class import DataClass, DataClassFormatType, required_field
 
 from adalflow.optim.grad_component import GradComponent
 from adalflow.core.generator import Generator
@@ -55,7 +55,11 @@
 __all__ = [
     "Component",
     "fun_to_component",
+    # dataclass
     "DataClass",
+    "DataClassFormatType",
+    "required_field",
+    # Container
     "Sequential",
     "GradComponent",
     "ModelClient",
diff --git a/adalflow/adalflow/components/output_parsers/dataclass_parser.py b/adalflow/adalflow/components/output_parsers/dataclass_parser.py
index 76a76e93..057f97d9 100644
--- a/adalflow/adalflow/components/output_parsers/dataclass_parser.py
+++ b/adalflow/adalflow/components/output_parsers/dataclass_parser.py
@@ -90,7 +90,6 @@ def get_output_format_str(self) -> str:
         else:
             schema = self._data_class.to_json_signature(include=self._output_fields)
             output_format_str = Prompt(template=JSON_OUTPUT_FORMAT)(schema=schema)
-        print(f"output_format_str: {output_format_str}")
         return output_format_str
 
     def get_input_str(self, input: DataClass) -> str:
diff --git a/adalflow/adalflow/core/generator.py b/adalflow/adalflow/core/generator.py
index 1f32b5ab..58c03244 100644
--- a/adalflow/adalflow/core/generator.py
+++ b/adalflow/adalflow/core/generator.py
@@ -125,8 +125,6 @@ def __init__(
         )
         self.cache_path = os.path.join(_cache_path, f"cache_{model_str}.db")
 
-        print(f"cache_path: {self.cache_path}")
-
         CachedEngine.__init__(self, cache_path=self.cache_path)
         Component.__init__(self)
         GradComponent.__init__(self)
@@ -167,6 +165,10 @@ def __init__(
         }
         self._teacher: Optional["Generator"] = None
 
+    def get_cache_path(self) -> str:
+        r"""Get the cache path for the generator."""
+        return self.cache_path
+
     @staticmethod
     def _get_default_mapping(
         output: "GeneratorOutput" = None,
@@ -269,11 +271,9 @@ def _compose_model_kwargs(self, **model_kwargs) -> Dict:
         return combined_model_kwargs
 
     def print_prompt(self, **kwargs) -> str:
-        # prompt_kwargs_str = _convert_prompt_kwargs_to_str(kwargs)
         return self.prompt.print_prompt(**kwargs)
 
     def get_prompt(self, **kwargs) -> str:
-        # prompt_kwargs_str = _convert_prompt_kwargs_to_str(kwargs)
         return self.prompt.call(**kwargs)
 
     def _extra_repr(self) -> str:
@@ -706,7 +706,6 @@ def _run_callbacks(
             model_kwargs=model_kwargs,
         )
         if output.error:
-            print(f"call back on failure: {output}")
             self.trigger_callbacks(
                 "on_failure",
                 output=output,
@@ -949,7 +948,6 @@ def create_teacher_generator(
     call_logger = GeneratorCallLogger(save_dir="traces")
 
     def on_complete(output, input, prompt_kwargs, model_kwargs, logger_call: Callable):
-        print(f"on_complet  output: {output}")
         logger_call(
             output=output,
             input=input,
@@ -958,13 +956,9 @@ def on_complete(output, input, prompt_kwargs, model_kwargs, logger_call: Callabl
         )
 
     for model in [llama3_model, gpt_3_model, gemini_model, claude_model]:
-        print(f"""model: {model["model_kwargs"]["model"]}""")
         generator = Generator(**model)
 
-        print("_kwargs: ", generator._kwargs)
-
         teacher = create_teacher_generator(generator, **claude_model)
-        print(f"teacher: {teacher}")
 
         call_logger.register_generator("generator", "generator_call")
         # setup the callback
@@ -978,7 +972,6 @@ def on_complete(output, input, prompt_kwargs, model_kwargs, logger_call: Callabl
                 "input_str": "Hello, world!",
             }
         )
-        print(f"output: {output}")
         break
 
     # test the backward engine
diff --git a/adalflow/adalflow/optim/trainer/trainer.py b/adalflow/adalflow/optim/trainer/trainer.py
index 2fcd6820..cfeed983 100644
--- a/adalflow/adalflow/optim/trainer/trainer.py
+++ b/adalflow/adalflow/optim/trainer/trainer.py
@@ -1471,37 +1471,16 @@ def _downsample_move_batch(
             if not isinstance(loss, Parameter):
                 raise ValueError("Loss should be a Parameter object")
         max_moving_batch_size = 20
-        # max_sampled_correct_size = 3
-        # max_sampled_error_size = 3
 
         correct_indices = [i for i, score in enumerate(acc_score_list) if score > 0.5]
         error_indices = [i for i, score in enumerate(acc_score_list) if score <= 0.5]
 
-        print(f"Moving batch correct size: {len(correct_indices)}")
-        print(f"Moving batch error size: {len(error_indices)}")
         if (
             len(error_indices) + len(correct_indices)
             <= max_moving_batch_size
             # and len(correct_indices) <= max_moving_batch_size
         ):
             return all_samples, all_losses, all_y_preds, acc_score_list
-        # sampled_error_indices = error_indices
-        # if len(correct_indices) > max_moving_batch_size:
-        #     sampled_error_indices = random.sample(
-        #         error_indices, min(max_moving_batch_size, len(error_indices))
-        #     )
-        # sampled_correct_indices = correct_indices
-        # if len(correct_indices) > max_moving_batch_size:
-        #     sampled_correct_indices = random.sample(
-        #         correct_indices,
-        #         min(
-        #             max_moving_batch_size,
-        #             len(correct_indices),
-        #         ),
-        #     )
-        # print(f"New moving batch size size: {len(sampled_error_indices)}")
-        # print(f"Subset Correct size: {len(sampled_correct_indices)}")
-        # new_sample_indices = sampled_error_indices + sampled_correct_indices
 
         # downsample from all samples
         new_sample_indices = random.sample(
diff --git a/docs/source/index.rst b/docs/source/index.rst
index 4c0a002d..0ccb2469 100644
--- a/docs/source/index.rst
+++ b/docs/source/index.rst
@@ -6,27 +6,27 @@
 
 
 
-.. .. raw:: html
-
-..    <p align="center">
-
-
-..     <a href="https://colab.research.google.com/drive/1TKw_JHE42Z_AWo8UuRYZCO2iuMgyslTZ?usp=sharing">
-..         <img alt="Try Quickstart in Colab" src="https://colab.research.google.com/assets/colab-badge.svg">
-..     </a>
-..    </p>
-
-..    <div style="text-align: center; margin-bottom: 20px;">
-..       <a href="https://pypi.org/project/adalflow/"><img src="https://img.shields.io/pypi/v/adalflow?style=flat-square" alt="PyPI Version"></a>
-..       <a href="https://github.com/SylphAI-Inc/AdalFlow" style="display: inline-block; margin-left: 10px;">
-..          <img src="https://img.shields.io/badge/GitHub-AdalFlow-blue?logo=github&style=flat-square" alt="GitHub Repo">
-..       </a>
-..       <a href="https://star-history.com/#SylphAI-Inc/LightRAG"><img src="https://img.shields.io/github/stars/SylphAI-Inc/LightRAG?style=flat-square" alt="GitHub Stars"></a>
-..       <a href="https://discord.gg/ezzszrRZvT">
-..         <img alt="discord-invite" src="https://dcbadge.vercel.app/api/server/ezzszrRZvT?style=flat">
-..       </a>
-..       <a href="https://opensource.org/license/MIT"><img src="https://img.shields.io/github/license/SylphAI-Inc/LightRAG" alt="License"></a>
-..    </div>
+.. raw:: html
+
+   <p align="center">
+
+
+    <a href="https://colab.research.google.com/drive/1TKw_JHE42Z_AWo8UuRYZCO2iuMgyslTZ?usp=sharing">
+        <img alt="Try Quickstart in Colab" src="https://colab.research.google.com/assets/colab-badge.svg">
+    </a>
+   </p>
+
+   <div style="text-align: center; margin-bottom: 20px;">
+      <a href="https://pypi.org/project/adalflow/"><img src="https://img.shields.io/pypi/v/adalflow?style=flat-square" alt="PyPI Version"></a>
+      <a href="https://github.com/SylphAI-Inc/AdalFlow" style="display: inline-block; margin-left: 10px;">
+         <img src="https://img.shields.io/badge/GitHub-AdalFlow-blue?logo=github&style=flat-square" alt="GitHub Repo">
+      </a>
+      <a href="https://star-history.com/#SylphAI-Inc/LightRAG"><img src="https://img.shields.io/github/stars/SylphAI-Inc/LightRAG?style=flat-square" alt="GitHub Stars"></a>
+      <a href="https://discord.gg/ezzszrRZvT">
+        <img alt="discord-invite" src="https://dcbadge.vercel.app/api/server/ezzszrRZvT?style=flat">
+      </a>
+      <a href="https://opensource.org/license/MIT"><img src="https://img.shields.io/github/license/SylphAI-Inc/LightRAG" alt="License"></a>
+   </div>
 
 
 
diff --git a/docs/source/use_cases/question_answering.rst b/docs/source/use_cases/question_answering.rst
index fe7e7b82..430b19cd 100644
--- a/docs/source/use_cases/question_answering.rst
+++ b/docs/source/use_cases/question_answering.rst
@@ -546,7 +546,7 @@ Now, we can use the trainer to train the model.
     ):
         adal_component = ObjectCountAdalComponent(
             **gpt_3_model,
-            teacher_model_config=gpt_3_model,
+            teacher_model_config=gpt_4o_model,
             text_optimizer_model_config=gpt_4o_model,
             backward_engine_model_config=gpt_4o_model
         )
diff --git a/use_cases/classification/train.py b/use_cases/classification/train.py
index 6b45e330..5b34d125 100644
--- a/use_cases/classification/train.py
+++ b/use_cases/classification/train.py
@@ -85,8 +85,8 @@ def train(
     model_client: adal.ModelClient,
     model_kwargs: Dict,
     train_batch_size=4,  # larger batch size is not that effective, probably because of llm's lost in the middle
-    raw_shots: int = 0,
-    bootstrap_shots: int = 1,
+    raw_shots: int = 36,
+    bootstrap_shots: int = 4,
     max_steps=1,
     num_workers=4,
     strategy="constrained",
@@ -113,7 +113,7 @@ def train(
         debug=debug,
         weighted_sampling=True,
         optimization_order=optimization_order,
-        exclude_input_fields_from_bootstrap_demos=True,
+        exclude_input_fields_from_bootstrap_demos=False,
     )
     print(trainer)
 
@@ -124,7 +124,7 @@ def train(
         # val_dataset=val_dataset,
         # test_dataset=test_dataset,
         debug=debug,
-        # resume_from_ckpt="/Users/liyin/.adalflow/ckpt/TrecClassifierAdal/constrained_max_steps_1_33c1a_run_1.json",
+        resume_from_ckpt="/Users/liyin/.adalflow/ckpt/TrecClassifierAdal/constrained_max_steps_12_5d1bf_run_1.json",
     )
 
 
diff --git a/use_cases/classification/trec_task_string_output.py b/use_cases/classification/trec_task_string_output.py
index 536874cc..98a42f91 100644
--- a/use_cases/classification/trec_task_string_output.py
+++ b/use_cases/classification/trec_task_string_output.py
@@ -3,7 +3,8 @@
 import adalflow as adal
 import re
 from adalflow.datasets.trec import _COARSE_LABELS_DESC, _COARSE_LABELS
-from use_cases.classification.trec_task import task_desc_template
+
+from use_cases.classification.trec_task_structured_output import task_desc_template
 
 template = r"""<START_OF_SYSTEM_MESSAGE>
 {{system_prompt}}
diff --git a/use_cases/classification/trec_task_structured_output.py b/use_cases/classification/trec_task_structured_output.py
index 71341e01..17066ebf 100644
--- a/use_cases/classification/trec_task_structured_output.py
+++ b/use_cases/classification/trec_task_structured_output.py
@@ -68,7 +68,7 @@ def __init__(self, model_client: adal.ModelClient, model_kwargs: Dict):
                 # data="You are a classifier. Given a question, classify it into one of the following classes based on what the question is seeking:\n\nFormat: class_index. class_name, class_description\n\n0. ABBR, Abbreviation\n1. ENTY, Entity\n2. DESC, Description and abstract concept\n3. HUM, Human being\n4. LOC, Location\n5. NUM, Numeric value\n\nPay special attention to questions about entities versus descriptions, as well as those asking for specific terms or people. Do not try to answer the question:",
                 # best  # data="You are a classifier. For each question given, classify it into one of the following classes:\n\nFormat: class_index. class_name, class_description\n\n0. ABBR, Abbreviation (includes initials)\n1. ENTY, Entity (includes products, languages, objects, etc.)\n2. DESC, Description and abstract concept (includes explanations)\n3. HUM, Human being (includes individuals, groups, etc.)\n4. LOC, Location (includes addresses, places, etc.)\n5. NUM, Numeric value (includes distances, dates, ages, etc.)\n\n- Focus on identifying the primary subject of the question and classifying based on what is being explicitly asked for.",
                 role_desc="Task description",
-                requires_opt=True,
+                requires_opt=False,
                 param_type=adal.ParameterType.PROMPT,
             ),
             "output_format_str": adal.Parameter(
diff --git a/use_cases/classification/visualize.py b/use_cases/classification/visualize.py
index ce3c94c9..016b3c37 100644
--- a/use_cases/classification/visualize.py
+++ b/use_cases/classification/visualize.py
@@ -26,31 +26,110 @@
     0.8819444444444444,
     0.8819444444444444,
 ]
+
+# constrained_max_steps_12_7fb88_run_1.json
+demo_only = [
+    0.8263888888888888,
+    0.8333333333333334,
+    0.8472222222222222,
+    0.8472222222222222,
+    0.8472222222222222,
+    0.8472222222222222,
+    0.8472222222222222,
+    0.8472222222222222,
+    0.8472222222222222,
+    0.8472222222222222,
+    0.8472222222222222,
+    0.8472222222222222,
+    0.8472222222222222,
+]
+# TrecClassifierAdal/constrained_max_steps_12_5d1bf_run_1.json
+text_only = [
+    0.8263888888888888,
+    0.8263888888888888,
+    0.8333333333333334,
+    0.8333333333333334,
+    0.8333333333333334,
+    0.8402777777777778,
+    0.8402777777777778,
+    0.8402777777777778,
+    0.8402777777777778,
+    0.8888888888888888,
+    0.8888888888888888,
+    0.8888888888888888,
+    0.8888888888888888,
+]
+
 import matplotlib.pyplot as plt
 
-methods = ["text_optimizer(Text-Grad 2.0)"] * 12 + [
-    "demo_optimizer (Learn-to-Reason)"
-] * 12
-plt.figure(figsize=(10, 6))
-plt.plot(
-    range(1, 13),
-    test_score_combo[:12],
-    marker="o",
-    label="text_optimizer(Text-Grad 2.0)",
-)
-plt.plot(
-    range(13, 25),
-    test_score_combo[12:24],
-    marker="o",
-    label="demo_optimizer(Learn-to-Reason)",
-)
-
-plt.axvline(x=12.5, color="gray", linestyle="--")  # Divider between methods
-
-plt.xlabel("Steps")
-plt.ylabel("Test Accuracy")
-plt.title("Test Accuracy Using Seqential Optimization on TREC-6 Classification")
-plt.legend()
-plt.grid(True)
-
-plt.show()
+
+def plot_sequential_traing(test_score_combo):
+    steps = 12
+    # methods = ["text_optimizer(Text-Grad 2.0)"] * steps + [
+    #     "demo_optimizer (Learn-to-Reason)"
+    # ] * steps
+    plt.figure(figsize=(10, 6))
+    plt.plot(
+        range(1, 13),
+        test_score_combo[:12],
+        marker="o",
+        label="text_optimizer(Text-Grad 2.0)",
+    )
+    plt.plot(
+        range(13, 25),
+        test_score_combo[steps : 2 * steps],
+        marker="o",
+        label="demo_optimizer(Learn-to-Reason)",
+    )
+
+    plt.axvline(x=12.5, color="gray", linestyle="--")  # Divider between methods
+
+    plt.xlabel("Steps")
+    plt.ylabel("Test Accuracy")
+    plt.title("Test Accuracy Using Seqential Optimization on TREC-6 Classification")
+    plt.legend()
+    plt.grid(True)
+
+    plt.show()
+
+
+def plot_demo_optimizer_only(demo_only):
+    plt.figure(figsize=(10, 6))
+    plt.plot(
+        range(1, len(demo_only) + 1),
+        demo_only,
+        marker="o",
+        label="demo_optimizer(Learn-to-Reason)",
+    )
+
+    plt.xlabel("Steps")
+    plt.ylabel("Test Accuracy")
+    plt.title("Test Accuracy Using Only Learn-to-Reason on TREC-6 Classification")
+    plt.legend()
+    plt.grid(True)
+
+    plt.show()
+
+
+def plot_text_optimizer_only(text_only):
+    plt.figure(figsize=(10, 6))
+    plt.plot(
+        range(1, len(text_only) + 1),
+        text_only,
+        marker="o",
+        label="text_optimizer(Text-Grad 2.0)",
+    )
+
+    plt.xlabel("Steps")
+    plt.ylabel("Test Accuracy")
+    plt.title("Test Accuracy Using Only Text-Grad 2.0 on TREC-6 Classification")
+    plt.legend()
+    plt.grid(True)
+
+    plt.show()
+
+
+if __name__ == "__main__":
+    # plot_sequential_traing(test_score_combo)
+    # plot_demo_optimizer_only(demo_only)  # Training time: 95.81865906715393s
+    plot_text_optimizer_only(text_only)  # Training time: 95.81865906715393s
diff --git a/use_cases/question_answering/bhh_object_count/prepare_trainer.py b/use_cases/question_answering/bhh_object_count/prepare_trainer.py
deleted file mode 100644
index b514524a..00000000
--- a/use_cases/question_answering/bhh_object_count/prepare_trainer.py
+++ /dev/null
@@ -1,129 +0,0 @@
-# from typing import Dict, Tuple, Callable, Any, List, TYPE_CHECKING
-
-# if TYPE_CHECKING:
-#     from adalflow.core.component import Component
-
-# from adalflow.optim.parameter import Parameter
-
-# from adalflow.optim.trainer.adal import AdalComponent
-# from adalflow.optim.text_grad.tgd_optimer import TGDOptimizer
-# from adalflow.core.generator import BackwardEngine
-# from adalflow.eval.answer_match_acc import AnswerMatchAcc
-# from adalflow.eval.base import EvaluationResult
-# from adalflow.optim.text_grad.text_loss_with_eval_fn import EvalFnToTextLoss
-# from adalflow.optim.few_shot.bootstrap_optimizer import BootstrapFewShot
-# from adalflow.optim.types import ObjectCountData
-
-
-# from use_cases.question_answering.bhh_object_count.task import ObjectCountTaskOriginal
-
-
-# class TGDWithEvalFnLoss(AdalComponent):
-#     def __init__(
-#         self,
-#         task: "Component",  # TODO: better check the type to be a class not an instance
-#         task_model_config: Dict,  # for task pipeline
-#         backward_engine_model_config: Dict,  # for computing gradients
-#         optimizer_model_config: Dict,  # for proposal
-#     ):
-#         super().__init__()
-
-#         self.task_model_config = task_model_config
-#         self.backward_engine_model_config = backward_engine_model_config
-#         self.optimizer_model_config = optimizer_model_config
-#         self.teacher_model_config = backward_engine_model_config
-
-#         self.backward_engine = BackwardEngine(
-#             **backward_engine_model_config, use_cache=True
-#         )
-#         if not task:
-#             self.task = ObjectCountTaskOriginal(**task_model_config)
-#         else:
-#             self.task = task(**task_model_config)
-#         self.evaluator = AnswerMatchAcc(type="exact_match")
-#         # self.configure_backward_engine()
-
-#     def handle_one_train_sample(self, sample: ObjectCountData) -> Tuple[Callable, Dict]:
-#         return self.task.call, {"question": sample.x, "id": sample.id}
-
-#     def handle_one_loss_sample(
-#         self, sample: ObjectCountData, y_pred: Any
-#     ) -> Tuple[Callable, Dict]:
-#         return self.loss_fn, {
-#             "kwargs": {
-#                 "y": y_pred,
-#                 "y_gt": Parameter(
-#                     data=sample.y,
-#                     role_desc="The ground truth(reference correct answer)",
-#                     name="y_gt",
-#                     requires_opt=False,
-#                 ),
-#             }
-#         }
-
-#     def evaluate_one_sample(self, sample: ObjectCountData, y_pred: Any) -> Any:
-#         return self.evaluator.compute_single_item(y_pred, sample.y)
-
-#     # TODO: remove this, one should be enough
-#     def evaluate_samples(
-#         self, samples: List[ObjectCountData], y_preds: List
-#     ) -> EvaluationResult:
-#         r"""Support both batch and list of samples"""
-#         y_gts = [sample.y for sample in samples]
-#         return self.evaluator.compute(y_preds, y_gts)
-
-#     # def train_step(self, batch, batch_idx, num_workers: int = 2) -> List:
-#     #     self.task.train()
-#     #     y_preds = super().pred_step(batch, batch_idx, num_workers)
-#     #     for i, y_pred in enumerate(y_preds):
-#     #         y_pred.name += f"y_pred_{i}"
-#     #     return y_preds
-
-#     def configure_optimizers(self):
-#         use_tgd = False
-#         tgd = None  # noqa F841
-#         parameters = []
-#         for name, param in self.task.named_parameters():
-#             param.name = name
-#             parameters.append(param)
-#         # config backend engine for tgd
-
-#         if use_tgd:
-#             self.configure_backward_engine()
-#             # noqa F841
-#             tgd = TGDOptimizer(  # noqa: F841
-#                 params=parameters,
-#                 **self.optimizer_model_config,
-#                 num_gradient_memory=0,
-#             )  # noqa:F841
-#         # config = FewShotConfig(
-#         #     num_shots=5,
-#         #     raw_shots=3,
-#         #     bootstrap_shots=2,
-#         # )
-
-#         fso = BootstrapFewShot(
-#             params=parameters,
-#             # few_shot_config=config,
-#             # sampler=RandomSampler(default_num_shots=5),
-#         )
-#         return [fso]
-
-#     def configure_backward_engine(self):
-#         self.backward_engine = BackwardEngine(**self.backward_engine_model_config)
-#         # add backward engine to the generator of the task
-#         self.task.llm_counter.set_backward_engine(self.backward_engine)
-
-#     def configure_teacher_generator(self):
-#         super().configure_teacher_generator(**self.teacher_model_config)
-
-#     def configure_loss_fn(self):
-#         # share the backward engine with the generator
-#         # if self.backward_engine is None:
-#         #     self.configure_backwar_engine()
-#         self.loss_fn = EvalFnToTextLoss(
-#             eval_fn=self.evaluator.compute_single_item,
-#             eval_fn_desc="ObjectCountingEvalFn, Output accuracy score: 1 for correct, 0 for incorrect",
-#             # backward_engine=self.backward_engine,
-#             backward_engine=None,
-#         )
diff --git a/use_cases/question_answering/bhh_object_count/task.py b/use_cases/question_answering/bhh_object_count/task.py
index 37d78def..3fc9b3de 100644
--- a/use_cases/question_answering/bhh_object_count/task.py
+++ b/use_cases/question_answering/bhh_object_count/task.py
@@ -9,57 +9,56 @@
 from use_cases.question_answering.bhh_object_count.data import (
     parse_integer_answer,
     ObjectCountPredData,
-    ObjectCountSimple,
 )
 
 
-class ObjectCountTaskOriginal(Component):
-    r"""Same system prompt as text-grad paper, but with our one message prompt template, which has better starting performance"""
-
-    def __init__(self, model_client, model_kwargs):
-        super().__init__()
-
-        template = """<START_OF_SYSTEM_PROMPT>{{system_prompt}}<END_OF_SYSTEM_PROMPT>{{input_str}}"""
-        # 1. set up system prompt, and define the parameters for optimization.
-        # NOTE: Dont use self.system_prompt
-        # use self. will double the parameters, so we dont need that as we want the parameter to be part of the generator
-        system_prompt = Parameter(
-            name="task_instruction",
-            data="You will answer a reasoning question. Think step by step. The last line of your response should be of the following format: 'Answer: $VALUE' where VALUE is a numerical value.",
-            role_desc="To give task instruction to the language model in the system prompt",
-            requires_opt=True,
-            param_type=ParameterType.NONE,
-            instruction_to_optimizer="You can show some examples if you think that will help.",
-        )
-        # TODO: sometimes the cache will collide, so we get different evaluation
-        self.llm_counter = Generator(
-            model_client=model_client,
-            model_kwargs=model_kwargs,
-            template=template,
-            prompt_kwargs={
-                "system_prompt": system_prompt,
-            },
-            output_processors=parse_integer_answer,
-            use_cache=True,
-        )
-        # TODO: make this data map function more robust (this is the final answer and the input to eval_fn)
-        # self.llm_counter.set_data_map_func(lambda x: x.data.answer)
-        print(f"llm_counter set_data_map_func, {self.llm_counter.data_map_func}")
-
-    # TODO: the error will be a context
-    def call(self, question: str, id: str = None) -> Any:  # Add id for tracing
-        output = self.llm_counter(
-            prompt_kwargs={"input_str": question, "id": id}
-        )  # already support both training (forward + call)
-
-        if not self.training:  # eval
-
-            if output.data is None:
-                print(f"Error in processing the question: {question}, output: {output}")
-                output = -1
-            else:
-                output = int(output.data)
-        return output
+# class ObjectCountTaskOriginal(Component):
+#     r"""Same system prompt as text-grad paper, but with our one message prompt template, which has better starting performance"""
+
+#     def __init__(self, model_client, model_kwargs):
+#         super().__init__()
+
+#         template = """<START_OF_SYSTEM_PROMPT>{{system_prompt}}<END_OF_SYSTEM_PROMPT>{{input_str}}"""
+#         # 1. set up system prompt, and define the parameters for optimization.
+#         # NOTE: Dont use self.system_prompt
+#         # use self. will double the parameters, so we dont need that as we want the parameter to be part of the generator
+#         system_prompt = Parameter(
+#             name="task_instruction",
+#             data="You will answer a reasoning question. Think step by step. The last line of your response should be of the following format: 'Answer: $VALUE' where VALUE is a numerical value.",
+#             role_desc="To give task instruction to the language model in the system prompt",
+#             requires_opt=True,
+#             param_type=ParameterType.NONE,
+#             instruction_to_optimizer="You can show some examples if you think that will help.",
+#         )
+#         # TODO: sometimes the cache will collide, so we get different evaluation
+#         self.llm_counter = Generator(
+#             model_client=model_client,
+#             model_kwargs=model_kwargs,
+#             template=template,
+#             prompt_kwargs={
+#                 "system_prompt": system_prompt,
+#             },
+#             output_processors=parse_integer_answer,
+#             use_cache=True,
+#         )
+#         # TODO: make this data map function more robust (this is the final answer and the input to eval_fn)
+#         # self.llm_counter.set_data_map_func(lambda x: x.data.answer)
+#         print(f"llm_counter set_data_map_func, {self.llm_counter.data_map_func}")
+
+#     # TODO: the error will be a context
+#     def call(self, question: str, id: str = None) -> Any:  # Add id for tracing
+#         output = self.llm_counter(
+#             prompt_kwargs={"input_str": question, "id": id}
+#         )  # already support both training (forward + call)
+
+#         if not self.training:  # eval
+
+#             if output.data is None:
+#                 print(f"Error in processing the question: {question}, output: {output}")
+#                 output = -1
+#             else:
+#                 output = int(output.data)
+#         return output
 
 
 # Few shot demonstration can be less effective when performance already high
@@ -88,7 +87,7 @@ def __init__(self, model_client: adal.ModelClient, model_kwargs: Dict):
             # data="You will answer a reasoning question. Think step by step. The last line of your response should be of the following format: 'Answer: $VALUE' where VALUE is a numerical value.",
             data="You will answer a reasoning question. Think step by step. The last line of your response should be of the following format: 'Answer: $VALUE' where VALUE is a numerical value.",
             role_desc="To give task instruction to the language model in the system prompt",
-            requires_opt=False,
+            requires_opt=True,
             param_type=ParameterType.PROMPT,
             instruction_to_optimizer="You can try to show examples to see if it helps.",
         )
@@ -118,63 +117,63 @@ def call(
         return output
 
 
-class ObjectCountTaskFewShot(Component):
-    r"""Same system prompt as text-grad paper, but with our one message prompt template, which has better starting performance"""
-
-    def __init__(self, model_client, model_kwargs):
-        super().__init__()
-
-        # 1. set up system prompt, and define the parameters for optimization.
-        # NOTE: Dont use self.system_prompt
-        # use self. will double the parameters, so we dont need that as we want the parameter to be part of the generator
-        system_prompt = Parameter(
-            name="task_instruction",
-            data="You will answer a reasoning question. Think step by step. The last line of your response should be of the following format: 'Answer: $VALUE' where VALUE is a numerical value.",
-            role_desc="To give task instruction to the language model in the system prompt",
-            requires_opt=True,
-            param_type=ParameterType.NONE,
-            # instruction_to_optimizer="You can show some examples if you think that will help.",
-        )
-        _few_shot_demos = Parameter(
-            name="few_shot_demos",
-            data=None,
-            role_desc="To provide few shot demos to the language model",
-            requires_opt=True,
-            param_type=ParameterType.DEMOS,
-        )
-
-        self.llm_counter = Generator(
-            model_client=model_client,
-            model_kwargs=model_kwargs,
-            template=few_shot_template,
-            prompt_kwargs={
-                "system_prompt": system_prompt,
-                "few_shot_demos": _few_shot_demos,
-            },
-            output_processors=parse_integer_answer,  # transform data field
-            use_cache=True,
-            demo_data_class=ObjectCountSimple,  # for output format
-            demo_data_class_input_mapping={"question": "input_str"},
-            demo_data_class_output_mapping={"answer": lambda x: x.raw_response},
-        )
-
-    def call(self, question: str, id: str = None) -> Any:  # Add id for tracing
-        output = self.llm_counter(
-            prompt_kwargs={
-                "input_str": question,
-                # "few_shot_demos": self._few_shot_demos.data,
-            },
-            id=id,
-        )  # already support both training (forward + call)
-
-        if not self.training:  # eval
-
-            if output.data is None:
-                print(f"Error in processing the question: {question}, output: {output}")
-                output = -1
-            else:
-                output = int(output.data)
-        return output
+# class ObjectCountTaskFewShot(Component):
+#     r"""Same system prompt as text-grad paper, but with our one message prompt template, which has better starting performance"""
+
+#     def __init__(self, model_client, model_kwargs):
+#         super().__init__()
+
+#         # 1. set up system prompt, and define the parameters for optimization.
+#         # NOTE: Dont use self.system_prompt
+#         # use self. will double the parameters, so we dont need that as we want the parameter to be part of the generator
+#         system_prompt = Parameter(
+#             name="task_instruction",
+#             data="You will answer a reasoning question. Think step by step. The last line of your response should be of the following format: 'Answer: $VALUE' where VALUE is a numerical value.",
+#             role_desc="To give task instruction to the language model in the system prompt",
+#             requires_opt=True,
+#             param_type=ParameterType.NONE,
+#             # instruction_to_optimizer="You can show some examples if you think that will help.",
+#         )
+#         _few_shot_demos = Parameter(
+#             name="few_shot_demos",
+#             data=None,
+#             role_desc="To provide few shot demos to the language model",
+#             requires_opt=True,
+#             param_type=ParameterType.DEMOS,
+#         )
+
+#         self.llm_counter = Generator(
+#             model_client=model_client,
+#             model_kwargs=model_kwargs,
+#             template=few_shot_template,
+#             prompt_kwargs={
+#                 "system_prompt": system_prompt,
+#                 "few_shot_demos": _few_shot_demos,
+#             },
+#             output_processors=parse_integer_answer,  # transform data field
+#             use_cache=True,
+#             demo_data_class=ObjectCountSimple,  # for output format
+#             demo_data_class_input_mapping={"question": "input_str"},
+#             demo_data_class_output_mapping={"answer": lambda x: x.raw_response},
+#         )
+
+#     def call(self, question: str, id: str = None) -> Any:  # Add id for tracing
+#         output = self.llm_counter(
+#             prompt_kwargs={
+#                 "input_str": question,
+#                 # "few_shot_demos": self._few_shot_demos.data,
+#             },
+#             id=id,
+#         )  # already support both training (forward + call)
+
+#         if not self.training:  # eval
+
+#             if output.data is None:
+#                 print(f"Error in processing the question: {question}, output: {output}")
+#                 output = -1
+#             else:
+#                 output = int(output.data)
+#         return output
 
 
 class ObjectCountTask(Component):
diff --git a/use_cases/question_answering/bhh_object_count/train.py b/use_cases/question_answering/bhh_object_count/train.py
deleted file mode 100644
index b458172a..00000000
--- a/use_cases/question_answering/bhh_object_count/train.py
+++ /dev/null
@@ -1,132 +0,0 @@
-import random
-from adalflow.optim.trainer.trainer import Trainer
-from adalflow.utils.data import Subset, DataLoader
-
-from use_cases.question_answering.bhh_object_count.task import (
-    ObjectCountTaskFewShot,
-    ObjectCountTaskOriginal,
-)
-from use_cases.question_answering.bhh_object_count.prepare_trainer import (
-    TGDWithEvalFnLoss,
-)
-from use_cases.question_answering.bhh_object_count.config import (
-    gpt_3_model,
-    gpt_4o_model,
-)
-from adalflow.datasets.big_bench_hard import BigBenchHard
-
-
-def train_object_count_text_grad_v1(
-    batch_size=6,
-    max_steps=1,
-    max_samples=2,
-    num_workers=2,
-    strategy="random",
-    debug=False,
-):
-
-    trainer = Trainer(
-        optimizer_type="text-grad",
-        strategy=strategy,
-        max_steps=max_steps,
-        num_workers=num_workers,
-        adaltask=TGDWithEvalFnLoss(
-            ObjectCountTaskOriginal, gpt_3_model, gpt_4o_model, gpt_4o_model
-        ),
-        ckpt_path="object_count_text_grad_random",
-    )
-
-    root = "cache_datasets"
-    train_dataset = BigBenchHard("BBH_object_counting", split="train", root=root)
-    val_dataset = BigBenchHard("BBH_object_counting", split="val", root=root)
-    test_dataset = BigBenchHard("BBH_object_counting", split="test", root=root)
-
-    def subset_dataset(dataset, num_samples):
-        num_samples = min(num_samples, len(dataset))
-        random_subset_indices = random.sample(range(len(dataset)), num_samples)
-        return Subset(dataset, random_subset_indices)
-
-    train_dataset = subset_dataset(train_dataset, max_samples)
-    val_dataset = subset_dataset(val_dataset, max_samples)
-    test_dataset = subset_dataset(test_dataset, max_samples)
-
-    train_dataloader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
-    trainer.fit(
-        train_loader=train_dataloader,
-        val_dataset=val_dataset,
-        test_dataset=test_dataset,
-        debug=debug,
-    )
-
-
-def train_object_count_few_shot(
-    total_shots=5,
-    raw_shots: int = 3,
-    bootstrap_shots: int = 2,
-    max_steps=1,
-    max_samples=2,
-    num_workers=2,
-    strategy="random",
-    debug=False,
-    **kwargs
-):
-    trainer = Trainer(
-        optimizer_type="text-grad",
-        strategy="random",
-        max_steps=max_steps,
-        num_workers=num_workers,
-        adaltask=TGDWithEvalFnLoss(
-            ObjectCountTaskFewShot, gpt_3_model, gpt_4o_model, gpt_4o_model
-        ),
-        ckpt_path="object_count_text_grad_few_shot",
-        save_traces=True,
-        raw_shots=raw_shots,
-        bootstrap_shots=bootstrap_shots,
-    )
-
-    root = "cache_datasets"
-    train_dataset = BigBenchHard("BBH_object_counting", split="train", root=root)
-    val_dataset = BigBenchHard("BBH_object_counting", split="val", root=root)
-    test_dataset = BigBenchHard("BBH_object_counting", split="test", root=root)
-
-    def subset_dataset(dataset, num_samples):
-        num_samples = min(num_samples, len(dataset))
-        random_subset_indices = random.sample(range(len(dataset)), num_samples)
-        return Subset(dataset, random_subset_indices)
-
-    train_dataset = subset_dataset(train_dataset, max_samples)
-    val_dataset = subset_dataset(val_dataset, max_samples)
-    test_dataset = subset_dataset(test_dataset, max_samples)
-
-    train_dataloader = DataLoader(  # noqa: F841
-        train_dataset, batch_size=1, shuffle=True
-    )  # noqa: F841
-    # TODO: make it take a dataset instead of data loader
-    trainer.fit(
-        train_dataset=train_dataset,
-        val_dataset=val_dataset,
-        test_dataset=test_dataset,
-        debug=debug,
-    )
-
-
-if __name__ == "__main__":
-    # train_object_count_text_grad_v1(
-    #     batch_size=12,
-    #     max_steps=10,  # large batch size helps
-    #     max_samples=100,
-    #     num_workers=4,
-    #     strategy="constrained",
-    #     debug=False,
-    # )
-
-    train_object_count_few_shot(
-        total_shots=5,
-        raw_shots=0,
-        bootstrap_shots=2,
-        max_steps=10,
-        max_samples=20,
-        num_workers=4,
-        strategy="random",
-        debug=False,
-    )
diff --git a/use_cases/question_answering/bhh_object_count/train_new.py b/use_cases/question_answering/bhh_object_count/train_new.py
index a56a7d6d..888a1ee7 100644
--- a/use_cases/question_answering/bhh_object_count/train_new.py
+++ b/use_cases/question_answering/bhh_object_count/train_new.py
@@ -118,11 +118,14 @@ def train(
     max_steps=1,
     num_workers=4,
     strategy="random",
+    optimization_order="sequential",
     debug=False,
+    resume_from_ckpt=None,
+    exclude_input_fields_from_bootstrap_demos=False,
 ):
     adal_component = ObjectCountAdalComponent(
         **gpt_3_model,
-        teacher_model_config=gpt_3_model,
+        teacher_model_config=gpt_4o_model,
         text_optimizer_model_config=gpt_4o_model,
         backward_engine_model_config=gpt_4o_model
     )
@@ -137,6 +140,8 @@ def train(
         bootstrap_shots=bootstrap_shots,
         debug=debug,
         weighted_sampling=True,
+        optimization_order=optimization_order,
+        exclude_input_fields_from_bootstrap_demos=exclude_input_fields_from_bootstrap_demos,
     )
     print(trainer)
 
@@ -145,15 +150,18 @@ def train(
         train_dataset=train_dataset,
         val_dataset=val_dataset,
         test_dataset=test_dataset,
-        debug=debug,
+        resume_from_ckpt=resume_from_ckpt,
     )
 
 
 if __name__ == "__main__":
 
     train(
-        debug=False, max_steps=24, strategy="constrained"
-    )  # TODO: few-shot constraint
+        debug=False,
+        max_steps=12,
+        strategy="constrained",
+        exclude_input_fields_from_bootstrap_demos=True,
+    )
 
     # train_diagnose(**gpt_3_model)
     # train_diagnose_teacher(**gpt_4o_model)

From 51cd53fecd633c9f21b070b1c12c645032b647b2 Mon Sep 17 00:00:00 2001
From: Li Yin <li.yin.gravity@gmail.com>
Date: Fri, 23 Aug 2024 19:36:43 -0700
Subject: [PATCH 2/8] add default set up in the Trainer, but need to (1)test
 (2) diagnose need to resume from ckpt

---
 README.md                                     |   1 +
 adalflow/CHANGELOG.md                         |   4 +
 adalflow/adalflow/eval/llm_as_judge.py        |  16 +-
 adalflow/adalflow/optim/trainer/adal.py       |  63 +-
 adalflow/adalflow/optim/trainer/trainer.py    |   2 +-
 adalflow/poetry.lock                          | 802 +++++++++++++++++-
 adalflow/pyproject.toml                       |   6 +-
 .../source/get_started/adalflow_in_15mins.rst |  12 +
 docs/source/use_cases/question_answering.rst  |  13 +
 poetry.lock                                   |   4 +-
 10 files changed, 906 insertions(+), 17 deletions(-)

diff --git a/README.md b/README.md
index e206e61f..88778c87 100644
--- a/README.md
+++ b/README.md
@@ -16,6 +16,7 @@
 </h2>
 
 
+
 <p align="center">
     <a href="https://colab.research.google.com/drive/1TKw_JHE42Z_AWo8UuRYZCO2iuMgyslTZ?usp=sharing">
         <img alt="Try Quickstart in Colab" src="https://colab.research.google.com/assets/colab-badge.svg">
diff --git a/adalflow/CHANGELOG.md b/adalflow/CHANGELOG.md
index e509f624..f90aa4b0 100644
--- a/adalflow/CHANGELOG.md
+++ b/adalflow/CHANGELOG.md
@@ -1,6 +1,10 @@
 ## [0.2.1] - 2024-09-01
 ### Added
 - `get_cache_path`, instead of print out the cache path all the time, we add a ``get_cache_path`` to get the cache path.
+- Make `huggingface datasets` as an optional dependency.
+### Modified
+- Add `template` to let users pass their own template, but need to have the same arguments as the default template.
+- Added `checkpoint resumt` in the `Trainer.diagnose` to show the newest performance and diagnostics on the checkpoint.
 ## [0.2.0] - 2024-08-20
 ### Added
 - Qdrant retriever.
diff --git a/adalflow/adalflow/eval/llm_as_judge.py b/adalflow/adalflow/eval/llm_as_judge.py
index 534e6910..119e263b 100644
--- a/adalflow/adalflow/eval/llm_as_judge.py
+++ b/adalflow/adalflow/eval/llm_as_judge.py
@@ -11,12 +11,14 @@
 
 log = logging.getLogger(__name__)
 
-DEFAULT_LLM_EVALUATOR_PROMPT = r"""
-<<SYS>>{# task desc #}
-You are a helpful assistant.
+DEFAULT_LLM_EVALUATOR_PROMPT = r"""<START_OF_SYSTEM_PROMPT>
+{# task desc #}
+You are an evaluator.
 Given the question, ground truth answer, and predicted answer, you need to answer the judgement query.
-Output True or False according to the judgement query.<</SYS>>
+Output True or False according to the judgement query.
+<END_OF_SYSTEM_PROMPT>
 ---------------------
+<START_OF_USER>
 {# question #}
 Question: {{question_str}}
 {# ground truth answer #}
@@ -26,7 +28,7 @@
 {# judgement question #}
 Judgement question: {{judgement_str}}
 {# assistant response #}
-You:
+<END_OF_USER>
 """
 
 
@@ -55,6 +57,7 @@ def __init__(
         self,
         model_client: Optional[ModelClient] = None,
         model_kwargs: Optional[Dict[str, Any]] = None,
+        template: Optional[str] = None,
     ):
         from adalflow.core.generator import Generator
 
@@ -70,10 +73,11 @@ def __init__(
                 )
             self.model_client = OpenAIClient()
         self.model_kwargs = model_kwargs or DEFAULT_LLM_EVALUATOR_MODEL_KWARGS
+        self.template = template or DEFAULT_LLM_EVALUATOR_PROMPT
         self.llm_evaluator = Generator(
             model_client=self.model_client,
             model_kwargs=self.model_kwargs,
-            template=DEFAULT_LLM_EVALUATOR_PROMPT,
+            template=self.template,
         )
 
     def call(
diff --git a/adalflow/adalflow/optim/trainer/adal.py b/adalflow/adalflow/optim/trainer/adal.py
index aa428798..8dd59025 100644
--- a/adalflow/adalflow/optim/trainer/adal.py
+++ b/adalflow/adalflow/optim/trainer/adal.py
@@ -5,6 +5,7 @@
 from tqdm import tqdm
 import numpy as np
 import warnings
+import logging as log
 
 if TYPE_CHECKING:
     from adalflow.core.model_client import ModelClient
@@ -21,6 +22,9 @@
 from adalflow.optim.optimizer import DemoOptimizer, TextOptimizer
 
 
+log = log.getLogger(__name__)
+
+
 class AdalComponent(Component):
     """Define a train, eval, and test step for a task pipeline.
 
@@ -133,12 +137,45 @@ def evaluate_one_sample(self, sample: Any, y_pred: Any, *args, **kwargs) -> floa
         The higher the score the better the prediction."""
         raise NotImplementedError("evaluate_one_sample method is not implemented")
 
-    def configure_optimizers(self, *args, **kwargs) -> Optimizer:
+    # def configure_optimizers(self, *args, **kwargs) -> Optimizer:
+    #     r"""Note: When you use text optimizor, ensure you call `configure_backward_engine_engine` too."""
+    #     raise NotImplementedError("configure_optimizers method is not implemented")
+
+    def configure_optimizers(self, *args, **kwargs) -> List[Optimizer]:
         r"""Note: When you use text optimizor, ensure you call `configure_backward_engine_engine` too."""
-        raise NotImplementedError("configure_optimizers method is not implemented")
+        if self._demo_optimizers is None:
+            self._demo_optimizers = self.configure_demo_optimizer_helper()
+        if self._text_optimizers is None:
+            if not self.text_optimizer_model_config:
+                raise ValueError("Text optimizer model config is not configured.")
+            if not self.text_optimizer_model_config.get("model_client"):
+                raise ValueError("Model client is not configured.")
+            if not self.text_optimizer_model_config.get("model_kwargs"):
+                raise ValueError("Model kwargs is not configured.")
+
+            self._text_optimizers = self.configure_text_optimizer_helper(
+                **self.text_optimizer_model_config
+            )
+        return self._demo_optimizers + self._text_optimizers
 
     def configure_backward_engine(self, *args, **kwargs):
-        raise NotImplementedError("configure_backward_engine method is not implemented")
+        r"""Configure a backward engine for all generators in the task for bootstrapping examples."""
+        # check if backward engine is already configured
+        if self.backward_engine:
+            log.warning("Backward engine is already configured.")
+        if not self.backward_engine_model_config:
+            raise ValueError("Backward engine model config is not configured.")
+        if not self.backward_engine_model_config.get("model_client"):
+            raise ValueError("Model client is not configured.")
+        if not self.backward_engine_model_config.get("model_kwargs"):
+            raise ValueError("Model kwargs is not configured.")
+        self.configure_backward_engine_helper(
+            model_client=self.backward_engine_model_config["model_client"],
+            model_kwargs=self.backward_engine_model_config["model_kwargs"],
+        )
+
+    # def configure_backward_engine(self, *args, **kwargs):
+    #     raise NotImplementedError("configure_backward_engine method is not implemented")
 
     def evaluate_samples(
         self, samples: Any, y_preds: List, metadata: Optional[Dict[str, Any]] = None
@@ -385,14 +422,28 @@ def loss_step(
                 tqdm_loader.update(1)
         return losses
 
+    # def configure_teacher_generator(self):
+    #     r"""Configure a teach generator for all generators in the task for bootstrapping examples.
+
+    #     You can call `configure_teacher_generator_helper` to easily configure it by passing the model_client and model_kwargs.
+    #     """
+    #     raise NotImplementedError(
+    #         "configure_teacher_generator method is not implemented"
+    #     )
+
+    # use default implementation
     def configure_teacher_generator(self):
         r"""Configure a teach generator for all generators in the task for bootstrapping examples.
 
         You can call `configure_teacher_generator_helper` to easily configure it by passing the model_client and model_kwargs.
         """
-        raise NotImplementedError(
-            "configure_teacher_generator method is not implemented"
-        )
+        if not self.teacher_model_config:
+            raise ValueError("Teacher model config is not configured.")
+        if not self.teacher_model_config.get("model_client"):
+            raise ValueError("Model client is not configured.")
+        if not self.teacher_model_config.get("model_kwargs"):
+            raise ValueError("Model kwargs is not configured.")
+        self.configure_teacher_generator_helper(**self.teacher_model_config)
 
     def configure_teacher_generator_helper(
         self,
diff --git a/adalflow/adalflow/optim/trainer/trainer.py b/adalflow/adalflow/optim/trainer/trainer.py
index cfeed983..4ac4b934 100644
--- a/adalflow/adalflow/optim/trainer/trainer.py
+++ b/adalflow/adalflow/optim/trainer/trainer.py
@@ -151,6 +151,7 @@ def __init__(
             exclude_input_fields_from_bootstrap_demos
         )
 
+    # TODO: need to support checkpoint resume too!
     def diagnose(self, dataset: Any, split: str = "train"):
         """Run an evaluation on the trainset to track all error response, and its raw response using AdaplComponent's default configure_callbacks
         Args:
@@ -1167,7 +1168,6 @@ def _fit_demos_random(
         )
         print(f"save to {self.ckpt_file}")
         print(f"Starting step: {starting_step}")
-        print(f"trainer_results: {trainer_results.steps}")
 
         self.adaltask.train()
         self.adaltask.trace()
diff --git a/adalflow/poetry.lock b/adalflow/poetry.lock
index 03f93823..682e5a8b 100644
--- a/adalflow/poetry.lock
+++ b/adalflow/poetry.lock
@@ -11,6 +11,143 @@ files = [
     {file = "absl_py-2.1.0-py3-none-any.whl", hash = "sha256:526a04eadab8b4ee719ce68f204172ead1027549089702d99b9059f129ff1308"},
 ]
 
+[[package]]
+name = "aiohappyeyeballs"
+version = "2.4.0"
+description = "Happy Eyeballs for asyncio"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "aiohappyeyeballs-2.4.0-py3-none-any.whl", hash = "sha256:7ce92076e249169a13c2f49320d1967425eaf1f407522d707d59cac7628d62bd"},
+    {file = "aiohappyeyeballs-2.4.0.tar.gz", hash = "sha256:55a1714f084e63d49639800f95716da97a1f173d46a16dfcfda0016abb93b6b2"},
+]
+
+[[package]]
+name = "aiohttp"
+version = "3.10.5"
+description = "Async http client/server framework (asyncio)"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "aiohttp-3.10.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:18a01eba2574fb9edd5f6e5fb25f66e6ce061da5dab5db75e13fe1558142e0a3"},
+    {file = "aiohttp-3.10.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:94fac7c6e77ccb1ca91e9eb4cb0ac0270b9fb9b289738654120ba8cebb1189c6"},
+    {file = "aiohttp-3.10.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2f1f1c75c395991ce9c94d3e4aa96e5c59c8356a15b1c9231e783865e2772699"},
+    {file = "aiohttp-3.10.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4f7acae3cf1a2a2361ec4c8e787eaaa86a94171d2417aae53c0cca6ca3118ff6"},
+    {file = "aiohttp-3.10.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:94c4381ffba9cc508b37d2e536b418d5ea9cfdc2848b9a7fea6aebad4ec6aac1"},
+    {file = "aiohttp-3.10.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c31ad0c0c507894e3eaa843415841995bf8de4d6b2d24c6e33099f4bc9fc0d4f"},
+    {file = "aiohttp-3.10.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0912b8a8fadeb32ff67a3ed44249448c20148397c1ed905d5dac185b4ca547bb"},
+    {file = "aiohttp-3.10.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0d93400c18596b7dc4794d48a63fb361b01a0d8eb39f28800dc900c8fbdaca91"},
+    {file = "aiohttp-3.10.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:d00f3c5e0d764a5c9aa5a62d99728c56d455310bcc288a79cab10157b3af426f"},
+    {file = "aiohttp-3.10.5-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:d742c36ed44f2798c8d3f4bc511f479b9ceef2b93f348671184139e7d708042c"},
+    {file = "aiohttp-3.10.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:814375093edae5f1cb31e3407997cf3eacefb9010f96df10d64829362ae2df69"},
+    {file = "aiohttp-3.10.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:8224f98be68a84b19f48e0bdc14224b5a71339aff3a27df69989fa47d01296f3"},
+    {file = "aiohttp-3.10.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d9a487ef090aea982d748b1b0d74fe7c3950b109df967630a20584f9a99c0683"},
+    {file = "aiohttp-3.10.5-cp310-cp310-win32.whl", hash = "sha256:d9ef084e3dc690ad50137cc05831c52b6ca428096e6deb3c43e95827f531d5ef"},
+    {file = "aiohttp-3.10.5-cp310-cp310-win_amd64.whl", hash = "sha256:66bf9234e08fe561dccd62083bf67400bdbf1c67ba9efdc3dac03650e97c6088"},
+    {file = "aiohttp-3.10.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:8c6a4e5e40156d72a40241a25cc226051c0a8d816610097a8e8f517aeacd59a2"},
+    {file = "aiohttp-3.10.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2c634a3207a5445be65536d38c13791904fda0748b9eabf908d3fe86a52941cf"},
+    {file = "aiohttp-3.10.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4aff049b5e629ef9b3e9e617fa6e2dfeda1bf87e01bcfecaf3949af9e210105e"},
+    {file = "aiohttp-3.10.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1942244f00baaacaa8155eca94dbd9e8cc7017deb69b75ef67c78e89fdad3c77"},
+    {file = "aiohttp-3.10.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e04a1f2a65ad2f93aa20f9ff9f1b672bf912413e5547f60749fa2ef8a644e061"},
+    {file = "aiohttp-3.10.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7f2bfc0032a00405d4af2ba27f3c429e851d04fad1e5ceee4080a1c570476697"},
+    {file = "aiohttp-3.10.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:424ae21498790e12eb759040bbb504e5e280cab64693d14775c54269fd1d2bb7"},
+    {file = "aiohttp-3.10.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:975218eee0e6d24eb336d0328c768ebc5d617609affaca5dbbd6dd1984f16ed0"},
+    {file = "aiohttp-3.10.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4120d7fefa1e2d8fb6f650b11489710091788de554e2b6f8347c7a20ceb003f5"},
+    {file = "aiohttp-3.10.5-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:b90078989ef3fc45cf9221d3859acd1108af7560c52397ff4ace8ad7052a132e"},
+    {file = "aiohttp-3.10.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:ba5a8b74c2a8af7d862399cdedce1533642fa727def0b8c3e3e02fcb52dca1b1"},
+    {file = "aiohttp-3.10.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:02594361128f780eecc2a29939d9dfc870e17b45178a867bf61a11b2a4367277"},
+    {file = "aiohttp-3.10.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:8fb4fc029e135859f533025bc82047334e24b0d489e75513144f25408ecaf058"},
+    {file = "aiohttp-3.10.5-cp311-cp311-win32.whl", hash = "sha256:e1ca1ef5ba129718a8fc827b0867f6aa4e893c56eb00003b7367f8a733a9b072"},
+    {file = "aiohttp-3.10.5-cp311-cp311-win_amd64.whl", hash = "sha256:349ef8a73a7c5665cca65c88ab24abe75447e28aa3bc4c93ea5093474dfdf0ff"},
+    {file = "aiohttp-3.10.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:305be5ff2081fa1d283a76113b8df7a14c10d75602a38d9f012935df20731487"},
+    {file = "aiohttp-3.10.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3a1c32a19ee6bbde02f1cb189e13a71b321256cc1d431196a9f824050b160d5a"},
+    {file = "aiohttp-3.10.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:61645818edd40cc6f455b851277a21bf420ce347baa0b86eaa41d51ef58ba23d"},
+    {file = "aiohttp-3.10.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c225286f2b13bab5987425558baa5cbdb2bc925b2998038fa028245ef421e75"},
+    {file = "aiohttp-3.10.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8ba01ebc6175e1e6b7275c907a3a36be48a2d487549b656aa90c8a910d9f3178"},
+    {file = "aiohttp-3.10.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8eaf44ccbc4e35762683078b72bf293f476561d8b68ec8a64f98cf32811c323e"},
+    {file = "aiohttp-3.10.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1c43eb1ab7cbf411b8e387dc169acb31f0ca0d8c09ba63f9eac67829585b44f"},
+    {file = "aiohttp-3.10.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:de7a5299827253023c55ea549444e058c0eb496931fa05d693b95140a947cb73"},
+    {file = "aiohttp-3.10.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4790f0e15f00058f7599dab2b206d3049d7ac464dc2e5eae0e93fa18aee9e7bf"},
+    {file = "aiohttp-3.10.5-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:44b324a6b8376a23e6ba25d368726ee3bc281e6ab306db80b5819999c737d820"},
+    {file = "aiohttp-3.10.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:0d277cfb304118079e7044aad0b76685d30ecb86f83a0711fc5fb257ffe832ca"},
+    {file = "aiohttp-3.10.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:54d9ddea424cd19d3ff6128601a4a4d23d54a421f9b4c0fff740505813739a91"},
+    {file = "aiohttp-3.10.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:4f1c9866ccf48a6df2b06823e6ae80573529f2af3a0992ec4fe75b1a510df8a6"},
+    {file = "aiohttp-3.10.5-cp312-cp312-win32.whl", hash = "sha256:dc4826823121783dccc0871e3f405417ac116055bf184ac04c36f98b75aacd12"},
+    {file = "aiohttp-3.10.5-cp312-cp312-win_amd64.whl", hash = "sha256:22c0a23a3b3138a6bf76fc553789cb1a703836da86b0f306b6f0dc1617398abc"},
+    {file = "aiohttp-3.10.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:7f6b639c36734eaa80a6c152a238242bedcee9b953f23bb887e9102976343092"},
+    {file = "aiohttp-3.10.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f29930bc2921cef955ba39a3ff87d2c4398a0394ae217f41cb02d5c26c8b1b77"},
+    {file = "aiohttp-3.10.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f489a2c9e6455d87eabf907ac0b7d230a9786be43fbe884ad184ddf9e9c1e385"},
+    {file = "aiohttp-3.10.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:123dd5b16b75b2962d0fff566effb7a065e33cd4538c1692fb31c3bda2bfb972"},
+    {file = "aiohttp-3.10.5-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b98e698dc34966e5976e10bbca6d26d6724e6bdea853c7c10162a3235aba6e16"},
+    {file = "aiohttp-3.10.5-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c3b9162bab7e42f21243effc822652dc5bb5e8ff42a4eb62fe7782bcbcdfacf6"},
+    {file = "aiohttp-3.10.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1923a5c44061bffd5eebeef58cecf68096e35003907d8201a4d0d6f6e387ccaa"},
+    {file = "aiohttp-3.10.5-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d55f011da0a843c3d3df2c2cf4e537b8070a419f891c930245f05d329c4b0689"},
+    {file = "aiohttp-3.10.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:afe16a84498441d05e9189a15900640a2d2b5e76cf4efe8cbb088ab4f112ee57"},
+    {file = "aiohttp-3.10.5-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:f8112fb501b1e0567a1251a2fd0747baae60a4ab325a871e975b7bb67e59221f"},
+    {file = "aiohttp-3.10.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:1e72589da4c90337837fdfe2026ae1952c0f4a6e793adbbfbdd40efed7c63599"},
+    {file = "aiohttp-3.10.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:4d46c7b4173415d8e583045fbc4daa48b40e31b19ce595b8d92cf639396c15d5"},
+    {file = "aiohttp-3.10.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:33e6bc4bab477c772a541f76cd91e11ccb6d2efa2b8d7d7883591dfb523e5987"},
+    {file = "aiohttp-3.10.5-cp313-cp313-win32.whl", hash = "sha256:c58c6837a2c2a7cf3133983e64173aec11f9c2cd8e87ec2fdc16ce727bcf1a04"},
+    {file = "aiohttp-3.10.5-cp313-cp313-win_amd64.whl", hash = "sha256:38172a70005252b6893088c0f5e8a47d173df7cc2b2bd88650957eb84fcf5022"},
+    {file = "aiohttp-3.10.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:f6f18898ace4bcd2d41a122916475344a87f1dfdec626ecde9ee802a711bc569"},
+    {file = "aiohttp-3.10.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:5ede29d91a40ba22ac1b922ef510aab871652f6c88ef60b9dcdf773c6d32ad7a"},
+    {file = "aiohttp-3.10.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:673f988370f5954df96cc31fd99c7312a3af0a97f09e407399f61583f30da9bc"},
+    {file = "aiohttp-3.10.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:58718e181c56a3c02d25b09d4115eb02aafe1a732ce5714ab70326d9776457c3"},
+    {file = "aiohttp-3.10.5-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4b38b1570242fbab8d86a84128fb5b5234a2f70c2e32f3070143a6d94bc854cf"},
+    {file = "aiohttp-3.10.5-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:074d1bff0163e107e97bd48cad9f928fa5a3eb4b9d33366137ffce08a63e37fe"},
+    {file = "aiohttp-3.10.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd31f176429cecbc1ba499d4aba31aaccfea488f418d60376b911269d3b883c5"},
+    {file = "aiohttp-3.10.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7384d0b87d4635ec38db9263e6a3f1eb609e2e06087f0aa7f63b76833737b471"},
+    {file = "aiohttp-3.10.5-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:8989f46f3d7ef79585e98fa991e6ded55d2f48ae56d2c9fa5e491a6e4effb589"},
+    {file = "aiohttp-3.10.5-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:c83f7a107abb89a227d6c454c613e7606c12a42b9a4ca9c5d7dad25d47c776ae"},
+    {file = "aiohttp-3.10.5-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:cde98f323d6bf161041e7627a5fd763f9fd829bcfcd089804a5fdce7bb6e1b7d"},
+    {file = "aiohttp-3.10.5-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:676f94c5480d8eefd97c0c7e3953315e4d8c2b71f3b49539beb2aa676c58272f"},
+    {file = "aiohttp-3.10.5-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:2d21ac12dc943c68135ff858c3a989f2194a709e6e10b4c8977d7fcd67dfd511"},
+    {file = "aiohttp-3.10.5-cp38-cp38-win32.whl", hash = "sha256:17e997105bd1a260850272bfb50e2a328e029c941c2708170d9d978d5a30ad9a"},
+    {file = "aiohttp-3.10.5-cp38-cp38-win_amd64.whl", hash = "sha256:1c19de68896747a2aa6257ae4cf6ef59d73917a36a35ee9d0a6f48cff0f94db8"},
+    {file = "aiohttp-3.10.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7e2fe37ac654032db1f3499fe56e77190282534810e2a8e833141a021faaab0e"},
+    {file = "aiohttp-3.10.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f5bf3ead3cb66ab990ee2561373b009db5bc0e857549b6c9ba84b20bc462e172"},
+    {file = "aiohttp-3.10.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1b2c16a919d936ca87a3c5f0e43af12a89a3ce7ccbce59a2d6784caba945b68b"},
+    {file = "aiohttp-3.10.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad146dae5977c4dd435eb31373b3fe9b0b1bf26858c6fc452bf6af394067e10b"},
+    {file = "aiohttp-3.10.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8c5c6fa16412b35999320f5c9690c0f554392dc222c04e559217e0f9ae244b92"},
+    {file = "aiohttp-3.10.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:95c4dc6f61d610bc0ee1edc6f29d993f10febfe5b76bb470b486d90bbece6b22"},
+    {file = "aiohttp-3.10.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:da452c2c322e9ce0cfef392e469a26d63d42860f829026a63374fde6b5c5876f"},
+    {file = "aiohttp-3.10.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:898715cf566ec2869d5cb4d5fb4be408964704c46c96b4be267442d265390f32"},
+    {file = "aiohttp-3.10.5-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:391cc3a9c1527e424c6865e087897e766a917f15dddb360174a70467572ac6ce"},
+    {file = "aiohttp-3.10.5-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:380f926b51b92d02a34119d072f178d80bbda334d1a7e10fa22d467a66e494db"},
+    {file = "aiohttp-3.10.5-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:ce91db90dbf37bb6fa0997f26574107e1b9d5ff939315247b7e615baa8ec313b"},
+    {file = "aiohttp-3.10.5-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:9093a81e18c45227eebe4c16124ebf3e0d893830c6aca7cc310bfca8fe59d857"},
+    {file = "aiohttp-3.10.5-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:ee40b40aa753d844162dcc80d0fe256b87cba48ca0054f64e68000453caead11"},
+    {file = "aiohttp-3.10.5-cp39-cp39-win32.whl", hash = "sha256:03f2645adbe17f274444953bdea69f8327e9d278d961d85657cb0d06864814c1"},
+    {file = "aiohttp-3.10.5-cp39-cp39-win_amd64.whl", hash = "sha256:d17920f18e6ee090bdd3d0bfffd769d9f2cb4c8ffde3eb203777a3895c128862"},
+    {file = "aiohttp-3.10.5.tar.gz", hash = "sha256:f071854b47d39591ce9a17981c46790acb30518e2f83dfca8db2dfa091178691"},
+]
+
+[package.dependencies]
+aiohappyeyeballs = ">=2.3.0"
+aiosignal = ">=1.1.2"
+async-timeout = {version = ">=4.0,<5.0", markers = "python_version < \"3.11\""}
+attrs = ">=17.3.0"
+frozenlist = ">=1.1.1"
+multidict = ">=4.5,<7.0"
+yarl = ">=1.0,<2.0"
+
+[package.extras]
+speedups = ["Brotli", "aiodns (>=3.2.0)", "brotlicffi"]
+
+[[package]]
+name = "aiosignal"
+version = "1.3.1"
+description = "aiosignal: a list of registered asynchronous callbacks"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "aiosignal-1.3.1-py3-none-any.whl", hash = "sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17"},
+    {file = "aiosignal-1.3.1.tar.gz", hash = "sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc"},
+]
+
+[package.dependencies]
+frozenlist = ">=1.1.0"
+
 [[package]]
 name = "annotated-types"
 version = "0.7.0"
@@ -69,6 +206,17 @@ doc = ["Sphinx (>=7)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphin
 test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (>=0.17)"]
 trio = ["trio (>=0.23)"]
 
+[[package]]
+name = "async-timeout"
+version = "4.0.3"
+description = "Timeout context manager for asyncio programs"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "async-timeout-4.0.3.tar.gz", hash = "sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f"},
+    {file = "async_timeout-4.0.3-py3-none-any.whl", hash = "sha256:7405140ff1230c310e51dc27b3145b9092d659ce68ff733fb0cefe3ee42be028"},
+]
+
 [[package]]
 name = "attrs"
 version = "23.2.0"
@@ -384,6 +532,65 @@ files = [
 docs = ["ipython", "matplotlib", "numpydoc", "sphinx"]
 tests = ["pytest", "pytest-cov", "pytest-xdist"]
 
+[[package]]
+name = "datasets"
+version = "2.21.0"
+description = "HuggingFace community-driven open-source library of datasets"
+optional = false
+python-versions = ">=3.8.0"
+files = [
+    {file = "datasets-2.21.0-py3-none-any.whl", hash = "sha256:25e4e097110ce28824b746a107727ada94024cba11db8bc588d468414692b65a"},
+    {file = "datasets-2.21.0.tar.gz", hash = "sha256:998f85a8460f1bd982e5bd058f8a0808eef424249e3df1e8cdd594ccd0dc8ba2"},
+]
+
+[package.dependencies]
+aiohttp = "*"
+dill = ">=0.3.0,<0.3.9"
+filelock = "*"
+fsspec = {version = ">=2023.1.0,<=2024.6.1", extras = ["http"]}
+huggingface-hub = ">=0.21.2"
+multiprocess = "*"
+numpy = ">=1.17"
+packaging = "*"
+pandas = "*"
+pyarrow = ">=15.0.0"
+pyyaml = ">=5.1"
+requests = ">=2.32.2"
+tqdm = ">=4.66.3"
+xxhash = "*"
+
+[package.extras]
+apache-beam = ["apache-beam (>=2.26.0)"]
+audio = ["librosa", "soundfile (>=0.12.1)", "soxr (>=0.4.0)"]
+benchmarks = ["tensorflow (==2.12.0)", "torch (==2.0.1)", "transformers (==4.30.1)"]
+dev = ["Pillow (>=9.4.0)", "absl-py", "decorator", "elasticsearch (<8.0.0)", "faiss-cpu (>=1.8.0.post1)", "jax (>=0.3.14)", "jaxlib (>=0.3.14)", "joblib (<1.3.0)", "joblibspark", "librosa", "lz4", "moto[server]", "polars[timezone] (>=0.20.0)", "protobuf (<4.0.0)", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "ruff (>=0.3.0)", "s3fs", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "soxr (>=0.4.0)", "sqlalchemy", "tensorflow (>=2.16.0)", "tensorflow (>=2.6.0)", "tensorflow (>=2.6.0)", "tiktoken", "torch", "torch (>=2.0.0)", "transformers", "transformers (>=4.42.0)", "typing-extensions (>=4.6.1)", "zstandard"]
+docs = ["s3fs", "tensorflow (>=2.6.0)", "torch", "transformers"]
+jax = ["jax (>=0.3.14)", "jaxlib (>=0.3.14)"]
+metrics-tests = ["Werkzeug (>=1.0.1)", "accelerate", "bert-score (>=0.3.6)", "jiwer", "langdetect", "mauve-text", "nltk (<3.8.2)", "requests-file (>=1.5.1)", "rouge-score", "sacrebleu", "sacremoses", "scikit-learn", "scipy", "sentencepiece", "seqeval", "six (>=1.15.0,<1.16.0)", "spacy (>=3.0.0)", "texttable (>=1.6.3)", "tldextract", "tldextract (>=3.1.0)", "toml (>=0.10.1)", "typer (<0.5.0)"]
+quality = ["ruff (>=0.3.0)"]
+s3 = ["s3fs"]
+tensorflow = ["tensorflow (>=2.6.0)"]
+tensorflow-gpu = ["tensorflow (>=2.6.0)"]
+tests = ["Pillow (>=9.4.0)", "absl-py", "decorator", "elasticsearch (<8.0.0)", "faiss-cpu (>=1.8.0.post1)", "jax (>=0.3.14)", "jaxlib (>=0.3.14)", "joblib (<1.3.0)", "joblibspark", "librosa", "lz4", "moto[server]", "polars[timezone] (>=0.20.0)", "protobuf (<4.0.0)", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "soxr (>=0.4.0)", "sqlalchemy", "tensorflow (>=2.16.0)", "tensorflow (>=2.6.0)", "tiktoken", "torch (>=2.0.0)", "transformers (>=4.42.0)", "typing-extensions (>=4.6.1)", "zstandard"]
+tests-numpy2 = ["Pillow (>=9.4.0)", "absl-py", "decorator", "elasticsearch (<8.0.0)", "jax (>=0.3.14)", "jaxlib (>=0.3.14)", "joblib (<1.3.0)", "joblibspark", "librosa", "lz4", "moto[server]", "polars[timezone] (>=0.20.0)", "protobuf (<4.0.0)", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "soxr (>=0.4.0)", "sqlalchemy", "tiktoken", "torch (>=2.0.0)", "typing-extensions (>=4.6.1)", "zstandard"]
+torch = ["torch"]
+vision = ["Pillow (>=9.4.0)"]
+
+[[package]]
+name = "dill"
+version = "0.3.8"
+description = "serialize all of Python"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "dill-0.3.8-py3-none-any.whl", hash = "sha256:c36ca9ffb54365bdd2f8eb3eff7d2a21237f8452b57ace88b1ac615b7e815bd7"},
+    {file = "dill-0.3.8.tar.gz", hash = "sha256:3ebe3c479ad625c4553aca177444d89b486b1d84982eeacded644afc0cf797ca"},
+]
+
+[package.extras]
+graph = ["objgraph (>=1.7.2)"]
+profile = ["gprof2dot (>=2022.7.29)"]
+
 [[package]]
 name = "diskcache"
 version = "5.6.3"
@@ -597,6 +804,92 @@ ufo = ["fs (>=2.2.0,<3)"]
 unicode = ["unicodedata2 (>=15.1.0)"]
 woff = ["brotli (>=1.0.1)", "brotlicffi (>=0.8.0)", "zopfli (>=0.1.4)"]
 
+[[package]]
+name = "frozenlist"
+version = "1.4.1"
+description = "A list-like structure which implements collections.abc.MutableSequence"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "frozenlist-1.4.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:f9aa1878d1083b276b0196f2dfbe00c9b7e752475ed3b682025ff20c1c1f51ac"},
+    {file = "frozenlist-1.4.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:29acab3f66f0f24674b7dc4736477bcd4bc3ad4b896f5f45379a67bce8b96868"},
+    {file = "frozenlist-1.4.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:74fb4bee6880b529a0c6560885fce4dc95936920f9f20f53d99a213f7bf66776"},
+    {file = "frozenlist-1.4.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:590344787a90ae57d62511dd7c736ed56b428f04cd8c161fcc5e7232c130c69a"},
+    {file = "frozenlist-1.4.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:068b63f23b17df8569b7fdca5517edef76171cf3897eb68beb01341131fbd2ad"},
+    {file = "frozenlist-1.4.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5c849d495bf5154cd8da18a9eb15db127d4dba2968d88831aff6f0331ea9bd4c"},
+    {file = "frozenlist-1.4.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9750cc7fe1ae3b1611bb8cfc3f9ec11d532244235d75901fb6b8e42ce9229dfe"},
+    {file = "frozenlist-1.4.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a9b2de4cf0cdd5bd2dee4c4f63a653c61d2408055ab77b151c1957f221cabf2a"},
+    {file = "frozenlist-1.4.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:0633c8d5337cb5c77acbccc6357ac49a1770b8c487e5b3505c57b949b4b82e98"},
+    {file = "frozenlist-1.4.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:27657df69e8801be6c3638054e202a135c7f299267f1a55ed3a598934f6c0d75"},
+    {file = "frozenlist-1.4.1-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:f9a3ea26252bd92f570600098783d1371354d89d5f6b7dfd87359d669f2109b5"},
+    {file = "frozenlist-1.4.1-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:4f57dab5fe3407b6c0c1cc907ac98e8a189f9e418f3b6e54d65a718aaafe3950"},
+    {file = "frozenlist-1.4.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:e02a0e11cf6597299b9f3bbd3f93d79217cb90cfd1411aec33848b13f5c656cc"},
+    {file = "frozenlist-1.4.1-cp310-cp310-win32.whl", hash = "sha256:a828c57f00f729620a442881cc60e57cfcec6842ba38e1b19fd3e47ac0ff8dc1"},
+    {file = "frozenlist-1.4.1-cp310-cp310-win_amd64.whl", hash = "sha256:f56e2333dda1fe0f909e7cc59f021eba0d2307bc6f012a1ccf2beca6ba362439"},
+    {file = "frozenlist-1.4.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:a0cb6f11204443f27a1628b0e460f37fb30f624be6051d490fa7d7e26d4af3d0"},
+    {file = "frozenlist-1.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b46c8ae3a8f1f41a0d2ef350c0b6e65822d80772fe46b653ab6b6274f61d4a49"},
+    {file = "frozenlist-1.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fde5bd59ab5357e3853313127f4d3565fc7dad314a74d7b5d43c22c6a5ed2ced"},
+    {file = "frozenlist-1.4.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:722e1124aec435320ae01ee3ac7bec11a5d47f25d0ed6328f2273d287bc3abb0"},
+    {file = "frozenlist-1.4.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2471c201b70d58a0f0c1f91261542a03d9a5e088ed3dc6c160d614c01649c106"},
+    {file = "frozenlist-1.4.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c757a9dd70d72b076d6f68efdbb9bc943665ae954dad2801b874c8c69e185068"},
+    {file = "frozenlist-1.4.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f146e0911cb2f1da549fc58fc7bcd2b836a44b79ef871980d605ec392ff6b0d2"},
+    {file = "frozenlist-1.4.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f9c515e7914626b2a2e1e311794b4c35720a0be87af52b79ff8e1429fc25f19"},
+    {file = "frozenlist-1.4.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:c302220494f5c1ebeb0912ea782bcd5e2f8308037b3c7553fad0e48ebad6ad82"},
+    {file = "frozenlist-1.4.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:442acde1e068288a4ba7acfe05f5f343e19fac87bfc96d89eb886b0363e977ec"},
+    {file = "frozenlist-1.4.1-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:1b280e6507ea8a4fa0c0a7150b4e526a8d113989e28eaaef946cc77ffd7efc0a"},
+    {file = "frozenlist-1.4.1-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:fe1a06da377e3a1062ae5fe0926e12b84eceb8a50b350ddca72dc85015873f74"},
+    {file = "frozenlist-1.4.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:db9e724bebd621d9beca794f2a4ff1d26eed5965b004a97f1f1685a173b869c2"},
+    {file = "frozenlist-1.4.1-cp311-cp311-win32.whl", hash = "sha256:e774d53b1a477a67838a904131c4b0eef6b3d8a651f8b138b04f748fccfefe17"},
+    {file = "frozenlist-1.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:fb3c2db03683b5767dedb5769b8a40ebb47d6f7f45b1b3e3b4b51ec8ad9d9825"},
+    {file = "frozenlist-1.4.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:1979bc0aeb89b33b588c51c54ab0161791149f2461ea7c7c946d95d5f93b56ae"},
+    {file = "frozenlist-1.4.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:cc7b01b3754ea68a62bd77ce6020afaffb44a590c2289089289363472d13aedb"},
+    {file = "frozenlist-1.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c9c92be9fd329ac801cc420e08452b70e7aeab94ea4233a4804f0915c14eba9b"},
+    {file = "frozenlist-1.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5c3894db91f5a489fc8fa6a9991820f368f0b3cbdb9cd8849547ccfab3392d86"},
+    {file = "frozenlist-1.4.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ba60bb19387e13597fb059f32cd4d59445d7b18b69a745b8f8e5db0346f33480"},
+    {file = "frozenlist-1.4.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8aefbba5f69d42246543407ed2461db31006b0f76c4e32dfd6f42215a2c41d09"},
+    {file = "frozenlist-1.4.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:780d3a35680ced9ce682fbcf4cb9c2bad3136eeff760ab33707b71db84664e3a"},
+    {file = "frozenlist-1.4.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9acbb16f06fe7f52f441bb6f413ebae6c37baa6ef9edd49cdd567216da8600cd"},
+    {file = "frozenlist-1.4.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:23b701e65c7b36e4bf15546a89279bd4d8675faabc287d06bbcfac7d3c33e1e6"},
+    {file = "frozenlist-1.4.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:3e0153a805a98f5ada7e09826255ba99fb4f7524bb81bf6b47fb702666484ae1"},
+    {file = "frozenlist-1.4.1-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:dd9b1baec094d91bf36ec729445f7769d0d0cf6b64d04d86e45baf89e2b9059b"},
+    {file = "frozenlist-1.4.1-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:1a4471094e146b6790f61b98616ab8e44f72661879cc63fa1049d13ef711e71e"},
+    {file = "frozenlist-1.4.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:5667ed53d68d91920defdf4035d1cdaa3c3121dc0b113255124bcfada1cfa1b8"},
+    {file = "frozenlist-1.4.1-cp312-cp312-win32.whl", hash = "sha256:beee944ae828747fd7cb216a70f120767fc9f4f00bacae8543c14a6831673f89"},
+    {file = "frozenlist-1.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:64536573d0a2cb6e625cf309984e2d873979709f2cf22839bf2d61790b448ad5"},
+    {file = "frozenlist-1.4.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:20b51fa3f588ff2fe658663db52a41a4f7aa6c04f6201449c6c7c476bd255c0d"},
+    {file = "frozenlist-1.4.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:410478a0c562d1a5bcc2f7ea448359fcb050ed48b3c6f6f4f18c313a9bdb1826"},
+    {file = "frozenlist-1.4.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c6321c9efe29975232da3bd0af0ad216800a47e93d763ce64f291917a381b8eb"},
+    {file = "frozenlist-1.4.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:48f6a4533887e189dae092f1cf981f2e3885175f7a0f33c91fb5b7b682b6bab6"},
+    {file = "frozenlist-1.4.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6eb73fa5426ea69ee0e012fb59cdc76a15b1283d6e32e4f8dc4482ec67d1194d"},
+    {file = "frozenlist-1.4.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fbeb989b5cc29e8daf7f976b421c220f1b8c731cbf22b9130d8815418ea45887"},
+    {file = "frozenlist-1.4.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:32453c1de775c889eb4e22f1197fe3bdfe457d16476ea407472b9442e6295f7a"},
+    {file = "frozenlist-1.4.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:693945278a31f2086d9bf3df0fe8254bbeaef1fe71e1351c3bd730aa7d31c41b"},
+    {file = "frozenlist-1.4.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:1d0ce09d36d53bbbe566fe296965b23b961764c0bcf3ce2fa45f463745c04701"},
+    {file = "frozenlist-1.4.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:3a670dc61eb0d0eb7080890c13de3066790f9049b47b0de04007090807c776b0"},
+    {file = "frozenlist-1.4.1-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:dca69045298ce5c11fd539682cff879cc1e664c245d1c64da929813e54241d11"},
+    {file = "frozenlist-1.4.1-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:a06339f38e9ed3a64e4c4e43aec7f59084033647f908e4259d279a52d3757d09"},
+    {file = "frozenlist-1.4.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b7f2f9f912dca3934c1baec2e4585a674ef16fe00218d833856408c48d5beee7"},
+    {file = "frozenlist-1.4.1-cp38-cp38-win32.whl", hash = "sha256:e7004be74cbb7d9f34553a5ce5fb08be14fb33bc86f332fb71cbe5216362a497"},
+    {file = "frozenlist-1.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:5a7d70357e7cee13f470c7883a063aae5fe209a493c57d86eb7f5a6f910fae09"},
+    {file = "frozenlist-1.4.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:bfa4a17e17ce9abf47a74ae02f32d014c5e9404b6d9ac7f729e01562bbee601e"},
+    {file = "frozenlist-1.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b7e3ed87d4138356775346e6845cccbe66cd9e207f3cd11d2f0b9fd13681359d"},
+    {file = "frozenlist-1.4.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c99169d4ff810155ca50b4da3b075cbde79752443117d89429595c2e8e37fed8"},
+    {file = "frozenlist-1.4.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:edb678da49d9f72c9f6c609fbe41a5dfb9a9282f9e6a2253d5a91e0fc382d7c0"},
+    {file = "frozenlist-1.4.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6db4667b187a6742b33afbbaf05a7bc551ffcf1ced0000a571aedbb4aa42fc7b"},
+    {file = "frozenlist-1.4.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:55fdc093b5a3cb41d420884cdaf37a1e74c3c37a31f46e66286d9145d2063bd0"},
+    {file = "frozenlist-1.4.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:82e8211d69a4f4bc360ea22cd6555f8e61a1bd211d1d5d39d3d228b48c83a897"},
+    {file = "frozenlist-1.4.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89aa2c2eeb20957be2d950b85974b30a01a762f3308cd02bb15e1ad632e22dc7"},
+    {file = "frozenlist-1.4.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:9d3e0c25a2350080e9319724dede4f31f43a6c9779be48021a7f4ebde8b2d742"},
+    {file = "frozenlist-1.4.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7268252af60904bf52c26173cbadc3a071cece75f873705419c8681f24d3edea"},
+    {file = "frozenlist-1.4.1-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:0c250a29735d4f15321007fb02865f0e6b6a41a6b88f1f523ca1596ab5f50bd5"},
+    {file = "frozenlist-1.4.1-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:96ec70beabbd3b10e8bfe52616a13561e58fe84c0101dd031dc78f250d5128b9"},
+    {file = "frozenlist-1.4.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:23b2d7679b73fe0e5a4560b672a39f98dfc6f60df63823b0a9970525325b95f6"},
+    {file = "frozenlist-1.4.1-cp39-cp39-win32.whl", hash = "sha256:a7496bfe1da7fb1a4e1cc23bb67c58fab69311cc7d32b5a99c2007b4b2a0e932"},
+    {file = "frozenlist-1.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:e6a20a581f9ce92d389a8c7d7c3dd47c81fd5d6e655c8dddf341e14aa48659d0"},
+    {file = "frozenlist-1.4.1-py3-none-any.whl", hash = "sha256:04ced3e6a46b4cfffe20f9ae482818e34eba9b5fb0ce4056e4cc9b6e212d09b7"},
+    {file = "frozenlist-1.4.1.tar.gz", hash = "sha256:c037a86e8513059a2613aaba4d817bb90b9d9b6b69aace3ce9c877e8c8ed402b"},
+]
+
 [[package]]
 name = "fsspec"
 version = "2024.6.1"
@@ -608,6 +901,9 @@ files = [
     {file = "fsspec-2024.6.1.tar.gz", hash = "sha256:fad7d7e209dd4c1208e3bbfda706620e0da5142bebbd9c384afb95b07e798e49"},
 ]
 
+[package.dependencies]
+aiohttp = {version = "<4.0.0a0 || >4.0.0a0,<4.0.0a1 || >4.0.0a1", optional = true, markers = "extra == \"http\""}
+
 [package.extras]
 abfs = ["adlfs"]
 adl = ["adlfs"]
@@ -1525,6 +1821,129 @@ docs = ["sphinx"]
 gmpy = ["gmpy2 (>=2.1.0a4)"]
 tests = ["pytest (>=4.6)"]
 
+[[package]]
+name = "multidict"
+version = "6.0.5"
+description = "multidict implementation"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "multidict-6.0.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:228b644ae063c10e7f324ab1ab6b548bdf6f8b47f3ec234fef1093bc2735e5f9"},
+    {file = "multidict-6.0.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:896ebdcf62683551312c30e20614305f53125750803b614e9e6ce74a96232604"},
+    {file = "multidict-6.0.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:411bf8515f3be9813d06004cac41ccf7d1cd46dfe233705933dd163b60e37600"},
+    {file = "multidict-6.0.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1d147090048129ce3c453f0292e7697d333db95e52616b3793922945804a433c"},
+    {file = "multidict-6.0.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:215ed703caf15f578dca76ee6f6b21b7603791ae090fbf1ef9d865571039ade5"},
+    {file = "multidict-6.0.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c6390cf87ff6234643428991b7359b5f59cc15155695deb4eda5c777d2b880f"},
+    {file = "multidict-6.0.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21fd81c4ebdb4f214161be351eb5bcf385426bf023041da2fd9e60681f3cebae"},
+    {file = "multidict-6.0.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3cc2ad10255f903656017363cd59436f2111443a76f996584d1077e43ee51182"},
+    {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:6939c95381e003f54cd4c5516740faba40cf5ad3eeff460c3ad1d3e0ea2549bf"},
+    {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:220dd781e3f7af2c2c1053da9fa96d9cf3072ca58f057f4c5adaaa1cab8fc442"},
+    {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:766c8f7511df26d9f11cd3a8be623e59cca73d44643abab3f8c8c07620524e4a"},
+    {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:fe5d7785250541f7f5019ab9cba2c71169dc7d74d0f45253f8313f436458a4ef"},
+    {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c1c1496e73051918fcd4f58ff2e0f2f3066d1c76a0c6aeffd9b45d53243702cc"},
+    {file = "multidict-6.0.5-cp310-cp310-win32.whl", hash = "sha256:7afcdd1fc07befad18ec4523a782cde4e93e0a2bf71239894b8d61ee578c1319"},
+    {file = "multidict-6.0.5-cp310-cp310-win_amd64.whl", hash = "sha256:99f60d34c048c5c2fabc766108c103612344c46e35d4ed9ae0673d33c8fb26e8"},
+    {file = "multidict-6.0.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:f285e862d2f153a70586579c15c44656f888806ed0e5b56b64489afe4a2dbfba"},
+    {file = "multidict-6.0.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:53689bb4e102200a4fafa9de9c7c3c212ab40a7ab2c8e474491914d2305f187e"},
+    {file = "multidict-6.0.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:612d1156111ae11d14afaf3a0669ebf6c170dbb735e510a7438ffe2369a847fd"},
+    {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7be7047bd08accdb7487737631d25735c9a04327911de89ff1b26b81745bd4e3"},
+    {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de170c7b4fe6859beb8926e84f7d7d6c693dfe8e27372ce3b76f01c46e489fcf"},
+    {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:04bde7a7b3de05732a4eb39c94574db1ec99abb56162d6c520ad26f83267de29"},
+    {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:85f67aed7bb647f93e7520633d8f51d3cbc6ab96957c71272b286b2f30dc70ed"},
+    {file = "multidict-6.0.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:425bf820055005bfc8aa9a0b99ccb52cc2f4070153e34b701acc98d201693733"},
+    {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d3eb1ceec286eba8220c26f3b0096cf189aea7057b6e7b7a2e60ed36b373b77f"},
+    {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:7901c05ead4b3fb75113fb1dd33eb1253c6d3ee37ce93305acd9d38e0b5f21a4"},
+    {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:e0e79d91e71b9867c73323a3444724d496c037e578a0e1755ae159ba14f4f3d1"},
+    {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:29bfeb0dff5cb5fdab2023a7a9947b3b4af63e9c47cae2a10ad58394b517fddc"},
+    {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e030047e85cbcedbfc073f71836d62dd5dadfbe7531cae27789ff66bc551bd5e"},
+    {file = "multidict-6.0.5-cp311-cp311-win32.whl", hash = "sha256:2f4848aa3baa109e6ab81fe2006c77ed4d3cd1e0ac2c1fbddb7b1277c168788c"},
+    {file = "multidict-6.0.5-cp311-cp311-win_amd64.whl", hash = "sha256:2faa5ae9376faba05f630d7e5e6be05be22913782b927b19d12b8145968a85ea"},
+    {file = "multidict-6.0.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:51d035609b86722963404f711db441cf7134f1889107fb171a970c9701f92e1e"},
+    {file = "multidict-6.0.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:cbebcd5bcaf1eaf302617c114aa67569dd3f090dd0ce8ba9e35e9985b41ac35b"},
+    {file = "multidict-6.0.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2ffc42c922dbfddb4a4c3b438eb056828719f07608af27d163191cb3e3aa6cc5"},
+    {file = "multidict-6.0.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ceb3b7e6a0135e092de86110c5a74e46bda4bd4fbfeeb3a3bcec79c0f861e450"},
+    {file = "multidict-6.0.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:79660376075cfd4b2c80f295528aa6beb2058fd289f4c9252f986751a4cd0496"},
+    {file = "multidict-6.0.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e4428b29611e989719874670fd152b6625500ad6c686d464e99f5aaeeaca175a"},
+    {file = "multidict-6.0.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d84a5c3a5f7ce6db1f999fb9438f686bc2e09d38143f2d93d8406ed2dd6b9226"},
+    {file = "multidict-6.0.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:76c0de87358b192de7ea9649beb392f107dcad9ad27276324c24c91774ca5271"},
+    {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:79a6d2ba910adb2cbafc95dad936f8b9386e77c84c35bc0add315b856d7c3abb"},
+    {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:92d16a3e275e38293623ebf639c471d3e03bb20b8ebb845237e0d3664914caef"},
+    {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:fb616be3538599e797a2017cccca78e354c767165e8858ab5116813146041a24"},
+    {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:14c2976aa9038c2629efa2c148022ed5eb4cb939e15ec7aace7ca932f48f9ba6"},
+    {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:435a0984199d81ca178b9ae2c26ec3d49692d20ee29bc4c11a2a8d4514c67eda"},
+    {file = "multidict-6.0.5-cp312-cp312-win32.whl", hash = "sha256:9fe7b0653ba3d9d65cbe7698cca585bf0f8c83dbbcc710db9c90f478e175f2d5"},
+    {file = "multidict-6.0.5-cp312-cp312-win_amd64.whl", hash = "sha256:01265f5e40f5a17f8241d52656ed27192be03bfa8764d88e8220141d1e4b3556"},
+    {file = "multidict-6.0.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:19fe01cea168585ba0f678cad6f58133db2aa14eccaf22f88e4a6dccadfad8b3"},
+    {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6bf7a982604375a8d49b6cc1b781c1747f243d91b81035a9b43a2126c04766f5"},
+    {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:107c0cdefe028703fb5dafe640a409cb146d44a6ae201e55b35a4af8e95457dd"},
+    {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:403c0911cd5d5791605808b942c88a8155c2592e05332d2bf78f18697a5fa15e"},
+    {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aeaf541ddbad8311a87dd695ed9642401131ea39ad7bc8cf3ef3967fd093b626"},
+    {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e4972624066095e52b569e02b5ca97dbd7a7ddd4294bf4e7247d52635630dd83"},
+    {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:d946b0a9eb8aaa590df1fe082cee553ceab173e6cb5b03239716338629c50c7a"},
+    {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b55358304d7a73d7bdf5de62494aaf70bd33015831ffd98bc498b433dfe5b10c"},
+    {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:a3145cb08d8625b2d3fee1b2d596a8766352979c9bffe5d7833e0503d0f0b5e5"},
+    {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:d65f25da8e248202bd47445cec78e0025c0fe7582b23ec69c3b27a640dd7a8e3"},
+    {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:c9bf56195c6bbd293340ea82eafd0071cb3d450c703d2c93afb89f93b8386ccc"},
+    {file = "multidict-6.0.5-cp37-cp37m-win32.whl", hash = "sha256:69db76c09796b313331bb7048229e3bee7928eb62bab5e071e9f7fcc4879caee"},
+    {file = "multidict-6.0.5-cp37-cp37m-win_amd64.whl", hash = "sha256:fce28b3c8a81b6b36dfac9feb1de115bab619b3c13905b419ec71d03a3fc1423"},
+    {file = "multidict-6.0.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:76f067f5121dcecf0d63a67f29080b26c43c71a98b10c701b0677e4a065fbd54"},
+    {file = "multidict-6.0.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b82cc8ace10ab5bd93235dfaab2021c70637005e1ac787031f4d1da63d493c1d"},
+    {file = "multidict-6.0.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:5cb241881eefd96b46f89b1a056187ea8e9ba14ab88ba632e68d7a2ecb7aadf7"},
+    {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8e94e6912639a02ce173341ff62cc1201232ab86b8a8fcc05572741a5dc7d93"},
+    {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:09a892e4a9fb47331da06948690ae38eaa2426de97b4ccbfafbdcbe5c8f37ff8"},
+    {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:55205d03e8a598cfc688c71ca8ea5f66447164efff8869517f175ea632c7cb7b"},
+    {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:37b15024f864916b4951adb95d3a80c9431299080341ab9544ed148091b53f50"},
+    {file = "multidict-6.0.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2a1dee728b52b33eebff5072817176c172050d44d67befd681609b4746e1c2e"},
+    {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:edd08e6f2f1a390bf137080507e44ccc086353c8e98c657e666c017718561b89"},
+    {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:60d698e8179a42ec85172d12f50b1668254628425a6bd611aba022257cac1386"},
+    {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:3d25f19500588cbc47dc19081d78131c32637c25804df8414463ec908631e453"},
+    {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:4cc0ef8b962ac7a5e62b9e826bd0cd5040e7d401bc45a6835910ed699037a461"},
+    {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:eca2e9d0cc5a889850e9bbd68e98314ada174ff6ccd1129500103df7a94a7a44"},
+    {file = "multidict-6.0.5-cp38-cp38-win32.whl", hash = "sha256:4a6a4f196f08c58c59e0b8ef8ec441d12aee4125a7d4f4fef000ccb22f8d7241"},
+    {file = "multidict-6.0.5-cp38-cp38-win_amd64.whl", hash = "sha256:0275e35209c27a3f7951e1ce7aaf93ce0d163b28948444bec61dd7badc6d3f8c"},
+    {file = "multidict-6.0.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:e7be68734bd8c9a513f2b0cfd508802d6609da068f40dc57d4e3494cefc92929"},
+    {file = "multidict-6.0.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1d9ea7a7e779d7a3561aade7d596649fbecfa5c08a7674b11b423783217933f9"},
+    {file = "multidict-6.0.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ea1456df2a27c73ce51120fa2f519f1bea2f4a03a917f4a43c8707cf4cbbae1a"},
+    {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cf590b134eb70629e350691ecca88eac3e3b8b3c86992042fb82e3cb1830d5e1"},
+    {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5c0631926c4f58e9a5ccce555ad7747d9a9f8b10619621f22f9635f069f6233e"},
+    {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dce1c6912ab9ff5f179eaf6efe7365c1f425ed690b03341911bf4939ef2f3046"},
+    {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0868d64af83169e4d4152ec612637a543f7a336e4a307b119e98042e852ad9c"},
+    {file = "multidict-6.0.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:141b43360bfd3bdd75f15ed811850763555a251e38b2405967f8e25fb43f7d40"},
+    {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:7df704ca8cf4a073334e0427ae2345323613e4df18cc224f647f251e5e75a527"},
+    {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:6214c5a5571802c33f80e6c84713b2c79e024995b9c5897f794b43e714daeec9"},
+    {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:cd6c8fca38178e12c00418de737aef1261576bd1b6e8c6134d3e729a4e858b38"},
+    {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:e02021f87a5b6932fa6ce916ca004c4d441509d33bbdbeca70d05dff5e9d2479"},
+    {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ebd8d160f91a764652d3e51ce0d2956b38efe37c9231cd82cfc0bed2e40b581c"},
+    {file = "multidict-6.0.5-cp39-cp39-win32.whl", hash = "sha256:04da1bb8c8dbadf2a18a452639771951c662c5ad03aefe4884775454be322c9b"},
+    {file = "multidict-6.0.5-cp39-cp39-win_amd64.whl", hash = "sha256:d6f6d4f185481c9669b9447bf9d9cf3b95a0e9df9d169bbc17e363b7d5487755"},
+    {file = "multidict-6.0.5-py3-none-any.whl", hash = "sha256:0d63c74e3d7ab26de115c49bffc92cc77ed23395303d496eae515d4204a625e7"},
+    {file = "multidict-6.0.5.tar.gz", hash = "sha256:f7e301075edaf50500f0b341543c41194d8df3ae5caf4702f2095f3ca73dd8da"},
+]
+
+[[package]]
+name = "multiprocess"
+version = "0.70.16"
+description = "better multiprocessing and multithreading in Python"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "multiprocess-0.70.16-pp310-pypy310_pp73-macosx_10_13_x86_64.whl", hash = "sha256:476887be10e2f59ff183c006af746cb6f1fd0eadcfd4ef49e605cbe2659920ee"},
+    {file = "multiprocess-0.70.16-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:d951bed82c8f73929ac82c61f01a7b5ce8f3e5ef40f5b52553b4f547ce2b08ec"},
+    {file = "multiprocess-0.70.16-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:37b55f71c07e2d741374998c043b9520b626a8dddc8b3129222ca4f1a06ef67a"},
+    {file = "multiprocess-0.70.16-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:ba8c31889abf4511c7308a8c52bb4a30b9d590e7f58523302ba00237702ca054"},
+    {file = "multiprocess-0.70.16-pp39-pypy39_pp73-macosx_10_13_x86_64.whl", hash = "sha256:0dfd078c306e08d46d7a8d06fb120313d87aa43af60d66da43ffff40b44d2f41"},
+    {file = "multiprocess-0.70.16-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:e7b9d0f307cd9bd50851afaac0dba2cb6c44449efff697df7c7645f7d3f2be3a"},
+    {file = "multiprocess-0.70.16-py310-none-any.whl", hash = "sha256:c4a9944c67bd49f823687463660a2d6daae94c289adff97e0f9d696ba6371d02"},
+    {file = "multiprocess-0.70.16-py311-none-any.whl", hash = "sha256:af4cabb0dac72abfb1e794fa7855c325fd2b55a10a44628a3c1ad3311c04127a"},
+    {file = "multiprocess-0.70.16-py312-none-any.whl", hash = "sha256:fc0544c531920dde3b00c29863377f87e1632601092ea2daca74e4beb40faa2e"},
+    {file = "multiprocess-0.70.16-py38-none-any.whl", hash = "sha256:a71d82033454891091a226dfc319d0cfa8019a4e888ef9ca910372a446de4435"},
+    {file = "multiprocess-0.70.16-py39-none-any.whl", hash = "sha256:a0bafd3ae1b732eac64be2e72038231c1ba97724b60b09400d68f229fcc2fbf3"},
+    {file = "multiprocess-0.70.16.tar.gz", hash = "sha256:161af703d4652a0e1410be6abccecde4a7ddffd19341be0a7011b94aeb171ac1"},
+]
+
+[package.dependencies]
+dill = ">=0.3.8"
+
 [[package]]
 name = "mypy"
 version = "1.11.1"
@@ -1860,6 +2279,79 @@ files = [
     {file = "packaging-24.1.tar.gz", hash = "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002"},
 ]
 
+[[package]]
+name = "pandas"
+version = "2.2.2"
+description = "Powerful data structures for data analysis, time series, and statistics"
+optional = false
+python-versions = ">=3.9"
+files = [
+    {file = "pandas-2.2.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:90c6fca2acf139569e74e8781709dccb6fe25940488755716d1d354d6bc58bce"},
+    {file = "pandas-2.2.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c7adfc142dac335d8c1e0dcbd37eb8617eac386596eb9e1a1b77791cf2498238"},
+    {file = "pandas-2.2.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4abfe0be0d7221be4f12552995e58723c7422c80a659da13ca382697de830c08"},
+    {file = "pandas-2.2.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8635c16bf3d99040fdf3ca3db669a7250ddf49c55dc4aa8fe0ae0fa8d6dcc1f0"},
+    {file = "pandas-2.2.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:40ae1dffb3967a52203105a077415a86044a2bea011b5f321c6aa64b379a3f51"},
+    {file = "pandas-2.2.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8e5a0b00e1e56a842f922e7fae8ae4077aee4af0acb5ae3622bd4b4c30aedf99"},
+    {file = "pandas-2.2.2-cp310-cp310-win_amd64.whl", hash = "sha256:ddf818e4e6c7c6f4f7c8a12709696d193976b591cc7dc50588d3d1a6b5dc8772"},
+    {file = "pandas-2.2.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:696039430f7a562b74fa45f540aca068ea85fa34c244d0deee539cb6d70aa288"},
+    {file = "pandas-2.2.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8e90497254aacacbc4ea6ae5e7a8cd75629d6ad2b30025a4a8b09aa4faf55151"},
+    {file = "pandas-2.2.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:58b84b91b0b9f4bafac2a0ac55002280c094dfc6402402332c0913a59654ab2b"},
+    {file = "pandas-2.2.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d2123dc9ad6a814bcdea0f099885276b31b24f7edf40f6cdbc0912672e22eee"},
+    {file = "pandas-2.2.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:2925720037f06e89af896c70bca73459d7e6a4be96f9de79e2d440bd499fe0db"},
+    {file = "pandas-2.2.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0cace394b6ea70c01ca1595f839cf193df35d1575986e484ad35c4aeae7266c1"},
+    {file = "pandas-2.2.2-cp311-cp311-win_amd64.whl", hash = "sha256:873d13d177501a28b2756375d59816c365e42ed8417b41665f346289adc68d24"},
+    {file = "pandas-2.2.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:9dfde2a0ddef507a631dc9dc4af6a9489d5e2e740e226ad426a05cabfbd7c8ef"},
+    {file = "pandas-2.2.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e9b79011ff7a0f4b1d6da6a61aa1aa604fb312d6647de5bad20013682d1429ce"},
+    {file = "pandas-2.2.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1cb51fe389360f3b5a4d57dbd2848a5f033350336ca3b340d1c53a1fad33bcad"},
+    {file = "pandas-2.2.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eee3a87076c0756de40b05c5e9a6069c035ba43e8dd71c379e68cab2c20f16ad"},
+    {file = "pandas-2.2.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:3e374f59e440d4ab45ca2fffde54b81ac3834cf5ae2cdfa69c90bc03bde04d76"},
+    {file = "pandas-2.2.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:43498c0bdb43d55cb162cdc8c06fac328ccb5d2eabe3cadeb3529ae6f0517c32"},
+    {file = "pandas-2.2.2-cp312-cp312-win_amd64.whl", hash = "sha256:d187d355ecec3629624fccb01d104da7d7f391db0311145817525281e2804d23"},
+    {file = "pandas-2.2.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:0ca6377b8fca51815f382bd0b697a0814c8bda55115678cbc94c30aacbb6eff2"},
+    {file = "pandas-2.2.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9057e6aa78a584bc93a13f0a9bf7e753a5e9770a30b4d758b8d5f2a62a9433cd"},
+    {file = "pandas-2.2.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:001910ad31abc7bf06f49dcc903755d2f7f3a9186c0c040b827e522e9cef0863"},
+    {file = "pandas-2.2.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:66b479b0bd07204e37583c191535505410daa8df638fd8e75ae1b383851fe921"},
+    {file = "pandas-2.2.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:a77e9d1c386196879aa5eb712e77461aaee433e54c68cf253053a73b7e49c33a"},
+    {file = "pandas-2.2.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:92fd6b027924a7e178ac202cfbe25e53368db90d56872d20ffae94b96c7acc57"},
+    {file = "pandas-2.2.2-cp39-cp39-win_amd64.whl", hash = "sha256:640cef9aa381b60e296db324337a554aeeb883ead99dc8f6c18e81a93942f5f4"},
+    {file = "pandas-2.2.2.tar.gz", hash = "sha256:9e79019aba43cb4fda9e4d983f8e88ca0373adbb697ae9c6c43093218de28b54"},
+]
+
+[package.dependencies]
+numpy = [
+    {version = ">=1.22.4", markers = "python_version < \"3.11\""},
+    {version = ">=1.26.0", markers = "python_version >= \"3.12\""},
+    {version = ">=1.23.2", markers = "python_version == \"3.11\""},
+]
+python-dateutil = ">=2.8.2"
+pytz = ">=2020.1"
+tzdata = ">=2022.7"
+
+[package.extras]
+all = ["PyQt5 (>=5.15.9)", "SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)", "beautifulsoup4 (>=4.11.2)", "bottleneck (>=1.3.6)", "dataframe-api-compat (>=0.1.7)", "fastparquet (>=2022.12.0)", "fsspec (>=2022.11.0)", "gcsfs (>=2022.11.0)", "html5lib (>=1.1)", "hypothesis (>=6.46.1)", "jinja2 (>=3.1.2)", "lxml (>=4.9.2)", "matplotlib (>=3.6.3)", "numba (>=0.56.4)", "numexpr (>=2.8.4)", "odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "pandas-gbq (>=0.19.0)", "psycopg2 (>=2.9.6)", "pyarrow (>=10.0.1)", "pymysql (>=1.0.2)", "pyreadstat (>=1.2.0)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "qtpy (>=2.3.0)", "s3fs (>=2022.11.0)", "scipy (>=1.10.0)", "tables (>=3.8.0)", "tabulate (>=0.9.0)", "xarray (>=2022.12.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)", "zstandard (>=0.19.0)"]
+aws = ["s3fs (>=2022.11.0)"]
+clipboard = ["PyQt5 (>=5.15.9)", "qtpy (>=2.3.0)"]
+compression = ["zstandard (>=0.19.0)"]
+computation = ["scipy (>=1.10.0)", "xarray (>=2022.12.0)"]
+consortium-standard = ["dataframe-api-compat (>=0.1.7)"]
+excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)"]
+feather = ["pyarrow (>=10.0.1)"]
+fss = ["fsspec (>=2022.11.0)"]
+gcp = ["gcsfs (>=2022.11.0)", "pandas-gbq (>=0.19.0)"]
+hdf5 = ["tables (>=3.8.0)"]
+html = ["beautifulsoup4 (>=4.11.2)", "html5lib (>=1.1)", "lxml (>=4.9.2)"]
+mysql = ["SQLAlchemy (>=2.0.0)", "pymysql (>=1.0.2)"]
+output-formatting = ["jinja2 (>=3.1.2)", "tabulate (>=0.9.0)"]
+parquet = ["pyarrow (>=10.0.1)"]
+performance = ["bottleneck (>=1.3.6)", "numba (>=0.56.4)", "numexpr (>=2.8.4)"]
+plot = ["matplotlib (>=3.6.3)"]
+postgresql = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "psycopg2 (>=2.9.6)"]
+pyarrow = ["pyarrow (>=10.0.1)"]
+spss = ["pyreadstat (>=1.2.0)"]
+sql-other = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)"]
+test = ["hypothesis (>=6.46.1)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)"]
+xml = ["lxml (>=4.9.2)"]
+
 [[package]]
 name = "parameterized"
 version = "0.9.0"
@@ -2070,6 +2562,57 @@ files = [
     {file = "protobuf-4.25.4.tar.gz", hash = "sha256:0dc4a62cc4052a036ee2204d26fe4d835c62827c855c8a03f29fe6da146b380d"},
 ]
 
+[[package]]
+name = "pyarrow"
+version = "17.0.0"
+description = "Python library for Apache Arrow"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "pyarrow-17.0.0-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:a5c8b238d47e48812ee577ee20c9a2779e6a5904f1708ae240f53ecbee7c9f07"},
+    {file = "pyarrow-17.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:db023dc4c6cae1015de9e198d41250688383c3f9af8f565370ab2b4cb5f62655"},
+    {file = "pyarrow-17.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da1e060b3876faa11cee287839f9cc7cdc00649f475714b8680a05fd9071d545"},
+    {file = "pyarrow-17.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75c06d4624c0ad6674364bb46ef38c3132768139ddec1c56582dbac54f2663e2"},
+    {file = "pyarrow-17.0.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:fa3c246cc58cb5a4a5cb407a18f193354ea47dd0648194e6265bd24177982fe8"},
+    {file = "pyarrow-17.0.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:f7ae2de664e0b158d1607699a16a488de3d008ba99b3a7aa5de1cbc13574d047"},
+    {file = "pyarrow-17.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:5984f416552eea15fd9cee03da53542bf4cddaef5afecefb9aa8d1010c335087"},
+    {file = "pyarrow-17.0.0-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:1c8856e2ef09eb87ecf937104aacfa0708f22dfeb039c363ec99735190ffb977"},
+    {file = "pyarrow-17.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2e19f569567efcbbd42084e87f948778eb371d308e137a0f97afe19bb860ccb3"},
+    {file = "pyarrow-17.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b244dc8e08a23b3e352899a006a26ae7b4d0da7bb636872fa8f5884e70acf15"},
+    {file = "pyarrow-17.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0b72e87fe3e1db343995562f7fff8aee354b55ee83d13afba65400c178ab2597"},
+    {file = "pyarrow-17.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:dc5c31c37409dfbc5d014047817cb4ccd8c1ea25d19576acf1a001fe07f5b420"},
+    {file = "pyarrow-17.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:e3343cb1e88bc2ea605986d4b94948716edc7a8d14afd4e2c097232f729758b4"},
+    {file = "pyarrow-17.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:a27532c38f3de9eb3e90ecab63dfda948a8ca859a66e3a47f5f42d1e403c4d03"},
+    {file = "pyarrow-17.0.0-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:9b8a823cea605221e61f34859dcc03207e52e409ccf6354634143e23af7c8d22"},
+    {file = "pyarrow-17.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f1e70de6cb5790a50b01d2b686d54aaf73da01266850b05e3af2a1bc89e16053"},
+    {file = "pyarrow-17.0.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0071ce35788c6f9077ff9ecba4858108eebe2ea5a3f7cf2cf55ebc1dbc6ee24a"},
+    {file = "pyarrow-17.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:757074882f844411fcca735e39aae74248a1531367a7c80799b4266390ae51cc"},
+    {file = "pyarrow-17.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:9ba11c4f16976e89146781a83833df7f82077cdab7dc6232c897789343f7891a"},
+    {file = "pyarrow-17.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:b0c6ac301093b42d34410b187bba560b17c0330f64907bfa4f7f7f2444b0cf9b"},
+    {file = "pyarrow-17.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:392bc9feabc647338e6c89267635e111d71edad5fcffba204425a7c8d13610d7"},
+    {file = "pyarrow-17.0.0-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:af5ff82a04b2171415f1410cff7ebb79861afc5dae50be73ce06d6e870615204"},
+    {file = "pyarrow-17.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:edca18eaca89cd6382dfbcff3dd2d87633433043650c07375d095cd3517561d8"},
+    {file = "pyarrow-17.0.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7c7916bff914ac5d4a8fe25b7a25e432ff921e72f6f2b7547d1e325c1ad9d155"},
+    {file = "pyarrow-17.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f553ca691b9e94b202ff741bdd40f6ccb70cdd5fbf65c187af132f1317de6145"},
+    {file = "pyarrow-17.0.0-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:0cdb0e627c86c373205a2f94a510ac4376fdc523f8bb36beab2e7f204416163c"},
+    {file = "pyarrow-17.0.0-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:d7d192305d9d8bc9082d10f361fc70a73590a4c65cf31c3e6926cd72b76bc35c"},
+    {file = "pyarrow-17.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:02dae06ce212d8b3244dd3e7d12d9c4d3046945a5933d28026598e9dbbda1fca"},
+    {file = "pyarrow-17.0.0-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:13d7a460b412f31e4c0efa1148e1d29bdf18ad1411eb6757d38f8fbdcc8645fb"},
+    {file = "pyarrow-17.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9b564a51fbccfab5a04a80453e5ac6c9954a9c5ef2890d1bcf63741909c3f8df"},
+    {file = "pyarrow-17.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:32503827abbc5aadedfa235f5ece8c4f8f8b0a3cf01066bc8d29de7539532687"},
+    {file = "pyarrow-17.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a155acc7f154b9ffcc85497509bcd0d43efb80d6f733b0dc3bb14e281f131c8b"},
+    {file = "pyarrow-17.0.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:dec8d129254d0188a49f8a1fc99e0560dc1b85f60af729f47de4046015f9b0a5"},
+    {file = "pyarrow-17.0.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:a48ddf5c3c6a6c505904545c25a4ae13646ae1f8ba703c4df4a1bfe4f4006bda"},
+    {file = "pyarrow-17.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:42bf93249a083aca230ba7e2786c5f673507fa97bbd9725a1e2754715151a204"},
+    {file = "pyarrow-17.0.0.tar.gz", hash = "sha256:4beca9521ed2c0921c1023e68d097d0299b62c362639ea315572a58f3f50fd28"},
+]
+
+[package.dependencies]
+numpy = ">=1.16.6"
+
+[package.extras]
+test = ["cffi", "hypothesis", "pandas", "pytest", "pytz"]
+
 [[package]]
 name = "pyasn1"
 version = "0.6.0"
@@ -2299,6 +2842,17 @@ files = [
 [package.extras]
 cli = ["click (>=5.0)"]
 
+[[package]]
+name = "pytz"
+version = "2024.1"
+description = "World timezone definitions, modern and historical"
+optional = false
+python-versions = "*"
+files = [
+    {file = "pytz-2024.1-py2.py3-none-any.whl", hash = "sha256:328171f4e3623139da4983451950b28e95ac706e13f3f2630a879749e7a8b319"},
+    {file = "pytz-2024.1.tar.gz", hash = "sha256:2a29735ea9c18baf14b448846bde5a48030ed267578472d8955cd0e7443a9812"},
+]
+
 [[package]]
 name = "pyyaml"
 version = "6.0.1"
@@ -3042,6 +3596,17 @@ files = [
     {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"},
 ]
 
+[[package]]
+name = "tzdata"
+version = "2024.1"
+description = "Provider of IANA time zone data"
+optional = false
+python-versions = ">=2"
+files = [
+    {file = "tzdata-2024.1-py2.py3-none-any.whl", hash = "sha256:9068bc196136463f5245e51efda838afa15aaeca9903f49050dfa2679db4d252"},
+    {file = "tzdata-2024.1.tar.gz", hash = "sha256:2674120f8d891909751c38abcdfd386ac0a5a1127954fbc332af6b5ceae07efd"},
+]
+
 [[package]]
 name = "uritemplate"
 version = "4.1.1"
@@ -3123,6 +3688,241 @@ MarkupSafe = ">=2.1.1"
 [package.extras]
 watchdog = ["watchdog (>=2.3)"]
 
+[[package]]
+name = "xxhash"
+version = "3.5.0"
+description = "Python binding for xxHash"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "xxhash-3.5.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ece616532c499ee9afbb83078b1b952beffef121d989841f7f4b3dc5ac0fd212"},
+    {file = "xxhash-3.5.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3171f693dbc2cef6477054a665dc255d996646b4023fe56cb4db80e26f4cc520"},
+    {file = "xxhash-3.5.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7c5d3e570ef46adaf93fc81b44aca6002b5a4d8ca11bd0580c07eac537f36680"},
+    {file = "xxhash-3.5.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7cb29a034301e2982df8b1fe6328a84f4b676106a13e9135a0d7e0c3e9f806da"},
+    {file = "xxhash-3.5.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5d0d307d27099bb0cbeea7260eb39ed4fdb99c5542e21e94bb6fd29e49c57a23"},
+    {file = "xxhash-3.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0342aafd421795d740e514bc9858ebddfc705a75a8c5046ac56d85fe97bf196"},
+    {file = "xxhash-3.5.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3dbbd9892c5ebffeca1ed620cf0ade13eb55a0d8c84e0751a6653adc6ac40d0c"},
+    {file = "xxhash-3.5.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4cc2d67fdb4d057730c75a64c5923abfa17775ae234a71b0200346bfb0a7f482"},
+    {file = "xxhash-3.5.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:ec28adb204b759306a3d64358a5e5c07d7b1dd0ccbce04aa76cb9377b7b70296"},
+    {file = "xxhash-3.5.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:1328f6d8cca2b86acb14104e381225a3d7b42c92c4b86ceae814e5c400dbb415"},
+    {file = "xxhash-3.5.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:8d47ebd9f5d9607fd039c1fbf4994e3b071ea23eff42f4ecef246ab2b7334198"},
+    {file = "xxhash-3.5.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b96d559e0fcddd3343c510a0fe2b127fbff16bf346dd76280b82292567523442"},
+    {file = "xxhash-3.5.0-cp310-cp310-win32.whl", hash = "sha256:61c722ed8d49ac9bc26c7071eeaa1f6ff24053d553146d5df031802deffd03da"},
+    {file = "xxhash-3.5.0-cp310-cp310-win_amd64.whl", hash = "sha256:9bed5144c6923cc902cd14bb8963f2d5e034def4486ab0bbe1f58f03f042f9a9"},
+    {file = "xxhash-3.5.0-cp310-cp310-win_arm64.whl", hash = "sha256:893074d651cf25c1cc14e3bea4fceefd67f2921b1bb8e40fcfeba56820de80c6"},
+    {file = "xxhash-3.5.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:02c2e816896dc6f85922ced60097bcf6f008dedfc5073dcba32f9c8dd786f3c1"},
+    {file = "xxhash-3.5.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6027dcd885e21581e46d3c7f682cfb2b870942feeed58a21c29583512c3f09f8"},
+    {file = "xxhash-3.5.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1308fa542bbdbf2fa85e9e66b1077eea3a88bef38ee8a06270b4298a7a62a166"},
+    {file = "xxhash-3.5.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c28b2fdcee797e1c1961cd3bcd3d545cab22ad202c846235197935e1df2f8ef7"},
+    {file = "xxhash-3.5.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:924361811732ddad75ff23e90efd9ccfda4f664132feecb90895bade6a1b4623"},
+    {file = "xxhash-3.5.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89997aa1c4b6a5b1e5b588979d1da048a3c6f15e55c11d117a56b75c84531f5a"},
+    {file = "xxhash-3.5.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:685c4f4e8c59837de103344eb1c8a3851f670309eb5c361f746805c5471b8c88"},
+    {file = "xxhash-3.5.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:dbd2ecfbfee70bc1a4acb7461fa6af7748ec2ab08ac0fa298f281c51518f982c"},
+    {file = "xxhash-3.5.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:25b5a51dc3dfb20a10833c8eee25903fd2e14059e9afcd329c9da20609a307b2"},
+    {file = "xxhash-3.5.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a8fb786fb754ef6ff8c120cb96629fb518f8eb5a61a16aac3a979a9dbd40a084"},
+    {file = "xxhash-3.5.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:a905ad00ad1e1c34fe4e9d7c1d949ab09c6fa90c919860c1534ff479f40fd12d"},
+    {file = "xxhash-3.5.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:963be41bcd49f53af6d795f65c0da9b4cc518c0dd9c47145c98f61cb464f4839"},
+    {file = "xxhash-3.5.0-cp311-cp311-win32.whl", hash = "sha256:109b436096d0a2dd039c355fa3414160ec4d843dfecc64a14077332a00aeb7da"},
+    {file = "xxhash-3.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:b702f806693201ad6c0a05ddbbe4c8f359626d0b3305f766077d51388a6bac58"},
+    {file = "xxhash-3.5.0-cp311-cp311-win_arm64.whl", hash = "sha256:c4dcb4120d0cc3cc448624147dba64e9021b278c63e34a38789b688fd0da9bf3"},
+    {file = "xxhash-3.5.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:14470ace8bd3b5d51318782cd94e6f94431974f16cb3b8dc15d52f3b69df8e00"},
+    {file = "xxhash-3.5.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:59aa1203de1cb96dbeab595ded0ad0c0056bb2245ae11fac11c0ceea861382b9"},
+    {file = "xxhash-3.5.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:08424f6648526076e28fae6ea2806c0a7d504b9ef05ae61d196d571e5c879c84"},
+    {file = "xxhash-3.5.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:61a1ff00674879725b194695e17f23d3248998b843eb5e933007ca743310f793"},
+    {file = "xxhash-3.5.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f2f2c61bee5844d41c3eb015ac652a0229e901074951ae48581d58bfb2ba01be"},
+    {file = "xxhash-3.5.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d32a592cac88d18cc09a89172e1c32d7f2a6e516c3dfde1b9adb90ab5df54a6"},
+    {file = "xxhash-3.5.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:70dabf941dede727cca579e8c205e61121afc9b28516752fd65724be1355cc90"},
+    {file = "xxhash-3.5.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e5d0ddaca65ecca9c10dcf01730165fd858533d0be84c75c327487c37a906a27"},
+    {file = "xxhash-3.5.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:3e5b5e16c5a480fe5f59f56c30abdeba09ffd75da8d13f6b9b6fd224d0b4d0a2"},
+    {file = "xxhash-3.5.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:149b7914451eb154b3dfaa721315117ea1dac2cc55a01bfbd4df7c68c5dd683d"},
+    {file = "xxhash-3.5.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:eade977f5c96c677035ff39c56ac74d851b1cca7d607ab3d8f23c6b859379cab"},
+    {file = "xxhash-3.5.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fa9f547bd98f5553d03160967866a71056a60960be00356a15ecc44efb40ba8e"},
+    {file = "xxhash-3.5.0-cp312-cp312-win32.whl", hash = "sha256:f7b58d1fd3551b8c80a971199543379be1cee3d0d409e1f6d8b01c1a2eebf1f8"},
+    {file = "xxhash-3.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:fa0cafd3a2af231b4e113fba24a65d7922af91aeb23774a8b78228e6cd785e3e"},
+    {file = "xxhash-3.5.0-cp312-cp312-win_arm64.whl", hash = "sha256:586886c7e89cb9828bcd8a5686b12e161368e0064d040e225e72607b43858ba2"},
+    {file = "xxhash-3.5.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:37889a0d13b0b7d739cfc128b1c902f04e32de17b33d74b637ad42f1c55101f6"},
+    {file = "xxhash-3.5.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:97a662338797c660178e682f3bc180277b9569a59abfb5925e8620fba00b9fc5"},
+    {file = "xxhash-3.5.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7f85e0108d51092bdda90672476c7d909c04ada6923c14ff9d913c4f7dc8a3bc"},
+    {file = "xxhash-3.5.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cd2fd827b0ba763ac919440042302315c564fdb797294d86e8cdd4578e3bc7f3"},
+    {file = "xxhash-3.5.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:82085c2abec437abebf457c1d12fccb30cc8b3774a0814872511f0f0562c768c"},
+    {file = "xxhash-3.5.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:07fda5de378626e502b42b311b049848c2ef38784d0d67b6f30bb5008642f8eb"},
+    {file = "xxhash-3.5.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c279f0d2b34ef15f922b77966640ade58b4ccdfef1c4d94b20f2a364617a493f"},
+    {file = "xxhash-3.5.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:89e66ceed67b213dec5a773e2f7a9e8c58f64daeb38c7859d8815d2c89f39ad7"},
+    {file = "xxhash-3.5.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:bcd51708a633410737111e998ceb3b45d3dbc98c0931f743d9bb0a209033a326"},
+    {file = "xxhash-3.5.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:3ff2c0a34eae7df88c868be53a8dd56fbdf592109e21d4bfa092a27b0bf4a7bf"},
+    {file = "xxhash-3.5.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:4e28503dccc7d32e0b9817aa0cbfc1f45f563b2c995b7a66c4c8a0d232e840c7"},
+    {file = "xxhash-3.5.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a6c50017518329ed65a9e4829154626f008916d36295b6a3ba336e2458824c8c"},
+    {file = "xxhash-3.5.0-cp313-cp313-win32.whl", hash = "sha256:53a068fe70301ec30d868ece566ac90d873e3bb059cf83c32e76012c889b8637"},
+    {file = "xxhash-3.5.0-cp313-cp313-win_amd64.whl", hash = "sha256:80babcc30e7a1a484eab952d76a4f4673ff601f54d5142c26826502740e70b43"},
+    {file = "xxhash-3.5.0-cp313-cp313-win_arm64.whl", hash = "sha256:4811336f1ce11cac89dcbd18f3a25c527c16311709a89313c3acaf771def2d4b"},
+    {file = "xxhash-3.5.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:6e5f70f6dca1d3b09bccb7daf4e087075ff776e3da9ac870f86ca316736bb4aa"},
+    {file = "xxhash-3.5.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2e76e83efc7b443052dd1e585a76201e40b3411fe3da7af4fe434ec51b2f163b"},
+    {file = "xxhash-3.5.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:33eac61d0796ca0591f94548dcfe37bb193671e0c9bcf065789b5792f2eda644"},
+    {file = "xxhash-3.5.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0ec70a89be933ea49222fafc3999987d7899fc676f688dd12252509434636622"},
+    {file = "xxhash-3.5.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd86b8e7f703ec6ff4f351cfdb9f428955859537125904aa8c963604f2e9d3e7"},
+    {file = "xxhash-3.5.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0adfbd36003d9f86c8c97110039f7539b379f28656a04097e7434d3eaf9aa131"},
+    {file = "xxhash-3.5.0-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:63107013578c8a730419adc05608756c3fa640bdc6abe806c3123a49fb829f43"},
+    {file = "xxhash-3.5.0-cp37-cp37m-musllinux_1_2_i686.whl", hash = "sha256:683b94dbd1ca67557850b86423318a2e323511648f9f3f7b1840408a02b9a48c"},
+    {file = "xxhash-3.5.0-cp37-cp37m-musllinux_1_2_ppc64le.whl", hash = "sha256:5d2a01dcce81789cf4b12d478b5464632204f4c834dc2d064902ee27d2d1f0ee"},
+    {file = "xxhash-3.5.0-cp37-cp37m-musllinux_1_2_s390x.whl", hash = "sha256:a9d360a792cbcce2fe7b66b8d51274ec297c53cbc423401480e53b26161a290d"},
+    {file = "xxhash-3.5.0-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:f0b48edbebea1b7421a9c687c304f7b44d0677c46498a046079d445454504737"},
+    {file = "xxhash-3.5.0-cp37-cp37m-win32.whl", hash = "sha256:7ccb800c9418e438b44b060a32adeb8393764da7441eb52aa2aa195448935306"},
+    {file = "xxhash-3.5.0-cp37-cp37m-win_amd64.whl", hash = "sha256:c3bc7bf8cb8806f8d1c9bf149c18708cb1c406520097d6b0a73977460ea03602"},
+    {file = "xxhash-3.5.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:74752ecaa544657d88b1d1c94ae68031e364a4d47005a90288f3bab3da3c970f"},
+    {file = "xxhash-3.5.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:dee1316133c9b463aa81aca676bc506d3f80d8f65aeb0bba2b78d0b30c51d7bd"},
+    {file = "xxhash-3.5.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:602d339548d35a8579c6b013339fb34aee2df9b4e105f985443d2860e4d7ffaa"},
+    {file = "xxhash-3.5.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:695735deeddfb35da1677dbc16a083445360e37ff46d8ac5c6fcd64917ff9ade"},
+    {file = "xxhash-3.5.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1030a39ba01b0c519b1a82f80e8802630d16ab95dc3f2b2386a0b5c8ed5cbb10"},
+    {file = "xxhash-3.5.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a5bc08f33c4966f4eb6590d6ff3ceae76151ad744576b5fc6c4ba8edd459fdec"},
+    {file = "xxhash-3.5.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:160e0c19ee500482ddfb5d5570a0415f565d8ae2b3fd69c5dcfce8a58107b1c3"},
+    {file = "xxhash-3.5.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:f1abffa122452481a61c3551ab3c89d72238e279e517705b8b03847b1d93d738"},
+    {file = "xxhash-3.5.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:d5e9db7ef3ecbfc0b4733579cea45713a76852b002cf605420b12ef3ef1ec148"},
+    {file = "xxhash-3.5.0-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:23241ff6423378a731d84864bf923a41649dc67b144debd1077f02e6249a0d54"},
+    {file = "xxhash-3.5.0-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:82b833d5563fefd6fceafb1aed2f3f3ebe19f84760fdd289f8b926731c2e6e91"},
+    {file = "xxhash-3.5.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:0a80ad0ffd78bef9509eee27b4a29e56f5414b87fb01a888353e3d5bda7038bd"},
+    {file = "xxhash-3.5.0-cp38-cp38-win32.whl", hash = "sha256:50ac2184ffb1b999e11e27c7e3e70cc1139047e7ebc1aa95ed12f4269abe98d4"},
+    {file = "xxhash-3.5.0-cp38-cp38-win_amd64.whl", hash = "sha256:392f52ebbb932db566973693de48f15ce787cabd15cf6334e855ed22ea0be5b3"},
+    {file = "xxhash-3.5.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:bfc8cdd7f33d57f0468b0614ae634cc38ab9202c6957a60e31d285a71ebe0301"},
+    {file = "xxhash-3.5.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e0c48b6300cd0b0106bf49169c3e0536408dfbeb1ccb53180068a18b03c662ab"},
+    {file = "xxhash-3.5.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fe1a92cfbaa0a1253e339ccec42dbe6db262615e52df591b68726ab10338003f"},
+    {file = "xxhash-3.5.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:33513d6cc3ed3b559134fb307aae9bdd94d7e7c02907b37896a6c45ff9ce51bd"},
+    {file = "xxhash-3.5.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:eefc37f6138f522e771ac6db71a6d4838ec7933939676f3753eafd7d3f4c40bc"},
+    {file = "xxhash-3.5.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a606c8070ada8aa2a88e181773fa1ef17ba65ce5dd168b9d08038e2a61b33754"},
+    {file = "xxhash-3.5.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:42eca420c8fa072cc1dd62597635d140e78e384a79bb4944f825fbef8bfeeef6"},
+    {file = "xxhash-3.5.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:604253b2143e13218ff1ef0b59ce67f18b8bd1c4205d2ffda22b09b426386898"},
+    {file = "xxhash-3.5.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:6e93a5ad22f434d7876665444a97e713a8f60b5b1a3521e8df11b98309bff833"},
+    {file = "xxhash-3.5.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:7a46e1d6d2817ba8024de44c4fd79913a90e5f7265434cef97026215b7d30df6"},
+    {file = "xxhash-3.5.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:30eb2efe6503c379b7ab99c81ba4a779748e3830241f032ab46bd182bf5873af"},
+    {file = "xxhash-3.5.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:c8aa771ff2c13dd9cda8166d685d7333d389fae30a4d2bb39d63ab5775de8606"},
+    {file = "xxhash-3.5.0-cp39-cp39-win32.whl", hash = "sha256:5ed9ebc46f24cf91034544b26b131241b699edbfc99ec5e7f8f3d02d6eb7fba4"},
+    {file = "xxhash-3.5.0-cp39-cp39-win_amd64.whl", hash = "sha256:220f3f896c6b8d0316f63f16c077d52c412619e475f9372333474ee15133a558"},
+    {file = "xxhash-3.5.0-cp39-cp39-win_arm64.whl", hash = "sha256:a7b1d8315d9b5e9f89eb2933b73afae6ec9597a258d52190944437158b49d38e"},
+    {file = "xxhash-3.5.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:2014c5b3ff15e64feecb6b713af12093f75b7926049e26a580e94dcad3c73d8c"},
+    {file = "xxhash-3.5.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fab81ef75003eda96239a23eda4e4543cedc22e34c373edcaf744e721a163986"},
+    {file = "xxhash-3.5.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4e2febf914ace002132aa09169cc572e0d8959d0f305f93d5828c4836f9bc5a6"},
+    {file = "xxhash-3.5.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5d3a10609c51da2a1c0ea0293fc3968ca0a18bd73838455b5bca3069d7f8e32b"},
+    {file = "xxhash-3.5.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:5a74f23335b9689b66eb6dbe2a931a88fcd7a4c2cc4b1cb0edba8ce381c7a1da"},
+    {file = "xxhash-3.5.0-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:2b4154c00eb22e4d543f472cfca430e7962a0f1d0f3778334f2e08a7ba59363c"},
+    {file = "xxhash-3.5.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d30bbc1644f726b825b3278764240f449d75f1a8bdda892e641d4a688b1494ae"},
+    {file = "xxhash-3.5.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6fa0b72f2423e2aa53077e54a61c28e181d23effeaafd73fcb9c494e60930c8e"},
+    {file = "xxhash-3.5.0-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:13de2b76c1835399b2e419a296d5b38dc4855385d9e96916299170085ef72f57"},
+    {file = "xxhash-3.5.0-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:0691bfcc4f9c656bcb96cc5db94b4d75980b9d5589f2e59de790091028580837"},
+    {file = "xxhash-3.5.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:297595fe6138d4da2c8ce9e72a04d73e58725bb60f3a19048bc96ab2ff31c692"},
+    {file = "xxhash-3.5.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cc1276d369452040cbb943300dc8abeedab14245ea44056a2943183822513a18"},
+    {file = "xxhash-3.5.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2061188a1ba352fc699c82bff722f4baacb4b4b8b2f0c745d2001e56d0dfb514"},
+    {file = "xxhash-3.5.0-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:38c384c434021e4f62b8d9ba0bc9467e14d394893077e2c66d826243025e1f81"},
+    {file = "xxhash-3.5.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:e6a4dd644d72ab316b580a1c120b375890e4c52ec392d4aef3c63361ec4d77d1"},
+    {file = "xxhash-3.5.0-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:531af8845aaadcadf951b7e0c1345c6b9c68a990eeb74ff9acd8501a0ad6a1c9"},
+    {file = "xxhash-3.5.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7ce379bcaa9fcc00f19affa7773084dd09f5b59947b3fb47a1ceb0179f91aaa1"},
+    {file = "xxhash-3.5.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd1b2281d01723f076df3c8188f43f2472248a6b63118b036e641243656b1b0f"},
+    {file = "xxhash-3.5.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9c770750cc80e8694492244bca7251385188bc5597b6a39d98a9f30e8da984e0"},
+    {file = "xxhash-3.5.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:b150b8467852e1bd844387459aa6fbe11d7f38b56e901f9f3b3e6aba0d660240"},
+    {file = "xxhash-3.5.0.tar.gz", hash = "sha256:84f2caddf951c9cbf8dc2e22a89d4ccf5d86391ac6418fe81e3c67d0cf60b45f"},
+]
+
+[[package]]
+name = "yarl"
+version = "1.9.4"
+description = "Yet another URL library"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "yarl-1.9.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a8c1df72eb746f4136fe9a2e72b0c9dc1da1cbd23b5372f94b5820ff8ae30e0e"},
+    {file = "yarl-1.9.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a3a6ed1d525bfb91b3fc9b690c5a21bb52de28c018530ad85093cc488bee2dd2"},
+    {file = "yarl-1.9.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c38c9ddb6103ceae4e4498f9c08fac9b590c5c71b0370f98714768e22ac6fa66"},
+    {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d9e09c9d74f4566e905a0b8fa668c58109f7624db96a2171f21747abc7524234"},
+    {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b8477c1ee4bd47c57d49621a062121c3023609f7a13b8a46953eb6c9716ca392"},
+    {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d5ff2c858f5f6a42c2a8e751100f237c5e869cbde669a724f2062d4c4ef93551"},
+    {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:357495293086c5b6d34ca9616a43d329317feab7917518bc97a08f9e55648455"},
+    {file = "yarl-1.9.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:54525ae423d7b7a8ee81ba189f131054defdb122cde31ff17477951464c1691c"},
+    {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:801e9264d19643548651b9db361ce3287176671fb0117f96b5ac0ee1c3530d53"},
+    {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e516dc8baf7b380e6c1c26792610230f37147bb754d6426462ab115a02944385"},
+    {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:7d5aaac37d19b2904bb9dfe12cdb08c8443e7ba7d2852894ad448d4b8f442863"},
+    {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:54beabb809ffcacbd9d28ac57b0db46e42a6e341a030293fb3185c409e626b8b"},
+    {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bac8d525a8dbc2a1507ec731d2867025d11ceadcb4dd421423a5d42c56818541"},
+    {file = "yarl-1.9.4-cp310-cp310-win32.whl", hash = "sha256:7855426dfbddac81896b6e533ebefc0af2f132d4a47340cee6d22cac7190022d"},
+    {file = "yarl-1.9.4-cp310-cp310-win_amd64.whl", hash = "sha256:848cd2a1df56ddbffeb375535fb62c9d1645dde33ca4d51341378b3f5954429b"},
+    {file = "yarl-1.9.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:35a2b9396879ce32754bd457d31a51ff0a9d426fd9e0e3c33394bf4b9036b099"},
+    {file = "yarl-1.9.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4c7d56b293cc071e82532f70adcbd8b61909eec973ae9d2d1f9b233f3d943f2c"},
+    {file = "yarl-1.9.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d8a1c6c0be645c745a081c192e747c5de06e944a0d21245f4cf7c05e457c36e0"},
+    {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4b3c1ffe10069f655ea2d731808e76e0f452fc6c749bea04781daf18e6039525"},
+    {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:549d19c84c55d11687ddbd47eeb348a89df9cb30e1993f1b128f4685cd0ebbf8"},
+    {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a7409f968456111140c1c95301cadf071bd30a81cbd7ab829169fb9e3d72eae9"},
+    {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e23a6d84d9d1738dbc6e38167776107e63307dfc8ad108e580548d1f2c587f42"},
+    {file = "yarl-1.9.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d8b889777de69897406c9fb0b76cdf2fd0f31267861ae7501d93003d55f54fbe"},
+    {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:03caa9507d3d3c83bca08650678e25364e1843b484f19986a527630ca376ecce"},
+    {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:4e9035df8d0880b2f1c7f5031f33f69e071dfe72ee9310cfc76f7b605958ceb9"},
+    {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:c0ec0ed476f77db9fb29bca17f0a8fcc7bc97ad4c6c1d8959c507decb22e8572"},
+    {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:ee04010f26d5102399bd17f8df8bc38dc7ccd7701dc77f4a68c5b8d733406958"},
+    {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:49a180c2e0743d5d6e0b4d1a9e5f633c62eca3f8a86ba5dd3c471060e352ca98"},
+    {file = "yarl-1.9.4-cp311-cp311-win32.whl", hash = "sha256:81eb57278deb6098a5b62e88ad8281b2ba09f2f1147c4767522353eaa6260b31"},
+    {file = "yarl-1.9.4-cp311-cp311-win_amd64.whl", hash = "sha256:d1d2532b340b692880261c15aee4dc94dd22ca5d61b9db9a8a361953d36410b1"},
+    {file = "yarl-1.9.4-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0d2454f0aef65ea81037759be5ca9947539667eecebca092733b2eb43c965a81"},
+    {file = "yarl-1.9.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:44d8ffbb9c06e5a7f529f38f53eda23e50d1ed33c6c869e01481d3fafa6b8142"},
+    {file = "yarl-1.9.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:aaaea1e536f98754a6e5c56091baa1b6ce2f2700cc4a00b0d49eca8dea471074"},
+    {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3777ce5536d17989c91696db1d459574e9a9bd37660ea7ee4d3344579bb6f129"},
+    {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9fc5fc1eeb029757349ad26bbc5880557389a03fa6ada41703db5e068881e5f2"},
+    {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ea65804b5dc88dacd4a40279af0cdadcfe74b3e5b4c897aa0d81cf86927fee78"},
+    {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa102d6d280a5455ad6a0f9e6d769989638718e938a6a0a2ff3f4a7ff8c62cc4"},
+    {file = "yarl-1.9.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:09efe4615ada057ba2d30df871d2f668af661e971dfeedf0c159927d48bbeff0"},
+    {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:008d3e808d03ef28542372d01057fd09168419cdc8f848efe2804f894ae03e51"},
+    {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:6f5cb257bc2ec58f437da2b37a8cd48f666db96d47b8a3115c29f316313654ff"},
+    {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:992f18e0ea248ee03b5a6e8b3b4738850ae7dbb172cc41c966462801cbf62cf7"},
+    {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:0e9d124c191d5b881060a9e5060627694c3bdd1fe24c5eecc8d5d7d0eb6faabc"},
+    {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:3986b6f41ad22988e53d5778f91855dc0399b043fc8946d4f2e68af22ee9ff10"},
+    {file = "yarl-1.9.4-cp312-cp312-win32.whl", hash = "sha256:4b21516d181cd77ebd06ce160ef8cc2a5e9ad35fb1c5930882baff5ac865eee7"},
+    {file = "yarl-1.9.4-cp312-cp312-win_amd64.whl", hash = "sha256:a9bd00dc3bc395a662900f33f74feb3e757429e545d831eef5bb280252631984"},
+    {file = "yarl-1.9.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:63b20738b5aac74e239622d2fe30df4fca4942a86e31bf47a81a0e94c14df94f"},
+    {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7d7f7de27b8944f1fee2c26a88b4dabc2409d2fea7a9ed3df79b67277644e17"},
+    {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c74018551e31269d56fab81a728f683667e7c28c04e807ba08f8c9e3bba32f14"},
+    {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ca06675212f94e7a610e85ca36948bb8fc023e458dd6c63ef71abfd482481aa5"},
+    {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5aef935237d60a51a62b86249839b51345f47564208c6ee615ed2a40878dccdd"},
+    {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2b134fd795e2322b7684155b7855cc99409d10b2e408056db2b93b51a52accc7"},
+    {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:d25039a474c4c72a5ad4b52495056f843a7ff07b632c1b92ea9043a3d9950f6e"},
+    {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:f7d6b36dd2e029b6bcb8a13cf19664c7b8e19ab3a58e0fefbb5b8461447ed5ec"},
+    {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:957b4774373cf6f709359e5c8c4a0af9f6d7875db657adb0feaf8d6cb3c3964c"},
+    {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:d7eeb6d22331e2fd42fce928a81c697c9ee2d51400bd1a28803965883e13cead"},
+    {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:6a962e04b8f91f8c4e5917e518d17958e3bdee71fd1d8b88cdce74dd0ebbf434"},
+    {file = "yarl-1.9.4-cp37-cp37m-win32.whl", hash = "sha256:f3bc6af6e2b8f92eced34ef6a96ffb248e863af20ef4fde9448cc8c9b858b749"},
+    {file = "yarl-1.9.4-cp37-cp37m-win_amd64.whl", hash = "sha256:ad4d7a90a92e528aadf4965d685c17dacff3df282db1121136c382dc0b6014d2"},
+    {file = "yarl-1.9.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:ec61d826d80fc293ed46c9dd26995921e3a82146feacd952ef0757236fc137be"},
+    {file = "yarl-1.9.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8be9e837ea9113676e5754b43b940b50cce76d9ed7d2461df1af39a8ee674d9f"},
+    {file = "yarl-1.9.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:bef596fdaa8f26e3d66af846bbe77057237cb6e8efff8cd7cc8dff9a62278bbf"},
+    {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2d47552b6e52c3319fede1b60b3de120fe83bde9b7bddad11a69fb0af7db32f1"},
+    {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:84fc30f71689d7fc9168b92788abc977dc8cefa806909565fc2951d02f6b7d57"},
+    {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4aa9741085f635934f3a2583e16fcf62ba835719a8b2b28fb2917bb0537c1dfa"},
+    {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:206a55215e6d05dbc6c98ce598a59e6fbd0c493e2de4ea6cc2f4934d5a18d130"},
+    {file = "yarl-1.9.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:07574b007ee20e5c375a8fe4a0789fad26db905f9813be0f9fef5a68080de559"},
+    {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5a2e2433eb9344a163aced6a5f6c9222c0786e5a9e9cac2c89f0b28433f56e23"},
+    {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:6ad6d10ed9b67a382b45f29ea028f92d25bc0bc1daf6c5b801b90b5aa70fb9ec"},
+    {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:6fe79f998a4052d79e1c30eeb7d6c1c1056ad33300f682465e1b4e9b5a188b78"},
+    {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:a825ec844298c791fd28ed14ed1bffc56a98d15b8c58a20e0e08c1f5f2bea1be"},
+    {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8619d6915b3b0b34420cf9b2bb6d81ef59d984cb0fde7544e9ece32b4b3043c3"},
+    {file = "yarl-1.9.4-cp38-cp38-win32.whl", hash = "sha256:686a0c2f85f83463272ddffd4deb5e591c98aac1897d65e92319f729c320eece"},
+    {file = "yarl-1.9.4-cp38-cp38-win_amd64.whl", hash = "sha256:a00862fb23195b6b8322f7d781b0dc1d82cb3bcac346d1e38689370cc1cc398b"},
+    {file = "yarl-1.9.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:604f31d97fa493083ea21bd9b92c419012531c4e17ea6da0f65cacdcf5d0bd27"},
+    {file = "yarl-1.9.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8a854227cf581330ffa2c4824d96e52ee621dd571078a252c25e3a3b3d94a1b1"},
+    {file = "yarl-1.9.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ba6f52cbc7809cd8d74604cce9c14868306ae4aa0282016b641c661f981a6e91"},
+    {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a6327976c7c2f4ee6816eff196e25385ccc02cb81427952414a64811037bbc8b"},
+    {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8397a3817d7dcdd14bb266283cd1d6fc7264a48c186b986f32e86d86d35fbac5"},
+    {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e0381b4ce23ff92f8170080c97678040fc5b08da85e9e292292aba67fdac6c34"},
+    {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:23d32a2594cb5d565d358a92e151315d1b2268bc10f4610d098f96b147370136"},
+    {file = "yarl-1.9.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ddb2a5c08a4eaaba605340fdee8fc08e406c56617566d9643ad8bf6852778fc7"},
+    {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:26a1dc6285e03f3cc9e839a2da83bcbf31dcb0d004c72d0730e755b33466c30e"},
+    {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:18580f672e44ce1238b82f7fb87d727c4a131f3a9d33a5e0e82b793362bf18b4"},
+    {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:29e0f83f37610f173eb7e7b5562dd71467993495e568e708d99e9d1944f561ec"},
+    {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:1f23e4fe1e8794f74b6027d7cf19dc25f8b63af1483d91d595d4a07eca1fb26c"},
+    {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:db8e58b9d79200c76956cefd14d5c90af54416ff5353c5bfd7cbe58818e26ef0"},
+    {file = "yarl-1.9.4-cp39-cp39-win32.whl", hash = "sha256:c7224cab95645c7ab53791022ae77a4509472613e839dab722a72abe5a684575"},
+    {file = "yarl-1.9.4-cp39-cp39-win_amd64.whl", hash = "sha256:824d6c50492add5da9374875ce72db7a0733b29c2394890aef23d533106e2b15"},
+    {file = "yarl-1.9.4-py3-none-any.whl", hash = "sha256:928cecb0ef9d5a7946eb6ff58417ad2fe9375762382f1bf5c55e61645f2c43ad"},
+    {file = "yarl-1.9.4.tar.gz", hash = "sha256:566db86717cf8080b99b58b083b773a908ae40f06681e87e589a976faf8246bf"},
+]
+
+[package.dependencies]
+idna = ">=2.0"
+multidict = ">=4.0"
+
 [[package]]
 name = "zipp"
 version = "3.19.2"
@@ -3153,4 +3953,4 @@ torch = ["torch"]
 [metadata]
 lock-version = "2.0"
 python-versions = ">=3.9, <4.0"
-content-hash = "d3db0807e4c11604fb8ec1326ec1f2fc5264765a5bf673f5351e1a30f75b98be"
+content-hash = "2738134e77ae589eba11e3bc0a2b15491ce297f655f0cb0d79e28fcb7f248a33"
diff --git a/adalflow/pyproject.toml b/adalflow/pyproject.toml
index 3cf1148e..03171615 100644
--- a/adalflow/pyproject.toml
+++ b/adalflow/pyproject.toml
@@ -44,6 +44,7 @@ tqdm = "^4.66.4"
 pyyaml = "^6.0.1"
 nest-asyncio = "^1.6.0"
 botocore = "^1.34.149"   # if not specified, it will fail to resolve the dependency
+diskcache = "^5.6.3"
 
 
 # Optional dependencies
@@ -57,7 +58,6 @@ anthropic = { version = "^0.31.1", optional = true }
 google-generativeai = { version = "^0.7.2", optional = true }
 cohere = { version = "^5.5.8", optional = true }
 ollama = { version = "^0.2.1", optional = true }
-diskcache = "^5.6.3"
 
 
 [tool.poetry.group.test.dependencies]
@@ -87,6 +87,9 @@ matplotlib = "^3.9.1"
 diskcache = "^5.6.3"
 
 
+[tool.poetry.group.extra.dependencies]
+datasets = "^2.21.0"
+
 [tool.poetry.extras] # allow pip install adalflow[openai, groq]
 openai = ["openai"]
 groq = ["groq"]
@@ -98,6 +101,7 @@ faiss-cpu = ["faiss-cpu"]
 sqlalchemy = ["sqlalchemy"]
 torch = ["torch"]
 ollama = ["ollama"]
+datasets = ["datasets"]
 
 
 # [[tool.poetry.source]]
diff --git a/docs/source/get_started/adalflow_in_15mins.rst b/docs/source/get_started/adalflow_in_15mins.rst
index 16d46e87..3c88f5f8 100644
--- a/docs/source/get_started/adalflow_in_15mins.rst
+++ b/docs/source/get_started/adalflow_in_15mins.rst
@@ -1,3 +1,15 @@
+.. raw:: html
+
+   <div style="display: flex; justify-content: flex-start; align-items: center; margin-bottom: 20px;">
+      <a href="https://colab.research.google.com/drive/1n3mHUWekTEYHiBdYBTw43TKlPN41A9za?usp=sharing" target="_blank" style="margin-right: 10px;">
+         <img alt="Try Quickstart in Colab" src="https://colab.research.google.com/assets/colab-badge.svg" style="vertical-align: middle;">
+      </a>
+      <a href="https://github.com/SylphAI-Inc/AdalFlow/tree/main/use_cases/question_answering/bhh_object_count" target="_blank" style="display: flex; align-items: center;">
+         <img src="https://github.githubassets.com/images/modules/logos_page/GitHub-Mark.png" alt="GitHub" style="height: 20px; width: 20px; margin-right: 5px;">
+         <span style="vertical-align: middle;"> Open Source Code</span>
+      </a>
+   </div>
+
 Learn AdalFlow in 15 minutes
 =============================
 
diff --git a/docs/source/use_cases/question_answering.rst b/docs/source/use_cases/question_answering.rst
index 430b19cd..e2e0029e 100644
--- a/docs/source/use_cases/question_answering.rst
+++ b/docs/source/use_cases/question_answering.rst
@@ -1,3 +1,16 @@
+.. raw:: html
+
+   <div style="display: flex; justify-content: flex-start; align-items: center; margin-bottom: 20px;">
+      <a href="https://colab.research.google.com/drive/1n3mHUWekTEYHiBdYBTw43TKlPN41A9za?usp=sharing" target="_blank" style="margin-right: 10px;">
+         <img alt="Try Quickstart in Colab" src="https://colab.research.google.com/assets/colab-badge.svg" style="vertical-align: middle;">
+      </a>
+      <a href="https://github.com/SylphAI-Inc/AdalFlow/tree/main/use_cases/question_answering/bhh_object_count" target="_blank" style="display: flex; align-items: center;">
+         <img src="https://github.githubassets.com/images/modules/logos_page/GitHub-Mark.png" alt="GitHub" style="height: 20px; width: 20px; margin-right: 5px;">
+         <span style="vertical-align: middle;"> Open Source Code</span>
+      </a>
+   </div>
+
+
 Introduction to AdalFlow library
 ===============================
 
diff --git a/poetry.lock b/poetry.lock
index 4cd24a29..7f3d0a10 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -3187,8 +3187,8 @@ files = [
 
 [package.dependencies]
 numpy = [
-    {version = ">=1.23.2", markers = "python_version == \"3.11\""},
     {version = ">=1.26.0", markers = "python_version >= \"3.12\""},
+    {version = ">=1.23.2", markers = "python_version == \"3.11\""},
 ]
 python-dateutil = ">=2.8.2"
 pytz = ">=2020.1"
@@ -3607,8 +3607,8 @@ files = [
 annotated-types = ">=0.4.0"
 pydantic-core = "2.20.1"
 typing-extensions = [
-    {version = ">=4.6.1", markers = "python_version < \"3.13\""},
     {version = ">=4.12.2", markers = "python_version >= \"3.13\""},
+    {version = ">=4.6.1", markers = "python_version < \"3.13\""},
 ]
 
 [package.extras]

From 60a0fe927585cd974d7821bc1d51c99ebe030cc0 Mon Sep 17 00:00:00 2001
From: Li Yin <li.yin.gravity@gmail.com>
Date: Mon, 26 Aug 2024 14:35:22 -0700
Subject: [PATCH 3/8] restructure

---
 README.md                                     |   3 +-
 adalflow/adalflow/datasets/big_bench_hard.py  |  19 +-
 benchmarks/hotpot_qa/adal_train.py            |   2 +-
 use_cases/classification/prepare_for_train.py |   2 +-
 use_cases/classification/train.py             |   2 +-
 .../classification/train_string_output.py     |   2 +-
 .../trec_task_structured_output.py            |   8 -
 .../bhh_object_count => }/config.py           |   0
 .../{ => bbh}/bhh_object_count/__init__.py    |   0
 .../{ => bbh}/bhh_object_count/data.py        |  10 -
 .../{ => bbh}/bhh_object_count/diagnose.py    |   2 +-
 .../bbh/bhh_object_count/task.py              |  93 ++++++
 .../{ => bbh}/bhh_object_count/train_new.py   |   2 +-
 .../bbh/word_sorting/data.py                  | 101 +++++++
 .../bbh/word_sorting/diagnose.py              |  69 +++++
 .../bbh/word_sorting/task.py                  |  93 ++++++
 .../bhh_object_count/task.py                  | 268 ------------------
 use_cases/{ => unsorted}/fewshot_qa.py        |   0
 use_cases/{ => unsorted}/llm_as_retriever.py  |   0
 use_cases/{ => unsorted}/rag.py               |   0
 use_cases/{ => unsorted}/rag_hotpotqa.py      |   0
 use_cases/{ => unsorted}/rag_optimized.py     |   0
 use_cases/{ => unsorted}/rag_yaml_config.py   |   0
 use_cases/{ => unsorted}/retriever_llm.py     |   0
 use_cases/{ => unsorted}/simple_qa.py         |   0
 .../{ => unsorted}/simple_qa_anthropic.py     |   0
 use_cases/{ => unsorted}/simple_qa_google.py  |   0
 use_cases/{ => unsorted}/simple_qa_groq.py    |   0
 use_cases/{ => unsorted}/simple_qa_memory.py  |   0
 use_cases/{ => unsorted}/simple_qa_stream.py  |   0
 .../{ => unsorted}/simple_qa_trainable.py     |   0
 use_cases/{ => unsorted}/simple_rag_bm_25.py  |   0
 use_cases/{ => unsorted}/use_embedder.py      |   0
 use_cases/{ => unsorted}/yaml_output.py       |   0
 34 files changed, 380 insertions(+), 296 deletions(-)
 rename use_cases/{question_answering/bhh_object_count => }/config.py (100%)
 rename use_cases/question_answering/{ => bbh}/bhh_object_count/__init__.py (100%)
 rename use_cases/question_answering/{ => bbh}/bhh_object_count/data.py (93%)
 rename use_cases/question_answering/{ => bbh}/bhh_object_count/diagnose.py (97%)
 create mode 100644 use_cases/question_answering/bbh/bhh_object_count/task.py
 rename use_cases/question_answering/{ => bbh}/bhh_object_count/train_new.py (98%)
 create mode 100644 use_cases/question_answering/bbh/word_sorting/data.py
 create mode 100644 use_cases/question_answering/bbh/word_sorting/diagnose.py
 create mode 100644 use_cases/question_answering/bbh/word_sorting/task.py
 delete mode 100644 use_cases/question_answering/bhh_object_count/task.py
 rename use_cases/{ => unsorted}/fewshot_qa.py (100%)
 rename use_cases/{ => unsorted}/llm_as_retriever.py (100%)
 rename use_cases/{ => unsorted}/rag.py (100%)
 rename use_cases/{ => unsorted}/rag_hotpotqa.py (100%)
 rename use_cases/{ => unsorted}/rag_optimized.py (100%)
 rename use_cases/{ => unsorted}/rag_yaml_config.py (100%)
 rename use_cases/{ => unsorted}/retriever_llm.py (100%)
 rename use_cases/{ => unsorted}/simple_qa.py (100%)
 rename use_cases/{ => unsorted}/simple_qa_anthropic.py (100%)
 rename use_cases/{ => unsorted}/simple_qa_google.py (100%)
 rename use_cases/{ => unsorted}/simple_qa_groq.py (100%)
 rename use_cases/{ => unsorted}/simple_qa_memory.py (100%)
 rename use_cases/{ => unsorted}/simple_qa_stream.py (100%)
 rename use_cases/{ => unsorted}/simple_qa_trainable.py (100%)
 rename use_cases/{ => unsorted}/simple_rag_bm_25.py (100%)
 rename use_cases/{ => unsorted}/use_embedder.py (100%)
 rename use_cases/{ => unsorted}/yaml_output.py (100%)

diff --git a/README.md b/README.md
index 88778c87..9e0a6c78 100644
--- a/README.md
+++ b/README.md
@@ -214,13 +214,14 @@ AdalFlow is named in honor of [Ada Lovelace](https://en.wikipedia.org/wiki/Ada_L
 
 # Acknowledgements
 
-Many existing works greatly inspired this project! Here is a non-exhaustive list:
+Many existing works greatly inspired AdalFlow library! Here is a non-exhaustive list:
 
 - 📚 [PyTorch](https://github.com/pytorch/pytorch/) for design philosophy and design pattern of ``Component``, ``Parameter``, ``Sequential``.
 - 📚 [Micrograd](https://github.com/karpathy/micrograd): A tiny autograd engine for our auto-differentiative architecture.
 - 📚 [Text-Grad](https://github.com/zou-group/textgrad) for the ``Textual Gradient Descent`` text optimizer.
 - 📚 [DSPy](https://github.com/stanfordnlp/dspy) for inspiring the ``__{input/output}__fields`` in our ``DataClass`` and the bootstrap few-shot optimizer.
 - 📚 [ORPO](https://github.com/google-deepmind/opro) for adding past text instruction along with its accuracy in the text optimizer.
+- 📚 [PyTorch Lightning](https://github.com/Lightning-AI/pytorch-lightning) for the ``AdalComponent`` and ``Trainer``.
 
 # Citation
 
diff --git a/adalflow/adalflow/datasets/big_bench_hard.py b/adalflow/adalflow/datasets/big_bench_hard.py
index c566f6f6..d706ac1e 100644
--- a/adalflow/adalflow/datasets/big_bench_hard.py
+++ b/adalflow/adalflow/datasets/big_bench_hard.py
@@ -15,6 +15,10 @@
 class BigBenchHard(Dataset):
     __doc__ = """Big Bench Hard dataset for object counting task.
 
+    You can find the task name from the following link:
+    https://github.com/suzgunmirac/BIG-Bench-Hard/tree/main/bbh
+
+
     Data will be saved to ~/.adalflow/cache_datasets/BBH_object_counting/{split}.csv
     if root is not specified.
 
@@ -22,6 +26,11 @@ class BigBenchHard(Dataset):
     - train: 50 examples
     - val: 50 examples
     - test: 100 examples
+
+    Args:
+        task_name (str): The name of the task. "BHH_{task_name}" is the task name in the dataset.
+        root (str, optional): Root directory of the dataset to save the data. Defaults to ~/.adalflow/cache_datasets/task_name.
+        split (str, optional): The dataset split, supports ``"train"`` (default), ``"val"`` and ``"test"``.
     """
 
     def __init__(
@@ -81,6 +90,9 @@ def _check_or_download_dataset(self, data_path: str = None, split: str = "train"
 
         examples = data["examples"]
 
+        # NOTE: better to shuffle the examples before splitting.
+        # We do this splitting in order to be consistent with text-grad paper.
+
         train_examples = [
             {"x": ex["input"], "y": ex["target"], "id": str(uuid.uuid4())}
             for ex in examples[:50]
@@ -91,8 +103,9 @@ def _check_or_download_dataset(self, data_path: str = None, split: str = "train"
         ]
         test_examples = [
             {"x": ex["input"], "y": ex["target"], "id": str(uuid.uuid4())}
-            for ex in examples[100:200]
+            for ex in examples[150:250]
         ]
+        # ensure the
 
         for split, examples in zip(
             ["train", "val", "test"], [train_examples, val_examples, test_examples]
@@ -114,7 +127,7 @@ def get_default_task_instruction(self):
 if __name__ == "__main__":
     from adalflow.datasets.big_bench_hard import BigBenchHard
 
-    dataset = BigBenchHard("BBH_object_counting", split="train")
-    print(dataset[0])
+    dataset = BigBenchHard("BBH_word_sorting", split="train")
+    print(dataset[0:10])
     print(len(dataset))
     print(dataset.get_default_task_instruction())
diff --git a/benchmarks/hotpot_qa/adal_train.py b/benchmarks/hotpot_qa/adal_train.py
index 616670b0..5d4a54ec 100644
--- a/benchmarks/hotpot_qa/adal_train.py
+++ b/benchmarks/hotpot_qa/adal_train.py
@@ -501,7 +501,7 @@ def validate_dspy_demos(
     # get_logger(level="DEBUG")
     trainset, valset, testset = load_datasets()
 
-    from use_cases.question_answering.bhh_object_count.config import (
+    from LightRAG.use_cases.config import (
         gpt_3_model,
         gpt_4o_model,
     )
diff --git a/use_cases/classification/prepare_for_train.py b/use_cases/classification/prepare_for_train.py
index f929c2b9..abbdd15e 100644
--- a/use_cases/classification/prepare_for_train.py
+++ b/use_cases/classification/prepare_for_train.py
@@ -85,7 +85,7 @@ def diagnose_string_output(
 
 
 if __name__ == "__main__":
-    from use_cases.question_answering.bhh_object_count.config import (
+    from LightRAG.use_cases.config import (
         gpt_4o_model,
     )
 
diff --git a/use_cases/classification/train.py b/use_cases/classification/train.py
index 5b34d125..95616783 100644
--- a/use_cases/classification/train.py
+++ b/use_cases/classification/train.py
@@ -7,7 +7,7 @@
 from use_cases.classification.data import load_datasets, TRECExtendedData
 
 from adalflow.eval.answer_match_acc import AnswerMatchAcc
-from use_cases.question_answering.bhh_object_count.config import (
+from LightRAG.use_cases.config import (
     gpt_3_model,
     gpt_4o_model,
 )
diff --git a/use_cases/classification/train_string_output.py b/use_cases/classification/train_string_output.py
index bf4b8f33..9ecdef27 100644
--- a/use_cases/classification/train_string_output.py
+++ b/use_cases/classification/train_string_output.py
@@ -7,7 +7,7 @@
 from use_cases.classification.data import load_datasets, TRECExtendedData
 
 from adalflow.eval.answer_match_acc import AnswerMatchAcc
-from use_cases.question_answering.bhh_object_count.config import (
+from LightRAG.use_cases.config import (
     gpt_3_model,
     gpt_4o_model,
 )
diff --git a/use_cases/classification/trec_task_structured_output.py b/use_cases/classification/trec_task_structured_output.py
index 17066ebf..98e3c184 100644
--- a/use_cases/classification/trec_task_structured_output.py
+++ b/use_cases/classification/trec_task_structured_output.py
@@ -51,14 +51,6 @@ def __init__(self, model_client: adal.ModelClient, model_kwargs: Dict):
         )
 
         prompt_kwargs = {
-            # "system_prompt": adal.Parameter(
-            #     data=self.parser.get_task_desc_str()
-            #     + "\n"
-            #     + self.parser.get_output_format_str(),
-            #     role_desc="Task description with output format requirements",
-            #     requires_opt=True,
-            #     param_type=adal.ParameterType.PROMPT,
-            # ),
             # NOTE: when the instruction is too long,
             # it is better to split it into two prompts it is more effective at training
             # 0.8056 val, 0.903 test
diff --git a/use_cases/question_answering/bhh_object_count/config.py b/use_cases/config.py
similarity index 100%
rename from use_cases/question_answering/bhh_object_count/config.py
rename to use_cases/config.py
diff --git a/use_cases/question_answering/bhh_object_count/__init__.py b/use_cases/question_answering/bbh/bhh_object_count/__init__.py
similarity index 100%
rename from use_cases/question_answering/bhh_object_count/__init__.py
rename to use_cases/question_answering/bbh/bhh_object_count/__init__.py
diff --git a/use_cases/question_answering/bhh_object_count/data.py b/use_cases/question_answering/bbh/bhh_object_count/data.py
similarity index 93%
rename from use_cases/question_answering/bhh_object_count/data.py
rename to use_cases/question_answering/bbh/bhh_object_count/data.py
index 52f02c00..aeac5fbc 100644
--- a/use_cases/question_answering/bhh_object_count/data.py
+++ b/use_cases/question_answering/bbh/bhh_object_count/data.py
@@ -8,16 +8,6 @@
 from adalflow.utils.data import subset_dataset
 
 
-@dataclass
-class ObjectCountPredData(DataClass):
-    """Dataclass for structed prediction"""
-
-    thought: str = field(metadata={"desc": "List your step by step reasoning"})
-    answer: int = field(
-        metadata={"desc": "The answer to the question, only numerical values"}
-    )
-
-
 # TODO: user dont have to specify, we can auto generate a dataclass
 @dataclass
 class ObjectCountSimple(DataClass):
diff --git a/use_cases/question_answering/bhh_object_count/diagnose.py b/use_cases/question_answering/bbh/bhh_object_count/diagnose.py
similarity index 97%
rename from use_cases/question_answering/bhh_object_count/diagnose.py
rename to use_cases/question_answering/bbh/bhh_object_count/diagnose.py
index b877358d..1530d714 100644
--- a/use_cases/question_answering/bhh_object_count/diagnose.py
+++ b/use_cases/question_answering/bbh/bhh_object_count/diagnose.py
@@ -43,7 +43,7 @@ def diagnose(
 
 
 if __name__ == "__main__":
-    from use_cases.question_answering.bhh_object_count.config import (
+    from LightRAG.use_cases.config import (
         gpt_3_model,
     )
 
diff --git a/use_cases/question_answering/bbh/bhh_object_count/task.py b/use_cases/question_answering/bbh/bhh_object_count/task.py
new file mode 100644
index 00000000..8fcaeff1
--- /dev/null
+++ b/use_cases/question_answering/bbh/bhh_object_count/task.py
@@ -0,0 +1,93 @@
+"""Prepare the task pipeline"""
+
+from adalflow.optim.parameter import ParameterType
+
+from use_cases.question_answering.bhh_object_count.data import (
+    parse_integer_answer,
+)
+
+
+# Few shot demonstration can be less effective when performance already high
+few_shot_template = r"""<START_OF_SYSTEM_PROMPT>
+{{system_prompt}}
+{# Few shot demos #}
+{% if few_shot_demos is not none %}
+Here are some examples:
+{{few_shot_demos}}
+{% endif %}
+<END_OF_SYSTEM_PROMPT>
+<START_OF_USER>
+{{input_str}}
+<END_OF_USER>
+"""
+
+from typing import Dict, Union
+import adalflow as adal
+
+
+class ObjectCountTaskPipeline(adal.Component):
+    def __init__(self, model_client: adal.ModelClient, model_kwargs: Dict):
+        super().__init__()
+
+        system_prompt = adal.Parameter(
+            # data="You will answer a reasoning question. Think step by step. The last line of your response should be of the following format: 'Answer: $VALUE' where VALUE is a numerical value.",
+            data="You will answer a reasoning question. Think step by step. The last line of your response should be of the following format: 'Answer: $VALUE' where VALUE is a numerical value.",
+            role_desc="To give task instruction to the language model in the system prompt",
+            requires_opt=True,
+            param_type=ParameterType.PROMPT,
+            instruction_to_optimizer="You can try to show examples to see if it helps.",
+        )
+        few_shot_demos = adal.Parameter(
+            data=None,
+            role_desc="To provide few shot demos to the language model",
+            requires_opt=True,
+            param_type=ParameterType.DEMOS,
+        )
+
+        self.llm_counter = adal.Generator(
+            model_client=model_client,
+            model_kwargs=model_kwargs,
+            template=few_shot_template,
+            prompt_kwargs={
+                "system_prompt": system_prompt,
+                "few_shot_demos": few_shot_demos,
+            },
+            output_processors=parse_integer_answer,
+            use_cache=True,
+        )
+
+    def call(
+        self, question: str, id: str = None
+    ) -> Union[adal.GeneratorOutput, adal.Parameter]:
+        output = self.llm_counter(prompt_kwargs={"input_str": question}, id=id)
+        return output
+
+
+def test_object_count_task():
+    from LightRAG.use_cases.config import gpt_3_model
+
+    question = "I have a flute, a piano, a trombone, four stoves, a violin, an accordion, a clarinet, a drum, two lamps, and a trumpet. How many musical instruments do I have?"
+    task_pipeline = ObjectCountTaskPipeline(**gpt_3_model)
+    print(task_pipeline)
+
+    answer = task_pipeline(question)
+    print(answer)
+
+    # set it to train mode
+    task_pipeline.train()
+    answer = task_pipeline(question, id="1")
+    print(answer)
+    print(f"full_response: {answer.full_response}")
+
+
+if __name__ == "__main__":
+
+    # task = ObjectCountTask(**gpt_3_model)
+    # task_original = ObjectCountTaskOriginal(**gpt_3_model)
+
+    # question = "I have a flute, a piano, a trombone, four stoves, a violin, an accordion, a clarinet, a drum, two lamps, and a trumpet. How many musical instruments do I have?"
+
+    # print(task(question))
+    # print(task_original(question))
+
+    test_object_count_task()
diff --git a/use_cases/question_answering/bhh_object_count/train_new.py b/use_cases/question_answering/bbh/bhh_object_count/train_new.py
similarity index 98%
rename from use_cases/question_answering/bhh_object_count/train_new.py
rename to use_cases/question_answering/bbh/bhh_object_count/train_new.py
index 888a1ee7..3b38898e 100644
--- a/use_cases/question_answering/bhh_object_count/train_new.py
+++ b/use_cases/question_answering/bbh/bhh_object_count/train_new.py
@@ -2,7 +2,7 @@
     ObjectCountTaskPipeline,
 )
 
-from use_cases.question_answering.bhh_object_count.config import (
+from LightRAG.use_cases.config import (
     gpt_3_model,
     gpt_4o_model,
 )
diff --git a/use_cases/question_answering/bbh/word_sorting/data.py b/use_cases/question_answering/bbh/word_sorting/data.py
new file mode 100644
index 00000000..30fc5b44
--- /dev/null
+++ b/use_cases/question_answering/bbh/word_sorting/data.py
@@ -0,0 +1,101 @@
+"""Prepare classes, components, functions that will prepare and post processing the data"""
+
+from dataclasses import dataclass, field
+import adalflow as adal
+
+from adalflow.core import DataClass
+from adalflow.datasets.big_bench_hard import BigBenchHard
+from adalflow.utils.data import subset_dataset
+
+
+# TODO: user dont have to specify, we can auto generate a dataclass
+@dataclass
+class QuestionAnswer(DataClass):
+    """Dataclass for string output"""
+
+    id: str = field(
+        default=None,
+        metadata={"desc": "The unique identifier of the example"},
+    )
+
+    question: str = field(
+        default=None,
+        metadata={"desc": "The question to be answered"},
+    )
+
+    answer: str = field(
+        default=None,
+        metadata={"desc": "The raw answer to the question"},  # teacher
+    )
+
+    score: float = field(
+        default=None,
+        metadata={
+            "desc": "The score of the answer, in range [0, 1]. The higher the better"
+        },
+    )  # score can be used as weight for demo, weight = score (the higher the more likely to be sampled)
+
+
+import re
+
+
+def _parse_integer_answer(answer: str):
+    """A function that parses the last integer from a string using regular expressions."""
+    try:
+        # Use regular expression to find all sequences of digits
+        numbers = re.findall(r"\d+", answer)
+        if numbers:
+            # Get the last number found
+            answer = int(numbers[-1])
+        else:
+            answer = -1
+    except ValueError:
+        answer = -1
+
+    return answer
+
+
+@adal.fun_to_component
+def parse_integer_answer(answer: str):
+    """A function that parses the last integer from a string using regular expressions."""
+    try:
+        # Use regular expression to find all sequences of digits
+        numbers = re.findall(r"\d+", answer)
+        if numbers:
+            # Get the last number found
+            answer = int(numbers[-1])
+        else:
+            answer = -1
+    except ValueError:
+        answer = -1
+
+    return answer
+
+
+def load_datasets(max_samples: int = None):
+    """Load the dataset"""
+    train_data = BigBenchHard(split="train", task_name="BBH_word_sorting")
+    val_data = BigBenchHard(split="val")
+    test_data = BigBenchHard(split="test")
+
+    # Limit the number of samples
+    if max_samples:
+        train_data = subset_dataset(train_data, max_samples)
+        val_data = subset_dataset(val_data, max_samples)
+        test_data = subset_dataset(test_data, max_samples)
+
+    return train_data, val_data, test_data
+
+
+if __name__ == "__main__":
+    # TODO: make the int parser to be able to extract the last int
+
+    tests = [
+        "Answer: 10",
+        "xxxx10XXXX100",
+        "Answer: 10.0",
+        "Answer: 10.0.0",
+        "1\n2 Answer:5",
+    ]
+    for text in tests:
+        print(f"{text} -> {parse_integer_answer(text)}")
diff --git a/use_cases/question_answering/bbh/word_sorting/diagnose.py b/use_cases/question_answering/bbh/word_sorting/diagnose.py
new file mode 100644
index 00000000..1530d714
--- /dev/null
+++ b/use_cases/question_answering/bbh/word_sorting/diagnose.py
@@ -0,0 +1,69 @@
+from typing import Dict
+import adalflow as adal
+from use_cases.question_answering.bhh_object_count.task import ObjectCountTaskPipeline
+
+from adalflow.datasets.types import Example
+from adalflow.eval.answer_match_acc import AnswerMatchAcc
+
+
+class ObjectCountAdalComponent(adal.AdalComponent):
+    def __init__(self, model_client: adal.ModelClient, model_kwargs: Dict):
+        task = ObjectCountTaskPipeline(model_client, model_kwargs)
+        eval_fn = AnswerMatchAcc(type="exact_match").compute_single_item
+        super().__init__(task=task, eval_fn=eval_fn)
+
+    def handle_one_task_sample(self, sample: Example):
+        return self.task.call, {"question": sample.question, "id": sample.id}
+
+    def evaluate_one_sample(
+        self, sample: Example, y_pred: adal.GeneratorOutput
+    ) -> float:
+        y_label = -1
+        if y_pred and y_pred.data:
+            y_label = y_pred.data
+        return self.eval_fn(y_label, sample.answer)
+
+
+# TODO: ensure it can work by dataset, because
+# right now it will put trainset, valset, testset all together
+def diagnose(
+    model_client: adal.ModelClient,
+    model_kwargs: Dict,
+) -> Dict:
+    from use_cases.question_answering.bhh_object_count.data import load_datasets
+
+    trainset, valset, testset = load_datasets()
+    # use max_samples=10 to test the code
+
+    adal_component = ObjectCountAdalComponent(model_client, model_kwargs)
+    trainer = adal.Trainer(adaltask=adal_component)
+    trainer.diagnose(dataset=trainset, split="train")
+    trainer.diagnose(dataset=valset, split="val")
+    trainer.diagnose(dataset=testset, split="test")
+
+
+if __name__ == "__main__":
+    from LightRAG.use_cases.config import (
+        gpt_3_model,
+    )
+
+    # from use_cases.question_answering.bhh_object_count.prepare_trainer import (
+    #     TGDWithEvalFnLoss,
+    # )
+
+    # from adalflow.optim.trainer.trainer import Trainer
+
+    # trainset, valset, testset = load_datasets(max_samples=10)
+    # adaltask = TGDWithEvalFnLoss(
+    #     task_model_config=llama3_model,
+    #     backward_engine_model_config=llama3_model,
+    #     optimizer_model_config=llama3_model,
+    # )
+
+    # trainer = Trainer(adaltask=adaltask)
+    # diagnose = trainer.diagnose(train_dataset=trainset)
+    # print(diagnose)
+
+    # Diagnostic results run on trainer set, with all inputs and outputs tracked in ckpt/TGDWithEvalFnLoss/llm_counter_call
+
+    diagnose(**gpt_3_model)
diff --git a/use_cases/question_answering/bbh/word_sorting/task.py b/use_cases/question_answering/bbh/word_sorting/task.py
new file mode 100644
index 00000000..8fcaeff1
--- /dev/null
+++ b/use_cases/question_answering/bbh/word_sorting/task.py
@@ -0,0 +1,93 @@
+"""Prepare the task pipeline"""
+
+from adalflow.optim.parameter import ParameterType
+
+from use_cases.question_answering.bhh_object_count.data import (
+    parse_integer_answer,
+)
+
+
+# Few shot demonstration can be less effective when performance already high
+few_shot_template = r"""<START_OF_SYSTEM_PROMPT>
+{{system_prompt}}
+{# Few shot demos #}
+{% if few_shot_demos is not none %}
+Here are some examples:
+{{few_shot_demos}}
+{% endif %}
+<END_OF_SYSTEM_PROMPT>
+<START_OF_USER>
+{{input_str}}
+<END_OF_USER>
+"""
+
+from typing import Dict, Union
+import adalflow as adal
+
+
+class ObjectCountTaskPipeline(adal.Component):
+    def __init__(self, model_client: adal.ModelClient, model_kwargs: Dict):
+        super().__init__()
+
+        system_prompt = adal.Parameter(
+            # data="You will answer a reasoning question. Think step by step. The last line of your response should be of the following format: 'Answer: $VALUE' where VALUE is a numerical value.",
+            data="You will answer a reasoning question. Think step by step. The last line of your response should be of the following format: 'Answer: $VALUE' where VALUE is a numerical value.",
+            role_desc="To give task instruction to the language model in the system prompt",
+            requires_opt=True,
+            param_type=ParameterType.PROMPT,
+            instruction_to_optimizer="You can try to show examples to see if it helps.",
+        )
+        few_shot_demos = adal.Parameter(
+            data=None,
+            role_desc="To provide few shot demos to the language model",
+            requires_opt=True,
+            param_type=ParameterType.DEMOS,
+        )
+
+        self.llm_counter = adal.Generator(
+            model_client=model_client,
+            model_kwargs=model_kwargs,
+            template=few_shot_template,
+            prompt_kwargs={
+                "system_prompt": system_prompt,
+                "few_shot_demos": few_shot_demos,
+            },
+            output_processors=parse_integer_answer,
+            use_cache=True,
+        )
+
+    def call(
+        self, question: str, id: str = None
+    ) -> Union[adal.GeneratorOutput, adal.Parameter]:
+        output = self.llm_counter(prompt_kwargs={"input_str": question}, id=id)
+        return output
+
+
+def test_object_count_task():
+    from LightRAG.use_cases.config import gpt_3_model
+
+    question = "I have a flute, a piano, a trombone, four stoves, a violin, an accordion, a clarinet, a drum, two lamps, and a trumpet. How many musical instruments do I have?"
+    task_pipeline = ObjectCountTaskPipeline(**gpt_3_model)
+    print(task_pipeline)
+
+    answer = task_pipeline(question)
+    print(answer)
+
+    # set it to train mode
+    task_pipeline.train()
+    answer = task_pipeline(question, id="1")
+    print(answer)
+    print(f"full_response: {answer.full_response}")
+
+
+if __name__ == "__main__":
+
+    # task = ObjectCountTask(**gpt_3_model)
+    # task_original = ObjectCountTaskOriginal(**gpt_3_model)
+
+    # question = "I have a flute, a piano, a trombone, four stoves, a violin, an accordion, a clarinet, a drum, two lamps, and a trumpet. How many musical instruments do I have?"
+
+    # print(task(question))
+    # print(task_original(question))
+
+    test_object_count_task()
diff --git a/use_cases/question_answering/bhh_object_count/task.py b/use_cases/question_answering/bhh_object_count/task.py
deleted file mode 100644
index 3fc9b3de..00000000
--- a/use_cases/question_answering/bhh_object_count/task.py
+++ /dev/null
@@ -1,268 +0,0 @@
-"""Prepare the task pipeline"""
-
-from typing import Any
-
-from adalflow.core import Component, Generator
-from adalflow.optim.parameter import Parameter, ParameterType
-from adalflow.components.output_parsers import YamlOutputParser
-
-from use_cases.question_answering.bhh_object_count.data import (
-    parse_integer_answer,
-    ObjectCountPredData,
-)
-
-
-# class ObjectCountTaskOriginal(Component):
-#     r"""Same system prompt as text-grad paper, but with our one message prompt template, which has better starting performance"""
-
-#     def __init__(self, model_client, model_kwargs):
-#         super().__init__()
-
-#         template = """<START_OF_SYSTEM_PROMPT>{{system_prompt}}<END_OF_SYSTEM_PROMPT>{{input_str}}"""
-#         # 1. set up system prompt, and define the parameters for optimization.
-#         # NOTE: Dont use self.system_prompt
-#         # use self. will double the parameters, so we dont need that as we want the parameter to be part of the generator
-#         system_prompt = Parameter(
-#             name="task_instruction",
-#             data="You will answer a reasoning question. Think step by step. The last line of your response should be of the following format: 'Answer: $VALUE' where VALUE is a numerical value.",
-#             role_desc="To give task instruction to the language model in the system prompt",
-#             requires_opt=True,
-#             param_type=ParameterType.NONE,
-#             instruction_to_optimizer="You can show some examples if you think that will help.",
-#         )
-#         # TODO: sometimes the cache will collide, so we get different evaluation
-#         self.llm_counter = Generator(
-#             model_client=model_client,
-#             model_kwargs=model_kwargs,
-#             template=template,
-#             prompt_kwargs={
-#                 "system_prompt": system_prompt,
-#             },
-#             output_processors=parse_integer_answer,
-#             use_cache=True,
-#         )
-#         # TODO: make this data map function more robust (this is the final answer and the input to eval_fn)
-#         # self.llm_counter.set_data_map_func(lambda x: x.data.answer)
-#         print(f"llm_counter set_data_map_func, {self.llm_counter.data_map_func}")
-
-#     # TODO: the error will be a context
-#     def call(self, question: str, id: str = None) -> Any:  # Add id for tracing
-#         output = self.llm_counter(
-#             prompt_kwargs={"input_str": question, "id": id}
-#         )  # already support both training (forward + call)
-
-#         if not self.training:  # eval
-
-#             if output.data is None:
-#                 print(f"Error in processing the question: {question}, output: {output}")
-#                 output = -1
-#             else:
-#                 output = int(output.data)
-#         return output
-
-
-# Few shot demonstration can be less effective when performance already high
-few_shot_template = r"""<START_OF_SYSTEM_PROMPT>
-{{system_prompt}}
-{# Few shot demos #}
-{% if few_shot_demos is not none %}
-Here are some examples:
-{{few_shot_demos}}
-{% endif %}
-<END_OF_SYSTEM_PROMPT>
-<START_OF_USER>
-{{input_str}}
-<END_OF_USER>
-"""
-
-from typing import Dict, Union
-import adalflow as adal
-
-
-class ObjectCountTaskPipeline(adal.Component):
-    def __init__(self, model_client: adal.ModelClient, model_kwargs: Dict):
-        super().__init__()
-
-        system_prompt = adal.Parameter(
-            # data="You will answer a reasoning question. Think step by step. The last line of your response should be of the following format: 'Answer: $VALUE' where VALUE is a numerical value.",
-            data="You will answer a reasoning question. Think step by step. The last line of your response should be of the following format: 'Answer: $VALUE' where VALUE is a numerical value.",
-            role_desc="To give task instruction to the language model in the system prompt",
-            requires_opt=True,
-            param_type=ParameterType.PROMPT,
-            instruction_to_optimizer="You can try to show examples to see if it helps.",
-        )
-        few_shot_demos = adal.Parameter(
-            data=None,
-            role_desc="To provide few shot demos to the language model",
-            requires_opt=True,
-            param_type=ParameterType.DEMOS,
-        )
-
-        self.llm_counter = adal.Generator(
-            model_client=model_client,
-            model_kwargs=model_kwargs,
-            template=few_shot_template,
-            prompt_kwargs={
-                "system_prompt": system_prompt,
-                "few_shot_demos": few_shot_demos,
-            },
-            output_processors=parse_integer_answer,
-            use_cache=True,
-        )
-
-    def call(
-        self, question: str, id: str = None
-    ) -> Union[adal.GeneratorOutput, adal.Parameter]:
-        output = self.llm_counter(prompt_kwargs={"input_str": question}, id=id)
-        return output
-
-
-# class ObjectCountTaskFewShot(Component):
-#     r"""Same system prompt as text-grad paper, but with our one message prompt template, which has better starting performance"""
-
-#     def __init__(self, model_client, model_kwargs):
-#         super().__init__()
-
-#         # 1. set up system prompt, and define the parameters for optimization.
-#         # NOTE: Dont use self.system_prompt
-#         # use self. will double the parameters, so we dont need that as we want the parameter to be part of the generator
-#         system_prompt = Parameter(
-#             name="task_instruction",
-#             data="You will answer a reasoning question. Think step by step. The last line of your response should be of the following format: 'Answer: $VALUE' where VALUE is a numerical value.",
-#             role_desc="To give task instruction to the language model in the system prompt",
-#             requires_opt=True,
-#             param_type=ParameterType.NONE,
-#             # instruction_to_optimizer="You can show some examples if you think that will help.",
-#         )
-#         _few_shot_demos = Parameter(
-#             name="few_shot_demos",
-#             data=None,
-#             role_desc="To provide few shot demos to the language model",
-#             requires_opt=True,
-#             param_type=ParameterType.DEMOS,
-#         )
-
-#         self.llm_counter = Generator(
-#             model_client=model_client,
-#             model_kwargs=model_kwargs,
-#             template=few_shot_template,
-#             prompt_kwargs={
-#                 "system_prompt": system_prompt,
-#                 "few_shot_demos": _few_shot_demos,
-#             },
-#             output_processors=parse_integer_answer,  # transform data field
-#             use_cache=True,
-#             demo_data_class=ObjectCountSimple,  # for output format
-#             demo_data_class_input_mapping={"question": "input_str"},
-#             demo_data_class_output_mapping={"answer": lambda x: x.raw_response},
-#         )
-
-#     def call(self, question: str, id: str = None) -> Any:  # Add id for tracing
-#         output = self.llm_counter(
-#             prompt_kwargs={
-#                 "input_str": question,
-#                 # "few_shot_demos": self._few_shot_demos.data,
-#             },
-#             id=id,
-#         )  # already support both training (forward + call)
-
-#         if not self.training:  # eval
-
-#             if output.data is None:
-#                 print(f"Error in processing the question: {question}, output: {output}")
-#                 output = -1
-#             else:
-#                 output = int(output.data)
-#         return output
-
-
-class ObjectCountTask(Component):
-    r"""We will use (1) structured output (2) train both system prompt and the output formating prompt"""
-
-    def __init__(self, model_client, model_kwargs):
-        super().__init__()
-        template = """<START_OF_SYSTEM_PROMPT>{{system_prompt}}<OUTPUT_FORMAT> {{output_format_str}}</OUTPUT_FORMAT></END_OF_SYSTEM_PROMPT>{{input_str}}"""
-        # data = (
-        #     "You will answer a reasoning question. Think step by step. The last line of your response should be of the following format: 'Answer: $VALUE' where VALUE is a numerical value.",
-        # )
-        # 1. set up system prompt, and define the parameters for optimization.
-        # NOTE: use self. will double the parameters, so we dont need that as we want the parameter to be part of the generator
-        system_prompt = Parameter(
-            name="task_instruction",
-            data="You will answer a reasoning question. Think step by step.",
-            role_desc="To give task instruction to the language model in the system prompt",
-            requires_opt=True,
-            # param_type=ParameterType.NONE,
-        )
-        instruction = "Do not change the fields in the JSON object. Only improve on the field descriptions."
-        output_format_str = Parameter(
-            name="output_format",
-            data="Respond with valid JSON object with the following schema:\n"
-            + ObjectCountPredData.to_json_signature(),
-            role_desc="To specify the LLM output format",
-            instruction_to_optimizer=instruction,
-            instruction_to_backward_engine=instruction,
-            # param_type=ParameterType.PROMPT,
-            requires_opt=True,
-        )
-        parser = YamlOutputParser(
-            data_class=ObjectCountPredData, return_data_class=True
-        )  # noqa: F841
-        self.llm_counter = Generator(
-            model_client=model_client,
-            model_kwargs=model_kwargs,
-            template=template,
-            prompt_kwargs={
-                "system_prompt": system_prompt,
-                "output_format_str": output_format_str,
-            },
-            output_processors=parser,
-        )
-        # TODO: make this data map function more robust (this is the final answer and the input to eval_fn)
-        self.llm_counter.set_data_map_func(lambda x: x.data.answer)
-        print(f"llm_counter set_data_map_func, {self.llm_counter.data_map_func}")
-
-    # TODO: the error will be a context
-    def call(self, question: str, id: str = None) -> Any:
-        output = self.llm_counter(
-            prompt_kwargs={"input_str": question, "id": id}
-        )  # already support both training (forward + call)
-
-        if not self.training:  # eval
-
-            if output.data is None:
-                print(f"Error in processing the question: {question}, output: {output}")
-                output = -1
-            else:
-                output = output.data.answer
-        return output
-
-
-def test_object_count_task():
-    from use_cases.question_answering.bhh_object_count.config import gpt_3_model
-
-    question = "I have a flute, a piano, a trombone, four stoves, a violin, an accordion, a clarinet, a drum, two lamps, and a trumpet. How many musical instruments do I have?"
-    task_pipeline = ObjectCountTaskPipeline(**gpt_3_model)
-    print(task_pipeline)
-
-    answer = task_pipeline(question)
-    print(answer)
-
-    # set it to train mode
-    task_pipeline.train()
-    answer = task_pipeline(question, id="1")
-    print(answer)
-    print(f"full_response: {answer.full_response}")
-
-
-if __name__ == "__main__":
-
-    # task = ObjectCountTask(**gpt_3_model)
-    # task_original = ObjectCountTaskOriginal(**gpt_3_model)
-
-    # question = "I have a flute, a piano, a trombone, four stoves, a violin, an accordion, a clarinet, a drum, two lamps, and a trumpet. How many musical instruments do I have?"
-
-    # print(task(question))
-    # print(task_original(question))
-
-    test_object_count_task()
diff --git a/use_cases/fewshot_qa.py b/use_cases/unsorted/fewshot_qa.py
similarity index 100%
rename from use_cases/fewshot_qa.py
rename to use_cases/unsorted/fewshot_qa.py
diff --git a/use_cases/llm_as_retriever.py b/use_cases/unsorted/llm_as_retriever.py
similarity index 100%
rename from use_cases/llm_as_retriever.py
rename to use_cases/unsorted/llm_as_retriever.py
diff --git a/use_cases/rag.py b/use_cases/unsorted/rag.py
similarity index 100%
rename from use_cases/rag.py
rename to use_cases/unsorted/rag.py
diff --git a/use_cases/rag_hotpotqa.py b/use_cases/unsorted/rag_hotpotqa.py
similarity index 100%
rename from use_cases/rag_hotpotqa.py
rename to use_cases/unsorted/rag_hotpotqa.py
diff --git a/use_cases/rag_optimized.py b/use_cases/unsorted/rag_optimized.py
similarity index 100%
rename from use_cases/rag_optimized.py
rename to use_cases/unsorted/rag_optimized.py
diff --git a/use_cases/rag_yaml_config.py b/use_cases/unsorted/rag_yaml_config.py
similarity index 100%
rename from use_cases/rag_yaml_config.py
rename to use_cases/unsorted/rag_yaml_config.py
diff --git a/use_cases/retriever_llm.py b/use_cases/unsorted/retriever_llm.py
similarity index 100%
rename from use_cases/retriever_llm.py
rename to use_cases/unsorted/retriever_llm.py
diff --git a/use_cases/simple_qa.py b/use_cases/unsorted/simple_qa.py
similarity index 100%
rename from use_cases/simple_qa.py
rename to use_cases/unsorted/simple_qa.py
diff --git a/use_cases/simple_qa_anthropic.py b/use_cases/unsorted/simple_qa_anthropic.py
similarity index 100%
rename from use_cases/simple_qa_anthropic.py
rename to use_cases/unsorted/simple_qa_anthropic.py
diff --git a/use_cases/simple_qa_google.py b/use_cases/unsorted/simple_qa_google.py
similarity index 100%
rename from use_cases/simple_qa_google.py
rename to use_cases/unsorted/simple_qa_google.py
diff --git a/use_cases/simple_qa_groq.py b/use_cases/unsorted/simple_qa_groq.py
similarity index 100%
rename from use_cases/simple_qa_groq.py
rename to use_cases/unsorted/simple_qa_groq.py
diff --git a/use_cases/simple_qa_memory.py b/use_cases/unsorted/simple_qa_memory.py
similarity index 100%
rename from use_cases/simple_qa_memory.py
rename to use_cases/unsorted/simple_qa_memory.py
diff --git a/use_cases/simple_qa_stream.py b/use_cases/unsorted/simple_qa_stream.py
similarity index 100%
rename from use_cases/simple_qa_stream.py
rename to use_cases/unsorted/simple_qa_stream.py
diff --git a/use_cases/simple_qa_trainable.py b/use_cases/unsorted/simple_qa_trainable.py
similarity index 100%
rename from use_cases/simple_qa_trainable.py
rename to use_cases/unsorted/simple_qa_trainable.py
diff --git a/use_cases/simple_rag_bm_25.py b/use_cases/unsorted/simple_rag_bm_25.py
similarity index 100%
rename from use_cases/simple_rag_bm_25.py
rename to use_cases/unsorted/simple_rag_bm_25.py
diff --git a/use_cases/use_embedder.py b/use_cases/unsorted/use_embedder.py
similarity index 100%
rename from use_cases/use_embedder.py
rename to use_cases/unsorted/use_embedder.py
diff --git a/use_cases/yaml_output.py b/use_cases/unsorted/yaml_output.py
similarity index 100%
rename from use_cases/yaml_output.py
rename to use_cases/unsorted/yaml_output.py

From 4335b42f97d3c9ee82ae8b40e96c5b4ed5177c16 Mon Sep 17 00:00:00 2001
From: Li Yin <li.yin.gravity@gmail.com>
Date: Mon, 26 Aug 2024 15:00:09 -0700
Subject: [PATCH 4/8] adding word sorting task

---
 adalflow/adalflow/datasets/big_bench_hard.py  |   7 +-
 use_cases/config.py                           |   4 +
 use_cases/question_answering/bbh/__init__.py  |   0
 .../bbh/bhh_object_count/data.py              | 135 ------------------
 .../bbh/bhh_object_count/task.py              |   4 +-
 .../bbh/{word_sorting => }/data.py            |  45 +++---
 .../bbh/word_sorting/diagnose.py              |  69 ---------
 .../bbh/word_sorting/task.py                  |  25 ++--
 8 files changed, 54 insertions(+), 235 deletions(-)
 create mode 100644 use_cases/question_answering/bbh/__init__.py
 delete mode 100644 use_cases/question_answering/bbh/bhh_object_count/data.py
 rename use_cases/question_answering/bbh/{word_sorting => }/data.py (77%)
 delete mode 100644 use_cases/question_answering/bbh/word_sorting/diagnose.py

diff --git a/adalflow/adalflow/datasets/big_bench_hard.py b/adalflow/adalflow/datasets/big_bench_hard.py
index d706ac1e..9ac007a6 100644
--- a/adalflow/adalflow/datasets/big_bench_hard.py
+++ b/adalflow/adalflow/datasets/big_bench_hard.py
@@ -63,7 +63,6 @@ def __init__(
                 self.data.append(
                     Example(question=row["x"], answer=row["y"], id=row["id"])
                 )  # dont use a tuple, use a dict {"x": ..., "y": ...}
-        self._task_description = "You will answer a reasoning question. Think step by step. The last line of your response should be of the following format: 'Answer: $VALUE' where VALUE is a numerical value."
 
     def _check_or_download_dataset(self, data_path: str = None, split: str = "train"):
 
@@ -120,8 +119,10 @@ def __getitem__(self, index) -> Example:
     def __len__(self):
         return len(self.data)
 
-    def get_default_task_instruction(self):
-        return self._task_description
+    @staticmethod
+    def get_default_task_instruction():
+        _task_description = "You will answer a reasoning question. Think step by step. The last line of your response should be of the following format: 'Answer: $VALUE' where VALUE is a numerical value."
+        return _task_description
 
 
 if __name__ == "__main__":
diff --git a/use_cases/config.py b/use_cases/config.py
index 0a4b70ca..4b76a496 100644
--- a/use_cases/config.py
+++ b/use_cases/config.py
@@ -1,6 +1,10 @@
 from adalflow.components.model_client.groq_client import GroqAPIClient
 from adalflow.components.model_client.openai_client import OpenAIClient
 
+from adalflow.utils import setup_env
+
+setup_env()
+
 
 llama3_model = {
     "model_client": GroqAPIClient(),
diff --git a/use_cases/question_answering/bbh/__init__.py b/use_cases/question_answering/bbh/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/use_cases/question_answering/bbh/bhh_object_count/data.py b/use_cases/question_answering/bbh/bhh_object_count/data.py
deleted file mode 100644
index aeac5fbc..00000000
--- a/use_cases/question_answering/bbh/bhh_object_count/data.py
+++ /dev/null
@@ -1,135 +0,0 @@
-"""Prepare classes, components, functions that will prepare and post processing the data"""
-
-from dataclasses import dataclass, field
-import adalflow as adal
-
-from adalflow.core import DataClass
-from adalflow.datasets.big_bench_hard import BigBenchHard
-from adalflow.utils.data import subset_dataset
-
-
-# TODO: user dont have to specify, we can auto generate a dataclass
-@dataclass
-class ObjectCountSimple(DataClass):
-    """Dataclass for string output"""
-
-    # inputs: Dict[str, Any] = field(
-    #     default=None,
-    #     metadata={"desc": "The inputs to the model"},
-    # )
-    id: str = field(
-        default=None,
-        metadata={"desc": "The unique identifier of the example"},
-    )
-
-    question: str = field(
-        default=None,
-        metadata={"desc": "The question to be answered"},
-    )
-
-    answer: str = field(
-        default=None,
-        metadata={"desc": "The raw answer to the question"},  # teacher
-    )
-
-    # student_answer: str = field(
-    #     default=None,
-    #     metadata={"desc": "The student answer to the question"},  # student
-    # )
-    # ground_truth: str = field(
-    #     default=None,
-    #     metadata={"desc": "The ground truth answer to the question"},
-    # )
-    score: float = field(
-        default=None,
-        metadata={
-            "desc": "The score of the answer, in range [0, 1]. The higher the better"
-        },
-    )  # score can be used as weight for demo, weight = score (the higher the more likely to be sampled)
-
-    # score will be weight (1-score) for sampling using ground truth as answer.
-
-
-# def _parse_integer_answer(answer: str, only_first_line: bool = False):
-#     """A function to component that will parse the answer from a string. Used for string output"""
-#     try:
-#         if only_first_line:
-#             answer = answer.strip().split("\n")[0]
-#         answer = answer.strip()
-#         # find the last token that has a number in it
-#         answer = [token for token in answer.split() if any(c.isdigit() for c in token)][
-#             -1
-#         ]
-#         answer = answer.split(".")[0]
-#         answer = "".join([c for c in answer if c.isdigit()])
-#         answer = int(answer)
-
-#     except (ValueError, IndexError):
-#         answer = 0
-
-#     return answer
-
-
-import re
-
-
-def _parse_integer_answer(answer: str):
-    """A function that parses the last integer from a string using regular expressions."""
-    try:
-        # Use regular expression to find all sequences of digits
-        numbers = re.findall(r"\d+", answer)
-        if numbers:
-            # Get the last number found
-            answer = int(numbers[-1])
-        else:
-            answer = -1
-    except ValueError:
-        answer = -1
-
-    return answer
-
-
-@adal.fun_to_component
-def parse_integer_answer(answer: str):
-    """A function that parses the last integer from a string using regular expressions."""
-    try:
-        # Use regular expression to find all sequences of digits
-        numbers = re.findall(r"\d+", answer)
-        if numbers:
-            # Get the last number found
-            answer = int(numbers[-1])
-        else:
-            answer = -1
-    except ValueError:
-        answer = -1
-
-    return answer
-
-
-def load_datasets(max_samples: int = None):
-    """Load the dataset"""
-    train_data = BigBenchHard(split="train")
-    val_data = BigBenchHard(split="val")
-    test_data = BigBenchHard(split="test")
-
-    # Limit the number of samples
-    if max_samples:
-        train_data = subset_dataset(train_data, max_samples)
-        val_data = subset_dataset(val_data, max_samples)
-        test_data = subset_dataset(test_data, max_samples)
-
-    return train_data, val_data, test_data
-
-
-if __name__ == "__main__":
-    # TODO: make the int parser to be able to extract the last int
-
-    tests = [
-        "Answer: 10",
-        "xxxx10XXXX100",
-        "Answer: 10.0",
-        "Answer: 10.0.0",
-        "1\n2 Answer:5",
-    ]
-    for text in tests:
-        print(f"{text} -> {parse_integer_answer(text)}")
diff --git a/use_cases/question_answering/bbh/bhh_object_count/task.py b/use_cases/question_answering/bbh/bhh_object_count/task.py
index 8fcaeff1..6ded1951 100644
--- a/use_cases/question_answering/bbh/bhh_object_count/task.py
+++ b/use_cases/question_answering/bbh/bhh_object_count/task.py
@@ -2,7 +2,7 @@
 
 from adalflow.optim.parameter import ParameterType
 
-from use_cases.question_answering.bhh_object_count.data import (
+from use_cases.question_answering.bbh.data import (
     parse_integer_answer,
 )
 
@@ -64,7 +64,7 @@ def call(
 
 
 def test_object_count_task():
-    from LightRAG.use_cases.config import gpt_3_model
+    from use_cases.config import gpt_3_model
 
     question = "I have a flute, a piano, a trombone, four stoves, a violin, an accordion, a clarinet, a drum, two lamps, and a trumpet. How many musical instruments do I have?"
     task_pipeline = ObjectCountTaskPipeline(**gpt_3_model)
diff --git a/use_cases/question_answering/bbh/word_sorting/data.py b/use_cases/question_answering/bbh/data.py
similarity index 77%
rename from use_cases/question_answering/bbh/word_sorting/data.py
rename to use_cases/question_answering/bbh/data.py
index 30fc5b44..768af463 100644
--- a/use_cases/question_answering/bbh/word_sorting/data.py
+++ b/use_cases/question_answering/bbh/data.py
@@ -1,5 +1,7 @@
 """Prepare classes, components, functions that will prepare and post processing the data"""
 
+import re
+
 from dataclasses import dataclass, field
 import adalflow as adal
 
@@ -8,9 +10,8 @@
 from adalflow.utils.data import subset_dataset
 
 
-# TODO: user dont have to specify, we can auto generate a dataclass
 @dataclass
-class QuestionAnswer(DataClass):
+class ObjectCountSimple(DataClass):
     """Dataclass for string output"""
 
     id: str = field(
@@ -33,26 +34,34 @@ class QuestionAnswer(DataClass):
         metadata={
             "desc": "The score of the answer, in range [0, 1]. The higher the better"
         },
-    )  # score can be used as weight for demo, weight = score (the higher the more likely to be sampled)
+    )
 
 
-import re
+@dataclass
+class QuestionAnswer(DataClass):
+    """Dataclass for string output"""
 
+    id: str = field(
+        default=None,
+        metadata={"desc": "The unique identifier of the example"},
+    )
 
-def _parse_integer_answer(answer: str):
-    """A function that parses the last integer from a string using regular expressions."""
-    try:
-        # Use regular expression to find all sequences of digits
-        numbers = re.findall(r"\d+", answer)
-        if numbers:
-            # Get the last number found
-            answer = int(numbers[-1])
-        else:
-            answer = -1
-    except ValueError:
-        answer = -1
+    question: str = field(
+        default=None,
+        metadata={"desc": "The question to be answered"},
+    )
 
-    return answer
+    answer: str = field(
+        default=None,
+        metadata={"desc": "The raw answer to the question"},  # teacher
+    )
+
+    score: float = field(
+        default=None,
+        metadata={
+            "desc": "The score of the answer, in range [0, 1]. The higher the better"
+        },
+    )  # score can be used as weight for demo, weight = score (the higher the more likely to be sampled)
 
 
 @adal.fun_to_component
@@ -74,7 +83,7 @@ def parse_integer_answer(answer: str):
 
 def load_datasets(max_samples: int = None):
     """Load the dataset"""
-    train_data = BigBenchHard(split="train", task_name="BBH_word_sorting")
+    train_data = BigBenchHard(split="train")
     val_data = BigBenchHard(split="val")
     test_data = BigBenchHard(split="test")
 
diff --git a/use_cases/question_answering/bbh/word_sorting/diagnose.py b/use_cases/question_answering/bbh/word_sorting/diagnose.py
deleted file mode 100644
index 1530d714..00000000
--- a/use_cases/question_answering/bbh/word_sorting/diagnose.py
+++ /dev/null
@@ -1,69 +0,0 @@
-from typing import Dict
-import adalflow as adal
-from use_cases.question_answering.bhh_object_count.task import ObjectCountTaskPipeline
-
-from adalflow.datasets.types import Example
-from adalflow.eval.answer_match_acc import AnswerMatchAcc
-
-
-class ObjectCountAdalComponent(adal.AdalComponent):
-    def __init__(self, model_client: adal.ModelClient, model_kwargs: Dict):
-        task = ObjectCountTaskPipeline(model_client, model_kwargs)
-        eval_fn = AnswerMatchAcc(type="exact_match").compute_single_item
-        super().__init__(task=task, eval_fn=eval_fn)
-
-    def handle_one_task_sample(self, sample: Example):
-        return self.task.call, {"question": sample.question, "id": sample.id}
-
-    def evaluate_one_sample(
-        self, sample: Example, y_pred: adal.GeneratorOutput
-    ) -> float:
-        y_label = -1
-        if y_pred and y_pred.data:
-            y_label = y_pred.data
-        return self.eval_fn(y_label, sample.answer)
-
-
-# TODO: ensure it can work by dataset, because
-# right now it will put trainset, valset, testset all together
-def diagnose(
-    model_client: adal.ModelClient,
-    model_kwargs: Dict,
-) -> Dict:
-    from use_cases.question_answering.bhh_object_count.data import load_datasets
-
-    trainset, valset, testset = load_datasets()
-    # use max_samples=10 to test the code
-
-    adal_component = ObjectCountAdalComponent(model_client, model_kwargs)
-    trainer = adal.Trainer(adaltask=adal_component)
-    trainer.diagnose(dataset=trainset, split="train")
-    trainer.diagnose(dataset=valset, split="val")
-    trainer.diagnose(dataset=testset, split="test")
-
-
-if __name__ == "__main__":
-    from LightRAG.use_cases.config import (
-        gpt_3_model,
-    )
-
-    # from use_cases.question_answering.bhh_object_count.prepare_trainer import (
-    #     TGDWithEvalFnLoss,
-    # )
-
-    # from adalflow.optim.trainer.trainer import Trainer
-
-    # trainset, valset, testset = load_datasets(max_samples=10)
-    # adaltask = TGDWithEvalFnLoss(
-    #     task_model_config=llama3_model,
-    #     backward_engine_model_config=llama3_model,
-    #     optimizer_model_config=llama3_model,
-    # )
-
-    # trainer = Trainer(adaltask=adaltask)
-    # diagnose = trainer.diagnose(train_dataset=trainset)
-    # print(diagnose)
-
-    # Diagnostic results run on trainer set, with all inputs and outputs tracked in ckpt/TGDWithEvalFnLoss/llm_counter_call
-
-    diagnose(**gpt_3_model)
diff --git a/use_cases/question_answering/bbh/word_sorting/task.py b/use_cases/question_answering/bbh/word_sorting/task.py
index 8fcaeff1..d5ffbe35 100644
--- a/use_cases/question_answering/bbh/word_sorting/task.py
+++ b/use_cases/question_answering/bbh/word_sorting/task.py
@@ -2,7 +2,7 @@
 
 from adalflow.optim.parameter import ParameterType
 
-from use_cases.question_answering.bhh_object_count.data import (
+from use_cases.question_answering.bbh.data import (
     parse_integer_answer,
 )
 
@@ -23,15 +23,16 @@
 
 from typing import Dict, Union
 import adalflow as adal
+from adalflow.datasets.big_bench_hard import BigBenchHard
 
 
-class ObjectCountTaskPipeline(adal.Component):
+class QuestionAnswerTaskPipeline(adal.Component):
     def __init__(self, model_client: adal.ModelClient, model_kwargs: Dict):
         super().__init__()
 
         system_prompt = adal.Parameter(
             # data="You will answer a reasoning question. Think step by step. The last line of your response should be of the following format: 'Answer: $VALUE' where VALUE is a numerical value.",
-            data="You will answer a reasoning question. Think step by step. The last line of your response should be of the following format: 'Answer: $VALUE' where VALUE is a numerical value.",
+            data=BigBenchHard.get_default_task_instruction(),
             role_desc="To give task instruction to the language model in the system prompt",
             requires_opt=True,
             param_type=ParameterType.PROMPT,
@@ -63,13 +64,21 @@ def call(
         return output
 
 
-def test_object_count_task():
-    from LightRAG.use_cases.config import gpt_3_model
+def test_word_sorting_task():
+    from use_cases.config import gpt_3_model
+    from use_cases.question_answering.bbh.data import load_datasets
 
-    question = "I have a flute, a piano, a trombone, four stoves, a violin, an accordion, a clarinet, a drum, two lamps, and a trumpet. How many musical instruments do I have?"
-    task_pipeline = ObjectCountTaskPipeline(**gpt_3_model)
+    task_pipeline = QuestionAnswerTaskPipeline(**gpt_3_model)
     print(task_pipeline)
 
+    train_dataset, val_dataset, test_dataset = load_datasets(
+        task_name="BBH_word_sorting"
+    )
+
+    example = train_dataset[0]
+    question = example["question"]
+    print(example)
+
     answer = task_pipeline(question)
     print(answer)
 
@@ -90,4 +99,4 @@ def test_object_count_task():
     # print(task(question))
     # print(task_original(question))
 
-    test_object_count_task()
+    test_word_sorting_task()

From 6b192d4a23bcb46b6e844960409d0e48c27b1043 Mon Sep 17 00:00:00 2001
From: Li Yin <li.yin.gravity@gmail.com>
Date: Mon, 26 Aug 2024 21:05:19 -0700
Subject: [PATCH 5/8] update adalflow poetry

---
 adalflow/adalflow/eval/llm_as_judge.py        |  36 +-
 adalflow/adalflow/optim/trainer/adal.py       | 141 +++-
 adalflow/poetry.lock                          |   5 +-
 .../question_answering_word_sort.rst          | 776 ++++++++++++++++++
 use_cases/config.py                           |  14 +
 use_cases/question_answering/bbh/data.py      |  21 +-
 .../__init__.py                               |   0
 .../diagnose.py                               |   4 +-
 .../task.py                                   |   0
 .../train_new.py                              |   4 +-
 .../bbh/word_sorting/diagnose.py              |  91 ++
 .../bbh/word_sorting/task.py                  |  14 +-
 12 files changed, 1047 insertions(+), 59 deletions(-)
 create mode 100644 docs/source/use_cases/question_answering_word_sort.rst
 rename use_cases/question_answering/bbh/{bhh_object_count => object_count}/__init__.py (100%)
 rename use_cases/question_answering/bbh/{bhh_object_count => object_count}/diagnose.py (93%)
 rename use_cases/question_answering/bbh/{bhh_object_count => object_count}/task.py (100%)
 rename use_cases/question_answering/bbh/{bhh_object_count => object_count}/train_new.py (97%)
 create mode 100644 use_cases/question_answering/bbh/word_sorting/diagnose.py

diff --git a/adalflow/adalflow/eval/llm_as_judge.py b/adalflow/adalflow/eval/llm_as_judge.py
index 119e263b..0594830e 100644
--- a/adalflow/adalflow/eval/llm_as_judge.py
+++ b/adalflow/adalflow/eval/llm_as_judge.py
@@ -1,6 +1,6 @@
 """This is the metric to use an LLM as a judge for evaluating the performance of predicted answers."""
 
-from typing import List, Dict, Any, Optional, TYPE_CHECKING
+from typing import List, Dict, Any, Optional, TYPE_CHECKING, Union, Literal
 import logging
 
 if TYPE_CHECKING:
@@ -13,9 +13,9 @@
 
 DEFAULT_LLM_EVALUATOR_PROMPT = r"""<START_OF_SYSTEM_PROMPT>
 {# task desc #}
-You are an evaluator.
-Given the question, ground truth answer, and predicted answer, you need to answer the judgement query.
-Output True or False according to the judgement query.
+You are an evaluator. Given the question, ground truth answer, and predicted answer,
+{# judgement question #}
+{{judgement_str}}
 <END_OF_SYSTEM_PROMPT>
 ---------------------
 <START_OF_USER>
@@ -25,12 +25,12 @@
 Ground truth answer: {{gt_answer_str}}
 {# predicted answer #}
 Predicted answer: {{pred_answer_str}}
-{# judgement question #}
-Judgement question: {{judgement_str}}
 {# assistant response #}
 <END_OF_USER>
 """
 
+DEFAULT_JUDGEMENT_QUERY = "Does the predicted answer contain the ground truth answer? Say True if yes, False if no."
+
 
 # print(f"globals: {globals()}")
 
@@ -58,6 +58,9 @@ def __init__(
         model_client: Optional[ModelClient] = None,
         model_kwargs: Optional[Dict[str, Any]] = None,
         template: Optional[str] = None,
+        jugement_query: Optional[str] = None,
+        output_type: Literal["bool", "float"] = "bool",
+        use_cache: bool = True,
     ):
         from adalflow.core.generator import Generator
 
@@ -78,11 +81,18 @@ def __init__(
             model_client=self.model_client,
             model_kwargs=self.model_kwargs,
             template=self.template,
+            use_cache=use_cache,
         )
+        self._jugement_query = jugement_query or DEFAULT_JUDGEMENT_QUERY
+        self.output_type = output_type
 
     def call(
-        self, question: str, gt_answer: str, pred_answer: str, judgement_query: str
-    ) -> bool:
+        self,
+        question: str,
+        gt_answer: str,
+        pred_answer: str,
+        judgement_query: Optional[str] = None,
+    ) -> Union[bool, float]:
         r"""
         Get the judgement of the predicted answer for a single question.
 
@@ -95,6 +105,7 @@ def call(
         Returns:
             bool: Judgement result.
         """
+        judgement_query = judgement_query or self._jugement_query
         output = self.llm_evaluator(
             prompt_kwargs={
                 "question_str": question,
@@ -106,12 +117,15 @@ def call(
 
         judgement = output.raw_response
         judgement = judgement.strip().lower()
+        output = False if self.output_type == "bool" else 0.0
         if "true" in judgement:
-            return True
+            output = True if self.output_type == "bool" else 1.0
         elif "false" in judgement:
-            return False
+            output = False if self.output_type == "bool" else 0.0
         else:
-            raise ValueError(f"Invalid judgement: {judgement}")
+            print(f"Invalid judgement: {judgement}, use False or 0.0 instead.")
+            # raise ValueError(f"Invalid judgement: {judgement}")
+        return output
 
 
 class LLMasJudge:
diff --git a/adalflow/adalflow/optim/trainer/adal.py b/adalflow/adalflow/optim/trainer/adal.py
index 8dd59025..fbd95239 100644
--- a/adalflow/adalflow/optim/trainer/adal.py
+++ b/adalflow/adalflow/optim/trainer/adal.py
@@ -178,35 +178,98 @@ def configure_backward_engine(self, *args, **kwargs):
     #     raise NotImplementedError("configure_backward_engine method is not implemented")
 
     def evaluate_samples(
-        self, samples: Any, y_preds: List, metadata: Optional[Dict[str, Any]] = None
+        self,
+        samples: Any,
+        y_preds: List,
+        metadata: Optional[Dict[str, Any]] = None,
+        num_workers: int = 2,
     ) -> EvaluationResult:
-        r"""Run evaluation on samples. Use ``evaluate_one_sample`` defined by the user.
+        r"""Run evaluation on samples using parallel processing. Utilizes ``evaluate_one_sample`` defined by the user.
 
         Metadata is used for storing context that you can find from generator input.
 
-        Note:
-            ensure it supports both Tuple(batch) and a list of any type (fits for datasets).
+        Args:
+            samples (Any): The input samples to evaluate.
+            y_preds (List): The predicted outputs corresponding to each sample.
+            metadata (Optional[Dict[str, Any]]): Optional metadata dictionary.
+            num_workers (int): Number of worker threads for parallel processing.
+
+        Returns:
+            EvaluationResult: An object containing the average score and per-item scores.
         """
         from adalflow.optim.parameter import Parameter
 
         if not isinstance(y_preds, list) or len(y_preds) == 0:
             raise ValueError(f"y_preds is not a list or empty: {y_preds}")
+
         y_pred_0 = y_preds[0]
         if isinstance(y_pred_0, Parameter):
             raise ValueError(f"y_pred_0 should not be a Parameter: {y_pred_0}")
-        if metadata is None:
-            acc_list = [
-                self.evaluate_one_sample(sample, y_pred)
-                for sample, y_pred in zip(samples, y_preds)
-            ]
-        else:
-            acc_list = [
-                self.evaluate_one_sample(sample, y_pred, metadata=metadata)
-                for sample, y_pred in zip(samples, y_preds)
-            ]
+
+        acc_list = [None] * len(samples)  # Initialize accuracy list to hold results
+
+        with concurrent.futures.ThreadPoolExecutor(max_workers=num_workers) as executor:
+
+            # 1. submit all the tasks
+            futures = {}
+
+            for i, (sample, y_pred) in enumerate(zip(samples, y_preds)):
+                if metadata is None:
+                    future = executor.submit(self.evaluate_one_sample, sample, y_pred)
+                else:
+                    future = executor.submit(
+                        self.evaluate_one_sample, sample, y_pred, metadata=metadata
+                    )
+                futures[future] = i
+
+            # 2. collect the results, update the progress bar
+            # Initialize progress bar once outside the loop
+            progress_bar = tqdm(
+                total=len(samples), desc="Evaluating", position=0, leave=True
+            )
+
+            for future in concurrent.futures.as_completed(futures):
+                i = futures[future]
+                acc_list[i] = (
+                    future.result()
+                )  # Place the result in the correct position
+                progress_bar.update(
+                    1
+                )  # Update progress bar after each result is collected
+
         avg_score = float(np.mean(np.array(acc_list)))
         return EvaluationResult(avg_score=avg_score, per_item_scores=acc_list)
 
+    # def evaluate_samples(
+    #     self, samples: Any, y_preds: List, metadata: Optional[Dict[str, Any]] = None
+    # ) -> EvaluationResult:
+    #     r"""Run evaluation on samples. Use ``evaluate_one_sample`` defined by the user.
+
+    #     Metadata is used for storing context that you can find from generator input.
+
+    #     Note:
+    #         ensure it supports both Tuple(batch) and a list of any type (fits for datasets).
+    #     """
+    #     from adalflow.optim.parameter import Parameter
+
+    #     if not isinstance(y_preds, list) or len(y_preds) == 0:
+    #         raise ValueError(f"y_preds is not a list or empty: {y_preds}")
+    #     y_pred_0 = y_preds[0]
+    #     if isinstance(y_pred_0, Parameter):
+    #         raise ValueError(f"y_pred_0 should not be a Parameter: {y_pred_0}")
+    #     if metadata is None:
+    #         acc_list = [
+    #             self.evaluate_one_sample(sample, y_pred)
+    #             for sample, y_pred in zip(samples, y_preds)
+    #         ]
+    #     else:
+    #         acc_list = [
+    #             self.evaluate_one_sample(sample, y_pred, metadata=metadata)
+    #             for sample, y_pred in zip(samples, y_preds)
+    #         ]
+    #     avg_score = float(np.mean(np.array(acc_list)))
+    #     return EvaluationResult(avg_score=avg_score, per_item_scores=acc_list)
+
     def _train_step(
         self,
         batch,
@@ -288,8 +351,10 @@ def train_step(self, batch, batch_idx, num_workers: int = 2) -> List:
         y_preds = [None] * len(batch)
         samples = [None] * len(batch)
         completed_indices = set()
+        index_to_score = {}  # for running evaluation
 
         with concurrent.futures.ThreadPoolExecutor(max_workers=num_workers) as executor:
+            # 1. submit all the tasks
             futures = []
             tqdm_loader = tqdm(batch, total=len(batch), desc="Loading Data")
             for i, sample in enumerate(tqdm_loader):
@@ -297,10 +362,11 @@ def train_step(self, batch, batch_idx, num_workers: int = 2) -> List:
                 future = executor.submit(task_call, **kwargs)
                 futures.append((future, i, sample))  # preserve the order of the samples
 
+            # 2. predict the results, update the progress bar
             tqdm_loader = tqdm(
                 total=len(futures),
                 position=0,
-                desc=f"Evaluating step: {batch_idx}",
+                desc=f"Prediting step: {batch_idx}",
             )
             for future, i, sample in futures:
                 y_pred = future.result()
@@ -311,13 +377,15 @@ def train_step(self, batch, batch_idx, num_workers: int = 2) -> List:
                 assert (
                     y_pred.id == sample.id
                 ), f"ID mismatch: {y_pred.id} != {sample.id}, type: {type(y_pred)}"
+
                 completed_indices.add(i)  # Mark this index as completed
 
                 if running_eval and not isinstance(y_pred, Parameter):
-                    # Perform running evaluation only on completed samples
-                    sorted_indices = sorted(completed_indices)
-                    completed_y_preds = [y_preds[idx] for idx in sorted_indices]
-                    completed_samples = [samples[idx] for idx in sorted_indices]
+                    # evaluate one sample
+                    score = self.evaluate_one_sample(sample, y_pred)
+                    index_to_score[i] = score
+                    eval_score = np.mean(list(index_to_score.values())).item()
+
                     # for y_pred, sample in zip(completed_y_preds, completed_samples):
                     #     print(f"y_pred: {y_pred.data}, sample: {sample.answer}")
                     # for y_pred, sample in zip(completed_y_preds, completed_samples):
@@ -326,10 +394,6 @@ def train_step(self, batch, batch_idx, num_workers: int = 2) -> List:
                     #             f"ID mismatch: {y_pred.id} != {sample.id}, type: {type(y_pred)}"
                     #         )
                     #     print(f"y_pred: {y_pred.data}, sample: {sample.answer}")
-                    # TODO: each time only call evaluate on one sample
-                    eval_score = self.evaluate_samples(
-                        completed_samples, completed_y_preds
-                    ).avg_score
 
                     remaining_samples = len(batch) - len(completed_indices)
                     max_score = (
@@ -340,10 +404,10 @@ def train_step(self, batch, batch_idx, num_workers: int = 2) -> List:
                         break
 
                     tqdm_loader.set_description(
-                        f"Evaluating step({batch_idx}): {round(eval_score,4)} across {len(completed_indices)} samples, Max potential: {round(max_score,4)}"
+                        f"Predicting: step({batch_idx}): {round(eval_score,4)} across {len(completed_indices)} samples, Max potential: {round(max_score,4)}"
                     )
                 else:
-                    tqdm_loader.set_description(f"Evaluating step({batch_idx})")
+                    tqdm_loader.set_description(f"Predicting: step({batch_idx})")
 
                 tqdm_loader.update(1)  # Update the progress bar
 
@@ -351,7 +415,7 @@ def train_step(self, batch, batch_idx, num_workers: int = 2) -> List:
         completed_y_preds = [y_preds[idx] for idx in sorted_indices]
         completed_samples = [samples[idx] for idx in sorted_indices]
 
-        return completed_y_preds, completed_samples
+        return completed_y_preds, completed_samples, index_to_score
 
     def train_step(self, batch, batch_idx, num_workers: int = 2) -> List:
         self.task.train()
@@ -384,12 +448,29 @@ def validation_step(self, batch, batch_idx, num_workers: int = 2) -> List:
         """
         # TODO: let use decide which mode to be
         self.task.eval()
-        completed_y_preds, completed_samples = self.pred_step(
+        completed_y_preds, completed_samples, index_to_score = self.pred_step(
             batch, batch_idx, num_workers, running_eval=True, min_score=minimum_score
         )
-        eval_results = self.evaluate_samples(
-            samples=completed_samples, y_preds=completed_y_preds
-        )
+        if index_to_score:
+            # compute score from index_to_score
+            print(
+                f"completed_samples: {len(completed_samples)}, len: {len(list(index_to_score.values()))}"
+            )
+            avg_score = np.mean(list(index_to_score.values())).item()
+            acc_list = [None] * len(index_to_score)
+            for i, score in index_to_score.items():
+                acc_list[i] = score
+            acc_list = list(index_to_score.values())
+            eval_results = EvaluationResult(
+                avg_score=avg_score, per_item_scores=acc_list
+            )
+        else:
+
+            eval_results = self.evaluate_samples(
+                samples=completed_samples,
+                y_preds=completed_y_preds,
+                num_workers=num_workers,
+            )
         return eval_results
 
     def loss_step(
diff --git a/adalflow/poetry.lock b/adalflow/poetry.lock
index 682e5a8b..1918e90d 100644
--- a/adalflow/poetry.lock
+++ b/adalflow/poetry.lock
@@ -2320,8 +2320,8 @@ files = [
 [package.dependencies]
 numpy = [
     {version = ">=1.22.4", markers = "python_version < \"3.11\""},
-    {version = ">=1.26.0", markers = "python_version >= \"3.12\""},
     {version = ">=1.23.2", markers = "python_version == \"3.11\""},
+    {version = ">=1.26.0", markers = "python_version >= \"3.12\""},
 ]
 python-dateutil = ">=2.8.2"
 pytz = ">=2020.1"
@@ -3941,6 +3941,7 @@ test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools",
 [extras]
 anthropic = ["anthropic"]
 cohere = ["cohere"]
+datasets = []
 faiss-cpu = ["faiss-cpu"]
 google-generativeai = ["google-generativeai"]
 groq = ["groq"]
@@ -3953,4 +3954,4 @@ torch = ["torch"]
 [metadata]
 lock-version = "2.0"
 python-versions = ">=3.9, <4.0"
-content-hash = "2738134e77ae589eba11e3bc0a2b15491ce297f655f0cb0d79e28fcb7f248a33"
+content-hash = "39c83de88b46df53f3909df37da0e43d48b6aa9590dc5c38e798f4e7adba310a"
diff --git a/docs/source/use_cases/question_answering_word_sort.rst b/docs/source/use_cases/question_answering_word_sort.rst
new file mode 100644
index 00000000..a349f6dd
--- /dev/null
+++ b/docs/source/use_cases/question_answering_word_sort.rst
@@ -0,0 +1,776 @@
+.. raw:: html
+
+   <div style="display: flex; justify-content: flex-start; align-items: center; margin-bottom: 20px;">
+      <a href="https://colab.research.google.com/drive/1n3mHUWekTEYHiBdYBTw43TKlPN41A9za?usp=sharing" target="_blank" style="margin-right: 10px;">
+         <img alt="Try Quickstart in Colab" src="https://colab.research.google.com/assets/colab-badge.svg" style="vertical-align: middle;">
+      </a>
+      <a href="https://github.com/SylphAI-Inc/AdalFlow/tree/main/use_cases/question_answering/bhh_object_count" target="_blank" style="display: flex; align-items: center;">
+         <img src="https://github.githubassets.com/images/modules/logos_page/GitHub-Mark.png" alt="GitHub" style="height: 20px; width: 20px; margin-right: 5px;">
+         <span style="vertical-align: middle;"> Open Source Code</span>
+      </a>
+   </div>
+
+
+Question Answer with LLM as Judge
+===================================
+
+
+AdalFlow provides token-efficient and high-performing prompt optimization within a unified framework.
+
+Overview
+----------------
+In this tutorial, we will implement and optimize a question-answering task pipeline. Specifically, the task is to count the total number of objects.
+Here is an example from the dataset:
+
+.. code-block:: python
+
+    question = "I have a flute, a piano, a trombone, four stoves, a violin, an accordion, a clarinet, a drum, two lamps, and a trumpet. How many musical instruments do I have?"
+
+
+For optimization, we will demonstrate both the instruction/prompt optimization and few-shot In-context Learning(ICL).
+
+**Instruction/prompt Optimization**
+
+We especially want to see how the auto-training pipeline performs with both good and bad starting prompts.
+
+With a low-performing starting prompt, our zero-shot optimizer states:
+
+.. list-table:: Scores by Method and Split On Low-performing Starting Prompt (gpt-3.5-turbo)
+   :header-rows: 1
+   :widths: 20 20 20 20
+
+   * - Method
+     - Train
+     - Val
+     - Test
+   * - Start (manual prompt)
+     - 0.84 (50 samples)
+     - 0.66 (50 samples)
+     - 0.77 (100 samples)
+   * - Optimized Zero-shot
+     - N/A
+     - 0.9 (**+36%**)
+     - 0.9 (**+25%**)
+
+It converged within 5 steps, with each batch containing only 4 samples.
+
+
+.. list-table:: Manual Prompt vs Optimized Prompt (gpt-3.5-turbo)
+   :header-rows: 1
+   :widths: 20 20
+
+   * - Method
+     - Prompt
+   * - Manual
+     - You will answer a reasoning question. The last line of your response should be of the following format: 'Answer: $VALUE' where VALUE is a numerical value.
+   * - Optimized (zero-shot) (90% on val, 90% on test)
+     - You will answer a reasoning question by performing detailed and careful counting of each item. Ensure no items, particularly those in plural form, are miscounted. The last line of your response should be formatted as follows: 'Answer: $VALUE' where VALUE is a numerical value.
+
+
+We will also demonstrate how to optimize an already high-performing task pipeline (~90% accuracy) to achieve even better results—a process that would be very challenging with manual prompt optimization.
+
+.. list-table:: Scores by Method and Split On High-performing Starting Prompt (gpt-3.5-turbo)
+   :header-rows: 1
+   :widths: 20 20 20 20
+
+   * - Method
+     - Train
+     - Val
+     - Test
+   * - Start (manual prompt)
+     - 0.88 (50 samples)
+     - 0.90 (50 samples)
+     - 0.87 (100 samples)
+   * - Optimized Zero-shot
+     - N/A
+     - 0.98 (**+8%**)
+     - 0.91 (**+4%**)
+
+
+.. list-table:: Manual Prompt vs Optimized Prompt
+   :header-rows: 1
+   :widths: 20 20
+
+   * - Method
+     - Prompt
+   * - Manual
+     - You will answer a reasoning question. Think step by step. The last line of your response should be of the following format: 'Answer: $VALUE' where VALUE is a numerical value.
+   * - Optimized (zero-shot) (92% on val, 91% on test)
+     - You will answer a reasoning question. Think step by step, and make sure to convert any numbers written in words into numerals. Double-check your calculations. The last line of your response should be of the following format: 'Answer: $VALUE' where VALUE is a numerical value.
+   * - Optimized (plus generated examples by itself) (98% on val, 91% on test)
+     - You will answer a reasoning question. Think step by step and double-check each calculation you make. Pay close attention to any numerical quantities in the text, converting written numbers into their numerical equivalents. Additionally, re-verify your final answer before concluding. The last line of your response should be of the following format: 'Answer: $VALUE' where VALUE is a numerical value. Here are some examples: 1. I have a flute, a piano, a trombone, four stoves, a violin, an accordion, a clarinet, a drum, two lamps, and a trumpet. How many musical instruments do I have? Answer: 8
+
+**Bootstrap Few-shot**
+
+We achieved 94% accuracy on the test split with just one bootstrap shot, using only the demonstration of the teacher model's response, surpassing the performance of all existing libraries.
+
+.. list-table:: Optimized Scores comparison on the same prompt on test set (gpt-3.5-turbo)
+   :header-rows: 1
+   :widths: 50 50
+
+   * - Method
+     - Test
+   * - Text-grad (start)
+     - 0.72
+   * - Text-grad (optimized)
+     - 0.89
+   * - AdalFlow (start)
+     - 0.87
+   * - AdalFlow(text-grad optimized)
+     - 0.91
+   * - AdalFlow ("Learn-to-reason" one-shot)
+     - **0.94**
+
+Now, let's get started on how to implement and achieve the results mentioned above together.
+
+
+Build the task pipeline
+--------------------------
+As we can leverage the optimizer to automatically optimize our task pipeline, we offer a quick way to build it.
+We'll instruct the LLM to respond with a chain of thought and end the response with the format Answer: $VALUE. We will use the following code to process it:
+
+.. code-block:: python
+
+    import adalflow as adal
+    import re
+
+    @adal.fun_to_component
+    def parse_integer_answer(answer: str):
+        """A function that parses the last integer from a string using regular expressions."""
+        try:
+            # Use regular expression to find all sequences of digits
+            numbers = re.findall(r"\d+", answer)
+            if numbers:
+                # Get the last number found
+                answer = int(numbers[-1])
+            else:
+                answer = -1
+        except ValueError:
+            answer = -1
+
+        return answer
+
+``adal.fun_to_component`` is a decorator that converts a function to a component so that we can pass it to the generator as a output processor.
+
+For the task, we will use a simple template taking three arguments: ``system_prompt``, ``few_shot_demos``, and ``input_str``.
+
+.. code-block:: python
+
+    few_shot_template = r"""<START_OF_SYSTEM_PROMPT>
+    {{system_prompt}}
+    {# Few shot demos #}
+    {% if few_shot_demos is not none %}
+    Here are some examples:
+    {{few_shot_demos}}
+    {% endif %}
+    <END_OF_SYSTEM_PROMPT>
+    <START_OF_USER>
+    {{input_str}}
+    <END_OF_USER>
+    """
+
+
+We will create two parameters for training the model: ``system_prompt`` and ``few_shot_demos``.
+We will initialize the ``Parameter`` with a ``role_desc`` and ``requires_opt`` to inform the ``backward_engine`` (for feedback/textual gradients) and
+the optimizer about the purpose of the parameter.
+Additionally, we need to set the ``param_type`` to ``ParameterType.PROMPT`` and ``ParameterType.DEMOS`` so that our trainer can configure the appropriate optimizer to optimize these parameters.
+
+Here is our task pipeline:
+
+.. code-block:: python
+
+    from typing import Dict, Union
+    import adalflow as adal
+
+
+    class ObjectCountTaskPipeline(adal.Component):
+        def __init__(self, model_client: adal.ModelClient, model_kwargs: Dict):
+            super().__init__()
+
+            system_prompt = adal.Parameter(
+                data="You will answer a reasoning question. Think step by step. The last line of your response should be of the following format: 'Answer: $VALUE' where VALUE is a numerical value.",
+                role_desc="To give task instruction to the language model in the system prompt",
+                requires_opt=True,
+                param_type=ParameterType.PROMPT,
+            )
+            few_shot_demos = adal.Parameter(
+                data=None,
+                role_desc="To provide few shot demos to the language model",
+                requires_opt=True,
+                param_type=ParameterType.DEMOS,
+            )
+
+            self.llm_counter = adal.Generator(
+                model_client=model_client,
+                model_kwargs=model_kwargs,
+                template=few_shot_template,
+                prompt_kwargs={
+                    "system_prompt": system_prompt,
+                    "few_shot_demos": few_shot_demos,
+                },
+                output_processors=parse_integer_answer,
+                use_cache=True,
+            )
+
+        def call(
+            self, question: str, id: str = None
+        ) -> Union[adal.GeneratorOutput, adal.Parameter]:
+            output = self.llm_counter(prompt_kwargs={"input_str": question}, id=id)
+            return output
+
+
+
+Here are a few points to keep in mind:
+
+1. Our task pipeline operates in both evaluation and training modes. By default, it will be in evaluation mode and will output a ``GeneratorOutput`` object.
+   When in training mode, it will output a ``Parameter`` object where the data attribute contains the raw output from ``GeneratorOutput``.
+   The entire GeneratorOutput object will be saved in the ``full_response`` attribute, allowing it to be used later for evaluation.
+   To specify which input should be passed to the evaluation function, we will assign it to the ``eval_input`` attribute.
+
+2. If we want to train using few-shot in-context learning, we need to assign an ``id`` to our LLM call. This ``id`` will be used to trace the few-shot examples automatically.
+
+Now, let's pass a ``gpt-3.5-turbo`` model to our task pipeline and test both training and evaluation modes.
+
+.. code-block:: python
+
+    from adalflow.components.model_client.openai_client import OpenAIClient
+
+    adal.setup_env()
+
+    gpt_3_model = {
+        "model_client": OpenAIClient(),
+        "model_kwargs": {
+            "model": "gpt-3.5-turbo",
+            "max_tokens": 2000,
+            "temperature": 0.0,
+            "top_p": 0.99,
+            "frequency_penalty": 0,
+            "presence_penalty": 0,
+            "stop": None,
+        },
+    }
+
+Here is the code to test the task pipeline:
+
+.. code-block:: python
+
+    question = "I have a flute, a piano, a trombone, four stoves, a violin, an accordion, a clarinet, a drum, two lamps, and a trumpet. How many musical instruments do I have?"
+    task_pipeline = ObjectCountTaskPipeline(**gpt_3_model)
+    print(task_pipeline)
+
+    answer = task_pipeline(question)
+    print(answer)
+
+    # set it to train mode
+    task_pipeline.train()
+    answer = task_pipeline(question, id="1")
+    print(answer)
+    print(f"full_response: {answer.full_response}")
+
+The answer for the eval mode:
+
+.. code-block:: python
+
+    GeneratorOutput(id="1", data=8, error=None, usage=CompletionUsage(completion_tokens=113, prompt_tokens=113, total_tokens=226), raw_response='To find the total number of musical instruments you have, you simply need to count the individual instruments you listed. \n\nCounting the instruments:\n1 flute\n1 piano\n1 trombone\n1 violin\n1 accordion\n1 clarinet\n1 drum\n1 trumpet\n\nAdding the number of stoves and lamps, which are not musical instruments:\n4 stoves\n2 lamps\n\nTotal number of musical instruments = 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 = 8\n\nAnswer: 8', metadata=None)
+
+The answer for the train mode:
+
+.. code-block:: python
+
+    Parameter(name=Generator_output, requires_opt=True, param_type=generator_output (The output of the generator.), role_desc=Output from (llm) Generator, data=To find the total number of musical instruments you have, you simply need to count the individual instruments you listed.
+
+    Counting the instruments:
+    1 flute
+    1 piano
+    1 trombone
+    1 violin
+    1 accordion
+    1 clarinet
+    1 drum
+    1 trumpet
+
+    Adding the number of stoves and lamps, which are not musical instruments:
+    4 stoves
+    2 lamps
+
+    Total number of musical instruments = 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 = 8
+
+    Answer: 8, predecessors={Parameter(name=To_give_ta, requires_opt=True, param_type=prompt (Instruction to the language model on task, data, and format.), role_desc=To give task instruction to the language model in the system prompt, data=You will answer a reasoning question. Think step by step. The last line of your response should be of the following format: 'Answer: $VALUE' where VALUE is a numerical value., predecessors=set(), gradients=set(),            raw_response=None, input_args=None, traces={}), Parameter(name=To_provide, requires_opt=True, param_type=demos (A few examples to guide the language model.), role_desc=To provide few shot demos to the language model, data=None, predecessors=set(), gradients=set(),            raw_response=None, input_args=None, traces={})}, gradients=set(),            raw_response=None, input_args={'prompt_kwargs': {'system_prompt': Parameter(name=To_give_ta, requires_opt=True, param_type=prompt (Instruction to the language model on task, data, and format.), role_desc=To give task instruction to the language model in the system prompt, data=You will answer a reasoning question. Think step by step. The last line of your response should be of the following format: 'Answer: $VALUE' where VALUE is a numerical value., predecessors=set(), gradients=set(),            raw_response=None, input_args=None, traces={}), 'few_shot_demos': Parameter(name=To_provide, requires_opt=True, param_type=demos (A few examples to guide the language model.), role_desc=To provide few shot demos to the language model, data=None, predecessors=set(), gradients=set(),            raw_response=None, input_args=None, traces={}), 'input_str': 'I have a flute, a piano, a trombone, four stoves, a violin, an accordion, a clarinet, a drum, two lamps, and a trumpet. How many musical instruments do I have?'}, 'model_kwargs': {'model': 'gpt-3.5-turbo', 'max_tokens': 2000, 'temperature': 0.0, 'top_p': 0.99, 'frequency_penalty': 0, 'presence_penalty': 0, 'stop': None}}, traces={})
+
+
+So far, we have completed the task pipeline and ensured it works in both evaluation and training modes. Of course, if the performance is already perfect, there may be no need for further training, but evaluation is still essential.
+
+Our training pipeline can assist with both training and evaluation.
+
+
+Evaluate the task pipeline
+----------------------------
+
+Before we start the training, we should prepare three datasets: train, validation, and test datasets. An initial evaluation is necessary to check two things:
+
+1. **Overall Performance on Each Data Split:** We need to assess the performance on each data split. If the accuracy does not meet the required standards, we must plan for further evaluation and adjustments.
+
+2. **Performance Consistency Across Datasets:** We need to ensure that each split (train, validation, and test) performs comparably. This consistency is crucial so that the train and validation sets can serve as reliable indicators of test performance.
+
+Datasets
+~~~~~~~~~~~~
+
+We have prepared the dataset at ``adalflow.datasets.big_bench_hard``.
+We can load it with the following code:
+
+.. code-block:: python
+
+    from adalflow.datasets.big_bench_hard import BigBenchHard
+    from adalflow.utils.data import subset_dataset
+
+    def load_datasets(max_samples: int = None):
+        """Load the dataset"""
+        train_data = BigBenchHard(split="train")
+        val_data = BigBenchHard(split="val")
+        test_data = BigBenchHard(split="test")
+
+        # Limit the number of samples
+        if max_samples:
+            train_data = subset_dataset(train_data, max_samples)
+            val_data = subset_dataset(val_data, max_samples)
+            test_data = subset_dataset(test_data, max_samples)
+
+        return train_data, val_data, test_data
+
+We have 50, 50, 100 samples in the train, val, and test datasets, respectively. Here is one example of the loaded data sample:
+
+.. code-block:: python
+
+    Example(id='b0cffa3e-9dc8-4d8e-82e6-9dd7d34128df', question='I have a flute, a piano, a trombone, four stoves, a violin, an accordion, a clarinet, a drum, two lamps, and a trumpet. How many musical instruments do I have?', answer='8')
+
+The data sample is already of type ``DataClass`` and each sample is assigned with an ``id``, a ``question``, and an ``answer``.
+To note that the answer is in `str` format.
+
+
+Diagnose the task pipeline
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+To evaluate the task pipeline using the :meth:`diagnose<optim.trainer.trainer.Trainer>` method provided by our trainer,
+we can take advantage of the :class:`AdalComponent<optim.trainer.adal.AdalComponent>` interface.
+This interface class should be subclassed, allowing us to leverage its parallel processing capabilities, callback configuration, optimizer configuration, and built-in support for the teacher/backward engine. The AdalComponent works similarly to how PyTorch Lightning's LightningModule interacts with its Trainer.
+
+Here’s the minimum code required to get started on evaluating the task pipeline:
+
+.. code-block:: python
+
+    from adalflow.datasets.types import Example
+    from adalflow.eval.answer_match_acc import AnswerMatchAcc
+
+
+    class ObjectCountAdalComponent(adal.AdalComponent):
+        def __init__(self, model_client: adal.ModelClient, model_kwargs: Dict):
+            task = ObjectCountTaskPipeline(model_client, model_kwargs)
+            eval_fn = AnswerMatchAcc(type="exact_match").compute_single_item
+            super().__init__(task=task, eval_fn=eval_fn)
+
+        def handle_one_task_sample(self, sample: Example):
+            return self.task.call, {"question": sample.question, "id": sample.id}
+
+        def evaluate_one_sample(
+            self, sample: Example, y_pred: adal.GeneratorOutput
+        ) -> float:
+            y_label = -1
+            if y_pred and y_pred.data:
+                y_label = y_pred.data
+            return self.eval_fn(y=y_label, y_gt=sample.answer)
+
+Now, lets use the trainer.
+
+
+.. code-block:: python
+
+    def diagnose(
+        model_client: adal.ModelClient,
+        model_kwargs: Dict,
+    ) -> Dict:
+        from use_cases.question_answering.bhh_object_count.data import load_datasets
+
+        trainset, valset, testset = load_datasets()
+
+        adal_component = ObjectCountAdalComponent(model_client, model_kwargs)
+        trainer = adal.Trainer(adaltask=adal_component)
+        trainer.diagnose(dataset=trainset, split="train")
+        trainer.diagnose(dataset=valset, split="val")
+        trainer.diagnose(dataset=testset, split="test")
+
+File structure:
+
+.. code-block:: bash
+
+    .adalflow/
+    ├── ckpt/
+    │   └── ObjectCountAdalComponent/
+    │       ├── diagnose_{train, val, test}/  # Directory for training data diagnostics
+    │       │   ├── llm_counter_call.jsonl    # Sorted by score from lowest to highest
+    │       │   ├── logger_metadata.jsonl
+    │       │   ├── llm_counter_diagnose.json # Contains samples with score < 0.5, sorted by score
+    │       │   └── stats.json
+
+
+
+.. note::
+
+   As we save all data in default at `~/.adalflow`, you can create a soft link to the current directory to access the data easily
+   in your code editor.
+
+The `llm_counter_call.jsonl` file will contain 6 keys:
+
+1. "prompt_kwargs": the prompt_kwargs used in the call of ``llm_counter``.
+2. "model_kwargs": the model_kwargs used in the call of ``llm_counter``.
+3. "input": Everything that passed to the model_client (LLM).
+4. "output": GeneratorOutput object.
+5. "score": the performance score of the model on the dataset split.
+6. "time_stamp": the time stamp of the call.
+
+The items are ranked from the lowest to the highest score. The score is the performance score of the model on the dataset split.
+If you have passed the ``id`` to the call, you will find it in the ``output``.
+
+In the ``{}_diagnose.json`` file, we save what can be used to manually diagnose the errors:
+- "id": the id of the sample.
+- "score": the performance score of the model on the dataset split.
+- "prompt_kwargs": the prompt_kwargs used in the call of ``llm_counter``.
+- "raw_response": the raw_response of the model.
+- "answer": the answer of the sample.
+- "dataset_item": the dataset item where you can find sample to compare with.
+
+
+Here is the stats:
+
+.. list-table:: Scores by Split
+   :header-rows: 1
+
+   * - Split
+     - Train
+     - Val
+     - Test
+   * - Score
+     - 0.88 (50)
+     - 0.90 (50)
+     - 0.87 (100)
+
+The model already performs quite well on the dataset.
+Let's see if we can optimize it further with either few-shot or zero-shot prompt optimization or even both.
+
+
+Train Setup
+------------------------------
+
+Prepare AdalComponent for training
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+To be able to train, we will add a few attributes and define a few methods in our ``ObjectCountAdalComponent`` class.
+
+First, ``loss_fn`` where we use ``ada.EvalFnToTextLoss`` to compute the loss(``Parameter``) where it takes the ``eval_fn`` and the ``eval_fn_desc`` at the initialization.
+This loss function will pass whatever user set at ``kwargs`` to the ``eval_fn`` and compute the loss and handle the ``textual gradient`` for the loss function.
+
+.. code-block:: python
+
+    class ObjectCountAdalComponent(adal.AdalComponent):
+        def __init__(
+            self,
+            model_client: adal.ModelClient,
+            model_kwargs: Dict,
+            backward_engine_model_config: Dict,
+            teacher_model_config: Dict,
+            text_optimizer_model_config: Dict,
+        ):
+            task = ObjectCountTaskPipeline(model_client, model_kwargs)
+            eval_fn = AnswerMatchAcc(type="exact_match").compute_single_item
+            loss_fn = adal.EvalFnToTextLoss(
+                eval_fn=eval_fn,
+                eval_fn_desc="exact_match: 1 if str(y) == str(y_gt) else 0",
+            )
+            super().__init__(task=task, eval_fn=eval_fn, loss_fn=loss_fn)
+
+            self.backward_engine_model_config = backward_engine_model_config
+            self.teacher_model_config = teacher_model_config
+            self.text_optimizer_model_config = text_optimizer_model_config
+
+
+
+Second, :meth:`handle_one_loss_sample` where we will return the loss function and the ``kwargs`` to the loss function.
+We need to convert the the ground truth into a ``Parameter`` and set the ``eval_input`` that will be used as value to the ``eval_fn``
+when we evaluate the model.
+
+.. code-block:: python
+
+    def handle_one_loss_sample(self, sample: Example, pred: adal.Parameter):
+        # prepare gt parameter
+        y_gt = adal.Parameter(
+            name="y_gt",
+            data=sample.answer,
+            eval_input=sample.answer,
+            requires_opt=False,
+        )
+
+        # pred's full_response is the output of the task pipeline which is GeneratorOutput
+        pred.eval_input = pred.full_response.data
+        return self.loss_fn, {"kwargs": {"y": pred, "y_gt": y_gt}}
+
+Third, if you intent to train ``ParameterType.PROMPT``, we will need to set the ``backward_engine`` which is a subclass of ``Generator`` with its own ``template``.
+We provided a ``configure_backward_engine_helper`` method to smooth this setup; it requires only the ``model_client`` and the ``model_kwargs``.
+
+.. code-block:: python
+
+    def configure_backward_engine(self):
+        super().configure_backward_engine_helper(
+            **self.backward_engine_model_config
+        )
+
+If we also need to train the ``ParameterType.DEMOS``, we will need to set the ``teacher_generator`` which is exactly the same setup as your ``llm_counter`` but
+with your configured ``model_client`` and ``model_kwargs``.
+
+.. code-block:: python
+
+    def configure_teacher_generator(self):
+        super().configure_teacher_generator_helper(
+            **self.teacher_generator_model_config
+        )
+
+
+Finally, we need to configure the optimizer. We will use both the ``DemoOptimizer`` (in default configured with ``adal.optim.few_shot.few_shot_optimizer.BootstrapFewShot``) and the ``PromptOptimizer`` (in default configured with ``adal.optim.text_grad.tgd_optimizer.TGDOptimizer``).
+
+.. code-block:: python
+
+    def configure_optimizers(self):
+        to = super().configure_text_optimizer_helper(**self.text_optimizer_model_config)
+        do = super().configure_demo_optimizer_helper()
+        return to  + do
+
+Use the trainer
+~~~~~~~~~~~~~~~~~~~~
+
+Now, we can use the trainer to train the model.
+
+.. code-block:: python
+
+    def train(
+        train_batch_size=4,  # larger batch size is not that effective, probably because of llm's lost in the middle
+        raw_shots: int = 1,
+        bootstrap_shots: int = 1,
+        max_steps=1,
+        num_workers=4,
+        strategy="random",
+        debug=False,
+    ):
+        adal_component = ObjectCountAdalComponent(
+            **gpt_3_model,
+            teacher_model_config=gpt_4o_model,
+            text_optimizer_model_config=gpt_4o_model,
+            backward_engine_model_config=gpt_4o_model
+        )
+        print(adal_component)
+        trainer = Trainer(
+            train_batch_size=train_batch_size,
+            strategy=strategy,
+            max_steps=max_steps,
+            num_workers=num_workers,
+            adaltask=adal_component,
+            raw_shots=raw_shots,
+            bootstrap_shots=bootstrap_shots,
+            debug=debug,
+            weighted_sampling=True,
+        )
+        print(trainer)
+
+        train_dataset, val_dataset, test_dataset = load_datasets()
+        trainer.fit(
+            train_dataset=train_dataset,
+            val_dataset=val_dataset,
+            test_dataset=test_dataset,
+            debug=debug,
+        )
+
+
+
+Train in Debug mode
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. code-block:: python
+
+    train(debug=True, max_steps=12, strategy="constrained")
+
+Using the ``debug`` will show us two samples: one successful and one failed sample.
+And it will not only check all necessary steps/methods to try its best to ensure you
+have implemented all parts correctly before the training on the whole dataset which can be expensive.
+Also, it is important to make sure the ``backward_engine`` is giving the right feedback and the ``optimizer`` is
+following the instruction to make correct proposal.
+
+Debug mode will turn on the log and set it to ``DEBUG`` level.
+
+.. code-block:: bash
+
+    .adalflow/
+    ├── ckpt/
+    │   └── ObjectCountAdalComponent/
+    │       ├── diagnose_{train, val, test}/  # Directory for training data diagnostics
+    │       │   ├── llm_counter_call.jsonl    # Sorted by score from lowest to highest
+    │       │   ├── logger_metadata.jsonl
+    │       │   ├── llm_counter_diagnose.json # Contains samples with score < 0.5, sorted by score
+    │       │   └── stats.json
+    │       ├── debug_text_grads                          # Directory for debug mode with text optimizer
+    │       │   ├── lib.log                    # Log file
+    │       │   ├── trace_graph_sum.png       # Trace graph with textual feedback and new proposed value
+    │       │   ├── trace_graph_sum_root.json # Json representation of the root loss node (sum of the success and fail loss)
+
+Here is how our trace_graph looks like: :doc:`trace_graph <../tutorials/trace_graph>`.
+
+
+Train with Text-Gradient Descent
+-----------------------------------
+To train, we simply set the ``debug`` to ``False``.
+
+To do textual-gradient descent training for our task pipeline, we will go back to the task pipeline to set the `requires_opt` to `False` for the `few_shot_demos` parameter and
+`requires_opt=True` for the `system_prompt` parameter.
+
+.. code-block:: python
+
+    system_prompt = adal.Parameter(
+                data="You will answer a reasoning question. Think step by step. The last line of your response should be of the following format: 'Answer: $VALUE' where VALUE is a numerical value.",
+                role_desc="To give task instruction to the language model in the system prompt",
+                requires_opt=True,
+                param_type=ParameterType.PROMPT,
+            )
+    few_shot_demos = adal.Parameter(
+        data=None,
+        role_desc="To provide few shot demos to the language model",
+        requires_opt=False,
+        param_type=ParameterType.DEMOS,
+    )
+
+For the text optimizer, we have two training strategy: ``random`` and ``constrained``.
+The ``random`` strategy runs a batch of loss and backward propagation and then validate it on the ``validation`` and ``test`` dataset at each step.
+This is a standard training strategy, and it is used by libraries like ``Dspy`` and ``Text-grad``.
+You can refer :meth:`optim.trainer.Trainer.fit` for more details.
+
+The ``constrained`` strategy is unique to AdalFlow library where it runs a moving batch capped at maximum 20 samples, and it subsample the correct and failed samples (each maximum at 4).
+Before it runs the validations on the full ``validation`` and ``test`` dataset, it will run a validation on the moving sampled subset and the moving batch. It will try 5 proposals on the moving batch and only let a proposal that can beat the current subset and moving batch performance before it can be validated on the full dataset.
+We find it often more effective than the ``random`` strategy.
+
+Additionally, we estimate the maximum validataion score each validation can get. Once we know the maximum score is below our minimum requirement (the last highest validation score), we stop the evaluation to save time and cost.
+
+After the training, we will all information saved in ``.adalflow/ckpt/ObjectCountAdalComponent/``.
+With file names like:
+
+.. code-block:: bash
+
+    .adalflow/
+    ├── ckpt/
+    │   └── ObjectCountAdalComponent/
+    │       random_max_steps_8_bb908_run_1.json # The last training run for random strategy
+    │       constrained_max_steps_8_a1754_run_1.json # The last training run for constrained strategy
+
+
+Here is an example of how our ckpt file looks like: :doc:`ckpt_file <../tutorials/ckpt_file>`.
+This file is a direct `to_dict`  (json) representation of :class:`TrainerResult<optim.types.TrainerResult>`.
+
+
+Train with Few-shot Bootstrap
+------------------------------
+As we have defined a ``ParameterType.DEMOS`` in our ``ObjectCountAdalComponent``, we can train the model with few-shot bootstrap.
+We will set ``raw_shots=0`` and ``bootstrap_shots=1`` in the ``train`` method.
+In default, our demonstrations use the teacher's direct raw response, with the purpose to teach the weaker model how to reason the answer.
+We call this "Learn to reason" few-shot bootstrap.
+
+Note: before we start the training, it will be worth to check if the teacher model is performing better so that the student can learn from the teacher.
+We can achieve this using the diagnose method while setting the `model_client` and `model_kwargs` to the teacher model.
+Additionally, ensure you set the `split` to `train_teacher` etc to ensure the previous diagnose on the student model is not overwritten.
+Here is the teach model performance on the zero-shot prompt:
+
+.. list-table:: Scores by teacher mode (gpt-4o) on the same high-performing starting prompt
+   :header-rows: 1
+   :widths: 20 20 20 20
+
+   * - Method
+     - Train
+     - Val
+     - Test
+   * - Start (manual prompt)
+     - 0.98 (50 samples)
+     - 1.0 (50 samples)
+     - 0.98 (100 samples)
+
+
+We will show how a single demonstration can help push the model performance to 92% on validation and 97% on test.
+
+To do few-shot for our task pipeline, we will go back to the task pipeline to set the `requires_opt` to `True` for the `few_shot_demos` parameter and
+turn off the `requires_opt` for the `system_prompt` parameter.
+
+.. code-block:: python
+
+    system_prompt = adal.Parameter(
+                data="You will answer a reasoning question. Think step by step. The last line of your response should be of the following format: 'Answer: $VALUE' where VALUE is a numerical value.",
+                role_desc="To give task instruction to the language model in the system prompt",
+                requires_opt=False,
+                param_type=ParameterType.PROMPT,
+            )
+    few_shot_demos = adal.Parameter(
+        data=None,
+        role_desc="To provide few shot demos to the language model",
+        requires_opt=True,
+        param_type=ParameterType.DEMOS,
+    )
+
+
+Here is our top performing few-shot example:
+
+.. list-table:: Scores for One-shot Bootstrap
+   :header-rows: 1
+   :widths: 10 40 25 25
+
+   * - Method
+     - Prompt
+     - Val
+     - Test
+   * - Start
+     - None
+     - 0.90
+     - 0.87
+   * - Optimized One-shot
+     - """Example: 'To find the total number of objects you have, you need to count each individual\n  item. In this case, you have:\n\n  1 microwave\n\n  1 lamp\n\n  4 cars\n\n  1 stove\n\n  1 toaster\n\n  1 bed\n\n\n  Adding these together:\n\n  1 + 1 + 4 + 1 + 1 + 1 = 9\n\n\n  Therefore, you have 9 objects in total.\n\n  Answer: 9'""
+     - 0.96 (**+6%**, 4% < teacher)
+     - 0.94 (**+7%**, 4% < teacher)
+
+
+
+
+
+Benchmarking
+------------------------------
+We compared our performance with text-grad. Here are our stats:
+The same prompt, text-grad gets 0.72 on the validation set. and it optimized it to 0.89.
+But text-grad use more lengthy prompt, where it takes more than 80s to run a backpropagation on a batch size of 4.
+Yet, we only take 12s.
+Also AdalFlow has better converage rate in general.
+We also leverage single message prompt, sending the whole template to the model's system message, making this whole development process easy.
+
+.. list-table:: Optimized Scores comparison on the same prompt on test set (gpt-3.5-turbo)
+   :header-rows: 1
+   :widths: 50 50
+
+   * - Method
+     - Test
+   * - Text-grad (start)
+     - 0.72
+   * - Text-grad (optimized)
+     - 0.89
+   * - AdalFlow (start)
+     - 0.87
+   * - AdalFlow(text-grad optimized)
+     - 0.91
+   * - AdalFlow ("Learn-to-reason" one-shot)
+     - **0.94**
+
+.. note::
+    In the start we use same prompt but we use a single template which achieves much better zero-shot performance than text-grad which sends the system prompt to system message and the input to user message.
+
+.. admonition:: References
+   :class: highlight
+
+   .. [1] Text-grad: https://arxiv.org/abs/2406.07496
+   .. [2] DsPy: https://arxiv.org/abs/2310.03714
+   .. [3] ORPO: https://arxiv.org/abs/2309.03409
diff --git a/use_cases/config.py b/use_cases/config.py
index 4b76a496..6eb5b3ed 100644
--- a/use_cases/config.py
+++ b/use_cases/config.py
@@ -25,6 +25,19 @@
     },
 }
 
+# https://openai.com/api/pricing/
+# use this for evaluation
+gpt_4o_mini_model = {
+    "model_client": OpenAIClient(),
+    "model_kwargs": {
+        "model": "gpt-4o-mini",
+        "temperature": 1,
+        "top_p": 0.99,
+        "max_tokens": 1000,
+        # "frequency_penalty": 1,  # high for nto repeating prompt
+    },
+}
+
 gpt_4o_model = {
     "model_client": OpenAIClient(),
     "model_kwargs": {
@@ -36,4 +49,5 @@
     },
 }
 
+
 dataset_path = "cache_datasets"
diff --git a/use_cases/question_answering/bbh/data.py b/use_cases/question_answering/bbh/data.py
index 768af463..c844c476 100644
--- a/use_cases/question_answering/bbh/data.py
+++ b/use_cases/question_answering/bbh/data.py
@@ -81,11 +81,24 @@ def parse_integer_answer(answer: str):
     return answer
 
 
-def load_datasets(max_samples: int = None):
+@adal.fun_to_component
+def extract_answer(answer: str) -> str:
+    try:
+        pattern = re.compile(r"Answer:\s*(.*)", re.DOTALL)
+        match = pattern.search(answer)
+        if match:
+            return match.group(1).strip()
+        else:
+            return ""
+    except ValueError:
+        return ""
+
+
+def load_datasets(max_samples: int = None, task_name: str = "BBH_object_count"):
     """Load the dataset"""
-    train_data = BigBenchHard(split="train")
-    val_data = BigBenchHard(split="val")
-    test_data = BigBenchHard(split="test")
+    train_data = BigBenchHard(split="train", task_name=task_name)
+    val_data = BigBenchHard(split="val", task_name=task_name)
+    test_data = BigBenchHard(split="test", task_name=task_name)
 
     # Limit the number of samples
     if max_samples:
diff --git a/use_cases/question_answering/bbh/bhh_object_count/__init__.py b/use_cases/question_answering/bbh/object_count/__init__.py
similarity index 100%
rename from use_cases/question_answering/bbh/bhh_object_count/__init__.py
rename to use_cases/question_answering/bbh/object_count/__init__.py
diff --git a/use_cases/question_answering/bbh/bhh_object_count/diagnose.py b/use_cases/question_answering/bbh/object_count/diagnose.py
similarity index 93%
rename from use_cases/question_answering/bbh/bhh_object_count/diagnose.py
rename to use_cases/question_answering/bbh/object_count/diagnose.py
index 1530d714..f1802657 100644
--- a/use_cases/question_answering/bbh/bhh_object_count/diagnose.py
+++ b/use_cases/question_answering/bbh/object_count/diagnose.py
@@ -1,6 +1,6 @@
 from typing import Dict
 import adalflow as adal
-from use_cases.question_answering.bhh_object_count.task import ObjectCountTaskPipeline
+from use_cases.question_answering.bbh.object_count.task import ObjectCountTaskPipeline
 
 from adalflow.datasets.types import Example
 from adalflow.eval.answer_match_acc import AnswerMatchAcc
@@ -30,7 +30,7 @@ def diagnose(
     model_client: adal.ModelClient,
     model_kwargs: Dict,
 ) -> Dict:
-    from use_cases.question_answering.bhh_object_count.data import load_datasets
+    from use_cases.question_answering.bbh.data import load_datasets
 
     trainset, valset, testset = load_datasets()
     # use max_samples=10 to test the code
diff --git a/use_cases/question_answering/bbh/bhh_object_count/task.py b/use_cases/question_answering/bbh/object_count/task.py
similarity index 100%
rename from use_cases/question_answering/bbh/bhh_object_count/task.py
rename to use_cases/question_answering/bbh/object_count/task.py
diff --git a/use_cases/question_answering/bbh/bhh_object_count/train_new.py b/use_cases/question_answering/bbh/object_count/train_new.py
similarity index 97%
rename from use_cases/question_answering/bbh/bhh_object_count/train_new.py
rename to use_cases/question_answering/bbh/object_count/train_new.py
index 3b38898e..e3ed9de7 100644
--- a/use_cases/question_answering/bbh/bhh_object_count/train_new.py
+++ b/use_cases/question_answering/bbh/object_count/train_new.py
@@ -1,4 +1,4 @@
-from use_cases.question_answering.bhh_object_count.task import (
+from use_cases.question_answering.bbh.object_count.task import (
     ObjectCountTaskPipeline,
 )
 
@@ -80,7 +80,7 @@ def train_diagnose(
     model_client: adal.ModelClient,
     model_kwargs: Dict,
 ) -> Dict:
-    from use_cases.question_answering.bhh_object_count.data import load_datasets
+    from use_cases.question_answering.bbh.data import load_datasets
 
     trainset, valset, testset = load_datasets()
 
diff --git a/use_cases/question_answering/bbh/word_sorting/diagnose.py b/use_cases/question_answering/bbh/word_sorting/diagnose.py
new file mode 100644
index 00000000..dcd490fc
--- /dev/null
+++ b/use_cases/question_answering/bbh/word_sorting/diagnose.py
@@ -0,0 +1,91 @@
+from typing import Dict
+import adalflow as adal
+from use_cases.question_answering.bbh.word_sorting.task import (
+    QuestionAnswerTaskPipeline,
+)
+
+from adalflow.datasets.types import Example
+from adalflow.eval.llm_as_judge import DefaultLLMJudge
+from use_cases.question_answering.bbh.data import load_datasets
+from use_cases.config import gpt_3_model
+
+
+judgement_query = r"""Does the predicted answer match with the ground truth answer? PLEASE Ignore the difference of separators between words.
+Say True if it matches, False if not.
+Example:
+Question: Sort the following words alphabetically: List: syndrome therefrom
+Ground truth answer: syndrome therefrom
+Predicted answer: syndrome, therefrom
+Answer: True"""
+
+
+class WordSortingAdalComponent(adal.AdalComponent):
+    def __init__(
+        self,
+        model_client: adal.ModelClient,
+        model_kwargs: Dict,
+        llm_judge_model_config: Dict,
+    ):
+        task = QuestionAnswerTaskPipeline(model_client, model_kwargs)
+
+        llm_judge = DefaultLLMJudge(
+            **llm_judge_model_config,
+            output_type="float",
+            jugement_query=judgement_query,
+            use_cache=True,
+        )
+        eval_fn = llm_judge.call
+        # eval_fn = lambda question, gt_answer, pred_answer: 1
+        super().__init__(task=task, eval_fn=eval_fn)
+
+    def handle_one_task_sample(self, sample: Example):
+        return self.task.call, {"question": sample.question, "id": sample.id}
+
+    def evaluate_one_sample(
+        self, sample: Example, y_pred: adal.GeneratorOutput
+    ) -> float:
+        y_label = ""
+        if y_pred and y_pred.data:
+            y_label = y_pred.data
+        return self.eval_fn(
+            question=sample.question, gt_answer=sample.answer, pred_answer=y_label
+        )
+
+
+def evaluate_one_sample():
+
+    trainset, valset, testset = load_datasets(task_name="BBH_word_sorting")
+    adal_component = WordSortingAdalComponent(
+        **gpt_3_model, llm_judge_model_config=gpt_3_model
+    )
+    example = trainset[1]
+    call, kwargs = adal_component.handle_one_task_sample(example)
+    output = call(**kwargs)
+    print(f"output: {output}")
+    print(f"trainset[0]: {example}")
+    score = adal_component.evaluate_one_sample(example, output)
+    print(score)
+
+
+def diagnose(
+    model_client: adal.ModelClient,
+    model_kwargs: Dict,
+) -> Dict:
+
+    trainset, valset, testset = load_datasets(task_name="BBH_word_sorting")
+    adal_component = WordSortingAdalComponent(
+        model_client, model_kwargs, llm_judge_model_config=gpt_3_model
+    )
+    trainer = adal.Trainer(adaltask=adal_component, num_workers=4)
+    trainer.diagnose(dataset=trainset, split="train")
+    trainer.diagnose(dataset=valset, split="val")
+    trainer.diagnose(dataset=testset, split="test")
+
+
+if __name__ == "__main__":
+    from use_cases.config import (
+        gpt_3_model,
+    )
+
+    # evaluate_one_sample()
+    diagnose(**gpt_3_model)
diff --git a/use_cases/question_answering/bbh/word_sorting/task.py b/use_cases/question_answering/bbh/word_sorting/task.py
index d5ffbe35..2ddcca6e 100644
--- a/use_cases/question_answering/bbh/word_sorting/task.py
+++ b/use_cases/question_answering/bbh/word_sorting/task.py
@@ -3,7 +3,7 @@
 from adalflow.optim.parameter import ParameterType
 
 from use_cases.question_answering.bbh.data import (
-    parse_integer_answer,
+    extract_answer,
 )
 
 
@@ -23,7 +23,6 @@
 
 from typing import Dict, Union
 import adalflow as adal
-from adalflow.datasets.big_bench_hard import BigBenchHard
 
 
 class QuestionAnswerTaskPipeline(adal.Component):
@@ -31,8 +30,7 @@ def __init__(self, model_client: adal.ModelClient, model_kwargs: Dict):
         super().__init__()
 
         system_prompt = adal.Parameter(
-            # data="You will answer a reasoning question. Think step by step. The last line of your response should be of the following format: 'Answer: $VALUE' where VALUE is a numerical value.",
-            data=BigBenchHard.get_default_task_instruction(),
+            data="You will answer a reasoning question. Think step by step. The last line of your response should be of the following format: 'Answer: $VALUE' where VALUE is the answer.",
             role_desc="To give task instruction to the language model in the system prompt",
             requires_opt=True,
             param_type=ParameterType.PROMPT,
@@ -45,7 +43,7 @@ def __init__(self, model_client: adal.ModelClient, model_kwargs: Dict):
             param_type=ParameterType.DEMOS,
         )
 
-        self.llm_counter = adal.Generator(
+        self.llm = adal.Generator(
             model_client=model_client,
             model_kwargs=model_kwargs,
             template=few_shot_template,
@@ -53,14 +51,14 @@ def __init__(self, model_client: adal.ModelClient, model_kwargs: Dict):
                 "system_prompt": system_prompt,
                 "few_shot_demos": few_shot_demos,
             },
-            output_processors=parse_integer_answer,
+            output_processors=extract_answer,
             use_cache=True,
         )
 
     def call(
         self, question: str, id: str = None
     ) -> Union[adal.GeneratorOutput, adal.Parameter]:
-        output = self.llm_counter(prompt_kwargs={"input_str": question}, id=id)
+        output = self.llm(prompt_kwargs={"input_str": question}, id=id)
         return output
 
 
@@ -76,7 +74,7 @@ def test_word_sorting_task():
     )
 
     example = train_dataset[0]
-    question = example["question"]
+    question = example.question
     print(example)
 
     answer = task_pipeline(question)

From 1d805b1f344cabaff72111710a91d7e9737e2fb8 Mon Sep 17 00:00:00 2001
From: Li Yin <li.yin.gravity@gmail.com>
Date: Mon, 26 Aug 2024 22:56:39 -0700
Subject: [PATCH 6/8] work count training fine, > 83%, +4 that sota

---
 adalflow/adalflow/core/generator.py           |   8 +-
 .../optim/few_shot/bootstrap_optimizer.py     |  23 ++-
 adalflow/adalflow/optim/parameter.py          |   2 +-
 .../optim/text_grad/text_loss_with_eval_fn.py |   1 -
 adalflow/adalflow/optim/trainer/adal.py       |   3 +
 adalflow/adalflow/optim/trainer/trainer.py    |  10 +-
 adalflow/tests/test_random_sample.py          |  51 ++++++
 .../bbh/object_count/train_new.py             |   5 +-
 .../bbh/word_sorting/train.py                 | 160 ++++++++++++++++++
 9 files changed, 250 insertions(+), 13 deletions(-)
 create mode 100644 adalflow/tests/test_random_sample.py
 create mode 100644 use_cases/question_answering/bbh/word_sorting/train.py

diff --git a/adalflow/adalflow/core/generator.py b/adalflow/adalflow/core/generator.py
index 58c03244..1996b19a 100644
--- a/adalflow/adalflow/core/generator.py
+++ b/adalflow/adalflow/core/generator.py
@@ -420,8 +420,12 @@ def forward(
         if self.mock_output:
             output = GeneratorOutput(data=self.mock_output_data)
         else:
-            if self.teacher_mode:
+            if self.teacher_mode and not isinstance(self, BackwardEngine):
                 if not self._teacher:
+                    print(
+                        f"prompt_kwargs: {prompt_kwargs}, model_kwargs: {model_kwargs}"
+                    )
+                    print(f"names: {self.name}")
                     raise ValueError("Teacher generator is not set.")
                 log.info(f"Using teacher: {self._teacher}")
                 input_args = {
@@ -848,6 +852,8 @@ def __init__(self, **kwargs):
             kwargs = {}
         kwargs["template"] = FEEDBACK_ENGINE_TEMPLATE
         super().__init__(**kwargs)
+        self.name = "BackwardEngine"
+        self.teacher_mode = False
 
     @staticmethod
     def failure_message_to_optimizer(
diff --git a/adalflow/adalflow/optim/few_shot/bootstrap_optimizer.py b/adalflow/adalflow/optim/few_shot/bootstrap_optimizer.py
index 7fd01f68..39ed7840 100644
--- a/adalflow/adalflow/optim/few_shot/bootstrap_optimizer.py
+++ b/adalflow/adalflow/optim/few_shot/bootstrap_optimizer.py
@@ -125,10 +125,17 @@ def sample(
                     raise ValueError(
                         f"score must be provided for each demo, id: {demo.id}, all scores: {self._teacher_scores}"
                     )
+                if demo_score < 0 or demo_score > 1:
+                    raise ValueError(f"score must be in range [0, 1], got {demo_score}")
 
                 w = demo_score
                 student_demo_score = self._student_scores.get(demo.id, None)
+
                 if student_demo_score is not None:
+                    if student_demo_score < 0 or student_demo_score > 1:
+                        raise ValueError(
+                            f"score must be in range [0, 1], got {student_demo_score}"
+                        )
                     # if demo.id in demos and demos[demo.id].score is not None:
                     w = (
                         w
@@ -138,6 +145,9 @@ def sample(
                     if w < 0:
                         w = 0
                 weights.append(w)
+            # avoid all 0 weights
+            if all(weight == 0 for weight in weights):
+                weights = [1.0] * len(augmented_options)
         # print(f"augs: {augmented_options}")
         sampled_augmented_demos = (
             random_sample(
@@ -163,8 +173,17 @@ def sample(
             # for those exist in the demos, assign higher score with failed demos
             for i, demo in enumerate(filtered_dataset):
                 student_demo_score = self._student_scores.get(demo.id, None)
+
                 if student_demo_score is not None:
-                    raw_weights[i] += 1 - student_demo_score
+                    # ensure the score is in range [0, 1]
+                    if student_demo_score < 0 or student_demo_score > 1:
+                        raise ValueError(
+                            f"score must be in range [0, 1], got {student_demo_score}"
+                        )
+                    raw_weights[i] = 1 - student_demo_score
+            if all(weight == 0 for weight in raw_weights):
+
+                raw_weights = [1.0] * len(filtered_dataset)  # Equal probability for all
         sampled_raw_demos = random_sample(
             filtered_dataset, raw_shots, replace=False, weights=raw_weights
         )
@@ -258,7 +277,7 @@ def step(self):
 
     def __str__(self) -> str:
         s = f"BootstrapFewShot(raw_shots={self._raw_shots}, bootstrap_shots={self._bootstrap_shots}, \
-            params={[p.name for p in self.params]}, dataset={len(self.dataset)})"
+            params={[p.name for p in self.params]}, dataset={len(self.dataset) if self.dataset else 0})"
         return s
 
     def __repr__(self) -> str:
diff --git a/adalflow/adalflow/optim/parameter.py b/adalflow/adalflow/optim/parameter.py
index 546d16bc..abad6e0c 100644
--- a/adalflow/adalflow/optim/parameter.py
+++ b/adalflow/adalflow/optim/parameter.py
@@ -463,7 +463,7 @@ def draw_graph(
             if filepath
             else os.path.join(root_path, "graphs", filename)
         )
-        print(f"Saving graph to {filepath}")
+        print(f"Saving graph to {filepath}.{format}")
 
         def wrap_text(text, width):
             """Wrap text to the specified width, considering HTML breaks."""
diff --git a/adalflow/adalflow/optim/text_grad/text_loss_with_eval_fn.py b/adalflow/adalflow/optim/text_grad/text_loss_with_eval_fn.py
index e1928308..4aca3b44 100644
--- a/adalflow/adalflow/optim/text_grad/text_loss_with_eval_fn.py
+++ b/adalflow/adalflow/optim/text_grad/text_loss_with_eval_fn.py
@@ -152,7 +152,6 @@ def forward(
         eval_inputs = {}
         for k, v in kwargs.items():
             eval_inputs[k] = v.eval_input
-
         score: float = self.eval_fn(**eval_inputs)
 
         # Create a parameter
diff --git a/adalflow/adalflow/optim/trainer/adal.py b/adalflow/adalflow/optim/trainer/adal.py
index fbd95239..69ae067d 100644
--- a/adalflow/adalflow/optim/trainer/adal.py
+++ b/adalflow/adalflow/optim/trainer/adal.py
@@ -67,6 +67,8 @@ def __init__(
         self.backward_engine_model_config = backward_engine_model_config
         self.teacher_model_config = teacher_model_config
         self.text_optimizer_model_config = text_optimizer_model_config
+        self._demo_optimizers = None
+        self._text_optimizers = None
 
     def _set_param_values(self, prompts: List[PromptData]):
         r"""Set the parameters for the task. Used to resume from ckpt."""
@@ -524,6 +526,7 @@ def configure_teacher_generator(self):
             raise ValueError("Model client is not configured.")
         if not self.teacher_model_config.get("model_kwargs"):
             raise ValueError("Model kwargs is not configured.")
+        print("Configuring teacher generator.")
         self.configure_teacher_generator_helper(**self.teacher_model_config)
 
     def configure_teacher_generator_helper(
diff --git a/adalflow/adalflow/optim/trainer/trainer.py b/adalflow/adalflow/optim/trainer/trainer.py
index 4ac4b934..969d38ac 100644
--- a/adalflow/adalflow/optim/trainer/trainer.py
+++ b/adalflow/adalflow/optim/trainer/trainer.py
@@ -338,6 +338,8 @@ def fit(
                     self.exclude_input_fields_from_bootstrap_demos
                 )
             self.adaltask.configure_teacher_generator()
+            print("Configured demo optimizers")
+            # return
         else:
             print("No trainable demo params to optimize")
             self.demo_optimizers = []
@@ -383,8 +385,8 @@ def fit(
 
         if debug:
             print("Debugging mode")
-            if len(self.text_optimizers) > 0:
-                self._fit_text_grads_one_step_for_debug(train_loader)
+            # if len(self.text_optimizers) > 0:
+            #     self._fit_text_grads_one_step_for_debug(train_loader)
 
             if len(self.demo_optimizers) > 0:
                 self._fit_demos_one_step_for_debug(
@@ -672,8 +674,8 @@ def _fit_demos_one_step_for_debug(
             )
             # for loss in losses_student:
             #     loss.backward()
-
-            eval_result = self.adaltask.evaluate_samples(batch, y_preds_student)
+            y_preds_outputs = [p.full_response for p in y_preds_student]
+            eval_result = self.adaltask.evaluate_samples(batch, y_preds_outputs)
             print(f"Eval result: {eval_result.avg_score}")
             eval_score_per_item = eval_result.per_item_scores
 
diff --git a/adalflow/tests/test_random_sample.py b/adalflow/tests/test_random_sample.py
new file mode 100644
index 00000000..3cc6f56f
--- /dev/null
+++ b/adalflow/tests/test_random_sample.py
@@ -0,0 +1,51 @@
+import unittest
+from typing import TypeVar
+
+# Assuming the random_sample function is defined here or imported
+T_co = TypeVar("T_co", covariant=True)
+
+
+from adalflow.core.functional import random_sample
+
+
+class TestRandomSample(unittest.TestCase):
+
+    def setUp(self):
+        """Set up a common dataset for testing."""
+        self.dataset = [1, 2, 3, 4, 5]
+
+    def test_random_sample_no_replacement(self):
+        """Test random sampling without replacement."""
+        result = random_sample(self.dataset, 3, replace=False)
+        self.assertEqual(len(result), 3)
+        self.assertTrue(all(item in self.dataset for item in result))
+        self.assertEqual(len(set(result)), 3)  # No duplicates
+
+    def test_random_sample_with_replacement(self):
+        """Test random sampling with replacement."""
+        result = random_sample(self.dataset, 10, replace=True)
+        self.assertEqual(len(result), 10)
+        self.assertTrue(all(item in self.dataset for item in result))
+
+    def test_weighted_sampling_all_zero_weights(self):
+        """Test weighted sampling with all zero weights."""
+        result = random_sample(self.dataset, 3, replace=False, weights=[0, 0, 0, 0, 0])
+        self.assertEqual(len(result), 3)
+        self.assertTrue(all(item in self.dataset for item in result))
+
+    def test_weights_none(self):
+        """Test weighted sampling with None weights."""
+        result = random_sample(self.dataset, 3, replace=False, weights=None)
+        self.assertEqual(len(result), 3)
+        self.assertTrue(all(item in self.dataset for item in result))
+
+    def test_weighted_sampling_normal_weights(self):
+        """Test weighted sampling with normal weights."""
+        weights = [0.1, 0.2, 0.3, 0.4, 0.0]
+        result = random_sample(self.dataset, 3, replace=False, weights=weights)
+        self.assertEqual(len(result), 3)
+        self.assertTrue(all(item in self.dataset for item in result))
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/use_cases/question_answering/bbh/object_count/train_new.py b/use_cases/question_answering/bbh/object_count/train_new.py
index e3ed9de7..254c7c96 100644
--- a/use_cases/question_answering/bbh/object_count/train_new.py
+++ b/use_cases/question_answering/bbh/object_count/train_new.py
@@ -12,6 +12,7 @@
 
 from adalflow.datasets.types import Example
 from adalflow.eval.answer_match_acc import AnswerMatchAcc
+from use_cases.question_answering.bbh.data import load_datasets
 
 
 class ObjectCountAdalComponent(adal.AdalComponent):
@@ -95,7 +96,6 @@ def train_diagnose_teacher(
     model_client: adal.ModelClient,
     model_kwargs: Dict,
 ) -> Dict:
-    from use_cases.question_answering.bhh_object_count.data import load_datasets
 
     trainset, valset, testset = load_datasets()
 
@@ -106,9 +106,6 @@ def train_diagnose_teacher(
     trainer.diagnose(dataset=testset, split="test_teacher")
 
 
-from use_cases.question_answering.bhh_object_count.data import load_datasets
-
-
 # You will answer a reasoning question. Think step by step and double-check each calculation you make. Pay close attention to any numerical quantities in the text, converting written numbers into their numerical equivalents. Additionally, re-verify your final answer before concluding. The last line of your response should be of the following format: 'Answer: $VALUE' where VALUE is a numerical value.
 # 0.98 val, 0.91 test
 def train(
diff --git a/use_cases/question_answering/bbh/word_sorting/train.py b/use_cases/question_answering/bbh/word_sorting/train.py
new file mode 100644
index 00000000..9cdb04ed
--- /dev/null
+++ b/use_cases/question_answering/bbh/word_sorting/train.py
@@ -0,0 +1,160 @@
+from typing import Dict
+import adalflow as adal
+from use_cases.question_answering.bbh.word_sorting.task import (
+    QuestionAnswerTaskPipeline,
+)
+
+from adalflow.datasets.types import Example
+from adalflow.eval.llm_as_judge import DefaultLLMJudge
+from use_cases.question_answering.bbh.data import load_datasets
+from use_cases.config import gpt_3_model, gpt_4o_model
+
+
+judgement_query = r"""Does the predicted answer match with the ground truth answer? PLEASE Ignore the difference of separators between words.
+Say True if it matches, False if not.
+Example:
+Question: Sort the following words alphabetically: List: syndrome therefrom
+Ground truth answer: syndrome therefrom
+Predicted answer: syndrome, therefrom
+Answer: True"""
+
+
+class WordSortingAdalComponent(adal.AdalComponent):
+    def __init__(
+        self,
+        model_client: adal.ModelClient,
+        model_kwargs: Dict,
+        llm_judge_model_config: Dict,
+        backward_engine_model_config: Dict = None,
+        teacher_model_config: Dict = None,
+        text_optimizer_model_config: Dict = None,
+    ):
+        task = QuestionAnswerTaskPipeline(model_client, model_kwargs)
+
+        llm_judge = DefaultLLMJudge(
+            **llm_judge_model_config,
+            output_type="float",
+            jugement_query=judgement_query,
+            use_cache=True,
+        )
+        eval_fn = llm_judge.call
+        loss_fn = adal.EvalFnToTextLoss(
+            eval_fn=eval_fn,
+            eval_fn_desc="exact_match: 1 if str(y) == str(y_gt) else 0",
+        )
+        # eval_fn = lambda question, gt_answer, pred_answer: 1
+        super().__init__(
+            task=task,
+            eval_fn=eval_fn,
+            loss_fn=loss_fn,
+            backward_engine_model_config=backward_engine_model_config,
+            teacher_model_config=teacher_model_config,
+            text_optimizer_model_config=text_optimizer_model_config,
+        )
+
+    def handle_one_task_sample(self, sample: Example):
+        return self.task.call, {"question": sample.question, "id": sample.id}
+
+    def evaluate_one_sample(
+        self, sample: Example, y_pred: adal.GeneratorOutput
+    ) -> float:
+        y_label = ""
+        if y_pred and y_pred.data:
+            y_label = y_pred.data
+
+        return self.eval_fn(
+            question=sample.question, gt_answer=sample.answer, pred_answer=y_label
+        )
+
+    def handle_one_loss_sample(self, sample: Example, pred: adal.Parameter):
+        # prepare gt parameter
+        y_gt = adal.Parameter(
+            name="y_gt",
+            data=sample.answer,
+            eval_input=sample.answer,
+            requires_opt=False,
+        )
+        pred.eval_input = pred.full_response.data  # processed
+        question_param = adal.Parameter(
+            name="question",
+            data=sample.question,
+            eval_input=sample.question,
+            requires_opt=False,
+        )
+
+        return self.loss_fn, {
+            "kwargs": {
+                "pred_answer": pred,
+                "gt_answer": y_gt,
+                "question": question_param,
+            }
+        }
+
+    # def configure_backward_engine(self):
+    #     super().configure_backward_engine_helper(**self.backward_engine_model_config)
+
+    # def configure_teacher_generator(self):
+    #     super().configure_teacher_generator_helper(**self.teacher_model_config)
+
+    # def configure_optimizers(
+    #     self,
+    # ):  # TODO: train the text optimizer and the demo optimizer at the same time
+    #     to = super().configure_text_optimizer_helper(**self.text_optimizer_model_config)
+    #     do = super().configure_demo_optimizer_helper()
+    #     return to + do
+
+
+def train(
+    train_batch_size=4,  # larger batch size is not that effective, probably because of llm's lost in the middle
+    raw_shots: int = 0,
+    bootstrap_shots: int = 1,
+    max_steps=1,
+    num_workers=4,
+    strategy="random",
+    optimization_order="sequential",
+    debug=False,
+    resume_from_ckpt=None,
+    exclude_input_fields_from_bootstrap_demos=False,
+):
+    adal_component = WordSortingAdalComponent(
+        **gpt_3_model,
+        teacher_model_config=gpt_4o_model,
+        text_optimizer_model_config=gpt_4o_model,
+        backward_engine_model_config=gpt_4o_model,
+        llm_judge_model_config=gpt_3_model,
+    )
+    print(adal_component)
+    trainer = adal.Trainer(
+        train_batch_size=train_batch_size,
+        adaltask=adal_component,
+        strategy=strategy,
+        max_steps=max_steps,
+        num_workers=num_workers,
+        raw_shots=raw_shots,
+        bootstrap_shots=bootstrap_shots,
+        debug=debug,
+        weighted_sampling=True,
+        optimization_order=optimization_order,
+        exclude_input_fields_from_bootstrap_demos=exclude_input_fields_from_bootstrap_demos,
+    )
+    print(trainer)
+
+    train_dataset, val_dataset, test_dataset = load_datasets(
+        task_name="BBH_word_sorting"
+    )
+    trainer.fit(
+        train_dataset=train_dataset,
+        val_dataset=val_dataset,
+        test_dataset=test_dataset,
+        resume_from_ckpt=resume_from_ckpt,
+    )
+
+
+if __name__ == "__main__":
+
+    train(
+        debug=False,
+        max_steps=12,
+        strategy="constrained",
+        exclude_input_fields_from_bootstrap_demos=False,
+    )

From 29652da6b6a8736fb78ac7b06a350a37a15256c5 Mon Sep 17 00:00:00 2001
From: Li Yin <li.yin.gravity@gmail.com>
Date: Tue, 27 Aug 2024 01:00:48 -0700
Subject: [PATCH 7/8] complete the llm as judge for word sorting 91% on test,
 10% higher than other library, due to restructure the task

---
 adalflow/adalflow/core/functional.py          |   2 +
 .../optim/few_shot/bootstrap_optimizer.py     |  14 +-
 .../bbh/word_sorting/task.py                  |   4 +-
 .../bbh/word_sorting/train.py                 |  15 +-
 .../bbh/word_sorting/train_paper.py           | 161 ++++++++++++++++++
 5 files changed, 183 insertions(+), 13 deletions(-)
 create mode 100644 use_cases/question_answering/bbh/word_sorting/train_paper.py

diff --git a/adalflow/adalflow/core/functional.py b/adalflow/adalflow/core/functional.py
index 7779d25d..14157535 100644
--- a/adalflow/adalflow/core/functional.py
+++ b/adalflow/adalflow/core/functional.py
@@ -1264,6 +1264,8 @@ def random_sample(
     Randomly sample num_shots from the dataset. If replace is True, sample with replacement.
     """
     dataset_size = len(dataset)
+    if dataset_size == 0:
+        return []
 
     if not replace and num_shots > dataset_size:
         log.debug(
diff --git a/adalflow/adalflow/optim/few_shot/bootstrap_optimizer.py b/adalflow/adalflow/optim/few_shot/bootstrap_optimizer.py
index 39ed7840..c8a51fe7 100644
--- a/adalflow/adalflow/optim/few_shot/bootstrap_optimizer.py
+++ b/adalflow/adalflow/optim/few_shot/bootstrap_optimizer.py
@@ -29,13 +29,13 @@ class BootstrapFewShot(DemoOptimizer):
         to prioritize failed demos but successful in augmented demos based on the evaluation score
         while we backpropagate the demo samples.
      2. In default, we exclude the input fields from the augmented demos. Our reserch finds that
-        using the reasoning demostrations from teacher model can be more effective to just take inputs and output
+        using the reasoning demostrations from teacher model can be more effective in some cases than taking both inputs and output
         samples and be more token efficient.
 
     Reference:
     - DsPy: Com-piling declarative language model calls into state-of-the-art pipelines.
     """
-    exclude_input_fields_from_bootstrap_demos: bool = False
+    exclude_input_fields_from_bootstrap_demos: bool
 
     def __init__(
         self,
@@ -145,9 +145,7 @@ def sample(
                     if w < 0:
                         w = 0
                 weights.append(w)
-            # avoid all 0 weights
-            if all(weight == 0 for weight in weights):
-                weights = [1.0] * len(augmented_options)
+
         # print(f"augs: {augmented_options}")
         sampled_augmented_demos = (
             random_sample(
@@ -159,6 +157,7 @@ def sample(
 
         # 2. sample from raw demos
         # exclude the sampled augmented demos
+        # TODO: ensure all data points has unique ids
         filtered_dataset = list(
             filter(
                 lambda x: x.id
@@ -166,6 +165,9 @@ def sample(
                 dataset,
             )
         )
+        if len(filtered_dataset) == 0:
+            # If no demos left we will get raw_weights [], sum to 0
+            return sampled_augmented_demos, []
         # assigne weights 0 to all options
         raw_weights = None
         if weighted:
@@ -181,9 +183,7 @@ def sample(
                             f"score must be in range [0, 1], got {student_demo_score}"
                         )
                     raw_weights[i] = 1 - student_demo_score
-            if all(weight == 0 for weight in raw_weights):
 
-                raw_weights = [1.0] * len(filtered_dataset)  # Equal probability for all
         sampled_raw_demos = random_sample(
             filtered_dataset, raw_shots, replace=False, weights=raw_weights
         )
diff --git a/use_cases/question_answering/bbh/word_sorting/task.py b/use_cases/question_answering/bbh/word_sorting/task.py
index 2ddcca6e..ace7d1e6 100644
--- a/use_cases/question_answering/bbh/word_sorting/task.py
+++ b/use_cases/question_answering/bbh/word_sorting/task.py
@@ -30,11 +30,11 @@ def __init__(self, model_client: adal.ModelClient, model_kwargs: Dict):
         super().__init__()
 
         system_prompt = adal.Parameter(
-            data="You will answer a reasoning question. Think step by step. The last line of your response should be of the following format: 'Answer: $VALUE' where VALUE is the answer.",
+            data="Sort the following words alphabetically. Think step by step. The last line of your response should be of the following format: 'Answer: $VALUE' where VALUE is the answer.",
             role_desc="To give task instruction to the language model in the system prompt",
             requires_opt=True,
             param_type=ParameterType.PROMPT,
-            instruction_to_optimizer="You can try to show examples to see if it helps.",
+            # instruction_to_optimizer="Try to instruct what exactly word sorting is and what to do if two words have the same starting letter.",
         )
         few_shot_demos = adal.Parameter(
             data=None,
diff --git a/use_cases/question_answering/bbh/word_sorting/train.py b/use_cases/question_answering/bbh/word_sorting/train.py
index 9cdb04ed..a2b1a418 100644
--- a/use_cases/question_answering/bbh/word_sorting/train.py
+++ b/use_cases/question_answering/bbh/word_sorting/train.py
@@ -10,10 +10,10 @@
 from use_cases.config import gpt_3_model, gpt_4o_model
 
 
-judgement_query = r"""Does the predicted answer match with the ground truth answer? PLEASE Ignore the difference of separators between words.
+judgement_query = r"""Does the predicted answer match with the ground truth answer where words are in exact same order? PLEASE Ignore the difference of separators between words.
 Say True if it matches, False if not.
 Example:
-Question: Sort the following words alphabetically: List: syndrome therefrom
+Question: List: syndrome therefrom
 Ground truth answer: syndrome therefrom
 Predicted answer: syndrome, therefrom
 Answer: True"""
@@ -142,6 +142,12 @@ def train(
     train_dataset, val_dataset, test_dataset = load_datasets(
         task_name="BBH_word_sorting"
     )
+    for dataset in [train_dataset, val_dataset, test_dataset]:
+        for example in dataset:
+            example.question = example.question.replace(
+                "Sort the following words alphabetically:", ""
+            )
+
     trainer.fit(
         train_dataset=train_dataset,
         val_dataset=val_dataset,
@@ -154,7 +160,8 @@ def train(
 
     train(
         debug=False,
-        max_steps=12,
+        max_steps=10,
         strategy="constrained",
-        exclude_input_fields_from_bootstrap_demos=False,
+        exclude_input_fields_from_bootstrap_demos=True,
+        # resume_from_ckpt="/Users/liyin/.adalflow/ckpt/WordSortingAdalComponent/constrained_max_steps_12_7dc6a_run_2.json",
     )
diff --git a/use_cases/question_answering/bbh/word_sorting/train_paper.py b/use_cases/question_answering/bbh/word_sorting/train_paper.py
new file mode 100644
index 00000000..6d3bd477
--- /dev/null
+++ b/use_cases/question_answering/bbh/word_sorting/train_paper.py
@@ -0,0 +1,161 @@
+from typing import Dict
+import adalflow as adal
+from use_cases.question_answering.bbh.word_sorting.task import (
+    QuestionAnswerTaskPipeline,
+)
+
+from adalflow.datasets.types import Example
+from adalflow.eval.llm_as_judge import DefaultLLMJudge
+from use_cases.question_answering.bbh.data import load_datasets
+from use_cases.config import gpt_3_model, gpt_4o_model
+
+
+judgement_query = r"""Does the predicted answer match with the ground truth answer? PLEASE Ignore the difference of separators between words.
+Say True if it matches, False if not.
+Example:
+Question: Sort the following words alphabetically: List: syndrome therefrom
+Ground truth answer: syndrome therefrom
+Predicted answer: syndrome, therefrom
+Answer: True"""
+
+
+class WordSortingAdalComponent(adal.AdalComponent):
+    def __init__(
+        self,
+        model_client: adal.ModelClient,
+        model_kwargs: Dict,
+        llm_judge_model_config: Dict,
+        backward_engine_model_config: Dict = None,
+        teacher_model_config: Dict = None,
+        text_optimizer_model_config: Dict = None,
+    ):
+        task = QuestionAnswerTaskPipeline(model_client, model_kwargs)
+
+        llm_judge = DefaultLLMJudge(
+            **llm_judge_model_config,
+            output_type="float",
+            jugement_query=judgement_query,
+            use_cache=True,
+        )
+        eval_fn = llm_judge.call
+        loss_fn = adal.EvalFnToTextLoss(
+            eval_fn=eval_fn,
+            eval_fn_desc="exact_match: 1 if str(y) == str(y_gt) else 0",
+        )
+        # eval_fn = lambda question, gt_answer, pred_answer: 1
+        super().__init__(
+            task=task,
+            eval_fn=eval_fn,
+            loss_fn=loss_fn,
+            backward_engine_model_config=backward_engine_model_config,
+            teacher_model_config=teacher_model_config,
+            text_optimizer_model_config=text_optimizer_model_config,
+        )
+
+    def handle_one_task_sample(self, sample: Example):
+        return self.task.call, {"question": sample.question, "id": sample.id}
+
+    def evaluate_one_sample(
+        self, sample: Example, y_pred: adal.GeneratorOutput
+    ) -> float:
+        y_label = ""
+        if y_pred and y_pred.data:
+            y_label = y_pred.data
+
+        return self.eval_fn(
+            question=sample.question, gt_answer=sample.answer, pred_answer=y_label
+        )
+
+    def handle_one_loss_sample(self, sample: Example, pred: adal.Parameter):
+        # prepare gt parameter
+        y_gt = adal.Parameter(
+            name="y_gt",
+            data=sample.answer,
+            eval_input=sample.answer,
+            requires_opt=False,
+        )
+        pred.eval_input = pred.full_response.data  # processed
+        question_param = adal.Parameter(
+            name="question",
+            data=sample.question,
+            eval_input=sample.question,
+            requires_opt=False,
+        )
+
+        return self.loss_fn, {
+            "kwargs": {
+                "pred_answer": pred,
+                "gt_answer": y_gt,
+                "question": question_param,
+            }
+        }
+
+    # def configure_backward_engine(self):
+    #     super().configure_backward_engine_helper(**self.backward_engine_model_config)
+
+    # def configure_teacher_generator(self):
+    #     super().configure_teacher_generator_helper(**self.teacher_model_config)
+
+    # def configure_optimizers(
+    #     self,
+    # ):  # TODO: train the text optimizer and the demo optimizer at the same time
+    #     to = super().configure_text_optimizer_helper(**self.text_optimizer_model_config)
+    #     do = super().configure_demo_optimizer_helper()
+    #     return to + do
+
+
+def train(
+    train_batch_size=4,  # larger batch size is not that effective, probably because of llm's lost in the middle
+    raw_shots: int = 0,
+    bootstrap_shots: int = 1,
+    max_steps=1,
+    num_workers=4,
+    strategy="random",
+    optimization_order="sequential",
+    debug=False,
+    resume_from_ckpt=None,
+    exclude_input_fields_from_bootstrap_demos=False,
+):
+    adal_component = WordSortingAdalComponent(
+        **gpt_3_model,
+        teacher_model_config=gpt_4o_model,
+        text_optimizer_model_config=gpt_4o_model,
+        backward_engine_model_config=gpt_4o_model,
+        llm_judge_model_config=gpt_3_model,
+    )
+    print(adal_component)
+    trainer = adal.Trainer(
+        train_batch_size=train_batch_size,
+        adaltask=adal_component,
+        strategy=strategy,
+        max_steps=max_steps,
+        num_workers=num_workers,
+        raw_shots=raw_shots,
+        bootstrap_shots=bootstrap_shots,
+        debug=debug,
+        weighted_sampling=True,
+        optimization_order=optimization_order,
+        exclude_input_fields_from_bootstrap_demos=exclude_input_fields_from_bootstrap_demos,
+    )
+    print(trainer)
+
+    train_dataset, val_dataset, test_dataset = load_datasets(
+        task_name="BBH_word_sorting"
+    )
+    trainer.fit(
+        train_dataset=train_dataset,
+        val_dataset=val_dataset,
+        test_dataset=test_dataset,
+        resume_from_ckpt=resume_from_ckpt,
+    )
+
+
+if __name__ == "__main__":
+
+    train(
+        debug=False,
+        max_steps=10,
+        strategy="constrained",
+        exclude_input_fields_from_bootstrap_demos=False,
+        # resume_from_ckpt="constrained_max_steps_12_7dc6a_run_2.json",
+    )

From 5288aeafd25606525ace7724ce3f7dad6649a4bc Mon Sep 17 00:00:00 2001
From: Li Yin <li.yin.gravity@gmail.com>
Date: Sun, 1 Sep 2024 08:47:54 -0700
Subject: [PATCH 8/8] docs

---
 adalflow/adalflow/core/generator.py |  4 ++++
 docs/source/tutorials/db.rst        | 22 +++++++++++-----------
 2 files changed, 15 insertions(+), 11 deletions(-)

diff --git a/adalflow/adalflow/core/generator.py b/adalflow/adalflow/core/generator.py
index 1996b19a..819daa3b 100644
--- a/adalflow/adalflow/core/generator.py
+++ b/adalflow/adalflow/core/generator.py
@@ -831,6 +831,10 @@ def _extra_repr(self) -> str:
         s = f"model_kwargs={self.model_kwargs}, "
         return s
 
+    def to_dict(self) -> Dict[str, Any]:
+        r"""Convert the generator to a dictionary."""
+        # exclude default functions
+
     @staticmethod
     def failure_message_to_backward_engine(
         gradient_response: GeneratorOutput,
diff --git a/docs/source/tutorials/db.rst b/docs/source/tutorials/db.rst
index 4694b59b..edfc15bf 100644
--- a/docs/source/tutorials/db.rst
+++ b/docs/source/tutorials/db.rst
@@ -1,14 +1,14 @@
-.. raw:: html
-
-   <div style="display: flex; justify-content: flex-start; align-items: center; margin-bottom: 20px;">
-      <a href="https://colab.research.google.com/drive/1_sGeHaKrwpI9RiL01g3cKyI2_5PJqZtr?usp=sharing" target="_blank" style="margin-right: 10px;">
-         <img alt="Try Quickstart in Colab" src="https://colab.research.google.com/assets/colab-badge.svg" style="vertical-align: middle;">
-      </a>
-      <a href="https://github.com/SylphAI-Inc/LightRAG/blob/main/tutorials/prompt_note.py" target="_blank" style="display: flex; align-items: center;">
-         <img src="https://github.githubassets.com/images/modules/logos_page/GitHub-Mark.png" alt="GitHub" style="height: 20px; width: 20px; margin-right: 5px;">
-         <span style="vertical-align: middle;"> Open Source Code</span>
-      </a>
-   </div>
+.. .. raw:: html
+
+..    <div style="display: flex; justify-content: flex-start; align-items: center; margin-bottom: 20px;">
+..       <a href="https://colab.research.google.com/drive/1_sGeHaKrwpI9RiL01g3cKyI2_5PJqZtr?usp=sharing" target="_blank" style="margin-right: 10px;">
+..          <img alt="Try Quickstart in Colab" src="https://colab.research.google.com/assets/colab-badge.svg" style="vertical-align: middle;">
+..       </a>
+..       <a href="https://github.com/SylphAI-Inc/LightRAG/blob/main/tutorials/prompt_note.py" target="_blank" style="display: flex; align-items: center;">
+..          <img src="https://github.githubassets.com/images/modules/logos_page/GitHub-Mark.png" alt="GitHub" style="height: 20px; width: 20px; margin-right: 5px;">
+..          <span style="vertical-align: middle;"> Open Source Code</span>
+..       </a>
+..    </div>
 
 
 Data (Database/Pipeline) & RAG