Merge branch 'refs/heads/main' into llm-handler-refactor

# Conflicts:
#	agentops/time_travel.py

.github/workflows/test-notebooks.yml

@@ -0,0 +1,88 @@
+name: Test Notebooks
+on:
+  workflow_dispatch: # Allows manual triggering
+jobs:
+  test-notebooks:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.11']
+      fail-fast: false
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -U jupyter
+    - name: Create .env file
+      run: |
+        echo "OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY }}" >> .env
+        echo "AGENTOPS_API_KEY=${{ secrets.AGENTOPS_API_KEY }}" >> .env
+        echo "CO_API_KEY=${{ secrets.CO_API_KEY }}" >> .env
+        echo "GROQ_API_KEY=${{ secrets.GROQ_API_KEY }}" >> .env
+        echo "MULTION_API_KEY=${{ secrets.MULTION_API_KEY }}" >> .env
+        echo "SERPER_API_KEY=${{ secrets.SERPER_API_KEY }}" >> .env
+    - name: Run notebooks and check for errors
+      run: |
+        mkdir -p logs
+        exit_code=0
+
+        exclude_notebooks=(
+          "./examples/crew/job_posting.ipynb"
+        )
+
+        for notebook in $(find . -name '*.ipynb'); do
+
+          skip=false
+          for excluded in "${exclude_notebooks[@]}"; do
+            if [[ "$notebook" == "$excluded" ]]; then
+              skip=true
+              echo "Skipping excluded notebook: $notebook"
+              break
+            fi
+          done
+
+          $skip && continue
+
+          notebook_name=$(basename "$notebook" .ipynb)
+          notebook_path=$(realpath "$notebook")
+          notebook_dir=$(dirname "$notebook_path")
+          
+          # Run the notebook
+          jupyter execute "$notebook_path" || true
+          
+          # Check if agentops.log was created
+          if [ -f "${notebook_dir}/agentops.log" ]; then
+            dest_log="logs/agentops-${notebook_name}.log"
+            mv "${notebook_dir}/agentops.log" "$dest_log"
+            
+            # Check agentops log for errors or warnings
+            if grep -E "ERROR|WARNING" "$dest_log"; then
+              echo "Errors or warnings found in $dest_log for Python ${{ matrix.python-version }}"
+              exit_code=1
+            else
+              echo "No errors or warnings found in $dest_log for Python ${{ matrix.python-version }}"
+            fi
+          else
+            echo "No agentops.log generated for $notebook_name"
+          fi
+        done
+
+        # Check if any logs were found
+        if [ $(find logs -name 'agentops-*.log' | wc -l) -eq 0 ]; then
+          echo "No agentops.log files were generated for any notebook"
+        fi
+
+        exit $exit_code
+
+    - name: Upload logs as artifacts
+      uses: actions/upload-artifact@v4
+      if: always()
+      with:
+        name: notebook-logs-${{ matrix.python-version }}
+        path: logs/agentops-*.log
+        if-no-files-found: warn

README.md

@@ -141,7 +141,7 @@ To use the handler, import and set
 import os
 from langchain.chat_models import ChatOpenAI
 from langchain.agents import initialize_agent, AgentType
-from agentops.langchain_callback_handler import LangchainCallbackHandler
+from agentops.partners.langchain_callback_handler import LangchainCallbackHandler
 
 AGENTOPS_API_KEY = os.environ['AGENTOPS_API_KEY']
 handler = LangchainCallbackHandler(api_key=AGENTOPS_API_KEY, tags=['Langchain Example'])

agentops/cli.py

@@ -32,10 +32,6 @@ def main():
         if args.branch_name:
             fetch_time_travel_id(args.branch_name)
         if args.on:
-            set_time_travel_active_state("on")
+            set_time_travel_active_state(True)
         if args.off:
-            set_time_travel_active_state("off")
-
-
-if __name__ == "__main__":
-    main()
+            set_time_travel_active_state(False)

agentops/client.py

@@ -82,6 +82,9 @@ def configure(
         )
 
     def initialize(self) -> Union[Session, None]:
+        if self.is_initialized:
+            return
+
         self.unsuppress_logs()
 
         if self._config.api_key is None:

agentops/partners/langchain_callback_handler.py

@@ -46,9 +46,8 @@ def __init__(
         endpoint: Optional[str] = None,
         max_wait_time: Optional[int] = None,
         max_queue_size: Optional[int] = None,
-        tags: Optional[List[str]] = None,
+        default_tags: Optional[List[str]] = None,
     ):
-
         logging_level = os.getenv("AGENTOPS_LOGGING_LEVEL")
         log_levels = {
             "CRITICAL": logging.CRITICAL,
@@ -64,12 +63,19 @@ def __init__(
             "endpoint": endpoint,
             "max_wait_time": max_wait_time,
             "max_queue_size": max_queue_size,
-            "tags": tags,
+            "default_tags": default_tags,
         }
 
-        self.ao_client = AOClient(
-            **{k: v for k, v in client_params.items() if v is not None}, override=False
-        )
+        self.ao_client = AOClient()
+        if self.ao_client.session_count == 0:
+            self.ao_client.configure(
+                **{k: v for k, v in client_params.items() if v is not None},
+                instrument_llm_calls=False,
+            )
+
+        if not self.ao_client.is_initialized:
+            self.ao_client.initialize()
+
         self.agent_actions: Dict[UUID, List[ActionEvent]] = defaultdict(list)
         self.events = Events()
 
@@ -93,7 +99,6 @@ def on_llm_start(
             },  # TODO: params is inconsistent, in ToolEvent we put it in logs
             model=get_model_from_kwargs(kwargs),
             prompt=prompts[0],
-            # tags=tags # TODO
         )
 
     @debug_print_function_params
@@ -156,15 +161,18 @@ def on_chain_start(
         metadata: Optional[Dict[str, Any]] = None,
         **kwargs: Any,
     ) -> Any:
-        self.events.chain[str(run_id)] = ActionEvent(
-            params={
-                **serialized,
-                **inputs,
-                **({} if metadata is None else metadata),
-                **kwargs,
-            },
-            action_type="chain",
-        )
+        try:
+            self.events.chain[str(run_id)] = ActionEvent(
+                params={
+                    **serialized,
+                    **inputs,
+                    **({} if metadata is None else metadata),
+                    **kwargs,
+                },
+                action_type="chain",
+            )
+        except Exception as e:
+            logger.warning(e)
 
     @debug_print_function_params
     def on_chain_end(
@@ -240,6 +248,8 @@ def on_tool_end(
                 details=output,
             )
             self.ao_client.record(error_event)
+        else:
+            self.ao_client.record(tool_event)
 
     @debug_print_function_params
     def on_tool_error(
@@ -357,7 +367,13 @@ def on_retry(
 
     @property
     def session_id(self):
-        return self.ao_client.current_session_id
+        raise DeprecationWarning(
+            "session_id is deprecated in favor of current_session_ids"
+        )
+
+    @property
+    def current_session_ids(self):
+        return self.ao_client.current_session_ids
 
 
 class AsyncLangchainCallbackHandler(AsyncCallbackHandler):

agentops/session.py

@@ -268,7 +268,7 @@ def _flush_queue(self) -> None:
         if not self.is_running:
             return
         with self.lock:
-            queue_copy = copy.deepcopy(self.queue)  # Copy the current items
+            queue_copy = self.queue[:]  # Copy the current items
             self.queue = []
 
             if len(queue_copy) > 0:

agentops/time_travel.py

@@ -1,10 +1,9 @@
 import json
 import yaml
+import os
 from .http_client import HttpClient
 from .exceptions import ApiServerException
-import os
 from .singleton import singleton
-from os import environ
 
 
 @singleton
@@ -32,13 +31,10 @@ def __init__(self):
 
 def fetch_time_travel_id(ttd_id):
     try:
-        endpoint = environ.get("AGENTOPS_API_ENDPOINT", "https://api.agentops.ai")
-        payload = json.dumps({"ttd_id": ttd_id}).encode("utf-8")
+        endpoint = os.environ.get("AGENTOPS_API_ENDPOINT", "https://api.agentops.ai")
         ttd_res = HttpClient.get(f"{endpoint}/v2/ttd/{ttd_id}")
         if ttd_res.code != 200:
-            raise Exception(
-                f"Failed to fetch TTD with status code {ttd_res.status_code}"
-            )
+            raise Exception(f"Failed to fetch TTD with status code {ttd_res.code}")
 
         prompt_to_returns_map = {
             "completion_overrides": {

docs/v0/logger.mdx

@@ -7,7 +7,7 @@ description: 'This document explains the use of the AgentOpsLogger
 
 ## AgentOps logger
 
-The AgentsOps logger is quick and simple way of integrating your existing 
+The AgentOps logger is quick and simple way of integrating your existing 
 agent codebase with AgentOps. It allows you to use your current logs as events 
 by extending the built-in Python logging system to emit events to AgentOps.
 

docs/v0/recording-events.mdx

@@ -35,7 +35,7 @@ From this point, simply call the .record() method in the AgentOps client:
 ao_client.record(Event("event_type1"))
 ```
 
-In AgentsOps, each session is associated with a number of "Events". Events have 
+In AgentOps, each session is associated with a number of "Events". Events have 
 must have an "event_type" which is any abitrary string of your choice. It might be something
 like "OpenAI Call". Events can also have other information such as the parameters of the operation,
 the returned data, alongside tags, etc.

docs/v1/examples/notebooks/multion/Step_by_step_web_browsing.html

@@ -330,8 +330,8 @@ <h1>Step Stream</h1>
 <div class="sourceCode" id="cb4"><pre
 class="sourceCode python"><code class="sourceCode python"><span id="cb4-1"><a href="#cb4-1" aria-hidden="true" tabindex="-1"></a><span class="im">import</span> os</span>
 <span id="cb4-2"><a href="#cb4-2" aria-hidden="true" tabindex="-1"></a></span>
-<span id="cb4-3"><a href="#cb4-3" aria-hidden="true" tabindex="-1"></a>os.environ[<span class="st">&quot;MULTION_API_KEY&quot;</span>] <span class="op">=</span> <span class="st">&quot;e8cbbd0f8fa042f49f267a44bf97425c&quot;</span></span>
-<span id="cb4-4"><a href="#cb4-4" aria-hidden="true" tabindex="-1"></a>os.environ[<span class="st">&quot;AGENTOPS_API_KEY&quot;</span>] <span class="op">=</span> <span class="st">&quot;a640373b-30ae-4655-a1f3-5caa882a8721&quot;</span></span></code></pre></div>
+<span id="cb4-3"><a href="#cb4-3" aria-hidden="true" tabindex="-1"></a>os.environ[<span class="st">&quot;MULTION_API_KEY&quot;</span>] <span class="op">=</span></span>
+<span id="cb4-4"><a href="#cb4-4" aria-hidden="true" tabindex="-1"></a>os.environ[<span class="st">&quot;AGENTOPS_API_KEY&quot;</span>] <span class="op">=</span></span></code></pre></div>
 </div>
 <div class="cell code" data-scrolled="true">
 <div class="sourceCode" id="cb5"><pre