Merge branch 'main' into concurrency-fixes

.github/workflows/test-notebooks.yml

@@ -1,14 +1,7 @@
 name: Test Notebooks
 on:
-  push:
-    branches:
-      - main
-    paths:
-      - "agentops/**"
-      - "examples/**"
-      - "tests/**"
-      - ".github/workflows/test-notebooks.yml"
-  pull_request_target:
+  pull_request:
+    types: [closed]
     branches:
       - main
     paths:
@@ -43,13 +36,17 @@ jobs:
         echo "GROQ_API_KEY=${{ secrets.GROQ_API_KEY }}" >> .env
         echo "MULTION_API_KEY=${{ secrets.MULTION_API_KEY }}" >> .env
         echo "SERPER_API_KEY=${{ secrets.SERPER_API_KEY }}" >> .env
+    - name: Install AgentOps from main branch and remove agentops install from notebooks
+      run: |
+        pip install git+https://github.com/AgentOps-AI/agentops.git@main
+        find . -name '*.ipynb' -exec sed -i '/^%pip install.*agentops/d' {} +
     - name: Run notebooks and check for errors
       run: |
         mkdir -p logs
         exit_code=0
 
         exclude_notebooks=(
-          "./examples/crew/job_posting.ipynb",
+          "./examples/crewai_examples/job_posting.ipynb",
           "./examples/demos/agentchat_agentops.ipynb"
         )
 

agentops/client.py

@@ -38,6 +38,7 @@ def __init__(self):
         self._llm_tracker: Optional[LlmTracker] = None
         self._sessions: List[Session] = active_sessions
         self._config = Configuration()
+        self._pre_init_queue = {"agents": []}
 
         self.configure(
             api_key=os.environ.get("AGENTOPS_API_KEY"),
@@ -106,6 +107,13 @@ def initialize(self) -> Union[Session, None]:
         if self._config.auto_start_session:
             session = self.start_session()
 
+        if session:
+            for agent_args in self._pre_init_queue["agents"]:
+                session.create_agent(
+                    name=agent_args["name"], agent_id=agent_args["agent_id"]
+                )
+            self._pre_init_queue["agents"] = []
+
         return session
 
     def _initialize_partner_framework(self) -> None:
@@ -234,6 +242,13 @@ def start_session(
             config=self._config,
         )
 
+        if self._pre_init_queue["agents"] and len(self._pre_init_queue["agents"]) > 0:
+            for agent_args in self._pre_init_queue["agents"]:
+                session.create_agent(
+                    name=agent_args["name"], agent_id=agent_args["agent_id"]
+                )
+            self._pre_init_queue["agents"] = []
+
         if not session.is_running:
             return logger.error("Failed to start session")
 
@@ -294,7 +309,9 @@ def create_agent(
             # if no session passed, assume single session
             session = self._safe_get_session()
             if session is None:
-                return
+                self._pre_init_queue["agents"].append(
+                    {"name": name, "agent_id": agent_id}
+                )
             session.create_agent(name=name, agent_id=agent_id)
 
         return agent_id

agentops/decorators.py

@@ -326,12 +326,6 @@ def new_init(self, *args, **kwargs):
 
                     original_init(self, *args, **kwargs)
 
-                    if not Client().is_initialized:
-                        Client().add_pre_init_warning(
-                            f"Failed to track an agent {name} because agentops.init() was not "
-                            + "called before initializing the agent with the @track_agent decorator."
-                        )
-
                     self.agent_ops_agent_id = str(uuid4())
 
                     session = kwargs.get("session", None)
@@ -345,12 +339,10 @@ def new_init(self, *args, **kwargs):
                     )
                 except AttributeError as e:
                     Client().add_pre_init_warning(
-                        f"Failed to track an agent {name} because agentops.init() was not "
-                        + "called before initializing the agent with the @track_agent decorator."
+                        f"Failed to track an agent {name} with the @track_agent decorator."
                     )
                     logger.warning(
-                        "Failed to track an agent. This often happens if agentops.init() was not "
-                        "called before initializing an agent with the @track_agent decorator."
+                        "Failed to track an agent with the @track_agent decorator."
                     )
                     original_init(self, *args, **kwargs)
 

agentops/time_travel.py

@@ -5,6 +5,8 @@
 from .exceptions import ApiServerException
 from .singleton import singleton
 
+ttd_prepend_string = "🖇️ Agentops: ⏰ Time Travel |"
+
 
 @singleton
 class TimeTravel:
@@ -47,9 +49,9 @@ def fetch_time_travel_id(ttd_id):
 
         set_time_travel_active_state(True)
     except ApiServerException as e:
-        print(f"🖇️ Agentops: ⏰ Time Travel Error - {e}")
+        print(f"{ttd_prepend_string} Error - {e}")
     except Exception as e:
-        print(f"🖇️ Agentops: ⏰ Time Travel Error - {e}")
+        print(f"{ttd_prepend_string} Error - {e}")
 
 
 def fetch_completion_override_from_time_travel_cache(kwargs):
@@ -64,14 +66,14 @@ def fetch_completion_override_from_time_travel_cache(kwargs):
 def find_cache_hit(prompt_messages, completion_overrides):
     if not isinstance(prompt_messages, (list, tuple)):
         print(
-            "🖇️ Agentops: ⏰ Time Travel Error - unexpected type for prompt_messages. Expected 'list' or 'tuple'. Got ",
+            f"{ttd_prepend_string} Error - unexpected type for prompt_messages. Expected 'list' or 'tuple'. Got ",
             type(prompt_messages),
         )
         return None
 
     if not isinstance(completion_overrides, dict):
         print(
-            "🖇️ Agentops: ⏰ Time Travel Error - unexpected type for completion_overrides. Expected 'dict'. Got ",
+            f"{ttd_prepend_string} Error - unexpected type for completion_overrides. Expected 'dict'. Got ",
             type(completion_overrides),
         )
         return None
@@ -80,15 +82,15 @@ def find_cache_hit(prompt_messages, completion_overrides):
             completion_override_dict = eval(key)
             if not isinstance(completion_override_dict, dict):
                 print(
-                    "🖇️ Agentops: ⏰ Time Travel Error - unexpected type for completion_override_dict. Expected 'dict'. Got ",
+                    f"{ttd_prepend_string} Error - unexpected type for completion_override_dict. Expected 'dict'. Got ",
                     type(completion_override_dict),
                 )
                 continue
 
             cached_messages = completion_override_dict.get("messages")
             if not isinstance(cached_messages, list):
                 print(
-                    "🖇️ Agentops: ⏰ Time Travel Error - unexpected type for cached_messages. Expected 'list'. Got ",
+                    f"{ttd_prepend_string} Error - unexpected type for cached_messages. Expected 'list'. Got ",
                     type(cached_messages),
                 )
                 continue
@@ -105,11 +107,11 @@ def find_cache_hit(prompt_messages, completion_overrides):
                 return value
         except (SyntaxError, ValueError, TypeError) as e:
             print(
-                f"🖇️ Agentops: ⏰ Time Travel Error - Error processing completion_overrides item: {e}"
+                f"{ttd_prepend_string} Error - Error processing completion_overrides item: {e}"
             )
         except Exception as e:
             print(
-                f"🖇️ Agentops: ⏰ Time Travel Error - Unexpected error in find_cache_hit: {e}"
+                f"{ttd_prepend_string} Error - Unexpected error in find_cache_hit: {e}"
             )
     return None
 
@@ -142,11 +144,11 @@ def set_time_travel_active_state(is_active: bool):
             yaml.dump(config, config_file)
         except:
             print(
-                f"🖇️ Agentops: ⏰ Time Travel Error - Unable to write to {config_path}. Time Travel not activated"
+                f"{ttd_prepend_string} Error - Unable to write to {config_path}. Time Travel not activated"
             )
             return
 
     if is_active:
-        print("🖇️ Agentops: ⏰ Time Travel Activated")
+        print(f"{ttd_prepend_string} Activated")
     else:
-        print("🖇️ Agentops: ⏰ Time Travel Deactivated")
+        print(f"{ttd_prepend_string} Deactivated")

examples/autogen/AgentChat.ipynb → examples/autogen_examples/AgentChat.ipynb

@@ -146,6 +146,7 @@
     "except StdinNotImplementedError:\n",
     "    # This is only necessary for AgentOps testing automation which is headless and will not have user input\n",
     "    print(\"Stdin not implemented. Skipping initiate_chat\")\n",
+    "    agentops.end_session(\"Indeterminate\")\n",
     "\n",
     "# Close your AgentOps session to indicate that it completed.\n",
     "agentops.end_session(\"Success\")\n",

examples/autogen/MathAgent.ipynb → examples/autogen_examples/MathAgent.ipynb

@@ -195,6 +195,7 @@
     "except StdinNotImplementedError:\n",
     "    # This is only necessary for AgentOps testing automation which is headless and will not have user input\n",
     "    print(\"Stdin not implemented. Skipping initiate_chat\")\n",
+    "    agentops.end_session(\"Indeterminate\")\n",
     "\n",
     "agentops.end_session(\"Success\")"
    ]

examples/crew/job_posting.ipynb → examples/crewai_examples/job_posting.ipynb

@@ -41,7 +41,10 @@
     "from crewai_tools.tools import WebsiteSearchTool, SerperDevTool, FileReadTool\n",
     "import agentops\n",
     "import os\n",
-    "from dotenv import load_dotenv"
+    "from dotenv import load_dotenv\n",
+    "from IPython.core.error import (\n",
+    "    StdinNotImplementedError,\n",
+    ")  # only needed by AgentOps testing automation"
    ]
   },
   {
@@ -244,8 +247,13 @@
     "    ],\n",
     ")\n",
     "\n",
-    "# Kick off the process\n",
-    "result = crew.kickoff()\n",
+    "try:\n",
+    "    # Kick off the process\n",
+    "    result = crew.kickoff()\n",
+    "except StdinNotImplementedError:\n",
+    "    # This is only necessary for AgentOps testing automation which is headless and will not have user input\n",
+    "    print(\"Stdin not implemented. Skipping kickoff()\")\n",
+    "    agentops.end_session(\"Indeterminate\")\n",
     "\n",
     "print(\"Job Posting Creation Process Completed.\")\n",
     "print(\"Final Job Posting:\")\n",

examples/demos/agentchat_agentops.ipynb

@@ -196,6 +196,7 @@
     "except StdinNotImplementedError:\n",
     "    # This is only necessary for AgentOps testing automation which is headless and will not have user input\n",
     "    print(\"Stdin not implemented. Skipping initiate_chat\")\n",
+    "    agentops.end_session(\"Indeterminate\")\n",
     "\n",
     "# Close your AgentOps session to indicate that it completed.\n",
     "agentops.end_session(\"Success\")"

examples/langchain/langchain_examples.ipynb → examples/langchain_examples/langchain_examples.ipynb

@@ -148,6 +148,10 @@
    },
    "outputs": [],
    "source": [
+    "agentops_handler = AgentOpsLangchainCallbackHandler(\n",
+    "    api_key=AGENTOPS_API_KEY, default_tags=[\"Langchain Example\"]\n",
+    ")\n",
+    "\n",
     "llm = ChatOpenAI(\n",
     "    openai_api_key=OPENAI_API_KEY, callbacks=[agentops_handler], model=\"gpt-3.5-turbo\"\n",
     ")\n",

tests/test_pre_init.py

@@ -0,0 +1,57 @@
+import pytest
+import requests_mock
+import time
+import agentops
+from agentops import record_action, track_agent
+from datetime import datetime
+from agentops.singleton import clear_singletons
+import contextlib
+
+jwts = ["some_jwt", "some_jwt2", "some_jwt3"]
+
+
+@pytest.fixture(autouse=True)
+def setup_teardown():
+    clear_singletons()
+    yield
+    agentops.end_all_sessions()  # teardown part
+
+
+@contextlib.contextmanager
+@pytest.fixture(autouse=True)
+def mock_req():
+    with requests_mock.Mocker() as m:
+        url = "https://api.agentops.ai"
+        m.post(url + "/v2/create_agent", text="ok")
+        m.post(url + "/v2/update_session", text="ok")
+        m.post(
+            url + "/v2/create_session", json={"status": "success", "jwt": "some_jwt"}
+        )
+
+        yield m
+
+
+@track_agent(name="TestAgent")
+class BasicAgent:
+    def __init__(self):
+        pass
+
+
+class TestPreInit:
+    def setup_method(self):
+        self.url = "https://api.agentops.ai"
+        self.api_key = "11111111-1111-4111-8111-111111111111"
+
+    def test_track_agent(self, mock_req):
+        agent = BasicAgent()
+
+        assert len(mock_req.request_history) == 0
+
+        agentops.init(api_key=self.api_key)
+
+        # Assert
+        # start session and create agent
+        assert len(mock_req.request_history) == 2
+        assert mock_req.last_request.headers["X-Agentops-Api-Key"] == self.api_key
+
+        agentops.end_session(end_state="Success")