Merge branch 'main' into eng-142-instrument-litellm

README.md

@@ -25,7 +25,7 @@
 
 Build your next agent with benchmarks, observability, and replay analytics. AgentOps is the toolkit for evaluating and developing robust and reliable AI agents.
 
-AgentOps is open beta. You can sign up for AgentOps [here](https://app.agentops.ai).
+You can sign up for AgentOps [here](https://app.agentops.ai).
 
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) ![PyPI - Version](https://img.shields.io/pypi/v/agentops) <a href="https://pepy.tech/project/agentops">
   <img src="https://static.pepy.tech/badge/agentops/month"> <a href="https://twitter.com/agentopsai">
@@ -49,16 +49,16 @@ Initialize the AgentOps client, and automatically get analytics on every LLM cal
 import agentops
 
 # Beginning of program's code (i.e. main.py, __init__.py)
-ao_client = agentops.Client(<INSERT YOUR API KEY HERE>)
+agentops.init(<INSERT YOUR API KEY HERE>)
 
 ...
 # (optional: record specific functions)
-@record_function('sample function being record')
+@agentops.record_function('sample function being record')
 def sample_function(...):
     ...
 
 # End of program
-ao_client.end_session('Success')
+agentops.end_session('Success')
 # Woohoo You're done 🎉
 ```
 

agentops/__init__.py

@@ -8,6 +8,7 @@
 from .enums import Models
 from .decorators import record_function
 from .agent import track_agent
+from .log_config import set_logging_level_info, set_logging_level_critial
 
 
 def init(api_key: Optional[str] = None,
@@ -18,7 +19,9 @@ def init(api_key: Optional[str] = None,
          tags: Optional[List[str]] = None,
          override: Optional[bool] = None,  # Deprecated
          instrument_llm_calls=True,
-         auto_start_session=True):
+         auto_start_session=True,
+         inherited_session_id: Optional[str] = None
+         ):
     """
         Initializes the AgentOps singleton pattern.
 
@@ -38,18 +41,23 @@ def init(api_key: Optional[str] = None,
             override (bool, optional): [Deprecated] Use `instrument_llm_calls` instead. Whether to instrument LLM calls and emit LLMEvents..
             instrument_llm_calls (bool): Whether to instrument LLM calls and emit LLMEvents..
             auto_start_session (bool): Whether to start a session automatically when the client is created.
+            inherited_session_id (optional, str): Init Agentops with an existing Session
         Attributes:
     """
-
-    Client(api_key=api_key,
-           parent_key=parent_key,
-           endpoint=endpoint,
-           max_wait_time=max_wait_time,
-           max_queue_size=max_queue_size,
-           tags=tags,
-           override=override,
-           instrument_llm_calls=instrument_llm_calls,
-           auto_start_session=auto_start_session)
+    set_logging_level_info()
+    c = Client(api_key=api_key,
+               parent_key=parent_key,
+               endpoint=endpoint,
+               max_wait_time=max_wait_time,
+               max_queue_size=max_queue_size,
+               tags=tags,
+               override=override,
+               instrument_llm_calls=instrument_llm_calls,
+               auto_start_session=auto_start_session,
+               inherited_session_id=inherited_session_id
+               )
+
+    return inherited_session_id or c.current_session_id
 
 
 def end_session(end_state: str,
@@ -66,7 +74,7 @@ def end_session(end_state: str,
     Client().end_session(end_state, end_state_reason, video)
 
 
-def start_session(tags: Optional[List[str]] = None, config: Optional[Configuration] = None):
+def start_session(tags: Optional[List[str]] = None, config: Optional[Configuration] = None, inherited_session_id: Optional[str] = None):
     """
         Start a new session for recording events.
 
@@ -75,7 +83,7 @@ def start_session(tags: Optional[List[str]] = None, config: Optional[Configurati
                 e.g. ["test_run"].
             config: (Configuration, optional): Client configuration object
     """
-    Client().start_session(tags, config)
+    return Client().start_session(tags, config, inherited_session_id)
 
 
 def record(event: Event | ErrorEvent):

agentops/agent.py

@@ -1,4 +1,4 @@
-import logging
+from .log_config import logger
 from uuid import uuid4
 from agentops import Client
 from inspect import isclass, isfunction
@@ -18,7 +18,7 @@ def new_init(self, *args, **kwargs):
                     self.agent_ops_agent_id = uuid4()
                     Client().create_agent(self.agent_ops_agent_id, self.agent_ops_agent_name)
                 except AttributeError as e:
-                    logging.error("AgentOps failed to track an agent. This often happens if agentops.init() was not "
+                    logger.warning("AgentOps failed to track an agent. This often happens if agentops.init() was not "
                                   "called before initializing an agent with the @track_agent decorator.")
                     raise e
 

agentops/client.py

@@ -14,11 +14,12 @@
 from uuid import uuid4
 from typing import Optional, List
 import traceback
-import logging
+from .log_config import logger, set_logging_level_info
 import inspect
 import atexit
 import signal
 import sys
+import threading
 
 from .meta_client import MetaClient
 from .config import Configuration, ConfigurationError
@@ -46,6 +47,7 @@ class Client(metaclass=MetaClient):
             override (bool, optional): [Deprecated] Use `instrument_llm_calls` instead. Whether to instrument LLM calls and emit LLMEvents..
             instrument_llm_calls (bool): Whether to instrument LLM calls and emit LLMEvents..
             auto_start_session (bool): Whether to start a session automatically when the client is created.
+            inherited_session_id (optional, str): Init Agentops with an existing Session
         Attributes:
             _session (Session, optional): A Session is a grouping of events (e.g. a run of your agent).
             _worker (Worker, optional): A Worker manages the event queue and sends session updates to the AgentOps api server
@@ -60,12 +62,13 @@ def __init__(self,
                  tags: Optional[List[str]] = None,
                  override: Optional[bool] = None,  # Deprecated
                  instrument_llm_calls=True,
-                 auto_start_session=True
+                 auto_start_session=True,
+                 inherited_session_id: Optional[str] = None
                  ):
 
         if override is not None:
-            logging.warning("🖇 AgentOps: The 'override' parameter is deprecated. Use 'instrument_llm_calls' instead.",
-                            DeprecationWarning, stacklevel=2)
+            logger.warning("🖇 AgentOps: The 'override' parameter is deprecated. Use 'instrument_llm_calls' instead.",
+                           DeprecationWarning, stacklevel=2)
             instrument_llm_calls = instrument_llm_calls or override
 
         self._session = None
@@ -84,7 +87,7 @@ def __init__(self,
         self._handle_unclean_exits()
 
         if auto_start_session:
-            self.start_session(tags, self.config)
+            self.start_session(tags, self.config, inherited_session_id)
 
         if instrument_llm_calls:
             self.llm_tracker = LlmTracker(self)
@@ -130,7 +133,7 @@ def record(self, event: Event | ErrorEvent):
         if self._session is not None and not self._session.has_ended:
             self._worker.add_event(event.__dict__)
         else:
-            logging.warning(
+            logger.warning(
                 "🖇 AgentOps: Cannot record event - no current session")
 
     def _record_event_sync(self, func, event_name, *args, **kwargs):
@@ -165,8 +168,7 @@ def _record_event_sync(self, func, event_name, *args, **kwargs):
             self.record(event)
 
         except Exception as e:
-            # TODO: add the stack trace
-            self.record(ErrorEvent(trigger_event=event, details={f"{type(e).__name__}": str(e)}))
+            self.record(ErrorEvent(trigger_event=event, exception=e))
 
             # Re-raise the exception
             raise
@@ -205,38 +207,42 @@ async def _record_event_async(self, func, event_name, *args, **kwargs):
             self.record(event)
 
         except Exception as e:
-            # TODO: add the stack trace
-            self.record(ErrorEvent(trigger_event=event, details={f"{type(e).__name__}": str(e)}))
+            self.record(ErrorEvent(trigger_event=event, exception=e))
 
             # Re-raise the exception
             raise
 
         return returns
 
-    def start_session(self, tags: Optional[List[str]] = None, config: Optional[Configuration] = None):
+    def start_session(self, tags: Optional[List[str]] = None, config: Optional[Configuration] = None, inherited_session_id: Optional[str] = None):
         """
             Start a new session for recording events.
 
             Args:
                 tags (List[str], optional): Tags that can be used for grouping or sorting later.
                     e.g. ["test_run"].
                 config: (Configuration, optional): Client configuration object
+                inherited_session_id (optional, str): assign session id to match existing Session
         """
+        set_logging_level_info()
+
         if self._session is not None:
-            return logging.warning("🖇 AgentOps: Cannot start session - session already started")
+            return logger.warning("🖇 AgentOps: Cannot start session - session already started")
 
         if not config and not self.config:
-            return logging.warning("🖇 AgentOps: Cannot start session - missing configuration")
+            return logger.warning("🖇 AgentOps: Cannot start session - missing configuration")
 
-        self._session = Session(uuid4(), tags or self._tags, host_env=get_host_env())
+        self._session = Session(inherited_session_id or uuid4(), tags or self._tags, host_env=get_host_env())
         self._worker = Worker(config or self.config)
         start_session_result = self._worker.start_session(self._session)
         if not start_session_result:
             self._session = None
-            return logging.warning("🖇 AgentOps: Cannot start session")
+            return logger.warning("🖇 AgentOps: Cannot start session")
+
+        logger.info('View info on this session at https://app.agentops.ai/drilldown?session_id={}'
+                    .format(self._session.session_id))
 
-        logging.info('View info on this session at https://app.agentops.ai/drilldown?session_id={}'
-                     .format(self._session.session_id))
+        return self._session.session_id
 
     def end_session(self,
                     end_state: str,
@@ -251,10 +257,10 @@ def end_session(self,
                 video (str, optional): The video screen recording of the session
         """
         if self._session is None or self._session.has_ended:
-            return logging.warning("🖇 AgentOps: Cannot end session - no current session")
+            return logger.warning("🖇 AgentOps: Cannot end session - no current session")
 
         if not any(end_state == state.value for state in EndState):
-            return logging.warning("🖇 AgentOps: Invalid end_state. Please use one of the EndState enums")
+            return logger.warning("🖇 AgentOps: Invalid end_state. Please use one of the EndState enums")
 
         self._session.video = video
         self._session.end_session(end_state, end_state_reason)
@@ -286,7 +292,7 @@ def signal_handler(signum, frame):
                     frame: The current stack frame.
             """
             signal_name = 'SIGINT' if signum == signal.SIGINT else 'SIGTERM'
-            logging.info(
+            logger.info(
                 f'🖇 AgentOps: {signal_name} detected. Ending session...')
             self.end_session(end_state='Fail',
                              end_state_reason=f'Signal {signal_name} detected')
@@ -311,15 +317,17 @@ def handle_exception(exc_type, exc_value, exc_traceback):
             # Then call the default excepthook to exit the program
             sys.__excepthook__(exc_type, exc_value, exc_traceback)
 
-        atexit.register(lambda: cleanup(end_state="Indeterminate",
-                        end_state_reason="Process exited without calling end_session()"))
-        signal.signal(signal.SIGINT, signal_handler)
-        signal.signal(signal.SIGTERM, signal_handler)
-        sys.excepthook = handle_exception
+        # if main thread
+        if isinstance(threading.current_thread(), threading._MainThread):
+            atexit.register(lambda: cleanup(end_state="Indeterminate",
+                            end_state_reason="Process exited without calling end_session()"))
+            signal.signal(signal.SIGINT, signal_handler)
+            signal.signal(signal.SIGTERM, signal_handler)
+            sys.excepthook = handle_exception
 
     @property
     def current_session_id(self):
-        return self._session.session_id
+        return self._session.session_id if self._session else None
 
     @property
     def api_key(self):

agentops/config.py

@@ -7,7 +7,7 @@
 
 from typing import Optional
 from os import environ
-import logging
+from .log_config import logger
 
 
 class Configuration:
@@ -143,4 +143,4 @@ class ConfigurationError(Exception):
 
     def __init__(self, message: str):
         super().__init__(message)
-        logging.warning(message)
+        logger.warning(message)

agentops/event.py

@@ -5,11 +5,12 @@
     Event: Represents discrete events to be recorded.
 """
 
-from dataclasses import dataclass, field
+from dataclasses import asdict, dataclass, field
 from typing import List, Optional
 from .helpers import get_ISO_time, check_call_stack_for_agent_id
 from .enums import EventType, Models
 from uuid import UUID, uuid4
+import traceback
 
 
 @dataclass
@@ -115,6 +116,7 @@ class ErrorEvent():
     For recording any errors e.g. ones related to agent execution
 
     trigger_event(Event, optional): The event object that triggered the error if applicable.
+    exception(BaseException, optional): The thrown exception. We will automatically parse the error_type and details from this.
     error_type(str, optional): The type of error e.g. "ValueError".
     code(str, optional): A code that can be used to identify the error e.g. 501.
     details(str, optional): Detailed information about the error.
@@ -123,17 +125,21 @@ class ErrorEvent():
 
     """
 
-    trigger_event: Optional[Event] = None  # TODO: remove from serialization?
+    trigger_event: Optional[Event] = None
+    exception: Optional[BaseException] = None
     error_type: Optional[str] = None
     code: Optional[str] = None
     details: Optional[str] = None
-    logs: Optional[str] = None
+    logs: Optional[str] = field(default_factory=traceback.format_exc)
     timestamp: str = field(default_factory=get_ISO_time)
 
     def __post_init__(self):
         self.event_type = EventType.ERROR.value
         if self.trigger_event:
             self.trigger_event_id = self.trigger_event.id
             self.trigger_event_type = self.trigger_event.event_type
-            # TODO: remove trigger_event from serialization
-            # e.g. field(repr=False, compare=False, hash=False, metadata={'serialize': False})
+            self.trigger_event = None  # removes trigger_event from serialization
+        if self.exception:
+            self.error_type = self.error_type or type(self.exception).__name__
+            self.details = self.details or str(self.exception)
+            self.exception = None  # removes exception from serialization

agentops/helpers.py

@@ -4,7 +4,7 @@
 from datetime import datetime
 import json
 import inspect
-import logging
+from .log_config import logger
 from uuid import UUID
 import os
 from importlib.metadata import version
@@ -85,7 +85,7 @@ def check_call_stack_for_agent_id() -> str | None:
             if var == "__main__":
                 return
             if hasattr(var, 'agent_ops_agent_id') and getattr(var, 'agent_ops_agent_id'):
-                logging.debug('LLM call from agent named: ' + getattr(var, 'agent_ops_agent_name'))
+                logger.debug('LLM call from agent named: ' + getattr(var, 'agent_ops_agent_name'))
                 return getattr(var, 'agent_ops_agent_id')
     return None
 
@@ -95,7 +95,7 @@ def get_agentops_version():
         pkg_version = version("agentops")
         return pkg_version
     except Exception as e:
-        logging.warning(f"Error reading package version: {e}")
+        logger.warning(f"Error reading package version: {e}")
         return None