Merge branch 'main' of https://github.com/AgentOps-AI/agentops

README.md

@@ -25,7 +25,7 @@
 
 Build your next agent with benchmarks, observability, and replay analytics. AgentOps is the toolkit for evaluating and developing robust and reliable AI agents.
 
-AgentOps is open beta. You can sign up for AgentOps [here](https://app.agentops.ai).
+You can sign up for AgentOps [here](https://app.agentops.ai).
 
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) ![PyPI - Version](https://img.shields.io/pypi/v/agentops) <a href="https://pepy.tech/project/agentops">
   <img src="https://static.pepy.tech/badge/agentops/month"> <a href="https://twitter.com/agentopsai">
@@ -49,16 +49,16 @@ Initialize the AgentOps client, and automatically get analytics on every LLM cal
 import agentops
 
 # Beginning of program's code (i.e. main.py, __init__.py)
-ao_client = agentops.Client(<INSERT YOUR API KEY HERE>)
+agentops.init(<INSERT YOUR API KEY HERE>)
 
 ...
 # (optional: record specific functions)
-@record_function('sample function being record')
+@agentops.record_function('sample function being record')
 def sample_function(...):
     ...
 
 # End of program
-ao_client.end_session('Success')
+agentops.end_session('Success')
 # Woohoo You're done 🎉
 ```
 

agentops/__init__.py

@@ -8,6 +8,7 @@
 from .enums import Models
 from .decorators import record_function
 from .agent import track_agent
+from .log_config import set_logging_level_info, set_logging_level_critial
 
 
 def init(api_key: Optional[str] = None,
@@ -41,7 +42,7 @@ def init(api_key: Optional[str] = None,
             inherited_session_id (optional, str): Init Agentops with an existing Session
         Attributes:
         """
-
+    set_logging_level_info()
     c = Client(api_key=api_key,
                parent_key=parent_key,
                endpoint=endpoint,
@@ -52,7 +53,7 @@ def init(api_key: Optional[str] = None,
                auto_start_session=auto_start_session,
                inherited_session_id=inherited_session_id
                )
-
+    
     return inherited_session_id or c.current_session_id
 
 

agentops/agent.py

@@ -1,4 +1,4 @@
-import logging
+from .log_config import logger
 from uuid import uuid4
 from agentops import Client
 from inspect import isclass, isfunction
@@ -18,7 +18,7 @@ def new_init(self, *args, **kwargs):
                     self.agent_ops_agent_id = uuid4()
                     Client().create_agent(self.agent_ops_agent_id, self.agent_ops_agent_name)
                 except AttributeError as e:
-                    logging.error("AgentOps failed to track an agent. This often happens if agentops.init() was not "
+                    logger.warning("AgentOps failed to track an agent. This often happens if agentops.init() was not "
                                   "called before initializing an agent with the @track_agent decorator.")
                     raise e
 

agentops/client.py

@@ -14,7 +14,7 @@
 from uuid import uuid4
 from typing import Optional, List
 import traceback
-import logging
+from .log_config import logger, set_logging_level_info
 import inspect
 import atexit
 import signal
@@ -115,7 +115,7 @@ def record(self, event: Event | ErrorEvent):
         if self._session is not None and not self._session.has_ended:
             self._worker.add_event(event.__dict__)
         else:
-            logging.warning(
+            logger.warning(
                 "🖇 AgentOps: Cannot record event - no current session")
 
     def _record_event_sync(self, func, event_name, *args, **kwargs):
@@ -150,8 +150,7 @@ def _record_event_sync(self, func, event_name, *args, **kwargs):
             self.record(event)
 
         except Exception as e:
-            # TODO: add the stack trace
-            self.record(ErrorEvent(trigger_event=event, details={f"{type(e).__name__}": str(e)}))
+            self.record(ErrorEvent(trigger_event=event, exception=e))
 
             # Re-raise the exception
             raise
@@ -190,8 +189,7 @@ async def _record_event_async(self, func, event_name, *args, **kwargs):
             self.record(event)
 
         except Exception as e:
-            # TODO: add the stack trace
-            self.record(ErrorEvent(trigger_event=event, details={f"{type(e).__name__}": str(e)}))
+            self.record(ErrorEvent(trigger_event=event, exception=e))
 
             # Re-raise the exception
             raise
@@ -208,20 +206,22 @@ def start_session(self, tags: Optional[List[str]] = None, config: Optional[Confi
             config: (Configuration, optional): Client configuration object
             inherited_session_id (optional, str): assign session id to match existing Session
         """
+        set_logging_level_info()
+
         if self._session is not None:
-            return logging.warning("🖇 AgentOps: Cannot start session - session already started")
+            return logger.warning("🖇 AgentOps: Cannot start session - session already started")
 
         if not config and not self.config:
-            return logging.warning("🖇 AgentOps: Cannot start session - missing configuration")
+            return logger.warning("🖇 AgentOps: Cannot start session - missing configuration")
 
         self._session = Session(inherited_session_id or uuid4(), tags or self._tags, host_env=get_host_env())
         self._worker = Worker(config or self.config)
         start_session_result = self._worker.start_session(self._session)
         if not start_session_result:
             self._session = None
-            return logging.warning("🖇 AgentOps: Cannot start session")
+            return logger.warning("🖇 AgentOps: Cannot start session")
 
-        logging.info('View info on this session at https://app.agentops.ai/drilldown?session_id={}'
+        logger.info('View info on this session at https://app.agentops.ai/drilldown?session_id={}'
                      .format(self._session.session_id))
 
         return self._session.session_id
@@ -239,10 +239,10 @@ def end_session(self,
             video (str, optional): The video screen recording of the session
         """
         if self._session is None or self._session.has_ended:
-            return logging.warning("🖇 AgentOps: Cannot end session - no current session")
+            return logger.warning("🖇 AgentOps: Cannot end session - no current session")
 
         if not any(end_state == state.value for state in EndState):
-            return logging.warning("🖇 AgentOps: Invalid end_state. Please use one of the EndState enums")
+            return logger.warning("🖇 AgentOps: Invalid end_state. Please use one of the EndState enums")
 
         self._session.video = video
         self._session.end_session(end_state, end_state_reason)
@@ -274,7 +274,7 @@ def signal_handler(signum, frame):
                 frame: The current stack frame.
             """
             signal_name = 'SIGINT' if signum == signal.SIGINT else 'SIGTERM'
-            logging.info(
+            logger.info(
                 f'🖇 AgentOps: {signal_name} detected. Ending session...')
             self.end_session(end_state='Fail',
                              end_state_reason=f'Signal {signal_name} detected')

agentops/config.py

@@ -7,7 +7,7 @@
 
 from typing import Optional
 from os import environ
-import logging
+from .log_config import logger
 
 
 class Configuration:
@@ -143,4 +143,4 @@ class ConfigurationError(Exception):
 
     def __init__(self, message: str):
         super().__init__(message)
-        logging.warning(message)
+        logger.warning(message)

agentops/event.py

@@ -5,11 +5,12 @@
     Event: Represents discrete events to be recorded.
 """
 
-from dataclasses import dataclass, field
+from dataclasses import asdict, dataclass, field
 from typing import List, Optional
 from .helpers import get_ISO_time, check_call_stack_for_agent_id
 from .enums import EventType, Models
 from uuid import UUID, uuid4
+import traceback
 
 
 @dataclass
@@ -115,6 +116,7 @@ class ErrorEvent():
     For recording any errors e.g. ones related to agent execution
 
     trigger_event(Event, optional): The event object that triggered the error if applicable.
+    exception(BaseException, optional): The thrown exception. We will automatically parse the error_type and details from this.
     error_type(str, optional): The type of error e.g. "ValueError".
     code(str, optional): A code that can be used to identify the error e.g. 501.
     details(str, optional): Detailed information about the error.
@@ -123,17 +125,21 @@ class ErrorEvent():
 
     """
 
-    trigger_event: Optional[Event] = None  # TODO: remove from serialization?
+    trigger_event: Optional[Event] = None
+    exception: Optional[BaseException] = None
     error_type: Optional[str] = None
     code: Optional[str] = None
     details: Optional[str] = None
-    logs: Optional[str] = None
+    logs: Optional[str] = field(default_factory=traceback.format_exc)
     timestamp: str = field(default_factory=get_ISO_time)
 
     def __post_init__(self):
         self.event_type = EventType.ERROR.value
         if self.trigger_event:
             self.trigger_event_id = self.trigger_event.id
             self.trigger_event_type = self.trigger_event.event_type
-            # TODO: remove trigger_event from serialization
-            # e.g. field(repr=False, compare=False, hash=False, metadata={'serialize': False})
+            self.trigger_event = None  # removes trigger_event from serialization
+        if self.exception:
+            self.error_type = self.error_type or type(self.exception).__name__
+            self.details = self.details or str(self.exception)
+            self.exception = None  # removes exception from serialization

agentops/helpers.py

@@ -4,7 +4,7 @@
 from datetime import datetime
 import json
 import inspect
-import logging
+from .log_config import logger
 from uuid import UUID
 import os
 from importlib.metadata import version
@@ -85,7 +85,7 @@ def check_call_stack_for_agent_id() -> str | None:
             if var == "__main__":
                 return
             if hasattr(var, 'agent_ops_agent_id') and getattr(var, 'agent_ops_agent_id'):
-                logging.debug('LLM call from agent named: ' + getattr(var, 'agent_ops_agent_name'))
+                logger.debug('LLM call from agent named: ' + getattr(var, 'agent_ops_agent_name'))
                 return getattr(var, 'agent_ops_agent_id')
     return None
 
@@ -95,7 +95,7 @@ def get_agentops_version():
         pkg_version = version("agentops")
         return pkg_version
     except Exception as e:
-        logging.warning(f"Error reading package version: {e}")
+        logger.warning(f"Error reading package version: {e}")
         return None
 
 

agentops/http_client.py

@@ -1,6 +1,6 @@
 from enum import Enum
 from typing import Optional
-import logging
+from .log_config import logger
 import requests
 from requests.adapters import Retry, HTTPAdapter
 
@@ -82,7 +82,7 @@ def post(url: str, payload: bytes, api_key: Optional[str] = None, parent_key: Op
         except requests.exceptions.Timeout:
             result.code = 408
             result.status = HttpStatus.TIMEOUT
-            logging.warning(
+            logger.warning(
                 '🖇 AgentOps: Could not post data - connection timed out')
         except requests.exceptions.HTTPError as e:
             try:
@@ -96,12 +96,12 @@ def post(url: str, payload: bytes, api_key: Optional[str] = None, parent_key: Op
             result.body = {'error': str(e)}
 
         if result.code == 401:
-            logging.warning(
+            logger.warning(
                 f'🖇 AgentOps: Could not post data - API server rejected your API key: {api_key}')
         if result.code == 400:
-            logging.warning(f'🖇 AgentOps: Could not post data - {result.body}')
+            logger.warning(f'🖇 AgentOps: Could not post data - {result.body}')
         if result.code == 500:
-            logging.warning(
+            logger.warning(
                 f'🖇 AgentOps: Could not post data - internal server error')
 
         return result

agentops/langchain_callback_handler.py

@@ -80,7 +80,7 @@ def on_llm_error(
         llm_event: LLMEvent = self.events.llm[str(run_id)]
         self.ao_client.record(llm_event)
 
-        error_event = ErrorEvent(trigger_event=llm_event, details=str(error), timestamp=get_ISO_time())
+        error_event = ErrorEvent(trigger_event=llm_event, exception=error)
         self.ao_client.record(error_event)
 
     @debug_print_function_params
@@ -106,8 +106,7 @@ def on_llm_end(
 
         if len(response.generations) == 0:
             # TODO: more descriptive error
-            error_event = ErrorEvent(trigger_event=self.events.llm[str(run_id)],
-                                     details="on_llm_end: No generations", timestamp=get_ISO_time())
+            error_event = ErrorEvent(trigger_event=self.events.llm[str(run_id)], error_type="NoGenerations", details="on_llm_end: No generations")
             self.ao_client.record(error_event)
 
     @debug_print_function_params
@@ -156,7 +155,7 @@ def on_chain_error(
         action_event: ActionEvent = self.events.chain[str(run_id)]
         self.ao_client.record(action_event)
 
-        error_event = ErrorEvent(trigger_event=action_event, details=str(error), timestamp=get_ISO_time())
+        error_event = ErrorEvent(trigger_event=action_event, exception=error)
         self.ao_client.record(error_event)
 
     @debug_print_function_params
@@ -199,7 +198,7 @@ def on_tool_end(
         # Tools are capable of failing `on_tool_end` quietly.
         # This is a workaround to make sure we can log it as an error.
         if kwargs.get('name') == '_Exception':
-            error_event = ErrorEvent(trigger_event=tool_event, details=output, timestamp=get_ISO_time())
+            error_event = ErrorEvent(trigger_event=tool_event, error_type="LangchainToolException", details=output)
             self.ao_client.record(error_event)
 
     @debug_print_function_params
@@ -214,7 +213,7 @@ def on_tool_error(
         tool_event: ToolEvent = self.events.tool[str(run_id)]
         self.ao_client.record(tool_event)
 
-        error_event = ErrorEvent(trigger_event=tool_event, details=str(error), timestamp=get_ISO_time())
+        error_event = ErrorEvent(trigger_event=tool_event, exception=error)
         self.ao_client.record(error_event)
 
     @debug_print_function_params
@@ -265,7 +264,7 @@ def on_retriever_error(
         action_event: ActionEvent = self.events.retriever[str(run_id)]
         self.ao_client.record(action_event)
 
-        error_event = ErrorEvent(trigger_event=action_event, details=str(error), timestamp=get_ISO_time())
+        error_event = ErrorEvent(trigger_event=action_event, exception=error)
         self.ao_client.record(error_event)
 
     @debug_print_function_params
@@ -405,7 +404,7 @@ async def on_llm_error(
         llm_event: LLMEvent = self.events.llm[str(run_id)]
         self.ao_client.record(llm_event)
 
-        error_event = ErrorEvent(trigger_event=llm_event, details=str(error), timestamp=get_ISO_time())
+        error_event = ErrorEvent(trigger_event=llm_event, exception=error)
         self.ao_client.record(error_event)
 
     @debug_print_function_params
@@ -431,8 +430,7 @@ async def on_llm_end(
 
         if len(response.generations) == 0:
             # TODO: more descriptive error
-            error_event = ErrorEvent(trigger_event=self.events.llm[str(run_id)],
-                                     details="on_llm_end: No generations", timestamp=get_ISO_time())
+            error_event = ErrorEvent(trigger_event=self.events.llm[str(run_id)], error_type="NoGenerations", details="on_llm_end: No generations")
             self.ao_client.record(error_event)
 
     @debug_print_function_params
@@ -481,7 +479,7 @@ async def on_chain_error(
         action_event: ActionEvent = self.events.chain[str(run_id)]
         self.ao_client.record(action_event)
 
-        error_event = ErrorEvent(trigger_event=action_event, details=str(error), timestamp=get_ISO_time())
+        error_event = ErrorEvent(trigger_event=action_event, exception=error)
         self.ao_client.record(error_event)
 
     @debug_print_function_params
@@ -524,7 +522,7 @@ async def on_tool_end(
         # Tools are capable of failing `on_tool_end` quietly.
         # This is a workaround to make sure we can log it as an error.
         if kwargs.get('name') == '_Exception':
-            error_event = ErrorEvent(trigger_event=tool_event, details=output, timestamp=get_ISO_time())
+            error_event = ErrorEvent(trigger_event=tool_event, error_type="LangchainToolException", details=output)
             self.ao_client.record(error_event)
 
     @debug_print_function_params
@@ -539,7 +537,7 @@ async def on_tool_error(
         tool_event: ToolEvent = self.events.tool[str(run_id)]
         self.ao_client.record(tool_event)
 
-        error_event = ErrorEvent(trigger_event=tool_event, details=str(error), timestamp=get_ISO_time())
+        error_event = ErrorEvent(trigger_event=tool_event, exception=error)
         self.ao_client.record(error_event)
 
     @debug_print_function_params
@@ -590,7 +588,7 @@ async def on_retriever_error(
         action_event: ActionEvent = self.events.retriever[str(run_id)]
         self.ao_client.record(action_event)
 
-        error_event = ErrorEvent(trigger_event=action_event, details=str(error), timestamp=get_ISO_time())
+        error_event = ErrorEvent(trigger_event=action_event, exception=error)
         self.ao_client.record(error_event)
 
     @debug_print_function_params