diff --git a/README.md b/README.md index 4d80ada2..1be7f0e9 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ Build your next agent with benchmarks, observability, and replay analytics. AgentOps is the toolkit for evaluating and developing robust and reliable AI agents. -AgentOps is open beta. You can sign up for AgentOps [here](https://app.agentops.ai). +You can sign up for AgentOps [here](https://app.agentops.ai). [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) ![PyPI - Version](https://img.shields.io/pypi/v/agentops) @@ -49,16 +49,16 @@ Initialize the AgentOps client, and automatically get analytics on every LLM cal import agentops # Beginning of program's code (i.e. main.py, __init__.py) -ao_client = agentops.Client() +agentops.init() ... # (optional: record specific functions) -@record_function('sample function being record') +@agentops.record_function('sample function being record') def sample_function(...): ... # End of program -ao_client.end_session('Success') +agentops.end_session('Success') # Woohoo You're done 🎉 ``` diff --git a/agentops/__init__.py b/agentops/__init__.py index 232b3e90..0de3019d 100755 --- a/agentops/__init__.py +++ b/agentops/__init__.py @@ -8,6 +8,7 @@ from .enums import Models from .decorators import record_function from .agent import track_agent +from .log_config import set_logging_level_info, set_logging_level_critial def init(api_key: Optional[str] = None, @@ -41,7 +42,7 @@ def init(api_key: Optional[str] = None, inherited_session_id (optional, str): Init Agentops with an existing Session Attributes: """ - + set_logging_level_info() c = Client(api_key=api_key, parent_key=parent_key, endpoint=endpoint, @@ -52,7 +53,7 @@ def init(api_key: Optional[str] = None, auto_start_session=auto_start_session, inherited_session_id=inherited_session_id ) - + return inherited_session_id or c.current_session_id diff --git a/agentops/agent.py b/agentops/agent.py index 2a0a005c..fd3c8df4 100644 --- a/agentops/agent.py +++ b/agentops/agent.py @@ -1,4 +1,4 @@ -import logging +from .log_config import logger from uuid import uuid4 from agentops import Client from inspect import isclass, isfunction @@ -18,7 +18,7 @@ def new_init(self, *args, **kwargs): self.agent_ops_agent_id = uuid4() Client().create_agent(self.agent_ops_agent_id, self.agent_ops_agent_name) except AttributeError as e: - logging.error("AgentOps failed to track an agent. This often happens if agentops.init() was not " + logger.warning("AgentOps failed to track an agent. This often happens if agentops.init() was not " "called before initializing an agent with the @track_agent decorator.") raise e diff --git a/agentops/client.py b/agentops/client.py index a4eaedea..fe64799a 100644 --- a/agentops/client.py +++ b/agentops/client.py @@ -14,7 +14,7 @@ from uuid import uuid4 from typing import Optional, List import traceback -import logging +from .log_config import logger, set_logging_level_info import inspect import atexit import signal @@ -115,7 +115,7 @@ def record(self, event: Event | ErrorEvent): if self._session is not None and not self._session.has_ended: self._worker.add_event(event.__dict__) else: - logging.warning( + logger.warning( "🖇 AgentOps: Cannot record event - no current session") def _record_event_sync(self, func, event_name, *args, **kwargs): @@ -150,8 +150,7 @@ def _record_event_sync(self, func, event_name, *args, **kwargs): self.record(event) except Exception as e: - # TODO: add the stack trace - self.record(ErrorEvent(trigger_event=event, details={f"{type(e).__name__}": str(e)})) + self.record(ErrorEvent(trigger_event=event, exception=e)) # Re-raise the exception raise @@ -190,8 +189,7 @@ async def _record_event_async(self, func, event_name, *args, **kwargs): self.record(event) except Exception as e: - # TODO: add the stack trace - self.record(ErrorEvent(trigger_event=event, details={f"{type(e).__name__}": str(e)})) + self.record(ErrorEvent(trigger_event=event, exception=e)) # Re-raise the exception raise @@ -208,20 +206,22 @@ def start_session(self, tags: Optional[List[str]] = None, config: Optional[Confi config: (Configuration, optional): Client configuration object inherited_session_id (optional, str): assign session id to match existing Session """ + set_logging_level_info() + if self._session is not None: - return logging.warning("🖇 AgentOps: Cannot start session - session already started") + return logger.warning("🖇 AgentOps: Cannot start session - session already started") if not config and not self.config: - return logging.warning("🖇 AgentOps: Cannot start session - missing configuration") + return logger.warning("🖇 AgentOps: Cannot start session - missing configuration") self._session = Session(inherited_session_id or uuid4(), tags or self._tags, host_env=get_host_env()) self._worker = Worker(config or self.config) start_session_result = self._worker.start_session(self._session) if not start_session_result: self._session = None - return logging.warning("🖇 AgentOps: Cannot start session") + return logger.warning("🖇 AgentOps: Cannot start session") - logging.info('View info on this session at https://app.agentops.ai/drilldown?session_id={}' + logger.info('View info on this session at https://app.agentops.ai/drilldown?session_id={}' .format(self._session.session_id)) return self._session.session_id @@ -239,10 +239,10 @@ def end_session(self, video (str, optional): The video screen recording of the session """ if self._session is None or self._session.has_ended: - return logging.warning("🖇 AgentOps: Cannot end session - no current session") + return logger.warning("🖇 AgentOps: Cannot end session - no current session") if not any(end_state == state.value for state in EndState): - return logging.warning("🖇 AgentOps: Invalid end_state. Please use one of the EndState enums") + return logger.warning("🖇 AgentOps: Invalid end_state. Please use one of the EndState enums") self._session.video = video self._session.end_session(end_state, end_state_reason) @@ -274,7 +274,7 @@ def signal_handler(signum, frame): frame: The current stack frame. """ signal_name = 'SIGINT' if signum == signal.SIGINT else 'SIGTERM' - logging.info( + logger.info( f'🖇 AgentOps: {signal_name} detected. Ending session...') self.end_session(end_state='Fail', end_state_reason=f'Signal {signal_name} detected') diff --git a/agentops/config.py b/agentops/config.py index f5751b24..6ea2f7db 100644 --- a/agentops/config.py +++ b/agentops/config.py @@ -7,7 +7,7 @@ from typing import Optional from os import environ -import logging +from .log_config import logger class Configuration: @@ -143,4 +143,4 @@ class ConfigurationError(Exception): def __init__(self, message: str): super().__init__(message) - logging.warning(message) + logger.warning(message) diff --git a/agentops/event.py b/agentops/event.py index 76deeafc..e0f017db 100644 --- a/agentops/event.py +++ b/agentops/event.py @@ -5,11 +5,12 @@ Event: Represents discrete events to be recorded. """ -from dataclasses import dataclass, field +from dataclasses import asdict, dataclass, field from typing import List, Optional from .helpers import get_ISO_time, check_call_stack_for_agent_id from .enums import EventType, Models from uuid import UUID, uuid4 +import traceback @dataclass @@ -115,6 +116,7 @@ class ErrorEvent(): For recording any errors e.g. ones related to agent execution trigger_event(Event, optional): The event object that triggered the error if applicable. + exception(BaseException, optional): The thrown exception. We will automatically parse the error_type and details from this. error_type(str, optional): The type of error e.g. "ValueError". code(str, optional): A code that can be used to identify the error e.g. 501. details(str, optional): Detailed information about the error. @@ -123,11 +125,12 @@ class ErrorEvent(): """ - trigger_event: Optional[Event] = None # TODO: remove from serialization? + trigger_event: Optional[Event] = None + exception: Optional[BaseException] = None error_type: Optional[str] = None code: Optional[str] = None details: Optional[str] = None - logs: Optional[str] = None + logs: Optional[str] = field(default_factory=traceback.format_exc) timestamp: str = field(default_factory=get_ISO_time) def __post_init__(self): @@ -135,5 +138,8 @@ def __post_init__(self): if self.trigger_event: self.trigger_event_id = self.trigger_event.id self.trigger_event_type = self.trigger_event.event_type - # TODO: remove trigger_event from serialization - # e.g. field(repr=False, compare=False, hash=False, metadata={'serialize': False}) + self.trigger_event = None # removes trigger_event from serialization + if self.exception: + self.error_type = self.error_type or type(self.exception).__name__ + self.details = self.details or str(self.exception) + self.exception = None # removes exception from serialization diff --git a/agentops/helpers.py b/agentops/helpers.py index 409889f2..71990c4b 100644 --- a/agentops/helpers.py +++ b/agentops/helpers.py @@ -4,7 +4,7 @@ from datetime import datetime import json import inspect -import logging +from .log_config import logger from uuid import UUID import os from importlib.metadata import version @@ -85,7 +85,7 @@ def check_call_stack_for_agent_id() -> str | None: if var == "__main__": return if hasattr(var, 'agent_ops_agent_id') and getattr(var, 'agent_ops_agent_id'): - logging.debug('LLM call from agent named: ' + getattr(var, 'agent_ops_agent_name')) + logger.debug('LLM call from agent named: ' + getattr(var, 'agent_ops_agent_name')) return getattr(var, 'agent_ops_agent_id') return None @@ -95,7 +95,7 @@ def get_agentops_version(): pkg_version = version("agentops") return pkg_version except Exception as e: - logging.warning(f"Error reading package version: {e}") + logger.warning(f"Error reading package version: {e}") return None diff --git a/agentops/http_client.py b/agentops/http_client.py index f1d205f0..2c743e22 100644 --- a/agentops/http_client.py +++ b/agentops/http_client.py @@ -1,6 +1,6 @@ from enum import Enum from typing import Optional -import logging +from .log_config import logger import requests from requests.adapters import Retry, HTTPAdapter @@ -82,7 +82,7 @@ def post(url: str, payload: bytes, api_key: Optional[str] = None, parent_key: Op except requests.exceptions.Timeout: result.code = 408 result.status = HttpStatus.TIMEOUT - logging.warning( + logger.warning( '🖇 AgentOps: Could not post data - connection timed out') except requests.exceptions.HTTPError as e: try: @@ -96,12 +96,12 @@ def post(url: str, payload: bytes, api_key: Optional[str] = None, parent_key: Op result.body = {'error': str(e)} if result.code == 401: - logging.warning( + logger.warning( f'🖇 AgentOps: Could not post data - API server rejected your API key: {api_key}') if result.code == 400: - logging.warning(f'🖇 AgentOps: Could not post data - {result.body}') + logger.warning(f'🖇 AgentOps: Could not post data - {result.body}') if result.code == 500: - logging.warning( + logger.warning( f'🖇 AgentOps: Could not post data - internal server error') return result diff --git a/agentops/langchain_callback_handler.py b/agentops/langchain_callback_handler.py index a79d3ea9..fda660c5 100644 --- a/agentops/langchain_callback_handler.py +++ b/agentops/langchain_callback_handler.py @@ -80,7 +80,7 @@ def on_llm_error( llm_event: LLMEvent = self.events.llm[str(run_id)] self.ao_client.record(llm_event) - error_event = ErrorEvent(trigger_event=llm_event, details=str(error), timestamp=get_ISO_time()) + error_event = ErrorEvent(trigger_event=llm_event, exception=error) self.ao_client.record(error_event) @debug_print_function_params @@ -106,8 +106,7 @@ def on_llm_end( if len(response.generations) == 0: # TODO: more descriptive error - error_event = ErrorEvent(trigger_event=self.events.llm[str(run_id)], - details="on_llm_end: No generations", timestamp=get_ISO_time()) + error_event = ErrorEvent(trigger_event=self.events.llm[str(run_id)], error_type="NoGenerations", details="on_llm_end: No generations") self.ao_client.record(error_event) @debug_print_function_params @@ -156,7 +155,7 @@ def on_chain_error( action_event: ActionEvent = self.events.chain[str(run_id)] self.ao_client.record(action_event) - error_event = ErrorEvent(trigger_event=action_event, details=str(error), timestamp=get_ISO_time()) + error_event = ErrorEvent(trigger_event=action_event, exception=error) self.ao_client.record(error_event) @debug_print_function_params @@ -199,7 +198,7 @@ def on_tool_end( # Tools are capable of failing `on_tool_end` quietly. # This is a workaround to make sure we can log it as an error. if kwargs.get('name') == '_Exception': - error_event = ErrorEvent(trigger_event=tool_event, details=output, timestamp=get_ISO_time()) + error_event = ErrorEvent(trigger_event=tool_event, error_type="LangchainToolException", details=output) self.ao_client.record(error_event) @debug_print_function_params @@ -214,7 +213,7 @@ def on_tool_error( tool_event: ToolEvent = self.events.tool[str(run_id)] self.ao_client.record(tool_event) - error_event = ErrorEvent(trigger_event=tool_event, details=str(error), timestamp=get_ISO_time()) + error_event = ErrorEvent(trigger_event=tool_event, exception=error) self.ao_client.record(error_event) @debug_print_function_params @@ -265,7 +264,7 @@ def on_retriever_error( action_event: ActionEvent = self.events.retriever[str(run_id)] self.ao_client.record(action_event) - error_event = ErrorEvent(trigger_event=action_event, details=str(error), timestamp=get_ISO_time()) + error_event = ErrorEvent(trigger_event=action_event, exception=error) self.ao_client.record(error_event) @debug_print_function_params @@ -405,7 +404,7 @@ async def on_llm_error( llm_event: LLMEvent = self.events.llm[str(run_id)] self.ao_client.record(llm_event) - error_event = ErrorEvent(trigger_event=llm_event, details=str(error), timestamp=get_ISO_time()) + error_event = ErrorEvent(trigger_event=llm_event, exception=error) self.ao_client.record(error_event) @debug_print_function_params @@ -431,8 +430,7 @@ async def on_llm_end( if len(response.generations) == 0: # TODO: more descriptive error - error_event = ErrorEvent(trigger_event=self.events.llm[str(run_id)], - details="on_llm_end: No generations", timestamp=get_ISO_time()) + error_event = ErrorEvent(trigger_event=self.events.llm[str(run_id)], error_type="NoGenerations", details="on_llm_end: No generations") self.ao_client.record(error_event) @debug_print_function_params @@ -481,7 +479,7 @@ async def on_chain_error( action_event: ActionEvent = self.events.chain[str(run_id)] self.ao_client.record(action_event) - error_event = ErrorEvent(trigger_event=action_event, details=str(error), timestamp=get_ISO_time()) + error_event = ErrorEvent(trigger_event=action_event, exception=error) self.ao_client.record(error_event) @debug_print_function_params @@ -524,7 +522,7 @@ async def on_tool_end( # Tools are capable of failing `on_tool_end` quietly. # This is a workaround to make sure we can log it as an error. if kwargs.get('name') == '_Exception': - error_event = ErrorEvent(trigger_event=tool_event, details=output, timestamp=get_ISO_time()) + error_event = ErrorEvent(trigger_event=tool_event, error_type="LangchainToolException", details=output) self.ao_client.record(error_event) @debug_print_function_params @@ -539,7 +537,7 @@ async def on_tool_error( tool_event: ToolEvent = self.events.tool[str(run_id)] self.ao_client.record(tool_event) - error_event = ErrorEvent(trigger_event=tool_event, details=str(error), timestamp=get_ISO_time()) + error_event = ErrorEvent(trigger_event=tool_event, exception=error) self.ao_client.record(error_event) @debug_print_function_params @@ -590,7 +588,7 @@ async def on_retriever_error( action_event: ActionEvent = self.events.retriever[str(run_id)] self.ao_client.record(action_event) - error_event = ErrorEvent(trigger_event=action_event, details=str(error), timestamp=get_ISO_time()) + error_event = ErrorEvent(trigger_event=action_event, exception=error) self.ao_client.record(error_event) @debug_print_function_params diff --git a/agentops/llm_tracker.py b/agentops/llm_tracker.py index 1aefc392..3602f0bc 100644 --- a/agentops/llm_tracker.py +++ b/agentops/llm_tracker.py @@ -2,7 +2,7 @@ import sys from importlib import import_module from packaging.version import parse -import logging +from .log_config import logger from .event import LLMEvent, ErrorEvent from .helpers import get_ISO_time, check_call_stack_for_agent_id import inspect @@ -58,9 +58,9 @@ def handle_stream_chunk(chunk): self.client.record(self.llm_event) except Exception as e: - self.client.record(ErrorEvent(trigger_event=self.llm_event, details={f"{type(e).__name__}": str(e)})) - # TODO: This error is specific to only one path of failure. Should be more generic or have different logging for different paths - logging.warning( + self.client.record(ErrorEvent(trigger_event=self.llm_event, exception=e)) + # TODO: This error is specific to only one path of failure. Should be more generic or have different logger for different paths + logger.warning( f"🖇 AgentOps: Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps") # if the response is a generator, decorate the generator @@ -97,9 +97,9 @@ def generator(): self.client.record(self.llm_event) except Exception as e: - self.client.record(ErrorEvent(trigger_event=self.llm_event, details={f"{type(e).__name__}": str(e)})) - # TODO: This error is specific to only one path of failure. Should be more generic or have different logging for different paths - logging.warning( + self.client.record(ErrorEvent(trigger_event=self.llm_event, exception=e)) + # TODO: This error is specific to only one path of failure. Should be more generic or have different logger for different paths + logger.warning( f"🖇 AgentOps: Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps") return response @@ -143,9 +143,9 @@ def handle_stream_chunk(chunk: ChatCompletionChunk): self.client.record(self.llm_event) except Exception as e: - self.client.record(ErrorEvent(trigger_event=self.llm_event, details={f"{type(e).__name__}": str(e)})) - # TODO: This error is specific to only one path of failure. Should be more generic or have different logging for different paths - logging.warning( + self.client.record(ErrorEvent(trigger_event=self.llm_event, exception=e)) + # TODO: This error is specific to only one path of failure. Should be more generic or have different logger for different paths + logger.warning( f"🖇 AgentOps: Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps") # if the response is a generator, decorate the generator @@ -188,9 +188,9 @@ async def async_generator(): self.client.record(self.llm_event) except Exception as e: - self.client.record(ErrorEvent(trigger_event=self.llm_event, details={f"{type(e).__name__}": str(e)})) - # TODO: This error is specific to only one path of failure. Should be more generic or have different logging for different paths - logging.warning( + self.client.record(ErrorEvent(trigger_event=self.llm_event, exception=e)) + # TODO: This error is specific to only one path of failure. Should be more generic or have different logger for different paths + logger.warning( f"🖇 AgentOps: Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps") return response diff --git a/agentops/log_config.py b/agentops/log_config.py new file mode 100644 index 00000000..86cdecaf --- /dev/null +++ b/agentops/log_config.py @@ -0,0 +1,10 @@ +import logging + +logger = logging.getLogger("agentops") +logger.setLevel(logging.CRITICAL) + +def set_logging_level_critial(): + logger.setLevel(logging.CRITICAL) + +def set_logging_level_info(): + logger.setLevel(logging.INFO) \ No newline at end of file diff --git a/agentops/meta_client.py b/agentops/meta_client.py index d90d8035..b95973fa 100644 --- a/agentops/meta_client.py +++ b/agentops/meta_client.py @@ -1,4 +1,4 @@ -import logging +from .log_config import logger import traceback from .host_env import get_host_env @@ -45,7 +45,7 @@ def wrapper(self, *args, **kwargs): try: return method(self, *args, **kwargs) except Exception as e: - logging.warning(f"🖇 AgentOps: Error: {e}") + logger.warning(f"🖇 AgentOps: Error: {e}") config = getattr(self, 'config', None) if config is not None: type(self).send_exception_to_server(e, self.config._api_key) diff --git a/examples/langchain_examples.ipynb b/examples/langchain_examples.ipynb index 2498faf7..e0bfa337 100644 --- a/examples/langchain_examples.ipynb +++ b/examples/langchain_examples.ipynb @@ -107,10 +107,7 @@ { "name": "stderr", "output_type": "stream", - "text": [ - "/Users/howardgil/Desktop/agentops/AgentOps-AI/agentops/env/lib/python3.12/site-packages/langchain_core/_api/deprecation.py:117: LangChainDeprecationWarning: The class `langchain_community.chat_models.openai.ChatOpenAI` was deprecated in langchain-community 0.0.10 and will be removed in 0.2.0. An updated version of the class exists in the langchain-openai package and should be used instead. To use it run `pip install -U langchain-openai` and import as `from langchain_openai import ChatOpenAI`.\n", - " warn_deprecated(\n" - ] + "text": [""] } ], "source": [ diff --git a/pyproject.toml b/pyproject.toml index 5de5b79b..db81f3b6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "agentops" -version = "0.1.4" +version = "0.1.6" authors = [ { name="Alex Reibman", email="areibman@gmail.com" }, { name="Shawn Qiu", email="siyangqiu@gmail.com" },