Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Feature] Better ErrorEvents #156

Merged
merged 10 commits into from
Apr 20, 2024
Merged

[Feature] Better ErrorEvents #156

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
0f11a9d
Setting session to none if server does not return 200 for /sessions
HowieG Apr 3, 2024
95c5098
Merge branch 'main' of https://github.com/AgentOps-AI/agentops
HowieG Apr 8, 2024
7a0a207
Merge branch 'main' of https://github.com/AgentOps-AI/agentops
HowieG Apr 12, 2024
b78b642
Merge branch 'main' of https://github.com/AgentOps-AI/agentops
HowieG Apr 19, 2024
3af4e9b
WIP. Not working
HowieG Apr 19, 2024
b838d60
Adding ErrorEvent magic, stripping out trigger_event and exception
HowieG Apr 19, 2024
6d31c7f
Modified existing ErrorEvents
HowieG Apr 19, 2024
5b71efc
Working
HowieG Apr 19, 2024
9cef633
Removed hardcoded error
HowieG Apr 19, 2024
4b8f1a2
Removed resolved comments
HowieG Apr 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 2 additions & 4 deletions agentops/client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -150,8 +150,7 @@ def _record_event_sync(self, func, event_name, *args, **kwargs):
self.record(event)

except Exception as e:
# TODO: add the stack trace
self.record(ErrorEvent(trigger_event=event, details={f"{type(e).__name__}": str(e)}))
self.record(ErrorEvent(trigger_event=event, exception=e))

# Re-raise the exception
raise
Expand Down Expand Up @@ -190,8 +189,7 @@ async def _record_event_async(self, func, event_name, *args, **kwargs):
self.record(event)

except Exception as e:
# TODO: add the stack trace
self.record(ErrorEvent(trigger_event=event, details={f"{type(e).__name__}": str(e)}))
self.record(ErrorEvent(trigger_event=event, exception=e))

# Re-raise the exception
raise
Expand Down
16 changes: 11 additions & 5 deletions agentops/event.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,12 @@
Event: Represents discrete events to be recorded.
"""

from dataclasses import dataclass, field
from dataclasses import asdict, dataclass, field
from typing import List, Optional
from .helpers import get_ISO_time, check_call_stack_for_agent_id
from .enums import EventType, Models
from uuid import UUID, uuid4
import traceback


@dataclass
Expand Down Expand Up @@ -115,6 +116,7 @@ class ErrorEvent():
For recording any errors e.g. ones related to agent execution

trigger_event(Event, optional): The event object that triggered the error if applicable.
exception(BaseException, optional): The thrown exception. We will automatically parse the error_type and details from this.
error_type(str, optional): The type of error e.g. "ValueError".
code(str, optional): A code that can be used to identify the error e.g. 501.
details(str, optional): Detailed information about the error.
Expand All @@ -123,17 +125,21 @@ class ErrorEvent():

"""

trigger_event: Optional[Event] = None # TODO: remove from serialization?
trigger_event: Optional[Event] = None
exception: Optional[BaseException] = None
error_type: Optional[str] = None
code: Optional[str] = None
details: Optional[str] = None
logs: Optional[str] = None
logs: Optional[str] = field(default_factory=traceback.format_exc)
timestamp: str = field(default_factory=get_ISO_time)

def __post_init__(self):
self.event_type = EventType.ERROR.value
if self.trigger_event:
self.trigger_event_id = self.trigger_event.id
self.trigger_event_type = self.trigger_event.event_type
# TODO: remove trigger_event from serialization
# e.g. field(repr=False, compare=False, hash=False, metadata={'serialize': False})
self.trigger_event = None # removes trigger_event from serialization
if self.exception:
self.error_type = self.error_type or type(self.exception).__name__
self.details = self.details or str(self.exception)
self.exception = None # removes exception from serialization
26 changes: 12 additions & 14 deletions agentops/langchain_callback_handler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ def on_llm_error(
llm_event: LLMEvent = self.events.llm[str(run_id)]
self.ao_client.record(llm_event)

error_event = ErrorEvent(trigger_event=llm_event, details=str(error), timestamp=get_ISO_time())
error_event = ErrorEvent(trigger_event=llm_event, exception=error)
self.ao_client.record(error_event)

@debug_print_function_params
Expand All @@ -106,8 +106,7 @@ def on_llm_end(

if len(response.generations) == 0:
# TODO: more descriptive error
error_event = ErrorEvent(trigger_event=self.events.llm[str(run_id)],
details="on_llm_end: No generations", timestamp=get_ISO_time())
error_event = ErrorEvent(trigger_event=self.events.llm[str(run_id)], error_type="NoGenerations", details="on_llm_end: No generations")
self.ao_client.record(error_event)

@debug_print_function_params
Expand Down Expand Up @@ -156,7 +155,7 @@ def on_chain_error(
action_event: ActionEvent = self.events.chain[str(run_id)]
self.ao_client.record(action_event)

error_event = ErrorEvent(trigger_event=action_event, details=str(error), timestamp=get_ISO_time())
error_event = ErrorEvent(trigger_event=action_event, exception=error)
self.ao_client.record(error_event)

@debug_print_function_params
Expand Down Expand Up @@ -199,7 +198,7 @@ def on_tool_end(
# Tools are capable of failing `on_tool_end` quietly.
# This is a workaround to make sure we can log it as an error.
if kwargs.get('name') == '_Exception':
error_event = ErrorEvent(trigger_event=tool_event, details=output, timestamp=get_ISO_time())
error_event = ErrorEvent(trigger_event=tool_event, error_type="LangchainToolException", details=output)
self.ao_client.record(error_event)

@debug_print_function_params
Expand All @@ -214,7 +213,7 @@ def on_tool_error(
tool_event: ToolEvent = self.events.tool[str(run_id)]
self.ao_client.record(tool_event)

error_event = ErrorEvent(trigger_event=tool_event, details=str(error), timestamp=get_ISO_time())
error_event = ErrorEvent(trigger_event=tool_event, exception=error)
self.ao_client.record(error_event)

@debug_print_function_params
Expand Down Expand Up @@ -265,7 +264,7 @@ def on_retriever_error(
action_event: ActionEvent = self.events.retriever[str(run_id)]
self.ao_client.record(action_event)

error_event = ErrorEvent(trigger_event=action_event, details=str(error), timestamp=get_ISO_time())
error_event = ErrorEvent(trigger_event=action_event, exception=error)
self.ao_client.record(error_event)

@debug_print_function_params
Expand Down Expand Up @@ -405,7 +404,7 @@ async def on_llm_error(
llm_event: LLMEvent = self.events.llm[str(run_id)]
self.ao_client.record(llm_event)

error_event = ErrorEvent(trigger_event=llm_event, details=str(error), timestamp=get_ISO_time())
error_event = ErrorEvent(trigger_event=llm_event, exception=error)
self.ao_client.record(error_event)

@debug_print_function_params
Expand All @@ -431,8 +430,7 @@ async def on_llm_end(

if len(response.generations) == 0:
# TODO: more descriptive error
error_event = ErrorEvent(trigger_event=self.events.llm[str(run_id)],
details="on_llm_end: No generations", timestamp=get_ISO_time())
error_event = ErrorEvent(trigger_event=self.events.llm[str(run_id)], error_type="NoGenerations", details="on_llm_end: No generations")
self.ao_client.record(error_event)

@debug_print_function_params
Expand Down Expand Up @@ -481,7 +479,7 @@ async def on_chain_error(
action_event: ActionEvent = self.events.chain[str(run_id)]
self.ao_client.record(action_event)

error_event = ErrorEvent(trigger_event=action_event, details=str(error), timestamp=get_ISO_time())
error_event = ErrorEvent(trigger_event=action_event, exception=error)
self.ao_client.record(error_event)

@debug_print_function_params
Expand Down Expand Up @@ -524,7 +522,7 @@ async def on_tool_end(
# Tools are capable of failing `on_tool_end` quietly.
# This is a workaround to make sure we can log it as an error.
if kwargs.get('name') == '_Exception':
error_event = ErrorEvent(trigger_event=tool_event, details=output, timestamp=get_ISO_time())
error_event = ErrorEvent(trigger_event=tool_event, error_type="LangchainToolException", details=output)
self.ao_client.record(error_event)

@debug_print_function_params
Expand All @@ -539,7 +537,7 @@ async def on_tool_error(
tool_event: ToolEvent = self.events.tool[str(run_id)]
self.ao_client.record(tool_event)

error_event = ErrorEvent(trigger_event=tool_event, details=str(error), timestamp=get_ISO_time())
error_event = ErrorEvent(trigger_event=tool_event, exception=error)
self.ao_client.record(error_event)

@debug_print_function_params
Expand Down Expand Up @@ -590,7 +588,7 @@ async def on_retriever_error(
action_event: ActionEvent = self.events.retriever[str(run_id)]
self.ao_client.record(action_event)

error_event = ErrorEvent(trigger_event=action_event, details=str(error), timestamp=get_ISO_time())
error_event = ErrorEvent(trigger_event=action_event, exception=error)
self.ao_client.record(error_event)

@debug_print_function_params
Expand Down
8 changes: 4 additions & 4 deletions agentops/llm_tracker.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ def handle_stream_chunk(chunk):

self.client.record(self.llm_event)
except Exception as e:
self.client.record(ErrorEvent(trigger_event=self.llm_event, details={f"{type(e).__name__}": str(e)}))
self.client.record(ErrorEvent(trigger_event=self.llm_event, exception=e))
# TODO: This error is specific to only one path of failure. Should be more generic or have different logging for different paths
logging.warning(
f"🖇 AgentOps: Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")
Expand Down Expand Up @@ -97,7 +97,7 @@ def generator():

self.client.record(self.llm_event)
except Exception as e:
self.client.record(ErrorEvent(trigger_event=self.llm_event, details={f"{type(e).__name__}": str(e)}))
self.client.record(ErrorEvent(trigger_event=self.llm_event, exception=e))
# TODO: This error is specific to only one path of failure. Should be more generic or have different logging for different paths
logging.warning(
f"🖇 AgentOps: Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")
Expand Down Expand Up @@ -143,7 +143,7 @@ def handle_stream_chunk(chunk: ChatCompletionChunk):

self.client.record(self.llm_event)
except Exception as e:
self.client.record(ErrorEvent(trigger_event=self.llm_event, details={f"{type(e).__name__}": str(e)}))
self.client.record(ErrorEvent(trigger_event=self.llm_event, exception=e))
# TODO: This error is specific to only one path of failure. Should be more generic or have different logging for different paths
logging.warning(
f"🖇 AgentOps: Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")
Expand Down Expand Up @@ -188,7 +188,7 @@ async def async_generator():

self.client.record(self.llm_event)
except Exception as e:
self.client.record(ErrorEvent(trigger_event=self.llm_event, details={f"{type(e).__name__}": str(e)}))
self.client.record(ErrorEvent(trigger_event=self.llm_event, exception=e))
# TODO: This error is specific to only one path of failure. Should be more generic or have different logging for different paths
logging.warning(
f"🖇 AgentOps: Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")
Expand Down
Loading