From 4f79bb0f4325a5e52c628acc0f2877c78100bb6f Mon Sep 17 00:00:00 2001 From: Braelyn Boynton Date: Fri, 3 May 2024 16:15:15 -0700 Subject: [PATCH 1/2] Indent fix, test fix and python version expansion (#186) * union types * add tags fix * cleanup * all py versions --- .github/workflows/python-testing.yml | 2 +- agentops/__init__.py | 4 ++-- agentops/agent.py | 4 +++- agentops/client.py | 5 ++--- agentops/event.py | 18 +++++++++--------- agentops/helpers.py | 4 +++- agentops/llm_tracker.py | 2 -- pyproject.toml | 2 +- tox.ini | 2 +- 9 files changed, 22 insertions(+), 21 deletions(-) diff --git a/.github/workflows/python-testing.yml b/.github/workflows/python-testing.yml index 73840652..36615c21 100644 --- a/.github/workflows/python-testing.yml +++ b/.github/workflows/python-testing.yml @@ -15,7 +15,7 @@ jobs: strategy: matrix: - python-version: [3.11] + python-version: [3.7,3.8,3.9,3.10,3.11,3.12] steps: - uses: actions/checkout@v2 diff --git a/agentops/__init__.py b/agentops/__init__.py index 6cd37a41..3d087f1e 100755 --- a/agentops/__init__.py +++ b/agentops/__init__.py @@ -1,6 +1,6 @@ # agentops/__init__.py from os import environ -from typing import Optional, List +from typing import Optional, List, Union from .client import Client from .config import Configuration @@ -86,7 +86,7 @@ def start_session(tags: Optional[List[str]] = None, config: Optional[Configurati return Client().start_session(tags, config, inherited_session_id) -def record(event: Event | ErrorEvent): +def record(event: Union[Event, ErrorEvent]): """ Record an event with the AgentOps service. diff --git a/agentops/agent.py b/agentops/agent.py index 66d03e67..b0611681 100644 --- a/agentops/agent.py +++ b/agentops/agent.py @@ -1,10 +1,12 @@ +from typing import Union + from .log_config import logger from uuid import uuid4 from agentops import Client from inspect import isclass, isfunction -def track_agent(name: str | None = None): +def track_agent(name: Union[str, None] = None): def decorator(obj): if name: obj.agent_ops_agent_name = name diff --git a/agentops/client.py b/agentops/client.py index aaed2225..11b4de24 100644 --- a/agentops/client.py +++ b/agentops/client.py @@ -12,7 +12,7 @@ from .worker import Worker from .host_env import get_host_env from uuid import uuid4 -from typing import Optional, List +from typing import Optional, List, Union import traceback from .log_config import logger, set_logging_level_info from decimal import Decimal @@ -114,7 +114,6 @@ def add_tags(self, tags: List[str]): self._session.tags = tags if self._session is not None and self._worker is not None: - self._session.tags = self._tags self._worker.update_session(self._session) def set_tags(self, tags: List[str]): @@ -130,7 +129,7 @@ def set_tags(self, tags: List[str]): self._session.tags = tags self._worker.update_session(self._session) - def record(self, event: Event | ErrorEvent): + def record(self, event: Union[Event, ErrorEvent]): """ Record an event with the AgentOps service. diff --git a/agentops/event.py b/agentops/event.py index 9eed63d9..5773bf17 100644 --- a/agentops/event.py +++ b/agentops/event.py @@ -6,7 +6,7 @@ """ from dataclasses import asdict, dataclass, field -from typing import Any, Dict, List, Optional, Sequence +from typing import Any, Dict, List, Optional, Sequence, Union from .helpers import get_ISO_time, check_call_stack_for_agent_id from .enums import EventType, Models from uuid import UUID, uuid4 @@ -38,8 +38,8 @@ class Event: """ event_type: str # EventType.ENUM.value - params: Optional[str | Dict[str, Any]] = None - returns: Optional[str | Dict[str, Any]] = None + params: Optional[Union[str, Dict[str, Any]]] = None + returns: Optional[Union[str, Dict[str, Any]]] = None init_timestamp: Optional[str] = field(default_factory=get_ISO_time) end_timestamp: str = field(default_factory=get_ISO_time) agent_id: Optional[UUID] = field(default_factory=check_call_stack_for_agent_id) @@ -60,7 +60,7 @@ class ActionEvent(Event): event_type: str = EventType.ACTION.value # TODO: Should not be optional, but non-default argument 'agent_id' follows default argument error action_type: Optional[str] = None - logs: Optional[str | Sequence[Any]] = None + logs: Optional[Union[str, Sequence[Any]]] = None screenshot: Optional[str] = None # May be needed if we keep Optional for agent_id @@ -86,11 +86,11 @@ class LLMEvent(Event): event_type: str = EventType.LLM.value thread_id: Optional[UUID] = None - prompt: Optional[str | List] = None + prompt: Optional[Union[str, List]] = None prompt_tokens: Optional[int] = None - completion: str | object = None + completion: Union[str, object] = None completion_tokens: Optional[int] = None - model: Optional[Models | str] = None + model: Optional[Union[Models, str]] = None @dataclass @@ -104,7 +104,7 @@ class ToolEvent(Event): """ event_type: str = EventType.TOOL.value name: Optional[str] = None - logs: Optional[str | dict] = None + logs: Optional[Union[str, dict]] = None # Does not inherit from Event because error will (optionally) be linked to an ActionEvent, LLMEvent, etc that will have the details @@ -129,7 +129,7 @@ class ErrorEvent(): exception: Optional[BaseException] = None error_type: Optional[str] = None code: Optional[str] = None - details: Optional[str | Dict[str, str]] = None + details: Optional[Union[str, Dict[str, str]]] = None logs: Optional[str] = field(default_factory=traceback.format_exc) timestamp: str = field(default_factory=get_ISO_time) diff --git a/agentops/helpers.py b/agentops/helpers.py index 371a90f7..1a8cfcad 100644 --- a/agentops/helpers.py +++ b/agentops/helpers.py @@ -4,6 +4,8 @@ from datetime import datetime import json import inspect +from typing import Union + from .log_config import logger from uuid import UUID import os @@ -76,7 +78,7 @@ def remove_none_values(value): return json.dumps(cleaned_obj, default=default) -def check_call_stack_for_agent_id() -> UUID | None: +def check_call_stack_for_agent_id() -> Union[UUID, None]: for frame_info in inspect.stack(): # Look through the call stack for the class that called the LLM local_vars = frame_info.frame.f_locals diff --git a/agentops/llm_tracker.py b/agentops/llm_tracker.py index 76d7daec..0a8c78fe 100644 --- a/agentops/llm_tracker.py +++ b/agentops/llm_tracker.py @@ -71,8 +71,6 @@ def handle_stream_chunk(chunk): kwargs_str = pprint.pformat(kwargs) chunk = pprint.pformat(chunk) logger.warning( - "🖇 AgentOps: Unable to parse a chunk for LLM call %s - skipping upload to AgentOps", - kwargs) f"🖇 AgentOps: Unable to parse a chunk for LLM call. Skipping upload to AgentOps\n" f"chunk:\n {chunk}\n" f"kwargs:\n {kwargs_str}\n" diff --git a/pyproject.toml b/pyproject.toml index a3f14a93..624946e4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ authors = [ ] description = "Python SDK for developing AI agent evals and observability" readme = "README.md" -requires-python = ">=3.10" +requires-python = ">=3.7" classifiers = [ "Programming Language :: Python :: 3", "License :: OSI Approved :: MIT License", diff --git a/tox.ini b/tox.ini index 217cd70b..fa53e078 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = py310,py311,py312 +envlist = py37,py38,py39,py310,py311,py312 [testenv] deps = From 388101a3ad9947e0eabd1048d7864da04cce6a5a Mon Sep 17 00:00:00 2001 From: Braelyn Boynton Date: Fri, 3 May 2024 16:24:46 -0700 Subject: [PATCH 2/2] add stop_instrumenting (#185) * add stop_instrumenting * version bump * test deps --- agentops/__init__.py | 4 ++++ agentops/client.py | 5 ++++- agentops/llm_tracker.py | 22 ++++++++++++++++++++-- pyproject.toml | 2 +- tox.ini | 2 ++ 5 files changed, 31 insertions(+), 4 deletions(-) diff --git a/agentops/__init__.py b/agentops/__init__.py index 3d087f1e..1c2d4dfa 100755 --- a/agentops/__init__.py +++ b/agentops/__init__.py @@ -9,6 +9,7 @@ from .decorators import record_function from .agent import track_agent from .log_config import set_logging_level_info, set_logging_level_critial +from .langchain_callback_handler import LangchainCallbackHandler, AsyncLangchainCallbackHandler def init(api_key: Optional[str] = None, @@ -128,3 +129,6 @@ def set_parent_key(parent_key): parent_key (str): The API key of the parent organization to set. """ Client().set_parent_key(parent_key) + +def stop_instrumenting(): + Client().stop_instrumenting() diff --git a/agentops/client.py b/agentops/client.py index 11b4de24..7dc21559 100644 --- a/agentops/client.py +++ b/agentops/client.py @@ -136,7 +136,7 @@ def record(self, event: Union[Event, ErrorEvent]): Args: event (Event): The event to record. """ - if not event.end_timestamp or event.init_timestamp == event.end_timestamp: + if isinstance(event, Event) and not event.end_timestamp or event.init_timestamp == event.end_timestamp: event.end_timestamp = get_ISO_time() if self._session is not None and not self._session.has_ended and self._worker is not None: if isinstance(event, ErrorEvent): @@ -359,3 +359,6 @@ def set_parent_key(self, parent_key: str): @property def parent_key(self): return self.config.parent_key + + def stop_instrumenting(self): + self.llm_tracker.stop_instrumenting() diff --git a/agentops/llm_tracker.py b/agentops/llm_tracker.py index 0a8c78fe..2b823fe9 100644 --- a/agentops/llm_tracker.py +++ b/agentops/llm_tracker.py @@ -10,6 +10,8 @@ from typing import Optional import pprint +original_create = None +original_create_async = None class LlmTracker: SUPPORTED_APIS = { @@ -230,6 +232,7 @@ def override_openai_v1_completion(self): from openai.resources.chat import completions # Store the original method + global original_create original_create = completions.Completions.create def patched_function(*args, **kwargs): @@ -245,12 +248,13 @@ def override_openai_v1_async_completion(self): from openai.resources.chat import completions # Store the original method - original_create = completions.AsyncCompletions.create + global original_create_async + original_create_async = completions.AsyncCompletions.create async def patched_function(*args, **kwargs): # Call the original function with its original arguments init_timestamp = get_ISO_time() - result = await original_create(*args, **kwargs) + result = await original_create_async(*args, **kwargs) return self._handle_response_v1_openai(result, kwargs, init_timestamp) # Override the original method with the patched one @@ -345,3 +349,17 @@ def override_api(self): # Patch openai