diff --git a/.github/workflows/python-testing.yml b/.github/workflows/python-testing.yml index 73840652..36615c21 100644 --- a/.github/workflows/python-testing.yml +++ b/.github/workflows/python-testing.yml @@ -15,7 +15,7 @@ jobs: strategy: matrix: - python-version: [3.11] + python-version: [3.7,3.8,3.9,3.10,3.11,3.12] steps: - uses: actions/checkout@v2 diff --git a/README.md b/README.md index 11d08f1e..db901126 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@
- Logo + Logo

diff --git a/agentops/__init__.py b/agentops/__init__.py index 6cd37a41..1c2d4dfa 100755 --- a/agentops/__init__.py +++ b/agentops/__init__.py @@ -1,6 +1,6 @@ # agentops/__init__.py from os import environ -from typing import Optional, List +from typing import Optional, List, Union from .client import Client from .config import Configuration @@ -9,6 +9,7 @@ from .decorators import record_function from .agent import track_agent from .log_config import set_logging_level_info, set_logging_level_critial +from .langchain_callback_handler import LangchainCallbackHandler, AsyncLangchainCallbackHandler def init(api_key: Optional[str] = None, @@ -86,7 +87,7 @@ def start_session(tags: Optional[List[str]] = None, config: Optional[Configurati return Client().start_session(tags, config, inherited_session_id) -def record(event: Event | ErrorEvent): +def record(event: Union[Event, ErrorEvent]): """ Record an event with the AgentOps service. @@ -128,3 +129,6 @@ def set_parent_key(parent_key): parent_key (str): The API key of the parent organization to set. """ Client().set_parent_key(parent_key) + +def stop_instrumenting(): + Client().stop_instrumenting() diff --git a/agentops/agent.py b/agentops/agent.py index fd3c8df4..b0611681 100644 --- a/agentops/agent.py +++ b/agentops/agent.py @@ -1,10 +1,12 @@ +from typing import Union + from .log_config import logger from uuid import uuid4 from agentops import Client from inspect import isclass, isfunction -def track_agent(name: str | None = None): +def track_agent(name: Union[str, None] = None): def decorator(obj): if name: obj.agent_ops_agent_name = name @@ -15,7 +17,7 @@ def decorator(obj): def new_init(self, *args, **kwargs): try: original_init(self, *args, **kwargs) - self.agent_ops_agent_id = uuid4() + self.agent_ops_agent_id = str(uuid4()) Client().create_agent(self.agent_ops_agent_id, self.agent_ops_agent_name) except AttributeError as e: logger.warning("AgentOps failed to track an agent. This often happens if agentops.init() was not " @@ -25,7 +27,7 @@ def new_init(self, *args, **kwargs): obj.__init__ = new_init elif isfunction(obj): - obj.agent_ops_agent_id = uuid4() + obj.agent_ops_agent_id = str(uuid4()) Client().create_agent(obj.agent_ops_agent_id, obj.agent_ops_agent_name) else: diff --git a/agentops/client.py b/agentops/client.py index 6df00bf7..8707aba4 100644 --- a/agentops/client.py +++ b/agentops/client.py @@ -12,7 +12,7 @@ from .worker import Worker from .host_env import get_host_env from uuid import uuid4 -from typing import Optional, List +from typing import Optional, List, Union import traceback from .log_config import logger, set_logging_level_info from decimal import Decimal @@ -73,9 +73,9 @@ def __init__(self, DeprecationWarning, stacklevel=2) instrument_llm_calls = instrument_llm_calls or override - self._session = None - self._worker = None - self._tags_for_future_session = None + self._session: Optional[Session] = None + self._worker: Optional[Worker] = None + self._tags: Optional[List[str]] = tags self._env_data_opt_out = os.getenv('AGENTOPS_ENV_DATA_OPT_OUT') and os.getenv( 'AGENTOPS_ENV_DATA_OPT_OUT').lower() == 'true' @@ -114,7 +114,7 @@ def add_tags(self, tags: List[str]): else: self._session.tags = tags - if self._session is not None: + if self._session is not None and self._worker is not None: self._worker.update_session(self._session) def set_tags(self, tags: List[str]): @@ -126,11 +126,11 @@ def set_tags(self, tags: List[str]): """ self._tags_for_future_session = tags - if self._session is not None: + if self._session is not None and self._worker is not None: self._session.tags = tags self._worker.update_session(self._session) - def record(self, event: Event | ErrorEvent): + def record(self, event: Union[Event, ErrorEvent]): """ Record an event with the AgentOps service. @@ -254,8 +254,8 @@ def start_session(self, tags: Optional[List[str]] = None, config: Optional[Confi self._session = None return logger.warning("🖇 AgentOps: Cannot start session") - logger.info('View info on this session at https://app.agentops.ai/drilldown?session_id={}' - .format(self._session.session_id)) + logger.info('View info on this session at https://app.agentops.ai/drilldown?session_id=%s', + self._session.session_id) return self._session.session_id @@ -276,10 +276,14 @@ def end_session(self, if not any(end_state == state.value for state in EndState): return logger.warning("🖇 AgentOps: Invalid end_state. Please use one of the EndState enums") + + if self._worker is None or self._worker._session is None: + return logger.warning("🖇 AgentOps: Cannot end session - no current worker or session") self._session.video = video self._session.end_session(end_state, end_state_reason) token_cost = self._worker.end_session(self._session) + if token_cost == 'unknown': print('🖇 AgentOps: Could not determine cost of run.') else: @@ -294,7 +298,7 @@ def create_agent(self, agent_id: str, name: str): self._worker.create_agent(agent_id, name) def _handle_unclean_exits(self): - def cleanup(end_state: Optional[str] = 'Fail', end_state_reason: Optional[str] = None): + def cleanup(end_state: str = 'Fail', end_state_reason: Optional[str] = None): # Only run cleanup function if session is created if self._session is not None: self.end_session(end_state=end_state, @@ -309,8 +313,7 @@ def signal_handler(signum, frame): frame: The current stack frame. """ signal_name = 'SIGINT' if signum == signal.SIGINT else 'SIGTERM' - logger.info( - f'🖇 AgentOps: {signal_name} detected. Ending session...') + logger.info('🖇 AgentOps: %s detected. Ending session...', signal_name) self.end_session(end_state='Fail', end_state_reason=f'Signal {signal_name} detected') sys.exit(0) @@ -363,3 +366,6 @@ def set_parent_key(self, parent_key: str): @property def parent_key(self): return self.config.parent_key + + def stop_instrumenting(self): + self.llm_tracker.stop_instrumenting() diff --git a/agentops/event.py b/agentops/event.py index 2d34c08a..b7efbc75 100644 --- a/agentops/event.py +++ b/agentops/event.py @@ -6,7 +6,7 @@ """ from dataclasses import asdict, dataclass, field -from typing import List, Optional +from typing import Any, Dict, List, Optional, Sequence, Union from .helpers import get_ISO_time, check_call_stack_for_agent_id from .enums import EventType, Models from uuid import UUID, uuid4 @@ -59,7 +59,7 @@ class ActionEvent(Event): event_type: str = EventType.ACTION.value # TODO: Should not be optional, but non-default argument 'agent_id' follows default argument error action_type: Optional[str] = None - logs: Optional[str] = None + logs: Optional[Union[str, Sequence[Any]]] = None screenshot: Optional[str] = None # May be needed if we keep Optional for agent_id @@ -85,11 +85,11 @@ class LLMEvent(Event): event_type: str = EventType.LLM.value thread_id: Optional[UUID] = None - prompt: str | List = None + prompt: Optional[Union[str, List]] = None prompt_tokens: Optional[int] = None - completion: str | object = None + completion: Union[str, object] = None completion_tokens: Optional[int] = None - model: Optional[Models | str] = None + model: Optional[Union[Models, str]] = None @dataclass @@ -103,7 +103,7 @@ class ToolEvent(Event): """ event_type: str = EventType.TOOL.value name: Optional[str] = None - logs: Optional[str | dict] = None + logs: Optional[Union[str, dict]] = None # Does not inherit from Event because error will (optionally) be linked to an ActionEvent, LLMEvent, etc that will have the details @@ -128,7 +128,7 @@ class ErrorEvent(): exception: Optional[BaseException] = None error_type: Optional[str] = None code: Optional[str] = None - details: Optional[str] = None + details: Optional[Union[str, Dict[str, str]]] = None logs: Optional[str] = field(default_factory=traceback.format_exc) timestamp: str = field(default_factory=get_ISO_time) diff --git a/agentops/helpers.py b/agentops/helpers.py index 71990c4b..1a8cfcad 100644 --- a/agentops/helpers.py +++ b/agentops/helpers.py @@ -1,9 +1,11 @@ -from pprint import pprint, pformat +from pprint import pformat from functools import wraps import time from datetime import datetime import json import inspect +from typing import Union + from .log_config import logger from uuid import UUID import os @@ -76,16 +78,16 @@ def remove_none_values(value): return json.dumps(cleaned_obj, default=default) -def check_call_stack_for_agent_id() -> str | None: +def check_call_stack_for_agent_id() -> Union[UUID, None]: for frame_info in inspect.stack(): # Look through the call stack for the class that called the LLM local_vars = frame_info.frame.f_locals for var in local_vars.values(): # We stop looking up the stack at main because after that we see global variables if var == "__main__": - return + return None if hasattr(var, 'agent_ops_agent_id') and getattr(var, 'agent_ops_agent_id'): - logger.debug('LLM call from agent named: ' + getattr(var, 'agent_ops_agent_name')) + logger.debug('LLM call from agent named: %s', getattr(var, 'agent_ops_agent_name')) return getattr(var, 'agent_ops_agent_id') return None @@ -95,7 +97,7 @@ def get_agentops_version(): pkg_version = version("agentops") return pkg_version except Exception as e: - logger.warning(f"Error reading package version: {e}") + logger.warning('Error reading package version: %s', e) return None diff --git a/agentops/http_client.py b/agentops/http_client.py index 2c743e22..fee12f98 100644 --- a/agentops/http_client.py +++ b/agentops/http_client.py @@ -28,9 +28,7 @@ class Response: def __init__(self, status: HttpStatus = HttpStatus.UNKNOWN, body: Optional[dict] = None): self.status: HttpStatus = status self.code: int = status.value - self.body = body - if not self.body: - self.body = {} + self.body = body if body else {} def parse(self, res: requests.models.Response): res_body = res.json() @@ -87,7 +85,7 @@ def post(url: str, payload: bytes, api_key: Optional[str] = None, parent_key: Op except requests.exceptions.HTTPError as e: try: result.parse(e.response) - except: + except Exception: result = Response() result.code = e.response.status_code result.status = Response.get_status(e.response.status_code) @@ -96,12 +94,12 @@ def post(url: str, payload: bytes, api_key: Optional[str] = None, parent_key: Op result.body = {'error': str(e)} if result.code == 401: - logger.warning( - f'🖇 AgentOps: Could not post data - API server rejected your API key: {api_key}') + logger.warning('🖇 AgentOps: Could not post data - API server rejected your API key: %s', + api_key) if result.code == 400: - logger.warning(f'🖇 AgentOps: Could not post data - {result.body}') + logger.warning('🖇 AgentOps: Could not post data - %s', result.body) if result.code == 500: logger.warning( - f'🖇 AgentOps: Could not post data - internal server error') + '🖇 AgentOps: Could not post data - internal server error') return result diff --git a/agentops/langchain_callback_handler.py b/agentops/langchain_callback_handler.py index fda660c5..65ee9e1d 100644 --- a/agentops/langchain_callback_handler.py +++ b/agentops/langchain_callback_handler.py @@ -34,7 +34,7 @@ def __init__(self, api_key: Optional[str] = None, max_queue_size: Optional[int] = None, tags: Optional[List[str]] = None): - client_params = { + client_params: Dict[str, Any] = { 'api_key': api_key, 'endpoint': endpoint, 'max_wait_time': max_wait_time, @@ -331,7 +331,7 @@ def __init__(self, api_key: Optional[str] = None, max_queue_size: Optional[int] = None, tags: Optional[List[str]] = None): - client_params = { + client_params: Dict[str, Any] = { 'api_key': api_key, 'endpoint': endpoint, 'max_wait_time': max_wait_time, diff --git a/agentops/llm_tracker.py b/agentops/llm_tracker.py index 507b8504..2b823fe9 100644 --- a/agentops/llm_tracker.py +++ b/agentops/llm_tracker.py @@ -7,8 +7,11 @@ from .event import LLMEvent, ErrorEvent from .helpers import get_ISO_time, check_call_stack_for_agent_id import inspect +from typing import Optional import pprint +original_create = None +original_create_async = None class LlmTracker: SUPPORTED_APIS = { @@ -27,6 +30,8 @@ class LlmTracker: def __init__(self, client): self.client = client + self.completion = "" + self.llm_event: Optional[LLMEvent] = None def _handle_response_v0_openai(self, response, kwargs, init_timestamp): """Handle responses for OpenAI versions None: """ Sets a url to the video recording of the session. @@ -38,7 +38,7 @@ def set_session_video(self, video: str): """ self.video = video - def end_session(self, end_state: str = "Indeterminate", end_state_reason: Optional[str] = None): + def end_session(self, end_state: str = "Indeterminate", end_state_reason: Optional[str] = None) -> None: """ End the session with a specified state, rating, and reason. diff --git a/agentops/worker.py b/agentops/worker.py index 8d9d3f93..eaa833ed 100644 --- a/agentops/worker.py +++ b/agentops/worker.py @@ -2,10 +2,10 @@ import threading import time from .http_client import HttpClient -from .config import Configuration, ConfigurationError +from .config import Configuration from .session import Session from .helpers import safe_serialize, filter_unjsonable -from typing import Dict +from typing import Dict, Optional class Worker: @@ -17,7 +17,7 @@ def __init__(self, config: Configuration) -> None: self.thread = threading.Thread(target=self.run) self.thread.daemon = True self.thread.start() - self._session: Session | None = None + self._session: Optional[Session] = None def add_event(self, event: dict) -> None: with self.lock: @@ -32,7 +32,7 @@ def flush_queue(self) -> None: self.queue = [] payload = { - "session_id": self._session.session_id, + "session_id": getattr(self._session, "session_id", None), "events": events } @@ -42,7 +42,7 @@ def flush_queue(self) -> None: self.config.api_key, self.config.parent_key) - def start_session(self, session: Session) -> None: + def start_session(self, session: Session) -> bool: self._session = session with self.lock: payload = { @@ -94,7 +94,7 @@ def create_agent(self, agent_id, name): payload = { "id": agent_id, "name": name, - "session_id": self._session.session_id + "session_id": getattr(self._session, "session_id", None), } serialized_payload = \ diff --git a/logo.png b/logo.png index 8a4b3fc4..9b390b6c 100644 Binary files a/logo.png and b/logo.png differ diff --git a/pyproject.toml b/pyproject.toml index cd97b497..b0af0351 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ authors = [ ] description = "Python SDK for developing AI agent evals and observability" readme = "README.md" -requires-python = ">=3.10" +requires-python = ">=3.7" classifiers = [ "Programming Language :: Python :: 3", "License :: OSI Approved :: MIT License", diff --git a/tox.ini b/tox.ini index 217cd70b..dd339225 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = py310,py311,py312 +envlist = py37,py38,py39,py310,py311,py312 [testenv] deps = @@ -17,6 +17,8 @@ deps = types-requests psutil openai + langchain-core + langchain commands = coverage run --source agentops -m pytest coverage report -m