From cf1ca9fd968d41312c9ee788a53e5017234371cf Mon Sep 17 00:00:00 2001 From: Howard Gil Date: Wed, 15 May 2024 15:55:12 -0700 Subject: [PATCH 1/6] Fixed typo that was causing crash when not using autogen (#207) * Fixed typo * Added error info --- agentops/client.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/agentops/client.py b/agentops/client.py index 92653f1e6..049b49dbc 100644 --- a/agentops/client.py +++ b/agentops/client.py @@ -82,7 +82,6 @@ def __init__(self, self._tags: Optional[List[str]] = tags self._tags_for_future_session: Optional[List[str]] = None - self._env_data_opt_out = os.getenv('AGENTOPS_ENV_DATA_OPT_OUT') and os.getenv( 'AGENTOPS_ENV_DATA_OPT_OUT').lower() == 'true' @@ -97,7 +96,8 @@ def __init__(self, self._handle_unclean_exits() - instrument_llm_calls, auto_start_session = self._check_for_partner_frameworks(instrument_llm_calls, auto_start_session) + instrument_llm_calls, auto_start_session = self._check_for_partner_frameworks( + instrument_llm_calls, auto_start_session) if auto_start_session: self.start_session(tags, self.config, inherited_session_id) @@ -113,11 +113,16 @@ def _check_for_partner_frameworks(self, instrument_llm_calls, auto_start_session for framework in partner_frameworks.keys(): if framework in sys.modules: self.add_tags([framework]) - if 'autogen': - import autogen - autogen.runtime_logging.start(logger_type="agentops") + if framework == 'autogen': + try: + import autogen + autogen.runtime_logging.start(logger_type="agentops") + except ImportError: + pass + except Exception as e: + logger.warning("🖇️ AgentOps: Failed to set up autogen logger with AgentOps. Error: " + e) - return partner_frameworks[framework] + return partner_frameworks[framework] return instrument_llm_calls, auto_start_session @@ -402,4 +407,5 @@ def parent_key(self): return self.config.parent_key def stop_instrumenting(self): - self.llm_tracker.stop_instrumenting() + if self.llm_tracker: + self.llm_tracker.stop_instrumenting() From 5b13f9b3eeda2832cbb7c15c91ac7a4f26c44fab Mon Sep 17 00:00:00 2001 From: Howard Gil Date: Wed, 15 May 2024 16:34:23 -0700 Subject: [PATCH 2/6] Fixed misordering of create_agent by adding named parameters (#208) --- agentops/agent.py | 6 +++--- agentops/client.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/agentops/agent.py b/agentops/agent.py index b06116815..0e070b62a 100644 --- a/agentops/agent.py +++ b/agentops/agent.py @@ -18,17 +18,17 @@ def new_init(self, *args, **kwargs): try: original_init(self, *args, **kwargs) self.agent_ops_agent_id = str(uuid4()) - Client().create_agent(self.agent_ops_agent_id, self.agent_ops_agent_name) + Client().create_agent(name=self.agent_ops_agent_name, agent_id=self.agent_ops_agent_id) except AttributeError as e: logger.warning("AgentOps failed to track an agent. This often happens if agentops.init() was not " - "called before initializing an agent with the @track_agent decorator.") + "called before initializing an agent with the @track_agent decorator.") raise e obj.__init__ = new_init elif isfunction(obj): obj.agent_ops_agent_id = str(uuid4()) - Client().create_agent(obj.agent_ops_agent_id, obj.agent_ops_agent_name) + Client().create_agent(name=obj.agent_ops_agent_name, agent_id=obj.agent_ops_agent_id) else: raise Exception("Invalid input, 'obj' must be a class or a function") diff --git a/agentops/client.py b/agentops/client.py index 049b49dbc..2d8abd96e 100644 --- a/agentops/client.py +++ b/agentops/client.py @@ -333,7 +333,7 @@ def create_agent(self, name: str, agent_id: Optional[str] = None) -> str: if agent_id is None: agent_id = str(uuid.uuid4()) if self._worker: - self._worker.create_agent(agent_id, name) + self._worker.create_agent(name=name, agent_id=agent_id) return agent_id def _handle_unclean_exits(self): From ed9d7b3ebda17627328aecec66e2cde9864f546b Mon Sep 17 00:00:00 2001 From: Howard Gil Date: Wed, 15 May 2024 17:44:27 -0700 Subject: [PATCH 3/6] Adding data to host_env (#205) * Added stuff to host_env * Undo last reset * Put back critical stuff * undoing whitespace change --- agentops/host_env.py | 73 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 71 insertions(+), 2 deletions(-) diff --git a/agentops/host_env.py b/agentops/host_env.py index 4eff0c89a..13d279436 100644 --- a/agentops/host_env.py +++ b/agentops/host_env.py @@ -2,6 +2,9 @@ import psutil import socket from .helpers import get_agentops_version +import importlib.metadata +import os +import sys def get_sdk_details(): @@ -9,6 +12,68 @@ def get_sdk_details(): return { "AgentOps SDK Version": get_agentops_version(), "Python Version": platform.python_version(), + "System Packages": get_sys_packages() + } + except: + return {} + + +def get_python_details(): + try: + return { + "Python Version": platform.python_version() + } + except: + return {} + + +def get_agentops_details(): + try: + return { + "AgentOps SDK Version": get_agentops_version() + } + except: + return {} + + +def get_sys_packages(): + sys_packages = {} + for module in sys.modules: + try: + version = importlib.metadata.version(module) + sys_packages[module] = version + except importlib.metadata.PackageNotFoundError: + # Skip built-in modules and those without package metadata + continue + + return sys_packages + + +def get_installed_packages(): + + try: + return { + # TODO: test + # TODO: add to opt out + "Installed Packages": {dist.metadata['Name']: dist.version for dist in importlib.metadata.distributions()} + } + except: + return {} + + +def get_current_directory(): + try: + return { + "Project Working Directory": os.getcwd() + } + except: + return {} + + +def get_virtual_env(): + try: + return { + "Virtual Environment": os.environ.get('VIRTUAL_ENV', None) } except: return {} @@ -51,7 +116,6 @@ def get_ram_details(): return {} - def get_disk_details(): partitions = psutil.disk_partitions() disk_info = {} @@ -71,7 +135,9 @@ def get_host_env(opt_out: bool = False): if opt_out: return { "SDK": get_sdk_details(), - "OS": get_os_details() + "OS": get_os_details(), + "Project Working Directory": get_current_directory(), + "Virtual Environment": get_virtual_env() } else: return { @@ -80,4 +146,7 @@ def get_host_env(opt_out: bool = False): "CPU": get_cpu_details(), "RAM": get_ram_details(), "Disk": get_disk_details(), + "Installed Packages": get_installed_packages(), + "Project Working Directory": get_current_directory(), + "Virtual Environment": get_virtual_env() } From 1ef9ef55e625e1ccfe5e67401feeacd7087545d6 Mon Sep 17 00:00:00 2001 From: Howard Gil Date: Thu, 16 May 2024 17:01:37 -0700 Subject: [PATCH 4/6] feature: additional cohere support and example script (#212) --- agentops/llm_tracker.py | 56 ++++++++++++++++++++++++++++++++------ examples/cohere.py | 30 -------------------- examples/cohere_example.py | 46 +++++++++++++++++++++++++++++++ 3 files changed, 94 insertions(+), 38 deletions(-) delete mode 100644 examples/cohere.py create mode 100644 examples/cohere_example.py diff --git a/agentops/llm_tracker.py b/agentops/llm_tracker.py index 9beb42eff..fd60f67d8 100644 --- a/agentops/llm_tracker.py +++ b/agentops/llm_tracker.py @@ -4,7 +4,7 @@ from importlib.metadata import version from packaging.version import Version, parse from .log_config import logger -from .event import LLMEvent, ErrorEvent +from .event import LLMEvent, ActionEvent, ToolEvent, ErrorEvent from .helpers import get_ISO_time, check_call_stack_for_agent_id import inspect from typing import Optional @@ -228,7 +228,7 @@ async def async_generator(): return response def _handle_response_cohere(self, response, kwargs, init_timestamp): - # TODO: """Handle responses for Cohere versions >v5.4.0""" + """Handle responses for Cohere versions >v5.4.0""" from cohere.types.non_streamed_chat_response import NonStreamedChatResponse from cohere.types.streamed_chat_response import ( StreamedChatResponse, @@ -248,6 +248,8 @@ def _handle_response_cohere(self, response, kwargs, init_timestamp): params=kwargs ) + self.action_events = {} + def handle_stream_chunk(chunk): # We take the first chunk and accumulate the deltas from all subsequent chunks to build one full chat completion @@ -261,21 +263,59 @@ def handle_stream_chunk(chunk): try: if isinstance(chunk, StreamedChatResponse_StreamEnd): - # Streaming is done. Record LLMEvent - # self.llm_event.returns.finish_reason = chunk.is_finished + # StreamedChatResponse_TextGeneration = LLMEvent self.llm_event.completion = { - "role": "assistant", "content": self.llm_event.completion} + "role": "assistant", "content": chunk.response.text} self.llm_event.end_timestamp = get_ISO_time() - self.client.record(self.llm_event) + # StreamedChatResponse_SearchResults = ActionEvent + search_results = chunk.response.search_results + for search_result in search_results: + query = search_result.search_query + if query.generation_id in self.action_events: + action_event = self.action_events[query.generation_id] + search_result_dict = search_result.dict() + del search_result_dict["search_query"] + action_event.returns = search_result_dict + action_event.end_timestamp = get_ISO_time() + + # StreamedChatResponse_CitationGeneration = ActionEvent + documents = {doc['id']: doc for doc in chunk.response.documents} + citations = chunk.response.citations + for citation in citations: + citation_id = f"{citation.start}.{citation.end}" + if citation_id in self.action_events: + action_event = self.action_events[citation_id] + citation_dict = citation.dict() + # Replace document_ids with the actual documents + citation_dict['documents'] = [documents[doc_id] + for doc_id in citation_dict['document_ids'] if doc_id in documents] + del citation_dict['document_ids'] + + action_event.returns = citation_dict + action_event.end_timestamp = get_ISO_time() + + for key, action_event in self.action_events.items(): + self.client.record(action_event) + elif isinstance(chunk, StreamedChatResponse_TextGeneration): self.llm_event.completion += chunk.text elif isinstance(chunk, StreamedChatResponse_ToolCallsGeneration): pass elif isinstance(chunk, StreamedChatResponse_CitationGeneration): - pass + for citation in chunk.citations: + self.action_events[f"{citation.start}.{citation.end}"] = ActionEvent( + action_type="citation", + init_timestamp=get_ISO_time(), + params=citation.text) elif isinstance(chunk, StreamedChatResponse_SearchQueriesGeneration): + for query in chunk.search_queries: + self.action_events[query.generation_id] = ActionEvent( + action_type="search_query", + init_timestamp=get_ISO_time(), + params=query.text) + elif isinstance(chunk, StreamedChatResponse_SearchResults): pass except Exception as e: @@ -498,7 +538,7 @@ def override_api(self): self._override_method(api, method_path, module) if api == 'cohere': - # Patch cohere vx.x.x+ methods + # Patch cohere v5.4.0+ methods module_version = version(api) if module_version is None: logger.warning(f'🖇 AgentOps: Cannot determine Cohere version. Only Cohere>=5.4.0 supported.') diff --git a/examples/cohere.py b/examples/cohere.py deleted file mode 100644 index 8e1ef3aa2..000000000 --- a/examples/cohere.py +++ /dev/null @@ -1,30 +0,0 @@ -import cohere -import agentops -from dotenv import load_dotenv -load_dotenv() - -agentops.init() -co = cohere.Client() - -chat = co.chat( - chat_history=[ - {"role": "USER", "message": "Who discovered gravity?"}, - { - "role": "CHATBOT", - "message": "The man who is widely credited with discovering gravity is Sir Isaac Newton", - }, - ], - message="What year was he born?", -) - -print(chat) - -stream = co.chat_stream( - message="Tell me a short story" -) - -for event in stream: - if event.event_type == "text-generation": - print(event.text, end='') - -agentops.end_session('Success') diff --git a/examples/cohere_example.py b/examples/cohere_example.py new file mode 100644 index 000000000..d7a7013ca --- /dev/null +++ b/examples/cohere_example.py @@ -0,0 +1,46 @@ +import cohere +import agentops # just +from dotenv import load_dotenv +load_dotenv() + +agentops.init(tags=["cohere", "agentops-demo"]) # three +co = cohere.Client() + +stream = co.chat_stream( + message="Tell me everything you can about AgentOps", + connectors=[{"id": "web-search"}] +) + +response = "" +for event in stream: + if event.event_type == "text-generation": + response += event.text + print(event.text, end='') + elif event.event_type == "stream-end": + print("\n") + print(event) + print("\n") + +stream = co.chat_stream( + chat_history=[ + {"role": "SYSTEM", "message": "You are Adam Silverman: die-hard advocate of AgentOps, leader in AI Agent observability"}, + { + "role": "CHATBOT", + "message": "How's your day going? I'd like to tell you about AgentOps: {response}", + }, + ], + message="Based on your newfound knowledge of AgentOps, is Cohere a suitable partner for them and how could they integrate?", + connectors=[{"id": "web-search"}] +) + +response = "" +for event in stream: + if event.event_type == "text-generation": + response += event.text + print(event.text, end='') + elif event.event_type == "stream-end": + print("\n") + print(event) + print("\n") + +agentops.end_session('Success') # lines From 27504f8377ebeda6bb2274a3baa1e23c33cf052e Mon Sep 17 00:00:00 2001 From: Alex Reibman Date: Thu, 16 May 2024 17:02:17 -0700 Subject: [PATCH 5/6] Readme: Collapsable sections (#211) --- README.md | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 8884092fb..33db385b0 100644 --- a/README.md +++ b/README.md @@ -36,9 +36,6 @@ Discord community channel - - - git commit activity @@ -122,6 +119,9 @@ pip install git+https://github.com/AgentOps-AI/crewAI.git@main AgentOps works seamlessly with applications built using Langchain. To use the handler, install Langchain as an optional dependency: +
+ Installation + ```shell pip install agentops[langchain] ``` @@ -151,13 +151,18 @@ agent = initialize_agent(tools, Check out the [Langchain Examples Notebook](./examples/langchain_examples.ipynb) for more details including Async handlers. -### Cohere +
+ +### Cohere ⌨️ First class support for Cohere(>=5.4.0). This is a living integration, should you need any added functionality please message us on Discord! - [AgentOps integration example](https://docs.agentops.ai/v1/integrations/cohere) - [Official Cohere documentation](https://docs.cohere.com/reference/about) +
+ Installation + ```bash pip install cohere ``` @@ -198,6 +203,8 @@ for event in stream: agentops.end_session('Success') ``` +
+ ### LlamaIndex 🦙 From 7fd7d106ca0e763d4d508228eda21c66e5c23413 Mon Sep 17 00:00:00 2001 From: Howard Gil Date: Thu, 16 May 2024 17:29:58 -0700 Subject: [PATCH 6/6] Adding session_id to developer_errors (#210) --- agentops/meta_client.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/agentops/meta_client.py b/agentops/meta_client.py index b95973fa0..68b840608 100644 --- a/agentops/meta_client.py +++ b/agentops/meta_client.py @@ -19,7 +19,7 @@ def __new__(cls, name, bases, dct): return super().__new__(cls, name, bases, dct) - def send_exception_to_server(cls, exception, api_key): + def send_exception_to_server(cls, exception, api_key, session): """Class method to send exception to server.""" if api_key: exception_type = type(exception).__name__ @@ -33,6 +33,9 @@ def send_exception_to_server(cls, exception, api_key): "host_env": get_host_env() } + if session: + developer_error["session_id"] = session.session_id + HttpClient.post("https://api.agentops.ai/developer_errors", safe_serialize(developer_error).encode("utf-8"), api_key=api_key) @@ -48,7 +51,7 @@ def wrapper(self, *args, **kwargs): logger.warning(f"🖇 AgentOps: Error: {e}") config = getattr(self, 'config', None) if config is not None: - type(self).send_exception_to_server(e, self.config._api_key) + type(self).send_exception_to_server(e, self.config._api_key, self._session) raise e return wrapper