Add AutoGen logger for generating report and integrating with OCI mon…

…itoring. (#1031)

ads/llm/autogen/__init__.py

@@ -0,0 +1,2 @@
+# Copyright (c) 2024, Oracle and/or its affiliates.  All rights reserved.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/

ads/llm/autogen/constants.py

@@ -0,0 +1,15 @@
+# Copyright (c) 2024, Oracle and/or its affiliates.  All rights reserved.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+
+class Events:
+    KEY = "event_name"
+
+    EXCEPTION = "exception"
+    LLM_CALL = "llm_call"
+    TOOL_CALL = "tool_call"
+    NEW_AGENT = "new_agent"
+    NEW_CLIENT = "new_client"
+    RECEIVED_MESSAGE = "received_message"
+    SESSION_START = "logging_session_start"
+    SESSION_STOP = "logging_session_stop"

ads/llm/autogen/reports/__init__.py

@@ -0,0 +1,2 @@
+# Copyright (c) 2024, Oracle and/or its affiliates.  All rights reserved.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/

ads/llm/autogen/reports/base.py

@@ -0,0 +1,67 @@
+# Copyright (c) 2024, Oracle and/or its affiliates.  All rights reserved.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+import json
+import logging
+import os
+
+from jinja2 import Environment, FileSystemLoader
+
+logger = logging.getLogger(__name__)
+
+
+class BaseReport:
+    """Base class containing utilities for generating reports."""
+
+    @staticmethod
+    def format_json_string(s) -> str:
+        """Formats the JSON string in markdown."""
+        return f"```json\n{json.dumps(json.loads(s), indent=2)}\n```"
+
+    @staticmethod
+    def _parse_date_time(datetime_string: str):
+        """Parses a datetime string in the logs into date and time.
+        Keeps only the seconds in the time.
+        """
+        date_str, time_str = datetime_string.split(" ", 1)
+        time_str = time_str.split(".", 1)[0]
+        return date_str, time_str
+
+    @staticmethod
+    def _preview_message(message: str, max_length=30) -> str:
+        """Shows the beginning part of a string message."""
+        # Return the entire string if it is less than the max_length
+        if len(message) <= max_length:
+            return message
+        # Go backward until we find the first whitespace
+        idx = 30
+        while not message[idx].isspace() and idx > 0:
+            idx -= 1
+        # If we found a whitespace
+        if idx > 0:
+            return message[:idx] + "..."
+        # If we didn't find a whitespace
+        return message[:30] + "..."
+
+    @classmethod
+    def _render_template(cls, template_path, **kwargs) -> str:
+        """Render Jinja template with kwargs."""
+        template_dir = os.path.join(os.path.dirname(__file__), "templates")
+        environment = Environment(
+            loader=FileSystemLoader(template_dir), autoescape=True
+        )
+        template = environment.get_template(template_path)
+        try:
+            html = template.render(**kwargs)
+        except Exception:
+            logger.error(
+                "Unable to render template %s with data:\n%s",
+                template_path,
+                str(kwargs),
+            )
+            return cls._render_template(
+                template_path=template_path,
+                sender=kwargs.get("sender", "N/A"),
+                content="TEMPLATE RENDER ERROR",
+                timestamp=kwargs.get("timestamp", ""),
+            )
+        return html

ads/llm/autogen/reports/data.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python
+# Copyright (c) 2024 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+"""Contains the data structure for logging and reporting."""
+import copy
+import json
+from dataclasses import asdict, dataclass, field
+from typing import Optional, Union
+
+from ads.llm.autogen.constants import Events
+
+
+@dataclass
+class LogData:
+    """Base class for the data field of LogRecord."""
+
+    def to_dict(self):
+        """Convert the log data to dictionary."""
+        return asdict(self)
+
+
+@dataclass
+class LogRecord:
+    """Represents a log record.
+
+    The `data` field is for pre-defined structured data, which should be an instance of LogData.
+    The `kwargs` field is for freeform key value pairs.
+    """
+
+    session_id: str
+    thread_id: int
+    timestamp: str
+    event_name: str
+    source_id: Optional[int] = None
+    source_name: Optional[str] = None
+    # Structured data for specific type of logs
+    data: Optional[LogData] = None
+    # Freeform data
+    kwargs: dict = field(default_factory=dict)
+
+    def to_dict(self):
+        """Convert the log record to dictionary."""
+        return asdict(self)
+
+    def to_string(self):
+        """Serialize the log record to JSON string."""
+        return json.dumps(self.to_dict(), default=str)
+
+    @classmethod
+    def from_dict(cls, data: dict) -> "LogRecord":
+        """Initializes a LogRecord object from dictionary."""
+        event_mapping = {
+            Events.NEW_AGENT: AgentData,
+            Events.TOOL_CALL: ToolCallData,
+            Events.LLM_CALL: LLMCompletionData,
+        }
+        if Events.KEY not in data:
+            raise KeyError("event_name not found in data.")
+
+        data = copy.deepcopy(data)
+
+        event_name = data["event_name"]
+        if event_name in event_mapping and data.get("data"):
+            data["data"] = event_mapping[event_name](**data.pop("data"))
+
+        return cls(**data)
+
+
+@dataclass
+class AgentData(LogData):
+    """Represents agent log Data."""
+
+    agent_name: str
+    agent_class: str
+    agent_module: Optional[str] = None
+    is_manager: Optional[bool] = None
+
+
+@dataclass
+class LLMCompletionData(LogData):
+    """Represents LLM completion log data."""
+
+    invocation_id: str
+    request: dict
+    response: dict
+    start_time: str
+    end_time: str
+    cost: Optional[float] = None
+    is_cached: Optional[bool] = None
+
+
+@dataclass
+class ToolCallData(LogData):
+    """Represents tool call log data."""
+
+    tool_name: str
+    start_time: str
+    end_time: str
+    agent_name: str
+    agent_class: str
+    agent_module: Optional[str] = None
+    input_args: dict = field(default_factory=dict)
+    returns: Optional[Union[str, list, dict, tuple]] = None