From 09725aa26357eb7430e70befa271cb32b9b137bb Mon Sep 17 00:00:00 2001
From: "panxuchen.pxc" <panxuchen.pxc@alibaba-inc.com>
Date: Wed, 17 Jan 2024 17:29:58 +0800
Subject: [PATCH 01/25] add sqlite monitor and unittest

---
 src/agentscope/constants.py     |   2 +
 src/agentscope/utils/monitor.py | 255 +++++++++++++++++++++++++++++++-
 tests/monitor_test.py           |  18 ++-
 3 files changed, 269 insertions(+), 6 deletions(-)

diff --git a/src/agentscope/constants.py b/src/agentscope/constants.py
index 997d2998f..54f52688f 100644
--- a/src/agentscope/constants.py
+++ b/src/agentscope/constants.py
@@ -23,6 +23,8 @@
 # for execute python
 _DEFAULT_PYPI_MIRROR = "http://mirrors.aliyun.com/pypi/simple/"
 _DEFAULT_TRUSTED_HOST = "mirrors.aliyun.com"
+# for monitor
+_DEFAULT_MONITOR_TABLE_NAME = "monitor_metrics"
 # for summarization
 _DEFAULT_SUMMARIZATION_PROMPT = """
 TEXT: {}
diff --git a/src/agentscope/utils/monitor.py b/src/agentscope/utils/monitor.py
index 4acdf82c0..825584de1 100644
--- a/src/agentscope/utils/monitor.py
+++ b/src/agentscope/utils/monitor.py
@@ -3,11 +3,15 @@
 
 import re
 import copy
+import sqlite3
 from abc import ABC
 from abc import abstractmethod
-from typing import Optional, Any
+from contextlib import contextmanager
+from typing import Optional, Any, Generator
 from loguru import logger
 
+from agentscope.constants import _DEFAULT_MONITOR_TABLE_NAME
+
 
 class MonitorBase(ABC):
     r"""Base interface of Monitor"""
@@ -263,10 +267,9 @@ def register(
 
     @return_false_if_not_exists
     def add(self, metric_name: str, value: float) -> bool:
-        self.metrics[metric_name]["value"] += value
         if (
             self.metrics[metric_name]["quota"] is not None
-            and self.metrics[metric_name]["value"]
+            and self.metrics[metric_name]["value"] + value
             > self.metrics[metric_name]["quota"]
         ):
             logger.warning(f"Metric [{metric_name}] quota exceeded.")
@@ -274,6 +277,7 @@ def add(self, metric_name: str, value: float) -> bool:
                 metric_name=metric_name,
                 quota=self.metrics[metric_name]["quota"],
             )
+        self.metrics[metric_name]["value"] += value
         return True
 
     def exists(self, metric_name: str) -> bool:
@@ -327,6 +331,251 @@ def get_metrics(self, filter_regex: Optional[str] = None) -> dict:
             }
 
 
+@contextmanager
+def sqlite_transaction(db_path: str) -> Generator:
+    """Get a sqlite transaction cursor.
+
+    Args:
+        db_path (`str`): path to the sqlite db file
+
+    Yields:
+        `Generator`: a cursor with transaction
+    """
+    conn = sqlite3.connect(db_path)
+    cursor = conn.cursor()
+    try:
+        conn.execute("BEGIN")
+        yield cursor
+        conn.commit()
+    except Exception as e:
+        conn.rollback()
+        raise e
+    finally:
+        cursor.close()
+        conn.close()
+
+
+@contextmanager
+def sqlite_cursor(db_path: str) -> Generator:
+    """Get a sqlite cursor.
+
+    Args:
+        db_path (`str`): path to the sqlite db file
+
+    Yields:
+        `Generator`: a cursor
+    """
+    conn = sqlite3.connect(db_path)
+    cursor = conn.cursor()
+    try:
+        yield cursor
+    finally:
+        cursor.close()
+        conn.close()
+
+
+class SqliteMonitor(MonitorBase):
+    """A monitor based on sqlite"""
+
+    def __init__(
+        self,
+        db_path: str,
+        table_name: str = _DEFAULT_MONITOR_TABLE_NAME,
+        drop_exists: bool = False,
+    ) -> None:
+        super().__init__()
+        self.db_path = db_path
+        self.table_name = table_name
+        self._create_monitor_table(drop_exists)
+
+    def _create_monitor_table(self, drop_exists: bool = False) -> None:
+        with sqlite_transaction(self.db_path) as cursor:
+            if drop_exists:
+                cursor.execute(f"DROP TABLE IF EXISTS {self.table_name};")
+            cursor.execute(
+                f"""
+                CREATE TABLE IF NOT EXISTS {self.table_name} (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    name TEXT UNIQUE NOT NULL,
+                    value REAL NOT NULL,
+                    quota REAL,
+                    unit TEXT
+                );""",
+            )
+
+    def register(
+        self,
+        metric_name: str,
+        metric_unit: Optional[str] = None,
+        quota: Optional[float] = None,
+    ) -> bool:
+        with sqlite_transaction(self.db_path) as cursor:
+            if self._exists(cursor, metric_name):
+                return False
+            cursor.execute(
+                f"""
+                INSERT INTO {self.table_name} (name, value, quota, unit)
+                VALUES (?, ?, ?, ?)
+            """,
+                (metric_name, 0.0, quota, metric_unit),
+            )
+            logger.info(
+                f"Register metric [{metric_name}] to SqliteMonitor with unit "
+                f"[{metric_unit}] and quota [{quota}]",
+            )
+            return True
+
+    def add(self, metric_name: str, value: float) -> bool:
+        with sqlite_transaction(self.db_path) as cursor:
+            if not self._exists(cursor, metric_name):
+                return False
+            cursor.execute(
+                f"""
+                UPDATE {self.table_name}
+                SET value = value + ?
+                WHERE name = ?
+            """,
+                (value, metric_name),
+            )
+            cursor.execute(
+                f"""
+                SELECT value, quota FROM {self.table_name}
+                WHERE name = ?""",
+                (metric_name,),
+            )
+            row = cursor.fetchone()
+            if row:
+                new_value, quota = row
+                if quota is not None and new_value > quota:
+                    logger.warning(f"Metric [{metric_name}] quota exceeded.")
+                    raise QuotaExceededError(
+                        metric_name=metric_name,
+                        quota=quota,
+                    )
+            return True
+
+    def clear(self, metric_name: str) -> bool:
+        with sqlite_transaction(self.db_path) as cursor:
+            if not self._exists(cursor, metric_name):
+                return False
+            cursor.execute(
+                f"""
+                UPDATE {self.table_name}
+                SET value = value + ?
+                WHERE name = ?
+            """,
+                (0.0, metric_name),
+            )
+            return True
+
+    def remove(self, metric_name: str) -> bool:
+        with sqlite_transaction(self.db_path) as cursor:
+            if not self._exists(cursor, metric_name):
+                return False
+            cursor.execute(
+                f"""
+                DELETE FROM {self.table_name}
+                WHERE name = ?""",
+                (metric_name,),
+            )
+        return True
+
+    def _get_metric(self, cursor: sqlite3.Cursor, metric_name: str) -> dict:
+        cursor.execute(
+            f"""
+            SELECT value, quota, unit FROM {self.table_name}
+            WHERE name = ?""",
+            (metric_name,),
+        )
+        row = cursor.fetchone()
+        if row:
+            value, quota, unit = row
+            return {
+                "value": value,
+                "quota": quota,
+                "unit": unit,
+            }
+        else:
+            raise RuntimeError(f"Fail to get metric {metric_name}")
+
+    def get_value(self, metric_name: str) -> Optional[float]:
+        with sqlite_cursor(self.db_path) as cursor:
+            if not self._exists(cursor, metric_name):
+                return None
+            metric = self._get_metric(cursor, metric_name)
+            return metric["value"]
+
+    def get_quota(self, metric_name: str) -> Optional[float]:
+        with sqlite_cursor(self.db_path) as cursor:
+            if not self._exists(cursor, metric_name):
+                return None
+            metric = self._get_metric(cursor, metric_name)
+            return metric["quota"]
+
+    def set_quota(self, metric_name: str, quota: float) -> bool:
+        with sqlite_transaction(self.db_path) as cursor:
+            if not self._exists(cursor, metric_name):
+                return False
+            cursor.execute(
+                f"""
+                UPDATE {self.table_name}
+                SET quota = ?
+                WHERE name = ?
+            """,
+                (quota, metric_name),
+            )
+            return True
+
+    def get_unit(self, metric_name: str) -> Optional[str]:
+        with sqlite_cursor(self.db_path) as cursor:
+            if not self._exists(cursor, metric_name):
+                return None
+            metric = self._get_metric(cursor, metric_name)
+            return metric["unit"]
+
+    def get_metric(self, metric_name: str) -> Optional[dict]:
+        with sqlite_cursor(self.db_path) as cursor:
+            if not self._exists(cursor, metric_name):
+                return None
+            return self._get_metric(cursor, metric_name)
+
+    def get_metrics(self, filter_regex: Optional[str] = None) -> dict:
+        with sqlite_cursor(self.db_path) as cursor:
+            cursor.execute(f"SELECT * FROM {self.table_name}")
+            rows = cursor.fetchall()
+            metrics = {
+                row[1]: {
+                    "value": row[2],
+                    "quota": row[3],
+                    "unit": row[4],
+                }
+                for row in rows
+            }
+        if filter_regex is None:
+            return metrics
+        else:
+            pattern = re.compile(filter_regex)
+            return {
+                key: value
+                for key, value in metrics.items()
+                if pattern.search(key)
+            }
+
+    def _exists(self, cursor: sqlite3.Cursor, name: str) -> bool:
+        cursor.execute(
+            f"""
+            SELECT 1 FROM {self.table_name}
+            WHERE name = ? LIMIT 1
+        """,
+            (name,),
+        )
+        return cursor.fetchone() is not None
+
+    def exists(self, metric_name: str) -> bool:
+        with sqlite_cursor(self.db_path) as cursor:
+            return self._exists(cursor, metric_name)
+
+
 class MonitorFactory:
     """Factory of Monitor.
 
diff --git a/tests/monitor_test.py b/tests/monitor_test.py
index ffbae1e81..d40340b20 100644
--- a/tests/monitor_test.py
+++ b/tests/monitor_test.py
@@ -4,10 +4,11 @@
 """
 
 import unittest
-
+import uuid
+import os
 from agentscope.utils import MonitorBase, QuotaExceededError, MonitorFactory
 
-from agentscope.utils.monitor import DictMonitor
+from agentscope.utils.monitor import DictMonitor, SqliteMonitor
 
 
 class MonitorFactoryTest(unittest.TestCase):
@@ -91,7 +92,7 @@ def test_add_clear_set_quota(self) -> None:
         self.assertTrue(self.monitor.set_quota("token_num", 200))
         # add success and check new value
         self.assertTrue(self.monitor.add("token_num", 10))
-        self.assertEqual(self.monitor.get_value("token_num"), 111)
+        self.assertEqual(self.monitor.get_value("token_num"), 20)
         # clear an existing metric
         self.assertTrue(self.monitor.clear("token_num"))
         # clear an not existing metric
@@ -166,3 +167,14 @@ class DictMonitorTest(MonitorTestBase):
 
     def get_monitor_instance(self) -> MonitorBase:
         return DictMonitor()
+
+
+class SqliteMonitorTest(MonitorTestBase):
+    """Test class for SqliteMonitor"""
+
+    def get_monitor_instance(self) -> MonitorBase:
+        self.db_path = f"./test-{uuid.uuid4()}.db"
+        return SqliteMonitor(self.db_path)
+
+    def tearDown(self) -> None:
+        os.remove(self.db_path)

From 0ad78565de940b80fa3f4ecb4f396511a9eaa9d7 Mon Sep 17 00:00:00 2001
From: "panxuchen.pxc" <panxuchen.pxc@alibaba-inc.com>
Date: Thu, 18 Jan 2024 11:47:28 +0800
Subject: [PATCH 02/25] add monitor

---
 src/agentscope/utils/monitor.py | 97 ++++++++++++++++++++++++++-------
 1 file changed, 76 insertions(+), 21 deletions(-)

diff --git a/src/agentscope/utils/monitor.py b/src/agentscope/utils/monitor.py
index 825584de1..25c7011ec 100644
--- a/src/agentscope/utils/monitor.py
+++ b/src/agentscope/utils/monitor.py
@@ -186,6 +186,16 @@ def get_metrics(self, filter_regex: Optional[str] = None) -> dict:
                 }
         """
 
+    @abstractmethod
+    def set_budget(self, model_name: str, value: float) -> None:
+        """Set budget to the monitor, the monitor will raise
+        QuotaExceededError, when budget is exceeded
+
+        Args:
+            model_name (`str`): model that requires budget
+            value (`float`): the budget value
+        """
+
 
 class QuotaExceededError(Exception):
     """An Exception used to indicate that a certain metric exceeds quota"""
@@ -330,6 +340,9 @@ def get_metrics(self, filter_regex: Optional[str] = None) -> dict:
                 if pattern.search(key)
             }
 
+    def set_budget(self, model_name: str, value: float) -> None:
+        logger.warning("DictMonitor doesn't support set_budget")
+
 
 @contextmanager
 def sqlite_transaction(db_path: str) -> Generator:
@@ -425,33 +438,41 @@ def register(
             )
             return True
 
-    def add(self, metric_name: str, value: float) -> bool:
-        with sqlite_transaction(self.db_path) as cursor:
-            if not self._exists(cursor, metric_name):
-                return False
-            cursor.execute(
-                f"""
+    def _add(
+        self,
+        cursor: sqlite3.Cursor,
+        metric_name: str,
+        value: float,
+    ) -> None:
+        cursor.execute(
+            f"""
                 UPDATE {self.table_name}
                 SET value = value + ?
                 WHERE name = ?
             """,
-                (value, metric_name),
-            )
-            cursor.execute(
-                f"""
+            (value, metric_name),
+        )
+        cursor.execute(
+            f"""
                 SELECT value, quota FROM {self.table_name}
                 WHERE name = ?""",
-                (metric_name,),
-            )
-            row = cursor.fetchone()
-            if row:
-                new_value, quota = row
-                if quota is not None and new_value > quota:
-                    logger.warning(f"Metric [{metric_name}] quota exceeded.")
-                    raise QuotaExceededError(
-                        metric_name=metric_name,
-                        quota=quota,
-                    )
+            (metric_name,),
+        )
+        row = cursor.fetchone()
+        if row:
+            new_value, quota = row
+            if quota is not None and new_value > quota:
+                logger.warning(f"Metric [{metric_name}] quota exceeded.")
+                raise QuotaExceededError(
+                    metric_name=metric_name,
+                    quota=quota,
+                )
+
+    def add(self, metric_name: str, value: float) -> bool:
+        with sqlite_transaction(self.db_path) as cursor:
+            if not self._exists(cursor, metric_name):
+                return False
+            self._add(cursor, metric_name, value)
             return True
 
     def clear(self, metric_name: str) -> bool:
@@ -575,6 +596,40 @@ def exists(self, metric_name: str) -> bool:
         with sqlite_cursor(self.db_path) as cursor:
             return self._exists(cursor, metric_name)
 
+    def update(self, **kwargs: Any) -> None:
+        with sqlite_transaction(self.db_path) as cursor:
+            for metric_name, value in kwargs.items():
+                self._add(cursor, metric_name, value)
+
+    def set_budget(self, model_name: str, value: float) -> None:
+        logger.info(f"set budget {value} to {model_name}")
+
+
+def get_pricing() -> dict:
+    """Get pricing as a dict
+
+    Returns:
+        `dict`: the dict with pricing information.
+    """
+    return {
+        'gpt-4-turbo': {
+            'input': 0.01,
+            'output': 0.03
+        },
+        'gpt-4': {
+            'input': 0.03,
+            'output': 0.06
+        },
+        'gpt-4-32k': {
+            'input': 0.06,
+            'output': 0.12
+        },
+        'gpt-3.5-turbo': {
+            'input': 0.001,
+            'output': 0.002
+        }
+    }
+
 
 class MonitorFactory:
     """Factory of Monitor.

From 301b2b77f5d71ee327363730ad00a10c1b06a148 Mon Sep 17 00:00:00 2001
From: "panxuchen.pxc" <panxuchen.pxc@alibaba-inc.com>
Date: Thu, 18 Jan 2024 16:17:36 +0800
Subject: [PATCH 03/25] use trigger to detect quota execced

---
 src/agentscope/utils/monitor.py | 156 +++++++++++++++++++++++---------
 1 file changed, 113 insertions(+), 43 deletions(-)

diff --git a/src/agentscope/utils/monitor.py b/src/agentscope/utils/monitor.py
index 25c7011ec..7befdd13e 100644
--- a/src/agentscope/utils/monitor.py
+++ b/src/agentscope/utils/monitor.py
@@ -187,22 +187,39 @@ def get_metrics(self, filter_regex: Optional[str] = None) -> dict:
         """
 
     @abstractmethod
-    def set_budget(self, model_name: str, value: float) -> None:
-        """Set budget to the monitor, the monitor will raise
-        QuotaExceededError, when budget is exceeded
+    def register_budget(
+        self,
+        model_name: str,
+        value: float,
+        prefix: Optional[str] = 'local'
+    ) -> bool:
+        """Register model call budget to the monitor, the monitor will raise
+        QuotaExceededError, when budget is exceeded.
 
         Args:
-            model_name (`str`): model that requires budget
-            value (`float`): the budget value
+            model_name (`str`): model that requires budget.
+            value (`float`): the budget value.
+            prefix (`Optional[str]`, default `None`): used to distinguish
+            multiple budget registrations for the same model. For multiple
+            registrations with the same `model_name` and `prefix`, only the
+            first time will take effect.
+
+        Returns:
+            `bool`: whether the operation success.
         """
 
 
 class QuotaExceededError(Exception):
     """An Exception used to indicate that a certain metric exceeds quota"""
 
-    def __init__(self, metric_name: str, quota: float) -> None:
-        self.message = f"Metric [{metric_name}] exceed quota [{quota}]"
-        super().__init__(self.message)
+    def __init__(self,
+                 metric_name: Optional[str] = None,
+                 quota: Optional[float] = None) -> None:
+        if metric_name is not None and quota is not None:
+            self.message = f"Metric [{metric_name}] exceeds quota [{quota}]"
+            super().__init__(self.message)
+        else:
+            super().__init__()
 
 
 def return_false_if_not_exists(  # type: ignore [no-untyped-def]
@@ -340,8 +357,12 @@ def get_metrics(self, filter_regex: Optional[str] = None) -> dict:
                 if pattern.search(key)
             }
 
-    def set_budget(self, model_name: str, value: float) -> None:
-        logger.warning("DictMonitor doesn't support set_budget")
+    def register_budget(self,
+                        model_name: str,
+                        value: float,
+                        prefix: Optional[str] = 'local') -> bool:
+        logger.warning("DictMonitor doesn't support register_budget")
+        return False
 
 
 @contextmanager
@@ -402,6 +423,7 @@ def __init__(
         self._create_monitor_table(drop_exists)
 
     def _create_monitor_table(self, drop_exists: bool = False) -> None:
+        """Internal method to create a table in sqlite3."""
         with sqlite_transaction(self.db_path) as cursor:
             if drop_exists:
                 cursor.execute(f"DROP TABLE IF EXISTS {self.table_name};")
@@ -415,6 +437,21 @@ def _create_monitor_table(self, drop_exists: bool = False) -> None:
                     unit TEXT
                 );""",
             )
+            cursor.execute(
+                f"""
+                CREATE TRIGGER IF NOT EXISTS {self.table_name}_quota_exceeded
+                BEFORE UPDATE ON {self.table_name}
+                FOR EACH ROW
+                WHEN OLD.quota is not NULL AND NEW.value > OLD.quota
+                BEGIN
+                    SELECT RAISE(FAIL, 'QuotaExceeded');
+                END;
+                """
+            )
+
+    def _get_trigger_name(self, metric_name: str) -> str:
+        """Get the name of the trigger on a certain metric"""
+        return f'{self.table_name}.{metric_name}.trigger'
 
     def register(
         self,
@@ -429,7 +466,7 @@ def register(
                 f"""
                 INSERT INTO {self.table_name} (name, value, quota, unit)
                 VALUES (?, ?, ?, ?)
-            """,
+                """,
                 (metric_name, 0.0, quota, metric_unit),
             )
             logger.info(
@@ -444,29 +481,17 @@ def _add(
         metric_name: str,
         value: float,
     ) -> None:
-        cursor.execute(
-            f"""
-                UPDATE {self.table_name}
-                SET value = value + ?
-                WHERE name = ?
-            """,
-            (value, metric_name),
-        )
-        cursor.execute(
-            f"""
-                SELECT value, quota FROM {self.table_name}
-                WHERE name = ?""",
-            (metric_name,),
-        )
-        row = cursor.fetchone()
-        if row:
-            new_value, quota = row
-            if quota is not None and new_value > quota:
-                logger.warning(f"Metric [{metric_name}] quota exceeded.")
-                raise QuotaExceededError(
-                    metric_name=metric_name,
-                    quota=quota,
-                )
+        try:
+            cursor.execute(
+                f"""
+                    UPDATE {self.table_name}
+                    SET value = value + ?
+                    WHERE name = ?
+                """,
+                (value, metric_name),
+            )
+        except sqlite3.IntegrityError as e:
+            raise QuotaExceededError() from e
 
     def add(self, metric_name: str, value: float) -> bool:
         with sqlite_transaction(self.db_path) as cursor:
@@ -601,8 +626,53 @@ def update(self, **kwargs: Any) -> None:
             for metric_name, value in kwargs.items():
                 self._add(cursor, metric_name, value)
 
-    def set_budget(self, model_name: str, value: float) -> None:
+    def _create_update_cost_trigger(
+        self,
+        token_metric: str,
+        cost_metric: str,
+        unit_price: float
+    ) -> bool:
+        with sqlite_transaction(self.db_path) as cursor:
+            cursor.execute(
+                f"""
+                CREATE TRIGGER IF NOT EXISTS
+                "{self.table_name}_{token_metric}_{cost_metric}_price"
+                AFTER UPDATE OF value ON "{self.table_name}"
+                FOR EACH ROW
+                WHEN NEW.name = '{token_metric}'
+                BEGIN
+                    UPDATE {self.table_name}
+                    SET value = value + (NEW.value - OLD.value) * {unit_price}
+                    WHERE name = '{cost_metric}';
+                END;
+                """
+            )
+
+    def register_budget(
+        self,
+        model_name: str,
+        value: float,
+        prefix: Optional[str] = None
+    ) -> bool:
         logger.info(f"set budget {value} to {model_name}")
+        pricing = get_pricing()
+        if model_name in pricing:
+            budget_metric_name = f'{prefix}.{model_name}.cost'
+            self.register(
+                metric_name=budget_metric_name,
+                metric_unit='dollor')
+            for metric_name, unit_price in pricing[model_name].items():
+                token_metric_name = f'{prefix}.{model_name}.{metric_name}'
+                self.register(
+                    metric_name=token_metric_name,
+                    metric_unit='token')
+                self._create_update_cost_trigger(
+                    token_metric_name, budget_metric_name, unit_price)
+            return True
+        else:
+            logger.warning(
+                f'Calculate budgets for model [{model_name}] is not supported')
+            return False
 
 
 def get_pricing() -> dict:
@@ -613,20 +683,20 @@ def get_pricing() -> dict:
     """
     return {
         'gpt-4-turbo': {
-            'input': 0.01,
-            'output': 0.03
+            'prompt_tokens': 0.01,
+            'completion_tokens': 0.03
         },
         'gpt-4': {
-            'input': 0.03,
-            'output': 0.06
+            'prompt_tokens': 0.03,
+            'completion_tokens': 0.06
         },
         'gpt-4-32k': {
-            'input': 0.06,
-            'output': 0.12
+            'prompt_tokens': 0.06,
+            'completion_tokens': 0.12
         },
         'gpt-3.5-turbo': {
-            'input': 0.001,
-            'output': 0.002
+            'prompt_tokens': 0.001,
+            'completion_tokens': 0.002
         }
     }
 

From 2b40bdbef17ea6141e7a3ab09e3647d8da0435c0 Mon Sep 17 00:00:00 2001
From: "panxuchen.pxc" <panxuchen.pxc@alibaba-inc.com>
Date: Thu, 18 Jan 2024 16:53:56 +0800
Subject: [PATCH 04/25] use trigger to calculate costs

---
 src/agentscope/utils/monitor.py | 27 +++++++++++++++------------
 tests/monitor_test.py           | 33 ++++++++++++++++++++++++++++++++-
 2 files changed, 47 insertions(+), 13 deletions(-)

diff --git a/src/agentscope/utils/monitor.py b/src/agentscope/utils/monitor.py
index 7befdd13e..dfd71c26e 100644
--- a/src/agentscope/utils/monitor.py
+++ b/src/agentscope/utils/monitor.py
@@ -639,11 +639,11 @@ def _create_update_cost_trigger(
                 "{self.table_name}_{token_metric}_{cost_metric}_price"
                 AFTER UPDATE OF value ON "{self.table_name}"
                 FOR EACH ROW
-                WHEN NEW.name = '{token_metric}'
+                WHEN NEW.name = "{token_metric}"
                 BEGIN
                     UPDATE {self.table_name}
                     SET value = value + (NEW.value - OLD.value) * {unit_price}
-                    WHERE name = '{cost_metric}';
+                    WHERE name = "{cost_metric}";
                 END;
                 """
             )
@@ -658,9 +658,12 @@ def register_budget(
         pricing = get_pricing()
         if model_name in pricing:
             budget_metric_name = f'{prefix}.{model_name}.cost'
-            self.register(
+            ok = self.register(
                 metric_name=budget_metric_name,
-                metric_unit='dollor')
+                metric_unit='dollor',
+                quota=value)
+            if not ok:
+                return False
             for metric_name, unit_price in pricing[model_name].items():
                 token_metric_name = f'{prefix}.{model_name}.{metric_name}'
                 self.register(
@@ -683,20 +686,20 @@ def get_pricing() -> dict:
     """
     return {
         'gpt-4-turbo': {
-            'prompt_tokens': 0.01,
-            'completion_tokens': 0.03
+            'prompt_tokens': 0.00001,
+            'completion_tokens': 0.00003
         },
         'gpt-4': {
-            'prompt_tokens': 0.03,
-            'completion_tokens': 0.06
+            'prompt_tokens': 0.00003,
+            'completion_tokens': 0.00006
         },
         'gpt-4-32k': {
-            'prompt_tokens': 0.06,
-            'completion_tokens': 0.12
+            'prompt_tokens': 0.00006,
+            'completion_tokens': 0.00012
         },
         'gpt-3.5-turbo': {
-            'prompt_tokens': 0.001,
-            'completion_tokens': 0.002
+            'prompt_tokens': 0.000001,
+            'completion_tokens': 0.000002
         }
     }
 
diff --git a/tests/monitor_test.py b/tests/monitor_test.py
index d40340b20..656257d6a 100644
--- a/tests/monitor_test.py
+++ b/tests/monitor_test.py
@@ -173,8 +173,39 @@ class SqliteMonitorTest(MonitorTestBase):
     """Test class for SqliteMonitor"""
 
     def get_monitor_instance(self) -> MonitorBase:
-        self.db_path = f"./test-{uuid.uuid4()}.db"
+        self.db_path = f"test-{uuid.uuid4()}.db"
         return SqliteMonitor(self.db_path)
 
     def tearDown(self) -> None:
         os.remove(self.db_path)
+
+    def test_register_budget(self) -> None:
+        """Test register_budget method of monitor"""
+        self.assertTrue(
+            self.monitor.register_budget(
+                model_name='gpt-4', value=5, prefix="agent_A")
+        )
+        # register an existing model with different prefix is ok
+        self.assertTrue(
+            self.monitor.register_budget(
+                model_name='gpt-4', value=15, prefix="agent_B")
+        )
+        gpt_4_3d = {
+            "agent_A.gpt-4.prompt_tokens": 50000,
+            "agent_A.gpt-4.completion_tokens": 25000,
+            "agent_A.gpt-4.total_tokens": 750000
+        }
+        # agentA uses 3 dollors
+        self.monitor.update(**gpt_4_3d)
+        # agentA uses another 3 dollors and exceeds quota
+        self.assertRaises(
+            QuotaExceededError,
+            self.monitor.update,
+            **gpt_4_3d
+        )
+        self.assertLess(self.monitor.get_value('agent_A.gpt-4.cost'), 5)
+        # register an existing model with existing prefix is wrong
+        self.assertFalse(
+            self.monitor.register_budget(
+                model_name='gpt-4', value=5, prefix="agent_A")
+        )

From 14cdebec65d082592a6bbde4473fa764e943649e Mon Sep 17 00:00:00 2001
From: "panxuchen.pxc" <panxuchen.pxc@alibaba-inc.com>
Date: Thu, 18 Jan 2024 16:59:55 +0800
Subject: [PATCH 05/25] fix unittest

---
 src/agentscope/utils/monitor.py | 78 +++++++++++++++++++--------------
 tests/monitor_test.py           | 26 ++++++++---
 2 files changed, 64 insertions(+), 40 deletions(-)

diff --git a/src/agentscope/utils/monitor.py b/src/agentscope/utils/monitor.py
index dfd71c26e..7781290f5 100644
--- a/src/agentscope/utils/monitor.py
+++ b/src/agentscope/utils/monitor.py
@@ -191,7 +191,7 @@ def register_budget(
         self,
         model_name: str,
         value: float,
-        prefix: Optional[str] = 'local'
+        prefix: Optional[str] = "local",
     ) -> bool:
         """Register model call budget to the monitor, the monitor will raise
         QuotaExceededError, when budget is exceeded.
@@ -212,9 +212,11 @@ def register_budget(
 class QuotaExceededError(Exception):
     """An Exception used to indicate that a certain metric exceeds quota"""
 
-    def __init__(self,
-                 metric_name: Optional[str] = None,
-                 quota: Optional[float] = None) -> None:
+    def __init__(
+        self,
+        metric_name: Optional[str] = None,
+        quota: Optional[float] = None,
+    ) -> None:
         if metric_name is not None and quota is not None:
             self.message = f"Metric [{metric_name}] exceeds quota [{quota}]"
             super().__init__(self.message)
@@ -357,10 +359,12 @@ def get_metrics(self, filter_regex: Optional[str] = None) -> dict:
                 if pattern.search(key)
             }
 
-    def register_budget(self,
-                        model_name: str,
-                        value: float,
-                        prefix: Optional[str] = 'local') -> bool:
+    def register_budget(
+        self,
+        model_name: str,
+        value: float,
+        prefix: Optional[str] = "local",
+    ) -> bool:
         logger.warning("DictMonitor doesn't support register_budget")
         return False
 
@@ -446,12 +450,12 @@ def _create_monitor_table(self, drop_exists: bool = False) -> None:
                 BEGIN
                     SELECT RAISE(FAIL, 'QuotaExceeded');
                 END;
-                """
+                """,
             )
 
     def _get_trigger_name(self, metric_name: str) -> str:
         """Get the name of the trigger on a certain metric"""
-        return f'{self.table_name}.{metric_name}.trigger'
+        return f"{self.table_name}.{metric_name}.trigger"
 
     def register(
         self,
@@ -630,8 +634,8 @@ def _create_update_cost_trigger(
         self,
         token_metric: str,
         cost_metric: str,
-        unit_price: float
-    ) -> bool:
+        unit_price: float,
+    ) -> None:
         with sqlite_transaction(self.db_path) as cursor:
             cursor.execute(
                 f"""
@@ -645,36 +649,42 @@ def _create_update_cost_trigger(
                     SET value = value + (NEW.value - OLD.value) * {unit_price}
                     WHERE name = "{cost_metric}";
                 END;
-                """
+                """,
             )
 
     def register_budget(
         self,
         model_name: str,
         value: float,
-        prefix: Optional[str] = None
+        prefix: Optional[str] = None,
     ) -> bool:
         logger.info(f"set budget {value} to {model_name}")
         pricing = get_pricing()
         if model_name in pricing:
-            budget_metric_name = f'{prefix}.{model_name}.cost'
+            budget_metric_name = f"{prefix}.{model_name}.cost"
             ok = self.register(
                 metric_name=budget_metric_name,
-                metric_unit='dollor',
-                quota=value)
+                metric_unit="dollor",
+                quota=value,
+            )
             if not ok:
                 return False
             for metric_name, unit_price in pricing[model_name].items():
-                token_metric_name = f'{prefix}.{model_name}.{metric_name}'
+                token_metric_name = f"{prefix}.{model_name}.{metric_name}"
                 self.register(
                     metric_name=token_metric_name,
-                    metric_unit='token')
+                    metric_unit="token",
+                )
                 self._create_update_cost_trigger(
-                    token_metric_name, budget_metric_name, unit_price)
+                    token_metric_name,
+                    budget_metric_name,
+                    unit_price,
+                )
             return True
         else:
             logger.warning(
-                f'Calculate budgets for model [{model_name}] is not supported')
+                f"Calculate budgets for model [{model_name}] is not supported",
+            )
             return False
 
 
@@ -685,22 +695,22 @@ def get_pricing() -> dict:
         `dict`: the dict with pricing information.
     """
     return {
-        'gpt-4-turbo': {
-            'prompt_tokens': 0.00001,
-            'completion_tokens': 0.00003
+        "gpt-4-turbo": {
+            "prompt_tokens": 0.00001,
+            "completion_tokens": 0.00003,
         },
-        'gpt-4': {
-            'prompt_tokens': 0.00003,
-            'completion_tokens': 0.00006
+        "gpt-4": {
+            "prompt_tokens": 0.00003,
+            "completion_tokens": 0.00006,
         },
-        'gpt-4-32k': {
-            'prompt_tokens': 0.00006,
-            'completion_tokens': 0.00012
+        "gpt-4-32k": {
+            "prompt_tokens": 0.00006,
+            "completion_tokens": 0.00012,
+        },
+        "gpt-3.5-turbo": {
+            "prompt_tokens": 0.000001,
+            "completion_tokens": 0.000002,
         },
-        'gpt-3.5-turbo': {
-            'prompt_tokens': 0.000001,
-            'completion_tokens': 0.000002
-        }
     }
 
 
diff --git a/tests/monitor_test.py b/tests/monitor_test.py
index 656257d6a..342d32e5c 100644
--- a/tests/monitor_test.py
+++ b/tests/monitor_test.py
@@ -183,17 +183,23 @@ def test_register_budget(self) -> None:
         """Test register_budget method of monitor"""
         self.assertTrue(
             self.monitor.register_budget(
-                model_name='gpt-4', value=5, prefix="agent_A")
+                model_name="gpt-4",
+                value=5,
+                prefix="agent_A",
+            ),
         )
         # register an existing model with different prefix is ok
         self.assertTrue(
             self.monitor.register_budget(
-                model_name='gpt-4', value=15, prefix="agent_B")
+                model_name="gpt-4",
+                value=15,
+                prefix="agent_B",
+            ),
         )
         gpt_4_3d = {
             "agent_A.gpt-4.prompt_tokens": 50000,
             "agent_A.gpt-4.completion_tokens": 25000,
-            "agent_A.gpt-4.total_tokens": 750000
+            "agent_A.gpt-4.total_tokens": 750000,
         }
         # agentA uses 3 dollors
         self.monitor.update(**gpt_4_3d)
@@ -201,11 +207,19 @@ def test_register_budget(self) -> None:
         self.assertRaises(
             QuotaExceededError,
             self.monitor.update,
-            **gpt_4_3d
+            **gpt_4_3d,
+        )
+        self.assertLess(
+            self.monitor.get_value(  # type: ignore [arg-type]
+                "agent_A.gpt-4.cost",
+            ),
+            5,
         )
-        self.assertLess(self.monitor.get_value('agent_A.gpt-4.cost'), 5)
         # register an existing model with existing prefix is wrong
         self.assertFalse(
             self.monitor.register_budget(
-                model_name='gpt-4', value=5, prefix="agent_A")
+                model_name="gpt-4",
+                value=5,
+                prefix="agent_A",
+            ),
         )

From f4ad4695ad2644f9a1b6a3a654405cac1496d3f2 Mon Sep 17 00:00:00 2001
From: "panxuchen.pxc" <panxuchen.pxc@alibaba-inc.com>
Date: Thu, 18 Jan 2024 19:13:35 +0800
Subject: [PATCH 06/25] init monitor in _init

---
 src/agentscope/_init.py                |  9 +++-
 src/agentscope/configs/model_config.py |  7 +++
 src/agentscope/constants.py            |  2 +
 src/agentscope/file_manager.py         | 10 ++++
 src/agentscope/models/openai_model.py  | 24 ++++++++--
 src/agentscope/utils/monitor.py        | 64 +++++++++++++++++++-------
 tests/monitor_test.py                  | 27 +++++++----
 7 files changed, 112 insertions(+), 31 deletions(-)

diff --git a/src/agentscope/_init.py b/src/agentscope/_init.py
index e96f523a6..7e2296563 100644
--- a/src/agentscope/_init.py
+++ b/src/agentscope/_init.py
@@ -10,10 +10,12 @@
 from ._runtime import Runtime
 from .file_manager import file_manager
 from .utils.logging_utils import LOG_LEVEL, setup_logger
+from .utils.monitor import MonitorFactory
 from .models import read_model_configs
+from .constants import _DEFAULT_DIR
+from .constants import _DEFAULT_LOG_LEVEL
+
 
-_DEFAULT_DIR = "./runs"
-_DEFAULT_LOG_LEVEL = "INFO"
 _INIT_SETTINGS = {}
 
 
@@ -85,6 +87,9 @@ def init(
     dir_log = str(file_manager.dir_log) if save_log else None
     setup_logger(dir_log, logger_level)
 
+    # Set monitor
+    _ = MonitorFactory.get_monitor(db_path=file_manager.file_db)
+
     # Load config and init agent by configs
     if agent_configs is not None:
         if isinstance(agent_configs, str):
diff --git a/src/agentscope/configs/model_config.py b/src/agentscope/configs/model_config.py
index 774be726a..452a4d2c7 100644
--- a/src/agentscope/configs/model_config.py
+++ b/src/agentscope/configs/model_config.py
@@ -3,6 +3,7 @@
 from typing import Any
 from ..constants import _DEFAULT_MAX_RETRIES
 from ..constants import _DEFAULT_MESSAGES_KEY
+from ..constants import _DEFAULT_API_BUDGET
 
 
 class CfgBase(dict):
@@ -57,6 +58,9 @@ class OpenAICfg(CfgBase):
     """The arguments used in openai api generation, e.g. `temperature`,
     `seed`."""
 
+    budget: float = _DEFAULT_API_BUDGET
+    """The total budget using this model. Set to `None` means no limit."""
+
 
 class PostApiCfg(CfgBase):
     """The config for Post API. The final request post will be
@@ -113,3 +117,6 @@ class PostApiCfg(CfgBase):
     """The key of the prompt messages in `requests.post()`,
     e.g. `request.post(json={${messages_key}: messages, **json_args})`. For
     huggingface and modelscope inference API, the key is `inputs`"""
+
+    budget: float = _DEFAULT_API_BUDGET
+    """The total budget using this model. Set to `None` means no limit."""
diff --git a/src/agentscope/constants.py b/src/agentscope/constants.py
index 54f52688f..938c12290 100644
--- a/src/agentscope/constants.py
+++ b/src/agentscope/constants.py
@@ -16,10 +16,12 @@
 _DEFAULT_SUBDIR_FILE = "file"
 _DEFAULT_SUBDIR_INVOKE = "invoke"
 _DEFAULT_IMAGE_NAME = "image_{}_{}.png"
+_DEFAULT_SQLITE_DB_FILE = "agentscope.db"
 # for model wrapper
 _DEFAULT_MAX_RETRIES = 3
 _DEFAULT_MESSAGES_KEY = "inputs"
 _DEFAULT_RETRY_INTERVAL = 1
+_DEFAULT_API_BUDGET = None
 # for execute python
 _DEFAULT_PYPI_MIRROR = "http://mirrors.aliyun.com/pypi/simple/"
 _DEFAULT_TRUSTED_HOST = "mirrors.aliyun.com"
diff --git a/src/agentscope/file_manager.py b/src/agentscope/file_manager.py
index db73cda40..d53f39fbf 100644
--- a/src/agentscope/file_manager.py
+++ b/src/agentscope/file_manager.py
@@ -14,6 +14,7 @@
     _DEFAULT_SUBDIR_CODE,
     _DEFAULT_SUBDIR_FILE,
     _DEFAULT_SUBDIR_INVOKE,
+    _DEFAULT_SQLITE_DB_FILE,
     _DEFAULT_IMAGE_NAME,
 )
 
@@ -48,6 +49,10 @@ def _get_and_create_subdir(self, subdir: str) -> str:
             os.makedirs(path)
         return path
 
+    def _get_file_path(self, file_name: str) -> str:
+        """Get the path of the file."""
+        return os.path.join(self.dir, Runtime.runtime_id, file_name)
+
     @property
     def dir_log(self) -> str:
         """The directory for saving logs."""
@@ -69,6 +74,11 @@ def dir_invoke(self) -> str:
         """The directory for saving api invocations."""
         return self._get_and_create_subdir(_DEFAULT_SUBDIR_INVOKE)
 
+    @property
+    def file_db(self) -> str:
+        """The path to the sqlite db file."""
+        return self._get_file_path(_DEFAULT_SQLITE_DB_FILE)
+
     def init(self, save_dir: str, save_api_invoke: bool = False) -> None:
         """Set the directory for saving files."""
         self.dir = save_dir
diff --git a/src/agentscope/models/openai_model.py b/src/agentscope/models/openai_model.py
index abe1f6bd8..3eed6c846 100644
--- a/src/agentscope/models/openai_model.py
+++ b/src/agentscope/models/openai_model.py
@@ -12,7 +12,8 @@
 except ImportError:
     openai = None
 
-from ..utils import MonitorFactory
+from ..utils.monitor import MonitorFactory
+from ..utils.monitor import full_name
 from ..utils import QuotaExceededError
 from ..utils.token_utils import get_openai_max_length
 
@@ -28,6 +29,7 @@ def __init__(
         organization: str = None,
         client_args: dict = None,
         generate_args: dict = None,
+        budget: float = None,
     ) -> None:
         """Initialize the openai client.
 
@@ -49,6 +51,9 @@ def __init__(
             generate_args (`dict`, default `None`):
                 The extra keyword arguments used in openai api generation,
                 e.g. `temperature`, `seed`.
+            budget (`float`, default `None`):
+                The total budget using this model. Set to `None` means no
+                limit.
         """
         super().__init__(name)
 
@@ -77,8 +82,18 @@ def __init__(
 
         # Set monitor accordingly
         self.monitor = None
+        self.budget = budget
+        self._register_budget()
         self._register_default_metrics()
 
+    def _register_budget(self) -> None:
+        self.monitor = MonitorFactory.get_monitor()
+        self.monitor.register_budget(
+            model_name=self.model_name,
+            value=self.budget,
+            prefix=self.model_name,
+        )
+
     def _register_default_metrics(self) -> None:
         """Register metrics to the monitor."""
         raise NotImplementedError(
@@ -95,7 +110,7 @@ def _metric(self, metric_name: str) -> str:
         Returns:
             `str`: Metric name of this wrapper.
         """
-        return f"{self.__class__.__name__}.{self.model_name}.{metric_name}"
+        return full_name(name=metric_name, prefix=self.model_name)
 
 
 class OpenAIChatWrapper(OpenAIWrapper):
@@ -193,7 +208,10 @@ def __call__(
 
         # step5: update monitor accordingly
         try:
-            self.monitor.update(**response.usage.model_dump())
+            self.monitor.update(
+                response.usage.model_dump(),
+                prefix=self.model_name,
+            )
         except QuotaExceededError as e:
             # TODO: optimize quota exceeded error handling process
             logger.error(e.message)
diff --git a/src/agentscope/utils/monitor.py b/src/agentscope/utils/monitor.py
index 7781290f5..1f6a68429 100644
--- a/src/agentscope/utils/monitor.py
+++ b/src/agentscope/utils/monitor.py
@@ -64,10 +64,10 @@ def add(self, metric_name: str, value: float) -> bool:
             `bool`: whether the operation success.
         """
 
-    def update(self, **kwargs: Any) -> None:
+    def update(self, values: dict, prefix: Optional[str] = None) -> None:
         """Update multiple metrics at once."""
-        for k, v in kwargs.items():
-            self.add(k, v)
+        for k, v in values:
+            self.add(full_name(prefix=prefix, name=k), v)
 
     @abstractmethod
     def clear(self, metric_name: str) -> bool:
@@ -200,15 +200,30 @@ def register_budget(
             model_name (`str`): model that requires budget.
             value (`float`): the budget value.
             prefix (`Optional[str]`, default `None`): used to distinguish
-            multiple budget registrations for the same model. For multiple
-            registrations with the same `model_name` and `prefix`, only the
-            first time will take effect.
+            multiple budget registrations. For multiple registrations with
+            the same `prefix`, only the first time will take effect.
 
         Returns:
             `bool`: whether the operation success.
         """
 
 
+def full_name(name: str, prefix: Optional[str] = None) -> str:
+    """get the full name of a metric.
+
+    Args:
+        metric_name (`str`): name of a metric.
+        prefix (` Optional[str]`, default `None`): metric prefix.
+
+    Returns:
+        `str`: the full name of the metric
+    """
+    if prefix is None:
+        return name
+    else:
+        return f"{prefix}.{name}"
+
+
 class QuotaExceededError(Exception):
     """An Exception used to indicate that a certain metric exceeds quota"""
 
@@ -625,10 +640,17 @@ def exists(self, metric_name: str) -> bool:
         with sqlite_cursor(self.db_path) as cursor:
             return self._exists(cursor, metric_name)
 
-    def update(self, **kwargs: Any) -> None:
+    def update(self, values: dict, prefix: Optional[str] = None) -> None:
         with sqlite_transaction(self.db_path) as cursor:
-            for metric_name, value in kwargs.items():
-                self._add(cursor, metric_name, value)
+            for metric_name, value in values.items():
+                self._add(
+                    cursor,
+                    full_name(
+                        name=metric_name,
+                        prefix=prefix,
+                    ),
+                    value,
+                )
 
     def _create_update_cost_trigger(
         self,
@@ -661,7 +683,10 @@ def register_budget(
         logger.info(f"set budget {value} to {model_name}")
         pricing = get_pricing()
         if model_name in pricing:
-            budget_metric_name = f"{prefix}.{model_name}.cost"
+            budget_metric_name = full_name(
+                name="cost",
+                prefix=prefix,
+            )
             ok = self.register(
                 metric_name=budget_metric_name,
                 metric_unit="dollor",
@@ -670,7 +695,10 @@ def register_budget(
             if not ok:
                 return False
             for metric_name, unit_price in pricing[model_name].items():
-                token_metric_name = f"{prefix}.{model_name}.{metric_name}"
+                token_metric_name = full_name(
+                    name=metric_name,
+                    prefix=prefix,
+                )
                 self.register(
                     metric_name=token_metric_name,
                     metric_unit="token",
@@ -721,24 +749,28 @@ class MonitorFactory:
 
         from agentscope.utils import MonitorFactory
         monitor = MonitorFactory.get_monitor()
-
     """
 
     _instance = None
 
     @classmethod
-    def get_monitor(cls, impl_type: Optional[str] = None) -> MonitorBase:
+    def get_monitor(
+        cls,
+        impl_type: Optional[str] = None,
+        **kwargs: Any,
+    ) -> MonitorBase:
         """Get the monitor instance.
 
         Returns:
             `MonitorBase`: the monitor instance.
         """
         if cls._instance is None:
-            # todo: init a specific monitor implementation by input args
-            if impl_type is None or impl_type.lower() == "dict":
+            if impl_type is None or impl_type.lower() == "sqlite":
+                cls._instance = SqliteMonitor(**kwargs)
+            elif impl_type.lower() == "dict":
                 cls._instance = DictMonitor()
             else:
                 raise NotImplementedError(
                     "Monitor with type [{type}] is not implemented.",
                 )
-        return cls._instance
+        return cls._instance  # type: ignore [return-value]
diff --git a/tests/monitor_test.py b/tests/monitor_test.py
index 342d32e5c..916c6d2c3 100644
--- a/tests/monitor_test.py
+++ b/tests/monitor_test.py
@@ -16,8 +16,8 @@ class MonitorFactoryTest(unittest.TestCase):
 
     def test_get_monitor(self) -> None:
         """Test get monitor method of MonitorFactory."""
-        monitor1 = MonitorFactory.get_monitor()
-        monitor2 = MonitorFactory.get_monitor()
+        monitor1 = MonitorFactory.get_monitor("dict")
+        monitor2 = MonitorFactory.get_monitor("dict")
         self.assertEqual(monitor1, monitor2)
         self.assertTrue(
             monitor1.register("token_num", metric_unit="token", quota=200),
@@ -185,7 +185,7 @@ def test_register_budget(self) -> None:
             self.monitor.register_budget(
                 model_name="gpt-4",
                 value=5,
-                prefix="agent_A",
+                prefix="agent_A.gpt-4",
             ),
         )
         # register an existing model with different prefix is ok
@@ -193,21 +193,22 @@ def test_register_budget(self) -> None:
             self.monitor.register_budget(
                 model_name="gpt-4",
                 value=15,
-                prefix="agent_B",
+                prefix="agent_B.gpt-4",
             ),
         )
         gpt_4_3d = {
-            "agent_A.gpt-4.prompt_tokens": 50000,
-            "agent_A.gpt-4.completion_tokens": 25000,
-            "agent_A.gpt-4.total_tokens": 750000,
+            "prompt_tokens": 50000,
+            "completion_tokens": 25000,
+            "total_tokens": 750000,
         }
         # agentA uses 3 dollors
-        self.monitor.update(**gpt_4_3d)
+        self.monitor.update(gpt_4_3d, prefix="agent_A.gpt-4")
         # agentA uses another 3 dollors and exceeds quota
         self.assertRaises(
             QuotaExceededError,
             self.monitor.update,
-            **gpt_4_3d,
+            gpt_4_3d,
+            "agent_A.gpt-4",
         )
         self.assertLess(
             self.monitor.get_value(  # type: ignore [arg-type]
@@ -220,6 +221,12 @@ def test_register_budget(self) -> None:
             self.monitor.register_budget(
                 model_name="gpt-4",
                 value=5,
-                prefix="agent_A",
+                prefix="agent_A.gpt-4",
             ),
         )
+        self.assertEqual(
+            self.monitor.get_value(  # type: ignore [arg-type]
+                "agent_A.gpt-4.cost",
+            ),
+            3,
+        )

From ec1f2f8bdd356e46e4cd66a274ee2bc4c4aa027f Mon Sep 17 00:00:00 2001
From: "panxuchen.pxc" <panxuchen.pxc@alibaba-inc.com>
Date: Thu, 18 Jan 2024 19:33:34 +0800
Subject: [PATCH 07/25] fix docstring

---
 src/agentscope/utils/monitor.py | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/src/agentscope/utils/monitor.py b/src/agentscope/utils/monitor.py
index 1f6a68429..9e95e8b45 100644
--- a/src/agentscope/utils/monitor.py
+++ b/src/agentscope/utils/monitor.py
@@ -200,8 +200,8 @@ def register_budget(
             model_name (`str`): model that requires budget.
             value (`float`): the budget value.
             prefix (`Optional[str]`, default `None`): used to distinguish
-            multiple budget registrations. For multiple registrations with
-            the same `prefix`, only the first time will take effect.
+                multiple budget registrations. For multiple registrations with
+                the same `prefix`, only the first time will take effect.
 
         Returns:
             `bool`: whether the operation success.
@@ -436,6 +436,15 @@ def __init__(
         table_name: str = _DEFAULT_MONITOR_TABLE_NAME,
         drop_exists: bool = False,
     ) -> None:
+        """Initialize a SqliteMonitor.
+
+        Args:
+            db_path (`str`): path to the sqlite db file.
+            table_name (`str`, optional): the table name used by the monitor.
+                Defaults to _DEFAULT_MONITOR_TABLE_NAME.
+            drop_exists (bool, optional): whether to delete the original table
+            when the table already exists. Defaults to False.
+        """
         super().__init__()
         self.db_path = db_path
         self.table_name = table_name
@@ -722,6 +731,7 @@ def get_pricing() -> dict:
     Returns:
         `dict`: the dict with pricing information.
     """
+    # TODO: get pricing from files
     return {
         "gpt-4-turbo": {
             "prompt_tokens": 0.00001,
@@ -761,6 +771,11 @@ def get_monitor(
     ) -> MonitorBase:
         """Get the monitor instance.
 
+        Args:
+            impl_type (`Optional[str]`, optional): the type of monitor,
+                currently supports `sqlite` and `dict`, the default is
+                `sqlite`.
+
         Returns:
             `MonitorBase`: the monitor instance.
         """

From aeea7682be4c421fb53845cc48a96e03988e39de Mon Sep 17 00:00:00 2001
From: "panxuchen.pxc" <panxuchen.pxc@alibaba-inc.com>
Date: Fri, 19 Jan 2024 12:23:52 +0800
Subject: [PATCH 08/25] opt naming

---
 src/agentscope/_init.py               |  2 +-
 src/agentscope/constants.py           |  2 +-
 src/agentscope/file_manager.py        |  6 +++---
 src/agentscope/models/openai_model.py |  4 ++--
 src/agentscope/utils/monitor.py       | 14 +++++++-------
 5 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/src/agentscope/_init.py b/src/agentscope/_init.py
index 7e2296563..6f75004b5 100644
--- a/src/agentscope/_init.py
+++ b/src/agentscope/_init.py
@@ -88,7 +88,7 @@ def init(
     setup_logger(dir_log, logger_level)
 
     # Set monitor
-    _ = MonitorFactory.get_monitor(db_path=file_manager.file_db)
+    _ = MonitorFactory.get_monitor(db_path=file_manager.path_db)
 
     # Load config and init agent by configs
     if agent_configs is not None:
diff --git a/src/agentscope/constants.py b/src/agentscope/constants.py
index 938c12290..5cc9fea61 100644
--- a/src/agentscope/constants.py
+++ b/src/agentscope/constants.py
@@ -16,7 +16,7 @@
 _DEFAULT_SUBDIR_FILE = "file"
 _DEFAULT_SUBDIR_INVOKE = "invoke"
 _DEFAULT_IMAGE_NAME = "image_{}_{}.png"
-_DEFAULT_SQLITE_DB_FILE = "agentscope.db"
+_DEFAULT_SQLITE_DB_PATH = "agentscope.db"
 # for model wrapper
 _DEFAULT_MAX_RETRIES = 3
 _DEFAULT_MESSAGES_KEY = "inputs"
diff --git a/src/agentscope/file_manager.py b/src/agentscope/file_manager.py
index d53f39fbf..87b8e959d 100644
--- a/src/agentscope/file_manager.py
+++ b/src/agentscope/file_manager.py
@@ -14,7 +14,7 @@
     _DEFAULT_SUBDIR_CODE,
     _DEFAULT_SUBDIR_FILE,
     _DEFAULT_SUBDIR_INVOKE,
-    _DEFAULT_SQLITE_DB_FILE,
+    _DEFAULT_SQLITE_DB_PATH,
     _DEFAULT_IMAGE_NAME,
 )
 
@@ -75,9 +75,9 @@ def dir_invoke(self) -> str:
         return self._get_and_create_subdir(_DEFAULT_SUBDIR_INVOKE)
 
     @property
-    def file_db(self) -> str:
+    def path_db(self) -> str:
         """The path to the sqlite db file."""
-        return self._get_file_path(_DEFAULT_SQLITE_DB_FILE)
+        return self._get_file_path(_DEFAULT_SQLITE_DB_PATH)
 
     def init(self, save_dir: str, save_api_invoke: bool = False) -> None:
         """Set the directory for saving files."""
diff --git a/src/agentscope/models/openai_model.py b/src/agentscope/models/openai_model.py
index 3eed6c846..fc8eff640 100644
--- a/src/agentscope/models/openai_model.py
+++ b/src/agentscope/models/openai_model.py
@@ -13,7 +13,7 @@
     openai = None
 
 from ..utils.monitor import MonitorFactory
-from ..utils.monitor import full_name
+from ..utils.monitor import get_full_name
 from ..utils import QuotaExceededError
 from ..utils.token_utils import get_openai_max_length
 
@@ -110,7 +110,7 @@ def _metric(self, metric_name: str) -> str:
         Returns:
             `str`: Metric name of this wrapper.
         """
-        return full_name(name=metric_name, prefix=self.model_name)
+        return get_full_name(name=metric_name, prefix=self.model_name)
 
 
 class OpenAIChatWrapper(OpenAIWrapper):
diff --git a/src/agentscope/utils/monitor.py b/src/agentscope/utils/monitor.py
index 9e95e8b45..a3d8f95f8 100644
--- a/src/agentscope/utils/monitor.py
+++ b/src/agentscope/utils/monitor.py
@@ -67,7 +67,7 @@ def add(self, metric_name: str, value: float) -> bool:
     def update(self, values: dict, prefix: Optional[str] = None) -> None:
         """Update multiple metrics at once."""
         for k, v in values:
-            self.add(full_name(prefix=prefix, name=k), v)
+            self.add(get_full_name(prefix=prefix, name=k), v)
 
     @abstractmethod
     def clear(self, metric_name: str) -> bool:
@@ -208,7 +208,7 @@ def register_budget(
         """
 
 
-def full_name(name: str, prefix: Optional[str] = None) -> str:
+def get_full_name(name: str, prefix: Optional[str] = None) -> str:
     """get the full name of a metric.
 
     Args:
@@ -654,7 +654,7 @@ def update(self, values: dict, prefix: Optional[str] = None) -> None:
             for metric_name, value in values.items():
                 self._add(
                     cursor,
-                    full_name(
+                    get_full_name(
                         name=metric_name,
                         prefix=prefix,
                     ),
@@ -690,9 +690,9 @@ def register_budget(
         prefix: Optional[str] = None,
     ) -> bool:
         logger.info(f"set budget {value} to {model_name}")
-        pricing = get_pricing()
+        pricing = _get_pricing()
         if model_name in pricing:
-            budget_metric_name = full_name(
+            budget_metric_name = get_full_name(
                 name="cost",
                 prefix=prefix,
             )
@@ -704,7 +704,7 @@ def register_budget(
             if not ok:
                 return False
             for metric_name, unit_price in pricing[model_name].items():
-                token_metric_name = full_name(
+                token_metric_name = get_full_name(
                     name=metric_name,
                     prefix=prefix,
                 )
@@ -725,7 +725,7 @@ def register_budget(
             return False
 
 
-def get_pricing() -> dict:
+def _get_pricing() -> dict:
     """Get pricing as a dict
 
     Returns:

From d0c405fccde4bc6a6b1a34016c5aeae44cf5063d Mon Sep 17 00:00:00 2001
From: "panxuchen.pxc" <panxuchen.pxc@alibaba-inc.com>
Date: Fri, 19 Jan 2024 14:56:14 +0800
Subject: [PATCH 09/25] remove dictmonitor

---
 src/agentscope/utils/monitor.py | 179 ++++----------------------------
 tests/monitor_test.py           |  20 ++--
 2 files changed, 28 insertions(+), 171 deletions(-)

diff --git a/src/agentscope/utils/monitor.py b/src/agentscope/utils/monitor.py
index a3d8f95f8..3d6e124a6 100644
--- a/src/agentscope/utils/monitor.py
+++ b/src/agentscope/utils/monitor.py
@@ -2,15 +2,17 @@
 """ Monitor for agentscope """
 
 import re
-import copy
 import sqlite3
 from abc import ABC
 from abc import abstractmethod
 from contextlib import contextmanager
-from typing import Optional, Any, Generator
+from typing import Optional, Generator
 from loguru import logger
 
-from agentscope.constants import _DEFAULT_MONITOR_TABLE_NAME
+from agentscope.constants import (
+    _DEFAULT_MONITOR_TABLE_NAME,
+    _DEFAULT_SQLITE_DB_PATH,
+)
 
 
 class MonitorBase(ABC):
@@ -229,159 +231,16 @@ class QuotaExceededError(Exception):
 
     def __init__(
         self,
-        metric_name: Optional[str] = None,
-        quota: Optional[float] = None,
+        name: str,
     ) -> None:
-        if metric_name is not None and quota is not None:
-            self.message = f"Metric [{metric_name}] exceeds quota [{quota}]"
-            super().__init__(self.message)
-        else:
-            super().__init__()
-
-
-def return_false_if_not_exists(  # type: ignore [no-untyped-def]
-    func,
-):
-    """A decorator used to check whether the attribute exists.
-    It will return False directly without executing the function,
-    if the metric does not exist.
-    """
-
-    def inner(
-        monitor: MonitorBase,
-        metric_name: str,
-        *args: tuple,
-        **kwargs: dict,
-    ) -> bool:
-        if not monitor.exists(metric_name):
-            logger.warning(f"Metric [{metric_name}] not exists.")
-            return False
-        return func(monitor, metric_name, *args, **kwargs)
-
-    return inner
-
-
-def return_none_if_not_exists(  # type: ignore [no-untyped-def]
-    func,
-):
-    """A decorator used to check whether the attribute exists.
-    It will return None directly without executing the function,
-    if the metric does not exist.
-    """
-
-    def inner(  # type: ignore [no-untyped-def]
-        monitor: MonitorBase,
-        metric_name: str,
-        *args: tuple,
-        **kwargs: dict,
-    ):
-        if not monitor.exists(metric_name):
-            logger.warning(f"Metric [{metric_name}] not exists.")
-            return None
-        return func(monitor, metric_name, *args, **kwargs)
-
-    return inner
-
-
-class DictMonitor(MonitorBase):
-    """MonitorBase implementation based on dictionary."""
-
-    def __init__(self) -> None:
-        self.metrics = {}
-
-    def register(
-        self,
-        metric_name: str,
-        metric_unit: Optional[str] = None,
-        quota: Optional[float] = None,
-    ) -> bool:
-        if metric_name in self.metrics:
-            logger.warning(f"Metric [{metric_name}] is already registered.")
-            return False
-        self.metrics[metric_name] = {
-            "value": 0.0,
-            "unit": metric_unit,
-            "quota": quota,
-        }
-        logger.info(
-            f"Register metric [{metric_name}] to Monitor with unit "
-            f"[{metric_unit}] and quota [{quota}]",
-        )
-        return True
-
-    @return_false_if_not_exists
-    def add(self, metric_name: str, value: float) -> bool:
-        if (
-            self.metrics[metric_name]["quota"] is not None
-            and self.metrics[metric_name]["value"] + value
-            > self.metrics[metric_name]["quota"]
-        ):
-            logger.warning(f"Metric [{metric_name}] quota exceeded.")
-            raise QuotaExceededError(
-                metric_name=metric_name,
-                quota=self.metrics[metric_name]["quota"],
-            )
-        self.metrics[metric_name]["value"] += value
-        return True
-
-    def exists(self, metric_name: str) -> bool:
-        return metric_name in self.metrics
-
-    @return_false_if_not_exists
-    def clear(self, metric_name: str) -> bool:
-        self.metrics[metric_name]["value"] = 0.0
-        return True
-
-    @return_false_if_not_exists
-    def remove(self, metric_name: str) -> bool:
-        self.metrics.pop(metric_name)
-        logger.info(f"Remove metric [{metric_name}] from monitor.")
-        return True
-
-    @return_none_if_not_exists
-    def get_value(self, metric_name: str) -> Optional[float]:
-        if metric_name not in self.metrics:
-            return None
-        return self.metrics[metric_name]["value"]
-
-    @return_none_if_not_exists
-    def get_unit(self, metric_name: str) -> Optional[str]:
-        if metric_name not in self.metrics:
-            return None
-        return self.metrics[metric_name]["unit"]
-
-    @return_none_if_not_exists
-    def get_quota(self, metric_name: str) -> Optional[float]:
-        return self.metrics[metric_name]["quota"]
-
-    @return_false_if_not_exists
-    def set_quota(self, metric_name: str, quota: float) -> bool:
-        self.metrics[metric_name]["quota"] = quota
-        return True
-
-    @return_none_if_not_exists
-    def get_metric(self, metric_name: str) -> Optional[dict]:
-        return copy.deepcopy(self.metrics[metric_name])
-
-    def get_metrics(self, filter_regex: Optional[str] = None) -> dict:
-        if filter_regex is None:
-            return copy.deepcopy(self.metrics)
-        else:
-            pattern = re.compile(filter_regex)
-            return {
-                key: copy.deepcopy(value)
-                for key, value in self.metrics.items()
-                if pattern.search(key)
-            }
+        """Init a QuotaExceedError instance.
 
-    def register_budget(
-        self,
-        model_name: str,
-        value: float,
-        prefix: Optional[str] = "local",
-    ) -> bool:
-        logger.warning("DictMonitor doesn't support register_budget")
-        return False
+        Args:
+            name (`str`): name of the metric which exceeds quota.
+        """
+        self.message = f"Metric [{name}] exceeds quota."
+        self.name = name
+        super().__init__(self.message)
 
 
 @contextmanager
@@ -519,7 +378,7 @@ def _add(
                 (value, metric_name),
             )
         except sqlite3.IntegrityError as e:
-            raise QuotaExceededError() from e
+            raise QuotaExceededError(metric_name) from e
 
     def add(self, metric_name: str, value: float) -> bool:
         with sqlite_transaction(self.db_path) as cursor:
@@ -767,23 +626,21 @@ class MonitorFactory:
     def get_monitor(
         cls,
         impl_type: Optional[str] = None,
-        **kwargs: Any,
+        db_path: str = _DEFAULT_SQLITE_DB_PATH,
     ) -> MonitorBase:
         """Get the monitor instance.
 
         Args:
             impl_type (`Optional[str]`, optional): the type of monitor,
-                currently supports `sqlite` and `dict`, the default is
-                `sqlite`.
+                currently supports `sqlite` only.
+            db_path (`Optional[str]`, optional): path to the sqlite db file.
 
         Returns:
             `MonitorBase`: the monitor instance.
         """
         if cls._instance is None:
             if impl_type is None or impl_type.lower() == "sqlite":
-                cls._instance = SqliteMonitor(**kwargs)
-            elif impl_type.lower() == "dict":
-                cls._instance = DictMonitor()
+                cls._instance = SqliteMonitor(db_path=db_path)
             else:
                 raise NotImplementedError(
                     "Monitor with type [{type}] is not implemented.",
diff --git a/tests/monitor_test.py b/tests/monitor_test.py
index 916c6d2c3..d16357227 100644
--- a/tests/monitor_test.py
+++ b/tests/monitor_test.py
@@ -8,16 +8,20 @@
 import os
 from agentscope.utils import MonitorBase, QuotaExceededError, MonitorFactory
 
-from agentscope.utils.monitor import DictMonitor, SqliteMonitor
+from agentscope.utils.monitor import SqliteMonitor
 
 
 class MonitorFactoryTest(unittest.TestCase):
     "Test class for MonitorFactory"
 
+    def setUp(self) -> None:
+        self.db_path = f"test-{uuid.uuid4()}.db"
+        _ = MonitorFactory.get_monitor(db_path=self.db_path)
+
     def test_get_monitor(self) -> None:
         """Test get monitor method of MonitorFactory."""
-        monitor1 = MonitorFactory.get_monitor("dict")
-        monitor2 = MonitorFactory.get_monitor("dict")
+        monitor1 = MonitorFactory.get_monitor()
+        monitor2 = MonitorFactory.get_monitor()
         self.assertEqual(monitor1, monitor2)
         self.assertTrue(
             monitor1.register("token_num", metric_unit="token", quota=200),
@@ -26,6 +30,9 @@ def test_get_monitor(self) -> None:
         self.assertTrue(monitor2.remove("token_num"))
         self.assertFalse(monitor1.exists("token_num"))
 
+    def tearDown(self) -> None:
+        os.remove(self.db_path)
+
 
 class MonitorTestBase(unittest.TestCase):
     """An abstract test class for MonitorBase interface"""
@@ -162,13 +169,6 @@ def test_get(self) -> None:
         )
 
 
-class DictMonitorTest(MonitorTestBase):
-    """Test class for DictMonitor"""
-
-    def get_monitor_instance(self) -> MonitorBase:
-        return DictMonitor()
-
-
 class SqliteMonitorTest(MonitorTestBase):
     """Test class for SqliteMonitor"""
 

From 6e1a9c5a90b9ce5203cd78256043e7a4aa13aeb9 Mon Sep 17 00:00:00 2001
From: "panxuchen.pxc" <panxuchen.pxc@alibaba-inc.com>
Date: Fri, 19 Jan 2024 15:24:53 +0800
Subject: [PATCH 10/25] fix file_manager init

---
 src/agentscope/file_manager.py  | 5 +++--
 src/agentscope/utils/monitor.py | 4 ----
 2 files changed, 3 insertions(+), 6 deletions(-)

diff --git a/src/agentscope/file_manager.py b/src/agentscope/file_manager.py
index 87b8e959d..ad397b8ab 100644
--- a/src/agentscope/file_manager.py
+++ b/src/agentscope/file_manager.py
@@ -82,8 +82,9 @@ def path_db(self) -> str:
     def init(self, save_dir: str, save_api_invoke: bool = False) -> None:
         """Set the directory for saving files."""
         self.dir = save_dir
-        if not os.path.exists(save_dir):
-            os.makedirs(save_dir)
+        runtime_dir = os.path.join(save_dir, Runtime.runtime_id)
+        if not os.path.exists(runtime_dir):
+            os.makedirs(runtime_dir)
 
         self.save_api_invoke = save_api_invoke
 
diff --git a/src/agentscope/utils/monitor.py b/src/agentscope/utils/monitor.py
index 3d6e124a6..bfa70ae6d 100644
--- a/src/agentscope/utils/monitor.py
+++ b/src/agentscope/utils/monitor.py
@@ -336,10 +336,6 @@ def _create_monitor_table(self, drop_exists: bool = False) -> None:
                 """,
             )
 
-    def _get_trigger_name(self, metric_name: str) -> str:
-        """Get the name of the trigger on a certain metric"""
-        return f"{self.table_name}.{metric_name}.trigger"
-
     def register(
         self,
         metric_name: str,

From e57ef98a600611116e10376f4304c6bb2fd632b4 Mon Sep 17 00:00:00 2001
From: "panxuchen.pxc" <panxuchen.pxc@alibaba-inc.com>
Date: Fri, 19 Jan 2024 15:40:33 +0800
Subject: [PATCH 11/25] debug monitor init

---
 src/agentscope/utils/monitor.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/agentscope/utils/monitor.py b/src/agentscope/utils/monitor.py
index bfa70ae6d..9ffd8eabe 100644
--- a/src/agentscope/utils/monitor.py
+++ b/src/agentscope/utils/monitor.py
@@ -308,6 +308,8 @@ def __init__(
         self.db_path = db_path
         self.table_name = table_name
         self._create_monitor_table(drop_exists)
+        logger.info(
+            f"SqliteMonitor initialization completed at [{self.db_path}]")
 
     def _create_monitor_table(self, drop_exists: bool = False) -> None:
         """Internal method to create a table in sqlite3."""
@@ -335,6 +337,9 @@ def _create_monitor_table(self, drop_exists: bool = False) -> None:
                 END;
                 """,
             )
+        logger.info(f'Init [{self.table_name}] as the monitor table')
+        logger.info(
+            f'Init [{self.table_name}_quota_exceeded] as the monitor trigger')
 
     def register(
         self,

From 586b24e00f33da97fc40e8dac5247756a0255ad5 Mon Sep 17 00:00:00 2001
From: "panxuchen.pxc" <panxuchen.pxc@alibaba-inc.com>
Date: Fri, 19 Jan 2024 16:55:09 +0800
Subject: [PATCH 12/25] fix monitor test

---
 tests/monitor_test.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/monitor_test.py b/tests/monitor_test.py
index d16357227..26ac418c8 100644
--- a/tests/monitor_test.py
+++ b/tests/monitor_test.py
@@ -31,6 +31,7 @@ def test_get_monitor(self) -> None:
         self.assertFalse(monitor1.exists("token_num"))
 
     def tearDown(self) -> None:
+        MonitorFactory._instance = None  # pylint: disable=W0212
         os.remove(self.db_path)
 
 

From a401aba6858df5dc9c236999213549877a717a9d Mon Sep 17 00:00:00 2001
From: "panxuchen.pxc" <panxuchen.pxc@alibaba-inc.com>
Date: Fri, 19 Jan 2024 16:58:41 +0800
Subject: [PATCH 13/25] fix format

---
 src/agentscope/utils/monitor.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/agentscope/utils/monitor.py b/src/agentscope/utils/monitor.py
index 9ffd8eabe..c734c1594 100644
--- a/src/agentscope/utils/monitor.py
+++ b/src/agentscope/utils/monitor.py
@@ -309,7 +309,8 @@ def __init__(
         self.table_name = table_name
         self._create_monitor_table(drop_exists)
         logger.info(
-            f"SqliteMonitor initialization completed at [{self.db_path}]")
+            f"SqliteMonitor initialization completed at [{self.db_path}]",
+        )
 
     def _create_monitor_table(self, drop_exists: bool = False) -> None:
         """Internal method to create a table in sqlite3."""
@@ -337,9 +338,10 @@ def _create_monitor_table(self, drop_exists: bool = False) -> None:
                 END;
                 """,
             )
-        logger.info(f'Init [{self.table_name}] as the monitor table')
+        logger.info(f"Init [{self.table_name}] as the monitor table")
         logger.info(
-            f'Init [{self.table_name}_quota_exceeded] as the monitor trigger')
+            f"Init [{self.table_name}_quota_exceeded] as the monitor trigger",
+        )
 
     def register(
         self,

From 11479053eb447148649a52839cd6da4e0f7cd6e0 Mon Sep 17 00:00:00 2001
From: "panxuchen.pxc" <panxuchen.pxc@alibaba-inc.com>
Date: Mon, 22 Jan 2024 10:27:43 +0800
Subject: [PATCH 14/25] fix workflow

---
 .github/workflows/sphinx_docs.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/sphinx_docs.yml b/.github/workflows/sphinx_docs.yml
index 90a519c53..0e28de101 100644
--- a/.github/workflows/sphinx_docs.yml
+++ b/.github/workflows/sphinx_docs.yml
@@ -37,7 +37,7 @@ jobs:
         name: SphinxDoc
         path: ${{ steps.deployment.outputs.artifact }}
     - uses: peaceiris/actions-gh-pages@v3
-      if: github.event_name == 'push' && github_ref == 'refs/heads/main'
+      if: github.event_name == 'push' && github.ref == 'refs/heads/main'
       with:
         github_token: ${{ secrets.GITHUB_TOKEN }}
         publish_dir: ${{ steps.deployment.outputs.artifact }}
\ No newline at end of file

From 4699b7a838d84e13df6c2c448bfb27376dd89aa2 Mon Sep 17 00:00:00 2001
From: "panxuchen.pxc" <panxuchen.pxc@alibaba-inc.com>
Date: Mon, 22 Jan 2024 10:36:38 +0800
Subject: [PATCH 15/25] test page deploy

---
 .github/workflows/sphinx_docs.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/sphinx_docs.yml b/.github/workflows/sphinx_docs.yml
index 0e28de101..2212815e0 100644
--- a/.github/workflows/sphinx_docs.yml
+++ b/.github/workflows/sphinx_docs.yml
@@ -8,6 +8,7 @@ on:
   push:
     branches:
       - main
+      - 'feature/*'
 
 jobs:
   pages:
@@ -37,7 +38,7 @@ jobs:
         name: SphinxDoc
         path: ${{ steps.deployment.outputs.artifact }}
     - uses: peaceiris/actions-gh-pages@v3
-      if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+      if: github.event_name == 'push' && github.ref == 'refs/heads/main' && startsWith(github.ref, 'refs/heads/feature/')
       with:
         github_token: ${{ secrets.GITHUB_TOKEN }}
         publish_dir: ${{ steps.deployment.outputs.artifact }}
\ No newline at end of file

From 70707bb681b221d24a58625d617b6ca353dc067d Mon Sep 17 00:00:00 2001
From: "panxuchen.pxc" <panxuchen.pxc@alibaba-inc.com>
Date: Mon, 22 Jan 2024 10:52:02 +0800
Subject: [PATCH 16/25] test page deploy

---
 .github/workflows/sphinx_docs.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/sphinx_docs.yml b/.github/workflows/sphinx_docs.yml
index 2212815e0..fc8ad2ee6 100644
--- a/.github/workflows/sphinx_docs.yml
+++ b/.github/workflows/sphinx_docs.yml
@@ -38,7 +38,7 @@ jobs:
         name: SphinxDoc
         path: ${{ steps.deployment.outputs.artifact }}
     - uses: peaceiris/actions-gh-pages@v3
-      if: github.event_name == 'push' && github.ref == 'refs/heads/main' && startsWith(github.ref, 'refs/heads/feature/')
+      if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/feature/'))
       with:
         github_token: ${{ secrets.GITHUB_TOKEN }}
         publish_dir: ${{ steps.deployment.outputs.artifact }}
\ No newline at end of file

From e0f119f13d734583c0e5b8fe0827444ac3563696 Mon Sep 17 00:00:00 2001
From: "panxuchen.pxc" <panxuchen.pxc@alibaba-inc.com>
Date: Mon, 22 Jan 2024 11:16:03 +0800
Subject: [PATCH 17/25] test page deploy

---
 .github/workflows/sphinx_docs.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/sphinx_docs.yml b/.github/workflows/sphinx_docs.yml
index fc8ad2ee6..459416632 100644
--- a/.github/workflows/sphinx_docs.yml
+++ b/.github/workflows/sphinx_docs.yml
@@ -38,7 +38,6 @@ jobs:
         name: SphinxDoc
         path: ${{ steps.deployment.outputs.artifact }}
     - uses: peaceiris/actions-gh-pages@v3
-      if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/feature/'))
       with:
         github_token: ${{ secrets.GITHUB_TOKEN }}
         publish_dir: ${{ steps.deployment.outputs.artifact }}
\ No newline at end of file

From 8aa47bbdfc306d3072972643b34a8e4d1ebfb9ab Mon Sep 17 00:00:00 2001
From: "panxuchen.pxc" <panxuchen.pxc@alibaba-inc.com>
Date: Mon, 22 Jan 2024 11:27:11 +0800
Subject: [PATCH 18/25] test page deploy

---
 .github/workflows/sphinx_docs.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/sphinx_docs.yml b/.github/workflows/sphinx_docs.yml
index 459416632..c1f1eb60c 100644
--- a/.github/workflows/sphinx_docs.yml
+++ b/.github/workflows/sphinx_docs.yml
@@ -8,7 +8,7 @@ on:
   push:
     branches:
       - main
-      - 'feature/*'
+      - 'feature/**'
 
 jobs:
   pages:
@@ -38,6 +38,7 @@ jobs:
         name: SphinxDoc
         path: ${{ steps.deployment.outputs.artifact }}
     - uses: peaceiris/actions-gh-pages@v3
+      if: ${{ github.event_name == 'push' && (github.ref_name == 'main' || startsWith(github.ref_name, 'feature')) }}
       with:
         github_token: ${{ secrets.GITHUB_TOKEN }}
         publish_dir: ${{ steps.deployment.outputs.artifact }}
\ No newline at end of file

From 0396ae9d1c921d3aefdb26a936b99a75d1916996 Mon Sep 17 00:00:00 2001
From: "panxuchen.pxc" <panxuchen.pxc@alibaba-inc.com>
Date: Mon, 22 Jan 2024 11:35:28 +0800
Subject: [PATCH 19/25] test page deploy

---
 .github/workflows/sphinx_docs.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/sphinx_docs.yml b/.github/workflows/sphinx_docs.yml
index c1f1eb60c..1235218e0 100644
--- a/.github/workflows/sphinx_docs.yml
+++ b/.github/workflows/sphinx_docs.yml
@@ -38,7 +38,7 @@ jobs:
         name: SphinxDoc
         path: ${{ steps.deployment.outputs.artifact }}
     - uses: peaceiris/actions-gh-pages@v3
-      if: ${{ github.event_name == 'push' && (github.ref_name == 'main' || startsWith(github.ref_name, 'feature')) }}
+      if: ${{ github.event_name == 'push' }}
       with:
         github_token: ${{ secrets.GITHUB_TOKEN }}
         publish_dir: ${{ steps.deployment.outputs.artifact }}
\ No newline at end of file

From f00fdd07943032b8bb4199bc22d826e6c914029e Mon Sep 17 00:00:00 2001
From: "panxuchen.pxc" <panxuchen.pxc@alibaba-inc.com>
Date: Mon, 22 Jan 2024 12:08:28 +0800
Subject: [PATCH 20/25] refactor sphinx doc workflow

---
 .github/workflows/sphinx_docs.yml            | 20 +++++-----
 docs/sphinx_doc/source/agentscope.agents.rst | 40 ++++++++++++++++++++
 2 files changed, 50 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/sphinx_docs.yml b/.github/workflows/sphinx_docs.yml
index 1235218e0..313d1dfd2 100644
--- a/.github/workflows/sphinx_docs.yml
+++ b/.github/workflows/sphinx_docs.yml
@@ -18,27 +18,27 @@ jobs:
       OS: ${{ matrix.os }}
       PYTHON: '3.9'
     steps:
-    - name: Checkout repository
+    - name: Checkout Repository
       uses: actions/checkout@master
     - name: Setup Python ${{ matrix.python-version }}
       uses: actions/setup-python@master
       with:
         python_version: ${{ matrix.python-version }}
-    - id: deployment
+    - name: Install Dependencies
+      run: |
+        pip install -q -e .[full]
+    - id: build
       name: Build Documentation
-      uses: sphinx-notes/pages@v3
-      with:
-        documentation_path: ./docs/sphinx_doc/source
-        python_version: ${{ matrix.python-version }}
-        publish: false
-        requirements_path: ./docs/sphinx_doc/requirements.txt
+      run: |
+        cd docs/sphinx_doc
+        make clean html
     - name: Upload Documentation
       uses: actions/upload-artifact@v3
       with:
         name: SphinxDoc
-        path: ${{ steps.deployment.outputs.artifact }}
+        path: 'docs/sphinx_doc/build'
     - uses: peaceiris/actions-gh-pages@v3
       if: ${{ github.event_name == 'push' }}
       with:
         github_token: ${{ secrets.GITHUB_TOKEN }}
-        publish_dir: ${{ steps.deployment.outputs.artifact }}
\ No newline at end of file
+        publish_dir: 'docs/sphinx_doc/build'
\ No newline at end of file
diff --git a/docs/sphinx_doc/source/agentscope.agents.rst b/docs/sphinx_doc/source/agentscope.agents.rst
index d8c576d1a..0b6520701 100644
--- a/docs/sphinx_doc/source/agentscope.agents.rst
+++ b/docs/sphinx_doc/source/agentscope.agents.rst
@@ -1,6 +1,14 @@
 Agents package
 ==========================
 
+operator module
+-------------------------------
+
+.. automodule:: agentscope.agents.operator
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
 agent module
 -------------------------------
 
@@ -13,6 +21,38 @@ rpc_agent module
 -------------------------------
 
 .. automodule:: agentscope.agents.rpc_agent
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+user_agent module
+-------------------------------
+
+.. automodule:: agentscope.agents.user_agent
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+dialog_agent module
+-------------------------------
+
+.. automodule:: agentscope.agents.dialog_agent
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+dict_dialog_agent module
+-------------------------------
+
+.. automodule:: agentscope.agents.dict_dialog_agent
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+rpc_dialog_agent module
+-------------------------------
+
+.. automodule:: agentscope.agents.dict_dialog_agent
    :members:
    :undoc-members:
    :show-inheritance:
\ No newline at end of file

From e9780a525f40b9124cdb32c16b06acccf041c3cb Mon Sep 17 00:00:00 2001
From: "panxuchen.pxc" <panxuchen.pxc@alibaba-inc.com>
Date: Mon, 22 Jan 2024 12:12:07 +0800
Subject: [PATCH 21/25] refactor sphinx doc workflow

---
 .github/workflows/sphinx_docs.yml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/sphinx_docs.yml b/.github/workflows/sphinx_docs.yml
index 313d1dfd2..48f8a847f 100644
--- a/.github/workflows/sphinx_docs.yml
+++ b/.github/workflows/sphinx_docs.yml
@@ -18,12 +18,11 @@ jobs:
       OS: ${{ matrix.os }}
       PYTHON: '3.9'
     steps:
-    - name: Checkout Repository
-      uses: actions/checkout@master
+    - uses: actions/checkout@master
     - name: Setup Python ${{ matrix.python-version }}
       uses: actions/setup-python@master
       with:
-        python_version: ${{ matrix.python-version }}
+        python-version: ${{ matrix.python-version }}
     - name: Install Dependencies
       run: |
         pip install -q -e .[full]

From f1c1ddf675748538e927f0019c9766083da6a3fa Mon Sep 17 00:00:00 2001
From: "panxuchen.pxc" <panxuchen.pxc@alibaba-inc.com>
Date: Mon, 22 Jan 2024 12:14:10 +0800
Subject: [PATCH 22/25] refactor sphinx doc workflow

---
 .github/workflows/sphinx_docs.yml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/sphinx_docs.yml b/.github/workflows/sphinx_docs.yml
index 48f8a847f..4ab69c525 100644
--- a/.github/workflows/sphinx_docs.yml
+++ b/.github/workflows/sphinx_docs.yml
@@ -12,8 +12,12 @@ on:
 
 jobs:
   pages:
-    runs-on: ubuntu-latest
     timeout-minutes: 20
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ['3.9']
     env:
       OS: ${{ matrix.os }}
       PYTHON: '3.9'

From cfa10e0fbd1fef26494b4d253ae4631a5a37d226 Mon Sep 17 00:00:00 2001
From: "panxuchen.pxc" <panxuchen.pxc@alibaba-inc.com>
Date: Mon, 22 Jan 2024 12:18:36 +0800
Subject: [PATCH 23/25] refactor sphinx doc workflow

---
 .github/workflows/sphinx_docs.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/sphinx_docs.yml b/.github/workflows/sphinx_docs.yml
index 4ab69c525..dcd62d9c1 100644
--- a/.github/workflows/sphinx_docs.yml
+++ b/.github/workflows/sphinx_docs.yml
@@ -44,4 +44,4 @@ jobs:
       if: ${{ github.event_name == 'push' }}
       with:
         github_token: ${{ secrets.GITHUB_TOKEN }}
-        publish_dir: 'docs/sphinx_doc/build'
\ No newline at end of file
+        publish_dir: 'docs/sphinx_doc/build/html'
\ No newline at end of file

From fc353655e155a019d48c5d75b2b0266a82561a90 Mon Sep 17 00:00:00 2001
From: "panxuchen.pxc" <panxuchen.pxc@alibaba-inc.com>
Date: Mon, 22 Jan 2024 12:23:32 +0800
Subject: [PATCH 24/25] fix doc worflow

---
 .github/workflows/sphinx_docs.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/sphinx_docs.yml b/.github/workflows/sphinx_docs.yml
index dcd62d9c1..86750643b 100644
--- a/.github/workflows/sphinx_docs.yml
+++ b/.github/workflows/sphinx_docs.yml
@@ -8,7 +8,6 @@ on:
   push:
     branches:
       - main
-      - 'feature/**'
 
 jobs:
   pages:
@@ -41,7 +40,7 @@ jobs:
         name: SphinxDoc
         path: 'docs/sphinx_doc/build'
     - uses: peaceiris/actions-gh-pages@v3
-      if: ${{ github.event_name == 'push' }}
+      if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
       with:
         github_token: ${{ secrets.GITHUB_TOKEN }}
         publish_dir: 'docs/sphinx_doc/build/html'
\ No newline at end of file

From f726353a41ded15e94a625a3daab7233b6d19d6f Mon Sep 17 00:00:00 2001
From: "panxuchen.pxc" <panxuchen.pxc@alibaba-inc.com>
Date: Mon, 22 Jan 2024 12:24:27 +0800
Subject: [PATCH 25/25] fix docstring

---
 src/agentscope/utils/monitor.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/agentscope/utils/monitor.py b/src/agentscope/utils/monitor.py
index c734c1594..48900648f 100644
--- a/src/agentscope/utils/monitor.py
+++ b/src/agentscope/utils/monitor.py
@@ -211,7 +211,7 @@ def register_budget(
 
 
 def get_full_name(name: str, prefix: Optional[str] = None) -> str:
-    """get the full name of a metric.
+    """Get the full name of a metric.
 
     Args:
         metric_name (`str`): name of a metric.