diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000000..0cf6b446c3b7 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,40 @@ +# Ignore everything by default, selectively add things to context +classic/run + +# AutoGPT +!classic/original_autogpt/autogpt/ +!classic/original_autogpt/pyproject.toml +!classic/original_autogpt/poetry.lock +!classic/original_autogpt/README.md +!classic/original_autogpt/tests/ + +# Benchmark +!classic/benchmark/agbenchmark/ +!classic/benchmark/pyproject.toml +!classic/benchmark/poetry.lock +!classic/benchmark/README.md + +# Forge +!classic/forge/ +!classic/forge/pyproject.toml +!classic/forge/poetry.lock +!classic/forge/README.md + +# Frontend +!classic/frontend/build/web/ + +# Platform +!autogpt_platform/ + +# Explicitly re-ignore some folders +.* +**/__pycache__ + +autogpt_platform/frontend/.next/ +autogpt_platform/frontend/node_modules +autogpt_platform/frontend/.env.example +autogpt_platform/frontend/.env.local +autogpt_platform/backend/.env +autogpt_platform/backend/.venv/ + +autogpt_platform/market/.env diff --git a/autogpt_platform/README.md b/autogpt_platform/README.md index 74b0e3e764af..4415ce1a3c6d 100644 --- a/autogpt_platform/README.md +++ b/autogpt_platform/README.md @@ -27,7 +27,7 @@ To run the AutoGPT Platform, follow these steps: 6. Run the following command: ``` - docker compose -f docker-compose.combined.yml up -d + docker compose up -d ``` @@ -46,8 +46,8 @@ To run the AutoGPT Platform, follow these steps: Here are some useful Docker Compose commands for managing your AutoGPT Platform: -- `docker compose -f docker-compose.combined.yml up -d`: Start the services in detached mode. -- `docker compose -f docker-compose.combined.yml stop`: Stop the running services without removing them. +- `docker compose up -d`: Start the services in detached mode. +- `docker compose stop`: Stop the running services without removing them. - `docker compose rm`: Remove stopped service containers. - `docker compose build`: Build or rebuild services. - `docker compose down`: Stop and remove containers, networks, and volumes. diff --git a/autogpt_platform/backend/backend/blocks/llm.py b/autogpt_platform/backend/backend/blocks/llm.py index d9a5d4691872..e2bdfd8c2485 100644 --- a/autogpt_platform/backend/backend/blocks/llm.py +++ b/autogpt_platform/backend/backend/blocks/llm.py @@ -320,7 +320,7 @@ def llm_call(input_data: AIStructuredResponseGeneratorBlock.Input) -> str: if output_name == "response": return output_data["response"] else: - raise output_data + raise RuntimeError(output_data) raise ValueError("Failed to get a response from the LLM.") def run(self, input_data: Input) -> BlockOutput: diff --git a/autogpt_platform/backend/backend/blocks/time_blocks.py b/autogpt_platform/backend/backend/blocks/time_blocks.py index 32d92964ab78..ad59a54f2a4e 100644 --- a/autogpt_platform/backend/backend/blocks/time_blocks.py +++ b/autogpt_platform/backend/backend/blocks/time_blocks.py @@ -23,7 +23,7 @@ def __init__(self): {"trigger": "Hello", "format": "{time}"}, ], test_output=[ - ("time", time.strftime("%H:%M:%S")), + ("time", lambda _: time.strftime("%H:%M:%S")), ], ) diff --git a/autogpt_platform/backend/backend/data/execution.py b/autogpt_platform/backend/backend/data/execution.py index 4ab9be025f14..8e3449dd5c68 100644 --- a/autogpt_platform/backend/backend/data/execution.py +++ b/autogpt_platform/backend/backend/data/execution.py @@ -396,19 +396,19 @@ def merge_execution_input(data: BlockInput) -> BlockInput: # Merge all input with _$_ into a single list. items = list(data.items()) - list_input: list[Any] = [] + for key, value in items: if LIST_SPLIT not in key: continue name, index = key.split(LIST_SPLIT) if not index.isdigit(): - list_input.append((name, value, 0)) - else: - list_input.append((name, value, int(index))) + raise ValueError(f"Invalid key: {key}, #{index} index must be an integer.") - for name, value, _ in sorted(list_input, key=lambda x: x[2]): data[name] = data.get(name, []) - data[name].append(value) + if int(index) >= len(data[name]): + # Pad list with empty string on missing indices. + data[name].extend([""] * (int(index) - len(data[name]) + 1)) + data[name][int(index)] = value # Merge all input with _#_ into a single dict. for key, value in items: diff --git a/autogpt_platform/backend/backend/data/graph.py b/autogpt_platform/backend/backend/data/graph.py index c058547ba18f..bbe9afc237a8 100644 --- a/autogpt_platform/backend/backend/data/graph.py +++ b/autogpt_platform/backend/backend/data/graph.py @@ -1,17 +1,20 @@ import asyncio import logging import uuid +from datetime import datetime, timezone from pathlib import Path from typing import Any, Literal import prisma.types -from prisma.models import AgentGraph, AgentNode, AgentNodeLink +from prisma.models import AgentGraph, AgentGraphExecution, AgentNode, AgentNodeLink +from prisma.types import AgentGraphInclude from pydantic import BaseModel, PrivateAttr from pydantic_core import PydanticUndefinedType from backend.blocks.basic import AgentInputBlock, AgentOutputBlock from backend.data.block import BlockInput, get_block, get_blocks from backend.data.db import BaseDbModel, transaction +from backend.data.execution import ExecutionStatus from backend.data.user import DEFAULT_USER_ID from backend.util import json @@ -77,16 +80,57 @@ def from_db(node: AgentNode): return obj +class ExecutionMeta(BaseDbModel): + execution_id: str + started_at: datetime + ended_at: datetime + duration: float + total_run_time: float + status: ExecutionStatus + + @staticmethod + def from_agent_graph_execution(execution: AgentGraphExecution): + now = datetime.now(timezone.utc) + start_time = execution.startedAt or execution.createdAt + end_time = execution.updatedAt or now + duration = (end_time - start_time).total_seconds() + + total_run_time = 0 + if execution.AgentNodeExecutions: + for node_execution in execution.AgentNodeExecutions: + node_start = node_execution.startedTime or now + node_end = node_execution.endedTime or now + total_run_time += (node_end - node_start).total_seconds() + + return ExecutionMeta( + id=execution.id, + execution_id=execution.id, + started_at=start_time, + ended_at=end_time, + duration=duration, + total_run_time=total_run_time, + status=ExecutionStatus(execution.executionStatus), + ) + + class GraphMeta(BaseDbModel): version: int = 1 is_active: bool = True is_template: bool = False - name: str description: str + executions: list[ExecutionMeta] | None = None @staticmethod def from_db(graph: AgentGraph): + if graph.AgentGraphExecution: + executions = [ + ExecutionMeta.from_agent_graph_execution(execution) + for execution in graph.AgentGraphExecution + ] + else: + executions = None + return GraphMeta( id=graph.id, version=graph.version, @@ -94,6 +138,7 @@ def from_db(graph: AgentGraph): is_template=graph.isTemplate, name=graph.name or "", description=graph.description or "", + executions=executions, ) @@ -337,6 +382,7 @@ async def get_node(node_id: str) -> Node: async def get_graphs_meta( + include_executions: bool = False, filter_by: Literal["active", "template"] | None = "active", user_id: str | None = None, ) -> list[GraphMeta]: @@ -345,6 +391,7 @@ async def get_graphs_meta( Default behaviour is to get all currently active graphs. Args: + include_executions: Whether to include executions in the graph metadata. filter_by: An optional filter to either select templates or active graphs. Returns: @@ -364,6 +411,13 @@ async def get_graphs_meta( where=where_clause, distinct=["id"], order={"version": "desc"}, + include=( + AgentGraphInclude( + AgentGraphExecution={"include": {"AgentNodeExecutions": True}} + ) + if include_executions + else None + ), ) if not graphs: diff --git a/autogpt_platform/backend/backend/executor/manager.py b/autogpt_platform/backend/backend/executor/manager.py index 257d4cc8260f..c52086de6b9a 100644 --- a/autogpt_platform/backend/backend/executor/manager.py +++ b/autogpt_platform/backend/backend/executor/manager.py @@ -69,20 +69,28 @@ def __init__( self.prefix = f"[ExecutionManager|uid:{user_id}|gid:{graph_id}|nid:{node_id}]|geid:{graph_eid}|nid:{node_eid}|{block_name}]" def info(self, msg: str, **extra): + msg = self._wrap(msg, **extra) logger.info(msg, extra={"json_fields": {**self.metadata, **extra}}) def warning(self, msg: str, **extra): + msg = self._wrap(msg, **extra) logger.warning(msg, extra={"json_fields": {**self.metadata, **extra}}) def error(self, msg: str, **extra): + msg = self._wrap(msg, **extra) logger.error(msg, extra={"json_fields": {**self.metadata, **extra}}) def debug(self, msg: str, **extra): + msg = self._wrap(msg, **extra) logger.debug(msg, extra={"json_fields": {**self.metadata, **extra}}) def exception(self, msg: str, **extra): + msg = self._wrap(msg, **extra) logger.exception(msg, extra={"json_fields": {**self.metadata, **extra}}) + def _wrap(self, msg: str, **extra): + return f"{self.prefix} {msg} {extra}" + T = TypeVar("T") ExecutionStream = Generator[NodeExecution, None, None] diff --git a/autogpt_platform/backend/backend/server/rest_api.py b/autogpt_platform/backend/backend/server/rest_api.py index 9a765d15a84c..9f3afd2fba68 100644 --- a/autogpt_platform/backend/backend/server/rest_api.py +++ b/autogpt_platform/backend/backend/server/rest_api.py @@ -326,9 +326,13 @@ def execute_graph_block( @classmethod async def get_graphs( - cls, user_id: Annotated[str, Depends(get_user_id)] + cls, + user_id: Annotated[str, Depends(get_user_id)], + with_runs: bool = False, ) -> list[graph_db.GraphMeta]: - return await graph_db.get_graphs_meta(filter_by="active", user_id=user_id) + return await graph_db.get_graphs_meta( + include_executions=with_runs, filter_by="active", user_id=user_id + ) @classmethod async def get_templates(cls) -> list[graph_db.GraphMeta]: diff --git a/autogpt_platform/backend/pyproject.toml b/autogpt_platform/backend/pyproject.toml index d014d724675e..c7c3be2005dc 100644 --- a/autogpt_platform/backend/pyproject.toml +++ b/autogpt_platform/backend/pyproject.toml @@ -6,6 +6,7 @@ authors = ["AutoGPT "] readme = "README.md" packages = [{ include = "backend" }] + [tool.poetry.dependencies] python = "^3.10" aio-pika = "^9.4.3" @@ -67,40 +68,6 @@ cli = "backend.cli:main" format = "linter:format" lint = "linter:lint" test = "run_tests:test" -# https://poethepoet.natn.io/index.html - -[tool.poe] -poetry_command = "" - -# poetry run poe xxx -[tool.poe.tasks] -test = "pytest" -build = ["test", "_dbuild"] - -# This might break your python install :) -install = ["build", "_dinstall"] - -# https://cx-freeze.readthedocs.io/en/stable/index.html -[tool.poe.tasks._dbuild] -cmd = "python setup.py build" - -[tool.poe.tasks.dist_app] -cmd = "python setup.py bdist_app" - -[tool.poe.tasks.dist_dmg] -cmd = "python setup.py bdist_dmg" - -[tool.poe.tasks.dist_msi] -cmd = "python setup.py bdist_msi" - -[tool.poe.tasks.dist_appimage] -cmd = "python setup.py bdist_appimage" - -[tool.poe.tasks.dist_deb] -cmd = "python setup.py bdist_deb" - -[tool.poe.tasks._dinstall] -cmd = "python setup.py install" [tool.pytest-watcher] now = false diff --git a/autogpt_platform/docker-compose.combined.yml b/autogpt_platform/docker-compose.combined.yml deleted file mode 100644 index e53ca786c9bf..000000000000 --- a/autogpt_platform/docker-compose.combined.yml +++ /dev/null @@ -1,147 +0,0 @@ -version: '3.8' - -networks: - app-network: - name: app-network - shared-network: - name: shared-network - -volumes: - db-config: - -x-agpt-services: - &agpt-services - networks: - - app-network - - shared-network - -x-supabase-services: - &supabase-services - networks: - - app-network - - shared-network - -services: - # AGPT services - postgres: - <<: *agpt-services - extends: - file: ./docker-compose.yml - service: postgres - - migrate: - <<: *agpt-services - extends: - file: ./docker-compose.yml - service: migrate - - redis: - <<: *agpt-services - extends: - file: ./docker-compose.yml - service: redis - - rest_server: - <<: *agpt-services - extends: - file: ./docker-compose.yml - service: rest_server - - executor: - <<: *agpt-services - extends: - file: ./docker-compose.yml - service: executor - - websocket_server: - <<: *agpt-services - extends: - file: ./docker-compose.yml - service: websocket_server - - market: - <<: *agpt-services - extends: - file: ./docker-compose.yml - service: market - -# frontend: -# <<: *agpt-services -# extends: -# file: ./docker-compose.yml -# service: frontend - - # Supabase services - studio: - <<: *supabase-services - extends: - file: ./supabase/docker/docker-compose.yml - service: studio - - kong: - <<: *supabase-services - extends: - file: ./supabase/docker/docker-compose.yml - service: kong - - auth: - <<: *supabase-services - extends: - file: ./supabase/docker/docker-compose.yml - service: auth - environment: - GOTRUE_MAILER_AUTOCONFIRM: true - - rest: - <<: *supabase-services - extends: - file: ./supabase/docker/docker-compose.yml - service: rest - - realtime: - <<: *supabase-services - extends: - file: ./supabase/docker/docker-compose.yml - service: realtime - - storage: - <<: *supabase-services - extends: - file: ./supabase/docker/docker-compose.yml - service: storage - - imgproxy: - <<: *supabase-services - extends: - file: ./supabase/docker/docker-compose.yml - service: imgproxy - - meta: - <<: *supabase-services - extends: - file: ./supabase/docker/docker-compose.yml - service: meta - - functions: - <<: *supabase-services - extends: - file: ./supabase/docker/docker-compose.yml - service: functions - - analytics: - <<: *supabase-services - extends: - file: ./supabase/docker/docker-compose.yml - service: analytics - - db: - <<: *supabase-services - extends: - file: ./supabase/docker/docker-compose.yml - service: db - - vector: - <<: *supabase-services - extends: - file: ./supabase/docker/docker-compose.yml - service: vector diff --git a/autogpt_platform/docker-compose.platform.yml b/autogpt_platform/docker-compose.platform.yml new file mode 100644 index 000000000000..a0f76cd34350 --- /dev/null +++ b/autogpt_platform/docker-compose.platform.yml @@ -0,0 +1,210 @@ +services: + postgres: + image: ankane/pgvector:latest + environment: + - POSTGRES_USER=agpt_user + - POSTGRES_PASSWORD=pass123 + - POSTGRES_DB=agpt_local + healthcheck: + test: pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB + interval: 10s + timeout: 5s + retries: 5 + ports: + - "5433:5432" + networks: + - app-network + + migrate: + build: + context: ../ + dockerfile: autogpt_platform/backend/Dockerfile + target: server + command: ["sh", "-c", "poetry run prisma migrate deploy"] + develop: + watch: + - path: ./ + target: autogpt_platform/backend/migrate + action: rebuild + depends_on: + postgres: + condition: service_healthy + environment: + - DATABASE_URL=postgresql://agpt_user:pass123@postgres:5432/agpt_local?connect_timeout=60 + networks: + - app-network + restart: on-failure + healthcheck: + test: ["CMD", "poetry", "run", "prisma", "migrate", "status"] + interval: 10s + timeout: 5s + retries: 5 + + redis: + image: redis:latest + command: redis-server --requirepass password + ports: + - "6379:6379" + networks: + - app-network + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 5s + retries: 5 + + rest_server: + build: + context: ../ + dockerfile: autogpt_platform/backend/Dockerfile + target: server + command: ["python", "-m", "backend.rest"] + develop: + watch: + - path: ./ + target: autogpt_platform/backend/ + action: rebuild + depends_on: + redis: + condition: service_healthy + postgres: + condition: service_healthy + migrate: + condition: service_completed_successfully + environment: + - SUPABASE_URL=http://kong:8000 + - SUPABASE_JWT_SECRET=your-super-secret-jwt-token-with-at-least-32-characters-long + - SUPABASE_ANON_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyAgCiAgICAicm9sZSI6ICJhbm9uIiwKICAgICJpc3MiOiAic3VwYWJhc2UtZGVtbyIsCiAgICAiaWF0IjogMTY0MTc2OTIwMCwKICAgICJleHAiOiAxNzk5NTM1NjAwCn0.dc_X5iR_VP_qT0zsiyj_I_OZ2T9FtRU2BBNWN8Bu4GE + - DATABASE_URL=postgresql://agpt_user:pass123@postgres:5432/agpt_local?connect_timeout=60 + - REDIS_HOST=redis + - REDIS_PORT=6379 + - REDIS_PASSWORD=password + - ENABLE_AUTH=true + - PYRO_HOST=0.0.0.0 + - EXECUTIONMANAGER_HOST=executor + ports: + - "8006:8006" + - "8003:8003" # execution scheduler + networks: + - app-network + + executor: + build: + context: ../ + dockerfile: autogpt_platform/backend/Dockerfile + target: server + command: ["python", "-m", "backend.exec"] + develop: + watch: + - path: ./ + target: autogpt_platform/backend/ + action: rebuild + depends_on: + redis: + condition: service_healthy + postgres: + condition: service_healthy + migrate: + condition: service_completed_successfully + environment: + - NEXT_PUBLIC_SUPABASE_URL=http://kong:8000 + - SUPABASE_JWT_SECRET=your-super-secret-jwt-token-with-at-least-32-characters-long + - SUPABASE_ANON_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyAgCiAgICAicm9sZSI6ICJhbm9uIiwKICAgICJpc3MiOiAic3VwYWJhc2UtZGVtbyIsCiAgICAiaWF0IjogMTY0MTc2OTIwMCwKICAgICJleHAiOiAxNzk5NTM1NjAwCn0.dc_X5iR_VP_qT0zsiyj_I_OZ2T9FtRU2BBNWN8Bu4GE + - DATABASE_URL=postgresql://agpt_user:pass123@postgres:5432/agpt_local?connect_timeout=60 + - REDIS_HOST=redis + - REDIS_PORT=6379 + - REDIS_PASSWORD=password + - ENABLE_AUTH=true + - PYRO_HOST=0.0.0.0 + - AGENTSERVER_HOST=rest_server + ports: + - "8002:8000" + networks: + - app-network + + websocket_server: + build: + context: ../ + dockerfile: autogpt_platform/backend/Dockerfile + target: server + command: ["python", "-m", "backend.ws"] + develop: + watch: + - path: ./ + target: autogpt_platform/backend/ + action: rebuild + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + migrate: + condition: service_completed_successfully + environment: + - SUPABASE_URL=http://kong:8000 + - SUPABASE_JWT_SECRET=your-super-secret-jwt-token-with-at-least-32-characters-long + - SUPABASE_ANON_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyAgCiAgICAicm9sZSI6ICJhbm9uIiwKICAgICJpc3MiOiAic3VwYWJhc2UtZGVtbyIsCiAgICAiaWF0IjogMTY0MTc2OTIwMCwKICAgICJleHAiOiAxNzk5NTM1NjAwCn0.dc_X5iR_VP_qT0zsiyj_I_OZ2T9FtRU2BBNWN8Bu4GE + - DATABASE_URL=postgresql://agpt_user:pass123@postgres:5432/agpt_local?connect_timeout=60 + - REDIS_HOST=redis + - REDIS_PORT=6379 + - REDIS_PASSWORD=password + - ENABLE_AUTH=true + - PYRO_HOST=0.0.0.0 + ports: + - "8001:8001" + networks: + - app-network + + market: + build: + context: ../ + dockerfile: autogpt_platform/market/Dockerfile + develop: + watch: + - path: ./ + target: autogpt_platform/market/ + action: rebuild + depends_on: + postgres: + condition: service_healthy + migrate: + condition: service_completed_successfully + environment: + - SUPABASE_URL=http://kong:8000 + - SUPABASE_JWT_SECRET=your-super-secret-jwt-token-with-at-least-32-characters-long + - SUPABASE_ANON_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyAgCiAgICAicm9sZSI6ICJhbm9uIiwKICAgICJpc3MiOiAic3VwYWJhc2UtZGVtbyIsCiAgICAiaWF0IjogMTY0MTc2OTIwMCwKICAgICJleHAiOiAxNzk5NTM1NjAwCn0.dc_X5iR_VP_qT0zsiyj_I_OZ2T9FtRU2BBNWN8Bu4GE + - DATABASE_URL=postgresql://agpt_user:pass123@postgres:5432/agpt_local?connect_timeout=60&schema=market + ports: + - "8015:8015" + networks: + - app-network + +# frontend: +# build: +# context: ../ +# dockerfile: autogpt_platform/frontend/Dockerfile +# target: dev +# depends_on: +# postgres: +# condition: service_healthy +# rest_server: +# condition: service_started +# websocket_server: +# condition: service_started +# migrate: +# condition: service_completed_successfully +# environment: +# - NEXT_PUBLIC_SUPABASE_URL=http://kong:8000 +# - NEXT_PUBLIC_SUPABASE_ANON_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyAgCiAgICAicm9sZSI6ICJhbm9uIiwKICAgICJpc3MiOiAic3VwYWJhc2UtZGVtbyIsCiAgICAiaWF0IjogMTY0MTc2OTIwMCwKICAgICJleHAiOiAxNzk5NTM1NjAwCn0.dc_X5iR_VP_qT0zsiyj_I_OZ2T9FtRU2BBNWN8Bu4GE +# - DATABASE_URL=postgresql://agpt_user:pass123@postgres:5432/agpt_local?connect_timeout=60 +# - NEXT_PUBLIC_AGPT_SERVER_URL=http://localhost:8006/api +# - NEXT_PUBLIC_AGPT_WS_SERVER_URL=ws://localhost:8001/ws +# - NEXT_PUBLIC_AGPT_MARKETPLACE_URL=http://localhost:8015/api/v1/market +# ports: +# - "3000:3000" +# networks: +# - app-network + +networks: + app-network: + driver: bridge diff --git a/autogpt_platform/docker-compose.yml b/autogpt_platform/docker-compose.yml index a0f76cd34350..305d4104a642 100644 --- a/autogpt_platform/docker-compose.yml +++ b/autogpt_platform/docker-compose.yml @@ -1,210 +1,147 @@ +version: '3.8' + +networks: + app-network: + name: app-network + shared-network: + name: shared-network + +volumes: + db-config: + +x-agpt-services: + &agpt-services + networks: + - app-network + - shared-network + +x-supabase-services: + &supabase-services + networks: + - app-network + - shared-network + services: + # AGPT services postgres: - image: ankane/pgvector:latest - environment: - - POSTGRES_USER=agpt_user - - POSTGRES_PASSWORD=pass123 - - POSTGRES_DB=agpt_local - healthcheck: - test: pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB - interval: 10s - timeout: 5s - retries: 5 - ports: - - "5433:5432" - networks: - - app-network + <<: *agpt-services + extends: + file: ./docker-compose.platform.yml + service: postgres migrate: - build: - context: ../ - dockerfile: autogpt_platform/backend/Dockerfile - target: server - command: ["sh", "-c", "poetry run prisma migrate deploy"] - develop: - watch: - - path: ./ - target: autogpt_platform/backend/migrate - action: rebuild - depends_on: - postgres: - condition: service_healthy - environment: - - DATABASE_URL=postgresql://agpt_user:pass123@postgres:5432/agpt_local?connect_timeout=60 - networks: - - app-network - restart: on-failure - healthcheck: - test: ["CMD", "poetry", "run", "prisma", "migrate", "status"] - interval: 10s - timeout: 5s - retries: 5 + <<: *agpt-services + extends: + file: ./docker-compose.platform.yml + service: migrate redis: - image: redis:latest - command: redis-server --requirepass password - ports: - - "6379:6379" - networks: - - app-network - healthcheck: - test: ["CMD", "redis-cli", "ping"] - interval: 10s - timeout: 5s - retries: 5 + <<: *agpt-services + extends: + file: ./docker-compose.platform.yml + service: redis rest_server: - build: - context: ../ - dockerfile: autogpt_platform/backend/Dockerfile - target: server - command: ["python", "-m", "backend.rest"] - develop: - watch: - - path: ./ - target: autogpt_platform/backend/ - action: rebuild - depends_on: - redis: - condition: service_healthy - postgres: - condition: service_healthy - migrate: - condition: service_completed_successfully - environment: - - SUPABASE_URL=http://kong:8000 - - SUPABASE_JWT_SECRET=your-super-secret-jwt-token-with-at-least-32-characters-long - - SUPABASE_ANON_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyAgCiAgICAicm9sZSI6ICJhbm9uIiwKICAgICJpc3MiOiAic3VwYWJhc2UtZGVtbyIsCiAgICAiaWF0IjogMTY0MTc2OTIwMCwKICAgICJleHAiOiAxNzk5NTM1NjAwCn0.dc_X5iR_VP_qT0zsiyj_I_OZ2T9FtRU2BBNWN8Bu4GE - - DATABASE_URL=postgresql://agpt_user:pass123@postgres:5432/agpt_local?connect_timeout=60 - - REDIS_HOST=redis - - REDIS_PORT=6379 - - REDIS_PASSWORD=password - - ENABLE_AUTH=true - - PYRO_HOST=0.0.0.0 - - EXECUTIONMANAGER_HOST=executor - ports: - - "8006:8006" - - "8003:8003" # execution scheduler - networks: - - app-network + <<: *agpt-services + extends: + file: ./docker-compose.platform.yml + service: rest_server executor: - build: - context: ../ - dockerfile: autogpt_platform/backend/Dockerfile - target: server - command: ["python", "-m", "backend.exec"] - develop: - watch: - - path: ./ - target: autogpt_platform/backend/ - action: rebuild - depends_on: - redis: - condition: service_healthy - postgres: - condition: service_healthy - migrate: - condition: service_completed_successfully - environment: - - NEXT_PUBLIC_SUPABASE_URL=http://kong:8000 - - SUPABASE_JWT_SECRET=your-super-secret-jwt-token-with-at-least-32-characters-long - - SUPABASE_ANON_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyAgCiAgICAicm9sZSI6ICJhbm9uIiwKICAgICJpc3MiOiAic3VwYWJhc2UtZGVtbyIsCiAgICAiaWF0IjogMTY0MTc2OTIwMCwKICAgICJleHAiOiAxNzk5NTM1NjAwCn0.dc_X5iR_VP_qT0zsiyj_I_OZ2T9FtRU2BBNWN8Bu4GE - - DATABASE_URL=postgresql://agpt_user:pass123@postgres:5432/agpt_local?connect_timeout=60 - - REDIS_HOST=redis - - REDIS_PORT=6379 - - REDIS_PASSWORD=password - - ENABLE_AUTH=true - - PYRO_HOST=0.0.0.0 - - AGENTSERVER_HOST=rest_server - ports: - - "8002:8000" - networks: - - app-network + <<: *agpt-services + extends: + file: ./docker-compose.platform.yml + service: executor websocket_server: - build: - context: ../ - dockerfile: autogpt_platform/backend/Dockerfile - target: server - command: ["python", "-m", "backend.ws"] - develop: - watch: - - path: ./ - target: autogpt_platform/backend/ - action: rebuild - depends_on: - postgres: - condition: service_healthy - redis: - condition: service_healthy - migrate: - condition: service_completed_successfully - environment: - - SUPABASE_URL=http://kong:8000 - - SUPABASE_JWT_SECRET=your-super-secret-jwt-token-with-at-least-32-characters-long - - SUPABASE_ANON_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyAgCiAgICAicm9sZSI6ICJhbm9uIiwKICAgICJpc3MiOiAic3VwYWJhc2UtZGVtbyIsCiAgICAiaWF0IjogMTY0MTc2OTIwMCwKICAgICJleHAiOiAxNzk5NTM1NjAwCn0.dc_X5iR_VP_qT0zsiyj_I_OZ2T9FtRU2BBNWN8Bu4GE - - DATABASE_URL=postgresql://agpt_user:pass123@postgres:5432/agpt_local?connect_timeout=60 - - REDIS_HOST=redis - - REDIS_PORT=6379 - - REDIS_PASSWORD=password - - ENABLE_AUTH=true - - PYRO_HOST=0.0.0.0 - ports: - - "8001:8001" - networks: - - app-network + <<: *agpt-services + extends: + file: ./docker-compose.platform.yml + service: websocket_server market: - build: - context: ../ - dockerfile: autogpt_platform/market/Dockerfile - develop: - watch: - - path: ./ - target: autogpt_platform/market/ - action: rebuild - depends_on: - postgres: - condition: service_healthy - migrate: - condition: service_completed_successfully - environment: - - SUPABASE_URL=http://kong:8000 - - SUPABASE_JWT_SECRET=your-super-secret-jwt-token-with-at-least-32-characters-long - - SUPABASE_ANON_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyAgCiAgICAicm9sZSI6ICJhbm9uIiwKICAgICJpc3MiOiAic3VwYWJhc2UtZGVtbyIsCiAgICAiaWF0IjogMTY0MTc2OTIwMCwKICAgICJleHAiOiAxNzk5NTM1NjAwCn0.dc_X5iR_VP_qT0zsiyj_I_OZ2T9FtRU2BBNWN8Bu4GE - - DATABASE_URL=postgresql://agpt_user:pass123@postgres:5432/agpt_local?connect_timeout=60&schema=market - ports: - - "8015:8015" - networks: - - app-network + <<: *agpt-services + extends: + file: ./docker-compose.platform.yml + service: market # frontend: -# build: -# context: ../ -# dockerfile: autogpt_platform/frontend/Dockerfile -# target: dev -# depends_on: -# postgres: -# condition: service_healthy -# rest_server: -# condition: service_started -# websocket_server: -# condition: service_started -# migrate: -# condition: service_completed_successfully -# environment: -# - NEXT_PUBLIC_SUPABASE_URL=http://kong:8000 -# - NEXT_PUBLIC_SUPABASE_ANON_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyAgCiAgICAicm9sZSI6ICJhbm9uIiwKICAgICJpc3MiOiAic3VwYWJhc2UtZGVtbyIsCiAgICAiaWF0IjogMTY0MTc2OTIwMCwKICAgICJleHAiOiAxNzk5NTM1NjAwCn0.dc_X5iR_VP_qT0zsiyj_I_OZ2T9FtRU2BBNWN8Bu4GE -# - DATABASE_URL=postgresql://agpt_user:pass123@postgres:5432/agpt_local?connect_timeout=60 -# - NEXT_PUBLIC_AGPT_SERVER_URL=http://localhost:8006/api -# - NEXT_PUBLIC_AGPT_WS_SERVER_URL=ws://localhost:8001/ws -# - NEXT_PUBLIC_AGPT_MARKETPLACE_URL=http://localhost:8015/api/v1/market -# ports: -# - "3000:3000" -# networks: -# - app-network +# <<: *agpt-services +# extends: +# file: ./docker-compose.platform.yml +# service: frontend -networks: - app-network: - driver: bridge + # Supabase services + studio: + <<: *supabase-services + extends: + file: ./supabase/docker/docker-compose.yml + service: studio + + kong: + <<: *supabase-services + extends: + file: ./supabase/docker/docker-compose.yml + service: kong + + auth: + <<: *supabase-services + extends: + file: ./supabase/docker/docker-compose.yml + service: auth + environment: + GOTRUE_MAILER_AUTOCONFIRM: true + + rest: + <<: *supabase-services + extends: + file: ./supabase/docker/docker-compose.yml + service: rest + + realtime: + <<: *supabase-services + extends: + file: ./supabase/docker/docker-compose.yml + service: realtime + + storage: + <<: *supabase-services + extends: + file: ./supabase/docker/docker-compose.yml + service: storage + + imgproxy: + <<: *supabase-services + extends: + file: ./supabase/docker/docker-compose.yml + service: imgproxy + + meta: + <<: *supabase-services + extends: + file: ./supabase/docker/docker-compose.yml + service: meta + + functions: + <<: *supabase-services + extends: + file: ./supabase/docker/docker-compose.yml + service: functions + + analytics: + <<: *supabase-services + extends: + file: ./supabase/docker/docker-compose.yml + service: analytics + + db: + <<: *supabase-services + extends: + file: ./supabase/docker/docker-compose.yml + service: db + + vector: + <<: *supabase-services + extends: + file: ./supabase/docker/docker-compose.yml + service: vector diff --git a/autogpt_platform/frontend/package.json b/autogpt_platform/frontend/package.json index c9b7e2dc72c6..5f9b5cad686e 100644 --- a/autogpt_platform/frontend/package.json +++ b/autogpt_platform/frontend/package.json @@ -3,7 +3,8 @@ "version": "0.1.0", "private": true, "scripts": { - "dev": "next dev", + "dev": "export NODE_ENV=development && next dev", + "dev:test": "export NODE_ENV=test && next dev", "build": "next build", "start": "next start", "lint": "next lint", diff --git a/autogpt_platform/frontend/sentry.client.config.ts b/autogpt_platform/frontend/sentry.client.config.ts index aad53ebbb71b..f37d5cda2394 100644 --- a/autogpt_platform/frontend/sentry.client.config.ts +++ b/autogpt_platform/frontend/sentry.client.config.ts @@ -7,6 +7,8 @@ import * as Sentry from "@sentry/nextjs"; Sentry.init({ dsn: "https://fe4e4aa4a283391808a5da396da20159@o4505260022104064.ingest.us.sentry.io/4507946746380288", + enabled: process.env.NODE_ENV !== "development", + // Add optional integrations for additional features integrations: [ Sentry.replayIntegration(), diff --git a/autogpt_platform/frontend/sentry.edge.config.ts b/autogpt_platform/frontend/sentry.edge.config.ts index 09903cfad599..8a566e17b783 100644 --- a/autogpt_platform/frontend/sentry.edge.config.ts +++ b/autogpt_platform/frontend/sentry.edge.config.ts @@ -8,6 +8,8 @@ import * as Sentry from "@sentry/nextjs"; Sentry.init({ dsn: "https://fe4e4aa4a283391808a5da396da20159@o4505260022104064.ingest.us.sentry.io/4507946746380288", + enabled: process.env.NODE_ENV !== "development", + // Define how likely traces are sampled. Adjust this value in production, or use tracesSampler for greater control. tracesSampleRate: 1, diff --git a/autogpt_platform/frontend/sentry.server.config.ts b/autogpt_platform/frontend/sentry.server.config.ts index db0cf30751fc..20f0df5a3983 100644 --- a/autogpt_platform/frontend/sentry.server.config.ts +++ b/autogpt_platform/frontend/sentry.server.config.ts @@ -8,6 +8,8 @@ import * as Sentry from "@sentry/nextjs"; Sentry.init({ dsn: "https://fe4e4aa4a283391808a5da396da20159@o4505260022104064.ingest.us.sentry.io/4507946746380288", + enabled: process.env.NODE_ENV !== "development", + // Define how likely traces are sampled. Adjust this value in production, or use tracesSampler for greater control. tracesSampleRate: 1, diff --git a/autogpt_platform/frontend/src/app/page.tsx b/autogpt_platform/frontend/src/app/page.tsx index e76636fc6bc9..7924883160a3 100644 --- a/autogpt_platform/frontend/src/app/page.tsx +++ b/autogpt_platform/frontend/src/app/page.tsx @@ -2,8 +2,8 @@ import React, { useCallback, useEffect, useMemo, useState } from "react"; import AutoGPTServerAPI, { - GraphMeta, - NodeExecutionResult, + GraphMetaWithRuns, + ExecutionMeta, } from "@/lib/autogpt-server-api"; import { Card } from "@/components/ui/card"; @@ -17,62 +17,37 @@ import { } from "@/components/monitor"; const Monitor = () => { - const [flows, setFlows] = useState([]); + const [flows, setFlows] = useState([]); const [flowRuns, setFlowRuns] = useState([]); - const [selectedFlow, setSelectedFlow] = useState(null); + const [selectedFlow, setSelectedFlow] = useState( + null, + ); const [selectedRun, setSelectedRun] = useState(null); const api = useMemo(() => new AutoGPTServerAPI(), []); - const refreshFlowRuns = useCallback( - (flowID: string) => { - // Fetch flow run IDs - api.listGraphRunIDs(flowID).then((runIDs) => - runIDs.map((runID) => { - let run; - if ( - (run = flowRuns.find((fr) => fr.id == runID)) && - !["waiting", "running"].includes(run.status) - ) { - return; - } - - // Fetch flow run - api.getGraphExecutionInfo(flowID, runID).then((execInfo) => - setFlowRuns((flowRuns) => { - if (execInfo.length == 0) return flowRuns; - - const flowRunIndex = flowRuns.findIndex((fr) => fr.id == runID); - const flowRun = flowRunFromNodeExecutionResults(execInfo); - if (flowRunIndex > -1) { - flowRuns.splice(flowRunIndex, 1, flowRun); - } else { - flowRuns.push(flowRun); - } - return [...flowRuns]; - }), - ); - }), + const fetchAgents = useCallback(() => { + api.listGraphsWithRuns().then((agent) => { + setFlows(agent); + const flowRuns = agent.flatMap((graph) => + graph.executions != null + ? graph.executions.map((execution) => + flowRunFromExecutionMeta(graph, execution), + ) + : [], ); - }, - [api, flowRuns], - ); - - const fetchFlowsAndRuns = useCallback(() => { - api.listGraphs().then((flows) => { - setFlows(flows); - flows.map((flow) => refreshFlowRuns(flow.id)); + setFlowRuns(flowRuns); }); - }, [api, refreshFlowRuns]); + }, [api]); - useEffect(() => fetchFlowsAndRuns(), [fetchFlowsAndRuns]); useEffect(() => { - const intervalId = setInterval( - () => flows.map((f) => refreshFlowRuns(f.id)), - 5000, - ); + fetchAgents(); + }, [api, fetchAgents]); + + useEffect(() => { + const intervalId = setInterval(() => fetchAgents(), 5000); return () => clearInterval(intervalId); - }, [flows, refreshFlowRuns]); + }, [fetchAgents, flows]); const column1 = "md:col-span-2 xl:col-span-3 xxl:col-span-2"; const column2 = "md:col-span-3 lg:col-span-2 xl:col-span-3 space-y-4"; @@ -87,7 +62,9 @@ const Monitor = () => { selectedFlow={selectedFlow} onSelectFlow={(f) => { setSelectedRun(null); - setSelectedFlow(f.id == selectedFlow?.id ? null : f); + setSelectedFlow( + f.id == selectedFlow?.id ? null : (f as GraphMetaWithRuns), + ); }} /> { ); }; -function flowRunFromNodeExecutionResults( - nodeExecutionResults: NodeExecutionResult[], +function flowRunFromExecutionMeta( + graphMeta: GraphMetaWithRuns, + executionMeta: ExecutionMeta, ): FlowRun { - // Determine overall status - let status: "running" | "waiting" | "success" | "failed" = "success"; - for (const execution of nodeExecutionResults) { - if (execution.status === "FAILED") { - status = "failed"; - break; - } else if (["QUEUED", "RUNNING"].includes(execution.status)) { - status = "running"; - break; - } else if (execution.status === "INCOMPLETE") { - status = "waiting"; - } - } - - // Determine aggregate startTime, endTime, and totalRunTime - const now = Date.now(); - const startTime = Math.min( - ...nodeExecutionResults.map((ner) => ner.add_time.getTime()), - now, - ); - const endTime = ["success", "failed"].includes(status) - ? Math.max( - ...nodeExecutionResults.map((ner) => ner.end_time?.getTime() || 0), - startTime, - ) - : now; - const duration = (endTime - startTime) / 1000; // Convert to seconds - const totalRunTime = - nodeExecutionResults.reduce( - (cum, node) => - cum + - ((node.end_time?.getTime() ?? now) - - (node.start_time?.getTime() ?? now)), - 0, - ) / 1000; - return { - id: nodeExecutionResults[0].graph_exec_id, - graphID: nodeExecutionResults[0].graph_id, - graphVersion: nodeExecutionResults[0].graph_version, - status, - startTime, - endTime, - duration, - totalRunTime, - nodeExecutionResults: nodeExecutionResults, - }; + id: executionMeta.execution_id, + graphID: graphMeta.id, + graphVersion: graphMeta.version, + status: executionMeta.status, + startTime: executionMeta.started_at, + endTime: executionMeta.ended_at, + duration: executionMeta.duration, + totalRunTime: executionMeta.total_run_time, + } as FlowRun; } export default Monitor; diff --git a/autogpt_platform/frontend/src/components/ProfileDropdown.tsx b/autogpt_platform/frontend/src/components/ProfileDropdown.tsx index 4da31b9ae7a8..98d4ad8c0fc6 100644 --- a/autogpt_platform/frontend/src/components/ProfileDropdown.tsx +++ b/autogpt_platform/frontend/src/components/ProfileDropdown.tsx @@ -42,7 +42,11 @@ const ProfileDropdown = () => { Admin Dashboard )} - supabase?.auth.signOut()}> + + supabase?.auth.signOut().then(() => router.replace("/login")) + } + > Log out diff --git a/autogpt_platform/frontend/src/lib/autogpt-server-api/baseClient.ts b/autogpt_platform/frontend/src/lib/autogpt-server-api/baseClient.ts index eb7c6a403777..f1293ed10986 100644 --- a/autogpt_platform/frontend/src/lib/autogpt-server-api/baseClient.ts +++ b/autogpt_platform/frontend/src/lib/autogpt-server-api/baseClient.ts @@ -5,7 +5,9 @@ import { GraphCreatable, GraphUpdateable, GraphMeta, + GraphMetaWithRuns, GraphExecuteResponse, + ExecutionMeta, NodeExecutionResult, User, AnalyticsMetrics, @@ -45,7 +47,12 @@ export default class BaseAutoGPTServerAPI { } async listGraphs(): Promise { - return this._get("/graphs"); + return this._get(`/graphs`); + } + + async listGraphsWithRuns(): Promise { + let graphs = await this._get(`/graphs?with_runs=true`); + return graphs.map(parseGraphMetaWithRuns); } async listTemplates(): Promise { @@ -195,6 +202,15 @@ export default class BaseAutoGPTServerAPI { response_data.detail, response, ); + + if ( + response.status === 403 && + response_data.detail === "Not authenticated" && + window // Browser environment only: redirect to login page. + ) { + window.location.href = "/login"; + } + throw new Error(`HTTP error ${response.status}! ${response_data.detail}`); } return response_data; @@ -319,3 +335,28 @@ function parseNodeExecutionResultTimestamps(result: any): NodeExecutionResult { end_time: result.end_time ? new Date(result.end_time) : undefined, }; } + +function parseGraphMetaWithRuns(result: any): GraphMetaWithRuns { + return { + ...result, + executions: result.executions.map(parseExecutionMetaTimestamps), + }; +} + +function parseExecutionMetaTimestamps(result: any): ExecutionMeta { + let status: "running" | "waiting" | "success" | "failed" = "success"; + if (result.status === "FAILED") { + status = "failed"; + } else if (["QUEUED", "RUNNING"].includes(result.status)) { + status = "running"; + } else if (result.status === "INCOMPLETE") { + status = "waiting"; + } + + return { + ...result, + status, + started_at: new Date(result.started_at).getTime(), + ended_at: result.ended_at ? new Date(result.ended_at).getTime() : undefined, + }; +} diff --git a/autogpt_platform/frontend/src/lib/autogpt-server-api/types.ts b/autogpt_platform/frontend/src/lib/autogpt-server-api/types.ts index b5a326745f7b..df9f691ceba1 100644 --- a/autogpt_platform/frontend/src/lib/autogpt-server-api/types.ts +++ b/autogpt_platform/frontend/src/lib/autogpt-server-api/types.ts @@ -139,6 +139,16 @@ export type LinkCreatable = Omit & { id?: string; }; +/* Mirror of autogpt_server/data/graph.py:ExecutionMeta */ +export type ExecutionMeta = { + execution_id: string; + started_at: number; + ended_at: number; + duration: number; + total_run_time: number; + status: "running" | "waiting" | "success" | "failed"; +}; + /* Mirror of backend/data/graph.py:GraphMeta */ export type GraphMeta = { id: string; @@ -149,6 +159,10 @@ export type GraphMeta = { description: string; }; +export type GraphMetaWithRuns = GraphMeta & { + executions: ExecutionMeta[]; +}; + /* Mirror of backend/data/graph.py:Graph */ export type Graph = GraphMeta & { nodes: Array; diff --git a/autogpt_platform/frontend/src/lib/utils.ts b/autogpt_platform/frontend/src/lib/utils.ts index 053724599d3d..8852dc4f42a1 100644 --- a/autogpt_platform/frontend/src/lib/utils.ts +++ b/autogpt_platform/frontend/src/lib/utils.ts @@ -152,13 +152,13 @@ export function setNestedProperty(obj: any, path: string, value: any) { export function removeEmptyStringsAndNulls(obj: any): any { if (Array.isArray(obj)) { - // If obj is an array, recursively remove empty strings and nulls from its elements - return obj - .map((item) => removeEmptyStringsAndNulls(item)) - .filter( - (item) => - item !== null && (typeof item !== "string" || item.trim() !== ""), - ); + // If obj is an array, recursively check each element, + // but element removal is avoided to prevent index changes. + return obj.map((item) => + item === undefined || item === null + ? "" + : removeEmptyStringsAndNulls(item), + ); } else if (typeof obj === "object" && obj !== null) { // If obj is an object, recursively remove empty strings and nulls from its properties for (const key in obj) { @@ -166,7 +166,8 @@ export function removeEmptyStringsAndNulls(obj: any): any { const value = obj[key]; if ( value === null || - (typeof value === "string" && value.trim() === "") + value === undefined || + (typeof value === "string" && value === "") ) { delete obj[key]; } else { diff --git a/classic/.dockerignore b/classic/.dockerignore deleted file mode 100644 index 38497eef3a17..000000000000 --- a/classic/.dockerignore +++ /dev/null @@ -1,28 +0,0 @@ -# Ignore everything by default, selectively add things to context -* - -# AutoGPT -!original_autogpt/autogpt/ -!original_autogpt/pyproject.toml -!original_autogpt/poetry.lock -!original_autogpt/README.md -!original_autogpt/tests/ - -# Benchmark -!benchmark/agbenchmark/ -!benchmark/pyproject.toml -!benchmark/poetry.lock -!benchmark/README.md - -# Forge -!forge/ -!forge/pyproject.toml -!forge/poetry.lock -!forge/README.md - -# Frontend -!frontend/build/web/ - -# Explicitly re-ignore some folders -.* -**/__pycache__ diff --git a/docs/content/server/setup.md b/docs/content/server/setup.md index c821964d6ce9..cf850cfae2d7 100644 --- a/docs/content/server/setup.md +++ b/docs/content/server/setup.md @@ -116,8 +116,8 @@ To run the server, navigate back to rnd (cd..) and run the following commands in ```bash cp supabase/docker/.env.example .env -docker compose -f docker-compose.combined.yml build -docker compose -f docker-compose.combined.yml up -d +docker compose build +docker compose up -d ``` In the other terminal from frontend, you can run the following command to start the frontend: