From 2c1484b8365515a4de0dab5224d7f2df41ec35df Mon Sep 17 00:00:00 2001 From: jessedrelick Date: Fri, 23 Aug 2024 23:47:47 -0400 Subject: [PATCH] v0.1.0 (#1) * init --- .github/workflows/main.yml | 28 +++ LICENSE | 202 +++++++++++++++++ README.md | 212 +++++++++++++++++- config/config.exs | 22 ++ coveralls.json | 7 + lib/agens.ex | 177 ++++++++++++++- lib/agens/agent.ex | 126 +++++++++++ lib/agens/job.ex | 318 +++++++++++++++++++++++++++ lib/agens/serving.ex | 115 ++++++++++ lib/agens/supervisor.ex | 49 +++++ lib/agens/tool.ex | 41 ++++ mix.exs | 90 +++++++- mix.lock | 24 +++ test/agens/agent_test.exs | 164 ++++++++++++++ test/agens/job_test.exs | 369 ++++++++++++++++++++++++++++++++ test/agens/serving_test.exs | 98 +++++++++ test/agens/tool_test.exs | 32 +++ test/agens_test.exs | 10 +- test/support/agent_case.ex | 56 +++++ test/support/helpers.ex | 51 +++++ test/support/serving.ex | 45 ++++ test/support/tools/noop_tool.ex | 18 ++ test/test_helper.exs | 2 +- 23 files changed, 2223 insertions(+), 33 deletions(-) create mode 100644 .github/workflows/main.yml create mode 100644 LICENSE create mode 100644 config/config.exs create mode 100644 coveralls.json create mode 100644 lib/agens/agent.ex create mode 100644 lib/agens/job.ex create mode 100644 lib/agens/serving.ex create mode 100644 lib/agens/supervisor.ex create mode 100644 lib/agens/tool.ex create mode 100644 test/agens/agent_test.exs create mode 100644 test/agens/job_test.exs create mode 100644 test/agens/serving_test.exs create mode 100644 test/agens/tool_test.exs create mode 100644 test/support/agent_case.ex create mode 100644 test/support/helpers.ex create mode 100644 test/support/serving.ex create mode 100644 test/support/tools/noop_tool.ex diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..1938761 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,28 @@ +name: build + +on: + push: + branches: + - main + pull_request: + branches: + - main + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: erlef/setup-beam@v1 + with: + otp-version: '27.0' + elixir-version: '1.17.2' + - run: mix deps.get + - run: mix compile --warnings-as-errors --force + - run: mix format --check-formatted + - run: mix dialyzer + - run: mix coveralls.json + - name: Codecov + uses: codecov/codecov-action@v3 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..3473570 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2024 Jesse Drelick + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md index cf98daf..1232efc 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,24 @@ -# Agens +![](https://github.com/jessedrelick/agens/actions/workflows/main.yml/badge.svg) +[![Hexdocs](https://img.shields.io/badge/hex-docs-blue.svg)](https://hexdocs.pm/agens) +[![Hex.pm](https://img.shields.io/hexpm/v/yourrepo.svg)](https://hex.pm/packages/agens) +[![codecov](https://codecov.io/gh/jessedrelick/agens/graph/badge.svg?token=KTJXB4SGCJ)](https://codecov.io/gh/jessedrelick/agens) -**TODO: Add description** +__Agens__ is an Elixir application designed to build multi-agent workflows with language models. -## Installation +Drawing inspiration from popular tools in the Python ecosystem, such as [LangChain](https://www.langchain.com/)/[LangGraph](https://www.langchain.com/langgraph) and [CrewAI](https://www.crewai.com/), __Agens__ showcases Elixir’s unique strengths in multi-agent workflows. While the ML/AI landscape is dominated by Python, Elixir’s use of the BEAM virtual machine and OTP (Open Telecom Platform), specifically GenServers and Supervisors, makes it particularly well-suited for these tasks. Agens aims to demonstrate how these inherent design features can be leveraged effectively. + +By combining Agens with powerful Elixir libraries like [Bumblebee](https://github.com/elixir-nx/bumblebee) and [Nx.Serving](https://hexdocs.pm/nx/Nx.Serving.html), along with [structured outputs in the OpenAI API](https://openai.com/index/introducing-structured-outputs-in-the-api/) and the continuous improvement of open-source language models, the reliance on Python for multi-agent workflows is significantly reduced. This shift allows Elixir’s concurrency model to truly shine. -If [available in Hex](https://hex.pm/docs/publish), the package can be installed -by adding `agens` to your list of dependencies in `mix.exs`: +> **⚠️ Experimental:** v0.1 +> +> Agens is currently an experimental project. As of version 0.1, it is primarily a proof-of-concept and learning tool. +> +> The next phase of the project focuses on developing real-world examples to uncover potential issues or gaps that the current test suite may not address. +> +> These examples are designed to not only help you get started but also to advance Agens towards becoming a production-ready tool, suitable for integration into new or existing Elixir applications. + +## Installation +Add `agens` to your list of dependencies in `mix.exs`: ```elixir def deps do @@ -15,7 +28,190 @@ def deps do end ``` -Documentation can be generated with [ExDoc](https://github.com/elixir-lang/ex_doc) -and published on [HexDocs](https://hexdocs.pm). Once published, the docs can -be found at . +## Configuration +Future versions of Agens will be configurable by providing options to `Agens.Supervisor` in order to [avoid using application configuration](https://hexdocs.pm/elixir/1.17.2/design-anti-patterns.html#using-application-configuration-for-libraries). For now, however, you can change the Agens `Registry` if needed via config: + +```elixir +config :agens, registry: Agens.Registry +``` + +In addition, you can also override the prompt prefixes: + +```elixir +config :agens, prompts: %{ + prompt: + {"Agent", + "You are a specialized agent with the following capabilities and expertise"}, + identity: + {"Identity", + "You are a specialized agent with the following capabilities and expertise"}, + context: {"Context", "The purpose or goal behind your tasks are to"}, + constraints: + {"Constraints", "You must operate with the following constraints or limitations"}, + examples: + {"Examples", + "You should consider the following examples before returning results"}, + reflection: + {"Reflection", + "You should reflect on the following factors before returning results"}, + instructions: + {"Tool Instructions", + "You should provide structured output for function calling based on the following instructions"}, + objective: {"Step Objective", "The objective of this step is to"}, + description: + {"Job Description", "This is part of multi-step job to achieve the following"}, + input: + {"Input", + "The following is the actual input from the user, system or another agent"} +} +``` + +See the [Prompting](#prompting) section below or `Agens.Message` for more information on prompt prefixes. + +## Usage +Building a multi-agent workflow with Agens involves a few different steps and core entities: + +--- +**1. Add the Agens Supervisor to your Supervision tree** + +This will start Agens as a supervised process inside your application: + +```elixir +Supervisor.start_link( + [ + {Agens.Supervisor, name: Agens.Supervisor} + ], + strategy: :one_for_one +) +``` + +See `Agens.Supervisor` for more information + +--- +**2. Start one or more Servings** + +A **Serving** is essentially a wrapper for language model inference. It can be an `Nx.Serving` struct, either returned by `Bumblebee` or manually created, or a `GenServer` that interfaces with the OpenAI API or other language model APIs. Technically, due to `GenServer` support, a Serving doesn't need to be limited to language models or machine learning—it can also handle regular API calls. + +```elixir +Application.put_env(:nx, :default_backend, EXLA.Backend) +auth_token = System.get_env("HF_AUTH_TOKEN") + +my_serving = fn -> + repo = {:hf, "mistralai/Mistral-7B-Instruct-v0.2", auth_token: auth_token} + + {:ok, model} = Bumblebee.load_model(repo, type: :bf16) + {:ok, tokenizer} = Bumblebee.load_tokenizer(repo) + {:ok, generation_config} = Bumblebee.load_generation_config(repo) + + Bumblebee.Text.generation(model, tokenizer, generation_config) +end + +serving_config = %Agens.Serving.Config{ + name: :my_serving, + serving: my_serving() +} + +{:ok, pid} = Agens.Serving.start(serving_config) +``` + +See `Agens.Serving` for more information + +--- +**3. Create and start one or more Agents** + +An **Agent** in the context of Agens is responsible for communicating with Servings and can provide additional context during these interactions. + +In practice, Agents typically have their own specialized tasks or capabilities while communicating with the same Serving. Many projects may use a single Serving, such as a language model (LM) or an LM API, but employ multiple Agents to perform different tasks using that Serving. + +Additionally, Agents can use modules implementing the `Agens.Tool` behaviour to extend their capabilities beyond standard LM inference, enabling function-calling and other advanced operations. + +```elixir +agent_config = %Agens.Agent.Config{ + name: :my_agent, + serving: :my_serving +} +{:ok, pid} = Agens.Agent.start(agent_config) +``` + +See `Agens.Agent` for more information + +--- +**4. Create and start one or more Jobs** + +While Agens is designed to be flexible enough to allow direct communication with an `Agens.Serving` or `Agens.Agent`, its primary goal is to facilitate a multi-agent workflow that uses various steps to achieve a final result. Each step (`Agens.Job.Step`) employs an Agent to accomplish its objective, and the results are then passed to the next step in the **Job**. Conditions can also be used to determine the routing between steps or to conclude the job. + +```elixir +job_config = %Agens.Job.Config{ + name: :my_job, + description: "an example job", + steps: [ + %Agens.Job.Step{ + agent: :my_agent, + objective: "first step objective" + }, + %Agens.Job.Step{ + agent: :my_agent, + conditions: %{ + "__DEFAULT__" => :end + } + } + ] +} +{:ok, pid} = Agens.Job.start(job_config) +Agens.Job.run(:my_job, "user input") +``` + +See `Agens.Job` for more information + +--- + +## Prompting +Agens provides a variety of different ways to customize the final prompt sent to the language model (LM) or Serving. A natural language string can be assigned to the entity's specialized field (see below), while `nil` values will omit that field from the final prompt. This approach allows for precise control over the prompt’s content. + +All fields with values, in addition to user input, will be included in the final prompt !!!!using the [in-context learning]() method!!!!. The goal should be to balance detailed prompts with efficient token usage by focusing on relevant fields and using concise language. This approach will yield the best results with minimal token usage, keeping costs low and performance high. + +### User/Agent +The `input` value is the only required field for building prompts. This value can be the initial value provided to `Agens.Job.run/2`, or the final result of a previous step (`Agens.Job.Step`). Both the `input` and `result` are stored in `Agens.Message`, which can also be used to send messages directly to `Agens.Agent` or `Agens.Serving` without being part of an `Agens.Job`. + +### Job +`Agens.Job.Config` uses the `description` field to configure the prompt for all messages within the Job. This field should be used carefully as it will be sent to the Serving with every prompt. + +### Step +`Agens.Job.Step` uses the `objective` field to customize the final prompt sent to the Serving. This can provide more specific information in the final prompt than the Job `description` or Agent `prompt`. + +### Agent +`Agens.Agent` provides the most advanced prompt capabilities. The `prompt` field of `Agens.Agent.Config` accepts either a simple string value, or an `Agens.Agent.Prompt` struct. The following fields, which are all optional, can be used with the struct approach: + +- `:identity` - a string representing the purpose and capabilities of the agent +- `:context` - a string representing the goal or purpose of the agent's actions +- `:constraints` - a string listing any constraints or limitations on the agent's actions +- `:examples` - a list of maps representing example inputs and outputs for the agent +- `:reflection` - a string representing any additional considerations or reflection the agent should make before returning results + +Keep in mind that a single agent can be used across multiple jobs, so it is best to restrict the agent prompt to specific capabilities and use `objective` on `Agens.Job.Step` or `description` on `Agens.Job.Config` for Job or Step-specific prompting. + +### Tool +When creating Tools with the `Agens.Tool` behaviour, the `c:Agens.Tool.instructions/0` callback can be used to include specific instructions in the final prompt. These instructions may also include examples, especially for structured output, which can be crucial for designing a Tool that delivers predictable results. + +It is important to note that these instructions are provided to the Serving **before** the Tool is used, ensuring that the language model (LM) supplies the correct inputs to the Tool. After receiving these inputs, the Tool should be able to generate the relevant arguments to make the function call, and finally provide the expected output for the next step of the job. + +See `Agens.Tool` for more information on using Tools. + +### Summary +- **User/Agent**: `input`/`result` +- **Job**: `description` +- **Agent**: `prompt` (`string` or `Agens.Agent.Prompt`) +- **Step**: `objective` +- **Tool**: `instructions` + +> **Note:** +> +> Depending on your use case, some fields may be more relevant than others. +> +> It’s often beneficial to be more descriptive at granular levels, such as the `objective` of `Agens.Job.Step` or the `instructions` for `Agens.Tool`, while taking a more minimal approach with higher-level fields, such as the `description` of `Agens.Job.Config` or the `prompt` of `Agens.Agent.Config`. + +## Name +The name Agens comes from the Latin word for 'Agents' or 'Actors.' It also draws from **intellectus agens**, a term in medieval philosophy meaning ['active intellect'](https://en.wikipedia.org/wiki/Active_intellect), which describes the mind’s ability to actively process and abstract information. This reflects the goal of the Agens project: to create intelligent, autonomous agents that manage workflows within the Elixir ecosystem. +## License +This project is licensed under the Apache License, Version 2.0. See the [LICENSE](./LICENSE) file for more details. diff --git a/config/config.exs b/config/config.exs new file mode 100644 index 0000000..d7ffecc --- /dev/null +++ b/config/config.exs @@ -0,0 +1,22 @@ +import Config + +config :agens, + registry: Agens.Registry, + prompts: %{ + prompt: + {"Agent", "You are a specialized agent with the following capabilities and expertise"}, + identity: + {"Identity", "You are a specialized agent with the following capabilities and expertise"}, + context: {"Context", "The purpose or goal behind your tasks are to"}, + constraints: + {"Constraints", "You must operate with the following constraints or limitations"}, + examples: {"Examples", "You should consider the following examples before returning results"}, + reflection: + {"Reflection", "You should reflect on the following factors before returning results"}, + instructions: + {"Tool Instructions", + "You should provide structured output for function calling based on the following instructions"}, + objective: {"Step Objective", "The objective of this step is to"}, + description: {"Job Description", "This is part of multi-step job to achieve the following"}, + input: {"Input", "The following is the actual input from the user, system or another agent"} + } diff --git a/coveralls.json b/coveralls.json new file mode 100644 index 0000000..3bd71a2 --- /dev/null +++ b/coveralls.json @@ -0,0 +1,7 @@ +{ + "skip_files": [ + "lib/agens/tool.ex", + "test/support/helpers.ex", + "test/support/serving.ex" + ] +} \ No newline at end of file diff --git a/lib/agens.ex b/lib/agens.ex index 7dad16f..299dfeb 100644 --- a/lib/agens.ex +++ b/lib/agens.ex @@ -1,18 +1,177 @@ +# Copyright 2024 Jesse Drelick +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + defmodule Agens do @moduledoc """ - Documentation for `Agens`. + Agens is used to create multi-agent workflows with language models. + + It is made up of the following core entities: + + - `Agens.Serving` - used to interact with language models + - `Agens.Agent` - used to interact with servings in a specialized manner + - `Agens.Job` - used to define multi-agent workflows """ - @doc """ - Hello world. + defmodule Message do + @moduledoc """ + The Message struct defines the details of a message passed between Agents, Jobs and Servings. - ## Examples + ## Fields - iex> Agens.hello() - :world + * `:parent_pid` - The process identifier of the parent/caller process. + * `:input` - The input string for the message. + * `:prompt` - The prompt string or `Agens.Agent.Prompt` struct for the message. + * `:result` - The result string for the message. + * `:agent_name` - The name of the `Agens.Agent`. + * `:serving_name` - The name of the `Agens.Serving`. + * `:job_name` - The name of the `Agens.Job`. + * `:job_description` - The description of the `Agens.Job` to be added to the LM prompt. + * `:step_index` - The index of the `Agens.Job.Step`. + * `:step_objective` - The objective of the `Agens.Job.Step` to be added to the LM prompt. + """ - """ - def hello do - :world + @type t :: %__MODULE__{ + parent_pid: pid() | nil, + input: String.t() | nil, + prompt: String.t() | Agens.Agent.Prompt.t() | nil, + result: String.t() | nil, + agent_name: atom() | nil, + serving_name: atom() | nil, + job_name: atom() | nil, + job_description: String.t() | nil, + step_index: non_neg_integer() | nil, + step_objective: String.t() | nil + } + + @enforce_keys [] + defstruct [ + :parent_pid, + :input, + :prompt, + :result, + :agent_name, + :serving_name, + :job_name, + :job_description, + :step_index, + :step_objective + ] + + alias Agens.{Agent, Serving} + + @registry Application.compile_env(:agens, :registry) + @fields Application.compile_env(:agens, :prompts) + + @doc """ + Sends an `Agens.Message` to an `Agens.Agent` + """ + @spec send(__MODULE__.t()) :: __MODULE__.t() | {:error, :agent_not_running} + def send(%__MODULE__{} = message) do + case Registry.lookup(@registry, message.agent_name) do + [{_, {agent_pid, config}}] when is_pid(agent_pid) -> + base = build_prompt(config, message) + prompt = "[INST]#{base}[/INST]" + + result = + message + |> Map.put(:serving_name, config.serving) + |> Map.put(:prompt, prompt) + |> Serving.run() + + message = Map.put(message, :result, result) + maybe_use_tool(message, config.tool) + + [] -> + {:error, :agent_not_running} + end + end + + @spec build_prompt(Agent.Config.t(), t()) :: String.t() + defp build_prompt(%Agent.Config{prompt: prompt, tool: tool}, %__MODULE__{} = message) do + %{ + objective: message.step_objective, + description: message.job_description + } + |> maybe_add_prompt(prompt) + |> maybe_add_tool(tool) + |> maybe_prep_input(message.input, tool) + |> Enum.reject(&filter_empty/1) + |> Enum.map(&field/1) + |> Enum.map(&to_prompt/1) + |> Enum.join("\n\n") + end + + defp filter_empty({_, value}), do: value == "" or is_nil(value) + + defp field({key, value}) do + {Map.get(@fields, key), value} + end + + defp to_prompt({{heading, detail}, value}) do + """ + ## #{heading} + #{detail}: #{value} + """ + end + + defp maybe_add_prompt(map, %Agent.Prompt{} = prompt), + do: prompt |> Map.from_struct() |> Map.merge(map) + + defp maybe_add_prompt(map, prompt) when is_binary(prompt), do: Map.put(map, :prompt, prompt) + defp maybe_add_prompt(map, _prompt), do: map + + defp maybe_add_tool(map, nil), do: map + defp maybe_add_tool(map, tool), do: Map.put(map, :instructions, tool.instructions()) + + defp maybe_prep_input(map, input, nil), do: Map.put(map, :input, input) + defp maybe_prep_input(map, input, tool), do: Map.put(map, :input, tool.pre(input)) + + @spec maybe_use_tool(__MODULE__.t(), module() | nil) :: __MODULE__.t() + defp maybe_use_tool(message, nil), do: message + + defp maybe_use_tool(%__MODULE__{} = message, tool) do + send( + message.parent_pid, + {:tool_started, {message.job_name, message.step_index}, message.result} + ) + + raw = + message.result + |> tool.to_args() + |> tool.execute() + + send(message.parent_pid, {:tool_raw, {message.job_name, message.step_index}, raw}) + + result = tool.post(raw) + + send(message.parent_pid, {:tool_result, {message.job_name, message.step_index}, result}) + + Map.put(message, :result, result) + end + end + + use DynamicSupervisor + + @doc false + @spec start_link(any()) :: :ignore | {:error, any()} | {:ok, pid()} + def start_link(_) do + DynamicSupervisor.start_link(__MODULE__, :ok, name: __MODULE__) + end + + @doc false + @spec init(any()) :: {:ok, any()} + def init(:ok) do + DynamicSupervisor.init(strategy: :one_for_one) end end diff --git a/lib/agens/agent.ex b/lib/agens/agent.ex new file mode 100644 index 0000000..6750aab --- /dev/null +++ b/lib/agens/agent.ex @@ -0,0 +1,126 @@ +defmodule Agens.Agent do + @moduledoc """ + The Agent module provides functions for starting, stopping and running Agents. + + `Agens.Agent` is the the primary entity powering `Agens`. It uses `Agens.Serving` to interact with language models through `Nx.Serving`, or with language model APIs through a `GenServer`. + + Agents can have detailed identities to further refine LM outputs, and are used together in multi-agent workflows via the `Agens.Job` module. + + Agent capabilities can be expanded even further with `Agens.Tool` modules, which are designed to handle LM functional calling. In future releases, Agents will also have access to RAG generations via knowledge base features. + """ + + defmodule Prompt do + @moduledoc """ + The Prompt struct represents an advanced prompt for an Agent process. + + All fields are optional and will only be included in the final prompt if they are not nil. + + ## Fields + - `:identity` - a string representing the purpose and capabilities of the agent + - `:context` - a string representing the goal or purpose of the agent's actions + - `:constraints` - a string listing any constraints or limitations on the agent's actions + - `:examples` - a list of example inputs and outputs for the agent + - `:reflection` - a string representing any additional considerations or reflection the agent should make before returning results + """ + + @type t :: %__MODULE__{ + identity: String.t() | nil, + context: String.t() | nil, + constraints: String.t() | nil, + examples: String.t() | nil, + reflection: String.t() | nil + } + + @enforce_keys [] + defstruct [:identity, :context, :constraints, :examples, :reflection] + end + + defmodule Config do + @moduledoc """ + The Config struct represents the configuration for an Agent process. + + ## Fields + - `:name` - The name of the Agent process. + - `:serving` - The serving module or `Nx.Serving` struct for the Agent. + - `:knowledge` - The knowledge base or data source of the Agent. Default is nil. (Coming soon) + - `:prompt` - The string or `Agens.Agent.Prompt` struct defining the Agent. Default is nil. + - `:tool` - The module implementing the `Agens.Tool` behaviour for the Agent. Default is nil. + """ + + @type t :: %__MODULE__{ + name: atom(), + serving: module() | Nx.Serving.t(), + knowledge: module() | nil, + prompt: Agens.Agent.Prompt.t() | String.t() | nil, + tool: module() | nil + } + + @enforce_keys [:name, :serving] + defstruct [:name, :serving, :knowledge, :prompt, :tool] + end + + use GenServer + + @registry Application.compile_env(:agens, :registry) + + @doc """ + Starts one or more `Agens.Agent` processes + """ + @spec start([Config.t()] | Config.t()) :: [{:ok, pid()}] | {:ok, pid()} + def start(configs) when is_list(configs) do + configs + |> Enum.map(fn config -> + start(config) + end) + end + + def start(%Config{} = config) do + spec = %{ + id: config.name, + start: {__MODULE__, :start_link, [config]} + # type: :worker, + # restart: :transient + } + + Agens + |> DynamicSupervisor.start_child(spec) + |> case do + {:ok, pid} when is_pid(pid) -> + Registry.register(@registry, config.name, {pid, config}) + {:ok, pid} + + {:error, {:already_started, pid}} = err when is_pid(pid) -> + err + end + end + + @doc """ + Stops an `Agens.Agent` process + """ + @spec stop(atom()) :: :ok | {:error, :agent_not_found} + def stop(agent_name) do + agent_name + |> Process.whereis() + |> case do + nil -> + {:error, :agent_not_found} + + pid -> + :ok = DynamicSupervisor.terminate_child(Agens, pid) + Registry.unregister(@registry, agent_name) + end + end + + @doc false + @spec start_link(Config.t()) :: GenServer.on_start() + def start_link(config) do + GenServer.start_link(__MODULE__, config, name: config.name) + end + + @doc false + @spec init(Config.t()) :: {:ok, map()} + @impl true + def init(_config) do + {:ok, %{}} + end +end diff --git a/lib/agens/job.ex b/lib/agens/job.ex new file mode 100644 index 0000000..e62651b --- /dev/null +++ b/lib/agens/job.ex @@ -0,0 +1,318 @@ +defmodule Agens.Job do + @moduledoc """ + A Job defines a multi-agent workflow through a sequence of steps. + + An `Agens.Job` is mainly a sequence of steps, defined with the `Agens.Job.Step` struct, used to create advanced multi-agent workflows. + + Conditions can be used in order to route to different steps based on a result, or can be used to end the Job. + + ### Events + Agens emits several events that can be handled by the caller using `handle_info/3` for purposes such as UI updates, pubsub, logging, persistence and other side effects. + + #### Job + ``` + {:job_started, job.name} + ``` + + Emitted when a job has started. + + ``` + {:job_ended, job.name, :completed | {:error, error}} + ``` + + Emitted when a job has ended, either due to completion or an error. + + #### Step + ``` + {:step_started, {job.name, step_index}, message.input} + ``` + + Emitted when a step has started. Includes the input data provided to the step, whether from the user or a previous step. + + ``` + {:step_result, {job.name, step_index}, message.result} + ``` + + Emitted when a result has been returned from the Serving. Includes the Serving result, which will be passed to the Tool (if applicable), conditions (if applicable), or the next step of the job. + + #### Tool + The following events are emitted only if the Agent has a Tool specified in `Agens.Agent.Config`: + + ``` + {:tool_started, {job.name, step_index}, message.result} + ``` + + Emitted when a Tool is about to be called. `message.result` here is the Serving result, which will be overriden by the value returned from the Tool prior to final output. + + ``` + {:tool_raw, {job.name, step_index}, message.raw} + ``` + + Emitted after completing the Tool function call. It provides the raw result of the Tool before any post-processing. + + ``` + {:tool_result, {job.name, step_index}, message.result} + ``` + + Emitted after post-processing of the raw Tool result. This is the final result of the Tool, which will be passed to conditions or the next step of the job. + """ + + defmodule Step do + @moduledoc """ + The Step struct defines a single step within a Job. + + ## Fields + - `agent` - The name of the agent to be used in the Step. + - `objective` - An optional string to be added to the LM prompt explaining the purpose of the Step. + - `conditions` - An optional conditions map to control flow based on the result of the agent. + """ + + @type t :: %__MODULE__{ + agent: atom(), + objective: String.t() | nil, + conditions: map() | nil + } + + @enforce_keys [:agent] + defstruct [:agent, :objective, :conditions] + end + + defmodule Config do + @moduledoc """ + The Config struct defines the details of a Job. + + ## Fields + - `name` - An atom that identifies the Job. + - `description` - An optional string to be added to the LM prompt that describes the basic goal of the Job. + - `steps` - A list of `Agens.Job.Step` structs that define the sequence of agent actions to be performed. + """ + + @type t :: %__MODULE__{ + name: atom(), + description: String.t() | nil, + steps: list(Step.t()) + } + + @enforce_keys [:name, :steps] + defstruct [:name, :description, :steps] + end + + defmodule State do + @moduledoc false + + @type t :: %__MODULE__{ + status: :init | :running | :error | :completed, + step_index: non_neg_integer() | nil, + config: Config.t(), + parent: pid() | nil + } + + @enforce_keys [:status, :config] + defstruct [:status, :step_index, :config, :parent] + end + + use GenServer + + alias Agens.Message + + @doc """ + Starts a new Job process using the provided `Agens.Job.Config`. + + `start/1` does not run the Job, only starts the supervised process. See `run/2` for running the Job. + """ + @spec start(Config.t()) :: {:ok, pid} | {:error, term} + def start(config) do + spec = child_spec(config) + + Agens + |> DynamicSupervisor.start_child(spec) + |> case do + {:ok, pid} = result when is_pid(pid) -> + result + + {:error, {:already_started, pid}} = error when is_pid(pid) -> + error + end + end + + @doc """ + Runs a Job with the given input by Job name or `pid`. + + A supervised process for the Job must be started first using `start/1`. + """ + @spec run(pid | atom, term) :: {:ok, term} | {:error, :job_not_found} + def run(name, input) when is_atom(name) do + name + |> Process.whereis() + |> case do + nil -> + {:error, :job_not_found} + + pid when is_pid(pid) -> + run(pid, input) + end + end + + def run(pid, input) when is_pid(pid) do + GenServer.call(pid, {:run, input}) + end + + @doc """ + Retrieves the Job configuration by Job name or `pid`. + """ + @spec get_config(pid | atom) :: {:ok, term} | {:error, :job_not_found} + def get_config(name) when is_atom(name) do + name + |> Process.whereis() + |> case do + nil -> + {:error, :job_not_found} + + pid when is_pid(pid) -> + get_config(pid) + end + end + + def get_config(pid) when is_pid(pid) do + GenServer.call(pid, :get_config) + end + + @doc false + def start_link(config) do + GenServer.start_link(__MODULE__, config, name: config.name) + end + + @doc false + def child_spec(config) do + %{ + id: config.name, + start: {__MODULE__, :start_link, [config]}, + type: :worker, + restart: :transient + } + end + + @doc false + @impl true + @spec init(Config.t()) :: + {:ok, State.t()} + | {:ok, State.t(), timeout() | :hibernate | {:continue, continue_arg :: term()}} + | :ignore + | {:stop, reason :: any()} + def init(config) do + {:ok, %State{status: :init, config: config}} + end + + @doc false + @impl true + @spec handle_call(:get_config, {pid, term}, State.t()) :: {:reply, Config.t(), State.t()} + def handle_call(:get_config, _from, state) do + {:reply, state.config, state} + end + + @doc false + @impl true + @spec handle_call({:run, String.t()}, {pid, term}, State.t()) :: {:reply, :ok, State.t()} + def handle_call({:run, input}, {parent, _}, state) do + new_state = %State{state | status: :running, step_index: 0, parent: parent} + {:reply, :ok, new_state, {:continue, {:run, input}}} + end + + @doc false + @impl true + @spec handle_continue({:run, String.t()}, State.t()) :: {:noreply, State.t()} + def handle_continue({:run, input}, %{config: %{name: name}} = state) do + send(state.parent, {:job_started, name}) + do_step(input, state) + {:noreply, state} + end + + @doc false + @impl true + @spec handle_cast({:next, Message.t()}, State.t()) :: {:noreply, State.t()} + def handle_cast({:next, %Message{} = message}, %State{step_index: index} = state) do + new_state = %State{state | step_index: index + 1} + do_step(message.result, new_state) + {:noreply, new_state} + end + + @doc false + @impl true + @spec handle_cast({:step, integer, Message.t()}, State.t()) :: {:noreply, State.t()} + def handle_cast({:step, index, %Message{} = message}, %State{} = state) do + unless is_integer(index) do + raise "Invalid step index: #{inspect(index)}" + end + + new_state = %State{state | step_index: index} + do_step(message.result, new_state) + {:noreply, new_state} + end + + @doc false + @impl true + @spec handle_cast(:end, State.t()) :: {:stop, :complete, State.t()} + def handle_cast(:end, %State{} = state) do + new_state = %State{state | status: :complete} + {:stop, :complete, new_state} + end + + @doc false + @impl true + @spec terminate(:complete | {:error, term}, State.t()) :: :ok + def terminate(:complete, %State{config: %{name: name}} = state) do + send(state.parent, {:job_ended, name, :complete}) + :ok + end + + def terminate({error, _}, %State{config: %{name: name}} = state) do + send(state.parent, {:job_ended, name, {:error, error}}) + :ok + end + + @doc false + @spec do_step(String.t(), State.t()) :: :ok + defp do_step(input, %State{config: job_config} = state) do + step = Enum.at(job_config.steps, state.step_index) + + message = %Message{ + parent_pid: state.parent, + input: input, + agent_name: step.agent, + job_name: job_config.name, + job_description: job_config.description, + step_index: state.step_index, + step_objective: step.objective + } + + send(state.parent, {:step_started, {message.job_name, message.step_index}, message.input}) + message = Message.send(message) + send(state.parent, {:step_result, {message.job_name, message.step_index}, message.result}) + + if step.conditions do + do_conditions(step.conditions, message) + else + GenServer.cast(self(), {:next, message}) + end + end + + @doc false + @spec do_conditions(map(), Message.t()) :: :ok + defp do_conditions(conditions, %Message{} = message) when is_map(conditions) do + conditions + |> Map.get(message.result) + |> case do + :end -> + GenServer.cast(self(), :end) + + nil -> + case Map.get(conditions, "__DEFAULT__") do + :end -> + GenServer.cast(self(), :end) + + step_index -> + GenServer.cast(self(), {:step, step_index, message}) + end + end + end +end diff --git a/lib/agens/serving.ex b/lib/agens/serving.ex new file mode 100644 index 0000000..d3ca871 --- /dev/null +++ b/lib/agens/serving.ex @@ -0,0 +1,115 @@ +defmodule Agens.Serving do + @moduledoc """ + The Serving module provides functions for starting, stopping and running Servings. + + `Agens.Serving` accepts a `GenServer` module or `Nx.Serving` struct for processing messages. + + `Agens.Serving` is decoupled from `Agens.Agent` in order to reuse a single LM across multiple agents. In most cases, however, you will only need to start one text generation serving to be used by most, if not all, agents. + + In some cases, you may have additional servings for more specific use cases such as image generation, speech recognition, etc. + + Servings were built with the `Bumblebee` library in mind, as well as `Nx.Serving`. `GenServer` is supported for working with LM APIs instead, which may be more cost effective and easier to get started with. + """ + + defmodule Config do + @moduledoc """ + The Config struct represents the configuration for a Serving process. + + ## Fields + - `:name` - The name of the `Agens.Serving` process. + - `:serving` - The `Nx.Serving` struct or `GenServer` module for the `Agens.Serving`. + """ + + @type t :: %__MODULE__{ + name: atom(), + serving: Nx.Serving.t() | module() + } + + @enforce_keys [:name, :serving] + defstruct [:name, :serving] + end + + alias Agens.Message + + @registry Application.compile_env(:agens, :registry) + + @doc """ + Starts an `Agens.Serving` process + """ + @spec start(Config.t()) :: {:ok, pid()} + def start(%Config{} = config) do + spec = %{ + id: config.name, + start: start_function(config) + } + + Agens + |> DynamicSupervisor.start_child(spec) + |> case do + {:ok, pid} when is_pid(pid) -> + Registry.register(@registry, config.name, {pid, config}) + {:ok, pid} + + {:error, {:already_started, pid}} = error when is_pid(pid) -> + error + end + end + + @doc """ + Stops an `Agens.Serving` process + """ + @spec stop(atom()) :: :ok | {:error, :serving_not_found} + def stop(serving_name) do + serving_name + |> Module.concat("Supervisor") + |> Process.whereis() + |> case do + nil -> + {:error, :serving_not_found} + + pid -> + :ok = DynamicSupervisor.terminate_child(Agens, pid) + Registry.unregister(@registry, serving_name) + end + end + + @doc """ + Executes an `Agens.Message` against an `Agens.Serving` + """ + @spec run(Message.t()) :: String.t() | {:error, :serving_not_running} + def run(%Message{} = message) do + case Registry.lookup(@registry, message.serving_name) do + [{_, {serving_pid, config}}] when is_pid(serving_pid) -> + do_run({serving_pid, config}, message) + + [] -> + {:error, :serving_not_running} + end + end + + @spec do_run({pid(), Config.t()}, Message.t()) :: String.t() + defp do_run({_, %Config{serving: %Nx.Serving{}}}, %Message{} = message) do + message.serving_name + |> Nx.Serving.batched_run(message.prompt) + |> case do + %{results: [%{text: result}]} -> result + result -> result + end + end + + defp do_run({serving_pid, _}, %Message{} = message) do + # GenServer.call(serving_name, {:run, message}) + GenServer.call(serving_pid, {:run, message}) + end + + @spec start_function(Config.t()) :: tuple() + defp start_function(%Config{serving: %Nx.Serving{} = serving} = config) do + {Nx.Serving, :start_link, [[serving: serving, name: config.name]]} + end + + # Module.concat with "Supervisor" for Nx.Serving parity + defp start_function(%Config{serving: serving} = config) when is_atom(serving) do + name = Module.concat(config.name, "Supervisor") + {serving, :start_link, [[name: name, config: config]]} + end +end diff --git a/lib/agens/supervisor.ex b/lib/agens/supervisor.ex new file mode 100644 index 0000000..f132b43 --- /dev/null +++ b/lib/agens/supervisor.ex @@ -0,0 +1,49 @@ +defmodule Agens.Supervisor do + @moduledoc """ + The Supervisor module for the Agens application. + + `Agens.Supervisor` starts a `DynamicSupervisor` for managing `Agens.Agent`, `Agens.Serving`, and `Agens.Job` processes. It also starts a `Registry` for keeping track of these processes. + + The Registry module can be overriden by your application config: + + ``` + config :agens, registry: MyApp.Registry + ``` + + In order to use `Agens` simply add `Agens.Supervisor` to your application supervision tree: + + ``` + Supervisor.start_link( + [ + {Agens.Supervisor, name: Agens.Supervisor} + ], + strategy: :one_for_one + ) + ``` + """ + use Supervisor + + @registry Application.compile_env(:agens, :registry) + + @doc false + @spec start_link(any()) :: Supervisor.on_start() + def start_link(init_arg) do + Supervisor.start_link(__MODULE__, init_arg, name: __MODULE__) + end + + @doc false + @impl true + @spec init(any()) :: + {:ok, + {:supervisor.sup_flags(), + [:supervisor.child_spec() | (old_erlang_child_spec :: :supervisor.child_spec())]}} + | :ignore + def init(_init_arg) do + children = [ + {Agens, name: Agens}, + {Registry, keys: :unique, name: @registry} + ] + + Supervisor.init(children, strategy: :one_for_one) + end +end diff --git a/lib/agens/tool.ex b/lib/agens/tool.ex new file mode 100644 index 0000000..7a265e4 --- /dev/null +++ b/lib/agens/tool.ex @@ -0,0 +1,41 @@ +defmodule Agens.Tool do + @moduledoc """ + The Tool behaviour. + + A Tool is a module that implements the `Agens.Tool` behaviour. It is used to define + the functionality of a tool that can be used by an `Agens.Agent`. + + A Tool defines the following callbacks: + + - `pre/1` - pre-processes the input from the previous step before it is added to the LM prompt. + - `instructions/0` - returns the tool-specific instructions for the LM. + - `to_args/1` - parses the LM result into arguments to be used by `execute/1`. + - `execute/1` - executes the tool with the given arguments. + - `post/1` - handles the various outputs of `execute/1`, whether a map or error tuple, and returns a string for the next Step of the Job. + """ + + @doc """ + Pre-process the input from previous Step before it is added to the LM prompt. + """ + @callback pre(input :: String.t()) :: String.t() + + @doc """ + Instructions for the LM, combined with the input from previous Step (and optionally pre-processed using `pre/1`) as well as the prompt from the Agent config. + """ + @callback instructions() :: String.t() + + @doc """ + Parse the LM result into arguments to be used by `execute/1`. + """ + @callback to_args(result :: binary()) :: keyword() + + @doc """ + Execute the tool with the given arguments. LM is responsible for generating arguments for the Tool based on Tool instructions, input from previous Step, Agent config and any additional context. + """ + @callback execute(args :: keyword()) :: map() | {:error, atom()} + + @doc """ + Handles the various outputs of `execute/1`, whether a map or error tuple, and returns a string for the next Step of the Job. + """ + @callback post(map() | {:error, atom()}) :: String.t() +end diff --git a/mix.exs b/mix.exs index fd06dfd..93dc700 100644 --- a/mix.exs +++ b/mix.exs @@ -1,15 +1,30 @@ defmodule Agens.MixProject do use Mix.Project + @version "0.1.0" + def project do [ app: :agens, - version: "0.1.0", + version: @version, elixir: "~> 1.15", + elixirc_paths: elixirc_paths(Mix.env()), start_permanent: Mix.env() == :prod, deps: deps(), - description: "Library for creating AI agents", - package: package() + description: + "Create multi-agent workflows with AI and Language Models using OTP components for reliable and scalable automation.", + package: package(), + docs: docs(), + aliases: aliases(), + test_coverage: [tool: ExCoveralls], + preferred_cli_env: [ + "test.all": :test, + "test.lm": :test, + coveralls: :test, + "coveralls.detail": :test, + "coveralls.html": :test, + "coveralls.json": :test + ] ] end @@ -23,17 +38,76 @@ defmodule Agens.MixProject do # Run "mix help deps" to learn about dependencies. defp deps do [ - {:ex_doc, ">= 0.0.0", only: :dev, runtime: false} - # {:dep_from_hexpm, "~> 0.3.0"}, - # {:dep_from_git, git: "https://github.com/elixir-lang/my_dep.git", tag: "0.1.0"} + {:nx, "~> 0.7.3"}, + {:bumblebee, "~> 0.5.3", only: :test}, + {:exla, "~> 0.7.0", only: :test}, + {:dialyxir, "~> 1.4", only: [:dev, :test], runtime: false}, + {:ex_doc, ">= 0.0.0", only: :dev, runtime: false}, + {:excoveralls, "~> 0.17.1", only: :test} ] end defp package do [ maintainers: ["Jesse Drelick"], - licenses: ["MIT"], - links: %{"GitHub" => "https://github.com/jessedrelick/agens"} + licenses: ["Apache-2.0"], + links: %{"GitHub" => "https://github.com/jessedrelick/agens"}, + keywords: [ + "AI", + "Agents", + "AI Agents", + "Multi-Agent Systems", + "LLM", + "Language Models", + "NLP", + "Task Orchestration", + "Workflow Automation", + "Bumblebee" + ], + categories: [ + "Machine Learning", + "Artificial Intelligence", + "Natural Language Processing", + "Automation" + ] + ] + end + + defp docs do + [ + main: "Agens", + extras: [{"README.md", [title: "Agens"]}, "LICENSE"], + source_url: "https://github.com/jessedrelick/agens", + groups_for_modules: [ + Agent: [ + Agens.Agent, + Agens.Agent.Config, + Agens.Agent.Prompt + ], + Job: [ + Agens.Job, + Agens.Job.State, + Agens.Job.Config, + Agens.Job.Step + ], + Serving: [ + Agens.Serving, + Agens.Serving.Config + ], + Tool: [ + Agens.Tool + ] + ] + ] + end + + defp elixirc_paths(:test), do: ["lib", "test/support"] + defp elixirc_paths(_), do: ["lib"] + + defp aliases do + [ + "test.all": ["test --include lm"], + "test.lm": ["test --only lm"] ] end end diff --git a/mix.lock b/mix.lock index e702a12..01a3e7b 100644 --- a/mix.lock +++ b/mix.lock @@ -1,8 +1,32 @@ %{ + "axon": {:hex, :axon, "0.6.1", "1d042fdba1c1b4413a3d65800524feebd1bc8ed218f8cdefe7a97510c3f427f3", [:mix], [{:kino, "~> 0.7", [hex: :kino, repo: "hexpm", optional: true]}, {:kino_vega_lite, "~> 0.1.7", [hex: :kino_vega_lite, repo: "hexpm", optional: true]}, {:nx, "~> 0.6.0 or ~> 0.7.0", [hex: :nx, repo: "hexpm", optional: false]}, {:polaris, "~> 0.1", [hex: :polaris, repo: "hexpm", optional: false]}, {:table_rex, "~> 3.1.1", [hex: :table_rex, repo: "hexpm", optional: true]}], "hexpm", "d6b0ae2f0dd284f6bf702edcab71e790d6c01ca502dd06c4070836554f5a48e1"}, + "bumblebee": {:hex, :bumblebee, "0.5.3", "151c215fd6014958dbfc322fe5f31b44d170293f69cfdca419936c81e39b1f64", [:mix], [{:axon, "~> 0.6.1", [hex: :axon, repo: "hexpm", optional: false]}, {:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:jason, "~> 1.4.0", [hex: :jason, repo: "hexpm", optional: false]}, {:nx, "~> 0.7.0", [hex: :nx, repo: "hexpm", optional: false]}, {:nx_image, "~> 0.1.0", [hex: :nx_image, repo: "hexpm", optional: false]}, {:nx_signal, "~> 0.2.0", [hex: :nx_signal, repo: "hexpm", optional: false]}, {:progress_bar, "~> 3.0", [hex: :progress_bar, repo: "hexpm", optional: false]}, {:safetensors, "~> 0.1.3", [hex: :safetensors, repo: "hexpm", optional: false]}, {:tokenizers, "~> 0.4", [hex: :tokenizers, repo: "hexpm", optional: false]}, {:unpickler, "~> 0.1.0", [hex: :unpickler, repo: "hexpm", optional: false]}, {:unzip, "~> 0.10.0", [hex: :unzip, repo: "hexpm", optional: false]}], "hexpm", "5518f11e424c431a9cbedc80e0d26525368f0b6e50572a674ff247ec3b26bdd7"}, + "castore": {:hex, :castore, "1.0.8", "dedcf20ea746694647f883590b82d9e96014057aff1d44d03ec90f36a5c0dc6e", [:mix], [], "hexpm", "0b2b66d2ee742cb1d9cb8c8be3b43c3a70ee8651f37b75a8b982e036752983f1"}, + "complex": {:hex, :complex, "0.5.0", "af2d2331ff6170b61bb738695e481b27a66780e18763e066ee2cd863d0b1dd92", [:mix], [], "hexpm", "2683bd3c184466cfb94fad74cbfddfaa94b860e27ad4ca1bffe3bff169d91ef1"}, + "decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"}, + "dialyxir": {:hex, :dialyxir, "1.4.3", "edd0124f358f0b9e95bfe53a9fcf806d615d8f838e2202a9f430d59566b6b53b", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "bf2cfb75cd5c5006bec30141b131663299c661a864ec7fbbc72dfa557487a986"}, "earmark_parser": {:hex, :earmark_parser, "1.4.41", "ab34711c9dc6212dda44fcd20ecb87ac3f3fce6f0ca2f28d4a00e4154f8cd599", [:mix], [], "hexpm", "a81a04c7e34b6617c2792e291b5a2e57ab316365c2644ddc553bb9ed863ebefa"}, + "elixir_make": {:hex, :elixir_make, "0.8.4", "4960a03ce79081dee8fe119d80ad372c4e7badb84c493cc75983f9d3bc8bde0f", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:certifi, "~> 2.0", [hex: :certifi, repo: "hexpm", optional: true]}], "hexpm", "6e7f1d619b5f61dfabd0a20aa268e575572b542ac31723293a4c1a567d5ef040"}, + "erlex": {:hex, :erlex, "0.2.7", "810e8725f96ab74d17aac676e748627a07bc87eb950d2b83acd29dc047a30595", [:mix], [], "hexpm", "3ed95f79d1a844c3f6bf0cea61e0d5612a42ce56da9c03f01df538685365efb0"}, "ex_doc": {:hex, :ex_doc, "0.34.2", "13eedf3844ccdce25cfd837b99bea9ad92c4e511233199440488d217c92571e8", [:mix], [{:earmark_parser, "~> 1.4.39", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "5ce5f16b41208a50106afed3de6a2ed34f4acfd65715b82a0b84b49d995f95c1"}, + "excoveralls": {:hex, :excoveralls, "0.17.1", "83fa7906ef23aa7fc8ad7ee469c357a63b1b3d55dd701ff5b9ce1f72442b2874", [:mix], [{:castore, "~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "95bc6fda953e84c60f14da4a198880336205464e75383ec0f570180567985ae0"}, + "exla": {:hex, :exla, "0.7.3", "51310270a0976974fc758f7b28ebd6ca8e099b3d6fc78b0d484c808e977cb914", [:make, :mix], [{:elixir_make, "~> 0.6", [hex: :elixir_make, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 1.0", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:nx, "~> 0.7.1", [hex: :nx, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:xla, "~> 0.6.0", [hex: :xla, repo: "hexpm", optional: false]}], "hexpm", "5b3d5741a24aada21d3b0feb4b99d1fc3c8457f995a63ea16684d8d5678b96ff"}, + "jason": {:hex, :jason, "1.4.3", "d3f984eeb96fe53b85d20e0b049f03e57d075b5acda3ac8d465c969a2536c17b", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "9a90e868927f7c777689baa16d86f4d0e086d968db5c05d917ccff6d443e58a3"}, "makeup": {:hex, :makeup, "1.1.2", "9ba8837913bdf757787e71c1581c21f9d2455f4dd04cfca785c70bbfff1a76a3", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "cce1566b81fbcbd21eca8ffe808f33b221f9eee2cbc7a1706fc3da9ff18e6cac"}, "makeup_elixir": {:hex, :makeup_elixir, "0.16.2", "627e84b8e8bf22e60a2579dad15067c755531fea049ae26ef1020cad58fe9578", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "41193978704763f6bbe6cc2758b84909e62984c7752b3784bd3c218bb341706b"}, "makeup_erlang": {:hex, :makeup_erlang, "1.0.1", "c7f58c120b2b5aa5fd80d540a89fdf866ed42f1f3994e4fe189abebeab610839", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "8a89a1eeccc2d798d6ea15496a6e4870b75e014d1af514b1b71fa33134f57814"}, "nimble_parsec": {:hex, :nimble_parsec, "1.4.0", "51f9b613ea62cfa97b25ccc2c1b4216e81df970acd8e16e8d1bdc58fef21370d", [:mix], [], "hexpm", "9c565862810fb383e9838c1dd2d7d2c437b3d13b267414ba6af33e50d2d1cf28"}, + "nimble_pool": {:hex, :nimble_pool, "1.1.0", "bf9c29fbdcba3564a8b800d1eeb5a3c58f36e1e11d7b7fb2e084a643f645f06b", [:mix], [], "hexpm", "af2e4e6b34197db81f7aad230c1118eac993acc0dae6bc83bac0126d4ae0813a"}, + "nx": {:hex, :nx, "0.7.3", "51ff45d9f9ff58b616f4221fa54ccddda98f30319bb8caaf86695234a469017a", [:mix], [{:complex, "~> 0.5", [hex: :complex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "5ff29af84f08db9bda66b8ef7ce92ab583ab4f983629fe00b479f1e5c7c705a6"}, + "nx_image": {:hex, :nx_image, "0.1.2", "0c6e3453c1dc30fc80c723a54861204304cebc8a89ed3b806b972c73ee5d119d", [:mix], [{:nx, "~> 0.4", [hex: :nx, repo: "hexpm", optional: false]}], "hexpm", "9161863c42405ddccb6dbbbeae078ad23e30201509cc804b3b3a7c9e98764b81"}, + "nx_signal": {:hex, :nx_signal, "0.2.0", "e1ca0318877b17c81ce8906329f5125f1e2361e4c4235a5baac8a95ee88ea98e", [:mix], [{:nx, "~> 0.6", [hex: :nx, repo: "hexpm", optional: false]}], "hexpm", "7247e5e18a177a59c4cb5355952900c62fdeadeb2bad02a9a34237b68744e2bb"}, + "polaris": {:hex, :polaris, "0.1.0", "dca61b18e3e801ecdae6ac9f0eca5f19792b44a5cb4b8d63db50fc40fc038d22", [:mix], [{:nx, "~> 0.5", [hex: :nx, repo: "hexpm", optional: false]}], "hexpm", "13ef2b166650e533cb24b10e2f3b8ab4f2f449ba4d63156e8c569527f206e2c2"}, + "progress_bar": {:hex, :progress_bar, "3.0.0", "f54ff038c2ac540cfbb4c2bfe97c75e7116ead044f3c2b10c9f212452194b5cd", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}], "hexpm", "6981c2b25ab24aecc91a2dc46623658e1399c21a2ae24db986b90d678530f2b7"}, + "rustler_precompiled": {:hex, :rustler_precompiled, "0.7.2", "097f657e401f02e7bc1cab808cfc6abdc1f7b9dc5e5adee46bf2fd8fdcce9ecf", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:rustler, "~> 0.23", [hex: :rustler, repo: "hexpm", optional: true]}], "hexpm", "7663faaeadc9e93e605164dcf9e69168e35f2f8b7f2b9eb4e400d1a8e0fe2999"}, + "safetensors": {:hex, :safetensors, "0.1.3", "7ff3c22391e213289c713898481d492c9c28a49ab1d0705b72630fb8360426b2", [:mix], [{:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:nx, "~> 0.5", [hex: :nx, repo: "hexpm", optional: false]}], "hexpm", "fe50b53ea59fde4e723dd1a2e31cfdc6013e69343afac84c6be86d6d7c562c14"}, + "telemetry": {:hex, :telemetry, "1.2.1", "68fdfe8d8f05a8428483a97d7aab2f268aaff24b49e0f599faa091f1d4e7f61c", [:rebar3], [], "hexpm", "dad9ce9d8effc621708f99eac538ef1cbe05d6a874dd741de2e689c47feafed5"}, + "tokenizers": {:hex, :tokenizers, "0.5.0", "9944bba07d0b92bbfb0b8f3eef5d3694e8582a84f4154f1c447ca091a303b82d", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:rustler, ">= 0.0.0", [hex: :rustler, repo: "hexpm", optional: true]}, {:rustler_precompiled, "~> 0.6", [hex: :rustler_precompiled, repo: "hexpm", optional: false]}], "hexpm", "9791879ce694f6ddd0df004d4dfa598ba406c516f8a7ad2162c84cb0f0b7a62f"}, + "unpickler": {:hex, :unpickler, "0.1.0", "c2262c0819e6985b761e7107546cef96a485f401816be5304a65fdd200d5bd6a", [:mix], [], "hexpm", "e2b3f61e62406187ac52afead8a63bfb4e49394028993f3c4c42712743cab79e"}, + "unzip": {:hex, :unzip, "0.10.0", "374e0059e48e982076f3fd22cd4817ab11016c1bae3f09421511901ddda95c5c", [:mix], [], "hexpm", "101c06b0fa97a858a83beb618f4bc20370624f73ab3954f756d9b52194056de6"}, + "xla": {:hex, :xla, "0.6.0", "67bb7695efa4a23b06211dc212de6a72af1ad5a9e17325e05e0a87e4c241feb8", [:make, :mix], [{:elixir_make, "~> 0.4", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "dd074daf942312c6da87c7ed61b62fb1a075bced157f1cc4d47af2d7c9f44fb7"}, } diff --git a/test/agens/agent_test.exs b/test/agens/agent_test.exs new file mode 100644 index 0000000..286ef81 --- /dev/null +++ b/test/agens/agent_test.exs @@ -0,0 +1,164 @@ +defmodule Agens.AgentTest do + use Test.Support.AgentCase, async: false + + alias Agens.{Agent, Message} + + defp start_agens(_ctx) do + {:ok, _pid} = start_supervised({Agens.Supervisor, name: Agens.Supervisor}) + :ok + end + + defp start_serving(_ctx) do + %Agens.Serving.Config{ + name: :text_generation, + serving: Test.Support.Serving.Stub + } + |> Agens.Serving.start() + + :ok + end + + describe "agents" do + setup [:start_agens, :start_serving] + + test "start agents" do + agents = + [ + %Agent.Config{ + name: :test_start_agent, + serving: :text_generation + } + ] + |> Agent.start() + + assert length(agents) == 1 + [{:ok, pid}] = agents + assert is_pid(pid) + end + + test "stop agent" do + agent_name = :test_stop_agent + input = "test stop" + + message = %Message{ + agent_name: agent_name, + input: input + } + + agents = + [ + %Agent.Config{ + name: agent_name, + serving: :text_generation + } + ] + |> Agent.start() + + assert length(agents) == 1 + [{:ok, pid}] = agents + assert is_pid(pid) + + assert Agent.stop(:test_stop_agent) == :ok + + result = Message.send(message) + assert result == {:error, :agent_not_running} + end + end + + describe "errors" do + setup [:start_agens, :start_serving] + + test "start running agent" do + agent = %Agent.Config{ + name: :running_agent, + serving: :text_generation + } + + {:ok, pid} = Agent.start(agent) + assert is_pid(pid) + + assert {:error, {:already_started, ^pid}} = Agent.start(agent) + end + + test "stop non-existent agent" do + result = Agent.stop(:missing_agent) + assert result == {:error, :agent_not_found} + end + + test "invalid message returns error" do + agent_name = :second_agent + input = "Here is some invalid input" + + [ + %Agent.Config{ + name: agent_name, + serving: :text_generation + } + ] + |> Agent.start() + + message = %Message{ + agent_name: agent_name, + input: input + } + + message = Message.send(message) + + assert message.result == "ERROR" + end + + test "message non-existent agent" do + message = %Message{ + agent_name: :missing_agent, + input: "J" + } + + result = Message.send(message) + assert result == {:error, :agent_not_running} + end + end + + describe "sequence" do + setup [:start_agens, :start_serving] + + test "message sequence without job" do + get_agent_configs() + |> Agent.start() + + input = "D" + + # 0 + message = %Message{agent_name: :first_agent, input: input} + %Message{result: result} = Message.send(message) + assert result == "C" + message = %Message{agent_name: :second_agent, input: result} + %Message{result: result} = Message.send(message) + assert result == "E" + message = %Message{agent_name: :verifier_agent, input: result} + %Message{result: result} = Message.send(message) + assert result == "E" + + # 1 + message = %Message{agent_name: :first_agent, input: result} + %Message{result: result} = Message.send(message) + assert result == "D" + message = %Message{agent_name: :second_agent, input: result} + %Message{result: result} = Message.send(message) + assert result == "F" + message = %Message{agent_name: :verifier_agent, input: result} + %Message{result: result} = Message.send(message) + assert result == "F" + + # # 2 + message = %Message{agent_name: :first_agent, input: result} + %Message{result: result} = Message.send(message) + assert result == "E" + message = %Message{agent_name: :second_agent, input: result} + %Message{result: result} = Message.send(message) + assert result == "G" + message = %Message{agent_name: :verifier_agent, input: result} + %Message{result: result} = Message.send(message) + assert result == "TRUE" + end + end +end diff --git a/test/agens/job_test.exs b/test/agens/job_test.exs new file mode 100644 index 0000000..4360b19 --- /dev/null +++ b/test/agens/job_test.exs @@ -0,0 +1,369 @@ +defmodule Agens.JobTest do + use Test.Support.AgentCase, async: false + + alias Agens.{Agent, Job} + alias Test.Support.Tools.NoopTool + + @lm_result_timeout 100_000 + + defp start_agens(_ctx) do + {:ok, _pid} = start_supervised({Agens.Supervisor, name: Agens.Supervisor}) + :ok + end + + defp start_serving(_ctx) do + %Agens.Serving.Config{ + name: :text_generation, + serving: Test.Support.Serving.Stub + } + |> Agens.Serving.start() + + :ok + end + + defp start_job(_ctx) do + get_agent_configs() + |> Agent.start() + + job = %Job.Config{ + name: :first_job, + description: "to create a sequence of steps", + steps: [ + %Job.Step{ + agent: :first_agent, + objective: "test step objective", + conditions: nil + }, + %Job.Step{ + agent: :second_agent, + conditions: nil + }, + %Job.Step{ + agent: :verifier_agent, + conditions: %{ + "TRUE" => :end, + "__DEFAULT__" => 0 + } + } + ] + } + + {:ok, pid} = Job.start(job) + + [ + job: job, + pid: pid + ] + end + + describe "errors" do + setup [:start_agens, :start_job] + + test "start running", %{job: job, pid: pid} do + assert is_pid(pid) + + assert {:error, {:already_started, ^pid}} = Job.start(job) + end + + test "job missing" do + assert {:error, :job_not_found} == Job.run(:missing_job, "input") + end + end + + describe "config" do + setup :start_agens + + test "config" do + job = %Job.Config{ + name: :job_config, + steps: [] + } + + {:ok, pid} = Job.start(job) + + assert is_pid(pid) + assert job == Job.get_config(pid) + assert job == Job.get_config(:job_config) + assert {:error, :job_not_found} == Job.get_config(:missing_job) + end + end + + describe "sequence" do + setup [:start_agens, :start_serving, :start_job] + + @tag capture_log: true + test "start", %{job: %{name: name}, pid: pid} do + input = "D" + + assert is_pid(pid) + assert Job.run(name, input) == :ok + + assert_receive {:job_started, ^name} + + # 0 + assert_receive {:step_started, {^name, 0}, "D"} + assert_receive {:step_result, {^name, 0}, "C"} + assert_receive {:step_started, {^name, 1}, "C"} + assert_receive {:step_result, {^name, 1}, "E"} + assert_receive {:step_started, {^name, 2}, "E"} + assert_receive {:step_result, {^name, 2}, "E"} + + # 1 + assert_receive {:step_started, {^name, 0}, "E"} + assert_receive {:step_result, {^name, 0}, "D"} + assert_receive {:step_started, {^name, 1}, "D"} + assert_receive {:step_result, {^name, 1}, "F"} + assert_receive {:step_started, {^name, 2}, "F"} + assert_receive {:step_result, {^name, 2}, "F"} + + # 2 + assert_receive {:step_started, {^name, 0}, "F"} + assert_receive {:step_result, {^name, 0}, "E"} + assert_receive {:step_started, {^name, 1}, "E"} + assert_receive {:step_result, {^name, 1}, "G"} + assert_receive {:step_started, {^name, 2}, "G"} + assert_receive {:step_result, {^name, 2}, "TRUE"} + + assert_receive {:job_ended, ^name, :complete} + end + end + + describe "restart" do + setup [:start_agens, :start_serving] + + @tag capture_log: true + test "crash" do + name = :crash_job + + [ + %Agent.Config{ + name: :first_agent, + serving: :text_generation + }, + %Agent.Config{ + name: :verifier_agent, + serving: :text_generation + } + ] + |> Agent.start() + + job = %Job.Config{ + name: name, + description: "to simulate a crash", + steps: [ + %Job.Step{ + agent: :first_agent + }, + %Job.Step{ + agent: :verifier_agent, + conditions: %{ + "TRUE" => :end, + "__DEFAULT__" => :invalid + } + } + ] + } + + {:ok, pid} = Job.start(job) + + input = "F" + assert is_pid(pid) + ref = Process.monitor(pid) + result = Job.run(name, input) + assert result == :ok + assert_receive {:job_started, ^name} + + assert_receive {:step_started, {^name, 0}, "F"} + assert_receive {:step_result, {^name, 0}, "E"} + assert_receive {:step_started, {^name, 1}, "E"} + assert_receive {:step_result, {^name, 1}, "E"} + + assert_receive {:job_ended, ^name, + {:error, %RuntimeError{message: "Invalid step index: :invalid"}}} + + assert_receive {:DOWN, ^ref, :process, ^pid, _reason} + Process.demonitor(ref) + refute Process.alive?(pid) + + # TODO: alternative to Process.sleep/1? + # custom send from start_link/1 or init/1: 'from' not available + # supervisor: does not send any message when restarting child + # monitor: only notifies of terminated process + Process.sleep(100) + new_pid = GenServer.whereis(name) + assert is_pid(new_pid) + refute pid == new_pid + assert Process.alive?(new_pid) + + result = Job.run(name, input) + assert result == :ok + assert_receive {:job_started, ^name} + end + end + + describe "tool use" do + setup [:start_agens, :start_serving, :start_job] + + @tag capture_log: true + test "noop tool" do + name = :noop_job + + job = %Job.Config{ + name: name, + description: "to test tool usage", + steps: [ + %Job.Step{ + agent: :first_agent, + objective: "", + conditions: nil + }, + %Job.Step{ + agent: :tool_agent, + conditions: %{ + "TRUE" => :end, + "__DEFAULT__" => 0 + } + } + ] + } + + {:ok, pid} = Job.start(job) + + input = "F" + assert is_pid(pid) + result = Job.run(name, input) + assert result == :ok + assert_receive {:job_started, ^name} + + assert_receive {:step_started, {^name, 0}, "F"} + assert_receive {:step_result, {^name, 0}, "E"} + assert_receive {:step_started, {^name, 1}, "E"} + assert_receive {:tool_started, {^name, 1}, "FALSE"} + assert_receive {:tool_raw, {^name, 1}, %{}} + assert_receive {:tool_result, {^name, 1}, "TRUE"} + assert_receive {:step_result, {^name, 1}, "TRUE"} + + assert_receive {:job_ended, ^name, :complete} + end + end + + describe "prompt" do + setup [:start_agens, :start_serving] + + @tag capture_log: true + test "full prompt" do + job_name = :test_prompt_job + agent_name = :test_prompt_agent + input = "test input" + + prompt = %Agent.Prompt{ + identity: "test agent identity", + constraints: "test agent constraints", + context: "test agent context", + reflection: "test agent reflection" + # TODO + # examples: [ + # %{input: "A", output: "C"}, + # %{input: "F", output: "H"}, + # %{input: "9vasg2rwe", output: "ERROR"} + # ], + } + + %Agent.Config{ + name: agent_name, + serving: :text_generation, + prompt: prompt, + tool: NoopTool + } + |> Agent.start() + + %Job.Config{ + name: job_name, + description: "test job description", + steps: [ + %Job.Step{ + agent: agent_name, + objective: "test step objective", + conditions: %{ + "__DEFAULT__" => :end + } + } + ] + } + |> Job.start() + + Job.run(job_name, input) + + assert_receive {:job_started, ^job_name} + + assert_receive {:step_started, {^job_name, 0}, ^input} + assert_receive {:tool_started, {^job_name, 0}, "sent 'test input' to: test_prompt_agent"} + assert_receive {:tool_raw, {^job_name, 0}, %{}} + assert_receive {:tool_result, {^job_name, 0}, "TRUE"} + assert_receive {:step_result, {^job_name, 0}, "TRUE"} + assert_receive {:job_ended, ^job_name, :complete} + end + end + + describe "lm" do + setup :start_agens + + @tag timeout: :infinity + @tag capture_log: true + @tag :lm + test "run job" do + name = :test_lm_job + input = "start real lm job" + + {:ok, pid} = + %Agens.Serving.Config{ + name: :text_generation_lm, + serving: Test.Support.Serving.LLM.get() + } + |> Agens.Serving.start() + + assert is_pid(pid) + + [ + %Agent.Config{ + name: :first_agent, + serving: :text_generation_lm + }, + %Agent.Config{ + name: :verifier_agent, + serving: :text_generation + } + ] + |> Agent.start() + + job = %Job.Config{ + name: name, + description: "to test a real lm using Nx.Serving", + steps: [ + %Job.Step{ + agent: :first_agent + }, + %Job.Step{ + agent: :verifier_agent, + conditions: %{ + "__DEFAULT__" => :end + } + } + ] + } + + {:ok, pid} = Job.start(job) + assert is_pid(pid) + assert Job.run(name, input) == :ok + + assert_receive {:job_started, ^name} + + assert_receive {:step_started, {^name, 0}, ^input} + assert_receive {:step_result, {^name, 0}, _}, @lm_result_timeout + assert_receive {:step_started, {^name, 1}, _} + assert_receive {:step_result, {^name, 1}, _}, @lm_result_timeout + + assert_receive {:job_ended, ^name, :complete} + end + end +end diff --git a/test/agens/serving_test.exs b/test/agens/serving_test.exs new file mode 100644 index 0000000..cae6ce6 --- /dev/null +++ b/test/agens/serving_test.exs @@ -0,0 +1,98 @@ +defmodule Agens.ServingTest do + use Test.Support.AgentCase, async: false + + alias Agens.{Message, Serving} + + defp start_agens(_ctx) do + {:ok, _pid} = start_supervised({Agens.Supervisor, name: Agens.Supervisor}) + :ok + end + + defp start_serving(_ctx) do + config = %Serving.Config{ + name: :serving_test, + serving: Test.Support.Serving.Stub + } + + {:ok, pid} = Serving.start(config) + + [ + config: config, + pid: pid + ] + end + + defmodule MyDefn do + import Nx.Defn + + defn print_and_multiply(x) do + x = print_value(x, label: "debug") + x * 2 + end + end + + describe "serving" do + setup [:start_agens, :start_serving] + + test "message", %{config: config} do + input = "input" + + message = + %Message{serving_name: config.name, input: input} + |> Serving.run() + + assert message == "sent '#{input}' to: " + end + + test "start running", %{config: config, pid: pid} do + assert is_pid(pid) + + assert {:error, {:already_started, ^pid}} = Serving.start(config) + end + + test "not running" do + assert {:error, :serving_not_running} == + Serving.run(%Message{serving_name: :serving_missing, input: "input"}) + end + end + + describe "stop" do + setup [:start_agens, :start_serving] + + test "stop", %{config: config} do + assert :ok == Serving.stop(config.name) + end + + test "stop missing" do + assert {:error, :serving_not_found} == + Serving.stop(:serving_missing) + end + end + + describe "nx" do + setup :start_agens + + test "message" do + serving_name = :nx_serving_test + + config = %Serving.Config{ + name: serving_name, + serving: Nx.Serving.new(fn opts -> Nx.Defn.jit(&MyDefn.print_and_multiply/1, opts) end) + } + + {:ok, pid} = Serving.start(config) + + assert is_pid(pid) + + batch = Nx.Batch.stack([Nx.tensor([1, 2, 3])]) + + message = %Message{serving_name: serving_name, prompt: batch} + + assert %Nx.Tensor{ + type: {:s, 64}, + shape: {1, 3}, + data: %Nx.BinaryBackend{state: _} + } = Serving.run(message) + end + end +end diff --git a/test/agens/tool_test.exs b/test/agens/tool_test.exs new file mode 100644 index 0000000..d272412 --- /dev/null +++ b/test/agens/tool_test.exs @@ -0,0 +1,32 @@ +defmodule Test.ToolTest do + use Test.Support.AgentCase, async: false + + defmodule TestTool do + @behaviour Agens.Tool + def pre(input), do: "pre: #{input}" + def instructions(), do: "test tool instructions" + def to_args(_input), do: [t: "e", s: "t"] + def execute(args), do: Enum.into(args, %{}) + def post(result), do: Enum.map(result, fn {k, v} -> "#{k}: #{v}" end) |> Enum.join(", ") + end + + describe "tool" do + test "all" do + input = "test" + result = "test" + + assert TestTool.pre(input) == "pre: #{input}" + assert TestTool.instructions() == "test tool instructions" + + args = TestTool.to_args(result) + assert args == [t: "e", s: "t"] + + result = TestTool.execute(args) + assert result == %{t: "e", s: "t"} + + post = TestTool.post(result) + assert post =~ "t: e" + assert post =~ "s: t" + end + end +end diff --git a/test/agens_test.exs b/test/agens_test.exs index dce752a..43290f9 100644 --- a/test/agens_test.exs +++ b/test/agens_test.exs @@ -1,8 +1,4 @@ -defmodule AgensTest do - use ExUnit.Case - doctest Agens - - test "greets the world" do - assert Agens.hello() == :world - end +defmodule Agens.AgensTest do + use Test.Support.AgentCase, async: false + # doctest Agens end diff --git a/test/support/agent_case.ex b/test/support/agent_case.ex new file mode 100644 index 0000000..1d9ecfe --- /dev/null +++ b/test/support/agent_case.ex @@ -0,0 +1,56 @@ +defmodule Test.Support.AgentCase do + use ExUnit.CaseTemplate + + using do + quote do + import Test.Support.AgentCase + import Test.Support.Helpers + end + end + + alias Agens.Agent + alias Test.Support + + def get_agent_configs() do + [ + %Agent.Config{ + name: :first_agent, + serving: :text_generation, + prompt: + "Return the capital letter one place before the letter in the English alphabet provided after 'Input: '. If you reach the start of the alphabet, cycle to the end of the alphabet i.e. 'Z'. For invalid input, which would be anything other than a single letter after 'Input: ' simply return 'ERROR'. The output response should only be the letter without any additional characters, tokens, or whitespace, or ERROR in case of invalid input.", + knowledge: "" + }, + %Agent.Config{ + name: :second_agent, + serving: :text_generation, + prompt: %Agent.Prompt{ + identity: + "You are an AI agent that takes an input letter of the English alphabet and returns the capital letter two places ahead of the letter. If the input is anything but a single letter, your return 'ERROR'", + constraints: + "Your output should only be a single capital letter from the English alphabet, or 'ERROR'", + # examples: [ + # %{input: "A", output: "C"}, + # %{input: "F", output: "H"}, + # %{input: "9vasg2rwe", output: "ERROR"} + # ], + reflection: + "Before returning a result please ensure it is either a capital letter of the English alphabet or 'ERROR'" + }, + # "[INST]Which letter comes after '#{msg}' in the English alphabet? Return the letter only, no extra words, characters or tokens.[/INST]" + # prompt: "Return the capital letter two places ahead of the letter in the English alphabet provided after 'User input: '. If you reach the end of the alphabet, cycle back to the beginning of the alphabet. For invalid input, which would be anything other than a single letter after 'User input: ' simply return 'ERROR'. The output response should only be the letter without any additional characters, tokens, or whitespace, or ERROR in case of invalid input.", + knowledge: "" + }, + %Agent.Config{ + name: :verifier_agent, + serving: :text_generation, + prompt: "Return 'TRUE' if input is 'G', otherwise return 'FALSE'", + knowledge: "" + }, + %Agent.Config{ + name: :tool_agent, + serving: :text_generation, + tool: Support.Tools.NoopTool + } + ] + end +end diff --git a/test/support/helpers.ex b/test/support/helpers.ex new file mode 100644 index 0000000..5f24dd3 --- /dev/null +++ b/test/support/helpers.ex @@ -0,0 +1,51 @@ +defmodule Test.Support.Helpers do + def post_process(text) do + cond do + String.contains?(text, "Based on the given input") -> + text + |> String.split("`") + |> Enum.at(3) + + String.contains?(text, "Here's a brief explanation of the logic behind the code:") -> + String.first(text) + + String.contains?(text, "Here's a Python solution") -> + String.first(text) + + String.contains?(text, "TRUE") -> + "TRUE" + + String.contains?(text, "FALSE") -> + "FALSE" + + true -> + text + end + end + + def map_input(:first_agent, input) do + %{ + "D" => "C", + "E" => "D", + "F" => "E" + } + |> Map.get(input, "ERROR") + end + + def map_input(:second_agent, input) do + %{ + "C" => "E", + "D" => "F", + "E" => "G" + } + |> Map.get(input, "ERROR") + end + + def map_input(:verifier_agent, input) do + if input == "G", do: "TRUE", else: input + end + + def map_input(:tool_agent, "E"), do: "FALSE" + + def map_input(agent, input), do: "sent '#{input}' to: #{agent}" +end diff --git a/test/support/serving.ex b/test/support/serving.ex new file mode 100644 index 0000000..79a0400 --- /dev/null +++ b/test/support/serving.ex @@ -0,0 +1,45 @@ +defmodule Test.Support.Serving do + defmodule LLM do + def get() do + IO.puts("Enabling EXLA Backend") + Application.put_env(:nx, :default_backend, EXLA.Backend) + auth_token = System.get_env("HF_AUTH_TOKEN") + repo = {:hf, "mistralai/Mistral-7B-Instruct-v0.2", auth_token: auth_token} + + IO.puts("Loading Model") + {:ok, model} = Bumblebee.load_model(repo, type: :bf16) + {:ok, tokenizer} = Bumblebee.load_tokenizer(repo) + {:ok, generation_config} = Bumblebee.load_generation_config(repo) + + IO.puts("Starting LLM") + serving = Bumblebee.Text.generation(model, tokenizer, generation_config) + IO.puts("LLM Ready") + + serving + end + end + + defmodule Stub do + use GenServer + + def get() do + __MODULE__ + end + + def start_link(opts) do + GenServer.start_link(__MODULE__, opts, opts) + end + + def init(opts) do + {:ok, opts} + end + + # Normally a Serving would use `message.prompt` rather than `message.input` + # The Job or Agent would use config and `message.input` to build `message.prompt` + # In this case, using `message.input` instead to map to a result simplifies testing + def handle_call({:run, %Agens.Message{} = message}, _, state) do + result = Test.Support.Helpers.map_input(message.agent_name, message.input) + {:reply, result, state} + end + end +end diff --git a/test/support/tools/noop_tool.ex b/test/support/tools/noop_tool.ex new file mode 100644 index 0000000..9adc250 --- /dev/null +++ b/test/support/tools/noop_tool.ex @@ -0,0 +1,18 @@ +defmodule Test.Support.Tools.NoopTool do + @behaviour Agens.Tool + + @impl true + def pre(input), do: input + + @impl true + def instructions(), do: "noop tool instructions" + + @impl true + def to_args(_input), do: [] + + @impl true + def execute(_args), do: %{} + + @impl true + def post(_result), do: "TRUE" +end diff --git a/test/test_helper.exs b/test/test_helper.exs index 869559e..3b025fb 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -1 +1 @@ -ExUnit.start() +ExUnit.start(exclude: [:lm])