feat(tests): add SQL tools and integration tests 🎉

- Introduced `SQLTestAgent` for testing SQL-related functionalities.
- Added support for `ListSQLDatabaseTool`, `InfoSQLDatabaseTool`, and `QuerySQLDataBaseTool`.
- Created `Organization` model, migration, and factory for test data.
- Enhanced `TestCase` with database migration and seeding.
- Updated documentation to include SQL toolset examples.

docs/tools/packaged-tools.md

@@ -115,3 +115,30 @@ protected function resolveTools(): array
  return [new ClearbitCompanyTool];
 }
 ```
+
+## SQL Database Tool
+
+The SQL Database toolset allows the agent to browse and answer questions based on the database your Laravel application is running. This toolset is adapted from the LangChain SQL Database tool: [LangChain SQL Database Tool](https://api.python.langchain.com/en/latest/_modules/langchain_community/tools/sql_database/tool.html).
+
+There are three tools your agent can use:
+
+- **`InfoSQLDatabaseTool`**: Retrieves the schema and sample rows from specified SQL tables using `Schema::getColumns($table)` and `DB::select("SELECT * FROM {$table} LIMIT 3")`.
+- **`ListSQLDatabaseTool`**: Lists the table names in your database using `Schema::getTables()`.
+- **`QuerySQLDataBaseTool`**: Executes a select query against the database using `DB::select($query)`.
+
+Typically, you would include all three tools in your agent:
+
+```php
+use UseTheFork\Synapse\Tools\SQL\InfoSQLDatabaseTool;
+use UseTheFork\Synapse\Tools\SQL\ListSQLDatabaseTool;
+use UseTheFork\Synapse\Tools\SQL\QuerySQLDataBaseTool;
+
+protected function resolveTools(): array
+{
+ return [
+ new InfoSQLDatabaseTool,
+ new ListSQLDatabaseTool,
+ new QuerySQLDataBaseTool,
+ ];
+}
+```

src/Tools/SQL/InfoSQLDatabaseTool.php

@@ -0,0 +1,33 @@
+<?php
+
+ declare(strict_types=1);
+
+ namespace UseTheFork\Synapse\Tools\SQL;
+
+ use Illuminate\Support\Facades\DB;
+ use Illuminate\Support\Facades\Schema;
+ use UseTheFork\Synapse\Tools\BaseTool;
+
+ final class InfoSQLDatabaseTool extends BaseTool
+ {
+
+ /**
+ * Get the schema and sample rows for the specified SQL tables.
+ *
+ * @param string $tables A comma seperated list of tables to get information about.
+ */
+ public function handle(
+ string $tables,
+ ): string {
+
+ $tableDescriptions = collect();
+ str($tables)->explode(',')->each(function ($table) use (&$tableDescriptions) {
+ $columns = json_encode(Schema::getColumns($table));
+ $sample = json_encode(DB::select("SELECT * FROM {$table} LIMIT 3"));
+ $tableDescriptions->push("### {$table}\n\n```json\n{$columns}\n\n```\n\n```json\n{$sample}\n\n```\n\n");
+
+ });
+
+ return "## Schema\n" . $tableDescriptions->implode("\n\n");
+ }
+ }

src/Tools/SQL/ListSQLDatabaseTool.php

@@ -0,0 +1,20 @@
+<?php
+
+ declare(strict_types=1);
+
+ namespace UseTheFork\Synapse\Tools\SQL;
+
+ use Illuminate\Support\Facades\Schema;
+ use UseTheFork\Synapse\Tools\BaseTool;
+
+ final class ListSQLDatabaseTool extends BaseTool
+ {
+ /**
+ * Tool for getting tables names. Output is a comma-separated list of tables in the database.
+ *
+ */
+ public function handle(): string {
+ $tables = collect(Schema::getTables());
+ return $tables->map(function ($table) { return $table['name']; })->implode(', ');
+ }
+ }

src/Tools/SQL/QuerySQLDataBaseTool.php

@@ -0,0 +1,30 @@
+<?php
+
+ declare(strict_types=1);
+
+ namespace UseTheFork\Synapse\Tools\SQL;
+
+ use Illuminate\Database\QueryException;
+ use Illuminate\Support\Facades\DB;
+ use UseTheFork\Synapse\Tools\BaseTool;
+
+ final class QuerySQLDataBaseTool extends BaseTool
+ {
+
+ /**
+ * Execute a SQL SELECT query against the database and get back the result. If the query is not correct, an error message will be returned. If an error is returned, rewrite the query, check the query, and try again.
+ *
+ * @param string $query A detailed and correct SQL SELECT query.
+ */
+ public function handle(
+ string $query,
+ ): string {
+ try {
+ $query = DB::select($query);
+ } catch (QueryException $e) {
+ return $e->getMessage();
+ }
+
+ return json_encode($query);
+ }
+ }

tests/Fixtures/Saloon/Tools/SQLTestAgent-16ade5c7afd2bc82237a2c2c38ac43da.json

@@ -0,0 +1,10 @@
+{
+ "statusCode": 200,
+ "headers": {
+ "Date": "Sat, 12 Oct 2024 15:46:24 GMT",
+ "Content-Type": "application\/json",
+ "Content-Length": "1044",
+ "Connection": "keep-alive"
+ },
+ "data": "{\n \"id\": \"chatcmpl-AHYbvg2Z022Dp9J1lKpW8A7JyNBqe\",\n \"object\": \"chat.completion\",\n \"created\": 1728747983,\n \"model\": \"gpt-4-turbo-2024-04-09\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": null,\n \"tool_calls\": [\n {\n \"id\": \"call_C9LE4B5gWbExU1y747Hgs2wZ\",\n \"type\": \"function\",\n \"function\": {\n \"name\": \"query_s_q_l_data_base_tool\",\n \"arguments\": \"{\\\"query\\\":\\\"SELECT COUNT(*) AS Total_Organizations, AVG(funding_rounds) AS Average_Funding_Rounds FROM organizations WHERE status = 'operating'\\\"}\"\n }\n }\n ],\n \"refusal\": null\n },\n \"logprobs\": null,\n \"finish_reason\": \"tool_calls\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 298,\n \"completion_tokens\": 48,\n \"total_tokens\": 346,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0\n }\n },\n \"system_fingerprint\": \"fp_83975a045a\"\n}\n"
+}

tests/Fixtures/Saloon/Tools/SQLTestAgent-17f20fbcbd7ad04b632e81d68f84cf54.json

@@ -0,0 +1,10 @@
+{
+ "statusCode": 200,
+ "headers": {
+ "Date": "Sat, 12 Oct 2024 15:46:31 GMT",
+ "Content-Type": "application\/json",
+ "Content-Length": "786",
+ "Connection": "keep-alive"
+ },
+ "data": "{\n \"id\": \"chatcmpl-AHYc1C2fumsoLczurFDZ8Kuns7CRP\",\n \"object\": \"chat.completion\",\n \"created\": 1728747989,\n \"model\": \"gpt-4-turbo-2024-04-09\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"```json\\n{\\n \\\"answer\\\": \\\"There are 100 organizations currently operating, and the average number of funding rounds for these organizations is 5.\\\"\\n}\\n```\",\n \"refusal\": null\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 1214,\n \"completion_tokens\": 34,\n \"total_tokens\": 1248,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0\n }\n },\n \"system_fingerprint\": \"fp_83975a045a\"\n}\n"
+}

tests/Fixtures/Saloon/Tools/SQLTestAgent-4c3078626dcfa6950ae2eb9ae69d71d9.json

@@ -0,0 +1,10 @@
+{
+ "statusCode": 200,
+ "headers": {
+ "Date": "Sat, 12 Oct 2024 15:46:26 GMT",
+ "Content-Type": "application/json",
+ "Content-Length": "925",
+ "Connection": "keep-alive"
+ },
+ "data": "{\n \"id\": \"chatcmpl-AHYbyju4ZTcAQag0iGgiU4S6DD3gd\",\n \"object\": \"chat.completion\",\n \"created\": 1728747986,\n \"model\": \"gpt-4-turbo-2024-04-09\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": null,\n \"tool_calls\": [\n {\n \"id\": \"call_9ABiqHRAVvqiks60jaHYI9L7\",\n \"type\": \"function\",\n \"function\": {\n \"name\": \"info_s_q_l_database_tool\",\n \"arguments\": \"{\\\"tables\\\":\\\"organizations\\\"}\"\n }\n }\n ],\n \"refusal\": null\n },\n \"logprobs\": null,\n \"finish_reason\": \"tool_calls\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 414,\n \"completion_tokens\": 18,\n \"total_tokens\": 432,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0\n }\n },\n \"system_fingerprint\": \"fp_4dba7dd7b3\"\n}\n"
+}

tests/Fixtures/Saloon/Tools/SQLTestAgent-5c0d5e203062af04810663fcedb6fcee.json

@@ -0,0 +1,10 @@
+{
+ "statusCode": 200,
+ "headers": {
+ "Date": "Sat, 12 Oct 2024 15:46:22 GMT",
+ "Content-Type": "application/json",
+ "Content-Length": "897",
+ "Connection": "keep-alive"
+ },
+ "data": "{\n \"id\": \"chatcmpl-AHYbuZGSMw35f3TwaP1nMvPUvhNmK\",\n \"object\": \"chat.completion\",\n \"created\": 1728747982,\n \"model\": \"gpt-4-turbo-2024-04-09\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": null,\n \"tool_calls\": [\n {\n \"id\": \"call_3UbiicoV2jrxS6qtWp2H2X3X\",\n \"type\": \"function\",\n \"function\": {\n \"name\": \"list_s_q_l_database_tool\",\n \"arguments\": \"{}\"\n }\n }\n ],\n \"refusal\": null\n },\n \"logprobs\": null,\n \"finish_reason\": \"tool_calls\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 241,\n \"completion_tokens\": 14,\n \"total_tokens\": 255,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0\n }\n },\n \"system_fingerprint\": \"fp_83975a045a\"\n}\n"
+}

tests/Fixtures/Saloon/Tools/SQLTestAgent-6c936275667473da519c385ffd4d808c.json

@@ -0,0 +1,10 @@
+{
+ "statusCode": 200,
+ "headers": {
+ "Date": "Sat, 12 Oct 2024 15:46:29 GMT",
+ "Content-Type": "application/json",
+ "Content-Length": "1050",
+ "Connection": "keep-alive"
+ },
+ "data": "{\n \"id\": \"chatcmpl-AHYbzWAqcQdUtpiRgQCj91a8MxLsO\",\n \"object\": \"chat.completion\",\n \"created\": 1728747987,\n \"model\": \"gpt-4-turbo-2024-04-09\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": null,\n \"tool_calls\": [\n {\n \"id\": \"call_HRfBVryYOnQu8uYZPZ6dePje\",\n \"type\": \"function\",\n \"function\": {\n \"name\": \"query_s_q_l_data_base_tool\",\n \"arguments\": \"{\\\"query\\\":\\\"SELECT COUNT(*) AS Total_Organizations, AVG(num_funding_rounds) AS Average_Funding_Rounds FROM organizations WHERE status = 'operating'\\\"}\"\n }\n }\n ],\n \"refusal\": null\n },\n \"logprobs\": null,\n \"finish_reason\": \"tool_calls\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 1135,\n \"completion_tokens\": 49,\n \"total_tokens\": 1184,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0\n }\n },\n \"system_fingerprint\": \"fp_83975a045a\"\n}\n"
+}

tests/TestCase.php

@@ -6,20 +6,16 @@
 
 use Illuminate\Contracts\Config\Repository;
 use Illuminate\Foundation\Bootstrap\LoadEnvironmentVariables;
+use Illuminate\Foundation\Testing\RefreshDatabase;
 use Orchestra\Testbench\Concerns\WithWorkbench;
 use Orchestra\Testbench\TestCase as Orchestra;
 use UseTheFork\Synapse\SynapseServiceProvider;
+use function Orchestra\Testbench\workbench_path;
 
 abstract class TestCase extends Orchestra
 {
  use WithWorkbench;
-
- protected function getPackageProviders($app): array
- {
- return [
- SynapseServiceProvider::class,
- ];
- }
+ use RefreshDatabase;
 
  /**
  * Define database migrations.
@@ -30,12 +26,16 @@ protected function defineDatabaseMigrations(): void
  $this->loadMigrationsFrom(
  __DIR__.'/../database/migrations'
  );
- }
 
- protected function defineEnvironment($app): void {}
+ $this->loadMigrationsFrom(
+ workbench_path('database/migrations')
+ );
+ }
 
  protected function defineDatabaseSeeders(): void {}
 
+ protected function defineEnvironment($app): void {}
+
  protected function getEnvironmentSetUp($app): void
  {
 
@@ -53,4 +53,11 @@ protected function getEnvironmentSetUp($app): void
 
  parent::getEnvironmentSetUp($app);
  }
+
+ protected function getPackageProviders($app): array
+ {
+ return [
+ SynapseServiceProvider::class,
+ ];
+ }
 }

tests/Tools/SQLToolTest.php

@@ -0,0 +1,124 @@
+<?php
+
+declare(strict_types=1);
+
+ use Saloon\Http\Faking\Fixture;
+ use Saloon\Http\Faking\MockClient;
+ use Saloon\Http\Faking\MockResponse;
+ use Saloon\Http\PendingRequest;
+ use UseTheFork\Synapse\Agent;
+ use UseTheFork\Synapse\Contracts\Agent\HasOutputSchema;
+ use UseTheFork\Synapse\Contracts\Integration;
+ use UseTheFork\Synapse\Contracts\Tool;
+ use UseTheFork\Synapse\Integrations\Connectors\OpenAI\Requests\ChatRequest;
+ use UseTheFork\Synapse\Integrations\OpenAIIntegration;
+ use UseTheFork\Synapse\Tools\BaseTool;
+ use UseTheFork\Synapse\Tools\SQL\InfoSQLDatabaseTool;
+ use UseTheFork\Synapse\Tools\SQL\ListSQLDatabaseTool;
+ use UseTheFork\Synapse\Tools\SQL\QuerySQLDataBaseTool;
+ use UseTheFork\Synapse\Traits\Agent\ValidatesOutputSchema;
+ use UseTheFork\Synapse\ValueObject\SchemaRule;
+ use Workbench\App\Models\Organization;
+
+ test('SQL Tool', function (): void {
+
+
+ for ($i = 0; $i < 100; $i++){
+ $org = new Organization();
+ $org->fill([
+ 'name' => "Foo - {$i}",
+ 'domain' => "Foo_{$i}.com",
+ 'country_code' => 'USA',
+ 'email' => '[email protected]',
+ 'city' => 'hartford',
+ 'status' => 'operating',
+ 'short_description' => 'lorem ipsum',
+ 'num_funding_rounds' => 5,
+ 'total_funding_usd' => 1000000,
+ 'founded_on' => '2024-03-01',
+ ]);
+ $org->save();
+ }
+
+ for ($i = 0; $i < 100; $i++){
+ $org = new Organization();
+ $org->fill([
+ 'name' => "Baz - {$i}",
+ 'domain' => "Baz_{$i}.com",
+ 'country_code' => 'USA',
+ 'email' => '[email protected]',
+ 'city' => 'hartford',
+ 'status' => 'closed',
+ 'short_description' => 'lorem ipsum',
+ 'num_funding_rounds' => 5,
+ 'total_funding_usd' => 1000000,
+ 'founded_on' => '2024-03-01',
+ ]);
+ $org->save();
+ }
+
+
+ class SQLTestAgent extends Agent implements HasOutputSchema
+ {
+ use ValidatesOutputSchema;
+
+ protected string $promptView = 'synapse::Prompts.SimplePrompt';
+
+ public function resolveIntegration(): Integration
+ {
+ return new OpenAIIntegration;
+ }
+
+ public function resolveOutputSchema(): array
+ {
+ return [
+ SchemaRule::make([
+ 'name' => 'answer',
+ 'rules' => 'required|string',
+ 'description' => 'your final answer to the query.',
+ ]),
+ ];
+ }
+
+ protected function resolveTools(): array
+ {
+ return [
+ new ListSQLDatabaseTool,
+ new InfoSQLDatabaseTool,
+ new QuerySQLDataBaseTool,
+ ];
+ }
+ }
+
+
+ MockClient::global([
+ ChatRequest::class => function (PendingRequest $pendingRequest): Fixture {
+ $hash = md5(json_encode($pendingRequest->body()->get('messages')));
+
+ return MockResponse::fixture("Tools/SQLTestAgent-{$hash}");
+ }
+ ]);
+
+ $agent = new SQLTestAgent;
+ $message = $agent->handle(['input' => 'How many organizations are operating and what is the average number of funding rounds for them?']);
+
+ $agentResponseArray = $message->toArray();
+ expect($agentResponseArray['content'])->toBeArray()
+ ->and($agentResponseArray['content'])->toHaveKey('answer')
+ ->and($agentResponseArray['content']['answer'])->toContain('There are 100 organizations currently operating, and the average number of funding rounds for these organizations is 5.');
+
+});
+
+test('Architecture', function (): void {
+
+ expect(ListSQLDatabaseTool::class)
+ ->toExtend(BaseTool::class)
+ ->toImplement(Tool::class)
+ ->and(InfoSQLDatabaseTool::class)
+ ->toExtend(BaseTool::class)
+ ->toImplement(Tool::class)
+ ->and(QuerySQLDataBaseTool::class)
+ ->toExtend(BaseTool::class)
+ ->toImplement(Tool::class);
+
+});

workbench/app/Models/Organization.php

@@ -0,0 +1,23 @@
+<?php
+
+namespace Workbench\App\Models;
+
+use Illuminate\Database\Eloquent\Factories\HasFactory;
+use Illuminate\Database\Eloquent\Model;
+use Workbench\Database\Factories\OrganizationFactory;
+
+class Organization extends Model
+{
+ use HasFactory;
+
+ public $timestamps = false;
+
+ protected $connection = 'testing';
+
+ protected $guarded = [];
+
+ public static function newFactory(): OrganizationFactory
+ {
+ return OrganizationFactory::new();
+ }
+}