diff --git a/sdks/python/examples/demo_data_generator.py b/sdks/python/examples/demo_data_generator.py new file mode 100644 index 000000000..87c51e5f0 --- /dev/null +++ b/sdks/python/examples/demo_data_generator.py @@ -0,0 +1,573 @@ +# In order to run this script, you will to have the opik and langchain SDK installed +# Assumes that OpenAI API key is set in the environment variable OPENAI_API_KEY +import datetime +from uuid_extensions import uuid7str +import requests + + +def make_http_request(base_url, message, workspace_name, comet_api_key): + try: + headers = {} + if workspace_name: + headers["Comet-Workspace"] = workspace_name + + if comet_api_key: + headers["authorization"] = f"{comet_api_key}" + + url = base_url + message["url"] + response = requests.request( + message["method"], url, json=message["payload"], headers=headers + ) + print(response.status_code) + except Exception as e: + print(e) + + +def create_experiment_items(experiment_id: str): + items = [ + { + "experiment_id": experiment_id, + "dataset_item_id": uuid7str(), + "trace_id": uuid7str(), + "input": { + "question": "What is the total revenue generated by each country?" + }, + "output": { + "output": 'SELECT c."Country", SUM(il."UnitPrice" * il."Quantity") AS "TotalRevenue"\nFROM "Invoice" i\nJOIN "Customer" c ON i."CustomerId" = c."CustomerId"\nJOIN "InvoiceLine" il ON i."InvoiceId" = il."InvoiceId"\nGROUP BY c."Country"\nORDER BY "TotalRevenue" DESC;\nLIMIT 5;' + }, + "score": 0, + "reason": '(sqlite3.ProgrammingError) You can only execute one statement at a time.\n[SQL: SELECT c."Country", SUM(il."UnitPrice" * il."Quantity") AS "TotalRevenue"\nFROM "Invoice" i\nJOIN "Customer" c ON i."CustomerId" = c."CustomerId"\nJOIN "InvoiceLine" il ON i."InvoiceId" = il."InvoiceId"\nGROUP BY c."Country"\nORDER BY "TotalRevenue" DESC;\nLIMIT 5;]\n(Background on this error at: https://sqlalche.me/e/20/f405)', + }, + { + "experiment_id": experiment_id, + "dataset_item_id": uuid7str(), + "trace_id": uuid7str(), + "input": { + "question": "Which employee has the highest average number of sales per month?" + }, + "output": { + "output": "SELECT e.FirstName, e.LastName, AVG(i.Total) as AvgSalesPerMonth\nFROM Employee e\nJOIN Customer c ON e.EmployeeId = c.SupportRepId\nJOIN Invoice i ON c.CustomerId = i.CustomerId\nGROUP BY e.EmployeeId\nORDER BY AvgSalesPerMonth DESC\nLIMIT 1;" + }, + "score": 1, + "reason": "Query ran successfully", + }, + { + "experiment_id": experiment_id, + "dataset_item_id": uuid7str(), + "trace_id": uuid7str(), + "input": { + "question": "What is the average revenue generated per customer in each genre?" + }, + "output": { + "output": 'SELECT g."Name" AS "Genre", AVG(il."UnitPrice" * il."Quantity") AS "Average Revenue per Customer"\nFROM "Genre" g\nJOIN "Track" t ON g."GenreId" = t."GenreId"\nJOIN "InvoiceLine" il ON t."TrackId" = il."TrackId"\nGROUP BY g."Name"\nORDER BY "Average Revenue per Customer" DESC\nLIMIT 5;' + }, + "score": 1, + "reason": "Query ran successfully", + }, + { + "experiment_id": experiment_id, + "dataset_item_id": uuid7str(), + "trace_id": uuid7str(), + "input": { + "question": "How many unique customers made at least one purchase in each year?" + }, + "output": { + "output": "SELECT COUNT(DISTINCT c.CustomerId) AS UniqueCustomers\nFROM Customer c\nWHERE c.CustomerId IN (\n SELECT i.CustomerId\n FROM Invoice i\n GROUP BY i.CustomerId\n HAVING COUNT(DISTINCT strftime('%Y', i.InvoiceDate)) = (SELECT COUNT(DISTINCT strftime('%Y', InvoiceDate)) FROM Invoice)\n)" + }, + "score": 1, + "reason": "Query ran successfully", + }, + { + "experiment_id": experiment_id, + "dataset_item_id": uuid7str(), + "trace_id": uuid7str(), + "input": { + "question": "Which customers have made purchases in every genre?" + }, + "output": { + "output": 'SELECT c."CustomerId", c."FirstName", c."LastName"\nFROM "Customer" c\nWHERE c."CustomerId" IN (\n SELECT i."CustomerId"\n FROM "Invoice" i\n JOIN "InvoiceLine" il ON i."InvoiceId" = il."InvoiceId"\n JOIN "Track" t ON il."TrackId" = t."TrackId"\n JOIN "Genre" g ON t."GenreId" = g."GenreId"\n GROUP BY i."CustomerId"\n HAVING COUNT(DISTINCT g."GenreId") = (SELECT COUNT(*) FROM "Genre")\n)\nORDER BY c."CustomerId"\nLIMIT 5;' + }, + "score": 1, + "reason": "Query ran successfully", + }, + { + "experiment_id": experiment_id, + "dataset_item_id": uuid7str(), + "trace_id": uuid7str(), + "input": { + "question": "What is the total revenue generated by each customer in France?" + }, + "output": { + "output": 'SELECT c."CustomerId", c."FirstName", c."LastName", c."Country", SUM(il."UnitPrice" * il."Quantity") AS "TotalRevenue"\nFROM "Customer" c\nJOIN "Invoice" i ON c."CustomerId" = i."CustomerId"\nJOIN "InvoiceLine" il ON i."InvoiceId" = il."InvoiceId"\nWHERE c."Country" = \'France\'\nGROUP BY c."CustomerId"\nORDER BY "TotalRevenue" DESC;\nLIMIT 5;' + }, + "score": 0, + "reason": '(sqlite3.ProgrammingError) You can only execute one statement at a time.\n[SQL: SELECT c."CustomerId", c."FirstName", c."LastName", c."Country", SUM(il."UnitPrice" * il."Quantity") AS "TotalRevenue"\nFROM "Customer" c\nJOIN "Invoice" i ON c."CustomerId" = i."CustomerId"\nJOIN "InvoiceLine" il ON i."InvoiceId" = il."InvoiceId"\nWHERE c."Country" = \'France\'\nGROUP BY c."CustomerId"\nORDER BY "TotalRevenue" DESC;\nLIMIT 5;]\n(Background on this error at: https://sqlalche.me/e/20/f405)', + }, + { + "experiment_id": experiment_id, + "dataset_item_id": uuid7str(), + "trace_id": uuid7str(), + "input": { + "question": "How does the average revenue per sale compare between digital and physical purchases?" + }, + "output": { + "output": "SELECT AVG(il.UnitPrice * il.Quantity) AS AverageRevenue, mt.Name AS MediaType\nFROM InvoiceLine il\nJOIN Track t ON il.TrackId = t.TrackId\nJOIN MediaType mt ON t.MediaTypeId = mt.MediaTypeId\nGROUP BY mt.Name\nORDER BY AverageRevenue DESC\nLIMIT 2;" + }, + "score": 1, + "reason": "Query ran successfully", + }, + { + "experiment_id": experiment_id, + "dataset_item_id": uuid7str(), + "trace_id": uuid7str(), + "input": { + "question": "Which employee has the highest average revenue per sale?" + }, + "output": { + "output": 'SELECT e."FirstName", e."LastName", AVG(il."UnitPrice") AS avg_revenue_per_sale\nFROM "Employee" e\nJOIN "Customer" c ON e."EmployeeId" = c."SupportRepId"\nJOIN "Invoice" i ON c."CustomerId" = i."CustomerId"\nJOIN "InvoiceLine" il ON i."InvoiceId" = il."InvoiceId"\nGROUP BY e."EmployeeId"\nORDER BY avg_revenue_per_sale DESC\nLIMIT 1;' + }, + "score": 1, + "reason": "Query ran successfully", + }, + { + "experiment_id": experiment_id, + "dataset_item_id": uuid7str(), + "trace_id": uuid7str(), + "input": { + "question": "What is the average revenue generated by each country?" + }, + "output": { + "output": 'SELECT c."Country", AVG(i."Total") AS "Average Revenue"\nFROM "Customer" c\nJOIN "Invoice" i ON c."CustomerId" = i."CustomerId"\nGROUP BY c."Country"\nORDER BY "Average Revenue" DESC\nLIMIT 5;' + }, + "score": 1.000000000, + "reason": "Query ran successfully", + }, + { + "experiment_id": experiment_id, + "dataset_item_id": uuid7str(), + "trace_id": uuid7str(), + "input": { + "question": "How many unique albums have been purchased by each customer?" + }, + "output": { + "output": 'SELECT c."CustomerId", c."FirstName", c."LastName", COUNT(DISTINCT i."AlbumId") AS "UniqueAlbumsPurchased"\nFROM "Customer" c\nJOIN "Invoice" i ON c."CustomerId" = i."CustomerId"\nJOIN "InvoiceLine" il ON i."InvoiceId" = il."InvoiceId"\nJOIN "Track" t ON il."TrackId" = t."TrackId"\nGROUP BY c."CustomerId"\nORDER BY "UniqueAlbumsPurchased" DESC\nLIMIT 5;' + }, + "score": 0, + "reason": '(sqlite3.OperationalError) no such column: i.AlbumId\n[SQL: SELECT c."CustomerId", c."FirstName", c."LastName", COUNT(DISTINCT i."AlbumId") AS "UniqueAlbumsPurchased"\nFROM "Customer" c\nJOIN "Invoice" i ON c."CustomerId" = i."CustomerId"\nJOIN "InvoiceLine" il ON i."InvoiceId" = il."InvoiceId"\nJOIN "Track" t ON il."TrackId" = t."TrackId"\nGROUP BY c."CustomerId"\nORDER BY "UniqueAlbumsPurchased" DESC\nLIMIT 5;]\n(Background on this error at: https://sqlalche.me/e/20/e3q8)', + }, + ] + + return items + + +def insert_openai_project_example(project_name, experiment_items): + trace_id = uuid7str() + span_id = uuid7str() + + prompt_response = ",\n".join([x["input"]["question"] for x in experiment_items]) + + start_time = datetime.datetime.now() + + messages = [ + { + "url": "/v1/private/traces", + "method": "POST", + "payload": { + "id": trace_id, + "name": "chat_completion_create", + "project_name": project_name, + "start_time": (start_time).isoformat() + "Z", + "end_time": (start_time + datetime.timedelta(seconds=4.5)).isoformat() + + "Z", + "input": { + "messages": [ + { + "role": "user", + "content": '\nCreate 10 different example questions a user might ask based on the Chinook Database.\n\nThese questions should be complex and require the model to think. They should include complex joins and window functions to answer.\n\nReturn the response as a json object with a "result" key and an array of strings with the question.\n', + } + ] + }, + "output": { + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": None, + "message": { + "content": f'{{\n "result": [\n{prompt_response}\n]\n}}', + "refusal": None, + "role": "assistant", + "function_call": None, + "tool_calls": None, + }, + } + ] + }, + "metadata": { + "created_from": "openai", + "type": "openai_chat", + "model": "gpt-3.5-turbo", + }, + "tags": ["openai"], + }, + }, + { + "url": "/v1/private/spans", + "method": "POST", + "payload": { + "id": span_id, + "trace_id": trace_id, + "project_name": project_name, + "name": "chat_completion_create", + "start_time": (start_time).isoformat() + "Z", + "end_time": (start_time + datetime.timedelta(seconds=4.5)).isoformat() + + "Z", + "type": "llm", + "input": { + "messages": [ + { + "role": "user", + "content": '\nCreate 10 different example questions a user might ask based on the Chinook Database.\n\nThese questions should be complex and require the model to think. They should include complex joins and window functions to answer.\n\nReturn the response as a json object with a "result" key and an array of strings with the question.\n', + } + ] + }, + "output": { + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": None, + "message": { + "content": '{\n "result": [\n "What is the total sales revenue for each genre in the last quarter?",\n "How does the average order value compare between customers from the USA and Canada?",\n "Which customer has spent the most money on multiple orders within a single month?",\n "Which album has the highest number of tracks and what is its total duration?",\n "How many unique customers have made purchases in each country?",\n "What is the average number of tracks purchased by customers who have purchased at least 10 tracks?",\n "In which month did the highest number of new customers make their first purchase?",\n "Which employee has the highest total sales revenue and how does it compare to the average employee?",\n "What is the total sales revenue for each city when considering only customers who have made more than one purchase?",\n "Which genre has the highest number of tracks and what is its total duration?",\n "What is the average number of distinct tracks purchased by customers who have purchased from at least three different genres?",\n "What is the average number of days between a customer\'s first and last purchase?",\n "How many customers have made a purchase in at least 3 different cities?",\n "Which employee has the highest average order value and how does it compare to the overall average?",\n "Which artist has the highest total sales revenue and how many unique customers have purchased tracks by that artist?",\n "What is the average number of tracks purchased by customers who have made a purchase within 7 days of signing up?",\n "Which country has the highest average sales revenue per customer?",\n "What is the total sales revenue for each year, broken down by country?",\n "Which artist has the highest average sales revenue per track and how many tracks have been sold by that artist?",\n "How does the distribution of total sales revenue vary between customers who have made at least two purchases and customers who have made only one purchase?"\n ]\n}', + "refusal": None, + "role": "assistant", + "function_call": None, + "tool_calls": None, + }, + } + ] + }, + "usage": { + "completion_tokens": 394, + "prompt_tokens": 70, + "total_tokens": 464, + }, + "metadata": { + "created_from": "openai", + "type": "openai_chat", + "model": "gpt-3.5-turbo", + }, + "tags": ["openai"], + }, + }, + ] + return messages + + +def insert_dataset_example(experiment_items): + dataset_items = [ + {"id": x["dataset_item_id"], "input": x["input"], "source": "sdk"} + for x in experiment_items + ] + + messages = [ + { + "url": "/v1/private/datasets", + "method": "POST", + "payload": {"name": "Demo Dataset"}, + }, + { + "url": "/v1/private/datasets/items", + "method": "PUT", + "payload": {"dataset_name": "Demo Dataset", "items": dataset_items}, + }, + ] + + return messages + + +def insert_lanchain_chain(project_name: str, trace_id: str, input: str, output: str): + start_time = datetime.datetime.now() + + messages = [ + { + "url": "/v1/private/traces", + "method": "POST", + "payload": { + "id": trace_id, + "name": "RunnableSequence", + "project_name": project_name, + "start_time": (start_time).isoformat() + "Z", + "end_time": (start_time + datetime.timedelta(seconds=1.3)).isoformat() + + "Z", + "input": {"question": input}, + "output": {"output": output}, + "tags": ["Langchain"], + }, + }, + { + "url": "/v1/private/spans", + "method": "POST", + "payload": { + "id": uuid7str(), + "project_name": project_name, + "trace_id": trace_id, + "name": "PromptTemplate", + "type": "general", + "start_time": (start_time).isoformat() + "Z", + "end_time": (start_time + datetime.timedelta(seconds=0.1)).isoformat() + + "Z", + "input": { + "input": f"{input}\nSQLQuery: ", + "table_info": '\nCREATE TABLE "Album" (\n\t"AlbumId" INTEGER NOT NULL, \n\t"Title" NVARCHAR(160) NOT NULL, \n\t"ArtistId" INTEGER NOT NULL, \n\tPRIMARY KEY ("AlbumId"), \n\tFOREIGN KEY("ArtistId") REFERENCES "Artist" ("ArtistId")\n)\n\n/*\n3 rows from Album table:\nAlbumId\tTitle\tArtistId\n1\tFor Those About To Rock We Salute You\t1\n2\tBalls to the Wall\t2\n3\tRestless and Wild\t2\n*/\n\n\nCREATE TABLE "Artist" (\n\t"ArtistId" INTEGER NOT NULL, \n\t"Name" NVARCHAR(120), \n\tPRIMARY KEY ("ArtistId")\n)\n\n/*\n3 rows from Artist table:\nArtistId\tName\n1\tAC/DC\n2\tAccept\n3\tAerosmith\n*/\n\n\nCREATE TABLE "Customer" (\n\t"CustomerId" INTEGER NOT NULL, \n\t"FirstName" NVARCHAR(40) NOT NULL, \n\t"LastName" NVARCHAR(20) NOT NULL, \n\t"Company" NVARCHAR(80), \n\t"Address" NVARCHAR(70), \n\t"City" NVARCHAR(40), \n\t"State" NVARCHAR(40), \n\t"Country" NVARCHAR(40), \n\t"PostalCode" NVARCHAR(10), \n\t"Phone" NVARCHAR(24), \n\t"Fax" NVARCHAR(24), \n\t"Email" NVARCHAR(60) NOT NULL, \n\t"SupportRepId" INTEGER, \n\tPRIMARY KEY ("CustomerId"), \n\tFOREIGN KEY("SupportRepId") REFERENCES "Employee" ("EmployeeId")\n)\n\n/*\n3 rows from Customer table:\nCustomerId\tFirstName\tLastName\tCompany\tAddress\tCity\tState\tCountry\tPostalCode\tPhone\tFax\tEmail\tSupportRepId\n1\tLuís\tGonçalves\tEmbraer - Empresa Brasileira de Aeronáutica S.A.\tAv. Brigadeiro Faria Lima, 2170\tSão José dos Campos\tSP\tBrazil\t12227-000\t+55 (12) 3923-5555\t+55 (12) 3923-5566\tluisg@embraer.com.br\t3\n2\tLeonie\tKöhler\tNone\tTheodor-Heuss-Straße 34\tStuttgart\tNone\tGermany\t70174\t+49 0711 2842222\tNone\tleonekohler@surfeu.de\t5\n3\tFrançois\tTremblay\tNone\t1498 rue Bélanger\tMontréal\tQC\tCanada\tH2G 1A7\t+1 (514) 721-4711\tNone\tftremblay@gmail.com\t3\n*/\n\n\nCREATE TABLE "Employee" (\n\t"EmployeeId" INTEGER NOT NULL, \n\t"LastName" NVARCHAR(20) NOT NULL, \n\t"FirstName" NVARCHAR(20) NOT NULL, \n\t"Title" NVARCHAR(30), \n\t"ReportsTo" INTEGER, \n\t"BirthDate" DATETIME, \n\t"HireDate" DATETIME, \n\t"Address" NVARCHAR(70), \n\t"City" NVARCHAR(40), \n\t"State" NVARCHAR(40), \n\t"Country" NVARCHAR(40), \n\t"PostalCode" NVARCHAR(10), \n\t"Phone" NVARCHAR(24), \n\t"Fax" NVARCHAR(24), \n\t"Email" NVARCHAR(60), \n\tPRIMARY KEY ("EmployeeId"), \n\tFOREIGN KEY("ReportsTo") REFERENCES "Employee" ("EmployeeId")\n)\n\n/*\n3 rows from Employee table:\nEmployeeId\tLastName\tFirstName\tTitle\tReportsTo\tBirthDate\tHireDate\tAddress\tCity\tState\tCountry\tPostalCode\tPhone\tFax\tEmail\n1\tAdams\tAndrew\tGeneral Manager\tNone\t1962-02-18 00:00:00\t2002-08-14 00:00:00\t11120 Jasper Ave NW\tEdmonton\tAB\tCanada\tT5K 2N1\t+1 (780) 428-9482\t+1 (780) 428-3457\tandrew@chinookcorp.com\n2\tEdwards\tNancy\tSales Manager\t1\t1958-12-08 00:00:00\t2002-05-01 00:00:00\t825 8 Ave SW\tCalgary\tAB\tCanada\tT2P 2T3\t+1 (403) 262-3443\t+1 (403) 262-3322\tnancy@chinookcorp.com\n3\tPeacock\tJane\tSales Support Agent\t2\t1973-08-29 00:00:00\t2002-04-01 00:00:00\t1111 6 Ave SW\tCalgary\tAB\tCanada\tT2P 5M5\t+1 (403) 262-3443\t+1 (403) 262-6712\tjane@chinookcorp.com\n*/\n\n\nCREATE TABLE "Genre" (\n\t"GenreId" INTEGER NOT NULL, \n\t"Name" NVARCHAR(120), \n\tPRIMARY KEY ("GenreId")\n)\n\n/*\n3 rows from Genre table:\nGenreId\tName\n1\tRock\n2\tJazz\n3\tMetal\n*/\n\n\nCREATE TABLE "Invoice" (\n\t"InvoiceId" INTEGER NOT NULL, \n\t"CustomerId" INTEGER NOT NULL, \n\t"InvoiceDate" DATETIME NOT NULL, \n\t"BillingAddress" NVARCHAR(70), \n\t"BillingCity" NVARCHAR(40), \n\t"BillingState" NVARCHAR(40), \n\t"BillingCountry" NVARCHAR(40), \n\t"BillingPostalCode" NVARCHAR(10), \n\t"Total" NUMERIC(10, 2) NOT NULL, \n\tPRIMARY KEY ("InvoiceId"), \n\tFOREIGN KEY("CustomerId") REFERENCES "Customer" ("CustomerId")\n)\n\n/*\n3 rows from Invoice table:\nInvoiceId\tCustomerId\tInvoiceDate\tBillingAddress\tBillingCity\tBillingState\tBillingCountry\tBillingPostalCode\tTotal\n1\t2\t2009-01-01 00:00:00\tTheodor-Heuss-Straße 34\tStuttgart\tNone\tGermany\t70174\t1.98\n2\t4\t2009-01-02 00:00:00\tUllevålsveien 14\tOslo\tNone\tNorway\t0171\t3.96\n3\t8\t2009-01-03 00:00:00\tGrétrystraat 63\tBrussels\tNone\tBelgium\t1000\t5.94\n*/\n\n\nCREATE TABLE "InvoiceLine" (\n\t"InvoiceLineId" INTEGER NOT NULL, \n\t"InvoiceId" INTEGER NOT NULL, \n\t"TrackId" INTEGER NOT NULL, \n\t"UnitPrice" NUMERIC(10, 2) NOT NULL, \n\t"Quantity" INTEGER NOT NULL, \n\tPRIMARY KEY ("InvoiceLineId"), \n\tFOREIGN KEY("TrackId") REFERENCES "Track" ("TrackId"), \n\tFOREIGN KEY("InvoiceId") REFERENCES "Invoice" ("InvoiceId")\n)\n\n/*\n3 rows from InvoiceLine table:\nInvoiceLineId\tInvoiceId\tTrackId\tUnitPrice\tQuantity\n1\t1\t2\t0.99\t1\n2\t1\t4\t0.99\t1\n3\t2\t6\t0.99\t1\n*/\n\n\nCREATE TABLE "MediaType" (\n\t"MediaTypeId" INTEGER NOT NULL, \n\t"Name" NVARCHAR(120), \n\tPRIMARY KEY ("MediaTypeId")\n)\n\n/*\n3 rows from MediaType table:\nMediaTypeId\tName\n1\tMPEG audio file\n2\tProtected AAC audio file\n3\tProtected MPEG-4 video file\n*/\n\n\nCREATE TABLE "Playlist" (\n\t"PlaylistId" INTEGER NOT NULL, \n\t"Name" NVARCHAR(120), \n\tPRIMARY KEY ("PlaylistId")\n)\n\n/*\n3 rows from Playlist table:\nPlaylistId\tName\n1\tMusic\n2\tMovies\n3\tTV Shows\n*/\n\n\nCREATE TABLE "PlaylistTrack" (\n\t"PlaylistId" INTEGER NOT NULL, \n\t"TrackId" INTEGER NOT NULL, \n\tPRIMARY KEY ("PlaylistId", "TrackId"), \n\tFOREIGN KEY("TrackId") REFERENCES "Track" ("TrackId"), \n\tFOREIGN KEY("PlaylistId") REFERENCES "Playlist" ("PlaylistId")\n)\n\n/*\n3 rows from PlaylistTrack table:\nPlaylistId\tTrackId\n1\t3402\n1\t3389\n1\t3390\n*/\n\n\nCREATE TABLE "Track" (\n\t"TrackId" INTEGER NOT NULL, \n\t"Name" NVARCHAR(200) NOT NULL, \n\t"AlbumId" INTEGER, \n\t"MediaTypeId" INTEGER NOT NULL, \n\t"GenreId" INTEGER, \n\t"Composer" NVARCHAR(220), \n\t"Milliseconds" INTEGER NOT NULL, \n\t"Bytes" INTEGER, \n\t"UnitPrice" NUMERIC(10, 2) NOT NULL, \n\tPRIMARY KEY ("TrackId"), \n\tFOREIGN KEY("MediaTypeId") REFERENCES "MediaType" ("MediaTypeId"), \n\tFOREIGN KEY("GenreId") REFERENCES "Genre" ("GenreId"), \n\tFOREIGN KEY("AlbumId") REFERENCES "Album" ("AlbumId")\n)\n\n/*\n3 rows from Track table:\nTrackId\tName\tAlbumId\tMediaTypeId\tGenreId\tComposer\tMilliseconds\tBytes\tUnitPrice\n1\tFor Those About To Rock (We Salute You)\t1\t1\t1\tAngus Young, Malcolm Young, Brian Johnson\t343719\t11170334\t0.99\n2\tBalls to the Wall\t2\t2\t1\tNone\t342562\t5510424\t0.99\n3\tFast As a Shark\t3\t2\t1\tF. Baltes, S. Kaufman, U. Dirkscneider & W. Hoffman\t230619\t3990994\t0.99\n*/', + }, + "output": { + "output": { + "text": f'You are a SQLite expert. Given an input question, first create a syntactically correct SQLite query to run, then look at the results of the query and return the answer to the input question.\nUnless the user specifies in the question a specific number of examples to obtain, query for at most 5 results using the LIMIT clause as per SQLite. You can order the results to return the most informative data in the database.\nNever query for all columns from a table. You must query only the columns that are needed to answer the question. Wrap each column name in double quotes (") to denote them as delimited identifiers.\nPay attention to use only the column names you can see in the tables below. Be careful to not query for columns that do not exist. Also, pay attention to which column is in which table.\nPay attention to use date(\'now\') function to get the current date, if the question involves "today".\n\nUse the following format:\n\nQuestion: Question here\nSQLQuery: SQL Query to run\nSQLResult: Result of the SQLQuery\nAnswer: Final answer here\n\nOnly use the following tables:\n\nCREATE TABLE "Album" (\n\t"AlbumId" INTEGER NOT NULL, \n\t"Title" NVARCHAR(160) NOT NULL, \n\t"ArtistId" INTEGER NOT NULL, \n\tPRIMARY KEY ("AlbumId"), \n\tFOREIGN KEY("ArtistId") REFERENCES "Artist" ("ArtistId")\n)\n\n/*\n3 rows from Album table:\nAlbumId\tTitle\tArtistId\n1\tFor Those About To Rock We Salute You\t1\n2\tBalls to the Wall\t2\n3\tRestless and Wild\t2\n*/\n\n\nCREATE TABLE "Artist" (\n\t"ArtistId" INTEGER NOT NULL, \n\t"Name" NVARCHAR(120), \n\tPRIMARY KEY ("ArtistId")\n)\n\n/*\n3 rows from Artist table:\nArtistId\tName\n1\tAC/DC\n2\tAccept\n3\tAerosmith\n*/\n\n\nCREATE TABLE "Customer" (\n\t"CustomerId" INTEGER NOT NULL, \n\t"FirstName" NVARCHAR(40) NOT NULL, \n\t"LastName" NVARCHAR(20) NOT NULL, \n\t"Company" NVARCHAR(80), \n\t"Address" NVARCHAR(70), \n\t"City" NVARCHAR(40), \n\t"State" NVARCHAR(40), \n\t"Country" NVARCHAR(40), \n\t"PostalCode" NVARCHAR(10), \n\t"Phone" NVARCHAR(24), \n\t"Fax" NVARCHAR(24), \n\t"Email" NVARCHAR(60) NOT NULL, \n\t"SupportRepId" INTEGER, \n\tPRIMARY KEY ("CustomerId"), \n\tFOREIGN KEY("SupportRepId") REFERENCES "Employee" ("EmployeeId")\n)\n\n/*\n3 rows from Customer table:\nCustomerId\tFirstName\tLastName\tCompany\tAddress\tCity\tState\tCountry\tPostalCode\tPhone\tFax\tEmail\tSupportRepId\n1\tLuís\tGonçalves\tEmbraer - Empresa Brasileira de Aeronáutica S.A.\tAv. Brigadeiro Faria Lima, 2170\tSão José dos Campos\tSP\tBrazil\t12227-000\t+55 (12) 3923-5555\t+55 (12) 3923-5566\tluisg@embraer.com.br\t3\n2\tLeonie\tKöhler\tNone\tTheodor-Heuss-Straße 34\tStuttgart\tNone\tGermany\t70174\t+49 0711 2842222\tNone\tleonekohler@surfeu.de\t5\n3\tFrançois\tTremblay\tNone\t1498 rue Bélanger\tMontréal\tQC\tCanada\tH2G 1A7\t+1 (514) 721-4711\tNone\tftremblay@gmail.com\t3\n*/\n\n\nCREATE TABLE "Employee" (\n\t"EmployeeId" INTEGER NOT NULL, \n\t"LastName" NVARCHAR(20) NOT NULL, \n\t"FirstName" NVARCHAR(20) NOT NULL, \n\t"Title" NVARCHAR(30), \n\t"ReportsTo" INTEGER, \n\t"BirthDate" DATETIME, \n\t"HireDate" DATETIME, \n\t"Address" NVARCHAR(70), \n\t"City" NVARCHAR(40), \n\t"State" NVARCHAR(40), \n\t"Country" NVARCHAR(40), \n\t"PostalCode" NVARCHAR(10), \n\t"Phone" NVARCHAR(24), \n\t"Fax" NVARCHAR(24), \n\t"Email" NVARCHAR(60), \n\tPRIMARY KEY ("EmployeeId"), \n\tFOREIGN KEY("ReportsTo") REFERENCES "Employee" ("EmployeeId")\n)\n\n/*\n3 rows from Employee table:\nEmployeeId\tLastName\tFirstName\tTitle\tReportsTo\tBirthDate\tHireDate\tAddress\tCity\tState\tCountry\tPostalCode\tPhone\tFax\tEmail\n1\tAdams\tAndrew\tGeneral Manager\tNone\t1962-02-18 00:00:00\t2002-08-14 00:00:00\t11120 Jasper Ave NW\tEdmonton\tAB\tCanada\tT5K 2N1\t+1 (780) 428-9482\t+1 (780) 428-3457\tandrew@chinookcorp.com\n2\tEdwards\tNancy\tSales Manager\t1\t1958-12-08 00:00:00\t2002-05-01 00:00:00\t825 8 Ave SW\tCalgary\tAB\tCanada\tT2P 2T3\t+1 (403) 262-3443\t+1 (403) 262-3322\tnancy@chinookcorp.com\n3\tPeacock\tJane\tSales Support Agent\t2\t1973-08-29 00:00:00\t2002-04-01 00:00:00\t1111 6 Ave SW\tCalgary\tAB\tCanada\tT2P 5M5\t+1 (403) 262-3443\t+1 (403) 262-6712\tjane@chinookcorp.com\n*/\n\n\nCREATE TABLE "Genre" (\n\t"GenreId" INTEGER NOT NULL, \n\t"Name" NVARCHAR(120), \n\tPRIMARY KEY ("GenreId")\n)\n\n/*\n3 rows from Genre table:\nGenreId\tName\n1\tRock\n2\tJazz\n3\tMetal\n*/\n\n\nCREATE TABLE "Invoice" (\n\t"InvoiceId" INTEGER NOT NULL, \n\t"CustomerId" INTEGER NOT NULL, \n\t"InvoiceDate" DATETIME NOT NULL, \n\t"BillingAddress" NVARCHAR(70), \n\t"BillingCity" NVARCHAR(40), \n\t"BillingState" NVARCHAR(40), \n\t"BillingCountry" NVARCHAR(40), \n\t"BillingPostalCode" NVARCHAR(10), \n\t"Total" NUMERIC(10, 2) NOT NULL, \n\tPRIMARY KEY ("InvoiceId"), \n\tFOREIGN KEY("CustomerId") REFERENCES "Customer" ("CustomerId")\n)\n\n/*\n3 rows from Invoice table:\nInvoiceId\tCustomerId\tInvoiceDate\tBillingAddress\tBillingCity\tBillingState\tBillingCountry\tBillingPostalCode\tTotal\n1\t2\t2009-01-01 00:00:00\tTheodor-Heuss-Straße 34\tStuttgart\tNone\tGermany\t70174\t1.98\n2\t4\t2009-01-02 00:00:00\tUllevålsveien 14\tOslo\tNone\tNorway\t0171\t3.96\n3\t8\t2009-01-03 00:00:00\tGrétrystraat 63\tBrussels\tNone\tBelgium\t1000\t5.94\n*/\n\n\nCREATE TABLE "InvoiceLine" (\n\t"InvoiceLineId" INTEGER NOT NULL, \n\t"InvoiceId" INTEGER NOT NULL, \n\t"TrackId" INTEGER NOT NULL, \n\t"UnitPrice" NUMERIC(10, 2) NOT NULL, \n\t"Quantity" INTEGER NOT NULL, \n\tPRIMARY KEY ("InvoiceLineId"), \n\tFOREIGN KEY("TrackId") REFERENCES "Track" ("TrackId"), \n\tFOREIGN KEY("InvoiceId") REFERENCES "Invoice" ("InvoiceId")\n)\n\n/*\n3 rows from InvoiceLine table:\nInvoiceLineId\tInvoiceId\tTrackId\tUnitPrice\tQuantity\n1\t1\t2\t0.99\t1\n2\t1\t4\t0.99\t1\n3\t2\t6\t0.99\t1\n*/\n\n\nCREATE TABLE "MediaType" (\n\t"MediaTypeId" INTEGER NOT NULL, \n\t"Name" NVARCHAR(120), \n\tPRIMARY KEY ("MediaTypeId")\n)\n\n/*\n3 rows from MediaType table:\nMediaTypeId\tName\n1\tMPEG audio file\n2\tProtected AAC audio file\n3\tProtected MPEG-4 video file\n*/\n\n\nCREATE TABLE "Playlist" (\n\t"PlaylistId" INTEGER NOT NULL, \n\t"Name" NVARCHAR(120), \n\tPRIMARY KEY ("PlaylistId")\n)\n\n/*\n3 rows from Playlist table:\nPlaylistId\tName\n1\tMusic\n2\tMovies\n3\tTV Shows\n*/\n\n\nCREATE TABLE "PlaylistTrack" (\n\t"PlaylistId" INTEGER NOT NULL, \n\t"TrackId" INTEGER NOT NULL, \n\tPRIMARY KEY ("PlaylistId", "TrackId"), \n\tFOREIGN KEY("TrackId") REFERENCES "Track" ("TrackId"), \n\tFOREIGN KEY("PlaylistId") REFERENCES "Playlist" ("PlaylistId")\n)\n\n/*\n3 rows from PlaylistTrack table:\nPlaylistId\tTrackId\n1\t3402\n1\t3389\n1\t3390\n*/\n\n\nCREATE TABLE "Track" (\n\t"TrackId" INTEGER NOT NULL, \n\t"Name" NVARCHAR(200) NOT NULL, \n\t"AlbumId" INTEGER, \n\t"MediaTypeId" INTEGER NOT NULL, \n\t"GenreId" INTEGER, \n\t"Composer" NVARCHAR(220), \n\t"Milliseconds" INTEGER NOT NULL, \n\t"Bytes" INTEGER, \n\t"UnitPrice" NUMERIC(10, 2) NOT NULL, \n\tPRIMARY KEY ("TrackId"), \n\tFOREIGN KEY("MediaTypeId") REFERENCES "MediaType" ("MediaTypeId"), \n\tFOREIGN KEY("GenreId") REFERENCES "Genre" ("GenreId"), \n\tFOREIGN KEY("AlbumId") REFERENCES "Album" ("AlbumId")\n)\n\n/*\n3 rows from Track table:\nTrackId\tName\tAlbumId\tMediaTypeId\tGenreId\tComposer\tMilliseconds\tBytes\tUnitPrice\n1\tFor Those About To Rock (We Salute You)\t1\t1\t1\tAngus Young, Malcolm Young, Brian Johnson\t343719\t11170334\t0.99\n2\tBalls to the Wall\t2\t2\t1\tNone\t342562\t5510424\t0.99\n3\tFast As a Shark\t3\t2\t1\tF. Baltes, S. Kaufman, U. Dirkscneider & W. Hoffman\t230619\t3990994\t0.99\n*/\n\nQuestion: {input}\nSQLQuery: ', + "type": "StringPromptValue", + } + }, + }, + }, + { + "url": "/v1/private/spans", + "method": "POST", + "payload": { + "id": uuid7str(), + "project_name": project_name, + "trace_id": trace_id, + "name": "ChatOpenAI", + "type": "llm", + "start_time": (start_time + datetime.timedelta(seconds=0.1)).isoformat() + + "Z", + "end_time": (start_time + datetime.timedelta(seconds=1.2)).isoformat() + + "Z", + "input": { + "prompts": [ + f'Human: You are a SQLite expert. Given an input question, first create a syntactically correct SQLite query to run, then look at the results of the query and return the answer to the input question.\nUnless the user specifies in the question a specific number of examples to obtain, query for at most 5 results using the LIMIT clause as per SQLite. You can order the results to return the most informative data in the database.\nNever query for all columns from a table. You must query only the columns that are needed to answer the question. Wrap each column name in double quotes (") to denote them as delimited identifiers.\nPay attention to use only the column names you can see in the tables below. Be careful to not query for columns that do not exist. Also, pay attention to which column is in which table.\nPay attention to use date(\'now\') function to get the current date, if the question involves "today".\n\nUse the following format:\n\nQuestion: Question here\nSQLQuery: SQL Query to run\nSQLResult: Result of the SQLQuery\nAnswer: Final answer here\n\nOnly use the following tables:\n\nCREATE TABLE "Album" (\n\t"AlbumId" INTEGER NOT NULL, \n\t"Title" NVARCHAR(160) NOT NULL, \n\t"ArtistId" INTEGER NOT NULL, \n\tPRIMARY KEY ("AlbumId"), \n\tFOREIGN KEY("ArtistId") REFERENCES "Artist" ("ArtistId")\n)\n\n/*\n3 rows from Album table:\nAlbumId\tTitle\tArtistId\n1\tFor Those About To Rock We Salute You\t1\n2\tBalls to the Wall\t2\n3\tRestless and Wild\t2\n*/\n\n\nCREATE TABLE "Artist" (\n\t"ArtistId" INTEGER NOT NULL, \n\t"Name" NVARCHAR(120), \n\tPRIMARY KEY ("ArtistId")\n)\n\n/*\n3 rows from Artist table:\nArtistId\tName\n1\tAC/DC\n2\tAccept\n3\tAerosmith\n*/\n\n\nCREATE TABLE "Customer" (\n\t"CustomerId" INTEGER NOT NULL, \n\t"FirstName" NVARCHAR(40) NOT NULL, \n\t"LastName" NVARCHAR(20) NOT NULL, \n\t"Company" NVARCHAR(80), \n\t"Address" NVARCHAR(70), \n\t"City" NVARCHAR(40), \n\t"State" NVARCHAR(40), \n\t"Country" NVARCHAR(40), \n\t"PostalCode" NVARCHAR(10), \n\t"Phone" NVARCHAR(24), \n\t"Fax" NVARCHAR(24), \n\t"Email" NVARCHAR(60) NOT NULL, \n\t"SupportRepId" INTEGER, \n\tPRIMARY KEY ("CustomerId"), \n\tFOREIGN KEY("SupportRepId") REFERENCES "Employee" ("EmployeeId")\n)\n\n/*\n3 rows from Customer table:\nCustomerId\tFirstName\tLastName\tCompany\tAddress\tCity\tState\tCountry\tPostalCode\tPhone\tFax\tEmail\tSupportRepId\n1\tLuís\tGonçalves\tEmbraer - Empresa Brasileira de Aeronáutica S.A.\tAv. Brigadeiro Faria Lima, 2170\tSão José dos Campos\tSP\tBrazil\t12227-000\t+55 (12) 3923-5555\t+55 (12) 3923-5566\tluisg@embraer.com.br\t3\n2\tLeonie\tKöhler\tNone\tTheodor-Heuss-Straße 34\tStuttgart\tNone\tGermany\t70174\t+49 0711 2842222\tNone\tleonekohler@surfeu.de\t5\n3\tFrançois\tTremblay\tNone\t1498 rue Bélanger\tMontréal\tQC\tCanada\tH2G 1A7\t+1 (514) 721-4711\tNone\tftremblay@gmail.com\t3\n*/\n\n\nCREATE TABLE "Employee" (\n\t"EmployeeId" INTEGER NOT NULL, \n\t"LastName" NVARCHAR(20) NOT NULL, \n\t"FirstName" NVARCHAR(20) NOT NULL, \n\t"Title" NVARCHAR(30), \n\t"ReportsTo" INTEGER, \n\t"BirthDate" DATETIME, \n\t"HireDate" DATETIME, \n\t"Address" NVARCHAR(70), \n\t"City" NVARCHAR(40), \n\t"State" NVARCHAR(40), \n\t"Country" NVARCHAR(40), \n\t"PostalCode" NVARCHAR(10), \n\t"Phone" NVARCHAR(24), \n\t"Fax" NVARCHAR(24), \n\t"Email" NVARCHAR(60), \n\tPRIMARY KEY ("EmployeeId"), \n\tFOREIGN KEY("ReportsTo") REFERENCES "Employee" ("EmployeeId")\n)\n\n/*\n3 rows from Employee table:\nEmployeeId\tLastName\tFirstName\tTitle\tReportsTo\tBirthDate\tHireDate\tAddress\tCity\tState\tCountry\tPostalCode\tPhone\tFax\tEmail\n1\tAdams\tAndrew\tGeneral Manager\tNone\t1962-02-18 00:00:00\t2002-08-14 00:00:00\t11120 Jasper Ave NW\tEdmonton\tAB\tCanada\tT5K 2N1\t+1 (780) 428-9482\t+1 (780) 428-3457\tandrew@chinookcorp.com\n2\tEdwards\tNancy\tSales Manager\t1\t1958-12-08 00:00:00\t2002-05-01 00:00:00\t825 8 Ave SW\tCalgary\tAB\tCanada\tT2P 2T3\t+1 (403) 262-3443\t+1 (403) 262-3322\tnancy@chinookcorp.com\n3\tPeacock\tJane\tSales Support Agent\t2\t1973-08-29 00:00:00\t2002-04-01 00:00:00\t1111 6 Ave SW\tCalgary\tAB\tCanada\tT2P 5M5\t+1 (403) 262-3443\t+1 (403) 262-6712\tjane@chinookcorp.com\n*/\n\n\nCREATE TABLE "Genre" (\n\t"GenreId" INTEGER NOT NULL, \n\t"Name" NVARCHAR(120), \n\tPRIMARY KEY ("GenreId")\n)\n\n/*\n3 rows from Genre table:\nGenreId\tName\n1\tRock\n2\tJazz\n3\tMetal\n*/\n\n\nCREATE TABLE "Invoice" (\n\t"InvoiceId" INTEGER NOT NULL, \n\t"CustomerId" INTEGER NOT NULL, \n\t"InvoiceDate" DATETIME NOT NULL, \n\t"BillingAddress" NVARCHAR(70), \n\t"BillingCity" NVARCHAR(40), \n\t"BillingState" NVARCHAR(40), \n\t"BillingCountry" NVARCHAR(40), \n\t"BillingPostalCode" NVARCHAR(10), \n\t"Total" NUMERIC(10, 2) NOT NULL, \n\tPRIMARY KEY ("InvoiceId"), \n\tFOREIGN KEY("CustomerId") REFERENCES "Customer" ("CustomerId")\n)\n\n/*\n3 rows from Invoice table:\nInvoiceId\tCustomerId\tInvoiceDate\tBillingAddress\tBillingCity\tBillingState\tBillingCountry\tBillingPostalCode\tTotal\n1\t2\t2009-01-01 00:00:00\tTheodor-Heuss-Straße 34\tStuttgart\tNone\tGermany\t70174\t1.98\n2\t4\t2009-01-02 00:00:00\tUllevålsveien 14\tOslo\tNone\tNorway\t0171\t3.96\n3\t8\t2009-01-03 00:00:00\tGrétrystraat 63\tBrussels\tNone\tBelgium\t1000\t5.94\n*/\n\n\nCREATE TABLE "InvoiceLine" (\n\t"InvoiceLineId" INTEGER NOT NULL, \n\t"InvoiceId" INTEGER NOT NULL, \n\t"TrackId" INTEGER NOT NULL, \n\t"UnitPrice" NUMERIC(10, 2) NOT NULL, \n\t"Quantity" INTEGER NOT NULL, \n\tPRIMARY KEY ("InvoiceLineId"), \n\tFOREIGN KEY("TrackId") REFERENCES "Track" ("TrackId"), \n\tFOREIGN KEY("InvoiceId") REFERENCES "Invoice" ("InvoiceId")\n)\n\n/*\n3 rows from InvoiceLine table:\nInvoiceLineId\tInvoiceId\tTrackId\tUnitPrice\tQuantity\n1\t1\t2\t0.99\t1\n2\t1\t4\t0.99\t1\n3\t2\t6\t0.99\t1\n*/\n\n\nCREATE TABLE "MediaType" (\n\t"MediaTypeId" INTEGER NOT NULL, \n\t"Name" NVARCHAR(120), \n\tPRIMARY KEY ("MediaTypeId")\n)\n\n/*\n3 rows from MediaType table:\nMediaTypeId\tName\n1\tMPEG audio file\n2\tProtected AAC audio file\n3\tProtected MPEG-4 video file\n*/\n\n\nCREATE TABLE "Playlist" (\n\t"PlaylistId" INTEGER NOT NULL, \n\t"Name" NVARCHAR(120), \n\tPRIMARY KEY ("PlaylistId")\n)\n\n/*\n3 rows from Playlist table:\nPlaylistId\tName\n1\tMusic\n2\tMovies\n3\tTV Shows\n*/\n\n\nCREATE TABLE "PlaylistTrack" (\n\t"PlaylistId" INTEGER NOT NULL, \n\t"TrackId" INTEGER NOT NULL, \n\tPRIMARY KEY ("PlaylistId", "TrackId"), \n\tFOREIGN KEY("TrackId") REFERENCES "Track" ("TrackId"), \n\tFOREIGN KEY("PlaylistId") REFERENCES "Playlist" ("PlaylistId")\n)\n\n/*\n3 rows from PlaylistTrack table:\nPlaylistId\tTrackId\n1\t3402\n1\t3389\n1\t3390\n*/\n\n\nCREATE TABLE "Track" (\n\t"TrackId" INTEGER NOT NULL, \n\t"Name" NVARCHAR(200) NOT NULL, \n\t"AlbumId" INTEGER, \n\t"MediaTypeId" INTEGER NOT NULL, \n\t"GenreId" INTEGER, \n\t"Composer" NVARCHAR(220), \n\t"Milliseconds" INTEGER NOT NULL, \n\t"Bytes" INTEGER, \n\t"UnitPrice" NUMERIC(10, 2) NOT NULL, \n\tPRIMARY KEY ("TrackId"), \n\tFOREIGN KEY("MediaTypeId") REFERENCES "MediaType" ("MediaTypeId"), \n\tFOREIGN KEY("GenreId") REFERENCES "Genre" ("GenreId"), \n\tFOREIGN KEY("AlbumId") REFERENCES "Album" ("AlbumId")\n)\n\n/*\n3 rows from Track table:\nTrackId\tName\tAlbumId\tMediaTypeId\tGenreId\tComposer\tMilliseconds\tBytes\tUnitPrice\n1\tFor Those About To Rock (We Salute You)\t1\t1\t1\tAngus Young, Malcolm Young, Brian Johnson\t343719\t11170334\t0.99\n2\tBalls to the Wall\t2\t2\t1\tNone\t342562\t5510424\t0.99\n3\tFast As a Shark\t3\t2\t1\tF. Baltes, S. Kaufman, U. Dirkscneider & W. Hoffman\t230619\t3990994\t0.99\n*/\n\nQuestion: {input}\nSQLQuery: ' + ] + }, + "output": { + "generations": [ + [ + { + "text": output, + "generation_info": { + "finish_reason": "stop", + "logprobs": None, + }, + "type": "ChatGeneration", + "message": { + "lc": 1, + "type": "constructor", + "id": [ + "langchain", + "schema", + "messages", + "AIMessage", + ], + "kwargs": { + "content": output, + "additional_kwargs": {"refusal": None}, + "response_metadata": { + "token_usage": { + "completion_tokens": 15, + "prompt_tokens": 2340, + "total_tokens": 2355, + }, + "model_name": "gpt-3.5-turbo-0125", + "system_fingerprint": None, + "finish_reason": "stop", + "logprobs": None, + }, + "type": "ai", + "id": "run-e236b2e4-43c5-47df-ac25-86f6b9767406-0", + "usage_metadata": { + "input_tokens": 2340, + "output_tokens": 15, + "total_tokens": 2355, + }, + "tool_calls": [], + "invalid_tool_calls": [], + }, + }, + } + ] + ], + "llm_output": { + "token_usage": { + "completion_tokens": 15, + "prompt_tokens": 2340, + "total_tokens": 2355, + }, + "model_name": "gpt-3.5-turbo-0125", + "system_fingerprint": None, + }, + "run": None, + }, + "metadata": { + "invocation_params": { + "model": "gpt-3.5-turbo", + "model_name": "gpt-3.5-turbo", + "stream": False, + "n": 1, + "temperature": 0.0, + "_type": "openai-chat", + "stop": ["\nSQLResult:"], + }, + "options": {"stop": ["\nSQLResult:"]}, + "batch_size": 1, + "metadata": { + "ls_provider": "openai", + "ls_model_name": "gpt-3.5-turbo", + "ls_model_type": "chat", + "ls_temperature": 0.0, + "ls_stop": ["\nSQLResult:"], + }, + }, + }, + }, + { + "url": "/v1/private/spans", + "method": "POST", + "payload": { + "id": uuid7str(), + "project_name": project_name, + "trace_id": trace_id, + "name": "StrOutputParser", + "type": "general", + "start_time": (start_time + datetime.timedelta(seconds=1.2)).isoformat() + + "Z", + "end_time": (start_time + datetime.timedelta(seconds=1.3)).isoformat() + + "Z", + "input": { + "input": { + "content": output, + "additional_kwargs": {"refusal": None}, + "response_metadata": { + "token_usage": { + "completion_tokens": 15, + "prompt_tokens": 2340, + "total_tokens": 2355, + }, + "model_name": "gpt-3.5-turbo-0125", + "system_fingerprint": None, + "finish_reason": "stop", + "logprobs": None, + }, + "type": "ai", + "name": None, + "id": "run-e236b2e4-43c5-47df-ac25-86f6b9767406-0", + "example": False, + "tool_calls": [], + "invalid_tool_calls": [], + "usage_metadata": { + "input_tokens": 2340, + "output_tokens": 15, + "total_tokens": 2355, + }, + } + }, + "output": {"output": output}, + }, + }, + ] + return messages + + +def insert_experiment_example(dataset_name, experiment_id): + messages = [ + { + "url": "/v1/private/experiments", + "method": "POST", + "payload": { + "name": "Demo Experiment", + "id": experiment_id, + "dataset_name": dataset_name, + }, + } + ] + return messages + + +def insert_experiment_items(project_name, experiment_items): + # Add messages for experiment items + messages = [ + { + "url": "/v1/private/experiments/items", + "method": "POST", + "payload": { + "experiment_items": [ + { + "experiment_id": item["experiment_id"], + "trace_id": item["trace_id"], + "dataset_item_id": item["dataset_item_id"], + } + for item in experiment_items + ] + }, + } + ] + + # Add messages for each trace + for item in experiment_items: + messages += insert_lanchain_chain( + project_name, + item["trace_id"], + item["input"]["question"], + item["output"]["output"], + ) + + # Add messages for feedback scores + messages += [ + { + "url": "/v1/private/traces/feedback-scores", + "method": "PUT", + "payload": { + "scores": [ + { + "id": item["trace_id"], + "project_name": project_name, + "name": "is_valid_sql", + "value": item["score"], + "reason": item["reason"], + "source": "sdk", + } + for item in experiment_items + ] + }, + } + ] + + return messages + + +def create_demo_data(base_url: str, workspace_name, comet_api_key): + # Set constants + experiment_id = uuid7str() + project_name = "Demo Project" + + experiment_items = create_experiment_items(experiment_id) + + messages = [] + # Create OpenAI trace + messages += insert_openai_project_example(project_name, experiment_items) + + # Create dataset + messages += insert_dataset_example(experiment_items) + + # Create experiment + messages += insert_experiment_example("Demo Dataset", experiment_id) + + # Insert experiment items + messages += insert_experiment_items(project_name, experiment_items) + + for message in messages: + make_http_request(base_url, message, workspace_name, comet_api_key) + + +if __name__ == "__main__": + base_url = "http://localhost:5173" + workspace_name = None + comet_api_key = None + create_demo_data(base_url, workspace_name, comet_api_key)