diff --git a/daras_ai_v2/language_model.py b/daras_ai_v2/language_model.py index d9cbfa944..27820e37b 100644 --- a/daras_ai_v2/language_model.py +++ b/daras_ai_v2/language_model.py @@ -19,6 +19,7 @@ ChatCompletionContentPartParam, ChatCompletionChunk, ) +from openai.types.completion_usage import CompletionUsage from daras_ai.image_input import gs_url_to_uri, bytes_to_cv2_img, cv2_img_to_bytes from daras_ai_v2.asr import get_google_auth_session @@ -636,6 +637,9 @@ def _run_self_hosted_chat( avoid_repetition: bool, stop: list[str] | None, ) -> list[dict]: + from usage_costs.cost_utils import record_cost_auto + from usage_costs.models import ModelSku + # sea lion doesnt support system prompt if model == LargeLanguageModels.sea_lion_7b_instruct.model_id: for i, entry in enumerate(messages): @@ -656,6 +660,19 @@ def _run_self_hosted_chat( repetition_penalty=1.15 if avoid_repetition else 1, ), ) + + if usage := ret.get("usage"): + record_cost_auto( + model=model, + sku=ModelSku.llm_prompt, + quantity=usage["prompt_tokens"], + ) + record_cost_auto( + model=model, + sku=ModelSku.llm_completion, + quantity=usage["completion_tokens"], + ) + return [ { "role": CHATML_ROLE_ASSISTANT, @@ -757,7 +774,7 @@ def _run_openai_chat( presence_penalty = 0 if isinstance(model, str): model = [model] - r, used_model = try_all( + completion, used_model = try_all( *[ _get_chat_completions_create( model=model_str, @@ -780,11 +797,10 @@ def _run_openai_chat( ], ) if stream: - return _stream_openai_chunked(r, used_model, messages) + return _stream_openai_chunked(completion, used_model, messages) else: - ret = [choice.message.dict() for choice in r.choices] - record_openai_llm_usage(used_model, messages, ret) - return ret + record_openai_llm_usage(used_model, completion.usage) + return [choice.message.dict() for choice in completion.choices] def _get_chat_completions_create(model: str, **kwargs): @@ -854,17 +870,18 @@ def _stream_openai_chunked( break if changed: yield ret + if completion_chunk.usage: + record_openai_llm_usage(used_model, completion_chunk.usage) # add the leftover chunks for entry in ret: entry["content"] += entry["chunk"] yield ret - record_openai_llm_usage(used_model, messages, ret) - def record_openai_llm_usage( - used_model: str, messages: list[ConversationEntry], choices: list[ConversationEntry] + used_model, + usage: CompletionUsage, ): from usage_costs.cost_utils import record_cost_auto from usage_costs.models import ModelSku @@ -872,16 +889,12 @@ def record_openai_llm_usage( record_cost_auto( model=used_model, sku=ModelSku.llm_prompt, - quantity=sum( - default_length_function(get_entry_text(entry)) for entry in messages - ), + quantity=usage.prompt_tokens, ) record_cost_auto( model=used_model, sku=ModelSku.llm_completion, - quantity=sum( - default_length_function(get_entry_text(entry)) for entry in choices - ), + quantity=usage.completion_tokens, ) diff --git a/poetry.lock b/poetry.lock index 05d2ccf1c..9785b6162 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. [[package]] name = "absl-py" @@ -2928,16 +2928,6 @@ files = [ {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-win32.whl", hash = "sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"}, @@ -3381,22 +3371,23 @@ signedtoken = ["cryptography (>=3.0.0)", "pyjwt (>=2.0.0,<3)"] [[package]] name = "openai" -version = "1.2.2" -description = "Client library for the openai API" +version = "1.35.7" +description = "The official Python library for the openai API" optional = false python-versions = ">=3.7.1" files = [ - {file = "openai-1.2.2-py3-none-any.whl", hash = "sha256:e9238e506b8ee8fc00af5b656de7907986317d7cfcf581964ff6e54a449be727"}, - {file = "openai-1.2.2.tar.gz", hash = "sha256:cbeff4eaf6cdcdccb24c9190d9880a827c034b221ed996201c10d578850b3db8"}, + {file = "openai-1.35.7-py3-none-any.whl", hash = "sha256:3d1e0b0aac9b0db69a972d36dc7efa7563f8e8d65550b27a48f2a0c2ec207e80"}, + {file = "openai-1.35.7.tar.gz", hash = "sha256:009bfa1504c9c7ef64d87be55936d142325656bbc6d98c68b669d6472e4beb09"}, ] [package.dependencies] -anyio = ">=3.5.0,<4" +anyio = ">=3.5.0,<5" distro = ">=1.7.0,<2" httpx = ">=0.23.0,<1" pydantic = ">=1.9.0,<3" +sniffio = "*" tqdm = ">4" -typing-extensions = ">=4.5,<5" +typing-extensions = ">=4.7,<5" [package.extras] datalib = ["numpy (>=1)", "pandas (>=1.2.3)", "pandas-stubs (>=1.1.0.11)"] @@ -4488,7 +4479,6 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, - {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -4496,15 +4486,8 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, - {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, - {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, - {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, - {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -4521,7 +4504,6 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, - {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -4529,7 +4511,6 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, - {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -6438,4 +6419,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.13" -content-hash = "036089f7da549adae7e644e866fbeb9be28036f3171f0e11fd0f1b799658eb31" +content-hash = "65a37e43b317f57251a4627758432f6786053554b463b6dcc8b7189c30a867b1" diff --git a/pyproject.toml b/pyproject.toml index 7680b5855..141ed91be 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ package-mode = false [tool.poetry.dependencies] python = ">=3.10,<3.13" streamlit = "^1.15.2" -openai = "^1.2.0" +openai = "^1.35.7" python-decouple = "^3.6" requests = "^2.28.1" glom = "^22.1.0" diff --git a/scripts/init_llm_pricing.py b/scripts/init_llm_pricing.py index dc51de566..afdc094b8 100644 --- a/scripts/init_llm_pricing.py +++ b/scripts/init_llm_pricing.py @@ -5,6 +5,27 @@ def run(): + # GPT-4o + + llm_pricing_create( + model_id="gpt-4o", + model_name=LargeLanguageModels.gpt_4_o.name, + unit_cost_input=5, + unit_cost_output=15, + unit_quantity=10**6, + provider=ModelProvider.openai, + pricing_url="https://openai.com/pricing", + ) + llm_pricing_create( + model_id="openai-gpt-4o-prod-eastus2-1", + model_name=LargeLanguageModels.gpt_4_o.name, + unit_cost_input=5, + unit_cost_output=15, + unit_quantity=10**6, + provider=ModelProvider.azure_openai, + pricing_url="https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/", + ) + # GPT-4-Turbo for model in ["gpt-4-0125-preview", "gpt-4-1106-preview"]: @@ -575,6 +596,18 @@ def run(): pricing_url="https://docs.anthropic.com/claude/docs/models-overview#model-comparison", ) + # SEA-LION + + llm_pricing_create( + model_id="aisingapore/sea-lion-7b-instruct", + model_name=LargeLanguageModels.sea_lion_7b_instruct.name, + unit_cost_input=5, + unit_cost_output=15, + unit_quantity=10**6, + provider=ModelProvider.aks, + notes="Same as GPT-4o. Note that the actual cost of this model is in GPU Milliseconds", + ) + def llm_pricing_create( model_id: str, @@ -583,7 +616,8 @@ def llm_pricing_create( unit_cost_output: float, unit_quantity: int, provider: ModelProvider, - pricing_url: str, + pricing_url: str = "", + notes: str = "", ): obj, created = ModelPricing.objects.get_or_create( model_id=model_id, @@ -609,6 +643,7 @@ def llm_pricing_create( category=category, provider=provider, pricing_url=pricing_url, + notes=notes, ), ) if created: diff --git a/usage_costs/admin.py b/usage_costs/admin.py index fbe367243..2d97d3422 100644 --- a/usage_costs/admin.py +++ b/usage_costs/admin.py @@ -1,13 +1,14 @@ from django.contrib import admin from bots.admin_links import open_in_new_tab, change_obj_url +from daras_ai.text_format import format_number_with_suffix from usage_costs import models class CostQtyMixin: @admin.display(description="Cost / Qty", ordering="unit_cost") - def cost_qty(self, obj): - return f"${obj.unit_cost.normalize()} / {obj.unit_quantity}" + def cost_qty(self, obj: models.ModelPricing | models.UsageCost): + return f"${obj.unit_cost.normalize()} / {format_number_with_suffix(obj.unit_quantity)}" @admin.register(models.UsageCost) diff --git a/usage_costs/models.py b/usage_costs/models.py index 6f6ed9234..bfea9ec61 100644 --- a/usage_costs/models.py +++ b/usage_costs/models.py @@ -89,7 +89,7 @@ class ModelPricing(models.Model): unit_cost = models.DecimalField( max_digits=max_digits, decimal_places=decimal_places, - help_text="The cost per unit.", + help_text="The cost per unit (in dollars).", ) unit_quantity = models.PositiveIntegerField( help_text="The quantity of the unit. (e.g. 1000 tokens)", default=1