Skip to content

Commit

Permalink
Merge pull request #159 from Aleph-Alpha/num-total-tokens-in-evaluation
Browse files Browse the repository at this point in the history
Return num_tokens_prompt_total for evaluation
  • Loading branch information
Khayet authored Jan 15, 2024
2 parents 94aae05 + 2afbaf3 commit 65688cd
Show file tree
Hide file tree
Showing 4 changed files with 14 additions and 5 deletions.
5 changes: 5 additions & 0 deletions Changelog.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,10 @@
# Changelog

## 7.0.0

- Added `num_tokens_prompt_total` to `EvaluationResponse`
- HTTP API version 1.16.0 or higher is required.

## 6.0.0

- Added `num_tokens_prompt_total` to the types below.
Expand Down
2 changes: 2 additions & 0 deletions aleph_alpha_client/evaluation.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,11 +53,13 @@ class EvaluationResponse:
model_version: str
message: Optional[str]
result: Dict[str, Any]
num_tokens_prompt_total: int

@staticmethod
def from_json(json: Dict[str, Any]) -> "EvaluationResponse":
return EvaluationResponse(
model_version=json["model_version"],
result=json["result"],
num_tokens_prompt_total=json["num_tokens_prompt_total"],
message=json.get("message"),
)
4 changes: 2 additions & 2 deletions aleph_alpha_client/version.py
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
__version__ = "6.0.0"
MIN_API_VERSION = "1.15.0"
__version__ = "7.0.0"
MIN_API_VERSION = "1.16.0"
8 changes: 5 additions & 3 deletions tests/test_evaluate.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ async def test_can_evaluate_with_async_client(
response = await async_client.evaluate(request, model=model_name)
assert response.model_version is not None
assert response.result is not None
assert response.num_tokens_prompt_total >= 1


# Client
Expand All @@ -34,7 +35,8 @@ def test_evaluate(sync_client: Client, model_name: str):
prompt=Prompt.from_text("hello"), completion_expected="world"
)

result = sync_client.evaluate(request, model=model_name)
response = sync_client.evaluate(request, model=model_name)

assert result.model_version is not None
assert result.result is not None
assert response.model_version is not None
assert response.result is not None
assert response.num_tokens_prompt_total >= 1

0 comments on commit 65688cd

Please sign in to comment.