Skip to content

Commit

Permalink
Updating harness examples
Browse files Browse the repository at this point in the history
  • Loading branch information
@NivekT
NivekT committed Jan 3, 2024
1 parent 37191aa commit 9cf3319
Showing 1 changed file with 251 additions and 49 deletions.
300 changes: 251 additions & 49 deletions examples/notebooks/remote/Saving_and_Loading_ExperimentHarness_to_Remote.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,14 +81,11 @@
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>template_index</th>\n",
" <th>templates</th>\n",
" <th>user_inputs</th>\n",
" <th>model</th>\n",
" <th>messages</th>\n",
" <th>temperature</th>\n",
" <th>top_p</th>\n",
" <th>n</th>\n",
" <th>presence_penalty</th>\n",
" <th>frequency_penalty</th>\n",
" <th>response</th>\n",
" <th>response_usage</th>\n",
" <th>latency</th>\n",
Expand All @@ -97,65 +94,65 @@
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>0</td>\n",
" <td>[{'role': 'system', 'content': 'You are a helpful assistant.'}, {'role': 'user', 'content': 'Who was the {{input}} president?'}]</td>\n",
" <td>{'input': 'first'}</td>\n",
" <td>gpt-3.5-turbo</td>\n",
" <td>[{'role': 'system', 'content': 'You are a helpful assistant.'}, {'role': 'user', 'content': 'Who was the first president?'}]</td>\n",
" <td>1.0</td>\n",
" <td>1.0</td>\n",
" <td>1</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>The first president of the United States was George Washington. He served two terms from 1789 to 1797.</td>\n",
" <td>{'completion_tokens': 24, 'prompt_tokens': 23, 'total_tokens': 47}</td>\n",
" <td>1.251051</td>\n",
" <td>The first president of the United States was George Washington. He served from 1789 to 1797.</td>\n",
" <td>{'completion_tokens': 22, 'prompt_tokens': 23, 'total_tokens': 45}</td>\n",
" <td>1.620099</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>1</td>\n",
" <td>[{'role': 'system', 'content': 'You are a helpful assistant.'}, {'role': 'user', 'content': 'Who was the {{input}} vice president?'}]</td>\n",
" <td>{'input': 'second'}</td>\n",
" <td>gpt-3.5-turbo</td>\n",
" <td>[{'role': 'system', 'content': 'You are a helpful assistant.'}, {'role': 'user', 'content': 'Who was the second president?'}]</td>\n",
" <td>1.0</td>\n",
" <td>1.0</td>\n",
" <td>1</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>The second president of the United States was John Adams. He served from 1797 to 1801.</td>\n",
" <td>{'completion_tokens': 22, 'prompt_tokens': 23, 'total_tokens': 45}</td>\n",
" <td>0.949733</td>\n",
" <td>0.657733</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>0</td>\n",
" <td>[{'role': 'system', 'content': 'You are a helpful assistant.'}, {'role': 'user', 'content': 'Who was the {{input}} president?'}]</td>\n",
" <td>{'input': 'first'}</td>\n",
" <td>gpt-3.5-turbo</td>\n",
" <td>[{'role': 'system', 'content': 'You are a helpful assistant.'}, {'role': 'user', 'content': 'Who was the first vice president?'}]</td>\n",
" <td>1.0</td>\n",
" <td>1.0</td>\n",
" <td>1</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>The first Vice President of the United States was John Adams. He served as Vice President under President George Washington from 1789 to 1797.</td>\n",
" <td>{'completion_tokens': 30, 'prompt_tokens': 24, 'total_tokens': 54}</td>\n",
" <td>0.936664</td>\n",
" <td>The first Vice President of the United States was John Adams. He served under President George Washington from 1789 to 1797.</td>\n",
" <td>{'completion_tokens': 27, 'prompt_tokens': 24, 'total_tokens': 51}</td>\n",
" <td>0.809008</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>1</td>\n",
" <td>[{'role': 'system', 'content': 'You are a helpful assistant.'}, {'role': 'user', 'content': 'Who was the {{input}} vice president?'}]</td>\n",
" <td>{'input': 'second'}</td>\n",
" <td>gpt-3.5-turbo</td>\n",
" <td>[{'role': 'system', 'content': 'You are a helpful assistant.'}, {'role': 'user', 'content': 'Who was the second vice president?'}]</td>\n",
" <td>1.0</td>\n",
" <td>1.0</td>\n",
" <td>1</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>The second vice president of the United States was Thomas Jefferson. He served as vice president from 1797 to 1801 under President John Adams.</td>\n",
" <td>The second Vice President of the United States was Thomas Jefferson. He served as Vice President under President John Adams from 1797 to 1801.</td>\n",
" <td>{'completion_tokens': 30, 'prompt_tokens': 24, 'total_tokens': 54}</td>\n",
" <td>0.971230</td>\n",
" <td>0.922675</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" template_index \\\n",
"0 0 \n",
"1 1 \n",
"2 0 \n",
"3 1 \n",
"\n",
" templates \\\n",
"0 [{'role': 'system', 'content': 'You are a helpful assistant.'}, {'role': 'user', 'content': 'Who was the {{input}} president?'}] \n",
"1 [{'role': 'system', 'content': 'You are a helpful assistant.'}, {'role': 'user', 'content': 'Who was the {{input}} vice president?'}] \n",
"2 [{'role': 'system', 'content': 'You are a helpful assistant.'}, {'role': 'user', 'content': 'Who was the {{input}} president?'}] \n",
"3 [{'role': 'system', 'content': 'You are a helpful assistant.'}, {'role': 'user', 'content': 'Who was the {{input}} vice president?'}] \n",
"\n",
" user_inputs model \\\n",
"0 {'input': 'first'} gpt-3.5-turbo \n",
"1 {'input': 'second'} gpt-3.5-turbo \n",
Expand All @@ -168,29 +165,23 @@
"2 [{'role': 'system', 'content': 'You are a helpful assistant.'}, {'role': 'user', 'content': 'Who was the first vice president?'}] \n",
"3 [{'role': 'system', 'content': 'You are a helpful assistant.'}, {'role': 'user', 'content': 'Who was the second vice president?'}] \n",
"\n",
" temperature top_p n presence_penalty frequency_penalty \\\n",
"0 1.0 1.0 1 0.0 0.0 \n",
"1 1.0 1.0 1 0.0 0.0 \n",
"2 1.0 1.0 1 0.0 0.0 \n",
"3 1.0 1.0 1 0.0 0.0 \n",
"\n",
" response \\\n",
"0 The first president of the United States was George Washington. He served two terms from 1789 to 1797. \n",
"0 The first president of the United States was George Washington. He served from 1789 to 1797. \n",
"1 The second president of the United States was John Adams. He served from 1797 to 1801. \n",
"2 The first Vice President of the United States was John Adams. He served as Vice President under President George Washington from 1789 to 1797. \n",
"3 The second vice president of the United States was Thomas Jefferson. He served as vice president from 1797 to 1801 under President John Adams. \n",
"2 The first Vice President of the United States was John Adams. He served under President George Washington from 1789 to 1797. \n",
"3 The second Vice President of the United States was Thomas Jefferson. He served as Vice President under President John Adams from 1797 to 1801. \n",
"\n",
" response_usage \\\n",
"0 {'completion_tokens': 24, 'prompt_tokens': 23, 'total_tokens': 47} \n",
"0 {'completion_tokens': 22, 'prompt_tokens': 23, 'total_tokens': 45} \n",
"1 {'completion_tokens': 22, 'prompt_tokens': 23, 'total_tokens': 45} \n",
"2 {'completion_tokens': 30, 'prompt_tokens': 24, 'total_tokens': 54} \n",
"2 {'completion_tokens': 27, 'prompt_tokens': 24, 'total_tokens': 51} \n",
"3 {'completion_tokens': 30, 'prompt_tokens': 24, 'total_tokens': 54} \n",
"\n",
" latency \n",
"0 1.251051 \n",
"1 0.949733 \n",
"2 0.936664 \n",
"3 0.971230 "
"0 1.620099 \n",
"1 0.657733 \n",
"2 0.809008 \n",
"3 0.922675 "
]
},
"metadata": {},
Expand All @@ -203,6 +194,217 @@
"harness.visualize()"
]
},
{
"cell_type": "markdown",
"id": "f246c958",
"metadata": {},
"source": [
"### Examples of `aggregation` API"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "88d81001",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>user_inputs</th>\n",
" <th>latency</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>((input, first),)</td>\n",
" <td>1.214553</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>((input, second),)</td>\n",
" <td>0.790204</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" user_inputs latency\n",
"0 ((input, first),) 1.214553\n",
"1 ((input, second),) 0.790204"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"agg_df = harness.aggregate(groupby_column='user_inputs', aggregate_columns='latency', method='mean')\n",
"agg_df"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "349bec6c",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>template_index</th>\n",
" <th>latency</th>\n",
" <th>templates</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>0</td>\n",
" <td>1.214553</td>\n",
" <td>[{'role': 'system', 'content': 'You are a helpful assistant.'}, {'role': 'user', 'content': 'Who was the {{input}} president?'}]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>1</td>\n",
" <td>0.790204</td>\n",
" <td>[{'role': 'system', 'content': 'You are a helpful assistant.'}, {'role': 'user', 'content': 'Who was the {{input}} vice president?'}]</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" template_index latency \\\n",
"0 0 1.214553 \n",
"1 1 0.790204 \n",
"\n",
" templates \n",
"0 [{'role': 'system', 'content': 'You are a helpful assistant.'}, {'role': 'user', 'content': 'Who was the {{input}} president?'}] \n",
"1 [{'role': 'system', 'content': 'You are a helpful assistant.'}, {'role': 'user', 'content': 'Who was the {{input}} vice president?'}] "
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"agg_df = harness.aggregate(groupby_column='templates', aggregate_columns='latency', method='mean')\n",
"agg_df"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "18419516",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>template_index</th>\n",
" <th>latency</th>\n",
" <th>latency</th>\n",
" <th>templates</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>0</td>\n",
" <td>1.214553</td>\n",
" <td>1.214553</td>\n",
" <td>[{'role': 'system', 'content': 'You are a helpful assistant.'}, {'role': 'user', 'content': 'Who was the {{input}} president?'}]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>1</td>\n",
" <td>0.790204</td>\n",
" <td>0.790204</td>\n",
" <td>[{'role': 'system', 'content': 'You are a helpful assistant.'}, {'role': 'user', 'content': 'Who was the {{input}} vice president?'}]</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" template_index latency latency \\\n",
"0 0 1.214553 1.214553 \n",
"1 1 0.790204 0.790204 \n",
"\n",
" templates \n",
"0 [{'role': 'system', 'content': 'You are a helpful assistant.'}, {'role': 'user', 'content': 'Who was the {{input}} president?'}] \n",
"1 [{'role': 'system', 'content': 'You are a helpful assistant.'}, {'role': 'user', 'content': 'Who was the {{input}} vice president?'}] "
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"agg_df = harness.aggregate(groupby_column='templates', aggregate_columns=['latency', 'latency'], method='mean')\n",
"agg_df"
]
},
{
"cell_type": "code",
"execution_count": 4,
Expand Down

0 comments on commit 9cf3319

Please sign in to comment.