diff --git a/notebooks/en/advanced_rag.ipynb b/notebooks/en/advanced_rag.ipynb index e2939838..c41f16b2 100644 --- a/notebooks/en/advanced_rag.ipynb +++ b/notebooks/en/advanced_rag.ipynb @@ -39,19 +39,7 @@ }, "outputs": [], "source": [ - "!pip install -q torch transformers transformers accelerate bitsandbytes langchain sentence-transformers faiss-gpu openpyxl pacmap datasets langchain-community ragatouille" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "8_Uyukt39-9J" - }, - "outputs": [], - "source": [ - "%reload_ext dotenv\n", - "%dotenv" + "!pip install -q torch transformers transformers accelerate bitsandbytes langchain sentence-transformers faiss-cpu openpyxl pacmap datasets langchain-community ragatouille" ] }, { diff --git a/notebooks/en/agent_rag.ipynb b/notebooks/en/agent_rag.ipynb index e3098da1..1441c238 100644 --- a/notebooks/en/agent_rag.ipynb +++ b/notebooks/en/agent_rag.ipynb @@ -85,33 +85,9 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Splitting documents...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|██████████| 2647/2647 [00:35<00:00, 74.10it/s] \n", - "/var/folders/6m/9b1tts6d5w960j80wbw9tx3m0000gn/T/ipykernel_19946/2464760906.py:37: LangChainDeprecationWarning: The class `HuggingFaceEmbeddings` was deprecated in LangChain 0.2.2 and will be removed in 1.0. An updated version of the class exists in the :class:`~langchain-huggingface package and should be used instead. To use it run `pip install -U :class:`~langchain-huggingface` and import as `from :class:`~langchain_huggingface import HuggingFaceEmbeddings``.\n", - " embedding_model = HuggingFaceEmbeddings(model_name=\"thenlper/gte-small\")\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Embedding documents... This should take a few minutes (5 minutes on MacBook with M1 Pro)\n" - ] - } - ], + "outputs": [], "source": [ "from tqdm import tqdm\n", "from transformers import AutoTokenizer\n", @@ -170,7 +146,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -230,13 +206,13 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ "from transformers.agents import HfApiEngine, ReactJsonAgent\n", "\n", - "llm_engine = HfApiEngine(\"meta-llama/Llama-3.1-70B-Instruct\")\n", + "llm_engine = HfApiEngine(\"Qwen/Qwen2.5-72B-Instruct\")\n", "\n", "retriever_tool = RetrieverTool(vectordb)\n", "agent = ReactJsonAgent(\n", @@ -255,15 +231,254 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 10, "metadata": {}, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[32;20;1m======== New task ========\u001b[0m\n", + "\u001b[37;1mHow can I push a model to the Hub?\u001b[0m\n", + "\u001b[38;20mSystem prompt is as follows:\u001b[0m\n", + "\u001b[38;20mYou are an expert assistant who can solve any task using JSON tool calls. You will be given a task to solve as best you can.\n", + "To do so, you have been given access to the following tools: 'retriever', 'final_answer'\n", + "The way you use the tools is by specifying a json blob, ending with ''.\n", + "Specifically, this json should have an `action` key (name of the tool to use) and an `action_input` key (input to the tool).\n", + "\n", + "The $ACTION_JSON_BLOB should only contain a SINGLE action, do NOT return a list of multiple actions. It should be formatted in json. Do not try to escape special characters. Here is the template of a valid $ACTION_JSON_BLOB:\n", + "{\n", + " \"action\": $TOOL_NAME,\n", + " \"action_input\": $INPUT\n", + "}\n", + "\n", + "Make sure to have the $INPUT as a dictionary in the right format for the tool you are using, and do not put variable names as input if you can find the right values.\n", + "\n", + "You should ALWAYS use the following format:\n", + "\n", + "Thought: you should always think about one action to take. Then use the action as follows:\n", + "Action:\n", + "$ACTION_JSON_BLOB\n", + "Observation: the result of the action\n", + "... (this Thought/Action/Observation can repeat N times, you should take several steps when needed. The $ACTION_JSON_BLOB must only use a SINGLE action at a time.)\n", + "\n", + "You can use the result of the previous action as input for the next action.\n", + "The observation will always be a string: it can represent a file, like \"image_1.jpg\".\n", + "Then you can use it as input for the next action. You can do it for instance as follows:\n", + "\n", + "Observation: \"image_1.jpg\"\n", + "\n", + "Thought: I need to transform the image that I received in the previous observation to make it green.\n", + "Action:\n", + "{\n", + " \"action\": \"image_transformer\",\n", + " \"action_input\": {\"image\": \"image_1.jpg\"}\n", + "}\n", + "\n", + "To provide the final answer to the task, use an action blob with \"action\": \"final_answer\" tool. It is the only way to complete the task, else you will be stuck on a loop. So your final output should look like this:\n", + "Action:\n", + "{\n", + " \"action\": \"final_answer\",\n", + " \"action_input\": {\"answer\": \"insert your final answer here\"}\n", + "}\n", + "\n", + "\n", + "Here are a few examples using notional tools:\n", + "---\n", + "Task: \"Generate an image of the oldest person in this document.\"\n", + "\n", + "Thought: I will proceed step by step and use the following tools: `document_qa` to find the oldest person in the document, then `image_generator` to generate an image according to the answer.\n", + "Action:\n", + "{\n", + " \"action\": \"document_qa\",\n", + " \"action_input\": {\"document\": \"document.pdf\", \"question\": \"Who is the oldest person mentioned?\"}\n", + "}\n", + "Observation: \"The oldest person in the document is John Doe, a 55 year old lumberjack living in Newfoundland.\"\n", + "\n", + "\n", + "Thought: I will now generate an image showcasing the oldest person.\n", + "Action:\n", + "{\n", + " \"action\": \"image_generator\",\n", + " \"action_input\": {\"prompt\": \"A portrait of John Doe, a 55-year-old man living in Canada.\"}\n", + "}\n", + "Observation: \"image.png\"\n", + "\n", + "Thought: I will now return the generated image.\n", + "Action:\n", + "{\n", + " \"action\": \"final_answer\",\n", + " \"action_input\": \"image.png\"\n", + "}\n", + "\n", + "---\n", + "Task: \"What is the result of the following operation: 5 + 3 + 1294.678?\"\n", + "\n", + "Thought: I will use python code evaluator to compute the result of the operation and then return the final answer using the `final_answer` tool\n", + "Action:\n", + "{\n", + " \"action\": \"python_interpreter\",\n", + " \"action_input\": {\"code\": \"5 + 3 + 1294.678\"}\n", + "}\n", + "Observation: 1302.678\n", + "\n", + "Thought: Now that I know the result, I will now return it.\n", + "Action:\n", + "{\n", + " \"action\": \"final_answer\",\n", + " \"action_input\": \"1302.678\"\n", + "}\n", + "\n", + "---\n", + "Task: \"Which city has the highest population , Guangzhou or Shanghai?\"\n", + "\n", + "Thought: I need to get the populations for both cities and compare them: I will use the tool `search` to get the population of both cities.\n", + "Action:\n", + "{\n", + " \"action\": \"search\",\n", + " \"action_input\": \"Population Guangzhou\"\n", + "}\n", + "Observation: ['Guangzhou has a population of 15 million inhabitants as of 2021.']\n", + "\n", + "\n", + "Thought: Now let's get the population of Shanghai using the tool 'search'.\n", + "Action:\n", + "{\n", + " \"action\": \"search\",\n", + " \"action_input\": \"Population Shanghai\"\n", + "}\n", + "Observation: '26 million (2019)'\n", + "\n", + "Thought: Now I know that Shanghai has a larger population. Let's return the result.\n", + "Action:\n", + "{\n", + " \"action\": \"final_answer\",\n", + " \"action_input\": \"Shanghai\"\n", + "}\n", + "\n", + "\n", + "Above example were using notional tools that might not exist for you. You only have acces to those tools:\n", + "\n", + "- retriever: Using semantic similarity, retrieves some documents from the knowledge base that have the closest embeddings to the input query.\n", + " Takes inputs: {'query': {'type': 'string', 'description': 'The query to perform. This should be semantically close to your target documents. Use the affirmative form rather than a question.'}}\n", + " Returns an output of type: string\n", + "\n", + "- final_answer: Provides a final answer to the given problem.\n", + " Takes inputs: {'answer': {'type': 'any', 'description': 'The final answer to the problem'}}\n", + " Returns an output of type: any\n", + "\n", + "Here are the rules you should always follow to solve your task:\n", + "1. ALWAYS provide a 'Thought:' sequence, and an 'Action:' sequence that ends with , else you will fail.\n", + "2. Always use the right arguments for the tools. Never use variable names in the 'action_input' field, use the value instead.\n", + "3. Call a tool only when needed: do not call the search agent if you do not need information, try to solve the task yourself.\n", + "4. Never re-do a tool call that you previously did with the exact same parameters.\n", + "\n", + "Now Begin! If you solve the task correctly, you will receive a reward of $1,000,000.\n", + "\u001b[0m\n", + "\u001b[38;20m===== New step =====\u001b[0m\n", + "===== Calling LLM with this last message: =====\n", + "{'role': , 'content': 'Task: How can I push a model to the Hub?'}\n", + "\u001b[38;20m===== Output message of the LLM: =====\u001b[0m\n", + "\u001b[38;20mThought: To find out how to push a model to the Hub, I need to search the knowledge base for relevant information using the 'retriever' tool.\n", + "Action:\n", + "{\n", + " \"action\": \"retriever\",\n", + " \"action_input\": {\"query\": \"push a model to the Hub\"}\n", + "}\u001b[0m\n", + "\u001b[38;20m===== Extracting action =====\u001b[0m\n", + "\u001b[33;1m=== Agent thoughts:\u001b[0m\n", + "\u001b[0mThought: To find out how to push a model to the Hub, I need to search the knowledge base for relevant information using the 'retriever' tool.\u001b[0m\n", + "\u001b[33;1m>>> Calling tool: 'retriever' with arguments: {'query': 'push a model to the Hub'}\u001b[0m\n", + "Retrieved documents:\n", + "===== Document 0 =====\n", + "# Step 7. Push everything to the Hub\n", + " api.upload_folder(\n", + " repo_id=repo_id,\n", + " folder_path=repo_local_path,\n", + " path_in_repo=\".\",\n", + " )\n", + "\n", + " print(\"Your model is pushed to the Hub. You can view your model here: \", repo_url)\n", + "```\n", + "\n", + "### .\n", + "\n", + "By using `push_to_hub` **you evaluate, record a replay, generate a model card of your agent and push it to the Hub**.===== Document 1 =====\n", + "```py\n", + ">>> trainer.push_to_hub()\n", + "```\n", + "\n", + "\n", + "Share a model to the Hub with [`PushToHubCallback`]. In the [`PushToHubCallback`] function, add:\n", + "\n", + "- An output directory for your model.\n", + "- A tokenizer.\n", + "- The `hub_model_id`, which is your Hub username and model name.\n", + "\n", + "```py\n", + ">>> from transformers import PushToHubCallback\n", + "\n", + ">>> push_to_hub_callback = PushToHubCallback(\n", + "... output_dir=\"./your_model_save_path\", tokenizer=tokenizer, hub_model_id=\"your-username/my-awesome-model\"\n", + "... )\n", + "```===== Document 2 =====\n", + "Let's pretend we've now fine-tuned the model. The next step would be to push it to the Hub! We can do this with the `timm.models.hub.push_to_hf_hub` function.\n", + "\n", + "```py\n", + ">>> model_cfg = dict(labels=['a', 'b', 'c', 'd'])\n", + ">>> timm.models.hub.push_to_hf_hub(model, 'resnet18-random', model_config=model_cfg)\n", + "```\n", + "\n", + "Running the above would push the model to `/resnet18-random` on the Hub. You can now share this model with your friends, or use it in your own code!\n", + "\n", + "## Loading a Model===== Document 3 =====\n", + "processor.push_to_hub(hub_model_id)\n", + "trainer.push_to_hub(**kwargs)\n", + "```\n", + "\n", + "# 4. Inference\n", + "\n", + "Now comes the exciting part, using our fine-tuned model! In this section, we'll show how you can load your model from the hub and use it for inference.===== Document 4 =====\n", + "Finally, if you want, you can push your model up to the hub. Here, we'll push it up if you specified `push_to_hub=True` in the training configuration. Note that in order to push to hub, you'll have to have git-lfs installed and be logged into your Hugging Face account (which can be done via `huggingface-cli login`).\n", + "\n", + "```python\n", + "kwargs = {\n", + " \"finetuned_from\": model.config._name_or_path,\n", + " \"tasks\": \"image-classification\",\n", + " \"dataset\": 'beans',\n", + " \"tags\": ['image-classification'],\n", + "}===== Document 5 =====\n", + "--push_to_hub\n", + "```===== Document 6 =====\n", + ". The second way to upload a model, though, is to call model.push_to_hub(). So this is more of a once-off method - it's not called regularly during training. You can just call this manually whenever you want to upload a model to the hub. So we recommend running this after the end of training, just to make sure that you have a commit message just to guarantee that this was the final version of the model at the end of training. And it just makes sure that you're working with the definitive end-of-training model and not accidentally using a model that's from a checkpoint somewhere along the way\n", + "\u001b[38;20m===== New step =====\u001b[0m\n", + "===== Calling LLM with this last message: =====\n", + "{'role': , 'content': '[OUTPUT OF STEP 0] -> Observation:\\nRetrieved documents:\\n===== Document 0 =====\\n# Step 7. Push everything to the Hub\\n api.upload_folder(\\n repo_id=repo_id,\\n folder_path=repo_local_path,\\n path_in_repo=\".\",\\n )\\n\\n print(\"Your model is pushed to the Hub. You can view your model here: \", repo_url)\\n```\\n\\n### .\\n\\nBy using `push_to_hub` **you evaluate, record a replay, generate a model card of your agent and push it to the Hub**.===== Document 1 =====\\n```py\\n>>> trainer.push_to_hub()\\n```\\n\\n\\nShare a model to the Hub with [`PushToHubCallback`]. In the [`PushToHubCallback`] function, add:\\n\\n- An output directory for your model.\\n- A tokenizer.\\n- The `hub_model_id`, which is your Hub username and model name.\\n\\n```py\\n>>> from transformers import PushToHubCallback\\n\\n>>> push_to_hub_callback = PushToHubCallback(\\n... output_dir=\"./your_model_save_path\", tokenizer=tokenizer, hub_model_id=\"your-username/my-awesome-model\"\\n... )\\n```===== Document 2 =====\\nLet\\'s pretend we\\'ve now fine-tuned the model. The next step would be to push it to the Hub! We can do this with the `timm.models.hub.push_to_hf_hub` function.\\n\\n```py\\n>>> model_cfg = dict(labels=[\\'a\\', \\'b\\', \\'c\\', \\'d\\'])\\n>>> timm.models.hub.push_to_hf_hub(model, \\'resnet18-random\\', model_config=model_cfg)\\n```\\n\\nRunning the above would push the model to `/resnet18-random` on the Hub. You can now share this model with your friends, or use it in your own code!\\n\\n## Loading a Model===== Document 3 =====\\nprocessor.push_to_hub(hub_model_id)\\ntrainer.push_to_hub(**kwargs)\\n```\\n\\n# 4. Inference\\n\\nNow comes the exciting part, using our fine-tuned model! In this section, we\\'ll show how you can load your model from the hub and use it for inference.===== Document 4 =====\\nFinally, if you want, you can push your model up to the hub. Here, we\\'ll push it up if you specified `push_to_hub=True` in the training configuration. Note that in order to push to hub, you\\'ll have to have git-lfs installed and be logged into your Hugging Face account (which can be done via `huggingface-cli login`).\\n\\n```python\\nkwargs = {\\n \"finetuned_from\": model.config._name_or_path,\\n \"tasks\": \"image-classification\",\\n \"dataset\": \\'beans\\',\\n \"tags\": [\\'image-classification\\'],\\n}===== Document 5 =====\\n--push_to_hub\\n```===== Document 6 =====\\n. The second way to upload a model, though, is to call model.push_to_hub(). So this is more of a once-off method - it\\'s not called regularly during training. You can just call this manually whenever you want to upload a model to the hub. So we recommend running this after the end of training, just to make sure that you have a commit message just to guarantee that this was the final version of the model at the end of training. And it just makes sure that you\\'re working with the definitive end-of-training model and not accidentally using a model that\\'s from a checkpoint somewhere along the way'}\n", + "\u001b[38;20m===== Output message of the LLM: =====\u001b[0m\n", + "\u001b[38;20mThought: I retrieved information on how to push a model to the Hub. Now, I will gather the main points and return the final answer using the `final_answer` tool.\n", + "Action:\n", + "{\n", + " \"action\": \"final_answer\",\n", + " \"action_input\": {\"answer\": \"There are several ways to push a model to the Hub:\\n1. Using `trainer.push_to_hub()` method.\\n2. Using `model.push_to_hub()` method, which is a once-off approach after training.\\n3. Using `timm.models.hub.push_to_hf_hub` function for specific frameworks.\\n4. Using `api.upload_folder` with `repo_id` and `folder_path` to upload the entire folder.\\n5. Setting `--push_to_hub` flag in the training configuration.\\nTo use these methods, ensure you have `git-lfs` installed and are logged into your Hugging Face account using `huggingface-cli login`.\"}\n", + "}\u001b[0m\n", + "\u001b[38;20m===== Extracting action =====\u001b[0m\n", + "\u001b[33;1m=== Agent thoughts:\u001b[0m\n", + "\u001b[0mThought: I retrieved information on how to push a model to the Hub. Now, I will gather the main points and return the final answer using the `final_answer` tool.\u001b[0m\n", + "\u001b[33;1m>>> Calling tool: 'final_answer' with arguments: {'answer': 'There are several ways to push a model to the Hub:\\n1. Using `trainer.push_to_hub()` method.\\n2. Using `model.push_to_hub()` method, which is a once-off approach after training.\\n3. Using `timm.models.hub.push_to_hf_hub` function for specific frameworks.\\n4. Using `api.upload_folder` with `repo_id` and `folder_path` to upload the entire folder.\\n5. Setting `--push_to_hub` flag in the training configuration.\\nTo use these methods, ensure you have `git-lfs` installed and are logged into your Hugging Face account using `huggingface-cli login`.'}\u001b[0m\n" + ] + }, { "name": "stdout", "output_type": "stream", "text": [ "Final output:\n", - "To push a model to the Hub, use `model.push_to_hub()`.\n" + "There are several ways to push a model to the Hub:\n", + "1. Using `trainer.push_to_hub()` method.\n", + "2. Using `model.push_to_hub()` method, which is a once-off approach after training.\n", + "3. Using `timm.models.hub.push_to_hf_hub` function for specific frameworks.\n", + "4. Using `api.upload_folder` with `repo_id` and `folder_path` to upload the entire folder.\n", + "5. Setting `--push_to_hub` flag in the training configuration.\n", + "To use these methods, ensure you have `git-lfs` installed and are logged into your Hugging Face account using `huggingface-cli login`.\n" ] } ], @@ -309,7 +524,9 @@ "source": [ "import logging\n", "\n", - "agent.logger.setLevel(logging.WARNING)" + "agent.logger.setLevel(logging.WARNING) # Let's reduce the agent's verbosity level\n", + "\n", + "eval_dataset = datasets.load_dataset(\"m-ric/huggingface_doc_qa_eval\", split=\"train\")" ] }, { @@ -355,7 +572,7 @@ "source": [ "from huggingface_hub import InferenceClient\n", "\n", - "reader_llm = InferenceClient(\"meta-llama/Llama-3.1-70B-Instruct\")\n", + "reader_llm = InferenceClient(\"Qwen/Qwen2.5-72B-Instruct\")\n", "\n", "outputs_standard_rag = []\n", "\n", @@ -399,7 +616,7 @@ }, { "cell_type": "code", - "execution_count": 68, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -432,60 +649,25 @@ }, { "cell_type": "code", - "execution_count": 69, + "execution_count": 19, "metadata": {}, "outputs": [], "source": [ "from huggingface_hub import InferenceClient\n", "\n", - "evaluation_client = InferenceClient(\"meta-llama/Meta-Llama-3-70B-Instruct\")" + "evaluation_client = InferenceClient(\"meta-llama/Llama-3.1-70B-Instruct\")" ] }, { "cell_type": "code", - "execution_count": 70, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|██████████| 65/65 [02:24<00:00, 2.23s/it]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Average score for agentic RAG: 78.5%\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|██████████| 65/65 [02:17<00:00, 2.12s/it]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Average score for standard RAG: 70.0%\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\n" - ] - } - ], + "outputs": [], "source": [ "import pandas as pd\n", "\n", - "for type, outputs in [\n", + "results = {}\n", + "for system_type, outputs in [\n", " (\"agentic\", outputs_agentic_rag),\n", " (\"standard\", outputs_standard_rag),\n", "]:\n", @@ -510,15 +692,44 @@ " except:\n", " print(f\"Parsing failed - output was: {eval_result}\")\n", "\n", - " results = pd.DataFrame.from_dict(outputs)\n", - " results = results.loc[~results[\"generated_answer\"].str.contains(\"Error\")]\n", - " results[\"eval_score_LLM_judge_int\"] = (\n", - " results[\"eval_score_LLM_judge\"].fillna(1).apply(lambda x: int(x))\n", + " results[system_type] = pd.DataFrame.from_dict(outputs)\n", + " results[system_type] = results[system_type].loc[~results[system_type][\"generated_answer\"].str.contains(\"Error\")]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Average score for agentic RAG: 86.9%\n", + "Average score for standard RAG: 73.1%\n" + ] + } + ], + "source": [ + "DEFAULT_SCORE = 2 # Give average score whenever scoring fails\n", + "def fill_score(x):\n", + " try:\n", + " return int(x)\n", + " except:\n", + " return DEFAULT_SCORE\n", + "\n", + "for system_type, outputs in [\n", + " (\"agentic\", outputs_agentic_rag),\n", + " (\"standard\", outputs_standard_rag),\n", + "]:\n", + "\n", + " results[system_type][\"eval_score_LLM_judge_int\"] = (\n", + " results[system_type][\"eval_score_LLM_judge\"].fillna(DEFAULT_SCORE).apply(fill_score)\n", " )\n", - " results[\"eval_score_LLM_judge_int\"] = (results[\"eval_score_LLM_judge_int\"] - 1) / 2\n", + " results[system_type][\"eval_score_LLM_judge_int\"] = (results[system_type][\"eval_score_LLM_judge_int\"] - 1) / 2\n", "\n", " print(\n", - " f\"Average score for {type} RAG: {results['eval_score_LLM_judge_int'].mean()*100:.1f}%\"\n", + " f\"Average score for {system_type} RAG: {results[system_type]['eval_score_LLM_judge_int'].mean()*100:.1f}%\"\n", " )" ] }, @@ -526,7 +737,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "**Let us recap: the Agent setup improves scores by 8.5% compared to a standard RAG!** (from 70.0% to 78.5%)\n", + "**Let us recap: the Agent setup improves scores by 14% compared to a standard RAG!** (from 73.1% to 86.9%)\n", "\n", "This is a great improvement, with a very simple setup 🚀\n", "\n", @@ -536,9 +747,9 @@ ], "metadata": { "kernelspec": { - "display_name": "cookbook2", + "display_name": "test2", "language": "python", - "name": "cookbook2" + "name": "test2" }, "language_info": { "codemirror_mode": {