QA_without_gpt2.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "e7be0dc1fa5b409d8fa40d865b90a9ad": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_010c7484292d459b83e74b1c5722f079",
              "IPY_MODEL_6f65810532ee4784be7412ef8b062110",
              "IPY_MODEL_68f2539fa69f4011b17b27a482b00a31"
            ],
            "layout": "IPY_MODEL_7abaef36c2ef475a8453188b04de33c1"
          }
        },
        "010c7484292d459b83e74b1c5722f079": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9783f03db1fe4cebbbb943f2fdbb2cd0",
            "placeholder": "​",
            "style": "IPY_MODEL_dc8bb0fafde144ce9595739c1f20cecd",
            "value": "Saving the dataset (1/1 shards): 100%"
          }
        },
        "6f65810532ee4784be7412ef8b062110": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bb5b383d0fa24000ba4fd2c5e72de383",
            "max": 75608,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_0c45f181710145cda0245e92b1399674",
            "value": 75608
          }
        },
        "68f2539fa69f4011b17b27a482b00a31": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_47b592e99a81416d81d79065d30187a9",
            "placeholder": "​",
            "style": "IPY_MODEL_4c702b0cfab14c13b16846e3d69f3472",
            "value": " 75608/75608 [00:00&lt;00:00, 365714.65 examples/s]"
          }
        },
        "7abaef36c2ef475a8453188b04de33c1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9783f03db1fe4cebbbb943f2fdbb2cd0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "dc8bb0fafde144ce9595739c1f20cecd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "bb5b383d0fa24000ba4fd2c5e72de383": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0c45f181710145cda0245e92b1399674": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "47b592e99a81416d81d79065d30187a9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4c702b0cfab14c13b16846e3d69f3472": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "cc14f0f486de4087b100359327c39473": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_f9cf7e7d6c734d538bb274bf2bfbb2a9",
              "IPY_MODEL_87b663fa3bf24592bc8cfd1607cb6b88",
              "IPY_MODEL_056a808775524397a76f54d681547306"
            ],
            "layout": "IPY_MODEL_17caee241c5f465da258249df8acc335"
          }
        },
        "f9cf7e7d6c734d538bb274bf2bfbb2a9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ba940d3a4eb344d9827d3e54b1f081f6",
            "placeholder": "​",
            "style": "IPY_MODEL_dea3570b32a445309898322147e051da",
            "value": "Saving the dataset (1/1 shards): 100%"
          }
        },
        "87b663fa3bf24592bc8cfd1607cb6b88": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9514a2c7e1764ea8a433dcfbad9da919",
            "max": 8663,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_1c599d7729534bfe944e4cbe58776e3c",
            "value": 8663
          }
        },
        "056a808775524397a76f54d681547306": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_368e779161794d13a2c5d70c19d4fad7",
            "placeholder": "​",
            "style": "IPY_MODEL_c5503b27ae0f4a6ea7f524f07b1be731",
            "value": " 8663/8663 [00:00&lt;00:00, 147553.74 examples/s]"
          }
        },
        "17caee241c5f465da258249df8acc335": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ba940d3a4eb344d9827d3e54b1f081f6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "dea3570b32a445309898322147e051da": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9514a2c7e1764ea8a433dcfbad9da919": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1c599d7729534bfe944e4cbe58776e3c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "368e779161794d13a2c5d70c19d4fad7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c5503b27ae0f4a6ea7f524f07b1be731": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "433e2e7105d2489d9f2259f2ba302170": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_613b448408464000925987bda4b59af5",
              "IPY_MODEL_9778356bb72d4a4b8b2efe57b43393e7",
              "IPY_MODEL_4e90f02fed5445bbb441c2b45499a9f9"
            ],
            "layout": "IPY_MODEL_0292c90e5aaa431a9001e024c30171ff"
          }
        },
        "613b448408464000925987bda4b59af5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1b9275a3aaab4c96b1a39e39f64e388d",
            "placeholder": "​",
            "style": "IPY_MODEL_3e379f763c474a0ebbe800b37e82286c",
            "value": "Saving the dataset (1/1 shards): 100%"
          }
        },
        "9778356bb72d4a4b8b2efe57b43393e7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_02916728551841a0a2325ec248e77f5e",
            "max": 87599,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_1ae6a9d529c14241b538e47b654768fb",
            "value": 87599
          }
        },
        "4e90f02fed5445bbb441c2b45499a9f9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b76b8a461c5b48e5910f72210931aec2",
            "placeholder": "​",
            "style": "IPY_MODEL_2eee3979b2db4bcd8666a312c75750b8",
            "value": " 87599/87599 [00:00&lt;00:00, 453860.59 examples/s]"
          }
        },
        "0292c90e5aaa431a9001e024c30171ff": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1b9275a3aaab4c96b1a39e39f64e388d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3e379f763c474a0ebbe800b37e82286c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "02916728551841a0a2325ec248e77f5e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1ae6a9d529c14241b538e47b654768fb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "b76b8a461c5b48e5910f72210931aec2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2eee3979b2db4bcd8666a312c75750b8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3e02f28e34c546f9b0bb363423c7e550": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_812785d0e6ff4904a1159c361311a7a7",
              "IPY_MODEL_08a8caa2cd014ae186c1659bfc722bea",
              "IPY_MODEL_09c0a27d936d4d10b8d81a7729f9bf88"
            ],
            "layout": "IPY_MODEL_f65cb838e6504f859d3e2ce60f5b3aab"
          }
        },
        "812785d0e6ff4904a1159c361311a7a7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3a19356bf5c949d4baffd7fd43231a7a",
            "placeholder": "​",
            "style": "IPY_MODEL_79b8e75a31bc43f29d2fe260b96ba88f",
            "value": "Saving the dataset (1/1 shards): 100%"
          }
        },
        "08a8caa2cd014ae186c1659bfc722bea": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0ea55c9248154dd496666754b42b2e3a",
            "max": 10570,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_238f757c32604a02a963aeaf7e7df55f",
            "value": 10570
          }
        },
        "09c0a27d936d4d10b8d81a7729f9bf88": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2e81c1b7723f44d0a22b54d99b60a6ff",
            "placeholder": "​",
            "style": "IPY_MODEL_23b2395e2a0e4b83b16ce862ace73937",
            "value": " 10570/10570 [00:00&lt;00:00, 246714.15 examples/s]"
          }
        },
        "f65cb838e6504f859d3e2ce60f5b3aab": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3a19356bf5c949d4baffd7fd43231a7a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "79b8e75a31bc43f29d2fe260b96ba88f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0ea55c9248154dd496666754b42b2e3a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "238f757c32604a02a963aeaf7e7df55f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "2e81c1b7723f44d0a22b54d99b60a6ff": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "23b2395e2a0e4b83b16ce862ace73937": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "cead3ede305b45f4a7761a2b5f7d0a8f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_3e4d665055854b548486c2e25e82ce2f",
              "IPY_MODEL_369d51df338842029bb275fda9146583",
              "IPY_MODEL_bf4b002f54e344a98c25e99b215c0e94"
            ],
            "layout": "IPY_MODEL_e6e67cfb9a4c4efbb1ae7256440993ec"
          }
        },
        "3e4d665055854b548486c2e25e82ce2f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6dd11437522044fbb9529839c41bdeb2",
            "placeholder": "​",
            "style": "IPY_MODEL_c6cba4d3c4804b65a6a60b24fd474d50",
            "value": "Saving the dataset (1/1 shards): 100%"
          }
        },
        "369d51df338842029bb275fda9146583": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_794dac0ac2ad4854ac79cf5daa71cb47",
            "max": 87599,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_3c6413e4cbd04c8098b6f96193664a36",
            "value": 87599
          }
        },
        "bf4b002f54e344a98c25e99b215c0e94": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_883339be841740d2a1f50e7b5d7c649b",
            "placeholder": "​",
            "style": "IPY_MODEL_a8f88596619443798a26f1fbfdb0d701",
            "value": " 87599/87599 [00:00&lt;00:00, 365221.70 examples/s]"
          }
        },
        "e6e67cfb9a4c4efbb1ae7256440993ec": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6dd11437522044fbb9529839c41bdeb2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c6cba4d3c4804b65a6a60b24fd474d50": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "794dac0ac2ad4854ac79cf5daa71cb47": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3c6413e4cbd04c8098b6f96193664a36": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "883339be841740d2a1f50e7b5d7c649b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a8f88596619443798a26f1fbfdb0d701": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "8048c51e4ba34395b64bde9a3dbe4a04": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_0965bb1cc6bb401d848f03c359862854",
              "IPY_MODEL_517b3b043e5f4f50a395686f854e6199",
              "IPY_MODEL_16b11fa4c81e4cee9c6c1501e12bdf70"
            ],
            "layout": "IPY_MODEL_77fb9d215b75412ba64dfb0fd650de91"
          }
        },
        "0965bb1cc6bb401d848f03c359862854": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9503946fbbee4d66917e60f947d976ba",
            "placeholder": "​",
            "style": "IPY_MODEL_9f204b229c244839b17d564effff2fae",
            "value": "Saving the dataset (1/1 shards): 100%"
          }
        },
        "517b3b043e5f4f50a395686f854e6199": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5970ad3d91a64f00a872b8ff2f77386c",
            "max": 10570,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_32f5d622186a43f89a4895a01ac7af3d",
            "value": 10570
          }
        },
        "16b11fa4c81e4cee9c6c1501e12bdf70": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a1c322177c56445b97b8d07b10b2131b",
            "placeholder": "​",
            "style": "IPY_MODEL_7452ba4b4d484de2bd3e387ebe68f749",
            "value": " 10570/10570 [00:00&lt;00:00, 184340.86 examples/s]"
          }
        },
        "77fb9d215b75412ba64dfb0fd650de91": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9503946fbbee4d66917e60f947d976ba": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9f204b229c244839b17d564effff2fae": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5970ad3d91a64f00a872b8ff2f77386c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "32f5d622186a43f89a4895a01ac7af3d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "a1c322177c56445b97b8d07b10b2131b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7452ba4b4d484de2bd3e387ebe68f749": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Vilmo18/Languages_Models_Without_Entity_Knowledge/blob/main/QA_without_gpt2.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Import"
      ],
      "metadata": {
        "id": "MF3sHWoQ1pJ3"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -q transformers datasets\n",
        "!pip install --upgrade pyarrow"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pk4P30JJudng",
        "outputId": "fd538df4-4066-4fc8-8361-55fad78a4d9a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m527.3/527.3 kB\u001b[0m \u001b[31m5.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m39.9/39.9 MB\u001b[0m \u001b[31m37.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m116.3/116.3 kB\u001b[0m \u001b[31m25.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m64.9/64.9 kB\u001b[0m \u001b[31m14.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m194.1/194.1 kB\u001b[0m \u001b[31m36.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m28.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "cudf-cu12 24.4.1 requires pyarrow<15.0.0a0,>=14.0.1, but you have pyarrow 17.0.0 which is incompatible.\n",
            "google-colab 1.0.0 requires requests==2.31.0, but you have requests 2.32.3 which is incompatible.\n",
            "ibis-framework 8.0.0 requires pyarrow<16,>=2, but you have pyarrow 17.0.0 which is incompatible.\u001b[0m\u001b[31m\n",
            "\u001b[0mRequirement already satisfied: pyarrow in /usr/local/lib/python3.10/dist-packages (17.0.0)\n",
            "Requirement already satisfied: numpy>=1.16.6 in /usr/local/lib/python3.10/dist-packages (from pyarrow) (1.25.2)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install accelerate -U"
      ],
      "metadata": {
        "id": "0mQj4LaG7Z-A"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": true,
        "id": "S77zsZak_gUb"
      },
      "outputs": [],
      "source": [
        "!pip install -q transformers datasets\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "QzKtbI85OEdn"
      },
      "outputs": [],
      "source": [
        "!pip install flair -q"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Import dataset"
      ],
      "metadata": {
        "id": "JkFMD7fv1uZi"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import random\n",
        "from datasets import load_dataset\n",
        "from flair.models import SequenceTagger\n",
        "from flair.data import Sentence\n",
        "\n",
        "# Load the SQuAD dataset\n",
        "dataset = load_dataset(\"squad\")\n",
        "\n",
        "# Load the NER model\n",
        "#tagger = SequenceTagger.load('flair/ner-english-ontonotes-large')\n",
        "\n",
        "# Define tags to be included\n",
        "included_tags = {\"EVENT\", \"FAC\", \"GPE\", \"LANGUAGE\", \"LAW\", \"LOCATION\", \"NORP\",\n",
        "                 \"ORG\", \"PERSON\", \"PRODUCT\", \"WORK_OF_ART\"}"
      ],
      "metadata": {
        "id": "_9kyVoAgrnEF",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3ae6cd77-9921-497a-de1c-8816ca95c328"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:89: UserWarning: \n",
            "The secret `HF_TOKEN` does not exist in your Colab secrets.\n",
            "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n",
            "You will be able to reuse this secret in all of your notebooks.\n",
            "Please note that authentication is recommended but still optional to access public models or datasets.\n",
            "  warnings.warn(\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import random\n",
        "from datasets import load_dataset, DatasetDict, Dataset\n",
        "from flair.models import SequenceTagger\n",
        "from flair.data import Sentence\n",
        "from tqdm import tqdm\n",
        "\n",
        "# Load the SQuAD dataset\n",
        "dataset = load_dataset(\"squad\")\n",
        "\n",
        "# Load the NER model\n",
        "tagger = SequenceTagger.load('flair/ner-english-ontonotes-large')\n",
        "\n",
        "# Define tags to be included\n",
        "included_tags = {\"EVENT\", \"FAC\", \"GPE\", \"LANGUAGE\", \"LAW\", \"LOCATION\", \"NORP\",\n",
        "                 \"ORG\", \"PERSON\", \"PRODUCT\", \"WORK_OF_ART\"}\n",
        "\n",
        "excluded_tags = {\"CARDINAL\", \"DATA\", \"MONEY\", \"ORDINAL\", \"PERCENT\", \"QUANTITY\", \"TIME\"}\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uYONIBhY5ZH9",
        "outputId": "831314c1-6ca2-4e8b-c467-8516761b8cac"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-04 08:39:18,927 SequenceTagger predicts: Dictionary with 76 tags: <unk>, O, B-CARDINAL, E-CARDINAL, S-PERSON, S-CARDINAL, S-PRODUCT, B-PRODUCT, I-PRODUCT, E-PRODUCT, B-WORK_OF_ART, I-WORK_OF_ART, E-WORK_OF_ART, B-PERSON, E-PERSON, S-GPE, B-DATE, I-DATE, E-DATE, S-ORDINAL, S-LANGUAGE, I-PERSON, S-EVENT, S-DATE, B-QUANTITY, E-QUANTITY, S-TIME, B-TIME, I-TIME, E-TIME, B-GPE, E-GPE, S-ORG, I-GPE, S-NORP, B-FAC, I-FAC, E-FAC, B-NORP, E-NORP, S-PERCENT, B-ORG, E-ORG, B-LANGUAGE, E-LANGUAGE, I-CARDINAL, I-ORG, S-WORK_OF_ART, I-QUANTITY, B-MONEY\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# train_split.save_to_disk(\"train_split\")\n",
        "# validation_split.save_to_disk(\"validation_split\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 81,
          "referenced_widgets": [
            "433e2e7105d2489d9f2259f2ba302170",
            "613b448408464000925987bda4b59af5",
            "9778356bb72d4a4b8b2efe57b43393e7",
            "4e90f02fed5445bbb441c2b45499a9f9",
            "0292c90e5aaa431a9001e024c30171ff",
            "1b9275a3aaab4c96b1a39e39f64e388d",
            "3e379f763c474a0ebbe800b37e82286c",
            "02916728551841a0a2325ec248e77f5e",
            "1ae6a9d529c14241b538e47b654768fb",
            "b76b8a461c5b48e5910f72210931aec2",
            "2eee3979b2db4bcd8666a312c75750b8",
            "3e02f28e34c546f9b0bb363423c7e550",
            "812785d0e6ff4904a1159c361311a7a7",
            "08a8caa2cd014ae186c1659bfc722bea",
            "09c0a27d936d4d10b8d81a7729f9bf88",
            "f65cb838e6504f859d3e2ce60f5b3aab",
            "3a19356bf5c949d4baffd7fd43231a7a",
            "79b8e75a31bc43f29d2fe260b96ba88f",
            "0ea55c9248154dd496666754b42b2e3a",
            "238f757c32604a02a963aeaf7e7df55f",
            "2e81c1b7723f44d0a22b54d99b60a6ff",
            "23b2395e2a0e4b83b16ce862ace73937"
          ]
        },
        "id": "rkaoluxFIhfO",
        "outputId": "3318273b-1029-4268-817d-6e472d7a80a3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Saving the dataset (0/1 shards):   0%|          | 0/87599 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "433e2e7105d2489d9f2259f2ba302170"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Saving the dataset (0/1 shards):   0%|          | 0/10570 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "3e02f28e34c546f9b0bb363423c7e550"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# import pickle\n",
        "# with open('entity_map_qa.pkl', 'wb') as fichier:\n",
        "#     pickle.dump(entity_mapping, fichier)\n",
        "\n",
        "# print(\"Save with succes.\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0808f81e-f5d8-44a7-c61b-f835ca61002a",
        "id": "DMtutFUhI4Bd"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Save with succes.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "sZ7gNcJkp5fc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "cKJF1wCUp5d5"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "K_DXGCmyp5cb"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import T5Tokenizer, T5ForConditionalGeneration, Trainer, TrainingArguments, DataCollatorForSeq2Seq\n",
        "from datasets import load_dataset, Dataset\n",
        "from datasets import load_dataset\n",
        "\n",
        "dataset = load_dataset(\"squad\")\n",
        "dataset"
      ],
      "metadata": {
        "id": "cfffnGNRw9Aa",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c4354001-44e9-43ec-9940-3e75b696c611"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "DatasetDict({\n",
              "    train: Dataset({\n",
              "        features: ['id', 'title', 'context', 'question', 'answers'],\n",
              "        num_rows: 87599\n",
              "    })\n",
              "    validation: Dataset({\n",
              "        features: ['id', 'title', 'context', 'question', 'answers'],\n",
              "        num_rows: 10570\n",
              "    })\n",
              "})"
            ]
          },
          "metadata": {},
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from flair.data import Sentence\n",
        "from flair.models import SequenceTagger\n",
        "from datasets import load_dataset\n",
        "from collections import defaultdict\n",
        "\n",
        "tagger = SequenceTagger.load('flair/ner-english-ontonotes-large')\n",
        "included_tags = {\"EVENT\", \"FAC\", \"GPE\", \"LANGUAGE\", \"LAW\", \"LOCATION\", \"NORP\",\n",
        "                 \"ORG\", \"PERSON\", \"PRODUCT\", \"WORK_OF_ART\"}\n",
        "\n"
      ],
      "metadata": {
        "collapsed": true,
        "id": "mu1UzEMEvMz0",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "57f34fd0-720f-4bb2-dca3-1949847c6276"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-04 08:40:57,695 SequenceTagger predicts: Dictionary with 76 tags: <unk>, O, B-CARDINAL, E-CARDINAL, S-PERSON, S-CARDINAL, S-PRODUCT, B-PRODUCT, I-PRODUCT, E-PRODUCT, B-WORK_OF_ART, I-WORK_OF_ART, E-WORK_OF_ART, B-PERSON, E-PERSON, S-GPE, B-DATE, I-DATE, E-DATE, S-ORDINAL, S-LANGUAGE, I-PERSON, S-EVENT, S-DATE, B-QUANTITY, E-QUANTITY, S-TIME, B-TIME, I-TIME, E-TIME, B-GPE, E-GPE, S-ORG, I-GPE, S-NORP, B-FAC, I-FAC, E-FAC, B-NORP, E-NORP, S-PERCENT, B-ORG, E-ORG, B-LANGUAGE, E-LANGUAGE, I-CARDINAL, I-ORG, S-WORK_OF_ART, I-QUANTITY, B-MONEY\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "nzhi0UiaWbPW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "5WduH4nDwxis"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "F2arWQcMwxgD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "UBb_hNxbwxdo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from flair.data import Sentence\n",
        "from flair.models import SequenceTagger\n",
        "from flair.splitter import SegtokSentenceSplitter\n",
        "from datasets import load_dataset\n",
        "import random\n",
        "from collections import defaultdict\n",
        "\n",
        "# Load the SQuAD dataset\n",
        "dataset = load_dataset(\"squad\")\n",
        "\n",
        "# Load the NER model\n",
        "tagger = SequenceTagger.load('flair/ner-english-ontonotes-large')\n",
        "\n",
        "# Define the included tags for anonymization\n",
        "included_tags = {\"EVENT\", \"FAC\", \"GPE\", \"LANGUAGE\", \"LAW\", \"LOCATION\", \"NORP\", \"ORG\", \"PERSON\", \"PRODUCT\", \"WORK_OF_ART\"}\n",
        "\n",
        "# Define a sentence splitter to process the text sentence by sentence\n",
        "splitter = SegtokSentenceSplitter()\n"
      ],
      "metadata": {
        "id": "6oOqW7JkM6GR",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1ee98660-a68f-45f5-d0a1-6aca818d0eab"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:89: UserWarning: \n",
            "The secret `HF_TOKEN` does not exist in your Colab secrets.\n",
            "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n",
            "You will be able to reuse this secret in all of your notebooks.\n",
            "Please note that authentication is recommended but still optional to access public models or datasets.\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 05:32:13,211 SequenceTagger predicts: Dictionary with 76 tags: <unk>, O, B-CARDINAL, E-CARDINAL, S-PERSON, S-CARDINAL, S-PRODUCT, B-PRODUCT, I-PRODUCT, E-PRODUCT, B-WORK_OF_ART, I-WORK_OF_ART, E-WORK_OF_ART, B-PERSON, E-PERSON, S-GPE, B-DATE, I-DATE, E-DATE, S-ORDINAL, S-LANGUAGE, I-PERSON, S-EVENT, S-DATE, B-QUANTITY, E-QUANTITY, S-TIME, B-TIME, I-TIME, E-TIME, B-GPE, E-GPE, S-ORG, I-GPE, S-NORP, B-FAC, I-FAC, E-FAC, B-NORP, E-NORP, S-PERCENT, B-ORG, E-ORG, B-LANGUAGE, E-LANGUAGE, I-CARDINAL, I-ORG, S-WORK_OF_ART, I-QUANTITY, B-MONEY\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Function to anonymize text and track entity positions with consistent mapping\n",
        "def anonymize_text(text, tagger, entity_mapping, used_ids):\n",
        "    sentences = splitter.split(text)  # Split the text into sentences\n",
        "    anonymized_text = text\n",
        "    shift = 0  # To track how much the text length changes\n",
        "    replacements = []\n",
        "\n",
        "    for sentence in sentences:\n",
        "        sentence_obj = Sentence(sentence.to_plain_string())\n",
        "        tagger.predict(sentence_obj)\n",
        "\n",
        "        # Process each entity within the sentence\n",
        "        for entity in sentence_obj.get_spans('ner'):\n",
        "            if entity.tag in included_tags:\n",
        "                original_entity = entity.text\n",
        "\n",
        "                # If the entity has been seen before, use the same ID, otherwise generate a new one\n",
        "                if original_entity in entity_mapping:\n",
        "                    anon_id = entity_mapping[original_entity]\n",
        "                else:\n",
        "                    while True:\n",
        "                        anon_id = f\"ent{random.randint(1, 90000)}\"\n",
        "                        if anon_id not in used_ids:\n",
        "                            break\n",
        "                    entity_mapping[original_entity] = anon_id\n",
        "                    used_ids.add(anon_id)\n",
        "\n",
        "                # Track replacements with start and end positions\n",
        "                start_idx = anonymized_text.find(original_entity, shift)  # find after shift\n",
        "                if start_idx != -1:\n",
        "                    end_idx = start_idx + len(original_entity)\n",
        "                    replacements.append((start_idx, end_idx, anon_id))\n",
        "                    shift = end_idx  # Update shift to not reprocess the same text\n",
        "\n",
        "    # Apply replacements in reverse order (to keep indices consistent)\n",
        "    for start_idx, end_idx, anon_id in sorted(replacements, reverse=True):\n",
        "        anonymized_text = anonymized_text[:start_idx] + anon_id + anonymized_text[end_idx:]\n",
        "\n",
        "    return anonymized_text, replacements\n",
        "\n",
        "# Function to adjust answer positions after anonymization\n",
        "def adjust_answer_positions(example, replacements):\n",
        "    for i, answer_start in enumerate(example['answers']['answer_start']):\n",
        "        for start_idx, end_idx, anon_id in replacements:\n",
        "            # If the answer starts inside the entity span, set it to the start of the anonymized entity\n",
        "            if start_idx <= answer_start < end_idx:\n",
        "                example['answers']['answer_start'][i] = start_idx\n",
        "            # Adjust positions if the answer is after the replaced entity\n",
        "            elif answer_start >= end_idx:\n",
        "                shift = len(anon_id) - (end_idx - start_idx)\n",
        "                example['answers']['answer_start'][i] += shift\n",
        "\n",
        "    return example\n",
        "\n",
        "# Function to anonymize an answer\n",
        "def anonymize_answer(answer_text, tagger, entity_mapping, used_ids):\n",
        "    anonymized_answer, _ = anonymize_text(answer_text, tagger, entity_mapping, used_ids)\n",
        "    return anonymized_answer\n",
        "\n",
        "# Function to anonymize the entire dataset example (context, question, and answers)\n",
        "def anonymize_example(example, tagger, entity_mapping, used_ids):\n",
        "    # Anonymize context and get replacements\n",
        "    article_text = example['context']\n",
        "    anonymized_article, context_replacements = anonymize_text(article_text, tagger, entity_mapping, used_ids)\n",
        "\n",
        "    # Adjust answer positions based on context replacements\n",
        "    example = adjust_answer_positions(example, context_replacements)\n",
        "\n",
        "    # Anonymize each answer\n",
        "    for i, answer_text in enumerate(example['answers']['text']):\n",
        "        anonymized_answer = anonymize_answer(answer_text, tagger, entity_mapping, used_ids)\n",
        "        example['answers']['text'][i] = anonymized_answer\n",
        "\n",
        "    # Anonymize question\n",
        "    question_text = example['question']\n",
        "    anonymized_question, _ = anonymize_text(question_text, tagger, entity_mapping, used_ids)\n",
        "\n",
        "    # Update the example with anonymized context and question\n",
        "    example['context'] = anonymized_article\n",
        "    example['question'] = anonymized_question\n",
        "\n",
        "    return example\n"
      ],
      "metadata": {
        "id": "2LX0DU0q4-_l"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import random\n",
        "from flair.data import Sentence\n",
        "from tqdm import tqdm\n",
        "\n",
        "# Function to anonymize text and track entity positions with consistent mapping\n",
        "def anonymize_text(text, tagger, entity_mapping, used_ids):\n",
        "    sentences = text.split('. ')  # Split the text into sentences\n",
        "    anonymized_text = text\n",
        "    shift = 0  # To track how much the text length changes\n",
        "    replacements = []\n",
        "\n",
        "    for sentence in sentences:\n",
        "        sentence_obj = Sentence(sentence)\n",
        "        tagger.predict(sentence_obj)\n",
        "\n",
        "        # Process each entity within the sentence\n",
        "        for entity in sentence_obj.get_spans('ner'):\n",
        "            if entity.tag in included_tags:\n",
        "                original_entity = entity.text\n",
        "\n",
        "                # If the entity has been seen before, use the same ID, otherwise generate a new one\n",
        "                if original_entity in entity_mapping:\n",
        "                    anon_id = entity_mapping[original_entity]\n",
        "                else:\n",
        "                    while True:\n",
        "                        anon_id = f\"ent{random.randint(1, 300000)}\"\n",
        "                        if anon_id not in used_ids:\n",
        "                            break\n",
        "                    entity_mapping[original_entity] = anon_id\n",
        "                    used_ids.add(anon_id)\n",
        "\n",
        "                # Track replacements with start and end positions\n",
        "                start_idx = anonymized_text.find(original_entity, shift)  # find after shift\n",
        "                if start_idx != -1:\n",
        "                    end_idx = start_idx + len(original_entity)\n",
        "                    replacements.append((start_idx, end_idx, anon_id))\n",
        "                    shift = end_idx  # Update shift to not reprocess the same text\n",
        "\n",
        "    # Apply replacements in reverse order (to keep indices consistent)\n",
        "    for start_idx, end_idx, anon_id in sorted(replacements, reverse=True):\n",
        "        anonymized_text = anonymized_text[:start_idx] + anon_id + anonymized_text[end_idx:]\n",
        "\n",
        "    return anonymized_text, replacements\n",
        "\n",
        "# Function to adjust answer positions after anonymization\n",
        "def adjust_answer_positions(example, replacements):\n",
        "    answer_starts = example['answers']['answer_start']\n",
        "\n",
        "    # Loop over each answer's start position\n",
        "    for i, answer_start in enumerate(answer_starts):\n",
        "        shift = 0  # This tracks the overall shift for each replacement\n",
        "\n",
        "        # Loop over the replacements and adjust positions\n",
        "        for start_idx, end_idx, anon_id in replacements:\n",
        "            original_entity_len = end_idx - start_idx\n",
        "            anon_entity_len = len(anon_id)\n",
        "\n",
        "            # If the answer start is within the replaced entity, shift to the start of the anonymized entity\n",
        "            if start_idx <= answer_start < end_idx:\n",
        "                example['answers']['answer_start'][i] = start_idx\n",
        "                break\n",
        "            # If the answer start is after the replaced entity, adjust for the shift caused by the length difference\n",
        "            elif answer_start >= end_idx:\n",
        "                shift += anon_entity_len - original_entity_len\n",
        "\n",
        "        # Apply the accumulated shift\n",
        "        example['answers']['answer_start'][i] += shift\n",
        "\n",
        "    return example\n",
        "\n",
        "# Function to anonymize the question and answers using the entity mapping from the context\n",
        "def anonymize_question_and_answers(question_text, answers_text, tagger, entity_mapping, used_ids):\n",
        "    # Anonymize the question using the context-based mapping\n",
        "    anonymized_question, _ = anonymize_text(question_text, tagger, entity_mapping, used_ids)\n",
        "\n",
        "    # Anonymize each answer using the context-based mapping\n",
        "    anonymized_answers = []\n",
        "    for answer_text in answers_text:\n",
        "        anonymized_answer, _ = anonymize_text(answer_text, tagger, entity_mapping, used_ids)\n",
        "        anonymized_answers.append(anonymized_answer)\n",
        "\n",
        "    return anonymized_question, anonymized_answers\n",
        "\n",
        "# Function to anonymize the entire dataset example (context, question, and answers)\n",
        "# and return the entity mapping as well\n",
        "def anonymize_example_with_mapping(example, tagger):\n",
        "    entity_mapping = {}  # Keep track of entity to anonymized ID mapping within one example\n",
        "    used_ids = set()     # Keep track of all used anonymized IDs within the example\n",
        "\n",
        "    # Step 1: Anonymize context and get replacements\n",
        "    article_text = example['context']\n",
        "    anonymized_article, context_replacements = anonymize_text(article_text, tagger, entity_mapping, used_ids)\n",
        "\n",
        "    # Step 2: Adjust answer positions based on context replacements\n",
        "    example = adjust_answer_positions(example, context_replacements)\n",
        "\n",
        "    # Step 3: Anonymize question and answers based on the context mapping\n",
        "    question_text = example['question']\n",
        "    answers_text = example['answers']['text']\n",
        "    anonymized_question, anonymized_answers = anonymize_question_and_answers(question_text, answers_text, tagger, entity_mapping, used_ids)\n",
        "\n",
        "    # Step 4: Update the example with anonymized context, question, and answers\n",
        "    example['context'] = anonymized_article\n",
        "    example['question'] = anonymized_question\n",
        "    example['answers']['text'] = anonymized_answers\n",
        "\n",
        "    # Return the anonymized example and the entity mapping\n",
        "    return example, entity_mapping\n",
        "\n",
        "# Function to anonymize a dataset (list of examples) and return all entity mappings\n",
        "def anonymize_dataset(dataset, tagger):\n",
        "    all_mappings = []  # List to store mappings for each example\n",
        "    anonymized_dataset = []  # Store anonymized examples\n",
        "\n",
        "    # Process each example in the dataset\n",
        "    for example in tqdm(dataset):\n",
        "        anonymized_example, entity_mapping = anonymize_example_with_mapping(example, tagger)\n",
        "        anonymized_dataset.append(anonymized_example)\n",
        "        all_mappings.append(entity_mapping)\n",
        "\n",
        "    return anonymized_dataset, all_mappings\n"
      ],
      "metadata": {
        "id": "bNAp3mfOI6VO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# import pickle\n",
        "# with open('all_mappings1.pkl', 'wb') as fichier:\n",
        "#     pickle.dump(all_mappings1, fichier)\n",
        "\n",
        "# print(\"Save with succes.\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "67OURmicedxQ",
        "outputId": "098ffe29-d9ea-4378-e20b-0ff719c4b2af"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Save with succes.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# import pickle\n",
        "# with open('all_mappings2.pkl', 'wb') as fichier:\n",
        "#     pickle.dump(all_mappings2, fichier)\n",
        "\n",
        "# print(\"Save with succes.\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3iDAlt8ieiRP",
        "outputId": "347e84a5-5e20-4e2a-f7b1-2d106401cafe"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Save with succes.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pickle\n",
        "with open('entity_map_qa.pkl', 'rb') as file:\n",
        "    entity_mapping = pickle.load(file)\n",
        "len(entity_mapping)"
      ],
      "metadata": {
        "id": "ZlHJcWAgeEiv"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Assuming you have a dataset and a NER tagger already set up\n",
        "anonymized_train, all_mappings1 = anonymize_dataset(dataset['train'], tagger)"
      ],
      "metadata": {
        "id": "5tNYISFvI8ZI",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2c21ae2a-3b69-4de8-f00e-1e5ffd20922f",
        "collapsed": true
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 13%|█▎        | 11200/87599 [27:07<2:04:19, 10.24it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 06:02:07,378 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n",
            "2024-10-05 06:02:07,469 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 13%|█▎        | 11202/87599 [27:07<2:02:04, 10.43it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 06:02:07,564 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 15%|█▍        | 12759/87599 [30:39<1:57:04, 10.65it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 06:05:38,762 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 15%|█▌        | 13148/87599 [31:24<1:54:02, 10.88it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 06:06:24,735 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 15%|█▌        | 13150/87599 [31:25<1:50:42, 11.21it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 06:06:24,823 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n",
            "2024-10-05 06:06:24,909 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 15%|█▌        | 13152/87599 [31:25<1:49:46, 11.30it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 06:06:24,997 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 15%|█▌        | 13182/87599 [31:28<2:09:36,  9.57it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 06:06:28,017 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 15%|█▌        | 13184/87599 [31:28<2:06:25,  9.81it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 06:06:28,129 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n",
            "2024-10-05 06:06:28,239 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 15%|█▌        | 13186/87599 [31:28<2:09:36,  9.57it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 06:06:28,348 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 15%|█▌        | 13187/87599 [31:28<2:10:48,  9.48it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 06:06:28,459 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 15%|█▌        | 13196/87599 [31:29<2:12:49,  9.34it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 06:06:29,368 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n",
            "2024-10-05 06:06:29,450 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 15%|█▌        | 13198/87599 [31:29<2:05:19,  9.89it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 06:06:29,540 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n",
            "2024-10-05 06:06:29,630 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 15%|█▌        | 13200/87599 [31:29<2:00:31, 10.29it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 06:06:29,716 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 18%|█▊        | 15393/87599 [36:08<2:03:16,  9.76it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 06:11:08,493 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n",
            "2024-10-05 06:11:08,573 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 18%|█▊        | 15395/87599 [36:08<1:53:45, 10.58it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 06:11:08,652 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n",
            "2024-10-05 06:11:08,729 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 18%|█▊        | 15397/87599 [36:09<1:46:59, 11.25it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 06:11:08,807 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 18%|█▊        | 15662/87599 [36:40<1:30:02, 13.32it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 06:11:40,001 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 18%|█▊        | 15664/87599 [36:40<1:33:34, 12.81it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 06:11:40,107 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 20%|██        | 17611/87599 [40:33<1:26:53, 13.43it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 06:15:33,539 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n",
            "2024-10-05 06:15:33,622 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 20%|██        | 17613/87599 [40:33<1:30:47, 12.85it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 06:15:33,710 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 21%|██        | 18144/87599 [41:34<1:55:26, 10.03it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 06:16:34,039 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 21%|██        | 18146/87599 [41:34<1:58:47,  9.74it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 06:16:34,171 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 21%|██        | 18147/87599 [41:34<2:04:47,  9.28it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 06:16:34,302 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 21%|██        | 18148/87599 [41:34<2:11:14,  8.82it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 06:16:34,437 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 21%|██        | 18225/87599 [41:43<2:00:26,  9.60it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 06:16:43,521 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n",
            "2024-10-05 06:16:43,726 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 21%|██        | 18227/87599 [41:44<2:34:43,  7.47it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 06:16:43,930 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 21%|██        | 18228/87599 [41:44<2:48:38,  6.86it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 06:16:44,129 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 21%|██        | 18229/87599 [41:44<3:01:18,  6.38it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 06:16:44,332 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 23%|██▎       | 20214/87599 [45:58<1:44:10, 10.78it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 06:20:58,235 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 23%|██▎       | 20218/87599 [45:58<1:42:19, 10.97it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 06:20:58,693 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 23%|██▎       | 20222/87599 [45:59<1:49:29, 10.26it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 06:20:59,114 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 23%|██▎       | 20320/87599 [46:10<1:47:30, 10.43it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 06:21:10,487 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 26%|██▌       | 22584/87599 [51:01<2:05:15,  8.65it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 06:26:00,913 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 30%|██▉       | 26109/87599 [59:17<3:29:44,  4.89it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 06:34:16,783 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n",
            "2024-10-05 06:34:16,784 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 30%|██▉       | 26110/87599 [59:17<3:24:12,  5.02it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 06:34:16,970 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n",
            "2024-10-05 06:34:16,971 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 30%|██▉       | 26111/87599 [59:17<3:22:51,  5.05it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 06:34:17,166 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n",
            "2024-10-05 06:34:17,167 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 30%|██▉       | 26112/87599 [59:17<3:22:52,  5.05it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 06:34:17,367 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n",
            "2024-10-05 06:34:17,368 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 34%|███▍      | 30162/87599 [1:10:17<2:09:58,  7.37it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 06:45:17,414 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 34%|███▍      | 30163/87599 [1:10:17<2:27:13,  6.50it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 06:45:17,616 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 34%|███▍      | 30164/87599 [1:10:17<2:41:18,  5.93it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 06:45:17,830 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 34%|███▍      | 30165/87599 [1:10:18<2:53:10,  5.53it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 06:45:18,031 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 39%|███▉      | 34008/87599 [1:20:35<2:24:40,  6.17it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 06:55:34,863 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 39%|███▉      | 34009/87599 [1:20:35<2:22:18,  6.28it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 06:55:35,015 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 39%|███▉      | 34010/87599 [1:20:35<2:19:11,  6.42it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 06:55:35,164 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 39%|███▉      | 34011/87599 [1:20:35<2:17:39,  6.49it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 06:55:35,310 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 39%|███▉      | 34571/87599 [1:22:00<1:29:37,  9.86it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 06:57:00,046 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 40%|███▉      | 34645/87599 [1:22:12<2:23:28,  6.15it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 06:57:12,003 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 40%|███▉      | 34775/87599 [1:22:32<1:45:07,  8.37it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 06:57:31,898 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 41%|████      | 35653/87599 [1:24:48<2:36:42,  5.52it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 06:59:48,057 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 41%|████      | 35654/87599 [1:24:48<2:34:25,  5.61it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 06:59:48,230 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 41%|████      | 35655/87599 [1:24:48<2:32:43,  5.67it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 06:59:48,399 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 41%|████      | 35656/87599 [1:24:48<2:31:53,  5.70it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 06:59:48,571 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 41%|████      | 35657/87599 [1:24:48<2:29:35,  5.79it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 06:59:48,734 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 41%|████      | 35658/87599 [1:24:49<2:27:46,  5.86it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 06:59:48,898 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 41%|████      | 35659/87599 [1:24:49<2:25:26,  5.95it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 06:59:49,063 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 41%|████      | 35660/87599 [1:24:49<2:24:57,  5.97it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 06:59:49,231 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 41%|████      | 36081/87599 [1:26:04<2:28:46,  5.77it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 07:01:03,936 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 41%|████▏     | 36157/87599 [1:26:17<2:19:42,  6.14it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 07:01:17,279 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 41%|████▏     | 36161/87599 [1:26:18<2:13:42,  6.41it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 07:01:17,903 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 41%|████▏     | 36212/87599 [1:26:28<2:15:48,  6.31it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 07:01:27,899 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 41%|████▏     | 36257/87599 [1:26:34<2:38:45,  5.39it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 07:01:34,529 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 44%|████▍     | 38466/87599 [1:32:45<2:08:06,  6.39it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 07:07:45,485 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 44%|████▍     | 38536/87599 [1:32:56<1:39:49,  8.19it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 07:07:56,768 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 44%|████▍     | 38546/87599 [1:32:58<2:03:50,  6.60it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 07:07:58,391 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 44%|████▍     | 38676/87599 [1:33:22<2:17:15,  5.94it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 07:08:22,669 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 44%|████▍     | 38704/87599 [1:33:28<2:25:23,  5.60it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 07:08:28,254 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 44%|████▍     | 38705/87599 [1:33:28<2:33:52,  5.30it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 07:08:28,461 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 46%|████▌     | 40067/87599 [1:37:22<2:18:38,  5.71it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 07:12:22,319 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 46%|████▌     | 40068/87599 [1:37:22<2:31:37,  5.22it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 07:12:22,554 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 46%|████▌     | 40069/87599 [1:37:22<2:40:58,  4.92it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 07:12:22,787 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 46%|████▌     | 40070/87599 [1:37:23<2:47:18,  4.73it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 07:12:23,008 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 46%|████▌     | 40071/87599 [1:37:23<2:49:43,  4.67it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 07:12:23,236 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 46%|████▌     | 40081/87599 [1:37:25<2:27:46,  5.36it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 07:12:25,203 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 46%|████▌     | 40082/87599 [1:37:25<2:25:11,  5.45it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 07:12:25,386 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 46%|████▌     | 40083/87599 [1:37:25<2:26:00,  5.42it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 07:12:25,572 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 46%|████▌     | 40084/87599 [1:37:25<2:26:24,  5.41it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 07:12:25,758 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 46%|████▌     | 40141/87599 [1:37:35<1:45:22,  7.51it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 07:12:35,069 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 47%|████▋     | 41274/87599 [1:40:44<2:21:57,  5.44it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 07:15:44,396 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 47%|████▋     | 41275/87599 [1:40:44<2:31:42,  5.09it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 07:15:44,625 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 47%|████▋     | 41276/87599 [1:40:44<2:37:22,  4.91it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 07:15:44,837 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 47%|████▋     | 41282/87599 [1:40:45<1:38:16,  7.85it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 07:15:45,645 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n",
            "2024-10-05 07:15:45,647 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 47%|████▋     | 41284/87599 [1:40:46<1:33:49,  8.23it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 07:15:45,793 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n",
            "2024-10-05 07:15:45,794 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 47%|████▋     | 41285/87599 [1:40:46<1:37:41,  7.90it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 07:15:45,939 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n",
            "2024-10-05 07:15:45,940 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 48%|████▊     | 41737/87599 [1:41:59<2:50:49,  4.47it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 07:16:59,039 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 52%|█████▏    | 45725/87599 [1:52:25<2:36:12,  4.47it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 07:27:25,812 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 53%|█████▎    | 46412/87599 [1:54:12<2:18:41,  4.95it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 07:29:12,690 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n",
            "2024-10-05 07:29:12,712 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 53%|█████▎    | 46413/87599 [1:54:13<2:19:58,  4.90it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 07:29:12,900 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n",
            "2024-10-05 07:29:12,922 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 53%|█████▎    | 46414/87599 [1:54:13<2:21:08,  4.86it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 07:29:13,112 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n",
            "2024-10-05 07:29:13,135 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 53%|█████▎    | 46415/87599 [1:54:13<2:23:49,  4.77it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 07:29:13,328 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n",
            "2024-10-05 07:29:13,351 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 53%|█████▎    | 46416/87599 [1:54:13<2:24:56,  4.74it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 07:29:13,543 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n",
            "2024-10-05 07:29:13,565 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 54%|█████▍    | 47418/87599 [1:56:58<1:18:53,  8.49it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 07:31:57,876 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 54%|█████▍    | 47573/87599 [1:57:19<1:23:31,  7.99it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 07:32:19,573 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 54%|█████▍    | 47680/87599 [1:57:34<1:51:11,  5.98it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 07:32:34,528 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 55%|█████▌    | 48373/87599 [1:59:20<1:51:05,  5.88it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 07:34:20,655 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 55%|█████▌    | 48374/87599 [1:59:21<1:50:25,  5.92it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 07:34:20,824 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 55%|█████▌    | 48375/87599 [1:59:21<1:50:56,  5.89it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 07:34:20,997 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 55%|█████▌    | 48376/87599 [1:59:21<1:50:43,  5.90it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 07:34:21,164 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 55%|█████▌    | 48377/87599 [1:59:21<1:50:25,  5.92it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 07:34:21,330 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 56%|█████▌    | 48684/87599 [2:00:12<1:30:40,  7.15it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 07:35:12,544 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 61%|██████    | 53105/87599 [2:12:06<1:27:37,  6.56it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 07:47:06,269 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 61%|██████    | 53106/87599 [2:12:06<1:35:22,  6.03it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 07:47:06,467 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 61%|██████    | 53107/87599 [2:12:06<1:40:21,  5.73it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 07:47:06,662 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 61%|██████    | 53108/87599 [2:12:07<1:43:10,  5.57it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 07:47:06,852 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 61%|██████    | 53109/87599 [2:12:07<1:46:40,  5.39it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 07:47:07,052 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 63%|██████▎   | 54997/87599 [2:17:16<1:44:07,  5.22it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 07:52:16,756 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 63%|██████▎   | 54998/87599 [2:17:17<1:46:02,  5.12it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 07:52:16,955 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 63%|██████▎   | 54999/87599 [2:17:17<1:45:45,  5.14it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 07:52:17,148 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 63%|██████▎   | 55000/87599 [2:17:17<1:46:03,  5.12it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 07:52:17,346 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 63%|██████▎   | 55001/87599 [2:17:17<1:46:22,  5.11it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 07:52:17,540 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 63%|██████▎   | 55002/87599 [2:17:17<1:46:33,  5.10it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 07:52:17,756 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n",
            "2024-10-05 07:52:17,757 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 63%|██████▎   | 55003/87599 [2:17:18<1:39:37,  5.45it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 07:52:17,918 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n",
            "2024-10-05 07:52:17,920 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 63%|██████▎   | 55004/87599 [2:17:18<1:36:02,  5.66it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 07:52:18,072 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n",
            "2024-10-05 07:52:18,073 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 63%|██████▎   | 55005/87599 [2:17:18<1:32:36,  5.87it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 07:52:18,229 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n",
            "2024-10-05 07:52:18,230 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 63%|██████▎   | 55006/87599 [2:17:18<1:30:37,  5.99it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 07:52:18,387 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n",
            "2024-10-05 07:52:18,388 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 63%|██████▎   | 55178/87599 [2:17:49<1:20:49,  6.69it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 07:52:49,090 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 63%|██████▎   | 55209/87599 [2:17:53<1:03:05,  8.56it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 07:52:53,523 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 63%|██████▎   | 55233/87599 [2:17:57<1:13:42,  7.32it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 07:52:57,239 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 63%|██████▎   | 55246/87599 [2:17:59<1:53:02,  4.77it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 07:52:59,503 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 63%|██████▎   | 55275/87599 [2:18:03<1:12:09,  7.47it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 07:53:03,786 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 66%|██████▋   | 58057/87599 [2:25:18<1:20:39,  6.10it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 08:00:18,434 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 67%|██████▋   | 58259/87599 [2:25:51<2:16:31,  3.58it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 08:00:51,628 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 67%|██████▋   | 58260/87599 [2:25:51<1:52:48,  4.33it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 08:00:51,747 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 67%|██████▋   | 58261/87599 [2:25:52<1:35:42,  5.11it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 08:00:51,858 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 67%|██████▋   | 58262/87599 [2:25:52<1:23:05,  5.88it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 08:00:51,968 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 68%|██████▊   | 59898/87599 [2:30:00<1:11:10,  6.49it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 08:04:59,929 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 68%|██████▊   | 59899/87599 [2:30:00<1:10:00,  6.59it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 08:05:00,071 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 68%|██████▊   | 59900/87599 [2:30:00<1:11:24,  6.46it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 08:05:00,236 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 68%|██████▊   | 59901/87599 [2:30:00<1:10:25,  6.56it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 08:05:00,385 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 68%|██████▊   | 59902/87599 [2:30:00<1:09:53,  6.60it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 08:05:00,531 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 70%|███████   | 61727/87599 [2:34:43<1:00:57,  7.07it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 08:09:43,361 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 70%|███████   | 61728/87599 [2:34:43<1:04:59,  6.63it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 08:09:43,530 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 70%|███████   | 61729/87599 [2:34:43<1:07:34,  6.38it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 08:09:43,701 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 70%|███████   | 61730/87599 [2:34:44<1:09:19,  6.22it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 08:09:43,875 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 70%|███████   | 61731/87599 [2:34:44<1:11:28,  6.03it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 08:09:44,046 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 72%|███████▏  | 62867/87599 [2:37:39<1:06:51,  6.17it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 08:12:39,760 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 75%|███████▍  | 65557/87599 [2:44:43<48:26,  7.58it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 08:19:43,360 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 75%|███████▍  | 65558/87599 [2:44:43<59:30,  6.17it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 08:19:43,591 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 75%|███████▍  | 65559/87599 [2:44:43<1:07:13,  5.46it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 08:19:43,831 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 75%|███████▍  | 65560/87599 [2:44:44<1:13:27,  5.00it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 08:19:44,069 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 75%|███████▍  | 65587/87599 [2:44:48<1:01:00,  6.01it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 08:19:48,510 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 75%|███████▍  | 65588/87599 [2:44:48<54:46,  6.70it/s]  "
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 08:19:48,618 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 75%|███████▍  | 65589/87599 [2:44:48<50:17,  7.29it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 08:19:48,726 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 78%|███████▊  | 68323/87599 [2:51:52<56:18,  5.71it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 08:26:52,682 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 79%|███████▉  | 69208/87599 [2:54:23<56:04,  5.47it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 08:29:23,191 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 79%|███████▉  | 69473/87599 [2:55:05<39:15,  7.70it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 08:30:05,464 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n",
            "2024-10-05 08:30:05,465 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 79%|███████▉  | 69474/87599 [2:55:05<48:01,  6.29it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 08:30:05,687 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n",
            "2024-10-05 08:30:05,688 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 79%|███████▉  | 69475/87599 [2:55:06<54:00,  5.59it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 08:30:05,912 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n",
            "2024-10-05 08:30:05,913 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 79%|███████▉  | 69476/87599 [2:55:06<58:00,  5.21it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 08:30:06,133 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n",
            "2024-10-05 08:30:06,134 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 79%|███████▉  | 69477/87599 [2:55:06<1:00:42,  4.98it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 08:30:06,355 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n",
            "2024-10-05 08:30:06,356 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 80%|████████  | 70214/87599 [2:56:59<57:46,  5.01it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 08:31:59,169 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 80%|████████  | 70365/87599 [2:57:24<45:56,  6.25it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 08:32:24,646 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 80%|████████  | 70369/87599 [2:57:25<56:19,  5.10it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 08:32:25,473 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 80%|████████  | 70372/87599 [2:57:26<59:18,  4.84it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 08:32:26,107 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 83%|████████▎ | 72289/87599 [3:02:38<33:39,  7.58it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 08:37:38,302 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 83%|████████▎ | 72914/87599 [3:04:15<28:35,  8.56it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 08:39:15,337 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 83%|████████▎ | 72915/87599 [3:04:15<31:59,  7.65it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 08:39:15,507 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 83%|████████▎ | 72916/87599 [3:04:15<34:55,  7.01it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 08:39:15,675 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 83%|████████▎ | 72917/87599 [3:04:16<36:45,  6.66it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 08:39:15,842 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 83%|████████▎ | 72918/87599 [3:04:16<38:08,  6.42it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 08:39:16,015 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 84%|████████▍ | 73836/87599 [3:06:55<30:52,  7.43it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 08:41:55,139 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 84%|████████▍ | 73929/87599 [3:07:12<25:12,  9.04it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 08:42:12,011 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 85%|████████▍ | 74320/87599 [3:08:15<32:42,  6.77it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 08:43:14,988 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 86%|████████▌ | 74899/87599 [3:09:43<35:16,  6.00it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 08:44:43,423 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 87%|████████▋ | 75993/87599 [3:12:40<34:55,  5.54it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 08:47:40,079 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 93%|█████████▎| 81393/87599 [3:27:04<15:47,  6.55it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:02:04,303 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 94%|█████████▍| 82614/87599 [3:30:22<10:52,  7.63it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:05:22,307 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 94%|█████████▍| 82615/87599 [3:30:22<11:29,  7.23it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:05:22,460 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 94%|█████████▍| 82616/87599 [3:30:22<11:47,  7.04it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:05:22,613 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 94%|█████████▍| 82617/87599 [3:30:22<11:59,  6.92it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:05:22,763 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 94%|█████████▍| 82618/87599 [3:30:23<12:11,  6.81it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:05:22,918 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 94%|█████████▍| 82633/87599 [3:30:26<14:21,  5.76it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:05:25,869 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 94%|█████████▍| 82634/87599 [3:30:26<14:47,  5.60it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:05:26,058 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 94%|█████████▍| 82635/87599 [3:30:26<14:59,  5.52it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:05:26,253 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 94%|█████████▍| 82636/87599 [3:30:26<15:24,  5.37it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:05:26,449 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 95%|█████████▌| 83420/87599 [3:32:28<13:55,  5.00it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:07:28,071 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|█████████▉| 87343/87599 [3:42:41<00:46,  5.46it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:17:41,269 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r100%|█████████▉| 87344/87599 [3:42:41<00:45,  5.57it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:17:41,442 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r100%|█████████▉| 87345/87599 [3:42:41<00:44,  5.65it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:17:41,609 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r100%|█████████▉| 87346/87599 [3:42:41<00:44,  5.70it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:17:41,791 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r100%|█████████▉| 87347/87599 [3:42:42<00:44,  5.67it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:17:41,963 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 87599/87599 [3:43:26<00:00,  6.53it/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Assuming you have a dataset and a NER tagger already set up\n",
        "anonymized_val, all_mappings2 = anonymize_dataset(dataset['validation'], tagger)"
      ],
      "metadata": {
        "id": "b6yXUszATVEP",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5110ede7-ecaa-4163-fbec-432a43dde217",
        "collapsed": true
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "  6%|▌         | 621/10570 [01:45<20:30,  8.08it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:20:13,847 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 21%|██▏       | 2265/10570 [06:44<26:30,  5.22it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:25:12,631 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 47%|████▋     | 4961/10570 [15:59<22:55,  4.08it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:34:27,386 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n",
            "2024-10-05 09:34:27,387 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 47%|████▋     | 4962/10570 [15:59<21:58,  4.25it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:34:27,604 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n",
            "2024-10-05 09:34:27,605 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 47%|████▋     | 4963/10570 [15:59<21:33,  4.33it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:34:27,819 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n",
            "2024-10-05 09:34:27,822 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 47%|████▋     | 4964/10570 [15:59<21:04,  4.43it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:34:28,033 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n",
            "2024-10-05 09:34:28,034 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 47%|████▋     | 4965/10570 [16:00<21:15,  4.39it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:34:28,268 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n",
            "2024-10-05 09:34:28,269 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 48%|████▊     | 5068/10570 [16:23<21:07,  4.34it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:34:52,166 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 48%|████▊     | 5073/10570 [16:25<24:17,  3.77it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:34:53,488 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 64%|██████▍   | 6811/10570 [22:09<14:32,  4.31it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:40:37,584 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 64%|██████▍   | 6812/10570 [22:09<14:55,  4.20it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:40:37,835 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 64%|██████▍   | 6813/10570 [22:09<15:04,  4.16it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:40:38,079 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 64%|██████▍   | 6814/10570 [22:10<15:14,  4.11it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:40:38,333 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 64%|██████▍   | 6815/10570 [22:10<15:03,  4.16it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:40:38,565 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 64%|██████▍   | 6816/10570 [22:10<15:21,  4.07it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:40:38,823 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 64%|██████▍   | 6817/10570 [22:10<15:33,  4.02it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:40:39,081 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 65%|██████▍   | 6818/10570 [22:11<15:46,  3.96it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:40:39,338 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 65%|██████▍   | 6819/10570 [22:11<15:51,  3.94it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:40:39,603 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 65%|██████▍   | 6820/10570 [22:11<15:55,  3.92it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:40:39,853 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 65%|██████▌   | 6883/10570 [22:26<13:11,  4.66it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:40:54,725 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n",
            "2024-10-05 09:40:54,726 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 65%|██████▌   | 6884/10570 [22:26<13:33,  4.53it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:40:54,959 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n",
            "2024-10-05 09:40:54,960 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 65%|██████▌   | 6885/10570 [22:26<13:48,  4.45it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:40:55,191 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n",
            "2024-10-05 09:40:55,192 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 65%|██████▌   | 6886/10570 [22:27<13:54,  4.42it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:40:55,422 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n",
            "2024-10-05 09:40:55,423 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 65%|██████▌   | 6887/10570 [22:27<14:00,  4.38it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:40:55,656 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n",
            "2024-10-05 09:40:55,657 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 69%|██████▉   | 7331/10570 [23:59<12:44,  4.24it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:42:28,255 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n",
            "2024-10-05 09:42:28,257 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 69%|██████▉   | 7332/10570 [24:00<14:40,  3.68it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:42:28,611 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n",
            "2024-10-05 09:42:28,612 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 69%|██████▉   | 7333/10570 [24:00<16:00,  3.37it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:42:28,964 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n",
            "2024-10-05 09:42:28,965 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 69%|██████▉   | 7334/10570 [24:00<16:34,  3.25it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:42:29,288 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n",
            "2024-10-05 09:42:29,289 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n",
            "2024-10-05 09:42:29,392 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n",
            "2024-10-05 09:42:29,393 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 69%|██████▉   | 7335/10570 [24:01<17:33,  3.07it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:42:29,689 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n",
            "2024-10-05 09:42:29,691 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 69%|██████▉   | 7336/10570 [24:01<18:12,  2.96it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:42:30,051 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n",
            "2024-10-05 09:42:30,053 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 69%|██████▉   | 7337/10570 [24:02<18:44,  2.87it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:42:30,412 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n",
            "2024-10-05 09:42:30,413 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 69%|██████▉   | 7338/10570 [24:02<18:54,  2.85it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:42:30,759 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n",
            "2024-10-05 09:42:30,760 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 69%|██████▉   | 7339/10570 [24:02<18:54,  2.85it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:42:31,108 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n",
            "2024-10-05 09:42:31,109 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 69%|██████▉   | 7340/10570 [24:03<18:50,  2.86it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:42:31,467 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n",
            "2024-10-05 09:42:31,468 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 76%|███████▌  | 8007/10570 [26:13<08:01,  5.32it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:44:41,264 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 76%|███████▌  | 8008/10570 [26:13<08:22,  5.10it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:44:41,483 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 76%|███████▌  | 8009/10570 [26:13<08:37,  4.95it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:44:41,695 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 76%|███████▌  | 8010/10570 [26:13<08:41,  4.91it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:44:41,907 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 76%|███████▌  | 8011/10570 [26:13<08:52,  4.81it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:44:42,123 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 86%|████████▌ | 9049/10570 [29:47<06:03,  4.18it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:48:15,415 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 94%|█████████▍| 9915/10570 [32:37<02:39,  4.10it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:51:05,721 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 94%|█████████▍| 9916/10570 [32:37<02:37,  4.17it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:51:05,955 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 94%|█████████▍| 9917/10570 [32:37<02:33,  4.25it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:51:06,176 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 94%|█████████▍| 9918/10570 [32:38<02:31,  4.30it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:51:06,404 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 94%|█████████▍| 9919/10570 [32:38<02:30,  4.32it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:51:06,631 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 94%|█████████▍| 9983/10570 [32:54<03:04,  3.18it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:51:22,634 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 94%|█████████▍| 9984/10570 [32:54<02:52,  3.39it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:51:22,881 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 94%|█████████▍| 9985/10570 [32:54<02:43,  3.59it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:51:23,127 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 94%|█████████▍| 9986/10570 [32:55<02:37,  3.71it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:51:23,377 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 94%|█████████▍| 9987/10570 [32:55<02:34,  3.76it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:51:23,636 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 95%|█████████▍| 10038/10570 [33:06<01:26,  6.18it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-05 09:51:34,322 Warning: An empty Sentence was created! Are there empty strings in your dataset?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 10570/10570 [35:06<00:00,  5.02it/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from datasets import Dataset\n",
        "\n",
        "# Assuming `anonymized_train` and `anonymized_val` are lists of dictionaries,\n",
        "# or each element in the list corresponds to an entry in the original dataset.\n",
        "\n",
        "# Convert the list of anonymized examples back to a Dataset object.\n",
        "anonymized_train_dataset = Dataset.from_dict({key: [d[key] for d in anonymized_train] for key in anonymized_train[0]})\n",
        "anonymized_val_dataset = Dataset.from_dict({key: [d[key] for d in anonymized_val] for key in anonymized_val[0]})\n"
      ],
      "metadata": {
        "id": "lpnn7wLAi9K5"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "anonymized_train_dataset"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XOzIgRNai_46",
        "outputId": "090abf67-77fc-4241-f08e-dc15fa89211a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Dataset({\n",
              "    features: ['id', 'title', 'context', 'question', 'answers'],\n",
              "    num_rows: 87599\n",
              "})"
            ]
          },
          "metadata": {},
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "del anonymized_combined"
      ],
      "metadata": {
        "id": "Y1KUnqTfdtJh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "anonymized_combined = DatasetDict({\n",
        "    'train': anonymized_train_dataset,\n",
        "    'validation': anonymized_val_dataset\n",
        "})"
      ],
      "metadata": {
        "id": "7CNBprpkZEIc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# anonymized_combined.save_to_disk(\"qa_good\")"
      ],
      "metadata": {
        "id": "b90d7-QHc6Kl",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 101,
          "referenced_widgets": [
            "cead3ede305b45f4a7761a2b5f7d0a8f",
            "3e4d665055854b548486c2e25e82ce2f",
            "369d51df338842029bb275fda9146583",
            "bf4b002f54e344a98c25e99b215c0e94",
            "e6e67cfb9a4c4efbb1ae7256440993ec",
            "6dd11437522044fbb9529839c41bdeb2",
            "c6cba4d3c4804b65a6a60b24fd474d50",
            "794dac0ac2ad4854ac79cf5daa71cb47",
            "3c6413e4cbd04c8098b6f96193664a36",
            "883339be841740d2a1f50e7b5d7c649b",
            "a8f88596619443798a26f1fbfdb0d701",
            "8048c51e4ba34395b64bde9a3dbe4a04",
            "0965bb1cc6bb401d848f03c359862854",
            "517b3b043e5f4f50a395686f854e6199",
            "16b11fa4c81e4cee9c6c1501e12bdf70",
            "77fb9d215b75412ba64dfb0fd650de91",
            "9503946fbbee4d66917e60f947d976ba",
            "9f204b229c244839b17d564effff2fae",
            "5970ad3d91a64f00a872b8ff2f77386c",
            "32f5d622186a43f89a4895a01ac7af3d",
            "a1c322177c56445b97b8d07b10b2131b",
            "7452ba4b4d484de2bd3e387ebe68f749"
          ]
        },
        "outputId": "d95c0d00-0124-4e1b-f5b0-0c80b57379ec"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Saving the dataset (0/1 shards):   0%|          | 0/87599 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "cead3ede305b45f4a7761a2b5f7d0a8f"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Saving the dataset (0/1 shards):   0%|          | 0/10570 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "8048c51e4ba34395b64bde9a3dbe4a04"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from datasets import load_from_disk # import the load_from_disk function from the datasets library\n",
        "\n",
        "anonymized_subset=load_from_disk(\"qa_good\")"
      ],
      "metadata": {
        "id": "4mD3eH9LthMI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "anonymized_subset['train'][7]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oEPQGDibYsiE",
        "outputId": "e26e88ef-3483-46cf-ee4e-383c1e0f4c32"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'id': '5733bf84d058e614000b61c0',\n",
              " 'title': 'University_of_Notre_Dame',\n",
              " 'context': 'As at most other universities, ent188895 students run a number of news media outlets. The nine student-run outlets include three newspapers, both a radio and television station, and several magazines and journals. Begun as a one-page journal in September 1876, the ent123800 magazine is issued twice monthly and claims to be the oldest continuous collegiate publication in ent115227. The other magazine, ent256017, is released twice a year and focuses on student literature and artwork. The ent122511 yearbook is published annually. The newspapers have varying publication interests, with ent205802 published daily and mainly reporting university and other news, and staffed by students from both ent292193 and ent168725. Unlike ent123800 and ent41822, ent205802 is an independent publication and does not have a faculty advisor or any editorial oversight from the ent101460. In 1987, when some students believed that ent205802 began to show a conservative bias, a liberal newspaper, ent48503 was published. Likewise, in 2003, when other students believed that the paper showed a liberal bias, the conservative paper ent12151 went into production. Neither paper is published as often as ent205802; however, all three are distributed to all students. Finally, in Spring 2008 an undergraduate journal for political science research, ent196439, made its debut.',\n",
              " 'question': 'What is the daily student paper at ent292193 called?',\n",
              " 'answers': {'answer_start': [589], 'text': ['ent205802']}}"
            ]
          },
          "metadata": {},
          "execution_count": 105
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "for i in range(20,30):\n",
        "    print(dataset['train'][i])"
      ],
      "metadata": {
        "id": "B_3TZQnyNWc_"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Initialize dictionaries to keep track of mappings and used IDs\n",
        "entity_mappings = defaultdict(str)\n",
        "used_ids = set()\n",
        "\n",
        "# Apply anonymization to the dataset\n",
        "anonymized_subset = dataset['train'].select(range(200)).map(lambda example: anonymize_example(example, tagger,entity_mappings, used_ids))\n",
        "\n"
      ],
      "metadata": {
        "collapsed": true,
        "id": "Ekp4wa79TyFh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for i in range(20,41):\n",
        "    print(dataset['train'][i])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GXEiqZEXfPN_",
        "outputId": "1a14fa86-08d6-49bd-f567-117064ae2970"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'id': '5733a70c4776f41900660f64', 'title': 'University_of_Notre_Dame', 'context': \"All of Notre Dame's undergraduate students are a part of one of the five undergraduate colleges at the school or are in the First Year of Studies program. The First Year of Studies program was established in 1962 to guide incoming freshmen in their first year at the school before they have declared a major. Each student is given an academic advisor from the program who helps them to choose classes that give them exposure to any major in which they are interested. The program also includes a Learning Resource Center which provides time management, collaborative learning, and subject tutoring. This program has been recognized previously, by U.S. News & World Report, as outstanding.\", 'question': 'What entity provides help with the management of time for new students at Notre Dame?', 'answers': {'text': ['Learning Resource Center'], 'answer_start': [496]}}\n",
            "{'id': '5733a70c4776f41900660f62', 'title': 'University_of_Notre_Dame', 'context': \"All of Notre Dame's undergraduate students are a part of one of the five undergraduate colleges at the school or are in the First Year of Studies program. The First Year of Studies program was established in 1962 to guide incoming freshmen in their first year at the school before they have declared a major. Each student is given an academic advisor from the program who helps them to choose classes that give them exposure to any major in which they are interested. The program also includes a Learning Resource Center which provides time management, collaborative learning, and subject tutoring. This program has been recognized previously, by U.S. News & World Report, as outstanding.\", 'question': 'How many colleges for undergraduates are at Notre Dame?', 'answers': {'text': ['five'], 'answer_start': [68]}}\n",
            "{'id': '5733a70c4776f41900660f63', 'title': 'University_of_Notre_Dame', 'context': \"All of Notre Dame's undergraduate students are a part of one of the five undergraduate colleges at the school or are in the First Year of Studies program. The First Year of Studies program was established in 1962 to guide incoming freshmen in their first year at the school before they have declared a major. Each student is given an academic advisor from the program who helps them to choose classes that give them exposure to any major in which they are interested. The program also includes a Learning Resource Center which provides time management, collaborative learning, and subject tutoring. This program has been recognized previously, by U.S. News & World Report, as outstanding.\", 'question': 'What was created at Notre Dame in 1962 to assist first year students?', 'answers': {'text': ['The First Year of Studies program'], 'answer_start': [155]}}\n",
            "{'id': '5733a70c4776f41900660f65', 'title': 'University_of_Notre_Dame', 'context': \"All of Notre Dame's undergraduate students are a part of one of the five undergraduate colleges at the school or are in the First Year of Studies program. The First Year of Studies program was established in 1962 to guide incoming freshmen in their first year at the school before they have declared a major. Each student is given an academic advisor from the program who helps them to choose classes that give them exposure to any major in which they are interested. The program also includes a Learning Resource Center which provides time management, collaborative learning, and subject tutoring. This program has been recognized previously, by U.S. News & World Report, as outstanding.\", 'question': 'Which organization declared the First Year of Studies program at Notre Dame \"outstanding?\"', 'answers': {'text': ['U.S. News & World Report'], 'answer_start': [647]}}\n",
            "{'id': '5733a7bd4776f41900660f6b', 'title': 'University_of_Notre_Dame', 'context': 'The university first offered graduate degrees, in the form of a Master of Arts (MA), in the 1854–1855 academic year. The program expanded to include Master of Laws (LL.M.) and Master of Civil Engineering in its early stages of growth, before a formal graduate school education was developed with a thesis not required to receive the degrees. This changed in 1924 with formal requirements developed for graduate degrees, including offering Doctorate (PhD) degrees. Today each of the five colleges offer graduate education. Most of the departments from the College of Arts and Letters offer PhD programs, while a professional Master of Divinity (M.Div.) program also exists. All of the departments in the College of Science offer PhD programs, except for the Department of Pre-Professional Studies. The School of Architecture offers a Master of Architecture, while each of the departments of the College of Engineering offer PhD programs. The College of Business offers multiple professional programs including MBA and Master of Science in Accountancy programs. It also operates facilities in Chicago and Cincinnati for its executive MBA program. Additionally, the Alliance for Catholic Education program offers a Master of Education program where students study at the university during the summer and teach in Catholic elementary schools, middle schools, and high schools across the Southern United States for two school years.', 'question': 'The granting of Doctorate degrees first occurred in what year at Notre Dame?', 'answers': {'text': ['1924'], 'answer_start': [358]}}\n",
            "{'id': '5733a7bd4776f41900660f6c', 'title': 'University_of_Notre_Dame', 'context': 'The university first offered graduate degrees, in the form of a Master of Arts (MA), in the 1854–1855 academic year. The program expanded to include Master of Laws (LL.M.) and Master of Civil Engineering in its early stages of growth, before a formal graduate school education was developed with a thesis not required to receive the degrees. This changed in 1924 with formal requirements developed for graduate degrees, including offering Doctorate (PhD) degrees. Today each of the five colleges offer graduate education. Most of the departments from the College of Arts and Letters offer PhD programs, while a professional Master of Divinity (M.Div.) program also exists. All of the departments in the College of Science offer PhD programs, except for the Department of Pre-Professional Studies. The School of Architecture offers a Master of Architecture, while each of the departments of the College of Engineering offer PhD programs. The College of Business offers multiple professional programs including MBA and Master of Science in Accountancy programs. It also operates facilities in Chicago and Cincinnati for its executive MBA program. Additionally, the Alliance for Catholic Education program offers a Master of Education program where students study at the university during the summer and teach in Catholic elementary schools, middle schools, and high schools across the Southern United States for two school years.', 'question': 'What type of degree is an M.Div.?', 'answers': {'text': ['Master of Divinity'], 'answer_start': [624]}}\n",
            "{'id': '5733a7bd4776f41900660f6e', 'title': 'University_of_Notre_Dame', 'context': 'The university first offered graduate degrees, in the form of a Master of Arts (MA), in the 1854–1855 academic year. The program expanded to include Master of Laws (LL.M.) and Master of Civil Engineering in its early stages of growth, before a formal graduate school education was developed with a thesis not required to receive the degrees. This changed in 1924 with formal requirements developed for graduate degrees, including offering Doctorate (PhD) degrees. Today each of the five colleges offer graduate education. Most of the departments from the College of Arts and Letters offer PhD programs, while a professional Master of Divinity (M.Div.) program also exists. All of the departments in the College of Science offer PhD programs, except for the Department of Pre-Professional Studies. The School of Architecture offers a Master of Architecture, while each of the departments of the College of Engineering offer PhD programs. The College of Business offers multiple professional programs including MBA and Master of Science in Accountancy programs. It also operates facilities in Chicago and Cincinnati for its executive MBA program. Additionally, the Alliance for Catholic Education program offers a Master of Education program where students study at the university during the summer and teach in Catholic elementary schools, middle schools, and high schools across the Southern United States for two school years.', 'question': 'Which program at Notre Dame offers a Master of Education degree?', 'answers': {'text': ['Alliance for Catholic Education'], 'answer_start': [1163]}}\n",
            "{'id': '5733a7bd4776f41900660f6a', 'title': 'University_of_Notre_Dame', 'context': 'The university first offered graduate degrees, in the form of a Master of Arts (MA), in the 1854–1855 academic year. The program expanded to include Master of Laws (LL.M.) and Master of Civil Engineering in its early stages of growth, before a formal graduate school education was developed with a thesis not required to receive the degrees. This changed in 1924 with formal requirements developed for graduate degrees, including offering Doctorate (PhD) degrees. Today each of the five colleges offer graduate education. Most of the departments from the College of Arts and Letters offer PhD programs, while a professional Master of Divinity (M.Div.) program also exists. All of the departments in the College of Science offer PhD programs, except for the Department of Pre-Professional Studies. The School of Architecture offers a Master of Architecture, while each of the departments of the College of Engineering offer PhD programs. The College of Business offers multiple professional programs including MBA and Master of Science in Accountancy programs. It also operates facilities in Chicago and Cincinnati for its executive MBA program. Additionally, the Alliance for Catholic Education program offers a Master of Education program where students study at the university during the summer and teach in Catholic elementary schools, middle schools, and high schools across the Southern United States for two school years.', 'question': 'In what year was a Master of Arts course first offered at Notre Dame?', 'answers': {'text': ['1854'], 'answer_start': [92]}}\n",
            "{'id': '5733a7bd4776f41900660f6d', 'title': 'University_of_Notre_Dame', 'context': 'The university first offered graduate degrees, in the form of a Master of Arts (MA), in the 1854–1855 academic year. The program expanded to include Master of Laws (LL.M.) and Master of Civil Engineering in its early stages of growth, before a formal graduate school education was developed with a thesis not required to receive the degrees. This changed in 1924 with formal requirements developed for graduate degrees, including offering Doctorate (PhD) degrees. Today each of the five colleges offer graduate education. Most of the departments from the College of Arts and Letters offer PhD programs, while a professional Master of Divinity (M.Div.) program also exists. All of the departments in the College of Science offer PhD programs, except for the Department of Pre-Professional Studies. The School of Architecture offers a Master of Architecture, while each of the departments of the College of Engineering offer PhD programs. The College of Business offers multiple professional programs including MBA and Master of Science in Accountancy programs. It also operates facilities in Chicago and Cincinnati for its executive MBA program. Additionally, the Alliance for Catholic Education program offers a Master of Education program where students study at the university during the summer and teach in Catholic elementary schools, middle schools, and high schools across the Southern United States for two school years.', 'question': 'Which department at Notre Dame is the only one to not offer a PhD program?', 'answers': {'text': ['Department of Pre-Professional Studies'], 'answer_start': [757]}}\n",
            "{'id': '5733ac31d058e614000b5ff3', 'title': 'University_of_Notre_Dame', 'context': \"The Joan B. Kroc Institute for International Peace Studies at the University of Notre Dame is dedicated to research, education and outreach on the causes of violent conflict and the conditions for sustainable peace. It offers PhD, Master's, and undergraduate degrees in peace studies. It was founded in 1986 through the donations of Joan B. Kroc, the widow of McDonald's owner Ray Kroc. The institute was inspired by the vision of the Rev. Theodore M. Hesburgh CSC, President Emeritus of the University of Notre Dame. The institute has contributed to international policy discussions about peace building practices.\", 'question': 'What institute at Notre Dame studies  the reasons for violent conflict?', 'answers': {'text': ['Joan B. Kroc Institute for International Peace Studies'], 'answer_start': [4]}}\n",
            "{'id': '5733ac31d058e614000b5ff6', 'title': 'University_of_Notre_Dame', 'context': \"The Joan B. Kroc Institute for International Peace Studies at the University of Notre Dame is dedicated to research, education and outreach on the causes of violent conflict and the conditions for sustainable peace. It offers PhD, Master's, and undergraduate degrees in peace studies. It was founded in 1986 through the donations of Joan B. Kroc, the widow of McDonald's owner Ray Kroc. The institute was inspired by the vision of the Rev. Theodore M. Hesburgh CSC, President Emeritus of the University of Notre Dame. The institute has contributed to international policy discussions about peace building practices.\", 'question': \"What is the title of Notre Dame's Theodore Hesburgh?\", 'answers': {'text': ['President Emeritus of the University of Notre Dame'], 'answer_start': [466]}}\n",
            "{'id': '5733ac31d058e614000b5ff4', 'title': 'University_of_Notre_Dame', 'context': \"The Joan B. Kroc Institute for International Peace Studies at the University of Notre Dame is dedicated to research, education and outreach on the causes of violent conflict and the conditions for sustainable peace. It offers PhD, Master's, and undergraduate degrees in peace studies. It was founded in 1986 through the donations of Joan B. Kroc, the widow of McDonald's owner Ray Kroc. The institute was inspired by the vision of the Rev. Theodore M. Hesburgh CSC, President Emeritus of the University of Notre Dame. The institute has contributed to international policy discussions about peace building practices.\", 'question': 'In what year was the Joan B. Kroc Institute for International Peace Studies founded?', 'answers': {'text': ['1986'], 'answer_start': [303]}}\n",
            "{'id': '5733ac31d058e614000b5ff5', 'title': 'University_of_Notre_Dame', 'context': \"The Joan B. Kroc Institute for International Peace Studies at the University of Notre Dame is dedicated to research, education and outreach on the causes of violent conflict and the conditions for sustainable peace. It offers PhD, Master's, and undergraduate degrees in peace studies. It was founded in 1986 through the donations of Joan B. Kroc, the widow of McDonald's owner Ray Kroc. The institute was inspired by the vision of the Rev. Theodore M. Hesburgh CSC, President Emeritus of the University of Notre Dame. The institute has contributed to international policy discussions about peace building practices.\", 'question': 'To whom was John B. Kroc married?', 'answers': {'text': ['Ray Kroc'], 'answer_start': [377]}}\n",
            "{'id': '5733ac31d058e614000b5ff7', 'title': 'University_of_Notre_Dame', 'context': \"The Joan B. Kroc Institute for International Peace Studies at the University of Notre Dame is dedicated to research, education and outreach on the causes of violent conflict and the conditions for sustainable peace. It offers PhD, Master's, and undergraduate degrees in peace studies. It was founded in 1986 through the donations of Joan B. Kroc, the widow of McDonald's owner Ray Kroc. The institute was inspired by the vision of the Rev. Theodore M. Hesburgh CSC, President Emeritus of the University of Notre Dame. The institute has contributed to international policy discussions about peace building practices.\", 'question': 'What company did Ray Kroc own?', 'answers': {'text': [\"McDonald's\"], 'answer_start': [360]}}\n",
            "{'id': '5733ad384776f41900660fec', 'title': 'University_of_Notre_Dame', 'context': 'The library system of the university is divided between the main library and each of the colleges and schools. The main building is the 14-story Theodore M. Hesburgh Library, completed in 1963, which is the third building to house the main collection of books. The front of the library is adorned with the Word of Life mural designed by artist Millard Sheets. This mural is popularly known as \"Touchdown Jesus\" because of its proximity to Notre Dame Stadium and Jesus\\' arms appearing to make the signal for a touchdown.', 'question': 'How many stories tall is the main library at Notre Dame?', 'answers': {'text': ['14'], 'answer_start': [136]}}\n",
            "{'id': '5733ad384776f41900660fed', 'title': 'University_of_Notre_Dame', 'context': 'The library system of the university is divided between the main library and each of the colleges and schools. The main building is the 14-story Theodore M. Hesburgh Library, completed in 1963, which is the third building to house the main collection of books. The front of the library is adorned with the Word of Life mural designed by artist Millard Sheets. This mural is popularly known as \"Touchdown Jesus\" because of its proximity to Notre Dame Stadium and Jesus\\' arms appearing to make the signal for a touchdown.', 'question': 'What is the name of the main library at Notre Dame?', 'answers': {'text': ['Theodore M. Hesburgh Library'], 'answer_start': [145]}}\n",
            "{'id': '5733ad384776f41900660fee', 'title': 'University_of_Notre_Dame', 'context': 'The library system of the university is divided between the main library and each of the colleges and schools. The main building is the 14-story Theodore M. Hesburgh Library, completed in 1963, which is the third building to house the main collection of books. The front of the library is adorned with the Word of Life mural designed by artist Millard Sheets. This mural is popularly known as \"Touchdown Jesus\" because of its proximity to Notre Dame Stadium and Jesus\\' arms appearing to make the signal for a touchdown.', 'question': 'In what year was the Theodore M. Hesburgh Library at Notre Dame finished?', 'answers': {'text': ['1963'], 'answer_start': [188]}}\n",
            "{'id': '5733ad384776f41900660fef', 'title': 'University_of_Notre_Dame', 'context': 'The library system of the university is divided between the main library and each of the colleges and schools. The main building is the 14-story Theodore M. Hesburgh Library, completed in 1963, which is the third building to house the main collection of books. The front of the library is adorned with the Word of Life mural designed by artist Millard Sheets. This mural is popularly known as \"Touchdown Jesus\" because of its proximity to Notre Dame Stadium and Jesus\\' arms appearing to make the signal for a touchdown.', 'question': 'Which artist created the mural on the Theodore M. Hesburgh Library?', 'answers': {'text': ['Millard Sheets'], 'answer_start': [344]}}\n",
            "{'id': '5733ad384776f41900660ff0', 'title': 'University_of_Notre_Dame', 'context': 'The library system of the university is divided between the main library and each of the colleges and schools. The main building is the 14-story Theodore M. Hesburgh Library, completed in 1963, which is the third building to house the main collection of books. The front of the library is adorned with the Word of Life mural designed by artist Millard Sheets. This mural is popularly known as \"Touchdown Jesus\" because of its proximity to Notre Dame Stadium and Jesus\\' arms appearing to make the signal for a touchdown.', 'question': 'What is a common name to reference the mural created by Millard Sheets at Notre Dame?', 'answers': {'text': ['Touchdown Jesus'], 'answer_start': [394]}}\n",
            "{'id': '5733ae924776f41900661014', 'title': 'University_of_Notre_Dame', 'context': 'Notre Dame is known for its competitive admissions, with the incoming class enrolling in fall 2015 admitting 3,577 from a pool of 18,156 (19.7%). The academic profile of the enrolled class continues to rate among the top 10 to 15 in the nation for national research universities. The university practices a non-restrictive early action policy that allows admitted students to consider admission to Notre Dame as well as any other colleges to which they were accepted. 1,400 of the 3,577 (39.1%) were admitted under the early action plan. Admitted students came from 1,311 high schools and the average student traveled more than 750 miles to Notre Dame, making it arguably the most representative university in the United States. While all entering students begin in the College of the First Year of Studies, 25% have indicated they plan to study in the liberal arts or social sciences, 24% in engineering, 24% in business, 24% in science, and 3% in architecture.', 'question': 'How many incoming students did Notre Dame admit in fall 2015?', 'answers': {'text': ['3,577'], 'answer_start': [109]}}\n",
            "{'id': '5733ae924776f41900661013', 'title': 'University_of_Notre_Dame', 'context': 'Notre Dame is known for its competitive admissions, with the incoming class enrolling in fall 2015 admitting 3,577 from a pool of 18,156 (19.7%). The academic profile of the enrolled class continues to rate among the top 10 to 15 in the nation for national research universities. The university practices a non-restrictive early action policy that allows admitted students to consider admission to Notre Dame as well as any other colleges to which they were accepted. 1,400 of the 3,577 (39.1%) were admitted under the early action plan. Admitted students came from 1,311 high schools and the average student traveled more than 750 miles to Notre Dame, making it arguably the most representative university in the United States. While all entering students begin in the College of the First Year of Studies, 25% have indicated they plan to study in the liberal arts or social sciences, 24% in engineering, 24% in business, 24% in science, and 3% in architecture.', 'question': 'What percentage of students were admitted to Notre Dame in fall 2015?', 'answers': {'text': ['19.7%'], 'answer_start': [138]}}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "for i in range(20,31):\n",
        "    print(anonymized_subset['train'][i])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tk_klFvkRfC7",
        "outputId": "a56d0e92-cff7-4b5f-d419-48999aa61f0a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'id': '5733a70c4776f41900660f64', 'title': 'University_of_Notre_Dame', 'context': 'All of ent195905 undergraduate students are a part of one of the five undergraduate colleges at the school or are in the ent80919 program. The First Year of Studies program was established in 1962 to guide incoming freshmen in their first year at the school before they have declared a major. Each student is given an academic advisor from the program who helps them to choose classes that give them exposure to any major in which they are interested. The program also includes ent209217 which provides time management, collaborative learning, and subject tutoring. This program has been recognized previously, by ent275506. ent102414, as outstanding.', 'question': 'What entity provides help with the management of time for new students at ent125700?', 'answers': {'answer_start': [478], 'text': ['ent161017']}}\n",
            "{'id': '5733a70c4776f41900660f62', 'title': 'University_of_Notre_Dame', 'context': 'All of ent105906 undergraduate students are a part of one of the five undergraduate colleges at the school or are in the ent11917 program. The First Year of Studies program was established in 1962 to guide incoming freshmen in their first year at the school before they have declared a major. Each student is given an academic advisor from the program who helps them to choose classes that give them exposure to any major in which they are interested. The program also includes ent273531 which provides time management, collaborative learning, and subject tutoring. This program has been recognized previously, by ent12209. ent111343, as outstanding.', 'question': 'How many colleges for undergraduates are at ent180150?', 'answers': {'answer_start': [65], 'text': ['five']}}\n",
            "{'id': '5733a70c4776f41900660f63', 'title': 'University_of_Notre_Dame', 'context': 'All of ent294239 undergraduate students are a part of one of the five undergraduate colleges at the school or are in the ent209459 program. The First Year of Studies program was established in 1962 to guide incoming freshmen in their first year at the school before they have declared a major. Each student is given an academic advisor from the program who helps them to choose classes that give them exposure to any major in which they are interested. The program also includes ent195487 which provides time management, collaborative learning, and subject tutoring. This program has been recognized previously, by ent88052. ent257868, as outstanding.', 'question': 'What was created at ent879 in 1962 to assist first year students?', 'answers': {'answer_start': [140], 'text': ['The First Year of Studies program']}}\n",
            "{'id': '5733a70c4776f41900660f65', 'title': 'University_of_Notre_Dame', 'context': 'All of ent258979 undergraduate students are a part of one of the five undergraduate colleges at the school or are in the ent87074 program. The First Year of Studies program was established in 1962 to guide incoming freshmen in their first year at the school before they have declared a major. Each student is given an academic advisor from the program who helps them to choose classes that give them exposure to any major in which they are interested. The program also includes ent138684 which provides time management, collaborative learning, and subject tutoring. This program has been recognized previously, by ent49511. ent172513, as outstanding.', 'question': 'Which organization declared the ent41752 Year of Studies program at ent234894 \"outstanding?\"', 'answers': {'answer_start': [614], 'text': ['ent49511. ent172854 ent256231']}}\n",
            "{'id': '5733a7bd4776f41900660f6b', 'title': 'University_of_Notre_Dame', 'context': 'The university first offered graduate degrees, in the form of ent201370 (ent67439), in the 1854–1855 academic year. The program expanded to include ent118530 of ent266030 (ent133239) and Master of Civil ent49979 in its early stages of growth, before a formal graduate school education was developed with a thesis not required to receive the degrees. This changed in 1924 with formal requirements developed for graduate degrees, including offering ent171294 (ent97349) degrees. Today each of the five colleges offer graduate education. Most of the departments from ent243222 offer ent97349 programs, while a professional ent118530 of ent84041 (ent51355) program also exists. All of the departments in ent260051 offer ent97349 programs, except for ent223578. ent238118 offers ent184201, while each of the departments of ent103769 offer ent97349 programs. ent162562 offers multiple professional programs including ent166106 and ent4336 programs. It also operates facilities in ent46047 and ent76343 for its executive ent166106 program. Additionally, the ent23282 program offers a Master of ent192095 program where students study at the university during the summer and teach in ent26053 elementary schools, middle schools, and high schools across the Southern ent293732 for two school years.', 'question': 'The granting of ent171294 degrees first occurred in what year at ent272266?', 'answers': {'answer_start': [366], 'text': ['1924']}}\n",
            "{'id': '5733a7bd4776f41900660f6c', 'title': 'University_of_Notre_Dame', 'context': 'The university first offered graduate degrees, in the form of ent61168 (ent58767), in the 1854–1855 academic year. The program expanded to include ent12428 of ent71257 (ent208809) and Master of Civil ent284661 in its early stages of growth, before a formal graduate school education was developed with a thesis not required to receive the degrees. This changed in 1924 with formal requirements developed for graduate degrees, including offering ent99857 (ent182250) degrees. Today each of the five colleges offer graduate education. Most of the departments from ent100955 offer ent182250 programs, while a professional ent12428 of ent125360 (ent130093) program also exists. All of the departments in ent5448 offer ent182250 programs, except for ent281244. ent194987 offers ent20385, while each of the departments of ent2181 offer ent182250 programs. ent140545 offers multiple professional programs including ent74669 and ent257805 programs. It also operates facilities in ent110077 and ent54103 for its executive ent74669 program. Additionally, the ent239841 program offers a Master of ent178469 program where students study at the university during the summer and teach in ent12642 elementary schools, middle schools, and high schools across the Southern ent18922 for two school years.', 'question': 'What type of degree is an ent130093?', 'answers': {'answer_start': [619], 'text': ['ent12698']}}\n",
            "{'id': '5733a7bd4776f41900660f6e', 'title': 'University_of_Notre_Dame', 'context': 'The university first offered graduate degrees, in the form of ent32998 (ent220838), in the 1854–1855 academic year. The program expanded to include ent206075 of ent263570 (ent153247) and Master of Civil ent29065 in its early stages of growth, before a formal graduate school education was developed with a thesis not required to receive the degrees. This changed in 1924 with formal requirements developed for graduate degrees, including offering ent64689 (ent278866) degrees. Today each of the five colleges offer graduate education. Most of the departments from ent51665 offer ent278866 programs, while a professional ent206075 of ent62755 (ent77633) program also exists. All of the departments in ent117403 offer ent278866 programs, except for ent4889. ent33262 offers ent75456, while each of the departments of ent107562 offer ent278866 programs. ent154023 offers multiple professional programs including ent207675 and ent12896 programs. It also operates facilities in ent47876 and ent86705 for its executive ent207675 program. Additionally, the ent167539 program offers a Master of ent39855 program where students study at the university during the summer and teach in ent249573 elementary schools, middle schools, and high schools across the Southern ent259826 for two school years.', 'question': 'Which program at ent104312 offers ent280744?', 'answers': {'answer_start': [1050], 'text': ['ent167539']}}\n",
            "{'id': '5733a7bd4776f41900660f6a', 'title': 'University_of_Notre_Dame', 'context': 'The university first offered graduate degrees, in the form of ent164153 (ent239297), in the 1854–1855 academic year. The program expanded to include ent149133 of ent284444 (ent108728) and Master of Civil ent89595 in its early stages of growth, before a formal graduate school education was developed with a thesis not required to receive the degrees. This changed in 1924 with formal requirements developed for graduate degrees, including offering ent42676 (ent116630) degrees. Today each of the five colleges offer graduate education. Most of the departments from ent128415 offer ent116630 programs, while a professional ent149133 of ent2189 (ent144971) program also exists. All of the departments in ent121726 offer ent116630 programs, except for ent187389. ent198076 offers ent27960, while each of the departments of ent231153 offer ent116630 programs. ent89055 offers multiple professional programs including ent109098 and ent118453 programs. It also operates facilities in ent137445 and ent274900 for its executive ent109098 program. Additionally, the ent49636 program offers a Master of ent215993 program where students study at the university during the summer and teach in ent18115 elementary schools, middle schools, and high schools across the Southern ent22287 for two school years.', 'question': 'In what year was a Master of Arts course first offered at ent46197?', 'answers': {'answer_start': [92], 'text': ['1854']}}\n",
            "{'id': '5733a7bd4776f41900660f6d', 'title': 'University_of_Notre_Dame', 'context': 'The university first offered graduate degrees, in the form of ent145510 (ent183503), in the 1854–1855 academic year. The program expanded to include ent290092 of ent7211 (ent190309) and Master of Civil ent88104 in its early stages of growth, before a formal graduate school education was developed with a thesis not required to receive the degrees. This changed in 1924 with formal requirements developed for graduate degrees, including offering ent22064 (ent248635) degrees. Today each of the five colleges offer graduate education. Most of the departments from ent258630 offer ent248635 programs, while a professional ent290092 of ent159887 (ent287445) program also exists. All of the departments in ent11713 offer ent248635 programs, except for ent188903. ent227263 offers ent136281, while each of the departments of ent92099 offer ent248635 programs. ent211995 offers multiple professional programs including ent124033 and ent263735 programs. It also operates facilities in ent235417 and ent80193 for its executive ent124033 program. Additionally, the ent285453 program offers a Master of ent123701 program where students study at the university during the summer and teach in ent140739 elementary schools, middle schools, and high schools across the Southern ent139848 for two school years.', 'question': 'Which department at ent3050 is the only one to not offer a ent248635 program?', 'answers': {'answer_start': [748], 'text': ['ent281981']}}\n",
            "{'id': '5733ac31d058e614000b5ff3', 'title': 'University_of_Notre_Dame', 'context': \"ent102234. ent139109 at ent139060 is dedicated to research, education and outreach on the causes of violent conflict and the conditions for sustainable peace. It offers ent184942, ent176103, and undergraduate degrees in peace studies. It was founded in 1986 through the donations of ent120351. ent68410, the widow of ent204743's owner ent197124. The institute was inspired by the vision of the Rev. ent252423. ent107775 CSC, President Emeritus of ent139060. The institute has contributed to international policy discussions about peace building practices.\", 'question': 'What institute at ent74950 studies  the reasons for violent conflict?', 'answers': {'answer_start': [0], 'text': ['ent120351. ent139109']}}\n",
            "{'id': '5733ac31d058e614000b5ff6', 'title': 'University_of_Notre_Dame', 'context': \"ent209496. ent126869 at ent239321 is dedicated to research, education and outreach on the causes of violent conflict and the conditions for sustainable peace. It offers ent249912, ent260311, and undergraduate degrees in peace studies. It was founded in 1986 through the donations of ent112305. ent229511, the widow of ent70078's owner ent103009. The institute was inspired by the vision of the Rev. ent235619. ent61564 CSC, President Emeritus of ent239321. The institute has contributed to international policy discussions about peace building practices.\", 'question': 'What is the title of ent200896 ent19475?', 'answers': {'answer_start': [424], 'text': ['President Emeritus of ent239321']}}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "dataset['train'][30]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "m_VkRF1gSUXx",
        "outputId": "44cfd938-858f-4b9d-831d-32d19648f6a1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'id': '5733ac31d058e614000b5ff6',\n",
              " 'title': 'University_of_Notre_Dame',\n",
              " 'context': \"The Joan B. Kroc Institute for International Peace Studies at the University of Notre Dame is dedicated to research, education and outreach on the causes of violent conflict and the conditions for sustainable peace. It offers PhD, Master's, and undergraduate degrees in peace studies. It was founded in 1986 through the donations of Joan B. Kroc, the widow of McDonald's owner Ray Kroc. The institute was inspired by the vision of the Rev. Theodore M. Hesburgh CSC, President Emeritus of the University of Notre Dame. The institute has contributed to international policy discussions about peace building practices.\",\n",
              " 'question': \"What is the title of Notre Dame's Theodore Hesburgh?\",\n",
              " 'answers': {'text': ['President Emeritus of the University of Notre Dame'],\n",
              "  'answer_start': [466]}}"
            ]
          },
          "metadata": {},
          "execution_count": 83
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "anonymized_subset['train'][17]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8iAHOY50aZI5",
        "outputId": "4d8c91db-6b98-417c-9f1d-f1062ecd6e07"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'id': '5733a6424776f41900660f4f',\n",
              " 'title': 'University_of_Notre_Dame',\n",
              " 'context': 'ent132326 was established in 1920, however, early courses in civil and mechanical engineering were a part of ent218001 since the 1870s. Today the college, housed in the ent163566, ent235446, and ent277423, includes five departments of study – aerospace and mechanical engineering, chemical and biomolecular engineering, civil engineering and geological sciences, computer science and engineering, and electrical engineering – with eight B.S. degrees offered. Additionally, the college offers five-year dual degree programs with the ent202884 and ent196167 awarding additional ent47679. and ent203525 (ent146980) degrees, respectively.',\n",
              " 'question': 'Before the creation of ent59705 similar studies were carried out at which ent32203 college?',\n",
              " 'answers': {'answer_start': [109], 'text': ['ent218001']}}"
            ]
          },
          "metadata": {},
          "execution_count": 106
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "TnoSoTk6aZGK"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# import pickle\n",
        "# with open('entity_map_qa_new.pkl', 'wb') as fichier:\n",
        "#     pickle.dump(entity_mappings, fichier)\n",
        "\n",
        "# print(\"Save with succes.\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xN0tWkzxOrRH",
        "outputId": "0dfaef95-827e-4972-9a39-a1dab6273d89"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Save with succes.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "9R3hqCcaOrOP"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "zy8u5Z5ALZo1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for i in range(0,10):\n",
        "    print(anonymized_subset['train'][i])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cFOC97KxWf_N",
        "outputId": "708251c9-fdde-42c6-eff6-3f0bdc201473"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'id': '5733be284776f41900661182', 'title': 'University_of_Notre_Dame', 'context': 'Architecturally, the school has a ent35615 character. Atop ent37396 gold dome is a golden statue of ent24885. Immediately in front of ent62154 and facing it, is a copper statue of Christ with arms upraised with the legend \"Venite Ad Me Omnes\". Next to ent62154 is ent42220. Immediately behind the basilica is the Grotto, a ent78988 place of prayer and reflection. It is a replica of the grotto at ent81722, ent73597 where ent24885 reputedly appeared to ent27892 in 1858. At the end of the main drive (and in a direct line that connects through 3 statues and ent62907), is a simple, modern stone statue of ent61792.', 'question': 'To whom did the Virgin Mary allegedly appear in 1858 in Lourdes France?', 'answers': {'text': ['Saint Bernadette Soubirous'], 'answer_start': [515]}}\n",
            "{'id': '5733be284776f4190066117f', 'title': 'University_of_Notre_Dame', 'context': 'Architecturally, the school has a ent35615 character. Atop ent37396 gold dome is a golden statue of ent24885. Immediately in front of ent62154 and facing it, is a copper statue of Christ with arms upraised with the legend \"Venite Ad Me Omnes\". Next to ent62154 is ent42220. Immediately behind the basilica is the Grotto, a ent78988 place of prayer and reflection. It is a replica of the grotto at ent81722, ent73597 where ent24885 reputedly appeared to ent27892 in 1858. At the end of the main drive (and in a direct line that connects through 3 statues and ent62907), is a simple, modern stone statue of ent61792.', 'question': 'What is in front of the Notre Dame Main Building?', 'answers': {'text': ['a copper statue of Christ'], 'answer_start': [188]}}\n",
            "{'id': '5733be284776f41900661180', 'title': 'University_of_Notre_Dame', 'context': 'Architecturally, the school has a ent35615 character. Atop ent37396 gold dome is a golden statue of ent24885. Immediately in front of ent62154 and facing it, is a copper statue of Christ with arms upraised with the legend \"Venite Ad Me Omnes\". Next to ent62154 is ent42220. Immediately behind the basilica is the Grotto, a ent78988 place of prayer and reflection. It is a replica of the grotto at ent81722, ent73597 where ent24885 reputedly appeared to ent27892 in 1858. At the end of the main drive (and in a direct line that connects through 3 statues and ent62907), is a simple, modern stone statue of ent61792.', 'question': 'The Basilica of the Sacred heart at Notre Dame is beside to which structure?', 'answers': {'text': ['the Main Building'], 'answer_start': [279]}}\n",
            "{'id': '5733be284776f41900661181', 'title': 'University_of_Notre_Dame', 'context': 'Architecturally, the school has a ent35615 character. Atop ent37396 gold dome is a golden statue of ent24885. Immediately in front of ent62154 and facing it, is a copper statue of Christ with arms upraised with the legend \"Venite Ad Me Omnes\". Next to ent62154 is ent42220. Immediately behind the basilica is the Grotto, a ent78988 place of prayer and reflection. It is a replica of the grotto at ent81722, ent73597 where ent24885 reputedly appeared to ent27892 in 1858. At the end of the main drive (and in a direct line that connects through 3 statues and ent62907), is a simple, modern stone statue of ent61792.', 'question': 'What is the Grotto at Notre Dame?', 'answers': {'text': ['a Marian place of prayer and reflection'], 'answer_start': [381]}}\n",
            "{'id': '5733be284776f4190066117e', 'title': 'University_of_Notre_Dame', 'context': 'Architecturally, the school has a ent35615 character. Atop ent37396 gold dome is a golden statue of ent24885. Immediately in front of ent62154 and facing it, is a copper statue of Christ with arms upraised with the legend \"Venite Ad Me Omnes\". Next to ent62154 is ent42220. Immediately behind the basilica is the Grotto, a ent78988 place of prayer and reflection. It is a replica of the grotto at ent81722, ent73597 where ent24885 reputedly appeared to ent27892 in 1858. At the end of the main drive (and in a direct line that connects through 3 statues and ent62907), is a simple, modern stone statue of ent61792.', 'question': 'What sits on top of the Main Building at Notre Dame?', 'answers': {'text': ['a golden statue of the Virgin Mary'], 'answer_start': [92]}}\n",
            "{'id': '5733bf84d058e614000b61be', 'title': 'University_of_Notre_Dame', 'context': 'As at most other universities, ent88716 students run a number of news media outlets. The nine student-run outlets include three newspapers, both a radio and television station, and several magazines and journals. Begun as a one-page journal in September 1876, the ent52014 magazine is issued twice monthly and claims to be the oldest continuous collegiate publication in ent70996. The other magazine, ent19781, is released twice a year and focuses on student literature and artwork. The ent46964 yearbook is published annually. The newspapers have varying publication interests, with ent60663 published daily and mainly reporting university and other news, and staffed by students from both ent4003 and ent85563. Unlike ent52014 and The ent46964, ent60663 is an independent publication and does not have a faculty advisor or any editorial oversight from the ent11889. In 1987, when some students believed that ent60663 began to show a conservative bias, a liberal newspaper, ent15513 was published. Likewise, in 2003, when other students believed that the paper showed a liberal bias, the conservative paper ent10078 went into production. Neither paper is published as often as ent60663; however, all three are distributed to all students. Finally, in Spring 2008 an undergraduate journal for political science research, ent60246, made its debut.', 'question': 'When did the Scholastic Magazine of Notre dame begin publishing?', 'answers': {'text': ['September 1876'], 'answer_start': [248]}}\n",
            "{'id': '5733bf84d058e614000b61bf', 'title': 'University_of_Notre_Dame', 'context': 'As at most other universities, ent88716 students run a number of news media outlets. The nine student-run outlets include three newspapers, both a radio and television station, and several magazines and journals. Begun as a one-page journal in September 1876, the ent52014 magazine is issued twice monthly and claims to be the oldest continuous collegiate publication in ent70996. The other magazine, ent19781, is released twice a year and focuses on student literature and artwork. The ent46964 yearbook is published annually. The newspapers have varying publication interests, with ent60663 published daily and mainly reporting university and other news, and staffed by students from both ent4003 and ent85563. Unlike ent52014 and The ent46964, ent60663 is an independent publication and does not have a faculty advisor or any editorial oversight from the ent11889. In 1987, when some students believed that ent60663 began to show a conservative bias, a liberal newspaper, ent15513 was published. Likewise, in 2003, when other students believed that the paper showed a liberal bias, the conservative paper ent10078 went into production. Neither paper is published as often as ent60663; however, all three are distributed to all students. Finally, in Spring 2008 an undergraduate journal for political science research, ent60246, made its debut.', 'question': \"How often is Notre Dame's the Juggler published?\", 'answers': {'text': ['twice'], 'answer_start': [441]}}\n",
            "{'id': '5733bf84d058e614000b61c0', 'title': 'University_of_Notre_Dame', 'context': 'As at most other universities, ent88716 students run a number of news media outlets. The nine student-run outlets include three newspapers, both a radio and television station, and several magazines and journals. Begun as a one-page journal in September 1876, the ent52014 magazine is issued twice monthly and claims to be the oldest continuous collegiate publication in ent70996. The other magazine, ent19781, is released twice a year and focuses on student literature and artwork. The ent46964 yearbook is published annually. The newspapers have varying publication interests, with ent60663 published daily and mainly reporting university and other news, and staffed by students from both ent4003 and ent85563. Unlike ent52014 and The ent46964, ent60663 is an independent publication and does not have a faculty advisor or any editorial oversight from the ent11889. In 1987, when some students believed that ent60663 began to show a conservative bias, a liberal newspaper, ent15513 was published. Likewise, in 2003, when other students believed that the paper showed a liberal bias, the conservative paper ent10078 went into production. Neither paper is published as often as ent60663; however, all three are distributed to all students. Finally, in Spring 2008 an undergraduate journal for political science research, ent60246, made its debut.', 'question': 'What is the daily student paper at Notre Dame called?', 'answers': {'text': ['The Observer'], 'answer_start': [598]}}\n",
            "{'id': '5733bf84d058e614000b61bd', 'title': 'University_of_Notre_Dame', 'context': 'As at most other universities, ent88716 students run a number of news media outlets. The nine student-run outlets include three newspapers, both a radio and television station, and several magazines and journals. Begun as a one-page journal in September 1876, the ent52014 magazine is issued twice monthly and claims to be the oldest continuous collegiate publication in ent70996. The other magazine, ent19781, is released twice a year and focuses on student literature and artwork. The ent46964 yearbook is published annually. The newspapers have varying publication interests, with ent60663 published daily and mainly reporting university and other news, and staffed by students from both ent4003 and ent85563. Unlike ent52014 and The ent46964, ent60663 is an independent publication and does not have a faculty advisor or any editorial oversight from the ent11889. In 1987, when some students believed that ent60663 began to show a conservative bias, a liberal newspaper, ent15513 was published. Likewise, in 2003, when other students believed that the paper showed a liberal bias, the conservative paper ent10078 went into production. Neither paper is published as often as ent60663; however, all three are distributed to all students. Finally, in Spring 2008 an undergraduate journal for political science research, ent60246, made its debut.', 'question': 'How many student news papers are found at Notre Dame?', 'answers': {'text': ['three'], 'answer_start': [126]}}\n",
            "{'id': '5733bf84d058e614000b61c1', 'title': 'University_of_Notre_Dame', 'context': 'As at most other universities, ent88716 students run a number of news media outlets. The nine student-run outlets include three newspapers, both a radio and television station, and several magazines and journals. Begun as a one-page journal in September 1876, the ent52014 magazine is issued twice monthly and claims to be the oldest continuous collegiate publication in ent70996. The other magazine, ent19781, is released twice a year and focuses on student literature and artwork. The ent46964 yearbook is published annually. The newspapers have varying publication interests, with ent60663 published daily and mainly reporting university and other news, and staffed by students from both ent4003 and ent85563. Unlike ent52014 and The ent46964, ent60663 is an independent publication and does not have a faculty advisor or any editorial oversight from the ent11889. In 1987, when some students believed that ent60663 began to show a conservative bias, a liberal newspaper, ent15513 was published. Likewise, in 2003, when other students believed that the paper showed a liberal bias, the conservative paper ent10078 went into production. Neither paper is published as often as ent60663; however, all three are distributed to all students. Finally, in Spring 2008 an undergraduate journal for political science research, ent60246, made its debut.', 'question': 'In what year did the student paper Common Sense begin publication at Notre Dame?', 'answers': {'text': ['1987'], 'answer_start': [908]}}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "4hAEMmySWf5b"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "A9xliVZSS4l6"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Anomysation"
      ],
      "metadata": {
        "id": "lTsHBOis2KB9"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "anonymized_train[1]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZuZN11h-s4jC",
        "outputId": "93622e95-bc83-4160-ff03-42f32348106d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'id': '5733be284776f4190066117f',\n",
              " 'title': 'University_of_Notre_Dame',\n",
              " 'context': 'Architecturally, the school has a Catholic character. Atop the Main Building\\'s gold dome is a golden statue of the Virgin Mary. Immediately in front of the Main Building and facing it, is a copper statue of Christ with arms upraised with the legend \"Venite Ad Me Omnes\". Next to the Main Building is the Basilica of the Sacred Heart. Immediately behind the basilica is the Grotto, a Marian place of prayer and reflection. It is a replica of the grotto at Lourdes, France where the Virgin Mary reputedly appeared to Saint Bernadette Soubirous in 1858. At the end of the main drive (and in a direct line that connects through 3 statues and the Gold Dome), is a simple, modern stone statue of Mary.',\n",
              " 'question': 'What is in front of the Notre Dame Main Building?',\n",
              " 'answers': {'text': ['a copper statue of Christ'], 'answer_start': [188]}}"
            ]
          },
          "metadata": {},
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "# Créer des objets Dataset pour chaque partie (train, validation, test)\n",
        "train_dataset = Dataset.from_dict({\n",
        "    'id': [example['id'] for example in anonymized_train],\n",
        "    'title': [example['title'] for example in anonymized_train],\n",
        "    'context': [example['context'] for example in anonymized_train],\n",
        "    'question': [example['question'] for example in anonymized_train],\n",
        "    'answers': [example['answers'] for example in anonymized_train]\n",
        "})\n",
        "\n",
        "val_dataset = Dataset.from_dict({\n",
        "    'id': [example['id'] for example in anonymized_val],\n",
        "    'title': [example['title'] for example in anonymized_val],\n",
        "    'context': [example['context'] for example in anonymized_val],\n",
        "    'question': [example['question'] for example in anonymized_val],\n",
        "    'answers': [example['answers'] for example in anonymized_val]\n",
        "})\n",
        "\n",
        "\n",
        "# Combiner en un DatasetDict\n",
        "final_dataset = DatasetDict({\n",
        "    'train': train_dataset,\n",
        "    'validation': val_dataset,\n",
        "})\n",
        "\n",
        "# Afficher des informations sur le DatasetDict final\n",
        "print(final_dataset)"
      ],
      "metadata": {
        "id": "MbR9wk7-WX59",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "70c44d7e-e93a-4918-b1c9-cfde0952f861"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "DatasetDict({\n",
            "    train: Dataset({\n",
            "        features: ['id', 'title', 'context', 'question', 'answers'],\n",
            "        num_rows: 75608\n",
            "    })\n",
            "    validation: Dataset({\n",
            "        features: ['id', 'title', 'context', 'question', 'answers'],\n",
            "        num_rows: 8663\n",
            "    })\n",
            "})\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# import pickle\n",
        "\n",
        "# with open('used_ids_qa.pkl', 'wb') as fichier:\n",
        "#     pickle.dump(used_ids, fichier)\n",
        "\n",
        "# print(\"Save with succes.\")"
      ],
      "metadata": {
        "id": "r79FwNFknZ-7",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "93ec1cb3-bb07-43f1-945d-c99a3ce21d14"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Save with succes.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "# with open('entity_map_qa.pkl', 'wb') as fichier:\n",
        "#     pickle.dump(entity_mapping, fichier)\n",
        "\n",
        "# print(\"Save with succes.\")"
      ],
      "metadata": {
        "id": "YTXEPLFwnYmD",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "097dd134-0f98-4eda-a8b0-d7e1fe281076"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Save with succes.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "with open('entity_map_qa.pkl', 'rb') as file:\n",
        "    dictionary = pickle.load(file)\n",
        "len(dictionary)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JeExJcQM6lYM",
        "outputId": "e54a249b-147b-4759-f068-394b7a02f348"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "10185"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# final_dataset.save_to_disk(\"qa\")"
      ],
      "metadata": {
        "id": "zGGSVH6mnM5A",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 81,
          "referenced_widgets": [
            "e7be0dc1fa5b409d8fa40d865b90a9ad",
            "010c7484292d459b83e74b1c5722f079",
            "6f65810532ee4784be7412ef8b062110",
            "68f2539fa69f4011b17b27a482b00a31",
            "7abaef36c2ef475a8453188b04de33c1",
            "9783f03db1fe4cebbbb943f2fdbb2cd0",
            "dc8bb0fafde144ce9595739c1f20cecd",
            "bb5b383d0fa24000ba4fd2c5e72de383",
            "0c45f181710145cda0245e92b1399674",
            "47b592e99a81416d81d79065d30187a9",
            "4c702b0cfab14c13b16846e3d69f3472",
            "cc14f0f486de4087b100359327c39473",
            "f9cf7e7d6c734d538bb274bf2bfbb2a9",
            "87b663fa3bf24592bc8cfd1607cb6b88",
            "056a808775524397a76f54d681547306",
            "17caee241c5f465da258249df8acc335",
            "ba940d3a4eb344d9827d3e54b1f081f6",
            "dea3570b32a445309898322147e051da",
            "9514a2c7e1764ea8a433dcfbad9da919",
            "1c599d7729534bfe944e4cbe58776e3c",
            "368e779161794d13a2c5d70c19d4fad7",
            "c5503b27ae0f4a6ea7f524f07b1be731"
          ]
        },
        "outputId": "157f4378-f7c8-4032-ff87-74fa107262d2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Saving the dataset (0/1 shards):   0%|          | 0/75608 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "e7be0dc1fa5b409d8fa40d865b90a9ad"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Saving the dataset (0/1 shards):   0%|          | 0/8663 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "cc14f0f486de4087b100359327c39473"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Question Answering without entity"
      ],
      "metadata": {
        "id": "wE_lvD4-2F-q"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from datasets import Dataset, DatasetDict, load_dataset, load_from_disk\n",
        "anonymized_subset=load_from_disk(\"qa_good\")\n"
      ],
      "metadata": {
        "id": "2h-YRHywnUsE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for i in range(0,5):\n",
        "    print(anonymized_subset['train'][i])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "collapsed": true,
        "id": "0Xn9hbsBwhWk",
        "outputId": "e5f5e004-d520-45c7-cc0c-30ef4fb45c30"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'id': '5733be284776f41900661182', 'title': 'University_of_Notre_Dame', 'context': 'Architecturally, the school has a ent221588 character. Atop ent194266 gold dome is a golden statue of ent79924. Immediately in front of ent123323 and facing it, is a copper statue of Christ with arms upraised with the legend \"Venite Ad Me Omnes\". Next to ent123323 is ent220757. Immediately behind the basilica is the Grotto, a ent108031 place of prayer and reflection. It is a replica of the grotto at ent4627, ent280607 where ent79924 reputedly appeared to ent86140 in 1858. At the end of the main drive (and in a direct line that connects through 3 statues and ent27423), is a simple, modern stone statue of ent12688.', 'question': 'To whom did ent79924 allegedly appear in 1858 in ent4627 ent280607?', 'answers': {'answer_start': [459], 'text': ['ent86140']}}\n",
            "{'id': '5733be284776f4190066117f', 'title': 'University_of_Notre_Dame', 'context': 'Architecturally, the school has a ent209755 character. Atop ent247916 gold dome is a golden statue of ent105054. Immediately in front of ent67377 and facing it, is a copper statue of Christ with arms upraised with the legend \"Venite Ad Me Omnes\". Next to ent67377 is ent46706. Immediately behind the basilica is the Grotto, a ent220409 place of prayer and reflection. It is a replica of the grotto at ent251073, ent224927 where ent105054 reputedly appeared to ent183170 in 1858. At the end of the main drive (and in a direct line that connects through 3 statues and ent136462), is a simple, modern stone statue of ent192356.', 'question': 'What is in front of ent146980?', 'answers': {'answer_start': [164], 'text': ['a copper statue of Christ']}}\n",
            "{'id': '5733be284776f41900661180', 'title': 'University_of_Notre_Dame', 'context': 'Architecturally, the school has a ent93676 character. Atop ent289216 gold dome is a golden statue of ent68466. Immediately in front of ent285133 and facing it, is a copper statue of Christ with arms upraised with the legend \"Venite Ad Me Omnes\". Next to ent285133 is ent182288. Immediately behind the basilica is the Grotto, a ent78755 place of prayer and reflection. It is a replica of the grotto at ent249113, ent154860 where ent68466 reputedly appeared to ent223217 in 1858. At the end of the main drive (and in a direct line that connects through 3 statues and ent209005), is a simple, modern stone statue of ent86434.', 'question': 'ent21984 at ent59357 is beside to which structure?', 'answers': {'answer_start': [254], 'text': ['ent285133']}}\n",
            "{'id': '5733be284776f41900661181', 'title': 'University_of_Notre_Dame', 'context': 'Architecturally, the school has a ent211620 character. Atop ent290591 gold dome is a golden statue of ent242973. Immediately in front of ent163650 and facing it, is a copper statue of Christ with arms upraised with the legend \"Venite Ad Me Omnes\". Next to ent163650 is ent265560. Immediately behind the basilica is the Grotto, a ent246550 place of prayer and reflection. It is a replica of the grotto at ent127621, ent85641 where ent242973 reputedly appeared to ent82706 in 1858. At the end of the main drive (and in a direct line that connects through 3 statues and ent56039), is a simple, modern stone statue of ent119577.', 'question': 'What is the Grotto at ent43664?', 'answers': {'answer_start': [327], 'text': ['a ent246550 place of prayer and reflection']}}\n",
            "{'id': '5733be284776f4190066117e', 'title': 'University_of_Notre_Dame', 'context': 'Architecturally, the school has a ent50978 character. Atop ent233959 gold dome is a golden statue of ent76927. Immediately in front of ent62805 and facing it, is a copper statue of Christ with arms upraised with the legend \"Venite Ad Me Omnes\". Next to ent62805 is ent59292. Immediately behind the basilica is the Grotto, a ent236256 place of prayer and reflection. It is a replica of the grotto at ent123934, ent117276 where ent76927 reputedly appeared to ent75012 in 1858. At the end of the main drive (and in a direct line that connects through 3 statues and ent146455), is a simple, modern stone statue of ent146306.', 'question': 'What sits on top of ent62805 at ent183623?', 'answers': {'answer_start': [82], 'text': ['a golden statue of ent76927']}}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import AutoTokenizer\n",
        "tokenizer = AutoTokenizer.from_pretrained(\"YvanCarre/anonym-gpt2\",use_fast=True)\n",
        "\n",
        "#tokenizer = AutoTokenizer.from_pretrained(\"YvanCarre/gpt2_anonym\",use_fast=True)\n",
        "\n",
        "\n",
        "tokenizer.do_lower_case = True\n",
        "tokenizer.pad_token = tokenizer.eos_token"
      ],
      "metadata": {
        "id": "37L621PAuI9o",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4550924c-e323-4eed-da8a-6615e7fe5de5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:89: UserWarning: \n",
            "The secret `HF_TOKEN` does not exist in your Colab secrets.\n",
            "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n",
            "You will be able to reuse this secret in all of your notebooks.\n",
            "Please note that authentication is recommended but still optional to access public models or datasets.\n",
            "  warnings.warn(\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def preprocess_function(examples):\n",
        "    questions = [q.strip() for q in examples[\"question\"]]\n",
        "    inputs = tokenizer(\n",
        "        questions,\n",
        "        examples[\"context\"],\n",
        "        max_length=512,\n",
        "        truncation=\"only_second\",\n",
        "        return_offsets_mapping=True,\n",
        "        padding=\"max_length\",\n",
        "\n",
        "    )\n",
        "\n",
        "    offset_mapping = inputs.pop(\"offset_mapping\")\n",
        "    answers = examples[\"answers\"]\n",
        "    start_positions = []\n",
        "    end_positions = []\n",
        "\n",
        "    for i, offset in enumerate(offset_mapping):\n",
        "        answer = answers[i]\n",
        "        start_char = answer[\"answer_start\"][0]\n",
        "        end_char = answer[\"answer_start\"][0] + len(answer[\"text\"][0])\n",
        "        sequence_ids = inputs.sequence_ids(i)\n",
        "\n",
        "        idx = 0\n",
        "        while idx < len(sequence_ids) and sequence_ids[idx] != 1:\n",
        "            idx += 1\n",
        "        context_start = idx\n",
        "        while idx < len(sequence_ids) and sequence_ids[idx] == 1:\n",
        "            idx += 1\n",
        "        context_end = idx - 1\n",
        "\n",
        "        if offset[context_start][0] > end_char or offset[context_end][1] < start_char:\n",
        "            start_positions.append(0)\n",
        "            end_positions.append(0)\n",
        "        else:\n",
        "\n",
        "            idx = context_start\n",
        "            while idx <= context_end and offset[idx][0] <= start_char:\n",
        "\n",
        "                idx += 1\n",
        "            start_positions.append(idx - 1)\n",
        "\n",
        "            idx = context_end\n",
        "            while idx >= context_start and offset[idx][1] >= end_char:\n",
        "                idx -= 1\n",
        "            end_positions.append(idx + 1)\n",
        "\n",
        "    inputs[\"start_positions\"] = start_positions\n",
        "    inputs[\"end_positions\"] = end_positions\n",
        "    return inputs"
      ],
      "metadata": {
        "id": "AORHuPh8MYHp"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tokenized_data = anonymized_subset.map(preprocess_function, batched=True)"
      ],
      "metadata": {
        "id": "GK1lPd6Yv3H7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tokenized_data"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wvACIAkWKIsN",
        "outputId": "57c0fcbd-da82-4344-f4c8-0954e7897b65"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "DatasetDict({\n",
              "    train: Dataset({\n",
              "        features: ['id', 'title', 'context', 'question', 'answers', 'input_ids', 'attention_mask', 'start_positions', 'end_positions'],\n",
              "        num_rows: 87599\n",
              "    })\n",
              "    validation: Dataset({\n",
              "        features: ['id', 'title', 'context', 'question', 'answers', 'input_ids', 'attention_mask', 'start_positions', 'end_positions'],\n",
              "        num_rows: 10570\n",
              "    })\n",
              "})"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "len(tokenizer)"
      ],
      "metadata": {
        "id": "INLzFeib3JoO",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "28f40d25-bea7-4196-ffe2-6b61aebaa9c4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "32000"
            ]
          },
          "metadata": {},
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import GPT2LMHeadModel, Trainer, TrainingArguments,GPT2Config\n",
        "from transformers import GPT2Tokenizer, GPT2ForQuestionAnswering, Trainer, TrainingArguments\n",
        "from transformers import AutoModelForQuestionAnswering, TrainingArguments, Trainer\n",
        "\n",
        "model = AutoModelForQuestionAnswering.from_pretrained(\"YvanCarre/anonym-gpt2\")"
      ],
      "metadata": {
        "id": "xhZ48dZ4624R",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c9fabf60-68b8-4f11-fd2b-c6e8479a0f96"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of GPT2ForQuestionAnswering were not initialized from the model checkpoint at YvanCarre/anonym-gpt2 and are newly initialized: ['qa_outputs.bias', 'qa_outputs.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "model.resize_token_embeddings(len(tokenizer))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "StL-wmUCM7zJ",
        "outputId": "7727a978-9f99-4ad9-fb86-15f48a4cfdbb"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Embedding(32000, 768)"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tokenized_data"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "h6OASSjWDrJ2",
        "outputId": "f9c94160-4666-4720-b0b1-f8738a6684bd"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "DatasetDict({\n",
              "    train: Dataset({\n",
              "        features: ['id', 'title', 'context', 'question', 'answers', 'input_ids', 'attention_mask', 'start_positions', 'end_positions'],\n",
              "        num_rows: 87599\n",
              "    })\n",
              "    validation: Dataset({\n",
              "        features: ['id', 'title', 'context', 'question', 'answers', 'input_ids', 'attention_mask', 'start_positions', 'end_positions'],\n",
              "        num_rows: 10570\n",
              "    })\n",
              "})"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "from transformers import Trainer, DataCollatorWithPadding\n",
        "from torch.utils.data import DataLoader  #\n",
        "from transformers import T5Tokenizer, T5ForConditionalGeneration, DataCollatorForSeq2Seq,DataCollatorForLanguageModeling\n",
        "\n",
        "data_collator = DataCollatorWithPadding(tokenizer=tokenizer)\n",
        "#data_collator = DataCollatorForSeq2Seq(tokenizer=tokenizer, model=model)\n",
        "#model.config.use_cache = False\n",
        "\n",
        "training_args = TrainingArguments(\n",
        "    output_dir=\"./results\",\n",
        "    evaluation_strategy=\"steps\",\n",
        "    warmup_steps=1000,\n",
        "    #gradient_accumulation_steps=2,\n",
        "    max_steps=10000,\n",
        "    lr_scheduler_type=\"linear\",\n",
        "    logging_steps=100,\n",
        "    learning_rate=3e-4,\n",
        "    per_device_train_batch_size=4,  # 24 - 8\n",
        "    weight_decay=0.01,\n",
        "    dataloader_num_workers=4,\n",
        "    save_steps=0.1,\n",
        "    load_best_model_at_end=True,\n",
        "    metric_for_best_model='eval_loss',\n",
        "    fp16=True,\n",
        "    #report_to=\"tensorboard\",\n",
        "    eval_steps=0.1,\n",
        "    seed=42,\n",
        "    #remove_unused_columns=False,\n",
        ")\n",
        "\n",
        "\n",
        "trainer = Trainer(\n",
        "    model=model,\n",
        "    args=training_args,\n",
        "    train_dataset=tokenized_data[\"train\"],\n",
        "    eval_dataset=tokenized_data[\"validation\"],\n",
        "    data_collator=data_collator,\n",
        ")\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fdHJ0yO4jIDF",
        "outputId": "6f19a18a-acae-4f9f-b0c3-7e382ccd874e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/transformers/training_args.py:1525: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n",
            "  warnings.warn(\n",
            "max_steps is given, it will override any value given in num_train_epochs\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "trainer.train()"
      ],
      "metadata": {
        "id": "V0Ij0_StVuWn",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 826
        },
        "outputId": "486d2378-0e57-44b3-ee61-71974a9381b5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m The `run_name` is currently set to the same value as `TrainingArguments.output_dir`. If this was not intended, please specify a different run name by setting the `TrainingArguments.run_name` parameter.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mcarre\u001b[0m (\u001b[33mcarre-su\u001b[0m). Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "wandb version 0.18.3 is available!  To upgrade, please run:\n",
              " $ pip install wandb --upgrade"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "Tracking run with wandb version 0.17.7"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "Run data is saved locally in <code>/content/wandb/run-20241015_065250-4loafgr4</code>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "Syncing run <strong><a href='https://wandb.ai/carre-su/huggingface/runs/4loafgr4' target=\"_blank\">./results</a></strong> to <a href='https://wandb.ai/carre-su/huggingface' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/run' target=\"_blank\">docs</a>)<br/>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              " View project at <a href='https://wandb.ai/carre-su/huggingface' target=\"_blank\">https://wandb.ai/carre-su/huggingface</a>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              " View run at <a href='https://wandb.ai/carre-su/huggingface/runs/4loafgr4' target=\"_blank\">https://wandb.ai/carre-su/huggingface/runs/4loafgr4</a>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/lib/python3.10/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock.\n",
            "  self.pid = os.fork()\n",
            "/usr/local/lib/python3.10/dist-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
            "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='5247' max='10000' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [ 5247/10000 1:55:36 < 1:44:45, 0.76 it/s, Epoch 0.48/1]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Step</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1000</td>\n",
              "      <td>2.632700</td>\n",
              "      <td>2.512508</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2000</td>\n",
              "      <td>2.283600</td>\n",
              "      <td>2.189015</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3000</td>\n",
              "      <td>2.066300</td>\n",
              "      <td>2.101694</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4000</td>\n",
              "      <td>2.054400</td>\n",
              "      <td>2.006934</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5000</td>\n",
              "      <td>2.039000</td>\n",
              "      <td>1.944141</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Network error resolved after 0:01:28.187385, resuming normal operation.\n",
            "/usr/local/lib/python3.10/dist-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
            "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
            "/usr/lib/python3.10/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock.\n",
            "  self.pid = os.fork()\n",
            "/usr/local/lib/python3.10/dist-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
            "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
            "/usr/lib/python3.10/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock.\n",
            "  self.pid = os.fork()\n",
            "/usr/lib/python3.10/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock.\n",
            "  self.pid = os.fork()\n",
            "/usr/local/lib/python3.10/dist-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
            "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
            "/usr/lib/python3.10/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock.\n",
            "  self.pid = os.fork()\n",
            "/usr/local/lib/python3.10/dist-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
            "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
            "/usr/lib/python3.10/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock.\n",
            "  self.pid = os.fork()\n",
            "/usr/local/lib/python3.10/dist-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
            "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "trainer.evaluate()"
      ],
      "metadata": {
        "id": "SaU9JVTL5Nz_",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 214
        },
        "outputId": "b20144c3-17bb-42ed-81cc-ba8d740b5c3f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/lib/python3.10/multiprocessing/popen_fork.py:66: RuntimeWarning: os.fork() was called. os.fork() is incompatible with multithreaded code, and JAX is multithreaded, so this will likely lead to a deadlock.\n",
            "  self.pid = os.fork()\n",
            "/usr/local/lib/python3.10/dist-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
            "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='661' max='661' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [661/661 03:10]\n",
              "    </div>\n",
              "    "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'eval_loss': 1.7574889659881592,\n",
              " 'eval_runtime': 190.4695,\n",
              " 'eval_samples_per_second': 55.494,\n",
              " 'eval_steps_per_second': 3.47,\n",
              " 'epoch': 0.639269406392694}"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Test"
      ],
      "metadata": {
        "id": "8B-QxUwB3kSY"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import GPT2Tokenizer, GPT2ForQuestionAnswering\n",
        "import torch\n",
        "\n",
        "def generate_answer(question, context):\n",
        "    inputs = tokenizer.encode_plus(question, context, return_tensors=\"pt\", max_length=384, truncation=True, padding=\"max_length\")\n",
        "\n",
        "    ## Move input tensors to the same device as the model\n",
        "    inputs = {k: v.to(model.device) for k, v in inputs.items()}  # Add this line\n",
        "\n",
        "    with torch.no_grad():\n",
        "        outputs = model(**inputs)\n",
        "    answer_start = torch.argmax(outputs.start_logits)\n",
        "    answer_end = torch.argmax(outputs.end_logits) + 1\n",
        "\n",
        "    ## Use the 'decode' method directly on the tokenizer\n",
        "    answer = tokenizer.decode(inputs[\"input_ids\"][0][answer_start:answer_end])\n",
        "    return answer"
      ],
      "metadata": {
        "id": "DBpswCEkmLhc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "context = \"The Eiffel Tower is a wrought-iron lattice tower on the Champ de Mars in Paris, France. It is named after the engineer Gustave Eiffel, whose company designed and built the tower. Constructed from 1887 to 1889 as the entrance arch to the 1889 World's Fair, it was initially criticized by some of France's leading artists and intellectuals for its design, but it has become a global cultural icon of France and one of the most recognizable structures in the world.\"\n",
        "\n",
        "questions = [\n",
        "    \"Who designed the Eiffel Tower?\",\n",
        "    \"When was the Eiffel Tower built?\",\n",
        "    \"Where is the Eiffel Tower located?\",\n",
        "    \"What was the initial reaction to the Eiffel Tower?\",\n",
        "    \"What is the Eiffel Tower made of?\"\n",
        "]\n",
        "\n",
        "\n",
        "for question in questions:\n",
        "    answer = generate_answer(question, context)\n",
        "    print(f\"Question: {question}\")\n",
        "    print(f\"Answer: {answer}\\n\")"
      ],
      "metadata": {
        "id": "Jbv05csIMkDC",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e5646a88-9bcc-4882-fd46-b2af3ae3d839"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Question: Who designed the Eiffel Tower?\n",
            "Answer:  Gustave E\n",
            "\n",
            "Question: When was the Eiffel Tower built?\n",
            "Answer: 1887 to 1889\n",
            "\n",
            "Question: Where is the Eiffel Tower located?\n",
            "Answer:  Champ de Mars\n",
            "\n",
            "Question: What was the initial reaction to the Eiffel Tower?\n",
            "Answer:  criticized by some of France's leading artists and intellectuals for its design\n",
            "\n",
            "Question: What is the Eiffel Tower made of?\n",
            "Answer:  wrought-iron lattice tower\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "idx=85\n",
        "\n",
        "test_question = anonymized_subset['validation'][idx]['question']\n",
        "test_context = anonymized_subset['validation'][idx]['context']\n",
        "generated_answer = generate_answer(test_question, test_context)\n",
        "print(f\"Generated answer: {generated_answer}\")\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e50c184f-8128-4a8b-8772-29dd3a6e6c0b",
        "id": "__6H0PeHGzvh"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Generated answer: ent54956\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "anonymized_subset['validation'][idx]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9ab0f78a-4671-4b44-d6f6-420e5766244e",
        "id": "dFXxGbXfGzvj"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'id': '56beaf5e3aeaaa14008c91fd',\n",
              " 'title': 'Super_Bowl_50',\n",
              " 'context': 'ent54956 broadcast ent136160 in the ent249541, and charged an average of $5 million for a 30-second commercial during the game. The ent136160 halftime show was headlined by the ent116144 rock group ent217509 with special guest performers ent130408 and ent114873, who headlined the ent286281 and ent299233 halftime shows, respectively. It was the third-most watched ent254499. broadcast ever.',\n",
              " 'question': 'Who was the broadcaster for ent136160 in ent185589?',\n",
              " 'answers': {'answer_start': [0, 0, 0],\n",
              "  'text': ['ent54956', 'ent54956', 'ent54956']}}"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Evaluate"
      ],
      "metadata": {
        "id": "EyPvgbng3mN5"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!rm -rf ./results"
      ],
      "metadata": {
        "id": "1EJYz-YWukAA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "# Reverse the dictionary to create a lookup for deanonymization\n",
        "def deanonymize_list(text, data):\n",
        "    reverse_data = {v: k for k, v in data.items()}\n",
        "\n",
        "    # Split the input text into words, preserving punctuation\n",
        "    words = text.split()\n",
        "\n",
        "    # Replace identifiers with corresponding keys, preserving punctuation\n",
        "    deanonymized_words = []\n",
        "    for word in words:\n",
        "        # Remove punctuation for lookup but keep it for the output\n",
        "        stripped_word = word.strip(',.')\n",
        "\n",
        "        # Get the deanonymized word, or keep the original word if not found\n",
        "        deanonymized_word = reverse_data.get(stripped_word, stripped_word)\n",
        "\n",
        "        # Reattach any stripped punctuation\n",
        "        if stripped_word != deanonymized_word:  # It was an entity that was replaced\n",
        "            deanonymized_word += word[len(stripped_word):]  # Add original punctuation\n",
        "\n",
        "        deanonymized_words.append(deanonymized_word)\n",
        "\n",
        "    # Join the replaced words back into a single string\n",
        "    return ' '.join(deanonymized_words)\n",
        "\n",
        "\n",
        "def deanonymize_entity_string(entity_string, data):\n",
        "    reverse_data = {v: k for k, v in data.items()}\n",
        "\n",
        "    # Split the input string by spaces to handle identifiers\n",
        "    words = entity_string.split()\n",
        "\n",
        "    # Replace identifiers with corresponding keys, preserving punctuation\n",
        "    deanonymized_words = []\n",
        "    for word in words:\n",
        "        # Remove punctuation for lookup but keep it for the output\n",
        "        stripped_word = word.strip(',.')\n",
        "\n",
        "        # Get the deanonymized word or keep the original if not found\n",
        "        deanonymized_word = reverse_data.get(stripped_word, stripped_word)\n",
        "\n",
        "        # Reattach any stripped punctuation\n",
        "        if stripped_word != deanonymized_word:  # It was an entity that was replaced\n",
        "            deanonymized_word += word[len(stripped_word):]  # Add original punctuation\n",
        "\n",
        "        deanonymized_words.append(deanonymized_word)\n",
        "\n",
        "    # Join the replaced words back into a single string\n",
        "    return ' '.join(deanonymized_words)"
      ],
      "metadata": {
        "id": "wZ8A0bNujFgJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import pickle\n",
        "with open('all_mappings2.pkl', 'rb') as file:\n",
        "    entity_mapping = pickle.load(file)\n",
        "len(entity_mapping)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TcJG9qqpqaZ0",
        "outputId": "fd1364ec-624e-4a03-fcbd-fb8ec4dc7c4e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "10570"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from tqdm import tqdm\n",
        "\n",
        "\n",
        "def evaluate_model_with_rouge(model, tokenizer, dataset):\n",
        "    exact_match_total = 0\n",
        "    f1_total = 0\n",
        "\n",
        "    num_samples = len(dataset['validation'])\n",
        "    for i in tqdm(range(num_samples)):\n",
        "        question = dataset['validation'][i]['question']\n",
        "        context = dataset['validation'][i]['context']\n",
        "        ground_truths = dataset['validation'][i]['answers']['text']\n",
        "\n",
        "        generated_answer = generate_answer(question, context)\n",
        "\n",
        "        exact_match_scores = [int(deanonymize_entity_string(generated_answer,entity_mapping[i]).strip() == deanonymize_entity_string(truth,entity_mapping[i]).strip()) for truth in ground_truths]\n",
        "        exact_match_total += max(exact_match_scores)\n",
        "\n",
        "        f1_scores = []\n",
        "\n",
        "        for truth in ground_truths:\n",
        "            generated_tokens = generated_answer.split()\n",
        "            ground_truth_tokens = truth.split()\n",
        "\n",
        "            common_tokens = set(generated_tokens) & set(ground_truth_tokens)\n",
        "            precision = len(common_tokens) / len(generated_tokens) if generated_tokens else 0\n",
        "            recall = len(common_tokens) / len(ground_truth_tokens) if ground_truth_tokens else 0\n",
        "            f1 = (2 * precision * recall) / (precision + recall) if (precision + recall) > 0 else 0\n",
        "\n",
        "            f1_scores.append(f1)\n",
        "\n",
        "        f1_total += max(f1_scores)\n",
        "\n",
        "    avg_exact_match = exact_match_total / num_samples\n",
        "    avg_f1_score = f1_total / num_samples\n",
        "\n",
        "    return avg_exact_match, avg_f1_score"
      ],
      "metadata": {
        "id": "IIhZBO6_GycZ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "avg_exact_match, avg_f1_score= evaluate_model_with_rouge(model, tokenizer, anonymized_subset)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xPeD22N7gtUU",
        "outputId": "8e77b59a-49b9-4d7c-c86e-0483324fc882"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 10570/10570 [09:42<00:00, 18.14it/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(f\"Exact Match: {avg_exact_match:.2f}\")\n",
        "print(f\"F1 Score: {avg_f1_score:.2f}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "v0X1yaWRSMhO",
        "outputId": "dbe00b9b-ea67-4c48-c774-a192051a816b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Exact Match: 0.54\n",
            "F1 Score: 0.61\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "GMCzuTtJSNkC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "FypyGf4wSNhV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(f\"Exact Match: {avg_exact_match:.2f}\")\n",
        "print(f\"F1 Score: {avg_f1_score:.2f}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tEHCPWElsNKD",
        "outputId": "3e55c4e5-7f84-4c23-faeb-c01b006ad919"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Exact Match: 0.54\n",
            "F1 Score: 0.61\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!rm -rf ./results"
      ],
      "metadata": {
        "id": "jvc84gZanEsh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "avg_exact_match, avg_f1_score, avg_rouge_1, avg_rouge_2, avg_rouge_l = evaluate_model_with_rouge(model, tokenizer, anonymized_subset)\n",
        "print(f\"Exact Match: {avg_exact_match:.2f}\")\n",
        "print(f\"F1 Score: {avg_f1_score:.2f}\")\n",
        "print(f\"ROUGE-1: {avg_rouge_1:.2f}\")\n",
        "print(f\"ROUGE-2: {avg_rouge_2:.2f}\")\n",
        "print(f\"ROUGE-L: {avg_rouge_l:.2f}\")"
      ],
      "metadata": {
        "id": "FciuwTsNI5gE",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "651b6e5a-b79e-4f2b-d421-49ef8c919822"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 8663/8663 [05:07<00:00, 28.21it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Exact Match: 0.55\n",
            "F1 Score: 0.66\n",
            "ROUGE-1: 0.67\n",
            "ROUGE-2: 0.35\n",
            "ROUGE-L: 0.67\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ]
    }
  ]
}