run_mmbt_masked_text_eval.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "run_mmbt_masked_text_eval.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "0ca7df64ca8244e98cb0d6724dd8d962": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_3a69c490d12043dc8180faeaac5e8a14",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_49ec20fb08194d999f84f2e96cfa4d37",
              "IPY_MODEL_5d34f1cb39304343aca9417601cdbc74"
            ]
          }
        },
        "3a69c490d12043dc8180faeaac5e8a14": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "49ec20fb08194d999f84f2e96cfa4d37": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_303893517dd542a7a83901690fa4e185",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 433,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 433,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1336fc1965614018988035bb49c9f415"
          }
        },
        "5d34f1cb39304343aca9417601cdbc74": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_cb6d348fa7e74b98887b8101cb8717e8",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 433/433 [00:03&lt;00:00, 143B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ebde45d6d4984df690a87507bed104fb"
          }
        },
        "303893517dd542a7a83901690fa4e185": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1336fc1965614018988035bb49c9f415": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "cb6d348fa7e74b98887b8101cb8717e8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ebde45d6d4984df690a87507bed104fb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5e74c0b7941147f082b8d6ad1555c13f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_b55c00e83f69407dbdfd2d3a3d66a0c5",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_d8b7de4933b54feeb35cb4d5959b0c43",
              "IPY_MODEL_5723b866327a48129eb27fdcf50cbfb5"
            ]
          }
        },
        "b55c00e83f69407dbdfd2d3a3d66a0c5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d8b7de4933b54feeb35cb4d5959b0c43": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_5e04e3d481ea4a779714fc708dae4ee9",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 231508,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 231508,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b46d09c599f44eecb2fbc5345f61ff07"
          }
        },
        "5723b866327a48129eb27fdcf50cbfb5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_16e56ec36d584b4ea02080c9c4166ad0",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 232k/232k [00:02&lt;00:00, 89.4kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_20b9b113fc43423badbd4f7c5666be6d"
          }
        },
        "5e04e3d481ea4a779714fc708dae4ee9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b46d09c599f44eecb2fbc5345f61ff07": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "16e56ec36d584b4ea02080c9c4166ad0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "20b9b113fc43423badbd4f7c5666be6d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a7d5d39ca3ab49f0bd108d4f3361cde7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_04daa824e2194f99a6a5a67ac8a629f6",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_5768962e41d942e3853a0105908be043",
              "IPY_MODEL_c0fde0bdf3b148638e6e923c5088af91"
            ]
          }
        },
        "04daa824e2194f99a6a5a67ac8a629f6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5768962e41d942e3853a0105908be043": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_c2740158e8324362aeb8cc4e12b8f837",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 466062,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 466062,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_03b16b3017774c20b78a2b406527a975"
          }
        },
        "c0fde0bdf3b148638e6e923c5088af91": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_79630b783e7743adbcd73d06a7c043db",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 466k/466k [00:01&lt;00:00, 242kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1f58a09026f1471db9b96b5e948b4fc2"
          }
        },
        "c2740158e8324362aeb8cc4e12b8f837": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "03b16b3017774c20b78a2b406527a975": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "79630b783e7743adbcd73d06a7c043db": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1f58a09026f1471db9b96b5e948b4fc2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7ebb047a1b634acfa0e76d1596de486f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_51626e10a65b4dbf9015c7f049f89434",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_19b3d2fd1d0b4bcc8b9fc97814cc9911",
              "IPY_MODEL_712806652a5b44fa87e12b37e867cf08"
            ]
          }
        },
        "51626e10a65b4dbf9015c7f049f89434": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "19b3d2fd1d0b4bcc8b9fc97814cc9911": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_beb393982e294193b770081a6c22fea3",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 28,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 28,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_433a139208af4c338b1459b61c6dd364"
          }
        },
        "712806652a5b44fa87e12b37e867cf08": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_dbed6565e1544740be34bbbb1217fa77",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 28.0/28.0 [00:00&lt;00:00, 38.4B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_af89355512b24ffb9c26ca6f1b29fd92"
          }
        },
        "beb393982e294193b770081a6c22fea3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "433a139208af4c338b1459b61c6dd364": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "dbed6565e1544740be34bbbb1217fa77": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "af89355512b24ffb9c26ca6f1b29fd92": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1bb55d4270c74c538d50e1a50a99c178": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_505703cb2ad240f2ac76c35d11b39e86",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_23b7a2c675564700abef9d95f9578f85",
              "IPY_MODEL_c88b55a1eefe4a7188d2c663ea6fbde1"
            ]
          }
        },
        "505703cb2ad240f2ac76c35d11b39e86": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "23b7a2c675564700abef9d95f9578f85": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_7018cdc3a08c42b986e011718b83b366",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 440473133,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 440473133,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5862080830654d67ae06f0baae6953b4"
          }
        },
        "c88b55a1eefe4a7188d2c663ea6fbde1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_ce07402385ec410aa7973f41822ab319",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 440M/440M [00:08&lt;00:00, 51.3MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9d3d4f858f4240659710f5ad63a34960"
          }
        },
        "7018cdc3a08c42b986e011718b83b366": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5862080830654d67ae06f0baae6953b4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ce07402385ec410aa7973f41822ab319": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9d3d4f858f4240659710f5ad63a34960": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/AxelAllen/Pre-trained-Multimodal-Text-Image-Classifier-in-a-Sparse-Data-Application/blob/master/run_mmbt_masked_text_eval.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dOnHITFCd61k"
      },
      "source": [
        "# Evaluating MMBT on Masked Text Test Partition\n",
        "\n",
        "This notebook shows the end-to-end pipeline for fine-tuning pre-trained MMBT model for multimodal (text and image) classification on our dataset.\n",
        "\n",
        "Parts of this pipeline are adapted from the\n",
        "Huggingface `run_mmimdb.py` script to execute the MMBT model. This code can\n",
        "be accessed [here.](https://github.com/huggingface/transformers/blob/8ea412a86faa8e9edeeb6b5c46b08def06aa03ea/examples/research_projects/mm-imdb/run_mmimdb.py#L305). \n",
        "\n",
        "The code is slightly modified from `run_mmbt.ipynb` notebook to evaluate trained MMBT models on test data with all the text inputs masked. i.e. to test how multimodally fine-tuned MMBT perform at image classification."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2CArCZbzeM5J"
      },
      "source": [
        "## Skip unless on Google Colab"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pG3qBIrWMnyJ",
        "outputId": "213356d5-ef94-4539-abe8-719a721682f7"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "id": "Y8fISAPBm1vW",
        "outputId": "5f231c8b-dfae-424e-f724-dc5ab702a7d2"
      },
      "source": [
        "%cd /content/drive/MyDrive/LAP_MMBT\n",
        "%pwd"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/drive/MyDrive/LAP_MMBT\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'/content/drive/MyDrive/LAP_MMBT'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VK2BAFISn86n"
      },
      "source": [
        "Before running the cell below, make sure to select 'GPU' runtime type"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DfB8X3Ldn7Ci",
        "outputId": "df2cf4be-09b4-4dfe-c86d-8a2e34895dad"
      },
      "source": [
        "import torch\n",
        "\n",
        "# If there's a GPU available...\n",
        "if torch.cuda.is_available():    \n",
        "\n",
        "    # Tell PyTorch to use the GPU.    \n",
        "    device = torch.device(\"cuda\")\n",
        "\n",
        "    print('There are %d GPU(s) available.' % torch.cuda.device_count())\n",
        "\n",
        "    print('We will use the GPU:', torch.cuda.get_device_name(0))\n",
        "\n",
        "# If not...\n",
        "else:\n",
        "    print('No GPU available, using the CPU instead.')\n",
        "    device = torch.device(\"cpu\")"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "There are 1 GPU(s) available.\n",
            "We will use the GPU: Tesla T4\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FcFSbljwoMmO"
      },
      "source": [
        "## Install Huggingface Library\n",
        "\n",
        "These should have been installed during your environment set-up; you only need to run these cells in Google Colab."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wxskHmKgoJCP",
        "outputId": "17e3cb3b-77ae-4a63-dc72-280bcd1b3130"
      },
      "source": [
        "!pip install transformers"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting transformers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/81/91/61d69d58a1af1bd81d9ca9d62c90a6de3ab80d77f27c5df65d9a2c1f5626/transformers-4.5.0-py3-none-any.whl (2.1MB)\n",
            "\u001b[K     |████████████████████████████████| 2.2MB 7.9MB/s \n",
            "\u001b[?25hRequirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from transformers) (3.8.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
            "Collecting tokenizers<0.11,>=0.10.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ae/04/5b870f26a858552025a62f1649c20d29d2672c02ff3c3fb4c688ca46467a/tokenizers-0.10.2-cp37-cp37m-manylinux2010_x86_64.whl (3.3MB)\n",
            "\u001b[K     |████████████████████████████████| 3.3MB 47.9MB/s \n",
            "\u001b[?25hRequirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers) (20.9)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.19.5)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.41.1)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.0.12)\n",
            "Collecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/08/cd/342e584ee544d044fb573ae697404ce22ede086c9e87ce5960772084cad0/sacremoses-0.0.44.tar.gz (862kB)\n",
            "\u001b[K     |████████████████████████████████| 870kB 42.6MB/s \n",
            "\u001b[?25hRequirement already satisfied: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.7.4.3)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.4.1)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2020.12.5)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.0.1)\n",
            "Building wheels for collected packages: sacremoses\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.44-cp37-none-any.whl size=886084 sha256=a80b2f231ea1326a1e081ea977f824be1cab04085f26d85ef3c83e5c64faf061\n",
            "  Stored in directory: /root/.cache/pip/wheels/3e/fb/c0/13ab4d63d537658f448366744654323077c4d90069b6512f3c\n",
            "Successfully built sacremoses\n",
            "Installing collected packages: tokenizers, sacremoses, transformers\n",
            "Successfully installed sacremoses-0.0.44 tokenizers-0.10.2 transformers-4.5.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lww2Jo_TGiXx"
      },
      "source": [
        "# Data directories and file paths\n",
        "\n",
        "Paths to data files options are provide in the following cell. Uncomment the train/val/test partitions according to the desired labeling scheme:\n",
        "\n",
        "- filenames with 'major' are labeled with the 'major' metadata column text\n",
        "- filenames without  'major'are labeled with the 'impression' metadata column text\n",
        "- filenames with 'multi' are labeled for multiclass classification\n",
        "- filename without 'multi' are labeled for binary classification\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "y7d6Ny-lGef1"
      },
      "source": [
        "#train_file = \"image_labels_impression_frontal_train.jsonl\"\n",
        "#val_file = \"image_labels_impression_frontal_val.jsonl\"\n",
        "#test_file = \"image_labels_impression_frontal_test.jsonl\"\n",
        "\n",
        "train_file = \"image_multi_labels_major_findings_frontal_train.jsonl\"\n",
        "val_file = \"image_multi_labels_major_findings_frontal_val.jsonl\"\n",
        "test_file = \"image_multi_labels_major_findings_frontal_test.jsonl\"\n",
        "\n",
        "\n",
        "#train_file = \"image_labels_major_findings_frontal_train.jsonl\"\n",
        "#val_file = \"image_labels_major_findings_frontal_val.jsonl\"\n",
        "#test_file = \"image_labels_major_findings_frontal_test.jsonl\"\n",
        "\n",
        "\n",
        "#train_file = \"image_labels_findings_frontal_train.jsonl\"\n",
        "#val_file = \"image_labels_findings_frontal_val.jsonl\"\n",
        "#test_file = \"image_labels_findings_frontal_test.jsonl\""
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rodBL17AuSJ6"
      },
      "source": [
        "## Import Required Modules"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dNP7dJV2hTKG"
      },
      "source": [
        "from textBert_utils import set_seed\n",
        "from MMBT.image import ImageEncoderDenseNet\n",
        "from MMBT.mmbt_config import MMBTConfig\n",
        "from MMBT.mmbt import MMBTForClassification"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sqM0DScsmCnX"
      },
      "source": [
        "from MMBT.mmbt_utils import JsonlDataset, get_image_transforms, get_labels, load_examples, collate_fn, get_multiclass_labels, collate_fn_mask_all_text, get_multiclass_criterion"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0wUXVNbSoBVo"
      },
      "source": [
        "import argparse"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Th_vPKXcpn4R"
      },
      "source": [
        "import glob\n",
        "import logging\n",
        "import random\n",
        "import json\n",
        "import os\n",
        "from collections import Counter\n",
        "import numpy as np\n",
        "from matplotlib.pyplot import imshow\n",
        "\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torchvision\n",
        "import torchvision.transforms as transforms\n",
        "from PIL import Image\n",
        "from torch.utils.data import Dataset\n",
        "from torch.utils.data import DataLoader, RandomSampler, SequentialSampler"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lWUF76E4y1CC"
      },
      "source": [
        "from sklearn.metrics import accuracy_score, f1_score\n",
        "from tqdm import tqdm, trange\n",
        "\n",
        "from transformers import (\n",
        "    WEIGHTS_NAME,\n",
        "    AdamW,\n",
        "    AutoConfig,\n",
        "    AutoModel,\n",
        "    AutoTokenizer,\n",
        "    get_linear_schedule_with_warmup,\n",
        ")\n",
        "\n",
        "try:\n",
        "    from torch.utils.tensorboard import SummaryWriter\n",
        "except ImportError:\n",
        "    from tensorboardX import SummaryWriter"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yfYeENaFDbmO"
      },
      "source": [
        "from transformers import (\n",
        "    WEIGHTS_NAME,\n",
        "    AdamW,\n",
        "    AutoConfig,\n",
        "    AutoModel,\n",
        "    AutoTokenizer,\n",
        "    get_linear_schedule_with_warmup,\n",
        ")"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aWgdU7lBnyFd"
      },
      "source": [
        "# Set-up Experiment Hyperparameters and Arguments\n",
        "\n",
        "Specify the training, validation, and test files to run the experiment on. The default here is running the model on 'impression' texts.  \n",
        "\n",
        "To re-make the training, validation, and test data, please refer to the information in the **data/** directory.  \n",
        "\n",
        "Change the default values in the parser.add_argument function for the hyperparameters that you want to specify in the following cell or use the default option.  \n",
        "\n",
        "For multiple experiment runs, please make sure to change the `output_dir` argument so that new results don't overwrit existing ones.\n",
        "\n",
        "The arguments specified here are the same as in the `run_mmimdb.py` file \n",
        "in the [Huggingface example implementation of MMBT.](https://github.com/huggingface/transformers/blob/8ea412a86faa8e9edeeb6b5c46b08def06aa03ea/examples/research_projects/mm-imdb/run_mmimdb.py#L305)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "urly6ofboEmU"
      },
      "source": [
        "parser = argparse.ArgumentParser(f'Project Hyperparameters and Other Configurations Argument Parser')\n",
        "\n",
        "parser = argparse.ArgumentParser()\n",
        "\n",
        "# Required parameters\n",
        "parser.add_argument(\n",
        "    \"--data_dir\",\n",
        "    default=\"data/json\",\n",
        "    type=str,\n",
        "    help=\"The input data dir. Should contain the .jsonl files.\",\n",
        ")\n",
        "parser.add_argument(\n",
        "    \"--model_name\",\n",
        "    default=\"bert-base-uncased\",\n",
        "    type=str,\n",
        "    help=\"model identifier from huggingface.co/models\",\n",
        ")\n",
        "parser.add_argument(\n",
        "    \"--output_dir\",\n",
        "    default=\"mmbt_output_findings_multi_major\",\n",
        "    type=str,\n",
        "    help=\"The output directory where the model predictions and checkpoints will be written.\",\n",
        ")\n",
        "\n",
        "    \n",
        "parser.add_argument(\n",
        "    \"--config_name\", default=\"bert-base-uncased\", type=str, help=\"Pretrained config name if not the same as model_name\"\n",
        ")\n",
        "parser.add_argument(\n",
        "    \"--tokenizer_name\",\n",
        "    default=\"bert-base-uncased\",\n",
        "    type=str,\n",
        "    help=\"Pretrained tokenizer name or path if not the same as model_name\",\n",
        ")\n",
        "\n",
        "parser.add_argument(\"--train_batch_size\", default=16, type=int, help=\"Batch size for training.\")\n",
        "parser.add_argument(\n",
        "    \"--eval_batch_size\", default=16, type=int, help=\"Batch size for evaluation.\"\n",
        ")\n",
        "parser.add_argument(\n",
        "    \"--max_seq_length\",\n",
        "    default=300,\n",
        "    type=int,\n",
        "    help=\"The maximum total input sequence length after tokenization. Sequences longer \"\n",
        "    \"than this will be truncated, sequences shorter will be padded.\",\n",
        ")\n",
        "parser.add_argument(\n",
        "    \"--num_image_embeds\", default=3, type=int, help=\"Number of Image Embeddings from the Image Encoder\"\n",
        ")\n",
        "parser.add_argument(\"--do_train\", default=True, type=bool, help=\"Whether to run training.\")\n",
        "parser.add_argument(\"--do_eval\", default=True, type=bool, help=\"Whether to run eval on the dev set.\")\n",
        "parser.add_argument(\n",
        "    \"--evaluate_during_training\", default=True, type=bool, help=\"Run evaluation during training at each logging step.\"\n",
        ")\n",
        "\n",
        "\n",
        "parser.add_argument(\n",
        "    \"--gradient_accumulation_steps\",\n",
        "    type=int,\n",
        "    default=1,\n",
        "    help=\"Number of updates steps to accumulate before performing a backward/update pass.\",\n",
        ")\n",
        "parser.add_argument(\"--learning_rate\", default=5e-5, type=float, help=\"The initial learning rate for Adam.\")\n",
        "parser.add_argument(\"--weight_decay\", default=0.1, type=float, help=\"Weight deay if we apply some.\")\n",
        "parser.add_argument(\"--adam_epsilon\", default=1e-8, type=float, help=\"Epsilon for Adam optimizer.\")\n",
        "parser.add_argument(\"--max_grad_norm\", default=1.0, type=float, help=\"Max gradient norm.\")\n",
        "parser.add_argument(\n",
        "    \"--num_train_epochs\", default=6.0, type=float, help=\"Total number of training epochs to perform.\"\n",
        ")\n",
        "parser.add_argument(\"--patience\", default=5, type=int, help=\"Patience for Early Stopping.\")\n",
        "parser.add_argument(\n",
        "    \"--max_steps\",\n",
        "    default=-1,\n",
        "    type=int,\n",
        "    help=\"If > 0: set total number of training steps to perform. Override num_train_epochs.\",\n",
        ")\n",
        "parser.add_argument(\"--warmup_steps\", default=0, type=int, help=\"Linear warmup over warmup_steps.\")\n",
        "\n",
        "parser.add_argument(\"--logging_steps\", type=int, default=25, help=\"Log every X updates steps.\")\n",
        "parser.add_argument(\"--save_steps\", type=int, default=25, help=\"Save checkpoint every X updates steps.\")\n",
        "parser.add_argument(\n",
        "    \"--eval_all_checkpoints\",\n",
        "    default=True, type=bool,\n",
        "    help=\"Evaluate all checkpoints starting with the same prefix as model_name ending and ending with step number\",\n",
        ")\n",
        "\n",
        "parser.add_argument(\"--num_workers\", type=int, default=8, help=\"number of worker threads for dataloading\")\n",
        "\n",
        "parser.add_argument(\"--seed\", type=int, default=42, help=\"random seed for initialization\")\n",
        "\n",
        "\n",
        "args = parser.parse_args(\"\")\n",
        "\n",
        "# Setup CUDA, GPU & distributed training\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "args.n_gpu = torch.cuda.device_count() if torch.cuda.is_available() else 0\n",
        "args.device = device\n",
        "\n",
        "# for multiclass labeling\n",
        "args.multiclass = True"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YBKm89HqqqO6"
      },
      "source": [
        "# Setup Train/Val/Test filenames\n",
        "args.train_file = train_file\n",
        "args.val_file = val_file\n",
        "args.test_file = test_file"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lKOToFXcKxs1"
      },
      "source": [
        "## Showing a sample from JsonDataset\n",
        "i.e. calling \"\\_\\_getitem\\_\\_\"\n",
        "\n",
        "Note:   \n",
        "image_end_token is the BERT token id for [SEP].   \n",
        "image_start_token is the BERT token id for [CLS]. \n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h_cC7-CQFKyP",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 213,
          "referenced_widgets": [
            "0ca7df64ca8244e98cb0d6724dd8d962",
            "3a69c490d12043dc8180faeaac5e8a14",
            "49ec20fb08194d999f84f2e96cfa4d37",
            "5d34f1cb39304343aca9417601cdbc74",
            "303893517dd542a7a83901690fa4e185",
            "1336fc1965614018988035bb49c9f415",
            "cb6d348fa7e74b98887b8101cb8717e8",
            "ebde45d6d4984df690a87507bed104fb",
            "5e74c0b7941147f082b8d6ad1555c13f",
            "b55c00e83f69407dbdfd2d3a3d66a0c5",
            "d8b7de4933b54feeb35cb4d5959b0c43",
            "5723b866327a48129eb27fdcf50cbfb5",
            "5e04e3d481ea4a779714fc708dae4ee9",
            "b46d09c599f44eecb2fbc5345f61ff07",
            "16e56ec36d584b4ea02080c9c4166ad0",
            "20b9b113fc43423badbd4f7c5666be6d",
            "a7d5d39ca3ab49f0bd108d4f3361cde7",
            "04daa824e2194f99a6a5a67ac8a629f6",
            "5768962e41d942e3853a0105908be043",
            "c0fde0bdf3b148638e6e923c5088af91",
            "c2740158e8324362aeb8cc4e12b8f837",
            "03b16b3017774c20b78a2b406527a975",
            "79630b783e7743adbcd73d06a7c043db",
            "1f58a09026f1471db9b96b5e948b4fc2",
            "7ebb047a1b634acfa0e76d1596de486f",
            "51626e10a65b4dbf9015c7f049f89434",
            "19b3d2fd1d0b4bcc8b9fc97814cc9911",
            "712806652a5b44fa87e12b37e867cf08",
            "beb393982e294193b770081a6c22fea3",
            "433a139208af4c338b1459b61c6dd364",
            "dbed6565e1544740be34bbbb1217fa77",
            "af89355512b24ffb9c26ca6f1b29fd92"
          ]
        },
        "outputId": "db81efae-cd23-4ed9-fa0f-9c44dc2572db"
      },
      "source": [
        "tokenizer = AutoTokenizer.from_pretrained(\n",
        "        args.tokenizer_name if args.tokenizer_name else args.model_name,\n",
        "        do_lower_case=True,\n",
        "        cache_dir=None,\n",
        "    )\n",
        "train_dataset = load_examples(tokenizer, args)"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "0ca7df64ca8244e98cb0d6724dd8d962",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=433.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "5e74c0b7941147f082b8d6ad1555c13f",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=231508.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "a7d5d39ca3ab49f0bd108d4f3361cde7",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=466062.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "7ebb047a1b634acfa0e76d1596de486f",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=28.0, style=ProgressStyle(description_w…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "S80TAkjvJ9ZC",
        "outputId": "ce9f9946-bc01-4fff-d5be-752f6f38241b"
      },
      "source": [
        "train_dataset[0]"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'image': tensor([[[-0.7650, -0.7479, -0.7308,  ..., -0.3541, -0.3369, -0.3198],\n",
              "          [-0.7137, -0.7137, -0.6794,  ..., -0.2171, -0.1828, -0.1999],\n",
              "          [-0.6109, -0.6109, -0.6109,  ..., -0.1143, -0.0801, -0.0801],\n",
              "          ...,\n",
              "          [ 1.8722,  1.9064,  1.9064,  ...,  1.6324,  1.6667,  1.7523],\n",
              "          [ 1.8893,  1.9064,  1.9407,  ...,  1.6153,  1.6838,  1.7523],\n",
              "          [ 1.8722,  1.9064,  1.9407,  ...,  1.6324,  1.7180,  1.7694]],\n",
              " \n",
              "         [[-0.6527, -0.6352, -0.6176,  ..., -0.2325, -0.2150, -0.1975],\n",
              "          [-0.6001, -0.6001, -0.5651,  ..., -0.0924, -0.0574, -0.0749],\n",
              "          [-0.4951, -0.4951, -0.4951,  ...,  0.0126,  0.0476,  0.0476],\n",
              "          ...,\n",
              "          [ 2.0434,  2.0784,  2.0784,  ...,  1.7983,  1.8333,  1.9209],\n",
              "          [ 2.0609,  2.0784,  2.1134,  ...,  1.7808,  1.8508,  1.9209],\n",
              "          [ 2.0434,  2.0784,  2.1134,  ...,  1.7983,  1.8859,  1.9384]],\n",
              " \n",
              "         [[-0.4275, -0.4101, -0.3927,  ..., -0.0092,  0.0082,  0.0256],\n",
              "          [-0.3753, -0.3753, -0.3404,  ...,  0.1302,  0.1651,  0.1476],\n",
              "          [-0.2707, -0.2707, -0.2707,  ...,  0.2348,  0.2696,  0.2696],\n",
              "          ...,\n",
              "          [ 2.2566,  2.2914,  2.2914,  ...,  2.0125,  2.0474,  2.1346],\n",
              "          [ 2.2740,  2.2914,  2.3263,  ...,  1.9951,  2.0648,  2.1346],\n",
              "          [ 2.2566,  2.2914,  2.3263,  ...,  2.0125,  2.0997,  2.1520]]]),\n",
              " 'image_end_token': tensor(102),\n",
              " 'image_start_token': tensor(101),\n",
              " 'label': tensor([0., 1., 0.]),\n",
              " 'sentence': tensor([ 8948,  3961, 23760, 10288,  9739,  5732,  1012,  2053,  2689,  1999,\n",
              "          1996,  2157,  2690, 21833,  6728,  6305,  9031,  1012,  2053, 22038,\n",
              "         20348, 29543,  2015,  2030, 11678,  1012, 21908, 28915,  2024,  4069,\n",
              "         25497,  1012])}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "v_qY3Xe7Owyq"
      },
      "source": [
        "\n",
        "### Evaluating Functions.\n",
        "\n",
        "Experimenting with masking the text input during testing."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hwg0asA2Zm6P"
      },
      "source": [
        "def evaluate(args, model, tokenizer, evaluate=True, test=False, prefix=\"\"):\n",
        "    \n",
        "    if test:\n",
        "        # start a separate tensorboard to track testing eval result\n",
        "        comment = f\"masked_text_test_{args.output_dir}_{args.eval_batch_size}\"\n",
        "        tb_writer = SummaryWriter(comment=comment)\n",
        "\n",
        "    eval_output_dir = args.output_dir\n",
        "    eval_dataset = load_examples(tokenizer, args, evaluate=evaluate, test=test)\n",
        "\n",
        "    if not os.path.exists(eval_output_dir):\n",
        "        os.makedirs(eval_output_dir)\n",
        "\n",
        "    # mask all text input\n",
        "    eval_sampler = SequentialSampler(eval_dataset)\n",
        "    eval_dataloader = DataLoader(\n",
        "        eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size, collate_fn=collate_fn_mask_all_text\n",
        "    )\n",
        "\n",
        "    # Eval!\n",
        "    logger.info(\"***** Running evaluation {} *****\".format(prefix))\n",
        "    logger.info(\"  Num examples = %d\", len(eval_dataset))\n",
        "    logger.info(\"  Batch size = %d\", args.eval_batch_size)\n",
        "    eval_loss = 0.0\n",
        "    nb_eval_steps = 0\n",
        "    preds = []\n",
        "    out_label_ids = []\n",
        "    for batch in tqdm(eval_dataloader, desc=\"Evaluating\"):\n",
        "        model.eval()\n",
        "        batch = tuple(t.to(args.device) for t in batch)\n",
        "\n",
        "        with torch.no_grad():\n",
        "            batch = tuple(t.to(args.device) for t in batch)\n",
        "            labels = batch[5]\n",
        "            input_ids = batch[0]\n",
        "            input_modal = batch[2]\n",
        "            attention_mask = batch[1]\n",
        "            modal_start_tokens = batch[3]\n",
        "            modal_end_tokens = batch[4]\n",
        "            \n",
        "            if args.multiclass:\n",
        "                outputs = model(\n",
        "                    input_modal,\n",
        "                    input_ids=input_ids,\n",
        "                    modal_start_tokens=modal_start_tokens,\n",
        "                    modal_end_tokens=modal_end_tokens,\n",
        "                    attention_mask=attention_mask,\n",
        "                    token_type_ids=None,\n",
        "                    modal_token_type_ids=None,\n",
        "                    position_ids=None,\n",
        "                    modal_position_ids=None,\n",
        "                    head_mask=None,\n",
        "                    inputs_embeds=None,\n",
        "                    labels=None,\n",
        "                    return_dict=True\n",
        "                )\n",
        "            else:\n",
        "                outputs = model(\n",
        "                    input_modal,\n",
        "                    input_ids=input_ids,\n",
        "                    modal_start_tokens=modal_start_tokens,\n",
        "                    modal_end_tokens=modal_end_tokens,\n",
        "                    attention_mask=attention_mask,\n",
        "                    token_type_ids=None,\n",
        "                    modal_token_type_ids=None,\n",
        "                    position_ids=None,\n",
        "                    modal_position_ids=None,\n",
        "                    head_mask=None,\n",
        "                    inputs_embeds=None,\n",
        "                    labels=labels,\n",
        "                    return_dict=True\n",
        "                )\n",
        "            #logits = outputs[0]  # model outputs are always tuple in transformers (see doc)\n",
        "            #tmp_eval_loss = criterion(logits, labels)\n",
        "            logits = outputs.logits\n",
        "            if args.multiclass:\n",
        "                criterion = get_multiclass_criterion(eval_dataset)\n",
        "                tmp_eval_loss = criterion(logits, labels)\n",
        "            else:\n",
        "                tmp_eval_loss = outputs.loss\n",
        "            eval_loss += tmp_eval_loss.mean().item()\n",
        "        nb_eval_steps += 1\n",
        "        # Move logits and labels to CPU\n",
        "        if args.multiclass:\n",
        "            pred = torch.sigmoid(logits).cpu().detach().numpy() > 0.5\n",
        "        else:            \n",
        "            pred = torch.nn.functional.softmax(logits, dim=1).argmax(dim=1).cpu().detach().numpy()\n",
        "        out_label_id = labels.detach().cpu().numpy()\n",
        "        preds.append(pred)\n",
        "        out_label_ids.append(out_label_id)\n",
        "\n",
        "    eval_loss = eval_loss / nb_eval_steps\n",
        "\n",
        "    result = {\"loss\": eval_loss}\n",
        "\n",
        "    if args.multiclass:\n",
        "        tgts = np.vstack(out_label_ids)\n",
        "        preds = np.vstack(preds)\n",
        "        result[\"macro_f1\"] = f1_score(tgts, preds, average=\"macro\")\n",
        "        result[\"micro_f1\"] = f1_score(tgts, preds, average=\"micro\")\n",
        "    else:\n",
        "        preds = [l for sl in preds for l in sl]\n",
        "        out_label_ids = [l for sl in out_label_ids for l in sl]\n",
        "        result[\"accuracy\"] = accuracy_score(out_label_ids, preds)\n",
        "\n",
        "    output_eval_file = os.path.join(eval_output_dir, prefix, \"masked_text_eval_results.txt\")\n",
        "    with open(output_eval_file, \"w\") as writer:\n",
        "        logger.info(\"***** Eval results {} *****\".format(prefix))\n",
        "        for key in sorted(result.keys()):\n",
        "            logger.info(\"  %s = %s\", key, str(result[key]))\n",
        "            writer.write(\"%s = %s\\n\" % (key, str(result[key])))\n",
        "            if test:\n",
        "                tb_writer.add_scalar(f'eval_{key}', result[key], nb_eval_steps)\n",
        "    \n",
        "    if test:\n",
        "        tb_writer.close()\n",
        "\n",
        "\n",
        "    return result\n"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-mrKUcxxcbs3"
      },
      "source": [
        "## Setting up the MMBT Model \n",
        "\n",
        "Set up logging and the MMBT Model. Similar to the text-only model, check points \n",
        "are saved during a similar customizable interval.\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-3-ZxIuRcdfO"
      },
      "source": [
        "# Setup logging\n",
        "logger = logging.getLogger(__name__)\n",
        "if not os.path.exists(args.output_dir):\n",
        "    os.makedirs(args.output_dir)\n",
        "logging.basicConfig(format=\"%(asctime)s - %(levelname)s - %(name)s -   %(message)s\",\n",
        "                    datefmt=\"%m/%d/%Y %H:%M:%S\",\n",
        "                    filename=os.path.join(args.output_dir, f\"masked_text_{os.path.splitext(args.test_file)[0]}_logging.txt\"),\n",
        "                    level=logging.INFO)\n",
        "logger.warning(\"device: %s, n_gpu: %s\",\n",
        "        args.device,\n",
        "        args.n_gpu\n",
        ")\n",
        "# Set the verbosity to info of the Transformers logger (on main process only):\n",
        "\n",
        "# Set seed\n",
        "set_seed(args)"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZDgzsLlcc-Uk",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 66,
          "referenced_widgets": [
            "1bb55d4270c74c538d50e1a50a99c178",
            "505703cb2ad240f2ac76c35d11b39e86",
            "23b7a2c675564700abef9d95f9578f85",
            "c88b55a1eefe4a7188d2c663ea6fbde1",
            "7018cdc3a08c42b986e011718b83b366",
            "5862080830654d67ae06f0baae6953b4",
            "ce07402385ec410aa7973f41822ab319",
            "9d3d4f858f4240659710f5ad63a34960"
          ]
        },
        "outputId": "65e0b5ed-1487-4d85-be5e-7c317ec8839c"
      },
      "source": [
        "# Setup model\n",
        "if args.multiclass:\n",
        "    labels = get_multiclass_labels()\n",
        "    num_labels = len(labels)\n",
        "else:\n",
        "    labels = get_labels()\n",
        "    num_labels = len(labels)\n",
        "transformer_config = AutoConfig.from_pretrained(args.config_name if args.config_name else args.model_name, num_labels=num_labels)\n",
        "tokenizer = AutoTokenizer.from_pretrained(args.output_dir)\n",
        "transformer = AutoModel.from_pretrained(args.model_name, config=transformer_config, cache_dir=None)\n",
        "img_encoder = ImageEncoderDenseNet(num_image_embeds=args.num_image_embeds)\n",
        "multimodal_config = MMBTConfig(transformer, img_encoder, num_labels=num_labels, modal_hidden_size=1024)\n",
        "\n",
        "logger.info(f\"Evaluation parameters: {args}\")"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "1bb55d4270c74c538d50e1a50a99c178",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=440473133.0, style=ProgressStyle(descri…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Se45Ve33_KAr"
      },
      "source": [
        "## Evaluating saved model checkpoints on the Test Set\n",
        "\n",
        "The test set has been modified to only provide the image embeddings tokens and all the text inputs are masked.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vo2zxNJDYbmh",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "73343bef-6628-45be-8027-52b426e06814"
      },
      "source": [
        "%pdb on\n",
        "# Evaluation\n",
        "results = {}\n",
        "if args.do_eval:\n",
        "    checkpoints = [args.output_dir]\n",
        "    if args.eval_all_checkpoints:\n",
        "        checkpoints = list(os.path.dirname(c) \n",
        "        for c in sorted(glob.glob(args.output_dir + \"/**/\" + \n",
        "                                  WEIGHTS_NAME, recursive=False)))\n",
        "        # recursive=False because otherwise the parent diretory gets included\n",
        "        # which is not what we want; only subdirectories\n",
        "\n",
        "    logger.info(\"Evaluate the following checkpoints: %s\", checkpoints)\n",
        "\n",
        "    for checkpoint in checkpoints:\n",
        "        global_step = checkpoint.split(\"-\")[-1] if len(checkpoints) > 1 else \"\"\n",
        "        prefix = checkpoint.split(\"/\")[-1] if checkpoint.find(\"checkpoint\") != -1 else \"\"\n",
        "        model = MMBTForClassification(transformer_config, multimodal_config)\n",
        "        checkpoint = os.path.join(checkpoint, 'pytorch_model.bin')\n",
        "        model.load_state_dict(torch.load(checkpoint))\n",
        "        model.to(args.device)\n",
        "        result = evaluate(args, model, tokenizer, evaluate=True, test=True, prefix=prefix)\n",
        "        result = dict((k + \"_{}\".format(global_step), v) for k, v in result.items())\n",
        "        results.update(result)\n",
        "\n",
        "results.keys()"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Automatic pdb calling has been turned ON\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Evaluating: 100%|██████████| 36/36 [02:40<00:00,  4.45s/it]\n",
            "Evaluating: 100%|██████████| 36/36 [00:10<00:00,  3.54it/s]\n",
            "Evaluating: 100%|██████████| 36/36 [00:10<00:00,  3.55it/s]\n",
            "Evaluating: 100%|██████████| 36/36 [00:10<00:00,  3.49it/s]\n",
            "Evaluating: 100%|██████████| 36/36 [00:10<00:00,  3.50it/s]\n",
            "Evaluating: 100%|██████████| 36/36 [00:10<00:00,  3.45it/s]\n",
            "Evaluating: 100%|██████████| 36/36 [00:10<00:00,  3.46it/s]\n",
            "Evaluating: 100%|██████████| 36/36 [00:10<00:00,  3.38it/s]\n",
            "Evaluating: 100%|██████████| 36/36 [00:10<00:00,  3.43it/s]\n",
            "Evaluating: 100%|██████████| 36/36 [00:10<00:00,  3.48it/s]\n",
            "Evaluating: 100%|██████████| 36/36 [00:10<00:00,  3.44it/s]\n",
            "Evaluating: 100%|██████████| 36/36 [00:10<00:00,  3.42it/s]\n",
            "Evaluating: 100%|██████████| 36/36 [00:10<00:00,  3.39it/s]\n",
            "Evaluating: 100%|██████████| 36/36 [00:10<00:00,  3.45it/s]\n",
            "Evaluating: 100%|██████████| 36/36 [00:10<00:00,  3.45it/s]\n",
            "Evaluating: 100%|██████████| 36/36 [00:10<00:00,  3.45it/s]\n",
            "Evaluating: 100%|██████████| 36/36 [00:10<00:00,  3.41it/s]\n",
            "Evaluating: 100%|██████████| 36/36 [00:10<00:00,  3.37it/s]\n",
            "Evaluating: 100%|██████████| 36/36 [00:10<00:00,  3.42it/s]\n",
            "Evaluating: 100%|██████████| 36/36 [00:10<00:00,  3.41it/s]\n",
            "Evaluating: 100%|██████████| 36/36 [00:10<00:00,  3.41it/s]\n",
            "Evaluating: 100%|██████████| 36/36 [00:10<00:00,  3.37it/s]\n",
            "Evaluating: 100%|██████████| 36/36 [00:10<00:00,  3.40it/s]\n",
            "Evaluating: 100%|██████████| 36/36 [00:10<00:00,  3.40it/s]\n",
            "Evaluating: 100%|██████████| 36/36 [00:10<00:00,  3.41it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "dict_keys(['loss_100', 'macro_f1_100', 'micro_f1_100', 'loss_125', 'macro_f1_125', 'micro_f1_125', 'loss_150', 'macro_f1_150', 'micro_f1_150', 'loss_175', 'macro_f1_175', 'micro_f1_175', 'loss_200', 'macro_f1_200', 'micro_f1_200', 'loss_225', 'macro_f1_225', 'micro_f1_225', 'loss_25', 'macro_f1_25', 'micro_f1_25', 'loss_250', 'macro_f1_250', 'micro_f1_250', 'loss_275', 'macro_f1_275', 'micro_f1_275', 'loss_300', 'macro_f1_300', 'micro_f1_300', 'loss_325', 'macro_f1_325', 'micro_f1_325', 'loss_350', 'macro_f1_350', 'micro_f1_350', 'loss_375', 'macro_f1_375', 'micro_f1_375', 'loss_400', 'macro_f1_400', 'micro_f1_400', 'loss_425', 'macro_f1_425', 'micro_f1_425', 'loss_450', 'macro_f1_450', 'micro_f1_450', 'loss_475', 'macro_f1_475', 'micro_f1_475', 'loss_50', 'macro_f1_50', 'micro_f1_50', 'loss_500', 'macro_f1_500', 'micro_f1_500', 'loss_525', 'macro_f1_525', 'micro_f1_525', 'loss_550', 'macro_f1_550', 'micro_f1_550', 'loss_575', 'macro_f1_575', 'micro_f1_575', 'loss_600', 'macro_f1_600', 'micro_f1_600', 'loss_625', 'macro_f1_625', 'micro_f1_625', 'loss_75', 'macro_f1_75', 'micro_f1_75'])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mu5ge_8Yjg6w",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e455e2d4-d76c-49b4-e5ca-cacc491fe25b"
      },
      "source": [
        "results"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'loss_100': 1.658436440759235,\n",
              " 'loss_125': 1.7200494209925334,\n",
              " 'loss_150': 1.6319334639443293,\n",
              " 'loss_175': 1.91707839568456,\n",
              " 'loss_200': 2.07403427362442,\n",
              " 'loss_225': 2.3141628238889904,\n",
              " 'loss_25': 1.1056627333164215,\n",
              " 'loss_250': 2.4645030399163566,\n",
              " 'loss_275': 2.7903796103265552,\n",
              " 'loss_300': 2.0998321142461567,\n",
              " 'loss_325': 2.337274544768863,\n",
              " 'loss_350': 2.8250158958964877,\n",
              " 'loss_375': 2.8545848296748266,\n",
              " 'loss_400': 2.4531858232286243,\n",
              " 'loss_425': 2.341753406657113,\n",
              " 'loss_450': 2.415829267766741,\n",
              " 'loss_475': 3.101353704929352,\n",
              " 'loss_50': 1.1591884659396277,\n",
              " 'loss_500': 2.997980647616916,\n",
              " 'loss_525': 3.041702992386288,\n",
              " 'loss_550': 3.06148824095726,\n",
              " 'loss_575': 3.090206411149767,\n",
              " 'loss_600': 3.1117879615889654,\n",
              " 'loss_625': 3.13253300719791,\n",
              " 'loss_75': 1.1982646253373888,\n",
              " 'macro_f1_100': 0.20829822623667935,\n",
              " 'macro_f1_125': 0.20422464638160143,\n",
              " 'macro_f1_150': 0.39636227492773307,\n",
              " 'macro_f1_175': 0.2538169308633725,\n",
              " 'macro_f1_200': 0.2401805037528204,\n",
              " 'macro_f1_225': 0.2469763660509451,\n",
              " 'macro_f1_25': 0.4579607095204797,\n",
              " 'macro_f1_250': 0.26239290595726245,\n",
              " 'macro_f1_275': 0.22886511202560586,\n",
              " 'macro_f1_300': 0.4072049265600748,\n",
              " 'macro_f1_325': 0.3329129392796949,\n",
              " 'macro_f1_350': 0.208803017893927,\n",
              " 'macro_f1_375': 0.2419995497817894,\n",
              " 'macro_f1_400': 0.33554918789722415,\n",
              " 'macro_f1_425': 0.3187748291752741,\n",
              " 'macro_f1_450': 0.33754729962160307,\n",
              " 'macro_f1_475': 0.24654396196009565,\n",
              " 'macro_f1_50': 0.3957294939915063,\n",
              " 'macro_f1_500': 0.2483887246773845,\n",
              " 'macro_f1_525': 0.24636098587640864,\n",
              " 'macro_f1_550': 0.25344443527761734,\n",
              " 'macro_f1_575': 0.2571933722777718,\n",
              " 'macro_f1_600': 0.2569437734187951,\n",
              " 'macro_f1_625': 0.2551975846518383,\n",
              " 'macro_f1_75': 0.4579607095204797,\n",
              " 'micro_f1_100': 0.2674486803519062,\n",
              " 'micro_f1_125': 0.3364016736401673,\n",
              " 'micro_f1_150': 0.5751748251748252,\n",
              " 'micro_f1_175': 0.2951985773562537,\n",
              " 'micro_f1_200': 0.39824561403508774,\n",
              " 'micro_f1_225': 0.3824768323504633,\n",
              " 'micro_f1_25': 0.5,\n",
              " 'micro_f1_250': 0.4178321678321679,\n",
              " 'micro_f1_275': 0.3867016622922134,\n",
              " 'micro_f1_300': 0.5528596187175043,\n",
              " 'micro_f1_325': 0.47818499127399655,\n",
              " 'micro_f1_350': 0.36920384951881013,\n",
              " 'micro_f1_375': 0.4000000000000001,\n",
              " 'micro_f1_400': 0.4991243432574431,\n",
              " 'micro_f1_425': 0.3781869688385269,\n",
              " 'micro_f1_450': 0.49393414211438474,\n",
              " 'micro_f1_475': 0.40244969378827644,\n",
              " 'micro_f1_50': 0.5651857246904588,\n",
              " 'micro_f1_500': 0.4049079754601227,\n",
              " 'micro_f1_525': 0.40315512708150736,\n",
              " 'micro_f1_550': 0.4101665205959685,\n",
              " 'micro_f1_575': 0.41403508771929826,\n",
              " 'micro_f1_600': 0.4136722173531989,\n",
              " 'micro_f1_625': 0.41191936897458364,\n",
              " 'micro_f1_75': 0.5}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fMeqM4JD_bbV"
      },
      "source": [
        "## Saving Test Eval Results\n",
        "\n",
        "The code automatically saved evaluation result from each checkpoint in its respective folder. This next cell simply saves all of them in one place."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BvH1zD7si_rg"
      },
      "source": [
        "with open(os.path.join(args.output_dir, f\"{os.path.splitext(args.test_file)[0]}_masked_text_eval_results.txt\"), mode='w', encoding='utf-8') as out_f:\n",
        "    print(results, file=out_f)"
      ],
      "execution_count": 21,
      "outputs": []
    }
  ]
}