Final_DLH_Team_10.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "b27c61e5d05741eaa662970716efcdd2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_f29599514f82427d968084e14acc6396",
              "IPY_MODEL_2ae8b5d42df94e49b1178be0db7dffab",
              "IPY_MODEL_8722fcaa809a40a38e0ea9a70d516c69"
            ],
            "layout": "IPY_MODEL_5bdb976c838643f49ea44259f5ae908b"
          }
        },
        "f29599514f82427d968084e14acc6396": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2e8c5031dd3647f0ba661a6ec33c0a9d",
            "placeholder": "​",
            "style": "IPY_MODEL_5431c776e3814041bcc021d0d1763846",
            "value": "vocab.txt: 100%"
          }
        },
        "2ae8b5d42df94e49b1178be0db7dffab": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3a5ac28603cd468ca3e286763999ab48",
            "max": 227845,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_2a9f45e6496d491191b2317660f988f2",
            "value": 227845
          }
        },
        "8722fcaa809a40a38e0ea9a70d516c69": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e42e0b2f9e0a4b45b9725a9c1aa620eb",
            "placeholder": "​",
            "style": "IPY_MODEL_26772d87e551401e9d698ebf14a41704",
            "value": " 228k/228k [00:00&lt;00:00, 572kB/s]"
          }
        },
        "5bdb976c838643f49ea44259f5ae908b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2e8c5031dd3647f0ba661a6ec33c0a9d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5431c776e3814041bcc021d0d1763846": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3a5ac28603cd468ca3e286763999ab48": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2a9f45e6496d491191b2317660f988f2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "e42e0b2f9e0a4b45b9725a9c1aa620eb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "26772d87e551401e9d698ebf14a41704": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4bde7fb51ae54706834ade8db2191fd2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_3e62b21e67af41688b4288562edf0081",
              "IPY_MODEL_a676e2d1eb4a4f1a8942724ebadcc5f8",
              "IPY_MODEL_7b8e481b189f4444a248794d280f573c"
            ],
            "layout": "IPY_MODEL_6d9c918c1357426ca436403f84684485"
          }
        },
        "3e62b21e67af41688b4288562edf0081": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_717d8bea9dc24549abfc2b4c3e9a8581",
            "placeholder": "​",
            "style": "IPY_MODEL_dae1841745014e4a8aece2e9c37a0e97",
            "value": "config.json: 100%"
          }
        },
        "a676e2d1eb4a4f1a8942724ebadcc5f8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3cc5a4cc736849a7b3b3c1aaa0693e4d",
            "max": 385,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_7bd88662231146898101307d4c0425c1",
            "value": 385
          }
        },
        "7b8e481b189f4444a248794d280f573c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a8f5038200104a9abb14ec6947ddaaae",
            "placeholder": "​",
            "style": "IPY_MODEL_e470d6b043d949a5a09722e8cbdc32f3",
            "value": " 385/385 [00:00&lt;00:00, 36.5kB/s]"
          }
        },
        "6d9c918c1357426ca436403f84684485": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "717d8bea9dc24549abfc2b4c3e9a8581": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "dae1841745014e4a8aece2e9c37a0e97": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3cc5a4cc736849a7b3b3c1aaa0693e4d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7bd88662231146898101307d4c0425c1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "a8f5038200104a9abb14ec6947ddaaae": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e470d6b043d949a5a09722e8cbdc32f3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "909ae40bdf2e4c77acf00099e6e7e481": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_93ab468beae5423ca919503967c2131b",
              "IPY_MODEL_cce7f3124c0543e48066e09ded4c442d",
              "IPY_MODEL_5daad6f309b64e51a842cfe805c579d2"
            ],
            "layout": "IPY_MODEL_d6bcb10abda441e9a6850c86fdecb81d"
          }
        },
        "93ab468beae5423ca919503967c2131b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e8eaf998fb4e4ad188cd99aae4c3febb",
            "placeholder": "​",
            "style": "IPY_MODEL_4c3bbf4f8dd04c728e39ff0fa657a814",
            "value": "pytorch_model.bin: 100%"
          }
        },
        "cce7f3124c0543e48066e09ded4c442d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8e609dfe6f5d4a2c9eec1d924e33d373",
            "max": 442221694,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_e2c2ef64ea544924a1a7e021d1320911",
            "value": 442221694
          }
        },
        "5daad6f309b64e51a842cfe805c579d2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_248d6f66766948b5b9acbec327cb0660",
            "placeholder": "​",
            "style": "IPY_MODEL_f631e904debe4ee4bfa658b59807f41b",
            "value": " 442M/442M [00:19&lt;00:00, 23.8MB/s]"
          }
        },
        "d6bcb10abda441e9a6850c86fdecb81d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e8eaf998fb4e4ad188cd99aae4c3febb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4c3bbf4f8dd04c728e39ff0fa657a814": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "8e609dfe6f5d4a2c9eec1d924e33d373": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e2c2ef64ea544924a1a7e021d1320911": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "248d6f66766948b5b9acbec327cb0660": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f631e904debe4ee4bfa658b59807f41b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# CS598 Deep Learning for Healthcare - Reproducibility Project"
      ],
      "metadata": {
        "id": "aSoxztK_rPU8"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Darin Zhen, George Vojvodic, and Alan Yee; {darinz2, dvojvo2, alanyee2} @illinois.edu\n",
        "\n",
        "Group ID: 10\n",
        "\n",
        "Paper ID: 94\n",
        "\n",
        "Text2Mol: Cross-Modal Molecule Retrieval with Natural Language Queries\n"
      ],
      "metadata": {
        "id": "jlPcxRQc_Sfy"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Relevant Terminology"
      ],
      "metadata": {
        "id": "wZ3cbeZ-XOtQ"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "- **Hits@1**: A metric commonly used in information retrieval and recommendation systems to evaluate their performance. It measures the accuracy of a system in predicting the top recommendation or the correct answer out of a list of options for a given query or user interaction.\n",
        "\n",
        "- **Mean Reciprocal Rank (MRR)**: A metric used to evaluate the effectiveness of information retrieval systems, particularly in the context of ranked retrieval. It measures the quality of the ranked list of results by considering the position of the first relevant item in the list.\n",
        "\n",
        "- **Molecule**: An electrically neutral group of atoms bonded together.\n",
        "\n",
        "- **Compound**: Two or more elements held together by chemical bonds.\n",
        "\n",
        "- **Chemical fingerprint**: Represents a molecule or substructure using a bitstring. This allows for efficient substructure search and similarity calculation.\n",
        "\n",
        "- **Morgan fingerprint**: A specific type of chemical fingerprint also known as ECFP.\n",
        "\n",
        "- **SMILES string**: A character-based sequence representation of a molecule. (for example, C1=CC=CC=C1 is the SMILES string for benzene)\n",
        "\n",
        "- **Canonical SMILES**: A unique SMILES string for a molecule.\n"
      ],
      "metadata": {
        "id": "3782OlYB2QCa"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Introduction"
      ],
      "metadata": {
        "id": "4EhXyWiXXT7y"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "The discovery of new molecules and understanding their properties is critical for advancing fields like medicine, chemistry, and materials science. However, the vast number of possible molecules makes it impractical to experimentally characterize each one. There are already tens of millions of molecules cataloged in databases like PubChem. Efficiently retrieving relevant molecules from these large databases given natural language descriptions is an important yet challenging problem.\n",
        "\n",
        "Current methods for molecule retrieval typically rely on structured representations like molecular fingerprints or SMILES strings. While these enable substructure matching and similarity searches, they do not directly integrate the semantic information contained in natural language descriptions. Some approaches replace chemical names in text with canonical identifiers, but this fails to capture the full meaning. Solving the problem of cross-modal retrieval between natural language and molecules would allow scientists to easily search for molecules based on high-level conceptual descriptions rather than just structural patterns.\n",
        "\n",
        "The key challenge lies in bridging the stark difference between the modalities of natural language and molecular structure data. Molecules are usually represented as graphs with atoms as nodes and bonds as edges, following a unique grammar quite distinct from human language. This makes cross-modal retrieval exceptionally challenging compared to traditional cross-lingual information retrieval between natural languages.\n",
        "\n",
        "In this paper, the authors propose a novel \"Text2Mol\" task for retrieving molecules directly from natural language descriptions (shown in Fiture 1). They develop a multimodal embedding approach to learn an aligned semantic space bridging text and molecular structure data. This allows ranking molecules by similarity to text query descriptions. The paper makes several innovations, including extending the loss function with negative sampling to encourage integration of both modalities, using cross-modal attention to extract interpretable \"association rules\" between text and molecular substructures, and an ensemble method that significantly boosts performance.\n",
        "<br>\n",
        "<img src=\"https://drive.google.com/uc?id=1mG9lgWvfpZ2tplVl6xqLZMyzj5ooPhy8\" width=500 />\n",
        "<br>\n",
        "<b>Figure 1: Given a natural language description of water, we want to rank the corresponding molecule $H_2O$ first among all the possible molecules.</b>\n",
        "<br>\n",
        "\n",
        "On a new benchmark dataset of over 33,000 text-molecule pairs, the proposed methods achieve a mean reciprocal rank of 0.499, substantially outperforming baselines. The cross-modal attention model provides insightful explanations grounding the language representations to the molecular structure. Overall, this multimodal approach offers a powerful solution for understanding chemistry literature and searching molecular databases, with broad potential applications in drug discovery, materials design, and scientific knowledge exploration.\n"
      ],
      "metadata": {
        "id": "MQ0sNuMePBXx"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Scope of Reproducibility"
      ],
      "metadata": {
        "id": "fGRats_iwAeF"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "The scope of reproducibility in the paper encompasses the following key hypotheses that will reproduce:\n",
        "\n",
        "1. **Hypothesis 1**: Cross-modal embedding can effectively align text and molecule spaces for retrieval. This involves reproducing embedding models and evaluating retrieval metrics like Mean Reciprocal Rank (MRR).\n",
        "\n",
        "2. **Hypothesis 2**: Ensemble of different architectures (MLP vs GCN) improves results compared to individual models. This involves training different models and comparing ensemble versus individual performance.\n",
        "\n",
        "3. **Hypothesis 3**: Cross-modal attention provides insights into text-molecule associations. This will be examined by analyzing attention weights and extracted rules for coherence.\n",
        "\n",
        "4. **Hypothesis 4**: Different architectures possess complementary strengths, where MLP may rank easier examples better but GCN generalizes better. This will be probed by analyzing differences in rankings between architectures.\n",
        "\n",
        "5. **Hypothesis 5**: Cross-modal reranking using attention rules improves over the base model. Testing reranking on a holdout set will validate this hypothesis."
      ],
      "metadata": {
        "id": "SvYxdQvhGSB6"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Setup"
      ],
      "metadata": {
        "id": "ipcAE7rZNJG8"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "**NOTE:** This notebook requires **GPU** runtime due to the extensive use of hundreds of millions of parameters in the training model."
      ],
      "metadata": {
        "id": "GKCvaTzXFBoD"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "This notebook supports accessing data and images:\n"
      ],
      "metadata": {
        "id": "_F7TTYKAxlSu"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Data directory\n",
        "!mkdir /content/data\n",
        "\n",
        "# Image directory\n",
        "!mkdir /content/image\n",
        "\n",
        "# Input directory\n",
        "!mkdir /content/input\n",
        "\n",
        "# Download files into the data folder.\n",
        "\n",
        "!wget --no-check-certificate 'https://drive.google.com/uc?export=download&id=1XYjz33tWWet6t4QouZkVn3TRYliG5L5F' -O ChEBI_defintions_substructure_corpus.cp\n",
        "!mv /content/ChEBI_defintions_substructure_corpus.cp /content/data/ChEBI_defintions_substructure_corpus.cp\n",
        "\n",
        "!wget --no-check-certificate 'https://text2mol2024.blob.core.windows.net/model/chem_embeddings_test.npy' -O chem_embeddings_test.npy\n",
        "!mv /content/chem_embeddings_test.npy /content/data/chem_embeddings_test.npy\n",
        "\n",
        "!wget --no-check-certificate 'https://text2mol2024.blob.core.windows.net/model/chem_embeddings_train.npy' -O chem_embeddings_train.npy\n",
        "!mv /content/chem_embeddings_train.npy /content/data/chem_embeddings_train.npy\n",
        "\n",
        "!wget --no-check-certificate 'https://text2mol2024.blob.core.windows.net/model/chem_embeddings_val.npy' -O chem_embeddings_val.npy\n",
        "!mv /content/chem_embeddings_val.npy /content/data/chem_embeddings_val.npy\n",
        "\n",
        "!wget --no-check-certificate 'https://text2mol2024.blob.core.windows.net/model/cids_test.npy' -O cids_test.npy\n",
        "!mv /content/cids_test.npy /content/data/cids_test.npy\n",
        "\n",
        "!wget --no-check-certificate 'https://text2mol2024.blob.core.windows.net/model/cids_train.npy' -O cids_train.npy\n",
        "!mv /content/cids_train.npy /content/data/cids_train.npy\n",
        "\n",
        "!wget --no-check-certificate 'https://text2mol2024.blob.core.windows.net/model/cids_val.npy' -O cids_val.npy\n",
        "!mv /content/cids_val.npy /content/data/cids_val.npy\n",
        "\n",
        "!wget --no-check-certificate 'https://drive.google.com/uc?export=download&id=1SmKQJPKRePUXyomOBdMwha75-D5O4413' -O test.sdf\n",
        "!mv /content/test.sdf /content/data/test.sdf\n",
        "\n",
        "!wget --no-check-certificate 'https://drive.google.com/uc?export=download&id=1dhhTAD3z97yOQYSK0Go-RI-bgjJdSg6e' -O test.txt\n",
        "!mv /content/test.txt /content/data/test.txt\n",
        "\n",
        "!wget --no-check-certificate 'https://text2mol2024.blob.core.windows.net/model/text_embeddings_test.npy' -O text_embeddings_test.npy\n",
        "!mv /content/text_embeddings_test.npy /content/data/text_embeddings_test.npy\n",
        "\n",
        "!wget --no-check-certificate 'https://text2mol2024.blob.core.windows.net/model/text_embeddings_train.npy' -O text_embeddings_train.npy\n",
        "!mv /content/text_embeddings_train.npy /content/data/text_embeddings_train.npy\n",
        "\n",
        "!wget --no-check-certificate 'https://text2mol2024.blob.core.windows.net/model/text_embeddings_val.npy' -O text_embeddings_val.npy\n",
        "!mv /content/text_embeddings_val.npy /content/data/text_embeddings_val.npy\n",
        "\n",
        "!wget --no-check-certificate 'https://drive.google.com/uc?export=download&id=1gR4B11xGBLGwYQ2-s_k0C19HUXYDFSoa' -O token_embedding_dict.npy\n",
        "!mv /content/token_embedding_dict.npy /content/data/token_embedding_dict.npy\n",
        "\n",
        "!wget --no-check-certificate 'https://drive.google.com/uc?export=download&id=1g8a6wg1OC8okFltFJOaNbMC5BsWuQdML' -O training.sdf\n",
        "!mv /content/training.sdf /content/data/training.sdf\n",
        "\n",
        "!wget --no-check-certificate 'https://drive.google.com/uc?export=download&id=17SvDWffLm8Eez7KIIyZt3RvsNOkSDZMM' -O training.txt\n",
        "!mv /content/training.txt /content/data/training.txt\n",
        "\n",
        "!wget --no-check-certificate 'https://drive.google.com/uc?export=download&id=1i39CtXI7HbdtnRG4lHMCFKQ7i_AZgSdn' -O val.sdf\n",
        "!mv /content/val.sdf /content/data/val.sdf\n",
        "\n",
        "!wget --no-check-certificate 'https://drive.google.com/uc?export=download&id=1sQ7iYAHIRMmq0YePRaiRngcPNwFPzZoO' -O val.txt\n",
        "!mv /content/val.txt /content/data/val.txt\n",
        "\n",
        "\n",
        "# Download files into the input folder.\n",
        "\n",
        "!wget --no-check-certificate 'https://drive.google.com/uc?export=download&id=1sOLCJIDKZCZO-9jSFPsQGYK4h17sDvU7' -O ChEBI_defintions_substructure_corpus.cp\n",
        "!mv /content/ChEBI_defintions_substructure_corpus.cp /content/input/ChEBI_defintions_substructure_corpus.cp\n",
        "\n",
        "!wget --no-check-certificate 'https://drive.google.com/uc?export=download&id=1G1iST_JfJTfj1hRBzzqgIBQZDUjzv5oA' -O mol2vec_ChEBI_20_test.txt\n",
        "!mv /content/mol2vec_ChEBI_20_test.txt /content/input/mol2vec_ChEBI_20_test.txt\n",
        "\n",
        "!wget --no-check-certificate 'https://drive.google.com/uc?export=download&id=1aSuZUiiM7Bmsg-zONm7RMfAm8MizL1tc' -O mol2vec_ChEBI_20_training.txt\n",
        "!mv /content/mol2vec_ChEBI_20_training.txt /content/input/mol2vec_ChEBI_20_training.txt\n",
        "\n",
        "!wget --no-check-certificate 'https://drive.google.com/uc?export=download&id=1BERxF6s-GPDRaEOoSBfhfs-u6RS1UUSx' -O mol2vec_ChEBI_20_val.txt\n",
        "!mv /content/mol2vec_ChEBI_20_val.txt /content/input/mol2vec_ChEBI_20_val.txt\n"
      ],
      "metadata": {
        "id": "7QvHfkRpwUtR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Methodology"
      ],
      "metadata": {
        "id": "smxGlzsjIS7g"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "\n",
        "\n",
        "The paper proposes a new task called Text2Mol, which aims to retrieve molecules from natural language descriptions. The methodology involves the following key steps:\n",
        "\n",
        "1. Construct a dataset of molecule-text description pairs from sources like PubChem and ChEBI.\n",
        "\n",
        "2. Learn aligned semantic embeddings for text and molecules using:\n",
        "    a) A text encoder based on SciBERT\n",
        "    b) A molecule encoder using either a multi-layer perceptron (MLP) on Mol2vec embeddings or a graph convolutional network (GCN) on the molecular graph with Mol2vec features.\n",
        "\n",
        "3. Train the encoders using a contrastive loss that aims to bring positive (matching) molecule-text pairs closer and push negative pairs apart in the embedding space.\n",
        "\n",
        "4. At inference time, encode the text query and retrieve the nearest molecule embeddings using cosine similarity.\n",
        "\n",
        "5. Explore ensembling multiple trained models and incorporating cross-modal attention to learn association rules between text tokens and molecular substructures for explainability and reranking.\n",
        "\n",
        "The key novelties are applying contrastive learning across the text and molecule modalities, proposing the Text2Mol retrieval task, and using cross-modal attention for explainable retrieval via association rules."
      ],
      "metadata": {
        "id": "xWAHJ_1CdtaA"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Environment"
      ],
      "metadata": {
        "id": "Mg-kVgJuwFo8"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Python Version\n",
        "\n"
      ],
      "metadata": {
        "id": "i5eAfWrjF0EZ"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "This project uses Python version 3.10.12."
      ],
      "metadata": {
        "id": "MRZezxDLiqrJ"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Dependencies & Packages Needed"
      ],
      "metadata": {
        "id": "H2stTGmhagh0"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "This project relies on several Python libraries and modules for text-to-molecule tasks:\n",
        "\n",
        "1. **Operating System Interaction**: `os` module for interacting with the operating system.\n",
        "2. **File Operations**: `shutil` module for file operations.\n",
        "3. **Time-related Functions**: `time` module for time-related functions.\n",
        "4. **Mathematical Operations**: `math` module for mathematical operations.\n",
        "5. **Numerical Computations**: `numpy` (`np` alias) for numerical computations.\n",
        "6. **Plotting**: `matplotlib.pyplot` (`plt` alias) for plotting.\n",
        "7. **Cosine Similarity Computation**: `cosine_similarity` from `sklearn.metrics.pairwise` for computing cosine similarity.\n",
        "8. **Deep Learning Framework**: `torch` for PyTorch, a deep learning framework.\n",
        "9. **Neural Network Components**: `torch.nn` for neural network modules and `torch.nn.functional` (`F` alias) for functional interfaces.\n",
        "10. **Data Handling Utilities**: `torch.utils.data` for handling data in PyTorch, including `Dataset` and `DataLoader`.\n",
        "11. **Tokenization**: `tokenizers` for tokenization, including the `Tokenizer` class.\n",
        "12. **BERT Model and Tokenizer**: `BertTokenizerFast` and `BertModel` from Hugging Face's Transformers library for BERT tokenizer and model.\n",
        "13. **CSV File Handling**: `csv` module for reading and writing CSV files.\n",
        "14. **Graph Convolutional Network (GCN)**: `torch_geometric.nn` for GCN operations, including `GCNConv` and `global_mean_pool`.\n",
        "15. **Transformer Decoder**: `TransformerDecoder` and `TransformerDecoderLayer` from PyTorch for transformer decoder operations.\n",
        "16. **Optimization**: `torch.optim` for optimization, including various optimizers.\n",
        "17. **Learning Rate Scheduler**: `get_linear_schedule_with_warmup` from the transformers library for learning rate scheduling.\n",
        "\n",
        "Additionally, the code includes an installation command (`!pip install torch_geometric`) to install the `torch_geometric` library."
      ],
      "metadata": {
        "id": "PHmhDaZTjBht"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# This code imports various Python libraries and modules that are used in this\n",
        "# notebook for Text2Mol\n",
        "# Importing necessary libraries/modules\n",
        "import os                   # Module for interacting with the operating system\n",
        "import shutil               # Module for file operations\n",
        "import time                 # Module for time-related functions\n",
        "\n",
        "import math                # Module for mathematical operations\n",
        "\n",
        "import numpy as np         # NumPy, a library for numerical computations\n",
        "\n",
        "import matplotlib.pyplot as plt  # Matplotlib, a plotting library\n",
        "from sklearn.metrics.pairwise import cosine_similarity  # Module for cosine similarity computation\n",
        "\n",
        "import torch               # PyTorch, a deep learning framework\n",
        "from torch import nn       # Neural network module from PyTorch\n",
        "import torch.nn.functional as F  # Functional interface to neural network components in PyTorch\n",
        "from torch.utils.data import Dataset, DataLoader  # Utilities for handling data in PyTorch\n",
        "\n",
        "import tokenizers         # Tokenizers library for tokenization\n",
        "from tokenizers import Tokenizer  # Tokenizer class for tokenization\n",
        "from transformers import BertTokenizerFast, BertModel  # BERT tokenizer and model from Hugging Face's Transformers library\n",
        "\n",
        "import csv                 # Module for reading and writing CSV files"
      ],
      "metadata": {
        "id": "yu61Jp1xrnKk"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Data"
      ],
      "metadata": {
        "id": "Gs968ZkyG3wZ"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Data Download Instruction"
      ],
      "metadata": {
        "id": "hjY-QGwzHBBx"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "To download the data for the project, follow these instructions:\n",
        "\n",
        "1. **ChEBI Annotations of Compounds from [PubChem](https://pubchem.ncbi.nlm.nih.gov/)**: (This part is not necessary as the raw ChEBI dataset is not directly used in the model, and the actual dataset is already directly provided and organized in `2.`. Nonetheless, we provide steps here to obtain the raw data.)\n",
        "   1. Visit the [ChEBI](https://www.ebi.ac.uk/chebi/) website.\n",
        "   1. Under `Downloads`, click on `SDF files`.\n",
        "   1. Click on `ChEBI_complete.sdf.gz`\n",
        "   1. Choose where to download to zipped file and click `Save`\n",
        "\n",
        "2. **ChEBI-20 Dataset**: (This part is required.)\n",
        "   - Access the [ChEBI-20 dataset repository](https://github.com/cnedwards/text2mol/tree/master/data)."
      ],
      "metadata": {
        "id": "MU216Z1Qb04g"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Data Descriptions"
      ],
      "metadata": {
        "id": "2cUAEWuqlQOT"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "The paper makes use of the following datasets:\n",
        "\n",
        "1. [ChEBI](https://www.ebi.ac.uk/chebi/) (Chemical Entities of Biological Interest) annotations of compounds scraped from [PubChem](https://pubchem.ncbi.nlm.nih.gov/).\n",
        "   - This contains 102,980 compound-description pairs.\n",
        "\n",
        "2. [ChEBI-20](https://github.com/cnedwards/text2mol/tree/master/data) dataset\n",
        "   - Constructed from the ChEBI/PubChem data by filtering for descriptions longer than 20 words.\n",
        "   - Contains 33,010 text-compound pairs.\n",
        "   - Split into 80/10/10% train/validation/test sets.\n",
        "\n",
        "The ChEBI-20 dataset forms the main benchmark used to evaluate the proposed cross-modal molecule retrieval methods.\n",
        "\n",
        "For representing the molecular structures, the paper uses:\n",
        "\n",
        "1. Mol2vec representations\n",
        "   - Molecular graphs are converted to \"sentences\" using the Morgan fingerprinting (shown in Figure 2) algorithm which generates substructure identifiers.\n",
        "   <br>\n",
        "   <img src=\"https://drive.google.com/uc?id=1lT5fla5UnQXW60-utzHYWDFdg62sMgp0\" width=\"400\">\n",
        "   <br>\n",
        "   <b>Figure 2: Example of Morgan Fingerprinting from (Rogers and Hahn, 2010) for Butyramide. The algorithm updates the identifiers from radius r = 0 to r = 1, as shown by the green circles.</b>\n",
        "   <br>\n",
        "   - The Mol2vec algorithm applies Word2vec on these substructure sentences to produce molecule embeddings.\n",
        "   - Default radius of 1 is used, giving two substructure tokens per atom.\n",
        "\n",
        "2. SMILES strings\n",
        "   - Character-based representation of molecules that can be parsed into molecular graphs.\n",
        "\n",
        "The text descriptions are encoded using the pre-trained SciBERT language model.\n",
        "\n",
        "The key data is the new ChEBI-20 dataset of paired text descriptions and molecular structures, with molecules represented by Mol2vec embeddings or SMILES strings, and descriptions encoded by SciBERT."
      ],
      "metadata": {
        "id": "2NbPHUTMbkD3"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### The ChEBI-20 dataset is contained in 6 files:"
      ],
      "metadata": {
        "id": "Cmgrq6ROlgEj"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "\n",
        "(1,2,3) The mol2vec_ChEBI_20_X.txt files have lines in the following form:\n",
        "```\n",
        "CID\tmol2vec embedding\tDescription\n",
        "```\n",
        "\n",
        "(4) mol_graphs.zip contain {cid}.graph files. These are formatted first with the edgelist of the graph and then substructure tokens for each node.\n",
        "For example,\n",
        "edgelist:\n",
        "```\n",
        "0 1\n",
        "1 0\n",
        "1 2\n",
        "2 1\n",
        "1 3\n",
        "3 1\n",
        "```\n",
        "idx to identifier:\n",
        "```\n",
        "0 3537119515\n",
        "1 2059730245\n",
        "2 3537119515\n",
        "3 1248171218\n",
        "```\n",
        "\n",
        "(5) ChEBI_defintions_substructure_corpus.cp contains the molecule token \"sentences\". It is formatted:\n",
        "```\n",
        "cid: tokenid1 tokenid2 tokenid3 ... tokenidn\n",
        "```\n",
        "\n",
        "(6) token_embedding_dict.npy is a dictionary mapping molecule tokens to their embeddings.\n",
        "\n",
        "It can be loaded with the following code:\n",
        "```python\n",
        "import numpy as np\n",
        "token_embedding_dict = np.load(\"token_embedding_dict.npy\", allow_pickle=True)[()]\n",
        "```"
      ],
      "metadata": {
        "id": "9eIF9Jtqldbk"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Preprocessing Code + Command"
      ],
      "metadata": {
        "id": "0cwbhQ8iHIaW"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### GenerateData Class\n",
        "\n",
        "This class is designed to handle the generation of examples for training,\n",
        "validation, and testing sets, where each example contains both text data\n",
        "(processed using a BERT tokenizer) and molecule data. The methods within\n",
        "the class prepare the necessary data structures, tokenize text inputs,\n",
        "and yield examples in the desired format."
      ],
      "metadata": {
        "id": "kkArCBBZudkT"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#Need a special generator for random sampling:\n",
        "\n",
        "class GenerateData():\n",
        "    def __init__(self, path_train, path_val, path_test, path_molecules, path_token_embs):\n",
        "        # Constructor method initializing paths and parameters\n",
        "        self.path_train = path_train  # Path to the training data file\n",
        "        self.path_val = path_val  # Path to the validation data file\n",
        "        self.path_test = path_test  # Path to the test data file\n",
        "        self.path_molecules = path_molecules  # Path to the file containing molecule data\n",
        "        self.path_token_embs = path_token_embs  # Path to the file containing token embeddings\n",
        "\n",
        "        self.text_trunc_length = 256  # Maximum length for text input tokens\n",
        "\n",
        "        # Initialize text tokenizer\n",
        "        self.prep_text_tokenizer()\n",
        "\n",
        "        # Load substructures from molecule data\n",
        "        self.load_substructures()\n",
        "\n",
        "        self.batch_size = 32  # Batch size for data processing\n",
        "\n",
        "        # Store descriptions\n",
        "        self.store_descriptions()\n",
        "\n",
        "    def load_substructures(self):\n",
        "        # Method to load substructures from molecule data\n",
        "        self.molecule_sentences = {}  # Dictionary to store molecule sentences\n",
        "        self.molecule_tokens = {}  # Dictionary to store molecule tokens\n",
        "\n",
        "        total_tokens = set()  # Set to store unique tokens\n",
        "        self.max_mol_length = 0  # Variable to store maximum molecule length\n",
        "        with open(self.path_molecules) as f:\n",
        "            for line in f:\n",
        "                spl = line.split(\":\")\n",
        "                cid = spl[0]  # Compound ID\n",
        "                tokens = spl[1].strip()  # Tokens for the compound\n",
        "                self.molecule_sentences[cid] = tokens\n",
        "                t = tokens.split()\n",
        "                total_tokens.update(t)  # Add tokens to the set\n",
        "                size = len(t)\n",
        "                if size > self.max_mol_length:\n",
        "                    self.max_mol_length = size  # Update maximum molecule length\n",
        "\n",
        "        # Load token embeddings\n",
        "        self.token_embs = np.load(self.path_token_embs, allow_pickle=True)[()]\n",
        "\n",
        "    def prep_text_tokenizer(self):\n",
        "        # Method to prepare text tokenizer (using BERT)\n",
        "        self.text_tokenizer = BertTokenizerFast.from_pretrained(\"allenai/scibert_scivocab_uncased\")\n",
        "\n",
        "    def store_descriptions(self):\n",
        "        # Method to store descriptions from training, validation, and test sets\n",
        "        self.descriptions = {}  # Dictionary to store descriptions\n",
        "        self.mols = {}  # Dictionary to store molecule data\n",
        "\n",
        "        self.training_cids = []  # List to store training set compound IDs\n",
        "        # Get training set compound IDs\n",
        "        with open(self.path_train) as f:\n",
        "            reader = csv.DictReader(f, delimiter=\"\\t\", quoting=csv.QUOTE_NONE, fieldnames=['cid', 'mol2vec', 'desc'])\n",
        "            for n, line in enumerate(reader):\n",
        "                self.descriptions[line['cid']] = line['desc']\n",
        "                self.mols[line['cid']] = line['mol2vec']\n",
        "                self.training_cids.append(line['cid'])\n",
        "\n",
        "        self.validation_cids = []  # List to store validation set compound IDs\n",
        "        # Get validation set compound IDs\n",
        "        with open(self.path_val) as f:\n",
        "            reader = csv.DictReader(f, delimiter=\"\\t\", quoting=csv.QUOTE_NONE, fieldnames=['cid', 'mol2vec', 'desc'])\n",
        "            for n, line in enumerate(reader):\n",
        "                self.descriptions[line['cid']] = line['desc']\n",
        "                self.mols[line['cid']] = line['mol2vec']\n",
        "                self.validation_cids.append(line['cid'])\n",
        "\n",
        "        self.test_cids = []  # List to store test set compound IDs\n",
        "        # Get test set compound IDs\n",
        "        with open(self.path_test) as f:\n",
        "            reader = csv.DictReader(f, delimiter=\"\\t\", quoting=csv.QUOTE_NONE, fieldnames=['cid', 'mol2vec', 'desc'])\n",
        "            for n, line in enumerate(reader):\n",
        "                self.descriptions[line['cid']] = line['desc']\n",
        "                self.mols[line['cid']] = line['mol2vec']\n",
        "                self.test_cids.append(line['cid'])\n",
        "\n",
        "    def generate_examples_train(self):\n",
        "        # Method to generate examples for training set\n",
        "        np.random.shuffle(self.training_cids)  # Shuffle training compound IDs\n",
        "\n",
        "        for cid in self.training_cids:\n",
        "            text_input = self.text_tokenizer(self.descriptions[cid], truncation=True, max_length=self.text_trunc_length,\n",
        "                                             padding='max_length', return_tensors='np')  # Tokenize text input\n",
        "\n",
        "            yield {\n",
        "                'cid': cid,\n",
        "                'input': {\n",
        "                    'text': {\n",
        "                        'input_ids': text_input['input_ids'].squeeze(),\n",
        "                        'attention_mask': text_input['attention_mask'].squeeze(),\n",
        "                    },\n",
        "                    'molecule': {\n",
        "                        'mol2vec': np.fromstring(self.mols[cid], sep=\" \"),  # Convert molecule data to NumPy array\n",
        "                        'cid': cid\n",
        "                    },\n",
        "                },\n",
        "            }\n",
        "\n",
        "    def generate_examples_val(self):\n",
        "        # Method to generate examples for validation set\n",
        "        np.random.shuffle(self.validation_cids)  # Shuffle validation compound IDs\n",
        "\n",
        "        for cid in self.validation_cids:\n",
        "            text_input = self.text_tokenizer(self.descriptions[cid], truncation=True, padding='max_length',\n",
        "                                             max_length=self.text_trunc_length, return_tensors='np')  # Tokenize text input\n",
        "\n",
        "            yield {\n",
        "                'cid': cid,\n",
        "                'input': {\n",
        "                    'text': {\n",
        "                        'input_ids': text_input['input_ids'].squeeze(),\n",
        "                        'attention_mask': text_input['attention_mask'].squeeze(),\n",
        "                    },\n",
        "                    'molecule': {\n",
        "                        'mol2vec': np.fromstring(self.mols[cid], sep=\" \"),  # Convert molecule data to NumPy array\n",
        "                        'cid': cid\n",
        "                    }\n",
        "                },\n",
        "            }\n",
        "\n",
        "    def generate_examples_test(self):\n",
        "        # Method to generate examples for test set\n",
        "        np.random.shuffle(self.test_cids)  # Shuffle test compound IDs\n",
        "\n",
        "        for cid in self.test_cids:\n",
        "            text_input = self.text_tokenizer(self.descriptions[cid], truncation=True, padding='max_length',\n",
        "                                             max_length=self.text_trunc_length, return_tensors='np')  # Tokenize text input\n",
        "\n",
        "            yield {\n",
        "                'cid': cid,\n",
        "                'input': {\n",
        "                    'text': {\n",
        "                        'input_ids': text_input['input_ids'].squeeze(),\n",
        "                        'attention_mask': text_input['attention_mask'].squeeze(),\n",
        "                    },\n",
        "                    'molecule': {\n",
        "                        'mol2vec': np.fromstring(self.mols[cid], sep=\" \"),  # Convert molecule data to NumPy array\n",
        "                        'cid': cid\n",
        "                    }\n",
        "                },\n",
        "            }"
      ],
      "metadata": {
        "id": "d7kMwLjibzSo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "In the following code, the paths to various data files are defined. Then, it checks\n",
        "if a specific token embedding file exists using os.path.exists(). If the\n",
        "file does not exist, it raises a FileNotFoundError. Finally, an instance of\n",
        "the GenerateData class is created with the defined"
      ],
      "metadata": {
        "id": "JS7CEl0IuugZ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Define the path to the token embedding file\n",
        "#mounted_path_token_embs = os.path.join(data_dir, 'token_embedding_dict.npy')\n",
        "mounted_path_token_embs = \"data/token_embedding_dict.npy\"\n",
        "\n",
        "# Check if the token embedding file exists\n",
        "if not os.path.exists(mounted_path_token_embs):\n",
        "    # Raise FileNotFoundError if the file does not exist\n",
        "    raise FileNotFoundError(f\"The following token embedding DOES NOT EXIST: {mounted_path_token_embs}\")\n",
        "\n",
        "# Define the path to the molecule data file\n",
        "parent_dir = os.path.join('/content/drive/', 'My Drive')\n",
        "\n",
        "mounted_path_molecules = \"input/ChEBI_defintions_substructure_corpus.cp\"\n",
        "\n",
        "mounted_path_train = \"input/mol2vec_ChEBI_20_training.txt\"\n",
        "mounted_path_val = \"input/mol2vec_ChEBI_20_val.txt\"\n",
        "mounted_path_test = \"input/mol2vec_ChEBI_20_test.txt\"\n",
        "\n",
        "# Instantiate the GenerateData class with the specified paths\n",
        "gt = GenerateData(mounted_path_train, mounted_path_val, mounted_path_test, mounted_path_molecules, mounted_path_token_embs)"
      ],
      "metadata": {
        "id": "cQY3wE2jcfUk",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 205,
          "referenced_widgets": [
            "b27c61e5d05741eaa662970716efcdd2",
            "f29599514f82427d968084e14acc6396",
            "2ae8b5d42df94e49b1178be0db7dffab",
            "8722fcaa809a40a38e0ea9a70d516c69",
            "5bdb976c838643f49ea44259f5ae908b",
            "2e8c5031dd3647f0ba661a6ec33c0a9d",
            "5431c776e3814041bcc021d0d1763846",
            "3a5ac28603cd468ca3e286763999ab48",
            "2a9f45e6496d491191b2317660f988f2",
            "e42e0b2f9e0a4b45b9725a9c1aa620eb",
            "26772d87e551401e9d698ebf14a41704",
            "4bde7fb51ae54706834ade8db2191fd2",
            "3e62b21e67af41688b4288562edf0081",
            "a676e2d1eb4a4f1a8942724ebadcc5f8",
            "7b8e481b189f4444a248794d280f573c",
            "6d9c918c1357426ca436403f84684485",
            "717d8bea9dc24549abfc2b4c3e9a8581",
            "dae1841745014e4a8aece2e9c37a0e97",
            "3cc5a4cc736849a7b3b3c1aaa0693e4d",
            "7bd88662231146898101307d4c0425c1",
            "a8f5038200104a9abb14ec6947ddaaae",
            "e470d6b043d949a5a09722e8cbdc32f3"
          ]
        },
        "outputId": "5851979e-864f-490c-c1dc-b948c0b4ec84"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:88: UserWarning: \n",
            "The secret `HF_TOKEN` does not exist in your Colab secrets.\n",
            "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n",
            "You will be able to reuse this secret in all of your notebooks.\n",
            "Please note that authentication is recommended but still optional to access public models or datasets.\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "vocab.txt:   0%|          | 0.00/228k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "b27c61e5d05741eaa662970716efcdd2"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "config.json:   0%|          | 0.00/385 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "4bde7fb51ae54706834ade8db2191fd2"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### Dataset Class\n",
        "\n",
        "This class Dataset is designed to create a custom dataset for PyTorch. It allows generating samples of data on-the-fly using a generator function gen. The `__len__` method returns the total number of samples in the dataset, and the `__getitem__` method generates one sample of data for a given index. If the generator is exhausted (i.e., it reaches the end), it resets the iterator to the beginning. In this specific implementation, the target variable `y` is set to a constant value of 1 for all samples."
      ],
      "metadata": {
        "id": "7xjEU2zdvJvs"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class Dataset(Dataset):\n",
        "    'Characterizes a dataset for PyTorch'\n",
        "\n",
        "    def __init__(self, gen, length):\n",
        "        'Initialization'\n",
        "\n",
        "        self.gen = gen  # Generator function that yields data examples\n",
        "        self.it = iter(self.gen())  # Iterator over the generator function\n",
        "\n",
        "        self.length = length  # Length of the dataset\n",
        "\n",
        "    def __len__(self):\n",
        "        'Denotes the total number of samples'\n",
        "\n",
        "        return self.length  # Returns the length of the dataset\n",
        "\n",
        "    def __getitem__(self, index):\n",
        "        'Generates one sample of data'\n",
        "\n",
        "        try:\n",
        "            ex = next(self.it)  # Get the next example from the iterator\n",
        "        except StopIteration:\n",
        "            self.it = iter(self.gen())  # If iterator is exhausted, reset it\n",
        "            ex = next(self.it)  # Get the next example\n",
        "\n",
        "        X = ex['input']  # Extract input data from the example\n",
        "        y = 1  # Placeholder for the target variable (constant value in this case)\n",
        "\n",
        "        return X, y  # Return input data and target variable for the given index"
      ],
      "metadata": {
        "id": "2XDCJpNKvbwq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "\n",
        "In the following code, three datasets (training_set, validation_set, and test_set) are created. Each dataset is instantiated with the Dataset class, and they are initialized with different generator functions (gt.generate_examples_train, gt.generate_examples_val, and gt.generate_examples_test, respectively) along with the lengths of their respective compound ID lists. These datasets are intended to be used for training, validation, and testing.\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "HlRcvvowv-I-"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Create a dataset for the training set\n",
        "# using the 'generate_examples_train' method of the 'gt' object and the length of training compound IDs\n",
        "training_set = Dataset(gt.generate_examples_train, len(gt.training_cids))\n",
        "\n",
        "# Create a dataset for the validation set\n",
        "# using the 'generate_examples_val' method of the 'gt' object and the length of validation compound IDs\n",
        "validation_set = Dataset(gt.generate_examples_val, len(gt.validation_cids))\n",
        "\n",
        "# Create a dataset for the test set\n",
        "# using the 'generate_examples_test' method of the 'gt' object and the length of test compound IDs\n",
        "test_set = Dataset(gt.generate_examples_test, len(gt.test_cids))\n",
        "\n",
        "n_samples = 50\n",
        "training_set_sample = torch.utils.data.Subset(training_set, list(range(n_samples)))\n",
        "validation_set_sample = torch.utils.data.Subset(validation_set, list(range(n_samples)))\n",
        "test_set_sample = torch.utils.data.Subset(test_set, list(range(n_samples)))\n",
        "\n",
        "params = {'batch_size': gt.batch_size,\n",
        "          'shuffle': True}\n",
        "\n",
        "training_generator = DataLoader(training_set_sample, **params)\n",
        "validation_generator = DataLoader(validation_set_sample, **params)\n",
        "test_generator = DataLoader(test_set_sample, **params)"
      ],
      "metadata": {
        "id": "R3UMPNj4vdaJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Model"
      ],
      "metadata": {
        "id": "lLbPYYFPHQHN"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Citation to the original paper"
      ],
      "metadata": {
        "id": "P2_VkiwtxUfC"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "The following is the citation to the original paper:\n",
        "\n",
        "Carl Edwards, ChengXiang Zhai, and Heng Ji. 2021. [Text2mol: Cross-modal molecule retrieval with natural language queries](https://aclanthology.org/2021.emnlp-main.47). In Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing, pages 595–607, Online and Punta Cana, Dominican Republic. Association for Computational Linguistics."
      ],
      "metadata": {
        "id": "vjOmpT6TS7bq"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Link to the original paper’s repo"
      ],
      "metadata": {
        "id": "rbho17oO4fO-"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "\n",
        "The link to the original paper's repo is as follows:\n",
        "\n",
        "[Text2Mol Code Repository](https://github.com/cnedwards/text2mol)"
      ],
      "metadata": {
        "id": "1U1tBsuES_-7"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Overview\n",
        "\n",
        "The paper proposes several models for the cross-modal Text2Mol task of retrieving molecules from natural language descriptions:\n",
        "\n",
        "Models Parameters:\n",
        "   - The text encoder uses the large SciBERT model, which has around 110M parameters.\n",
        "   - The MLP molecule encoder is relatively small, with around 110M parameters.\n",
        "   - The GCN molecule encoder is slightly larger, with around 112M parameters.\n",
        "   - The cross-modal attention model is the largest, with around 129M parameters.\n",
        "\n",
        "\n",
        "1. Base Models:\n",
        "\n",
        "   a. Text Encoder:\n",
        "   \n",
        "   Uses the SciBERT language model to encode the text description, followed by a linear projection to an embedding space and layer normalization.\n",
        "   \n",
        "   b. Molecule Encoder:\n",
        "    - MLP Encoder: Takes the Mol2vec embedding as input, passes it through a multi-layer perceptron (MLP), and projects to the joint embedding space.\n",
        "      \n",
        "    - GCN Encoder: Incorporates the molecular graph structure by using a Graph Convolutional Network (GCN) on the Mol2vec token embeddings as node features.\n",
        "\n",
        "   \n",
        "   The text and molecule embeddings are mapped to an aligned semantic space, where cosine similarity is used to retrieve/rank molecules given a text query.\n",
        "\n",
        "2. Cross-Modal Attention Model (shown in Figure 3):\n",
        "   - Uses a transformer decoder with cross-modal attention between the text (from SciBERT) and molecule representations (from the GCN encoder).\n",
        "   - Allows learning \"association rules\" between text tokens and molecular substructures from the attention weights.\n",
        "   - Association rules are used for explainability and to rerank retrieved molecules.\n",
        "   <img src=\"https://drive.google.com/uc?id=1_b4MWeiDDRpDKtY44MJMQEaUSZtWjChI\" width=500 />\n",
        "   <br>\n",
        "   <b>Figure 3: Model architecture for the cross-modal attention extension and association rules. </b>\n",
        "   <br>\n",
        "\n",
        "3. Ensemble Model:\n",
        "   - Takes a weighted average of the rankings from different base model instances (e.g. MLP1, MLP2, GCN1, etc.) to create an ensemble ranking.\n",
        "   $$\n",
        "S(m) = \\sum_i w_i R_i(m) \\ \\ \\ \\ \\ \\ \\ \\ s.t. \\sum_i w_i = 1\n",
        "$$\n",
        "<b>Score as a weighted average for some molecule m where $R_i$ is the rank assigned to that molecule by model $i$ and $w_i$ is the model weight.</b><br>\n",
        "   - Improves performance significantly by combining models trained with different initializations.\n",
        "\n",
        "4. Loss Functions:\n",
        "\n",
        "   a. Base Models: Symmetric contrastive loss adapted from CLIP, using the other samples in a minibatch as negatives.\n",
        "   \n",
        "   b. Cross-Modal Attention: Modified contrastive loss incorporating random negative text descriptions to force cross-modal integration.\n",
        "\n",
        "The models are trained on the ChEBI-20 dataset, with molecules represented as SMILES strings or Mol2vec embeddings, and evaluated on molecular retrieval metrics like mean reciprocal rank."
      ],
      "metadata": {
        "id": "3muyDPFPbozY"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "This class Model defines a neural network model for processing text and molecule data. It consists of layers for text and molecule processing, including linear layers, activation functions, layer normalization, and dropout. The text data is processed using a BERT-based transformer model, while the molecule data is processed through fully connected layers. The model outputs scaled representations of text and molecule data."
      ],
      "metadata": {
        "id": "0xbdezr18o1o"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class Model(nn.Module):\n",
        "    def __init__(self, ntoken, ninp, nout, nhid, dropout=0.5):\n",
        "        super(Model, self).__init__()\n",
        "\n",
        "        # Define layers for text processing\n",
        "        self.text_hidden1 = nn.Linear(ninp, nout)  # Linear layer for text input\n",
        "\n",
        "        # Define parameters\n",
        "        self.ninp = ninp  # Dimension of input embeddings\n",
        "        self.nhid = nhid  # Dimension of hidden layers\n",
        "        self.nout = nout  # Dimension of output layer\n",
        "\n",
        "        # Dropout layer\n",
        "        self.drop = nn.Dropout(p=dropout)\n",
        "\n",
        "        # Define layers for molecule processing\n",
        "        self.mol_hidden1 = nn.Linear(nout, nhid)  # First hidden layer for molecule input\n",
        "        self.mol_hidden2 = nn.Linear(nhid, nhid)  # Second hidden layer for molecule input\n",
        "        self.mol_hidden3 = nn.Linear(nhid, nout)  # Output layer for molecule input\n",
        "\n",
        "        # Temperature parameter for scaling logits\n",
        "        self.temp = nn.Parameter(torch.Tensor([0.07]))\n",
        "        self.register_parameter('temp', self.temp)\n",
        "\n",
        "        # Layer normalization for text and molecule representations\n",
        "        self.ln1 = nn.LayerNorm(nout)  # LayerNorm for molecule representation\n",
        "        self.ln2 = nn.LayerNorm(nout)  # LayerNorm for text representation\n",
        "\n",
        "        # Activation functions\n",
        "        self.relu = nn.ReLU()\n",
        "        self.selu = nn.SELU()\n",
        "\n",
        "        # List to store parameters excluding those from the BERT model\n",
        "        self.other_params = list(self.parameters())  # Get all parameters except those from BERT\n",
        "\n",
        "        # Load BERT-based transformer model for text representation\n",
        "        self.text_transformer_model = BertModel.from_pretrained('allenai/scibert_scivocab_uncased')\n",
        "        self.text_transformer_model.train()\n",
        "\n",
        "    def forward(self, text, molecule, text_mask=None, molecule_mask=None):\n",
        "        # Forward pass of the model\n",
        "\n",
        "        # Process text input using BERT-based transformer model\n",
        "        text_encoder_output = self.text_transformer_model(text, attention_mask=text_mask)\n",
        "        text_x = text_encoder_output['pooler_output']  # Extract text representation from BERT pooler output\n",
        "        text_x = self.text_hidden1(text_x)  # Apply linear transformation to text representation\n",
        "\n",
        "        # Process molecule input through fully connected layers\n",
        "        x = self.relu(self.mol_hidden1(molecule))  # First hidden layer with ReLU activation\n",
        "        x = self.relu(self.mol_hidden2(x))  # Second hidden layer with ReLU activation\n",
        "        x = self.mol_hidden3(x)  # Output layer for molecule input\n",
        "\n",
        "        # Apply layer normalization\n",
        "        x = self.ln1(x)  # LayerNorm for molecule representation\n",
        "        text_x = self.ln2(text_x)  # LayerNorm for text representation\n",
        "\n",
        "        # Scale logits using temperature parameter\n",
        "        x = x * torch.exp(self.temp)  # Apply temperature scaling to molecule representation\n",
        "        text_x = text_x * torch.exp(self.temp)  # Apply temperature scaling to text representation\n",
        "\n",
        "        return text_x, x  # Return text and molecule representations"
      ],
      "metadata": {
        "id": "Hh1IiHDF84Q1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "This code creates an instance of the `Model` class, which represents a neural network model for processing text and molecule data. The parameters passed to the constructor (`ntoken`, `ninp`, `nhid`, and `nout`) define the architecture of the model. In this specific instantiation:\n",
        "- `ntoken` is set to the size of the vocabulary used by the text tokenizer (`gt.text_tokenizer.vocab_size`).\n",
        "- `ninp` is set to `768`, which is the dimensionality of the input embeddings typically used in BERT-based models.\n",
        "- `nhid` is set to `600`, representing the dimensionality of the hidden layers.\n",
        "- `nout` is set to `300`, representing the dimensionality of the output layer."
      ],
      "metadata": {
        "id": "v16HwwmU9HTn"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "ninp = 768\n",
        "nhid = 600\n",
        "nout = 300"
      ],
      "metadata": {
        "id": "m55HXJeFaUgX"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Instantiate the Model class with the specified parameters\n",
        "# Parameters:\n",
        "# - ntoken: Size of the vocabulary for the text tokenizer\n",
        "# - ninp: Dimensionality of the input embeddings (768 for BERT-based models)\n",
        "# - nhid: Dimensionality of the hidden layers\n",
        "# - nout: Dimensionality of the output layer\n",
        "model = Model(ntoken=gt.text_tokenizer.vocab_size, ninp=ninp, nhid=nhid, nout=nout)"
      ],
      "metadata": {
        "id": "gnq0SEYJ_WQB",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 49,
          "referenced_widgets": [
            "909ae40bdf2e4c77acf00099e6e7e481",
            "93ab468beae5423ca919503967c2131b",
            "cce7f3124c0543e48066e09ded4c442d",
            "5daad6f309b64e51a842cfe805c579d2",
            "d6bcb10abda441e9a6850c86fdecb81d",
            "e8eaf998fb4e4ad188cd99aae4c3febb",
            "4c3bbf4f8dd04c728e39ff0fa657a814",
            "8e609dfe6f5d4a2c9eec1d924e33d373",
            "e2c2ef64ea544924a1a7e021d1320911",
            "248d6f66766948b5b9acbec327cb0660",
            "f631e904debe4ee4bfa658b59807f41b"
          ]
        },
        "outputId": "c87a9f9e-fce7-4aff-d468-a48709abecea"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "pytorch_model.bin:   0%|          | 0.00/442M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "909ae40bdf2e4c77acf00099e6e7e481"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Model Descriptions and Implementation Code"
      ],
      "metadata": {
        "id": "sDIvMPTN36xA"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### MLP molecule encoder"
      ],
      "metadata": {
        "id": "jcXOF4ILt4eg"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Multi-layer perceptron (MLP) is one of two architectures for molecule encoding. MLP takes two different kinds of layers for the Mol2vec embedding, one kind of layer for the molecule processing (as defined in `self.mol_hidden1` and `self.mol_hidden2`) and another kind of layer for the text processing (as defined in `self.text_hidden1`). For the molecular input, the model applies that input through linear projection (as defined in `self.mol_hidden3`) and layer normalization (as defined in `self.ln1`), and for the text input, the model applies the BERT-based transformer model (as defined in `self.text_transformer_model`) and layer normalization (as defined in `self.ln2`). Together, both the word embeddings and the molecular representation create a trainable representation of the input Mol2vec embedding."
      ],
      "metadata": {
        "id": "Mhll14FXGctR"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class MLPModel(nn.Module):\n",
        "    def __init__(self, ninp, nout, nhid):\n",
        "        super(MLPModel, self).__init__()\n",
        "\n",
        "        # Define layers for text processing\n",
        "        self.text_hidden1 = nn.Linear(ninp, nout)\n",
        "\n",
        "        # Define parameters\n",
        "        self.ninp = ninp  # Dimension of input embeddings\n",
        "        self.nhid = nhid  # Dimension of hidden layers\n",
        "        self.nout = nout  # Dimension of output layer\n",
        "\n",
        "        # Define layers for molecule processing\n",
        "        self.mol_hidden1 = nn.Linear(nout, nhid)  # First hidden layer for molecule input\n",
        "        self.mol_hidden2 = nn.Linear(nhid, nhid)  # Second hidden layer for molecule input\n",
        "        self.mol_hidden3 = nn.Linear(nhid, nout)  # Output layer for molecule input\n",
        "\n",
        "        # Temperature parameter for scaling logits\n",
        "        self.temp = nn.Parameter(torch.Tensor([0.07]))\n",
        "        self.register_parameter('temp', self.temp)\n",
        "\n",
        "        # Layer normalization for text and molecule representations\n",
        "        self.ln1 = nn.LayerNorm(nout)  # LayerNorm for molecule representation\n",
        "        self.ln2 = nn.LayerNorm(nout)  # LayerNorm for text representation\n",
        "\n",
        "        # Activation functions\n",
        "        self.relu = nn.ReLU()\n",
        "        self.selu = nn.SELU()\n",
        "\n",
        "        # List to store parameters excluding those from the BERT model\n",
        "        self.other_params = list(self.parameters())\n",
        "\n",
        "        # Load BERT-based transformer model for text representation\n",
        "        self.text_transformer_model = BertModel.from_pretrained('allenai/scibert_scivocab_uncased')\n",
        "        self.text_transformer_model.train()\n",
        "\n",
        "    def forward(self, text, molecule, text_mask = None):\n",
        "        \"\"\"Forward pass of the model\"\"\"\n",
        "\n",
        "        # Process text input using BERT-based transformer model\n",
        "        text_encoder_output = self.text_transformer_model(text, attention_mask=text_mask)\n",
        "        text_x = text_encoder_output['pooler_output']  # Extract text representation from BERT pooler output\n",
        "        text_x = self.text_hidden1(text_x)  # Apply linear transformation to text representation\n",
        "\n",
        "        # Process molecule input through fully connected layers\n",
        "        x = self.relu(self.mol_hidden1(molecule))  # First hidden layer with ReLU activation\n",
        "        x = self.relu(self.mol_hidden2(x))  # Second hidden layer with ReLU activation\n",
        "        x = self.mol_hidden3(x)  # Output layer for molecule input\n",
        "\n",
        "        # Apply layer normalization\n",
        "        x = self.ln1(x)  # LayerNorm for molecule representation\n",
        "        text_x = self.ln2(text_x)  # LayerNorm for text representation\n",
        "\n",
        "        # Scale logits using temperature parameter\n",
        "        x = x * torch.exp(self.temp)  # Apply temperature scaling to molecule representation\n",
        "        text_x = text_x * torch.exp(self.temp)  # Apply temperature scaling to text representation\n",
        "\n",
        "        return text_x, x # Return text and molecule representations"
      ],
      "metadata": {
        "id": "6u_BopHgZ6c2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### GCN molecule encoder"
      ],
      "metadata": {
        "id": "J1fh_Mort6IE"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Graph convolutional network (GCN) is one of two architectures for molecule encoding. Unlike MLP, however, GCN explicitly takes in the molecular graph as input with the Mol2vec token embeddings as features instead of directly taking in Mol2vec embeddings as input. GCN runs the aforementioned token features into a three-layer GCN (as defined in `self.conv1`, `self.conv2`, and `self.conv3`) in order to create node representations for each atom in a given molecule. These node representations are then passed into a readout layer via global mean pooling in order to produce a new input for molecule processing. Through this approach, the model can explicitly learn the graph structure.\n",
        "\n",
        "Then, like MLP, GCN takes two different kinds of layers for the Mol2vec embedding, one kind of layer for the molecule processing (as defined in `self.mol_hidden1` and `self.mol_hidden2`) and another kind of layer for the text processing (as defined in `self.text_hidden1`). For the molecular input, the model applies that input through linear projection (as defined in `self.mol_hidden3`) and layer normalization (as defined in `self.ln1`), and for the text input, the model applies the BERT-based transformer model (as defined in `self.text_transformer_model`) and layer normalization (as defined in `self.ln2`). Together, both the word embeddings and the molecular representation create a trainable representation of the input Mol2vec embedding."
      ],
      "metadata": {
        "id": "4MCJQ1lBVGTK"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class GCNModel(nn.Module):\n",
        "    def __init__(self, num_node_features, ninp, nout, nhid, graph_hidden_channels):\n",
        "        super(GCNModel, self).__init__()\n",
        "\n",
        "        # Define layers for text processing\n",
        "        self.text_hidden1 = nn.Linear(ninp, nout)\n",
        "\n",
        "        # Define parameters\n",
        "        self.ninp = ninp  # Dimension of input embeddings\n",
        "        self.nhid = nhid  # Dimension of hidden layers\n",
        "        self.nout = nout  # Dimension of output layer\n",
        "\n",
        "        # Temperature parameter for scaling logits\n",
        "        self.temp = nn.Parameter(torch.Tensor([0.07]))\n",
        "        self.register_parameter('temp', self.temp)\n",
        "\n",
        "        # Layer normalization for text and molecule representations\n",
        "        self.ln1 = nn.LayerNorm(nout)  # LayerNorm for molecule representation\n",
        "        self.ln2 = nn.LayerNorm(nout)  # LayerNorm for text representation\n",
        "\n",
        "        # Activation functions\n",
        "        self.relu = nn.ReLU()\n",
        "        self.selu = nn.SELU()\n",
        "\n",
        "        # GCN Convolution layers\n",
        "        self.conv1 = GCNConv(num_node_features, graph_hidden_channels)\n",
        "        self.conv2 = GCNConv(graph_hidden_channels, graph_hidden_channels)\n",
        "        self.conv3 = GCNConv(graph_hidden_channels, graph_hidden_channels)\n",
        "\n",
        "        # Define layers for molecule processing\n",
        "        self.mol_hidden1 = nn.Linear(graph_hidden_channels, nhid)  # First hidden layer for molecule input\n",
        "        self.mol_hidden2 = nn.Linear(nhid, nhid)  # Second hidden layer for molecule input\n",
        "        self.mol_hidden3 = nn.Linear(nhid, nout)  # Output layer for molecule input\n",
        "\n",
        "        # List to store parameters excluding those from the BERT model\n",
        "        self.other_params = list(self.parameters())\n",
        "\n",
        "        # Load BERT-based transformer model for text representation\n",
        "        self.text_transformer_model = BertModel.from_pretrained('allenai/scibert_scivocab_uncased')\n",
        "        self.text_transformer_model.train()\n",
        "\n",
        "    def forward(self, text, graph_batch, text_mask=None, molecule_mask=None):\n",
        "        \"\"\"Forward pass of the model\"\"\"\n",
        "\n",
        "        # Process text input using BERT-based transformer model\n",
        "        text_encoder_output = self.text_transformer_model(text, attention_mask=text_mask)\n",
        "        text_x = text_encoder_output['pooler_output']  # Extract text representation from BERT pooler output\n",
        "        text_x = self.text_hidden1(text_x)  # Apply linear transformation to text representation\n",
        "\n",
        "        # Obtain node embeddings\n",
        "        x = graph_batch.x\n",
        "        edge_index = graph_batch.edge_index\n",
        "        batch = graph_batch.batch\n",
        "\n",
        "        # Process molecule token input through convolution layers\n",
        "        x = self.relu(self.conv1(x, edge_index)) # First convolution layer with ReLU activation\n",
        "        x = self.relu(self.conv2(x, edge_index)) # Second convolution layer with ReLU activation\n",
        "        x = self.conv3(x, edge_index) # Output convolution layer for molecule input\n",
        "\n",
        "        # Readout layer\n",
        "        x = global_mean_pool(x, batch)  # [batch_size, graph_hidden_channels]\n",
        "\n",
        "        # Process molecule input through fully connected layers\n",
        "        x = self.relu(mol_hidden1(x)) # First hidden layer with ReLU activation\n",
        "        x = self.relu(mol_hidden2(x)) # Second hidden layer with ReLU activation\n",
        "        x = self.mol_hidden3(x) # Output layer for molecule input\n",
        "\n",
        "        # Apply layer normalization\n",
        "        x = self.ln1(x)  # LayerNorm for molecule representation\n",
        "        text_x = self.ln2(text_x)  # LayerNorm for text representation\n",
        "\n",
        "        # Scale logits using temperature parameter\n",
        "        x = x * torch.exp(self.temp)  # Apply temperature scaling to molecule representation\n",
        "        text_x = text_x * torch.exp(self.temp)  # Apply temperature scaling to text representation\n",
        "\n",
        "        return text_x, x # Return text and molecule representations"
      ],
      "metadata": {
        "id": "fTGlFNyRZ65p"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### Cross-Modal Attention Model"
      ],
      "metadata": {
        "id": "AZF5AZ22t8Pr"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Cross-Modal Attention Model provides better explainability and reranking via attention as association rules. Like the GCN implementation code, the Cross-Modal Attention Model takes in the molecular graph as input with the Mol2vec token embeddings as features instead of directly taking in Mol2vec embeddings as input. The model runs the aforementioned token features into a three-layer GCN (as defined in `self.conv1`, `self.conv2`, and `self.conv3`) in order to create node representations for each atom in a given molecule. These node representations are then passed into a readout layer via global mean pooling in order to produce a new input for molecule processing.\n",
        "\n",
        "Then, like the previous models, this model takes two different kinds of layers for the Mol2vec embedding, one layer for the molecule processing (as defined in `self.mol_hidden1` and `self.mol_hidden2`) and another layer for the text processing (as defined in `self.text_hidden1` and `self.text_hidden2`). For the molecular input, the model applies that input through linear projection (as defined in `self.mol_hidden3`) and layer normalization (as defined in `self.ln1`). For the text input, the model applies the BERT-based transformer model (as defined in `self.text_transformer_model`) which serves as the source sequence, and then via a transformer decoder (as defined in `self.text_transformer_decoder`) uses the node representations from the three-layer GCN as the target sequence. Through this approach, attentions are extracted to learn the association between text and molecule. Together, both the word embeddings and the molecular representation create a trainable representation of the input Mol2vec embedding."
      ],
      "metadata": {
        "id": "XBZyuRn_yL3x"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class AttentionModel(nn.Module):\n",
        "\n",
        "    def __init__(self, num_node_features, ninp, nout, nhid, nhead, nlayers, graph_hidden_channels, mol_trunc_length, temp, dropout=0.5):\n",
        "        super(AttentionModel, self).__init__()\n",
        "\n",
        "        # Define layers for text processing\n",
        "        self.text_hidden1 = nn.Linear(ninp, nhid)\n",
        "        self.text_hidden2 = nn.Linear(nhid, nout)\n",
        "\n",
        "        # Define parameters\n",
        "        self.ninp = ninp  # Dimension of input embeddings\n",
        "        self.nhid = nhid  # Dimension of hidden layers\n",
        "        self.nout = nout  # Dimension of output layer\n",
        "        self.num_node_features = num_node_features # Number of node features\n",
        "        self.graph_hidden_channels = graph_hidden_channels # Number of graph hidden chanels\n",
        "        self.mol_trunc_length = mol_trunc_length # Allowable length in molecule\n",
        "\n",
        "        # Dropout layer\n",
        "        self.drop = nn.Dropout(p=dropout)\n",
        "\n",
        "        # Set up decoder\n",
        "        decoder_layers = TransformerDecoderLayer(ninp, nhead, nhid, dropout)\n",
        "        self.text_transformer_decoder = TransformerDecoder(decoder_layers, nlayers)\n",
        "\n",
        "        # Temperature parameter for scaling logits\n",
        "        self.temp = nn.Parameter(torch.Tensor([temp]))\n",
        "        self.register_parameter( 'temp' , self.temp )\n",
        "\n",
        "        # Layer normalization for text and molecule representations\n",
        "        self.ln1 = nn.LayerNorm(nout)  # LayerNorm for molecule representation\n",
        "        self.ln2 = nn.LayerNorm(nout)  # LayerNorm for text representation\n",
        "\n",
        "        # Activation functions\n",
        "        self.relu = nn.ReLU()\n",
        "        self.selu = nn.SELU()\n",
        "\n",
        "        # GCN Convolution layers\n",
        "        self.conv1 = GCNConv(self.num_node_features, graph_hidden_channels)\n",
        "        self.conv2 = GCNConv(graph_hidden_channels, graph_hidden_channels)\n",
        "        self.conv3 = GCNConv(graph_hidden_channels, graph_hidden_channels)\n",
        "\n",
        "        # Define layers for molecule processing\n",
        "        self.mol_hidden1 = nn.Linear(graph_hidden_channels, nhid)  # First hidden layer for molecule input\n",
        "        self.mol_hidden2 = nn.Linear(nhid, nout) # Output layer for molecule input\n",
        "\n",
        "        # List to store parameters excluding those from the BERT model\n",
        "        self.other_params = list(self.parameters())\n",
        "\n",
        "        # Load BERT-based transformer model for text representation\n",
        "        self.text_transformer_model = BertModel.from_pretrained('allenai/scibert_scivocab_uncased')\n",
        "        self.text_transformer_model.train()\n",
        "\n",
        "        self.device = 'cpu'\n",
        "\n",
        "    def set_device(self, dev):\n",
        "        self.to(dev)\n",
        "        self.device = dev\n",
        "\n",
        "    def forward(self, text, graph_batch, text_mask=None, molecule_mask=None):\n",
        "        \"\"\"Forward pass of the model\"\"\"\n",
        "\n",
        "        # Process text input using BERT-based transformer model\n",
        "        text_encoder_output = self.text_transformer_model(text, attention_mask=text_mask)\n",
        "\n",
        "        # Obtain node embeddings\n",
        "        x = graph_batch.x\n",
        "        edge_index = graph_batch.edge_index\n",
        "        batch = graph_batch.batch\n",
        "\n",
        "        # Process molecule input through convolution layers\n",
        "        x = self.relu(self.conv1(x, edge_index)) # First convolution layer with ReLU activation\n",
        "        x = self.relu(self.conv2(x, edge_index)) # Second convolution layer with ReLU activation\n",
        "        mol_x = self.conv3(x, edge_index) # Output layer for molecule input\n",
        "\n",
        "        # Turn pytorch geometric output into the correct format for transformer\n",
        "        # Requires recovering the nodes from each graph into a separate dimension\n",
        "        node_features = torch.zeros((graph_batch.num_graphs, self.mol_trunc_length, self.graph_hidden_channels)).to(self.device)\n",
        "        for i, p in enumerate(graph_batch.ptr):\n",
        "            if p == 0:\n",
        "                old_p = p\n",
        "                continue\n",
        "            node_features[i - 1, :p-old_p, :] = mol_x[old_p:torch.min(p, old_p + self.mol_trunc_length), :]\n",
        "            old_p = p\n",
        "        node_features = torch.transpose(node_features, 0, 1)\n",
        "\n",
        "        # Decode initial encoding\n",
        "        text_output = self.text_transformer_decoder(\n",
        "            text_encoder_output['last_hidden_state'].transpose(0,1),\n",
        "            node_features,\n",
        "            tgt_key_padding_mask=text_mask==0,\n",
        "            memory_key_padding_mask=~molecule_mask\n",
        "        )\n",
        "\n",
        "        # Readout layer\n",
        "        x = global_mean_pool(mol_x, batch)  # [batch_size, graph_hidden_channels]\n",
        "\n",
        "        # Process molecule input through fully connected layers\n",
        "        x = self.relu(self.mol_hidden1(x))\n",
        "        x = self.mol_hidden2(x)\n",
        "\n",
        "        # Extract text representation from CLS pooler output\n",
        "        text_x = torch.tanh(self.text_hidden1(text_output[0,:,:])) # [CLS] pooler\n",
        "        text_x = self.text_hidden2(text_x) # Apply linear transformation to text representation\n",
        "\n",
        "        # Apply layer normalization\n",
        "        x = self.ln1(x)  # LayerNorm for molecule representation\n",
        "        text_x = self.ln2(text_x)  # LayerNorm for text representation\n",
        "\n",
        "        # Scale logits using temperature parameter\n",
        "        x = x * torch.exp(self.temp)  # Apply temperature scaling to molecule representation\n",
        "        text_x = text_x * torch.exp(self.temp)  # Apply temperature scaling to text representation\n",
        "\n",
        "        return text_x, x  # Return text and molecule representations"
      ],
      "metadata": {
        "id": "wo3eGfSOZ7Qa"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Pretrained Models"
      ],
      "metadata": {
        "id": "YdkECU4CLMF9"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "The weights and embeddings of the pretrained model are provided here for your reference. In the evaluation metrics section, we load the pretrained model weights. Links for downloading each model are included to facilitate analysis and processing.\n",
        "\n",
        "1. [MLP1 Reproduce Weights](https://drive.google.com/file/d/17v3foaQJvnUBPrO-ioc-Bui1T5UtLWyK/view?usp=sharing)\n",
        "2. [GCN1 Reproduce Weights](https://drive.google.com/file/d/1gfsfvVB8iYjU_i9hqv_t4FjpWaZQB_af/view?usp=sharing)\n",
        "\n",
        "8. [MLP2 Reproduce Weights](https://drive.google.com/file/d/1BuKhDd8IALIZNuwHRVavj0tRai0Qmyuh/view?usp=sharing)\n",
        "7. [GCN2 Reproduce Weights](https://drive.google.com/file/d/1pbWFMyrTZBQAhqvUnkZDHbyIYaXJvj4j/view?usp=sharing)\n",
        "\n",
        "8. [MLP3 Reproduce Weights](https://drive.google.com/file/d/1L72QOt9qNw5lJgoRGnREqQNBhEcI77Au/view?usp=sharing)\n",
        "7. [GCN3 Reproduce Weights](https://drive.google.com/file/d/1DXxplaCS-DnG-kwuyJSuFjiTbvjgBo02/view?usp=sharing)\n",
        "\n",
        "8. [MLP4 Reproduce Weights](https://drive.google.com/file/d/1bhNeRZKkMIHOU-qB46IRimE0O3euAPRN/view?usp=sharing)\n",
        "7. [GCN4 Reproduce Weights](https://drive.google.com/file/d/19DNUPN581gkYTKooApnUW4HccgUPNxaD/view?usp=sharing)\n",
        "\n",
        "7. [MHA1: Reproduce Weights](https://drive.google.com/file/d/1S3AcCZK1C9JLZXa3MVvQ0kzOIj3yFSoK/view?usp=sharing)\n",
        "7. [MHA2: Reproduce Weights](https://drive.google.com/file/d/1S3AcCZK1C9JLZXa3MVvQ0kzOIj3yFSoK/view?usp=sharing)\n",
        "6. [MHA3: Reproduce Weights](https://drive.google.com/file/d/14-ECz6PqnqFjrcuUFzFYSD4e6Gcm7hgM/view?usp=sharing)\n",
        "\n",
        "1. [MLP1: Experiements Weights and Embeddings](https://drive.google.com/file/d/1ebDVr72e5ZnA9Mo9AZ03Ci4B79M7tu6n/view?usp=sharing)\n",
        "2. [MLP2: Experiements Weights and Embeddings](https://drive.google.com/file/d/1APEndZ0G-ZwkzrmUYQhn7S1orIx_4qf-/view?usp=sharing)\n",
        "3. [MLP3: Experiements Weights and Embeddings](https://drive.google.com/file/d/1y1nm8l3C8ugZoTOeJP0Qx1Sx4bA1OlpN/view?usp=sharing)\n",
        "4. [GCN1: Experiements Weights and Embeddings](https://drive.google.com/file/d/1KWbFEDSJZBZNaBRLIQxjFrRfHCmLNMo9/view?usp=sharing)\n",
        "5. [GCN2: Experiements Weights and Embeddings](https://drive.google.com/file/d/1tv6yYVhuNcYuIEQZQaW94kzvayGyAI8T/view?usp=sharing)\n",
        "9. [Experiements MLP1 Weights](https://drive.google.com/file/d/1qleJlAs6G6-GHgUFvcR3unkmvxyN7QWs/view?usp=sharing)\n",
        "10. [Experiements MLP2 Weights](https://drive.google.com/file/d/1rxEbt4XsZbv_xo0mQJzXxmk8cHPssWgs/view?usp=sharing)\n",
        "11. [Experiements MLP3 Weights](https://drive.google.com/file/d/1ASg580a7BIsWMteNnIJi_3-SLEv4YP6o/view?usp=sharing)\n",
        "\n",
        "12. [Ablation GCN1 Weights and Embeddings](https://drive.google.com/file/d/1KWbFEDSJZBZNaBRLIQxjFrRfHCmLNMo9/view?usp=sharing)\n",
        "13. [Ablation GCN2 Weights and Embeddings](https://drive.google.com/file/d/1tv6yYVhuNcYuIEQZQaW94kzvayGyAI8T/view?usp=sharing)"
      ],
      "metadata": {
        "id": "CiwRFDZKEa1r"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Training"
      ],
      "metadata": {
        "id": "F-QRUrzFALPC"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Hyperparameters"
      ],
      "metadata": {
        "id": "Z6MiV4vFPOx5"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "**The training uses the following hyperparameters:**\n",
        "\n",
        "Text Encoder:\n",
        "- Uses SciBERT model\n",
        "- Finetuning learning rate of 3e-5\n",
        "\n",
        "Molecule Encoders:\n",
        "- MLP: 600 hidden units\n",
        "- GCN: 3 layers\n",
        "\n",
        "Mol2vec Parameters:\n",
        "- Radius = 1 (for Morgan fingerprints)\n",
        "- Threshold for unknown tokens = 3  \n",
        "- Embedding dimension = 300\n",
        "- Window size = 10\n",
        "\n",
        "Training:\n",
        "- Adam optimizer\n",
        "- MLP/GCN learning rate = 1e-4\n",
        "- Linear annealing of learning rate with 1,000 warmup steps\n",
        "- Trained for 40 epochs\n",
        "- Batch size = 32\n",
        "- Temperature parameter τ = 0.07 (for contrastive loss)\n",
        "- Use first 256 text tokens\n",
        "\n",
        "Cross-Modal Attention Model:\n",
        "- 3 layer transformer decoder\n",
        "- Attends to first 512 molecule substructures\n",
        "- 128M parameters\n",
        "\n",
        "Association Rules:\n",
        "- Consider 1-to-1 rules with confidence > 0.1 and support > 2\n",
        "$$\n",
        "supp(r) = \\sum_{p \\in P} \\sum_{\\substack{t' \\in p_t \\\\ m' \\in p_m}} 𝟙_{\\substack{t=t' \\\\ m=m'}} a_{t', m'}\n",
        "$$\n",
        "<b>Support for a rule r from t (text token) to m (molecule token) as the sum of all attentions.</b>\n",
        "$$\n",
        "conf(t ⟹ m) = \\frac{supp(t,m)}{\\sum_{t' \\in T} supp(t',m)}\n",
        "$$\n",
        "<b>Confidence from every text token t to every molecule token m, divided by the support of all the fules using t, where T is the set of all text tokens.</b>\n",
        "\n",
        "- Taking top 10 confidence values for reranking\n",
        "\n",
        "The MLP has around 111M parameters and the GCN has 112M parameters."
      ],
      "metadata": {
        "id": "pA-la0adT6ob"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Computational Requirements"
      ],
      "metadata": {
        "id": "W9sum-UD92_w"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "**The computational requirements are as follows:**\n",
        "\n",
        "* The training used a combination of NVIDIA V100 and T4 GPUs for 40 epochs.\n",
        "* Training the MLP and GCN models took around 7 hours each on a V100 GPU and 13 hours each on a T4 GPU.\n",
        "*  Training the cross-modal attention model took around 9 hours on a V100 GPU and 14 hours on a T4 GPU.\n",
        "* Average runtime training used an NVIDIA V100 GPU take 10 minutes (T4 GPU take 18 minutes) for each epoch is for the MLP & GCN models and 14 minutes using NVIDIA V100 GPU (22 minutes using NVIDIA T4 GPU) for the Cross-Modal Attention Model"
      ],
      "metadata": {
        "id": "CYk_ZyCLUEaj"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Training Code"
      ],
      "metadata": {
        "id": "JvEZyKnjLDlL"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "In the following code:\n",
        "- An optimizer (Adam) is initialized to update model parameters during training. It uses different learning rates for parameters of the main model (`model.other_params`) and parameters of the BERT-based model (`bert_params`).\n",
        "- A linear learning rate scheduler with warmup is created. It adjusts the learning rate during training according to the specified warmup steps and total training steps."
      ],
      "metadata": {
        "id": "KvpUggyXF10n"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import torch.optim as optim  # Importing the optimizer module from PyTorch\n",
        "from transformers.optimization import get_linear_schedule_with_warmup  # Importing the learning rate scheduler from the transformers library\n",
        "\n",
        "# Define the number of epochs for training\n",
        "epochs = 1\n",
        "\n",
        "# Initial learning rate for the optimizer\n",
        "init_lr = 1e-4\n",
        "\n",
        "# Learning rate for the BERT-based model\n",
        "bert_lr = 3e-5\n",
        "\n",
        "# Get the parameters of the BERT-based model\n",
        "bert_params = list(model.text_transformer_model.parameters())\n",
        "\n",
        "# Initialize the optimizer with Adam optimizer\n",
        "# Separate learning rates can be specified for different parameter groups\n",
        "optimizer = optim.Adam([\n",
        "                {'params': model.other_params},  # Parameters excluding those from BERT\n",
        "                {'params': bert_params, 'lr': bert_lr}  # Parameters of the BERT model with custom learning rate\n",
        "            ], lr=init_lr)  # Initial learning rate for all parameters\n",
        "\n",
        "# Define the number of warmup steps for the scheduler\n",
        "num_warmup_steps = 1000\n",
        "\n",
        "# Calculate the total number of training steps\n",
        "num_training_steps = epochs * len(training_generator) - num_warmup_steps\n",
        "\n",
        "# Create a linear scheduler with warmup\n",
        "scheduler = get_linear_schedule_with_warmup(optimizer,\n",
        "                                            num_warmup_steps=num_warmup_steps,\n",
        "                                            num_training_steps=num_training_steps)"
      ],
      "metadata": {
        "id": "k2gOnzlABE5k"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "In the following code:\n",
        "- The first line checks if CUDA (GPU) is available. If it is, the device is set to the first GPU (`\"cuda:0\"`); otherwise, it defaults to the CPU (`\"cpu\"`).\n",
        "- The second line prints out the selected device.\n",
        "- The third line moves (or transfers) the model (`model`) to the selected device. This means that all computations involving the model will be performed on this device. If CUDA (GPU) is available, the model is transferred to the GPU; otherwise, it remains on the CPU."
      ],
      "metadata": {
        "id": "9z3rBnDTGNMP"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Check if CUDA (GPU) is available, and set the device accordingly\n",
        "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "# Print the selected device (CUDA/GPU or CPU)\n",
        "print(device)\n",
        "\n",
        "# Transfer the model to the selected device (CUDA/GPU or CPU)\n",
        "tmp = model.to(device)"
      ],
      "metadata": {
        "id": "dI6vkDPeGQvh",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "69fe5826-3193-4809-a6a6-a73690323ae5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "cuda:0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "In the following code:\n",
        "- The `nn.CrossEntropyLoss()` function defines the Cross Entropy Loss criterion, which is commonly used for classification tasks.\n",
        "- The `loss_func` function takes two vectors `v1` and `v2` and computes the loss between them. It does this by first computing the logits (unnormalized scores) using matrix multiplication between `v1` and the transpose of `v2`. Then, it generates labels based on the number of rows in the logits. Finally, it computes the loss using Cross Entropy Loss for the original logits and their transposition and sums both losses together. This is a customized loss function tailored for the specific task or model."
      ],
      "metadata": {
        "id": "BxyeNt8mGoN7"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Define the loss function using Cross Entropy Loss\n",
        "criterion = nn.CrossEntropyLoss()\n",
        "\n",
        "# Custom loss function that computes the loss between two vectors (v1 and v2)\n",
        "def loss_func(v1, v2):\n",
        "    # Compute logits by matrix multiplication between v1 and the transpose of v2\n",
        "    logits = torch.matmul(v1, torch.transpose(v2, 0, 1))\n",
        "\n",
        "    # Generate labels based on the number of rows in logits\n",
        "    labels = torch.arange(logits.shape[0]).to(device)\n",
        "\n",
        "    # Compute the loss using Cross Entropy Loss for the original logits and their transposition\n",
        "    # and sum both losses\n",
        "    return criterion(logits, labels) + criterion(torch.transpose(logits, 0, 1), labels)"
      ],
      "metadata": {
        "id": "tqeiMeiuGrLN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "This code trains and validates a neural network model for multiple epochs using the specified training and validation data generators. During training, it records training and validation losses and accuracies for each epoch. It also saves the model weights after each epoch if the validation loss improves. Finally, it saves the final model weights after training all epochs."
      ],
      "metadata": {
        "id": "izbBZ5fYGujm"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Lists to store training and validation losses\n",
        "train_losses = []\n",
        "val_losses = []\n",
        "\n",
        "# Lists to store training and validation accuracies\n",
        "train_acc = []\n",
        "val_acc = []\n",
        "\n",
        "# Directory to save model outputs\n",
        "mounted_path = \"MLP_outputs/\"\n",
        "\n",
        "# Create the directory if it doesn't exist\n",
        "if not os.path.exists(mounted_path):\n",
        "    os.mkdir(mounted_path)\n",
        "\n",
        "# Loop over epochs\n",
        "for epoch in range(epochs):\n",
        "    # Training\n",
        "    start_time = time.time()  # Record the start time of the epoch\n",
        "    running_loss = 0.0  # Initialize running loss\n",
        "    running_acc = 0.0  # Initialize running accuracy\n",
        "    model.train()  # Set the model to training mode\n",
        "    for i, d in enumerate(training_generator):\n",
        "        batch, labels = d  # Retrieve batch data and labels\n",
        "        # Transfer batch data to GPU\n",
        "        text_mask = batch['text']['attention_mask'].bool()  # Retrieve attention mask for text\n",
        "        text = batch['text']['input_ids'].to(device)  # Transfer text input to GPU\n",
        "        text_mask = text_mask.to(device)  # Transfer text mask to GPU\n",
        "        molecule = batch['molecule']['mol2vec'].float().to(device)  # Transfer molecule input to GPU\n",
        "\n",
        "        # Forward pass\n",
        "        text_out, chem_out = model(text, molecule, text_mask)  # Get model outputs\n",
        "        loss = loss_func(text_out, chem_out).to(device)  # Calculate loss\n",
        "        running_loss += loss.item()  # Accumulate loss\n",
        "\n",
        "        # Backward pass and optimization\n",
        "        optimizer.zero_grad()  # Clear gradients\n",
        "        loss.backward()  # Backpropagation\n",
        "        optimizer.step()  # Optimization step\n",
        "\n",
        "        scheduler.step()  # Update learning rate scheduler\n",
        "\n",
        "        # Print progress every 100 batches\n",
        "        if (i+1) % 100 == 0:\n",
        "            print(i+1, \"batches trained. Avg loss:\\t\", running_loss / (i+1), \". Avg ms/step =\", 1000*(time.time()-start_time)/(i+1))\n",
        "\n",
        "    # Calculate average training loss and accuracy for the epoch\n",
        "    train_losses.append(running_loss / (i+1))\n",
        "    train_acc.append(running_acc / (i+1))\n",
        "\n",
        "    # Print training loss and duration for the epoch\n",
        "    print(\"Epoch\", epoch, \"training loss:\\t\\t\", running_loss / (i+1), \". Time =\", (time.time()-start_time), \"seconds.\")\n",
        "\n",
        "    # Validation\n",
        "    model.eval()  # Set the model to evaluation mode\n",
        "    with torch.set_grad_enabled(False):  # Disable gradient calculation\n",
        "        start_time = time.time()  # Record the start time of the epoch\n",
        "        running_acc = 0.0  # Initialize running accuracy\n",
        "        running_loss = 0.0  # Initialize running loss\n",
        "        for i, d in enumerate(validation_generator):\n",
        "            batch, labels = d  # Retrieve batch data and labels\n",
        "            # Transfer batch data to GPU\n",
        "            text_mask = batch['text']['attention_mask'].bool()  # Retrieve attention mask for text\n",
        "            text = batch['text']['input_ids'].to(device)  # Transfer text input to GPU\n",
        "            text_mask = text_mask.to(device)  # Transfer text mask to GPU\n",
        "            molecule = batch['molecule']['mol2vec'].float().to(device)  # Transfer molecule input to GPU\n",
        "\n",
        "            # Forward pass\n",
        "            text_out, chem_out = model(text, molecule, text_mask)  # Get model outputs\n",
        "            loss = loss_func(text_out, chem_out).to(device)  # Calculate loss\n",
        "            running_loss += loss.item()  # Accumulate loss\n",
        "\n",
        "            # Print progress every 100 batches\n",
        "            if (i+1) % 100 == 0:\n",
        "                print(i+1, \"batches eval. Avg loss:\\t\", running_loss / (i+1), \". Avg ms/step =\", 1000*(time.time()-start_time)/(i+1))\n",
        "\n",
        "        # Calculate average validation loss and accuracy for the epoch\n",
        "        val_losses.append(running_loss / (i+1))\n",
        "        val_acc.append(running_acc / (i+1))\n",
        "\n",
        "        # Save the model with the lowest validation loss\n",
        "        min_loss = np.min(val_losses)\n",
        "        if val_losses[-1] == min_loss:\n",
        "            torch.save(model.state_dict(), mounted_path + 'weights_pretrained.{epoch:02d}-{min_loss:.2f}.pt'.format(epoch = epoch, min_loss = min_loss))\n",
        "\n",
        "    # Print validation loss and duration for the epoch\n",
        "    print(\"Epoch\", epoch, \"validation loss:\\t\", running_loss / (i+1), \". Time =\", (time.time()-start_time), \"seconds.\")\n",
        "\n",
        "# Save the final model weights\n",
        "torch.save(model.state_dict(), mounted_path + \"final_weights.\"+str(epochs)+\".pt\")"
      ],
      "metadata": {
        "id": "w9QCWLARItU7",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "95d5369f-d7a6-4240-d07d-76b684dcce92"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 0 training loss:\t\t 33.4815149307251 . Time = 2.7659153938293457 seconds.\n",
            "Epoch 0 validation loss:\t 32.926008224487305 . Time = 2.233452320098877 seconds.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Evaluation"
      ],
      "metadata": {
        "id": "ZBv6u2zdVEJ3"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Metrics Descriptions"
      ],
      "metadata": {
        "id": "-Y7bCGVTAtah"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "\n",
        "\n",
        "The paper evaluates the proposed Text2Mol methods using the following metrics:\n",
        "\n",
        "1. Mean Reciprocal Rank (MRR):\n",
        "This is the main evaluation metric used. It is calculated as:\n",
        "\n",
        "$$MRR = \\frac{1}{n} \\sum _{i=1} ^n \\frac{1}{R_i}$$\n",
        "\n",
        "- Where n is the number of queries, and $R_i$ is the rank of the correct (relevant) molecule for the $i$-th query text description.\n",
        "\n",
        "- Higher MRR values are better, with a perfect MRR of 1.0 if the correct molecule is ranked 1st for every query.\n",
        "\n",
        "2. Hits@K:\n",
        "This measures the percentage of queries for which the correct molecule is ranked among the top K results. It is calculated as:\n",
        "\n",
        "$$\\text{Hits}@K = \\frac{1}{n} \\sum _{i=1} ^n 1 _{R_i \\le K}$$\n",
        "\n",
        "- Specifically, the paper report Hits@1 and Hits@10.\n",
        "\n",
        "- Hits@1 is the percentage of queries where the correct molecule is ranked 1st.\n",
        "\n",
        "- Hits@10 is the percentage where the correct molecule appears in the top 10 rankings.\n",
        "\n",
        "3. Mean Rank:\n",
        "This is a secondary metric which reports the average rank of the correct molecules across all queries. It is calculated as:\n",
        "\n",
        "$$ \\text{MeanRank} = \\frac{1}{n} \\sum _{i=1} ^n R_i$$\n",
        "\n",
        "- Where n is the number of queries, and $R_i$ is the rank of the correct (relevant) molecule for the $i$-th query text description.\n",
        "\n",
        "- A lower mean rank value is better.\n",
        "\n",
        "The metrics are calculated on the test set of the ChEBI-20 dataset containing 33,010 text-molecule pairs split into train/val/test.\n",
        "\n",
        "The paper reports achieving an MRR of 0.499, Hits@1 of 34.4%, and Hits@10 of 81.1% on the test set using their best ensemble model, significantly outperforming baselines.\n",
        "\n",
        "MRR is the primary ranking metric, supplemented by Hits@K percentages and mean rank, evaluated on the held-out test portion of their new ChEBI-20 benchmark dataset."
      ],
      "metadata": {
        "id": "3YpQqZi6Cpcn"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Evaluation code"
      ],
      "metadata": {
        "id": "6Dkx5qk0U2d9"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "The following code loads embeddings and identifiers for chemical compounds."
      ],
      "metadata": {
        "id": "4XMJesWEJdYb"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from os import path as osp\n",
        "\n",
        "# Load training, validation, and test data for chemical compound identifiers (cids)\n",
        "cids_train = np.load(\"data/cids_train.npy\", allow_pickle=True)\n",
        "cids_val = np.load(\"data/cids_val.npy\", allow_pickle=True)\n",
        "cids_test = np.load(\"data/cids_test.npy\", allow_pickle=True)\n",
        "\n",
        "\n",
        "# Load training, validation, and test data for text embeddings\n",
        "text_embeddings_train = np.load(\"data/text_embeddings_train.npy\", allow_pickle=True)\n",
        "text_embeddings_val = np.load(\"data/text_embeddings_val.npy\", allow_pickle=True)\n",
        "text_embeddings_test = np.load(\"data/text_embeddings_test.npy\")\n",
        "\n",
        "# Load training, validation, and test data for chemical embeddings\n",
        "chem_embeddings_train = np.load(\"data/chem_embeddings_train.npy\", allow_pickle=True)\n",
        "chem_embeddings_val = np.load(\"data/chem_embeddings_val.npy\", allow_pickle=True)\n",
        "chem_embeddings_test = np.load(\"data/chem_embeddings_test.npy\", allow_pickle=True)\n",
        "\n",
        "# Print message indicating that embeddings have been loaded\n",
        "print('Loaded embeddings')\n",
        "\n",
        "# Combine text embeddings from all splits (train, val, test) into a single array\n",
        "all_text_embeddings = np.concatenate((text_embeddings_train, text_embeddings_val, text_embeddings_test), axis=0)\n",
        "# Combine chemical embeddings from all splits (train, val, test) into a single array\n",
        "all_mol_embeddings = np.concatenate((chem_embeddings_train, chem_embeddings_val, chem_embeddings_test), axis=0)\n",
        "\n",
        "# Concatenate all compound identifiers from train, val, and test sets into a single array\n",
        "all_cids = np.concatenate((cids_train, cids_val, cids_test), axis=0)\n",
        "\n",
        "# Calculate the number of samples in each split\n",
        "n_train = len(cids_train)\n",
        "n_val = len(cids_val)\n",
        "n_test = len(cids_test)\n",
        "\n",
        "# Calculate the total number of samples across all splits\n",
        "n = n_train + n_val + n_test\n",
        "\n",
        "# Define offsets for validation and test sets relative to the training set\n",
        "offset_val = n_train\n",
        "offset_test = n_train + n_val"
      ],
      "metadata": {
        "id": "zbCTcOMiCo1U",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1d48c717-a5d1-487c-9ceb-81be24d562e0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loaded embeddings\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "The following code defines a function memory_efficient_similarity_matrix_custom that calculates cosine similarity in a memory-efficient manner by processing data in chunks. It then applies this function to calculate cosine similarity between text embeddings and all molecule embeddings for the training, validation, and test sets."
      ],
      "metadata": {
        "id": "3aSELBqtJwQp"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Define a function to calculate cosine similarity in a memory-efficient manner\n",
        "def memory_efficient_similarity_matrix_custom(func, embedding1, embedding2, chunk_size=1000):\n",
        "    # Determine the number of rows in the first embedding array\n",
        "    rows = embedding1.shape[0]\n",
        "\n",
        "    # Calculate the number of chunks needed based on the chunk size\n",
        "    num_chunks = int(np.ceil(rows / chunk_size))\n",
        "\n",
        "    # Iterate over each chunk\n",
        "    for i in range(num_chunks):\n",
        "        # Determine the end index of the current chunk, accounting for the last chunk potentially being smaller\n",
        "        end_chunk = (i + 1) * chunk_size if (i + 1) * chunk_size < rows else rows\n",
        "\n",
        "        # Generate cosine similarity values for the current chunk and yield the result\n",
        "        yield func(embedding1[i * chunk_size:end_chunk, :], embedding2)\n",
        "\n",
        "# Calculate cosine similarity between text embeddings of the training set and all molecule embeddings\n",
        "text_chem_cos = memory_efficient_similarity_matrix_custom(cosine_similarity, text_embeddings_train, all_mol_embeddings)\n",
        "\n",
        "# Calculate cosine similarity between text embeddings of the validation set and all molecule embeddings\n",
        "text_chem_cos_val = memory_efficient_similarity_matrix_custom(cosine_similarity, text_embeddings_val, all_mol_embeddings)\n",
        "\n",
        "# Calculate cosine similarity between text embeddings of the test set and all molecule embeddings\n",
        "text_chem_cos_test = memory_efficient_similarity_matrix_custom(cosine_similarity, text_embeddings_test, all_mol_embeddings)\n"
      ],
      "metadata": {
        "id": "JVn6399qKD6i"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "The following code defines a function get_ranks to calculate ranks and update average ranks for samples in the training, validation, and test sets based on their cosine similarity scores. It iterates over the cosine similarity scores matrix and computes ranks for each sample, updating both individual ranks and average ranks arrays accordingly."
      ],
      "metadata": {
        "id": "RPWJvKVsKHO0"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Initialize arrays to store average ranks for each sample in the training, validation, and test sets\n",
        "# tr_avg_ranks = np.zeros((n_train, n))\n",
        "# val_avg_ranks = np.zeros((n_val, n))\n",
        "test_avg_ranks = np.zeros((n_test, n))\n",
        "\n",
        "# Initialize lists to store individual ranks for each sample in the training, validation, and test sets\n",
        "ranks_train = []\n",
        "ranks_val = []\n",
        "ranks_test = []\n",
        "\n",
        "# Define a function to calculate ranks and update average ranks\n",
        "def get_ranks(text_chem_cos, ranks_avg, offset, split=\"\"):\n",
        "    # Initialize a temporary list to store individual ranks\n",
        "    ranks_tmp = []\n",
        "    # Initialize a counter to keep track of all iterations\n",
        "    j = 0\n",
        "\n",
        "    # Iterate over each embedding in the cosine similarity matrix\n",
        "    for l, emb in enumerate(text_chem_cos):\n",
        "        # Iterate over each row in the embedding\n",
        "        for k in range(emb.shape[0]):\n",
        "            # Get the locations of the compound identifiers sorted by cosine similarity (descending order)\n",
        "            cid_locs = np.argsort(emb[k, :])[::-1]\n",
        "            # Get the ranks of the compound identifiers\n",
        "            ranks = np.argsort(cid_locs)\n",
        "\n",
        "            # Update the average ranks array by adding the ranks for the current sample\n",
        "            ranks_avg[j, :] = ranks_avg[j, :] + ranks\n",
        "\n",
        "            # Calculate the rank of the current sample\n",
        "            rank = ranks[j + offset] + 1\n",
        "            # Append the rank to the temporary list\n",
        "            ranks_tmp.append(rank)\n",
        "\n",
        "            # Increment the counter\n",
        "            j += 1\n",
        "            # Print progress message after processing every 1000 samples\n",
        "            if j % 1000 == 0:\n",
        "                print(j, split + \" processed\")\n",
        "\n",
        "    # Convert the temporary list of ranks to a numpy array and return it\n",
        "    return np.array(ranks_tmp)"
      ],
      "metadata": {
        "id": "8hn8rlI6Ka-T"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "The following code defines a function print_ranks to print statistics based on ranks, such as mean rank, hits at various ranks, and mean reciprocal rank (MRR). Then, it calculates ranks for the training, validation, and test sets using the get_ranks function and prints statistics for each set accordingly. Finally, it stores the ranks for each set in their respective variables (ranks_train, ranks_val, ranks_test)."
      ],
      "metadata": {
        "id": "ZhzU8XjmKdjb"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Define a function to print statistics based on ranks\n",
        "def print_ranks(ranks, split):\n",
        "    # Print the split type (e.g., \"Training\", \"Validation\", \"Test\")\n",
        "    print(split + \" Model:\")\n",
        "    # Print the mean rank\n",
        "    print(\"Mean rank:\", np.mean(ranks))\n",
        "    # Print the percentage of hits at ranks 1, 10, 100, 500, and 1000\n",
        "    print(\"Hits at 1:\", np.mean(ranks <= 1))\n",
        "    print(\"Hits at 10:\", np.mean(ranks <= 10))\n",
        "    print(\"Hits at 100:\", np.mean(ranks <= 100))\n",
        "    print(\"Hits at 500:\", np.mean(ranks <= 500))\n",
        "    print(\"Hits at 1000:\", np.mean(ranks <= 1000))\n",
        "    # Print the mean reciprocal rank (MRR)\n",
        "    print(\"MRR:\", np.mean(1 / ranks))\n",
        "    print()\n",
        "\n",
        "# Calculate ranks for the test set\n",
        "ranks_tmp = get_ranks(text_chem_cos_test, test_avg_ranks, offset=offset_test, split=\"test\")\n",
        "# Print statistics for the test set\n",
        "print_ranks(ranks_tmp, split=\"Test\")\n",
        "# Store the ranks for the test set\n",
        "ranks_test = ranks_tmp"
      ],
      "metadata": {
        "id": "Ch_yuaSHKzNg",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "cc5656bd-e57f-48ca-81a6-05b9ef913a5a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "1000 test processed\n",
            "2000 test processed\n",
            "3000 test processed\n",
            "Test Model:\n",
            "Mean rank: 24.18539836413208\n",
            "Hits at 1: 0.3477734019993941\n",
            "Hits at 10: 0.8382308391396547\n",
            "Hits at 100: 0.9760678582247804\n",
            "Hits at 500: 0.9933353529233565\n",
            "Hits at 1000: 0.9966676764616783\n",
            "MRR: 0.5105107506724086\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Results"
      ],
      "metadata": {
        "id": "PwQBw7mkVO41"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "The results are evaluated using the following metrics:\n",
        "\n",
        "1. **Mean Reciprocal Rank (MRR):**\n",
        "This is the main evaluation metric used. Higher MRR values are better, with a perfect MRR of 1.0 if the correct molecule is ranked 1st for every query.\n",
        "\n",
        "2. **Hits@K:**\n",
        "This measures the percentage of queries for which the correct molecule is ranked among the top K results.\n",
        "\n",
        "- Hits@1 is the percentage of queries where the correct molecule is ranked 1st.\n",
        "\n",
        "- Hits@10 is the percentage where the correct molecule appears in the top 10 rankings.\n",
        "\n",
        "3. **Mean Rank:**\n",
        "This is a metric which reports the average rank of the correct molecules across all queries. A lower mean rank value is better."
      ],
      "metadata": {
        "id": "U0fxJdYLRZo2"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Reproduction Results"
      ],
      "metadata": {
        "id": "DV-2vYl2pyJb"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### **Baseline Models:**"
      ],
      "metadata": {
        "id": "oyujrvNsn70H"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "The MLP and GCN encoders.\n",
        "\n",
        "**MLP Training Results**\n",
        "\n",
        "| Model | Mean Rank  | MRR   | Hits@1 | Hits@10 |\n",
        "|---    |---         |---    |---     |---      |\n",
        "|MLP1   | 3.61      | 0.604 | 43.2%  | 93.6%   |\n",
        "|MLP2   | 3.60      | 0.606 | 43.4%  | 93.8%   |\n",
        "|MLP3   | 3.59      | 0.608 | 43.8%  | 93.5%   |\n",
        "\n",
        "**GCN Training Results**\n",
        "\n",
        "| Model | Mean Rank | MRR   | Hits@1 | Hits@10 |\n",
        "|---    |---        |---    |---     |---      |\n",
        "|GCN1   | 3.77     | 0.604 | 43.3%  | 93.4%   |\n",
        "|GCN2   | 3.68     | 0.603 | 43.0%  | 93.3%   |\n",
        "|GCN3   | 3.85     | 0.600 | 42.4%  | 92.7%   |\n",
        "\n",
        "\n",
        "**MLP Test Results**\n",
        "\n",
        "| Model | Mean Rank  | MRR   | Hits@1 | Hits@10 |\n",
        "|---    |---         |---    |---     |---      |\n",
        "|MLP1   | 24.18      | 0.509 | 34.6%  | 83.7%   |\n",
        "|MLP2   | 24.20      | 0.502 | 33.4%  | 83.8%   |\n",
        "|MLP3   | 29.62      | 0.514 | 34.9%  | 84.0%   |\n",
        "\n",
        "**GCN Test Results**\n",
        "\n",
        "| Model | Mean Rank | MRR   | Hits@1 | Hits@10 |\n",
        "|---    |---        |---    |---     |---      |\n",
        "|GCN1   | 26.59     | 0.495 | 33.3%  | 82.3%   |\n",
        "|GCN2   | 25.09     | 0.498 | 33.5%  | 82.2%   |\n",
        "|GCN3   | 26.88     | 0.476 | 32.7%  | 81.9%   |"
      ],
      "metadata": {
        "id": "24rocx3nEG7Z"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### **Ensemble Approach:**"
      ],
      "metadata": {
        "id": "LdgmB_4JqIxL"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Ensembling multiple models with the same architecture (MLP or GCN).\n",
        "\n",
        "**Ensemble Training Results**\n",
        "\n",
        "| Model         | Mean Rank  | MRR   | Hits@1 | Hits@10 |\n",
        "|---            |---         |---    |---     |---      |\n",
        "|MLP-Ensemble   | 3.57      | 0.608 | 43.6%  | 93.7%   |\n",
        "|GCN-Ensemble   | 3.69      | 0.604 | 43.3%  | 93.4%   |\n",
        "\n",
        "**Ensemble Test Results**\n",
        "\n",
        "| Model         | Mean Rank  | MRR   | Hits@1 | Hits@10 |\n",
        "|---            |---         |---    |---     |---      |\n",
        "|MLP-Ensemble   | 24.18      | 0.517 | 35.3%  | 84.2%   |\n",
        "|GCN-Ensemble   | 25.06      | 0.499 | 33.7%  | 82.3%   |\n"
      ],
      "metadata": {
        "id": "VmEYBDY-N33u"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### **Cross-Architecture Ensemble:**"
      ],
      "metadata": {
        "id": "bXOMuFh2OLgv"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Ensembling across MLP and GCN architectures.\n",
        "\n",
        "**All-Ensemble Training Results**\n",
        "\n",
        "| Model         | Mean Rank  | MRR   | Hits@1 | Hits@10 |\n",
        "|---            |---         |---    |---     |---      |\n",
        "|All-Ensemble   | 3.61       | 0.606 | 43.5%  | 93.7%   |\n",
        "\n",
        "**All-Ensemble Test Results**\n",
        "\n",
        "| Model         | Mean Rank  | MRR   | Hits@1 | Hits@10 |\n",
        "|---            |---         |---    |---     |---      |\n",
        "|All-Ensemble   | 24.18      | 0.509 | 34.6%  | 83.8%   |"
      ],
      "metadata": {
        "id": "0qxyRAe-OPEy"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### **Multi-Headed Attention:**"
      ],
      "metadata": {
        "id": "t3qnyyDGMTbs"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Attention Mechanism Results.\n",
        "\n",
        "**Attention and FPGrowth Training Results**\n",
        "\n",
        "| Model             | Mean Rank  | MRR   | Hits@1 | Hits@10 |\n",
        "|---                |---         |---    |---     |---      |\n",
        "|MLP1+Attn1         | 3.62       | 0.605 | 43.3%  | 93.7%   |\n",
        "|MLP2+Attn2         | 3.62       | 0.605 | 43.3%  | 93.7%   |\n",
        "\n",
        "**Attention and FPGrowth Test Results**\n",
        "\n",
        "| Model             | Mean Rank  | MRR   | Hits@1 | Hits@10 |\n",
        "|---                |---         |---    |---     |---      |\n",
        "|MLP1+Attn1         | 32.19      | 0.501 | 33.3%  | 83.9%   |\n",
        "|MLP1+FPGrowth1     | 32.20      | 0.500 | 33.1%  | 83.9%   |\n",
        "|MLP2+Attn2         | 32.20      | 0.502 | 33.3%  | 83.9%   |\n",
        "|MLP2+FPGrowth2     | 32.20      | 0.500 | 33.1%  | 83.9%   |"
      ],
      "metadata": {
        "id": "z1AU2r-GMY3X"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Experiments and Ablation Study Results"
      ],
      "metadata": {
        "id": "5-5Rl3mMZ4LL"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### **Experiments Results:**"
      ],
      "metadata": {
        "id": "HSghBgCRq3y7"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "\n",
        "Additional experiments to test the impact of different hyperparameter settings.\n",
        "\n",
        "**MLP Results (1 Epoch)**\n",
        "\n",
        "| Model | Mean Rank  | MRR   | Hits@1 | Hits@10 |\n",
        "|---    |---         |---    |---     |---      |\n",
        "|MLP1 (600 hidden units, 300 Embeddings)   | 582.33      | 0.026 | 0.58%  | 5.2%   |\n",
        "|MLP2 (400 hidden units, 200 Embeddings)  | 650.64      | 0.013 | 0.36%  | 3.9%   |\n",
        "|MLP3 (200 hidden units, 100 Embeddings)  | 780.45      | 0.008 | 0.28%  | 1.9%   |\n",
        "\n",
        "\n",
        "**MLP Results (5 Epochs)**\n",
        "\n",
        "| Model | Mean Rank  | MRR   | Hits@1 | Hits@10 |\n",
        "|---    |---         |---    |---     |---      |\n",
        "|MLP1 (600 hidden units, 300 Embeddings)   | 121.60      | 0.108 | 4.1%  | 24.4%   |\n",
        "|MLP2 (400 hidden units, 200 Embeddings)  | 138.82      | 0.055 | 2.4%  | 15.9%   |\n",
        "|MLP3 (200 hidden units, 100 Embeddings)  | 160.53      | 0.023 | 1.8%  | 12.3%   |\n",
        "\n",
        "**MLP Results (30 Epochs)**\n",
        "\n",
        "| Model | Mean Rank  | MRR   | Hits@1 | Hits@10 |\n",
        "|---    |---         |---    |---     |---      |\n",
        "|MLP1 (600 hidden units, 300 Embeddings)   | 55.28      | 0.227 | 20.6%  | 58.7%   |\n",
        "|MLP2 (400 hidden units, 200 Embeddings)  | 90.45      | 0.175 | 15.4%  | 45.9%   |\n",
        "|MLP3 (200 hidden units, 100 Embeddings)  | 130.54      | 0.128 | 10.8%  | 30.3%   |\n",
        "\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "WwL2S_nWZ8VU"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### **Ablation Study Results:**"
      ],
      "metadata": {
        "id": "wec1gnIFralj"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Analyze the impact of each component of the model architecture: Contribution of the GCN module compared to using only Mol2vec.\n",
        "\n",
        "**Mol2Vec without GCN Training Results**\n",
        "\n",
        "| Model | Mean Rank  | MRR   | Hits@1 | Hits@10 |\n",
        "|---    |---         |---    |---     |---      |\n",
        "|MLP1   | 585.51      | 0.032 | 0.82%  | 6.5%   |\n",
        "|MLP2   | 580.56      | 0.035 | 0.81%  | 7.1%   |\n",
        "|MLP3   | 590.35      | 0.038 | 0.83%  | 7.0%   |\n",
        "\n",
        "**GCN Training Results**\n",
        "\n",
        "| Model | Mean Rank | MRR   | Hits@1 | Hits@10 |\n",
        "|---    |---        |---    |---     |---      |\n",
        "|GCN1   | 12.85     | 0.388 | 23.9%  | 67.1%   |\n",
        "|GCN2   | 12.99     | 0.365 | 23.5%  | 67.8%   |\n",
        "|GCN3   | 12.74     | 0.371 | 23.4%  | 67.5%   |\n",
        "\n",
        "\n",
        "**Mol2Vec without GCN Test Results**\n",
        "\n",
        "| Model | Mean Rank  | MRR   | Hits@1 | Hits@10 |\n",
        "|---    |---         |---    |---     |---      |\n",
        "|MLP1   | 681.47      | 0.031 | 0.87%  | 6.7%   |\n",
        "|MLP2   | 685.64      | 0.030 | 0.83%  | 6.6%   |\n",
        "|MLP3   | 690.56      | 0.029 | 0.84%  | 6.3%   |\n",
        "\n",
        "**GCN Test Results**\n",
        "\n",
        "| Model | Mean Rank | MRR   | Hits@1 | Hits@10 |\n",
        "|---    |---        |---    |---     |---      |\n",
        "|GCN1   | 57.58     | 0.285 | 20.7%  | 57.6%   |\n",
        "|GCN2   | 58.95     | 0.256 | 20.4%  | 56.9%   |\n",
        "|GCN3   | 55.47     | 0.294 | 20.8%  | 56.8%   |\n",
        "\n"
      ],
      "metadata": {
        "id": "qpFBHoZzrdOG"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Analysis"
      ],
      "metadata": {
        "id": "8EAWAy_LwHlV"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### **Analysis of Reproduction Results:**"
      ],
      "metadata": {
        "id": "JFMamK6_VasL"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "The MLP and GCN models exhibit complementary strengths, with GCN better at harder examples in training set, while MLP better on test set hard cases.\n",
        "\n",
        "Ensembling multiple models, especially across architectures, provides a substantial boost in generalization performance compared to the individual models. The MLP encoder slightly outperform the GCN, but combining them yields the best overall results. The attention mechanism does not significantly improve over the base ensemble models on this task.\n",
        "\n",
        "1. The MLP and GCN Encoders:\n",
        "- Both the MLP and GCN encoders achieve very good performance on the training set, with mean ranks around 3.6-3.8, MRR around 0.60-0.61, Hits@1 around 42-44%, and Hits@10 above 92%.\n",
        "- On the test set, the performance drops notably, with mean ranks increasing to around 24-30, MRR dropping to around 0.48-0.51, Hits@1 dropping to around 32-35%, and Hits@10 dropping to around 82-84%.\n",
        "- The MLP encoder perform slightly better than the GCN encoder on both the training and test sets across most metrics.\n",
        "\n",
        "2. Ensembling Same Architecture Models:\n",
        "- Ensembling multiple MLP or GCN models trained with different initializations provides a nice boost over the individual models.\n",
        "- For the MLP ensemble on the test set, the mean rank remains around 24, but MRR improves to 0.517, Hits@1 to 35.3%, and Hits@10 to 84.2%.\n",
        "- For the GCN ensemble, the mean rank is around 25, MRR is 0.499, Hits@1 is 33.7%, and Hits@10 is 82.3%.\n",
        "- Ensembling helps improve generalization by combining the strengths of separately trained models\n",
        "\n",
        "3. Ensembling Across Architectures:\n",
        "- Ensembling across both MLP and GCN architectures provides the best overall performance.\n",
        "- On the training set, the All-Ensemble mean rank is 3.61, MRR is 0.606, Hits@1 is 43.5%, and Hits@10 is 93.7% - very competitive with the individual architecture ensembles.\n",
        "- On the test set, the All-Ensemble mean rank is 24.18, MRR is 0.509, Hits@1 is 34.6%, and Hits@10 is 83.8%.\n",
        "- Combining both architecture types allows the ensemble to leverage their complementary strengths for improved generalization.\n",
        "\n",
        "4. Multi-Headed Attention Mechanism:\n",
        "- Adding an attention mechanism and reranking with FPGrowth association rules provides some improvements over the baselines on the training set, maintaining similar performance.\n",
        "- However, on the test set, the attention and FPGrowth models underperform compared to the baselines and ensembles.\n",
        "- Their mean ranks increase to around 32, MRR drops to around 0.50-0.51, Hits@1 drops to around 33%, and Hits@10 remains around 83.9%.\n",
        "- While the attention can provide explainability, it does not seem to significantly boost the ranking performance in this case."
      ],
      "metadata": {
        "id": "vMnolBfOMiim"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### **Analysis of Validation MRR Values:**"
      ],
      "metadata": {
        "id": "rL1MDbV5Vkny"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "<img src=\"https://drive.google.com/uc?id=1SnYWIHCrN0M229_xpj3E6y-6MeHcGRPT\" width=500 />\n",
        "<br>\n",
        "<b>\n",
        "Figure 4: Validation MRR values for different combinations of architectures. The axes indicate the number of each architecture used. Ensembles with both architectures are more effective.\n",
        "</b>\n",
        "<br>\n",
        "\n",
        "The analysis of the validation MRR values for different combinations of the MLP and GCN architectures in the ensemble approach is shown in Figure 4 above.\n",
        "\n",
        "Key observations from the analysis:\n",
        "\n",
        "1. Using a combination of both MLP and GCN models in the ensemble leads to higher validation MRR compared to using only one architecture.\n",
        "\n",
        "2. The validation MRR is clearly lower in the lower-left corners of the plot, where only rankings from one model are used (i.e., the other two models have zero weight).\n",
        "\n",
        "3. The best validation MRR is achieved when the weights are more balanced between the three models, rather than heavily skewed towards a single model.\n",
        "\n",
        "This demonstrates that the complementary strengths of the MLP and GCN architectures can be effectively leveraged through the ensemble approach. By combining the rankings from multiple models, the ensemble is able to outperform the individual models and achieve better overall retrieval performance on the validation set."
      ],
      "metadata": {
        "id": "r4vAFuPGM71m"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### **Analysis of queries that are predicted correctly by all-ensembles:**\n"
      ],
      "metadata": {
        "id": "and5tBZjO04I"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "\n",
        "<img src=\"https://drive.google.com/uc?id=1gHyuEf2mWQA-GbYt-kE8HOYqn3SUp6Gh\" width=500 />\n",
        "<br>\n",
        "<b>\n",
        "Figure 5: Example queries that are predicted correctly by All-Ensemble.\n",
        "</b>\n",
        "<br>\n",
        "\n",
        "The examples of queries that are predicted correctly by the All-Ensemble model, which combines the MLP and GCN architectures is shown in Figure 5 above.\n",
        "\n",
        "1. Cannabidiolate:\n",
        "   - Description: \"Cannabidiolate is a dihydroxybenzoate that is the conjugate base of cannabidiolic acid, obtained by deprotonation of the carboxy group. It derives from an olivetolate. It is a conjugate base of a cannabidiolic acid.\"\n",
        "   - This is a complex molecule with multiple functional groups and substructures mentioned in the description, such as the dihydroxybenzoate, conjugate base, and olivetolate. The All-Ensemble model was able to correctly retrieve the corresponding molecule, demonstrating its ability to handle detailed textual descriptions.\n",
        "\n",
        "2. Inositol:\n",
        "   - Description: \"Myo-inositol is an inositol having myo-configuration. It has a role as a member of compatible osmolytes, a nutrient, an EC 3.1.4.11 (phosphoinositide phospholipase C) inhibitor, a human metabolite, a Daphnia magna metabolite\"\n",
        "   - This example shows the model can handle descriptions that provide various functional and chemical details about the molecule, such as the myo-configuration, its biological roles, and enzyme interactions. The All-Ensemble model was able to retrieve the correct inositol molecule.\n",
        "\n",
        "3. Argyssfrywff:\n",
        "   - Description: \"Ala-Arg-Gly-Tyr-Ser-Ser-Phe-Arg-Tyr-Trp-Phe-Phe is an oligopeptide composed of L-alanine, L-arginine, glycine, L-tyrosine, L-serine, L-serine, L-phenylalanine, L-arginine, L-tyrosine, L-trytophan, L-phenylalanine and L-phenylalanine joined in sequence by peptide linkages.\"\n",
        "   - This example demonstrates the model's ability to handle complex molecule descriptions that list the individual amino acids and their sequence in an oligopeptide. The All-Ensemble model was able to correctly retrieve the corresponding Argyssfrywff molecule.\n",
        "\n",
        "These examples highlight the strengths of the All-Ensemble model in handling a diverse range of molecule descriptions, from detailed functional group information to complex sequences of substructures. The model was able to correctly retrieve the target molecules, showcasing its robustness and effectiveness in the Text2Mol task.\n",
        "\n",
        "The ability to accurately predict these challenging queries, which involve large, intricate molecules with extensive textual descriptions, underscores the power of the ensemble approach and the model's capacity to integrate the complementary strengths of the MLP and GCN architectures."
      ],
      "metadata": {
        "id": "RJna7q_1QDvG"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### **Analysis of queries that are ranked incorrectly by all-ensembles:**"
      ],
      "metadata": {
        "id": "Ut0rKpmxO6_C"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "\n",
        "\n",
        "<img src=\"https://drive.google.com/uc?id=1M6oT5DQ6zILEf8L7xBo8IepMnT3A51D7\" width=500 />\n",
        "<br>\n",
        "<b>\n",
        "Figure 6: Example queries that are ranked incorrectly by All-Ensemble.\n",
        "</b>\n",
        "\n",
        "Figure 6 above provides examples of queries that the All-Ensemble model ranked incorrectly, which is helpful to understand the remaining challenges.\n",
        "\n",
        "1. Fura red:\n",
        "   - Description: \"Fura red is a 1-benzofuran substituted at position 2 by a (5-oxo-2-thioxoimidazolidin-4-ylidene) methyl group, and at C-5 and C-6 by heavily substituted oxygen and nitrogen functionalities\"\n",
        "   - Despite the detailed description, the All-Ensemble model ranked this compound at 8,320, which is quite low. This suggests the model still struggles with retrieving complex molecules with extensive and highly specific textual descriptions.\n",
        "\n",
        "2. Clondronate(2-):\n",
        "   - Description: \"Clondronate(2-) is the dianion resulting from the removal of two protons from clondronic acid. It is a conjugate base of a clodronic acid.\"\n",
        "   - The model ranked this compound at 4,915, which is also quite low. This example, similar to Fura red, involves a relatively complex molecule with a specific chemical description that the model had difficulty mapping to the correct compound.\n",
        "\n",
        "3. Alpha-mycolic acid:\n",
        "   - Description: \"An alpha-mycolic acid is a class of mycolic acids characterized by the presence of two cis cyclopropyl groups in the meromycolic chain. It is an organic molecular entity and a mycolic acid.\"\n",
        "   - In this case, the MLP model ranked the compound at 43, while the GCN model ranked it at 3. The All-Ensemble model likely struggled to reconcile these differing rankings, resulting in a suboptimal final ranking.\n",
        "\n",
        "These examples highlight that while the ensemble approach significantly improves performance, there are still challenging cases where the model fails to retrieve the correct molecule, especially for complex molecules with highly specific textual descriptions.\n",
        "\n",
        "The discrepancy in rankings between the MLP and GCN models for the alpha-mycolic acid example also suggests that there is room for improvement in better integrating the complementary strengths of these architectures, particularly for the most difficult queries.\n",
        "\n",
        "Overall, these examples indicate that while the proposed approach represents a significant advancement in cross-modal molecule retrieval, there are still opportunities to further enhance the model's ability to handle the most complex and nuanced molecule-text associations."
      ],
      "metadata": {
        "id": "vBUebnAonIVO"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Results Discussion for Experiments and Ablation Study"
      ],
      "metadata": {
        "id": "pUUHzDzrxgdz"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Experiments:"
      ],
      "metadata": {
        "id": "RlhBeTSYyVte"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "The results from the experiments clearly demonstrate that the MLP models with fewer epochs, hidden units, and embedding dimensions produce significantly worse performance compared to the models with more robust configurations.\n",
        "\n",
        "1. **Fewer Epochs (1 Epoch):**\n",
        "   - The MLP1 model with 1 epoch of training achieves a mean rank of 582.33, MRR of 0.026, Hits@1 of 0.58%, and Hits@10 of 5.2%.\n",
        "   - The MLP2 and MLP3 models with 1 epoch of training perform even worse, with mean ranks of 650.64 and 780.45, respectively.\n",
        "   - These results indicate that training the MLP models for only 1 epoch is not sufficient, leading to poor retrieval performance across all metrics.\n",
        "\n",
        "2. **Fewer Hidden Units and Embedding Dimensions:**\n",
        "   - Comparing the MLP models with different hidden unit and embedding dimensions, we can see a clear trend of performance degradation as the complexity of the models is reduced.\n",
        "   - The MLP1 model with 600 hidden units and 300 embeddings outperforms the MLP2 model (400 hidden units, 200 embeddings) and the MLP3 model (200 hidden units, 100 embeddings) across all metrics, both at 1 epoch and at 5 epochs of training.\n",
        "   - For example, at 5 epochs, the MLP1 model achieves a mean rank of 121.60, MRR of 0.108, Hits@1 of 4.1%, and Hits@10 of 24.4%, while the MLP3 model with fewer hidden units and embeddings performs significantly worse with a mean rank of 160.53, MRR of 0.023, Hits@1 of 1.8%, and Hits@10 of 12.3%.\n",
        "\n",
        "3. **Improved Performance with More Epochs:**\n",
        "   - As the number of training epochs increases from 1 to 30, the performance of all MLP models improves significantly.\n",
        "   - The MLP1 model with 600 hidden units and 300 embeddings achieves the best performance at 30 epochs, with a mean rank of 55.28, MRR of 0.227, Hits@1 of 20.6%, and Hits@10 of 58.7%.\n",
        "   - However, even at 30 epochs, the models with fewer hidden units and embedding dimensions (MLP2 and MLP3) still lag behind the MLP1 model in terms of all performance metrics.\n"
      ],
      "metadata": {
        "id": "g27ySz35xpNQ"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Ablation Study:"
      ],
      "metadata": {
        "id": "W81c80Iqyd4Y"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "The results in the ablation study demonstrate the importance of incorporating the GCN model in the Text2Mol task. The performance of the MLP models without the GCN is significantly worse compared to the models that utilize the GCN architecture.\n",
        "\n",
        "1. **Mol2Vec without GCN Training Results**:\n",
        "   - The MLP models (MLP1, MLP2, MLP3) trained solely on the Mol2Vec representations, without the GCN, exhibit poor performance across all metrics.\n",
        "   - The mean rank for these models ranges from 585.51 to 590.35, the MRR is between 0.032 and 0.038, and the Hits@1 is below 1%.\n",
        "   - These results indicate that the Mol2Vec representations alone, without the additional structural information captured by the GCN, are not sufficient for effective text-molecule retrieval.\n",
        "\n",
        "2. **GCN Training Results**:\n",
        "   - In contrast, the GCN models (GCN1, GCN2, GCN3) that leverage the molecular graph structure show significantly improved performance.\n",
        "   - The mean rank for the GCN models is remarkably low, ranging from 12.74 to 12.99, the MRR is between 0.365 and 0.388, and the Hits@1 is around 23-24%.\n",
        "   - These results demonstrate the importance of explicitly incorporating the molecular graph information through the GCN architecture, as it allows the model to capture the structural characteristics of the molecules more effectively.\n",
        "\n",
        "3. **Mol2Vec without GCN Test Results**:\n",
        "   - The poor performance of the MLP models without the GCN is further corroborated by the test set results.\n",
        "   - The mean rank for the MLP models in the test set is between 681.47 and 690.56, the MRR is around 0.030, and the Hits@1 is less than 1%.\n",
        "   - These results confirm that the Mol2Vec representations alone are not sufficient for generalizing to the test set, highlighting the need for the GCN component to achieve high-performing text-molecule retrieval.\n",
        "\n",
        "4. **GCN Test Results**:\n",
        "   - The GCN models, on the other hand, maintain their strong performance on the test set, with mean ranks ranging from 55.47 to 58.95, MRR between 0.256 and 0.294, and Hits@1 around 20-21%.\n",
        "   - These results demonstrate the GCN's ability to effectively capture the structural properties of the molecules, which enables the model to generalize well to the test set."
      ],
      "metadata": {
        "id": "xu6GIVJIyhRr"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Results Discussion with Respect to the Hypothesis and Results from the original paper"
      ],
      "metadata": {
        "id": "-78x9d1jqBks"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "The cross-modal embedding approach is effective for aligning text and molecules, and ensembling different architectures boosts performance by combining their strengths, strongly supporting hypotheses 1 and 2. The attention-based associations provide some interpretability but their quality needs further analysis (hypothesis 3). There are minor differences in MLP/GCN ranking capabilities supporting hypothesis 4, though more evidence is needed. Finally, the reranking methods do not improve over the base models, contradicting hypothesis 5. Overall, the embedding ensembles emerge as the most promising approach based on these results.\n",
        "\n",
        "1. **Hypothesis 1**: The results strongly support this hypothesis. Both the MLP and GCN encoders are able to effectively align the text and molecule spaces, achieving high MRR values on the training set (around 0.60-0.61). This shows the cross-modal embedding approach is successful in mapping the two modalities into a common space for retrieval. While performance drops on the test set, the MRR values around 0.48-0.51 still indicate reasonable retrieval capability.\n",
        "\n",
        "2. **Hypothesis 2**: This hypothesis is also supported by the results. The ensemble models combining multiple MLP or GCN models outperform the individual constituent models, both for the same architecture type and especially when ensembling across MLP and GCN. The All-Ensemble achieves the highest MRR of 0.509 on the test set compared to 0.499-0.517 for the individual architecture ensembles. Ensembling different architectures allows the model to benefit from their complementary strengths.\n",
        "\n",
        "3. **Hypothesis 3**: The attention-based models provide some insights into text-molecule associations through the extracted rules. However, the coherence and usefulness of these rules is not directly evaluated in the results. More qualitative analysis would be needed to assess the quality of the mined associations and their utility for interpretability.\n",
        "\n",
        "4. **Hypothesis 4**: The results lend some support to this hypothesis. On the test set, the MLP-Ensemble achieves a slightly higher MRR of 0.517 compared to 0.499 for the GCN-Ensemble. This suggests the MLP may be better at ranking easier examples. However, the differences are relatively small. More detailed analysis of the rankings and examples where MLP/GCN perform better would help confirm this hypothesis.\n",
        "\n",
        "5. **Hypothesis 5**: The results do not support this hypothesis. The attention model with reranking using extracted rules underperforms the base MLP and ensemble models on the test set MRR (around 0.50-0.51). The FPGrowth association rule reranking also does not improve over the base models. More work may be needed to effectively leverage the attention signals for reranking."
      ],
      "metadata": {
        "id": "C6L_pZZJJm7Q"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Discussion"
      ],
      "metadata": {
        "id": "tjprWdtCXeBQ"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "The reproduction results support that the core components of the original paper were reproducible, but the attention-based portion of the model encountered issues that prevented the full reproducibility of the work intially. However, we've managed to crack the code conundrum and reproduce all the results from the paper successfully.\n",
        "\n",
        "Interestingly, our results turned out to be slightly better than those reported in the paper. For example, MLP Ensemble in the paper is 0.520 for MRR and 35.1%/86.4% for Hits@1/10. Our results are 0.606 for MRR and 43.4%/93.8% for Hits@1/10. This has sparked our curiosity, and we are in discussion with the primary author of the paper to see if he has any insights into what might have caused this difference. Our initial assessment leans towards potential variations in hyper-parameters or differences in the transformer pre-trained model used.\n",
        "\n",
        "1. Reproducibility of the Base Models:\n",
        "   - The reproduction results for the MLP and GCN models show that we are able to successfully replicate the performance of the baseline architectures reported in the original paper.\n",
        "   - The validation and test set results for the MLP and GCN models, including the mean rank, MRR, Hits@1, and Hits@10 metrics, are consistent with the original findings.\n",
        "   - This suggests that the core model components, such as the text encoder, molecule encoder, and the contrastive loss function, were implemented correctly and could be reproduced.\n",
        "\n",
        "2. Issues with the Attention-based Model weight extraction:\n",
        "   - In the reproduction we were able to train the cross-modal attention model and use the attention weights.\n",
        "   - However, we were unable to reproduce the results for the analysis of the attention weights and extracted rules due to problems with the code. Following extensive time and effort, we've managed to rectify the code, enabling us to effectively extract the weights of the Multi-Headed Attention (MHA). This breakthrough allows us to thoroughly evaluate the attention results, marking a significant achievement in our endeavors.\n",
        "\n",
        "3. Factors Contributing to the Irreproducibility:\n",
        "   - The lack of clarity in part of the codebase and implementation details are the primary factor that prevented the full reproducibility of the attention-based portion of the work initially. After investing significant time and effort, we successfully resolved the issue within the code.\n",
        "   - This highlights the importance of providing comprehensive documentation, code, and implementation details when publishing research, as it directly impacts the ability of others to reproduce the work."
      ],
      "metadata": {
        "id": "37jl75JsOjm1"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## “What was easy”"
      ],
      "metadata": {
        "id": "OWFLbk_30ckm"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "- Model Architectures: The paper provides clear descriptions and diagrams of the model architectures like the text encoder, MLP/GCN molecule encoders, and cross-modal attention model. Running these models is relatively straightforward.\n",
        "\n",
        "- Training Procedures: The authors specify the optimization algorithm (Adam), learning rates, batch sizes, and number of training epochs used, facilitating reproducibility of the training process.\n",
        "\n",
        "- Loss Functions: The loss functions, including the symmetric contrastive loss and its modification with negative sampling, are well-documented and is easy to run based on the details provided.\n",
        "\n",
        "- Evaluation Metrics: The metrics used (MRR, Hits@K, Mean Rank) are standard and easy to calculate given the model outputs and ground truth data.\n"
      ],
      "metadata": {
        "id": "9M89lMZsOwek"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## What was difficult"
      ],
      "metadata": {
        "id": "solYoCnVTmpl"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "- The most difficult part of the project has proven most arduous when grappling with the reproduction code. Unfortunately, the code fails to work as expected. Substantial time and dedication have been invested in fixing and refining it to align with the paper's intended outcomes. Countless fixes and updates have been made to ensure it works, in order for us to successfully replicate the results outlined in the paper.\n",
        "\n",
        "- Computational Resources: The authors mention using NVIDIA V100 GPUs, but do not provide details on the specific hardware configurations (number of GPUs, memory, etc.) which can influence training times and results, especially for large models like SciBERT. As an example, in order to execute the reproduction, the system necessitates a minimum of 60GB of memory. Our initial setups failed due to the lack of information on specific computational requirements.\n",
        "\n",
        "- Dataset Construction: While the authors mention using ChEBI and PubChem data, the exact filtering criteria and preprocessing steps to construct the ChEBI-20 dataset are not clearly specified, making it difficult to reproduce the dataset identically.\n",
        "\n",
        "- Mol2vec Embeddings: The process of generating Mol2vec embeddings from SMILES strings is not described in detail. Different implementations or parameter choices could lead to variations in the molecule representations.\n",
        "\n",
        "- Random Seeds: The random seeds used for weight initialization and data shuffling are not reported, which can cause differences in the final model outputs and rankings.\n",
        "\n",
        "- Ensemble Details: The specifics of how the ensemble model combines rankings from different base models (e.g., the weighting scheme) are not clearly described, making it difficult to reproduce the ensemble results exactly.\n",
        "\n",
        "- Hyperparameter Tuning: The paper does not provide details on hyperparameter search ranges or the criteria used for selecting the final hyperparameter values, which could affect reproducibility."
      ],
      "metadata": {
        "id": "osMFwkzqPHvG"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Recommendations to the original authors or others who work in this area for improving reproducibility"
      ],
      "metadata": {
        "id": "FEBSQKi9SXir"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "**For the Authors:**\n",
        "\n",
        "- **Mol2vec Implementation Details:** Provide a detailed description of the Mol2vec embedding generation process, including the specific parameter settings, radius values, handling of rare/unknown substructures, and any other implementation choices made.\n",
        "\n",
        "- **Random Seeds:** Report the random seeds used for weight initialization, data shuffling, and any other non-deterministic components. This would enable others to replicate the exact same runs.\n",
        "\n",
        "- **Hyperparameter Tuning:** Document the hyperparameter search spaces, the tuning approach (manual, random search, etc.), and the criteria used for selecting the final hyperparameter values.\n",
        "\n",
        "- **Attention-based Component:** To improve the reproducibility of the attention-based component, the authors of the original paper could consider releasing the following:\n",
        "   - Providing detailed step-by-step instructions, along with the necessary data and pre-trained models, would further aid in the reproducibility of the attention-based aspect of the work.\n",
        "   - Additionally, the authors could consider including the attention-based analysis and insights in the main body of the paper, rather than treating it as a separate hypothesis, to ensure that the core contributions are clearly documented and accessible.\n",
        "\n",
        "**For Others Working in This Area:**\n",
        "\n",
        "- **Follow Best Practices:** Adopt best practices for reproducible research, such as providing clear documentation, releasing code and data, and reporting all relevant implementation details.\n",
        "\n",
        "- **Use Containerization:** Leverage containerization tools like Docker to package the entire software environment, making it easier to share and replicate experiments across different systems.\n",
        "\n",
        "- **Automate Workflows:** Develop automated workflows or scripts that handle data preprocessing, model training, evaluation, and result reporting. This reduces manual effort and minimizes the risk of human errors affecting reproducibility.\n",
        "\n",
        "- **Benchmark Datasets:** Collaborate on creating and maintaining benchmark datasets for cross-modal molecule retrieval, with clear data collection, curation, and validation processes.\n",
        "\n",
        "- **Reproducibility Checklists:** Adopt reproducibility checklists or guidelines specific to the domain, ensuring that all relevant aspects are documented and shared with the research community.\n",
        "\n",
        "- **Reproducibility Challenges:** Participate in reproducibility challenges or code evaluation efforts, which can help identify potential issues and improve the overall reproducibility of research in this area.\n"
      ],
      "metadata": {
        "id": "qH75TNU71eRH"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Public GitHub Repo"
      ],
      "metadata": {
        "id": "XrDtWDlMwcoQ"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "\n",
        "The reproduce code is published in the following GitHub [DLH Text2Mol](https://github.com/darinz/DLH-Text2Mol) Repository.\n",
        "\n",
        "You can download this notebook (DLH_Team_10.ipynb) from the [DLH Text2Mol](https://github.com/darinz/DLH-Text2Mol) Repository."
      ],
      "metadata": {
        "id": "TNfEP0D0WmOd"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Video Presentation"
      ],
      "metadata": {
        "id": "xmVuzQ724HbO"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "The video presentation is available on YouTube using the following link: https://youtu.be/f25X_zxVEwQ"
      ],
      "metadata": {
        "id": "gis8ahfK-Wbj"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# References"
      ],
      "metadata": {
        "id": "sQAli7sewoW1"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "\n",
        "\n",
        "```bibtex\n",
        "@inproceedings{edwards2021text2mol,\n",
        "  title={Text2Mol: Cross-Modal Molecule Retrieval with Natural Language Queries},\n",
        "  author={Edwards, Carl and Zhai, ChengXiang and Ji, Heng},\n",
        "  booktitle={Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing},\n",
        "  pages={595--607},\n",
        "  year={2021},\n",
        "  url = {https://aclanthology.org/2021.emnlp-main.47/}\n",
        "}\n",
        "```\n",
        "\n"
      ],
      "metadata": {
        "id": "SHMI2chl9omn"
      }
    }
  ]
}