DataPrepocessing.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"DataPrepocessing","provenance":[],"collapsed_sections":[],"machine_shape":"hm"},"kernelspec":{"display_name":"Python 3","name":"python3"},"language_info":{"name":"python"},"widgets":{"application/vnd.jupyter.widget-state+json":{"17972f00f5174456a6826d5160ceeeef":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_edd44ae3c3f745d7a40f24d89994c8d2","IPY_MODEL_2a85af42b57e4f83b9f5ab529e5256bd"],"layout":"IPY_MODEL_da4b2b99391942d2a17c3012274f1994"}},"edd44ae3c3f745d7a40f24d89994c8d2":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"Downloading: 100%","description_tooltip":null,"layout":"IPY_MODEL_037826093c754649a921b8642f03753d","max":689,"min":0,"orientation":"horizontal","style":"IPY_MODEL_38eb97458e9b42fba80711989ccf9df8","value":689}},"2a85af42b57e4f83b9f5ab529e5256bd":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_24caa273d3e0451aadd9bb7f2fbe5e26","placeholder":"​","style":"IPY_MODEL_81f8960d6b1b4916b79ce14512b500e2","value":" 689/689 [00:02&lt;00:00, 264B/s]"}},"da4b2b99391942d2a17c3012274f1994":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"037826093c754649a921b8642f03753d":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"38eb97458e9b42fba80711989ccf9df8":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":"initial"}},"24caa273d3e0451aadd9bb7f2fbe5e26":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"81f8960d6b1b4916b79ce14512b500e2":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"0bebbb11af324bd8b47f1ebd86cee164":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_ab5cd1791416490fa7d4ac3e8e9005fe","IPY_MODEL_0065185116fb47d4ba26bb82d004faa5"],"layout":"IPY_MODEL_79cf476811f74502b85964a43cbdd97f"}},"ab5cd1791416490fa7d4ac3e8e9005fe":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"Downloading: 100%","description_tooltip":null,"layout":"IPY_MODEL_9084ba816b564fd29011b68bbb6806e1","max":231508,"min":0,"orientation":"horizontal","style":"IPY_MODEL_51274cd0c5c2457fbc4c6e174f086aa5","value":231508}},"0065185116fb47d4ba26bb82d004faa5":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_407fd88ec7e24230903ccd391d04e72c","placeholder":"​","style":"IPY_MODEL_c59c6ac231a6441492758af81c6ab59e","value":" 232k/232k [00:01&lt;00:00, 169kB/s]"}},"79cf476811f74502b85964a43cbdd97f":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"9084ba816b564fd29011b68bbb6806e1":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"51274cd0c5c2457fbc4c6e174f086aa5":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":"initial"}},"407fd88ec7e24230903ccd391d04e72c":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"c59c6ac231a6441492758af81c6ab59e":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"beaf09c2052948d3b018c7e3bad9b772":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_41d1d7ab35fe495ea1dd846d50981008","IPY_MODEL_49cd106830de47b684bfb743ea818804"],"layout":"IPY_MODEL_7f31f06ac87d4ce8af878257d090075b"}},"41d1d7ab35fe495ea1dd846d50981008":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"Downloading: 100%","description_tooltip":null,"layout":"IPY_MODEL_aa743b4f2a534ff781d27562c5e178d5","max":112,"min":0,"orientation":"horizontal","style":"IPY_MODEL_7a9cdb191ae44e02941e6b0d3923f281","value":112}},"49cd106830de47b684bfb743ea818804":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_fcd717a1f9804dffb585b36302a73f2c","placeholder":"​","style":"IPY_MODEL_1191bbb189094e47a8ab0c0e5b18ba92","value":" 112/112 [00:00&lt;00:00, 171B/s]"}},"7f31f06ac87d4ce8af878257d090075b":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"aa743b4f2a534ff781d27562c5e178d5":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"7a9cdb191ae44e02941e6b0d3923f281":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":"initial"}},"fcd717a1f9804dffb585b36302a73f2c":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"1191bbb189094e47a8ab0c0e5b18ba92":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"afdd685619954083bd74e0fb023697b6":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_2b35f47086bc4c1b98cee9e531813224","IPY_MODEL_1e81341e012640a78b1f9fbfe8b23777"],"layout":"IPY_MODEL_f059f234ba7244129e592509587d9615"}},"2b35f47086bc4c1b98cee9e531813224":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"Downloading: 100%","description_tooltip":null,"layout":"IPY_MODEL_7414d6b34bb04996bd20cdda764eef45","max":252,"min":0,"orientation":"horizontal","style":"IPY_MODEL_7ca1d4599d0c4b409e625b70fd297486","value":252}},"1e81341e012640a78b1f9fbfe8b23777":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_11842ad53bd1461db4844ddfb78b24ab","placeholder":"​","style":"IPY_MODEL_beaad27eb3ba45ed9f13cec7c131b40e","value":" 252/252 [00:00&lt;00:00, 3.91kB/s]"}},"f059f234ba7244129e592509587d9615":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"7414d6b34bb04996bd20cdda764eef45":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"7ca1d4599d0c4b409e625b70fd297486":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":"initial"}},"11842ad53bd1461db4844ddfb78b24ab":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"beaad27eb3ba45ed9f13cec7c131b40e":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"412e823e9ebf4e2585d8a0cdb35baad5":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_9f8ebaad5e65459d83b733192ededa8e","IPY_MODEL_a3b875f8384b45de80572cc4a5333130"],"layout":"IPY_MODEL_05d6a9f3664643bbaf66787364836d33"}},"9f8ebaad5e65459d83b733192ededa8e":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"Downloading: 100%","description_tooltip":null,"layout":"IPY_MODEL_ae3976a71b4b4a35ba1953bde354bd63","max":437998343,"min":0,"orientation":"horizontal","style":"IPY_MODEL_23e3a78dacca4415ac0cfc8096fe0337","value":437998343}},"a3b875f8384b45de80572cc4a5333130":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_e1004bd075df472688b69337e04af14a","placeholder":"​","style":"IPY_MODEL_5b82dc843c5a4222a2a04e6d647a8936","value":" 438M/438M [00:12&lt;00:00, 36.3MB/s]"}},"05d6a9f3664643bbaf66787364836d33":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"ae3976a71b4b4a35ba1953bde354bd63":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"23e3a78dacca4415ac0cfc8096fe0337":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":"initial"}},"e1004bd075df472688b69337e04af14a":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"5b82dc843c5a4222a2a04e6d647a8936":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}}}}},"cells":[{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"3kI3esy4CGcc","executionInfo":{"status":"ok","timestamp":1638308555096,"user_tz":480,"elapsed":5944,"user":{"displayName":"Shilpa Nair","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"14689458463016539498"}},"outputId":"825ca562-6c66-477f-f741-72ba6fc454d2"},"source":["!pip install emoji\n","!pip install contractions\n","!pip install spacy_langdetect\n","!pip install pycld2\n","!pip install simcse transformers==4.2.1 scipy==1.5.4 datasets==1.2.1 pandas==1.1.5 scikit-learn==0.24.0 prettytable==2.1.0 gradio torch setuptools==49.3.0"],"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Collecting emoji\n","  Downloading emoji-1.6.1.tar.gz (170 kB)\n","\u001b[?25l\r\u001b[K     |██                              | 10 kB 23.0 MB/s eta 0:00:01\r\u001b[K     |███▉                            | 20 kB 27.6 MB/s eta 0:00:01\r\u001b[K     |█████▉                          | 30 kB 12.5 MB/s eta 0:00:01\r\u001b[K     |███████▊                        | 40 kB 9.6 MB/s eta 0:00:01\r\u001b[K     |█████████▋                      | 51 kB 5.1 MB/s eta 0:00:01\r\u001b[K     |███████████▋                    | 61 kB 5.4 MB/s eta 0:00:01\r\u001b[K     |█████████████▌                  | 71 kB 5.8 MB/s eta 0:00:01\r\u001b[K     |███████████████▍                | 81 kB 6.5 MB/s eta 0:00:01\r\u001b[K     |█████████████████▍              | 92 kB 6.7 MB/s eta 0:00:01\r\u001b[K     |███████████████████▎            | 102 kB 5.2 MB/s eta 0:00:01\r\u001b[K     |█████████████████████▏          | 112 kB 5.2 MB/s eta 0:00:01\r\u001b[K     |███████████████████████▏        | 122 kB 5.2 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████       | 133 kB 5.2 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████     | 143 kB 5.2 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████   | 153 kB 5.2 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▉ | 163 kB 5.2 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 170 kB 5.2 MB/s \n","\u001b[?25hBuilding wheels for collected packages: emoji\n","  Building wheel for emoji (setup.py) ... \u001b[?25l\u001b[?25hdone\n","  Created wheel for emoji: filename=emoji-1.6.1-py3-none-any.whl size=169314 sha256=d62aaed24ab57a663c0f8030fd2cca9c99e63f127dca8c13b80beed9249379f2\n","  Stored in directory: /root/.cache/pip/wheels/ea/5f/d3/03d313ddb3c2a1a427bb4690f1621eea60fe6f2a30cc95940f\n","Successfully built emoji\n","Installing collected packages: emoji\n","Successfully installed emoji-1.6.1\n"]}]},{"cell_type":"code","metadata":{"id":"kbp6QS4kCPyZ"},"source":["import json\n","import pandas as pd\n","from nltk import word_tokenize\n","import nltk\n","nltk.download('punkt')\n","from nltk.tokenize import MWETokenizer\n","import zipfile\n","import re\n","from emoji import UNICODE_EMOJI\n","import os\n","import contractions\n","from time import time\n","from datetime import datetime\n","import csv\n","import torch\n","from simcse import SimCSE\n","import pandas as pd\n","import heapq\n","import numpy as np\n"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"2cBi1oi9MtDF","executionInfo":{"status":"ok","timestamp":1638243102425,"user_tz":480,"elapsed":27956,"user":{"displayName":"Shilpa Nair","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"15472047455778914696"}},"outputId":"943710ce-0866-4c8e-bef5-588839ad1647"},"source":["from google.colab import drive\n","drive.mount('/content/drive')"],"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Mounted at /content/drive\n"]}]},{"cell_type":"code","metadata":{"id":"ZnBrZDsiNLvD"},"source":["folder_path = 'drive/MyDrive/CSCI_544_NLP_Project/Twitter_Bot_Detection/Data/'"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"6059e6c7"},"source":["def preprocessing(main_zip_file, file, include_meta=True):\n","    data = None\n","    with zipfile.ZipFile(main_zip_file) as z:\n","        file = [x for x in z.namelist() if file in x]\n","        assert len(file)==1\n","        file = file[0]\n","        with z.open(file, 'r') as f:\n","            data = f.read()\n","    data = json.loads(data.decode('utf-8'))\n","    \n","    # rearrange - data into tweets\n","    data, user_metadata = restructure(data)\n","    data['orig_tweet'] = data['tweet']\n","\n","    #ROUND 2: remove RT\n","    data = remove_rt(data)\n","    \n","    # replace urls to <url> - retain the info in another way?\n","    url_re = 'https?:\\/\\/(www\\.)?[-a-zA-Z0-9@:%.\\+~#=]{1,256}\\.[a-zA-Z0-9()]{1,6}([-a-zA-Z0-9()@:%\\+.~#?&//=]*)'\n","    replace_in_tweet(data, url_re, ' <url> ')\n","    \n","    # replace @userhandle with <user>\n","    # replace_in_tweet(data, '@\\w+', ' <user> ')\n","    \n","    # replace #hashtag with <hashtag>\n","    # replace_in_tweet(data, '#\\w+', ' <hashtag> ')\n","    \n","    # replace numbers with <number>\n","    # replace_in_tweet(data, '\\d+', ' <number> ')\n","    \n","    # replace emojis with tags\n","    data['tweet'] = data['tweet'].apply(space_out_emojis)\n","    \n","    #removing extra spaces\n","    data['tweet'] = data['tweet'].str.replace('\\s+', ' ')\n","    data['tweet'] = data['tweet'].apply(lambda x: x.strip())\n","\n","    #handle contractions\n","    # print('Starting contractions')\n","    def fn(x):\n","        res = x\n","        try:\n","            res = contractions.fix(x)\n","        except Exception as e:\n","            pass #seems to be from other languages - handle if not filtering?\n","        return res\n","    data['tweet'] = data['tweet'].apply(lambda x: fn(x))\n","    # print('Handling contractions')\n","    \n","    # tokenize words\n","    # mwe_tokens = MWETokenizer([('<','user','>'),('<','hashtag','>'), ('<','url','>'), ('<','number','>')])\n","    # data['tweet'] = data['tweet'].apply(lambda x: mwe_tokens.tokenize(word_tokenize(x)))\n","    \n","    # replace all uppercase - token+<allcaps> - another way to handle case?\n","    # data['tweet'] = data['tweet'].apply(handle_allcaps)\n","    \n","    # replace lowercase\n","    # data['tweet'] = data['tweet'].apply(lambda x: x.lower())\n","\n","    data['tweet'] = data['tweet'].apply(handle_extended)\n","\n","    #ROUND 3: metadata\n","    if include_meta:\n","        add_metadata(data, user_metadata)\n","    \n","    # replace same letter repeated 3+ times - needed?\n","    # add info to word vectors - variable - future work\n","    return data, user_metadata"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"1MvDUHg2-sG4"},"source":["# def handle_allcaps(tweet):\n","#     tokens = word_tokenize(tweet)\n","#     for token in tokens:\n","#         if token.isupper():\n","#             tweet = tweet.replace(token, token.lower()+'_allcaps_')\n","#     return tweet"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"aWMG9Do_rd8D","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1638052602865,"user_tz":480,"elapsed":238,"user":{"displayName":"Shilpa Nair","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"14689458463016539498"}},"outputId":"09fc1d14-1e65-4d1f-e00d-b87e10af9a62"},"source":["def handle_extended(tweet):\n","    tokens = word_tokenize(tweet)\n","    for token in tokens:\n","        res=re.sub(r'([a-zA-Z])\\1{2,}', r'\\1', token)\n","        if res!=token:\n","            tweet = tweet.replace(token, token+\"<ext>\")\n","    return tweet\n","    # re.sub(r'([a-zA-Z])\\1{2,}', r'\\1', 'whaaaaaaaat')\n","    #extended tokens\n","\n","print(handle_extended(\"whaaaaaattttt are you saying. are you crazzzzyyyyyyyyy!!! Loooooookkkk!!!!\"))"],"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["whaaaaaattttt<ext> are you saying. are you crazzzzyyyyyyyyy<ext>!!! Loooooookkkk<ext>!!!!\n"]}]},{"cell_type":"code","metadata":{"id":"69b97702"},"source":["def space_out_emojis(tweet):\n","    all_emojis = UNICODE_EMOJI['en'].keys()\n","    tweet_emojis = {x for x in all_emojis if x in tweet}\n","    for emoji in tweet_emojis:\n","        tweet = re.sub(re.escape(emoji), ' '+emoji+' ', tweet)\n","    return tweet"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"7bba1535"},"source":["def restructure(data):\n","    tweet_data = []\n","    user_data = []\n","    for instance in data:\n","        if type(instance['tweet']) != type(None):\n","            for tweet in instance['tweet']:\n","                tweet_data.append( [instance['ID'], tweet, instance['label']] )\n","        user_meta = eval(str(instance))\n","        del user_meta['tweet']\n","        user_data.append(user_meta)\n","    return pd.DataFrame(tweet_data, columns=['id', 'tweet', 'label']), user_data"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"0721365d"},"source":["def replace_in_tweet(data, re_to_replace, replace_with):\n","    data['tweet'] = data['tweet'].apply(lambda x: re.sub(re_to_replace, replace_with, x))"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"5ef9ce17"},"source":["def write_triplets(data, filename):\n","    data['tweet'] = data['tweet'].apply(lambda x: ' '.join(x))\n","    bots = data[data['label']=='1'].sample(30000)['tweet']\n","    humans = data[data['label']=='0'].sample(30000)['tweet']\n","    out1 = pd.DataFrame(zip(bots[:10000], bots[10000:20000], humans[20000:]), columns=['sent0', 'sent1', 'hard_neg'])\n","    out2 = pd.DataFrame(zip(humans[:10000], humans[10000:20000], bots[20000:]), columns=['sent0', 'sent1', 'hard_neg'])\n","    out = pd.concat([out1, out2])\n","    out.to_csv(filename, index=False)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"nnewk-RJcunI"},"source":["# def write_triplets_with_similarity(data, filename):\n","#     data['tweet'] = data['tweet'].apply(lambda x: ' '.join(x))\n","#     bots_tweets = data[data['label']=='1']['tweet']\n","#     humans_tweets = data[data['label']=='0']['tweet']\n","\n","#     #For each tweet, find top 2 similar positive and top 2 similar negative\n","#     out1 = pd.DataFrame(zip(bots[:10000], bots[10000:20000], humans[20000:]), columns=['sent0', 'sent1', 'hard_neg'])\n","#     out2 = pd.DataFrame(zip(humans[:10000], humans[10000:20000], bots[20000:]), columns=['sent0', 'sent1', 'hard_neg'])\n","#     out = pd.concat([out1, out2])\n","#     out.to_csv(filename, index=False)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"4cCC9qxYb3Sn"},"source":["def remove_rt(data):\n","    columns=['id', 'tweet', 'label', 'orig_tweet']\n","    data['is_retweet'] = data['orig_tweet'].apply(lambda x: x[:3] == 'RT ')\n","    bots = data[data['label']=='1']\n","    humans = data[data['label']=='0']\n","    print('Ratio of RT tweets by bots:', len(bots[bots['is_retweet']]), '/', len(bots), '(',len(bots[bots['is_retweet']])*100/len(bots), '%)' )\n","    print('Ratio of RT tweets by humans:', len(humans[humans['is_retweet']]), '/', len(humans), '(',len(humans[humans['is_retweet']])*100/len(humans), '%)'  )\n","    data = data[~(data['is_retweet'])].copy()\n","    data.drop(columns=['is_retweet'], inplace=True)\n","    return data"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"w7po5bskmdLB"},"source":["def add_metadata(data, user_metadata):\n","\n","    metadata = {x['ID']:x['profile'] for x in user_metadata}\n","    #variable length of user_name and text desc, profile image url\n","    def concat_meta(prof):\n","        params= [\"screen_name\", \"protected\", \"verified\", \"default_profile\", \"default_profile_image\",\n","                  \"followers_count\", \"friends_count\", \"listed_count\", \"favourites_count\", \"statuses_count\"]\n","        meta_text = [ prof[x].strip() for x in params ]\n","        meta_text[1] = 'protected' if meta_text[1]=='True' else 'unprotected'\n","        meta_text[2] = 'verified' if meta_text[2]=='True' else 'unverified'\n","        meta_text[3] = 'defprof' if meta_text[3]=='True' else 'nondefprof'\n","        meta_text[4] = 'defimg' if meta_text[4]=='True' else 'nondefimg'\n","        return ' '.join( meta_text )\n","\n","    data['tweet'] = data.apply(lambda x: concat_meta(metadata[x['id']]) + \" \" + x['tweet'], axis=1)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"51ffb254"},"source":["# for dataset in ('train', 'dev', 'test'):\n","#     data, user_metadata = preprocessing(folder_path+'Twibot-20.zip', dataset+'.json')\n","#     write_triplets(data, folder_path+dataset+'_triplets.csv')"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"UPVSvarX5WqE"},"source":["def generate_preprocessed_data(dataset):\n","    start = time()\n","    data, user_metadata = preprocessing(folder_path+'Twibot-20.zip', dataset+'.json')\n","    # data['tweet'] = data['tweet'].apply(lambda x: ' '.join(x))\n","    print('Remaining bot tweets:', len(data[data['label']=='1']))\n","    print('Remaining human tweets:', len(data[data['label']=='0']))\n","    print('Time taken:', (time() - start))\n","    return data"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"fc2c65e2"},"source":["# dataset = 'test'\n","# data, user_metadata = preprocessing(folder_path+'Twibot-20.zip', dataset+'.json')\n","# # data['tweet'] = data['tweet'].apply(lambda x: ' '.join(x))\n","# print('Remaining bot tweets:', len(data[data['label']=='1']))\n","# print('Remaining human tweets:', len(data[data['label']=='0']))\n","# data"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"9878505c","executionInfo":{"elapsed":122844,"status":"ok","timestamp":1637584365638,"user":{"displayName":"zeeshan ahmad","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"15348670853435150367"},"user_tz":480},"outputId":"728d1b9b-42c7-4ad6-a8a1-b45466f3a2cf"},"source":["# dataset = 'dev'\n","# data, user_metadata = preprocessing(folder_path+'Twibot-20.zip', dataset+'.json')\n","# # data['tweet'] = data['tweet'].apply(lambda x: ' '.join(x))\n","# print('Remaining bot tweets:', len(data[data['label']=='1']))\n","# print('Remaining human tweets:', len(data[data['label']=='0']))"],"execution_count":null,"outputs":[{"name":"stdout","output_type":"stream","text":["Ratio of RT tweets by bots: 78591 / 212950 ( 36.905846442826956 %)\n","Ratio of RT tweets by humans: 48586 / 188575 ( 25.764815060320828 %)\n","Remaining bot tweets: 134359\n","Remaining human tweets: 139989\n"]}]},{"cell_type":"code","metadata":{"id":"vfWSRPjFcNtZ"},"source":["# joined_data = pd.concat([spacy_ref, data], axis=1)\n","def join_spacy_data(data, dataset):\n","    spacy_ref = pd.read_csv(os.path.join(folder_path, 'english_labelled_ref_data', dataset+'_tweets_spacy_labelled.csv'))\n","    assert len(data)==len(spacy_ref), \"Row mismatched with ref data\"\n","    # assert len\n","    cols = spacy_ref.columns\n","    spacy_ref.rename(columns={x:x+'_spc' for x in cols}, inplace=True)\n","    print(len(spacy_ref))\n","    joined = pd.merge(spacy_ref, data, left_index=True, right_index=True)\n","    print(len(joined))\n","    joined = joined[['id', 'tweet', 'label', 'orig_tweet', 'lang_spc', 'lang_code_spc']]\n","    return joined\n","# joined[joined['orig_tweet_x']==joined['orig_tweet_y']].count()\n","# data.reset_index(inplace=True, drop=True)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"hPk8kmpuEnFK","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1638053457223,"user_tz":480,"elapsed":818133,"user":{"displayName":"Shilpa Nair","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"14689458463016539498"}},"outputId":"b1656b88-4e6f-4292-aebe-cd612caa787f"},"source":["for dataset in ['train', 'test', 'dev']:\n","    data = generate_preprocessed_data(dataset)\n","    data.reset_index(inplace=True)\n","    joined_data = join_spacy_data(data, dataset)\n","    joined_data.to_csv(os.path.join(folder_path, 'data_round_3_corrected', dataset+'_tweets_spacy_round3.csv'), index=False)"],"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Ratio of RT tweets by bots: 289171 / 756619 ( 38.218839336574945 %)\n","Ratio of RT tweets by humans: 163623 / 641791 ( 25.494748290331277 %)\n","Remaining bot tweets: 467448\n","Remaining human tweets: 478168\n","Time taken: 548.1311504840851\n","945616\n","945616\n","Ratio of RT tweets by bots: 39970 / 102927 ( 38.83334790676888 %)\n","Ratio of RT tweets by humans: 25391 / 96926 ( 26.196273445721477 %)\n","Remaining bot tweets: 62957\n","Remaining human tweets: 71535\n","Time taken: 79.39990496635437\n","134492\n","134492\n","Ratio of RT tweets by bots: 78591 / 212950 ( 36.905846442826956 %)\n","Ratio of RT tweets by humans: 48586 / 188575 ( 25.764815060320828 %)\n","Remaining bot tweets: 134359\n","Remaining human tweets: 139989\n","Time taken: 159.53771948814392\n","274348\n","274348\n"]}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"s0RJVNDXBZGY","executionInfo":{"status":"ok","timestamp":1638053635153,"user_tz":480,"elapsed":4871,"user":{"displayName":"Shilpa Nair","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"14689458463016539498"}},"outputId":"284ac83f-48df-4881-99e9-d1fdac9c9cd8"},"source":["spacy_ref = pd.read_csv(os.path.join(folder_path, 'english_labelled_ref_data', 'train_tweets_spacy_labelled.csv'))\n","len(spacy_ref[spacy_ref['lang_code']=='en']), len(spacy_ref)"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["(623464, 945616)"]},"metadata":{},"execution_count":34}]},{"cell_type":"code","metadata":{"id":"7GL5lFhf8pmD"},"source":["# dataset = 'test'\n","# spacy_data = pd.read_csv(os.path.join(folder_path, 'english_labelled_ref_data', dataset+'_tweets_spacy_labelled.csv'))\n","# spacy_data['lang'] = spacy_data['lang'].apply(lambda x: eval(x))\n","# spacy_data['lang_code'] = spacy_data['lang'].apply(lambda x: x['language'] if x['score']>=0.99 else 'unsure') # run for dev, test"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"7PLcCCLCFOU4"},"source":["# spacy_data[spacy_data['lang_code']=='so'] #can cross check with pycld2 to classify better\n","# spacy_data['lang_code'].unique()\n","spacy_data.to_csv(os.path.join(folder_path, 'english_labelled_ref_data', dataset+'_tweets_spacy_labelled_new.csv'), index=False)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"AGsAvsWv8prf"},"source":["from spacy_langdetect import LanguageDetector\n","import spacy\n","def find_spacy_labels(data):\n","    lang_detect = spacy.load('en') #defaults to english\n","    lang_detect.add_pipe(LanguageDetector(), name='language_detector', last=True)\n","\n","    def detect_language(text, lang_detect):\n","        doc = lang_detect(text)\n","        return doc._.language\n","    data['lang'] = data['tweet'].apply(lambda x: detect_language(x, lang_detect))\n","    data['lang_code'] = data['lang'].apply(lambda x: x['language'] if x['score']>=0.99 else 'unsure')"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":419},"id":"gHqXcZLUctjb","executionInfo":{"status":"ok","timestamp":1637896191131,"user_tz":480,"elapsed":394,"user":{"displayName":"Rishibha Bansal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"07562060109829238780"}},"outputId":"c4be1c57-c2f9-4833-9b8d-6096b958e5c4"},"source":["joined_data"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>id</th>\n","      <th>tweet</th>\n","      <th>label</th>\n","      <th>orig_tweet</th>\n","      <th>lang_spc</th>\n","      <th>lang_code_spc</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>1</th>\n","      <td>17461978</td>\n","      <td>SHAQ False True False False 15349596 692 45568...</td>\n","      <td>0</td>\n","      <td>Who has time for receipts? Not me. @epson rece...</td>\n","      <td>{'language': 'en', 'score': 0.9999971783640074}</td>\n","      <td>en</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>17461978</td>\n","      <td>SHAQ False True False False 15349596 692 45568...</td>\n","      <td>0</td>\n","      <td>Steady wants to encourage you to invest in you...</td>\n","      <td>{'language': 'en', 'score': 0.9999972659917247}</td>\n","      <td>en</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>17461978</td>\n","      <td>SHAQ False True False False 15349596 692 45568...</td>\n","      <td>0</td>\n","      <td>Good one, @rishid. But let’s see if y'all can ...</td>\n","      <td>{'language': 'sq', 'score': 0.7142845190395024}</td>\n","      <td>unsure</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>17461978</td>\n","      <td>SHAQ False True False False 15349596 692 45568...</td>\n","      <td>0</td>\n","      <td>#lsunationalchamps\\n</td>\n","      <td>{'language': 'en', 'score': 0.9999970673587957}</td>\n","      <td>en</td>\n","    </tr>\n","    <tr>\n","      <th>5</th>\n","      <td>17461978</td>\n","      <td>SHAQ False True False False 15349596 692 45568...</td>\n","      <td>0</td>\n","      <td>I stand with the student athletes! #WeWantToPl...</td>\n","      <td>{'language': 'en', 'score': 0.9999974093430682}</td>\n","      <td>en</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>945608</th>\n","      <td>926345425</td>\n","      <td>Tamare_xo False False False False 824 456 1 16...</td>\n","      <td>0</td>\n","      <td>Getting ready and realize I can’t find my lash...</td>\n","      <td>{'language': 'en', 'score': 0.9999956733790624}</td>\n","      <td>en</td>\n","    </tr>\n","    <tr>\n","      <th>945609</th>\n","      <td>926345425</td>\n","      <td>Tamare_xo False False False False 824 456 1 16...</td>\n","      <td>0</td>\n","      <td>Parents can be so judgemental sometimes\\n</td>\n","      <td>{'language': 'en', 'score': 0.9999975935611402}</td>\n","      <td>en</td>\n","    </tr>\n","    <tr>\n","      <th>945610</th>\n","      <td>926345425</td>\n","      <td>Tamare_xo False False False False 824 456 1 16...</td>\n","      <td>0</td>\n","      <td>@djshella876_ Me woulda more likely fi tell yo...</td>\n","      <td>{'language': 'en', 'score': 0.9999967992298511}</td>\n","      <td>en</td>\n","    </tr>\n","    <tr>\n","      <th>945611</th>\n","      <td>926345425</td>\n","      <td>Tamare_xo False False False False 824 456 1 16...</td>\n","      <td>0</td>\n","      <td>🤔 https://t.co/Rn2j0xoM6q\\n</td>\n","      <td>{'language': 'en', 'score': 0.9999977318834589}</td>\n","      <td>en</td>\n","    </tr>\n","    <tr>\n","      <th>945613</th>\n","      <td>926345425</td>\n","      <td>Tamare_xo False False False False 824 456 1 16...</td>\n","      <td>0</td>\n","      <td>Always https://t.co/u1yDufe0Q8\\n</td>\n","      <td>{'language': 'en', 'score': 0.9999969869334313}</td>\n","      <td>en</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>658573 rows × 6 columns</p>\n","</div>"],"text/plain":["               id  ... lang_code_spc\n","1        17461978  ...            en\n","2        17461978  ...            en\n","3        17461978  ...        unsure\n","4        17461978  ...            en\n","5        17461978  ...            en\n","...           ...  ...           ...\n","945608  926345425  ...            en\n","945609  926345425  ...            en\n","945610  926345425  ...            en\n","945611  926345425  ...            en\n","945613  926345425  ...            en\n","\n","[658573 rows x 6 columns]"]},"metadata":{},"execution_count":55}]},{"cell_type":"code","metadata":{"id":"7zhjLWH2DF3J"},"source":["# joined['ext'] = joined['tweet'].apply(lambda x: re.search( , x))\n","# re.sub(r'([a-zA-Z])\\1{2,}', r'\\1', 'whaaaaaaaat')"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"344f7514"},"source":["#also try spacy, pycld2: https://towardsdatascience.com/4-python-libraries-to-detect-english-and-non-english-language-c82ad3efd430\n","# from spacy_langdetect import LanguageDetector\n","# import spacy\n","# lang_detect = spacy.load('en') #defaults to english\n","# lang_detect.add_pipe(LanguageDetector(), name='language_detector', last=True)\n","\n","# def detect_language(text, lang_detect):\n","#     doc = lang_detect(text)\n","#     return doc._.language\n","\n","# TRY IF WE NEED MULTIPLE LANGUAGES:\n","# start = time()\n","# import pycld2 as cld2\n","# text_content = \"Er lebt mit seinen Eltern und seiner Schwester in Berlin. Welcome, to this world of Data Scientist. Today is a lovely day.\"\n","# text_content = \"\"\" A accès aux chiens et aux frontaux qui lui ont été il peut consulter et modifier ses collections et exporter Cet article concerne le pays européen aujourd’hui appelé République française. \n","# Pour d’autres usages du nom France, Pour une aide rapide et effective, veuiller trouver votre aide dans le menu ci-dessus. \n","# Welcome, to this world of Data Scientist. Today is a lovely day.\"\"\"\n","# def detect_language(text):\n","#     detected_language = ((0, 0, 'f', 'failed_char'),)\n","#     try:\n","#         # check_for = {'', ''} #\\x92, \\x82\n","#         # for ch in check_for:\n","#         #     if ch in text:\n","#         #         text = re.sub(ch, '', text)\n","#         _, _, _, detected_language = cld2.detect(text,  returnVectors=True)\n","#     except Exception as e:\n","#         print(text, e, '\\n')\n","#     if not detected_language:\n","#         return 'en'\n","#     return max(detected_language, key=lambda x:x[1])[-1]\n","\n","# data['lang'] = data['tweet'].apply(lambda x: detect_language(x))\n","# print(time() - start)\n"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"dwGqPL4uIiaV","executionInfo":{"status":"ok","timestamp":1637833827145,"user_tz":480,"elapsed":5866250,"user":{"displayName":"Rishibha Bansal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"07562060109829238780"}},"outputId":"7a6045e8-f22a-45f4-e638-b05eb30ce70f"},"source":["# start = time()\n","# dataset = 'dev'\n","# data, user_metadata = preprocessing(folder_path+'Twibot-20.zip', dataset+'.json', include_meta=False)\n","# data['lang'] = data['tweet'].apply(lambda x: detect_language(x, lang_detect))\n","# data.to_csv(folder_path+'dev_tweets_spacy_labelled.csv', index=False)\n","# print(time() - start)"],"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Ratio of RT tweets by bots: 78591 / 212950 ( 36.905846442826956 %)\n","Ratio of RT tweets by humans: 48586 / 188575 ( 25.764815060320828 %)\n","5865.719421386719\n"]}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"e1qzo2-ZIieW","executionInfo":{"status":"ok","timestamp":1637836731183,"user_tz":480,"elapsed":2903229,"user":{"displayName":"Rishibha Bansal","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"07562060109829238780"}},"outputId":"95f42a44-72dd-434e-b241-5e361e968b1d"},"source":["# start = time()\n","# dataset = 'test'\n","# data, user_metadata = preprocessing(folder_path+'Twibot-20.zip', dataset+'.json', include_meta=False)\n","# data['lang'] = data['tweet'].apply(lambda x: detect_language(x, lang_detect))\n","# data.to_csv(folder_path+'test_tweets_spacy_labelled.csv', index=False)\n","# print(time() - start)"],"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Ratio of RT tweets by bots: 39970 / 102927 ( 38.83334790676888 %)\n","Ratio of RT tweets by humans: 25391 / 96926 ( 26.196273445721477 %)\n","2902.9452855587006\n"]}]},{"cell_type":"code","metadata":{"id":"LCpMmR89tmB4"},"source":["#*********************** ROUND 2 **************************"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"8o1got2ICQIG"},"source":["torch.cuda.is_available()\n","dtype = torch.cuda.FloatTensor"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"lf39yRP3CXYI"},"source":["data_folder = 'drive/MyDrive/CSCI_544_NLP_Project/Twitter_Bot_Detection/Data/'"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":266,"referenced_widgets":["17972f00f5174456a6826d5160ceeeef","edd44ae3c3f745d7a40f24d89994c8d2","2a85af42b57e4f83b9f5ab529e5256bd","da4b2b99391942d2a17c3012274f1994","037826093c754649a921b8642f03753d","38eb97458e9b42fba80711989ccf9df8","24caa273d3e0451aadd9bb7f2fbe5e26","81f8960d6b1b4916b79ce14512b500e2","0bebbb11af324bd8b47f1ebd86cee164","ab5cd1791416490fa7d4ac3e8e9005fe","0065185116fb47d4ba26bb82d004faa5","79cf476811f74502b85964a43cbdd97f","9084ba816b564fd29011b68bbb6806e1","51274cd0c5c2457fbc4c6e174f086aa5","407fd88ec7e24230903ccd391d04e72c","c59c6ac231a6441492758af81c6ab59e","beaf09c2052948d3b018c7e3bad9b772","41d1d7ab35fe495ea1dd846d50981008","49cd106830de47b684bfb743ea818804","7f31f06ac87d4ce8af878257d090075b","aa743b4f2a534ff781d27562c5e178d5","7a9cdb191ae44e02941e6b0d3923f281","fcd717a1f9804dffb585b36302a73f2c","1191bbb189094e47a8ab0c0e5b18ba92","afdd685619954083bd74e0fb023697b6","2b35f47086bc4c1b98cee9e531813224","1e81341e012640a78b1f9fbfe8b23777","f059f234ba7244129e592509587d9615","7414d6b34bb04996bd20cdda764eef45","7ca1d4599d0c4b409e625b70fd297486","11842ad53bd1461db4844ddfb78b24ab","beaad27eb3ba45ed9f13cec7c131b40e","412e823e9ebf4e2585d8a0cdb35baad5","9f8ebaad5e65459d83b733192ededa8e","a3b875f8384b45de80572cc4a5333130","05d6a9f3664643bbaf66787364836d33","ae3976a71b4b4a35ba1953bde354bd63","23e3a78dacca4415ac0cfc8096fe0337","e1004bd075df472688b69337e04af14a","5b82dc843c5a4222a2a04e6d647a8936"]},"id":"LOql_FPLCXVp","executionInfo":{"elapsed":18031,"status":"ok","timestamp":1637785873184,"user":{"displayName":"zeeshan ahmad","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"15348670853435150367"},"user_tz":480},"outputId":"5fd494f0-b51d-4c47-98d7-29feb081bafd"},"source":["model = SimCSE(\"princeton-nlp/sup-simcse-bert-base-uncased\")\n","## is supervised and case insensitive"],"execution_count":null,"outputs":[{"data":{"text/html":["\n","  <style>\n","    pre {\n","        white-space: pre-wrap;\n","    }\n","  </style>\n","  "],"text/plain":["<IPython.core.display.HTML object>"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"17972f00f5174456a6826d5160ceeeef","version_major":2,"version_minor":0},"text/plain":["HBox(children=(FloatProgress(value=0.0, description='Downloading', max=689.0, style=ProgressStyle(description_…"]},"metadata":{},"output_type":"display_data"},{"name":"stdout","output_type":"stream","text":["\n"]},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"0bebbb11af324bd8b47f1ebd86cee164","version_major":2,"version_minor":0},"text/plain":["HBox(children=(FloatProgress(value=0.0, description='Downloading', max=231508.0, style=ProgressStyle(descripti…"]},"metadata":{},"output_type":"display_data"},{"name":"stdout","output_type":"stream","text":["\n"]},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"beaf09c2052948d3b018c7e3bad9b772","version_major":2,"version_minor":0},"text/plain":["HBox(children=(FloatProgress(value=0.0, description='Downloading', max=112.0, style=ProgressStyle(description_…"]},"metadata":{},"output_type":"display_data"},{"name":"stdout","output_type":"stream","text":["\n"]},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"afdd685619954083bd74e0fb023697b6","version_major":2,"version_minor":0},"text/plain":["HBox(children=(FloatProgress(value=0.0, description='Downloading', max=252.0, style=ProgressStyle(description_…"]},"metadata":{},"output_type":"display_data"},{"name":"stdout","output_type":"stream","text":["\n"]},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"412e823e9ebf4e2585d8a0cdb35baad5","version_major":2,"version_minor":0},"text/plain":["HBox(children=(FloatProgress(value=0.0, description='Downloading', max=437998343.0, style=ProgressStyle(descri…"]},"metadata":{},"output_type":"display_data"},{"name":"stdout","output_type":"stream","text":["\n"]}]},{"cell_type":"code","metadata":{"id":"qo16RNDJCXHv"},"source":["data = pd.read_csv(data_folder+'train_tweets_spacy_labelled.csv')"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"0PJs1MwkCXEu"},"source":["data = data[data['lang_code']=='en'].reset_index(drop=True)\n","data = data.drop(columns=['orig_tweet', 'lang', 'lang_code'])"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"WvAgdfV4CRmV"},"source":["from IPython.display import HTML, display\n","\n","def set_css():\n","  display(HTML('''\n","  <style>\n","    pre {\n","        white-space: pre-wrap;\n","    }\n","  </style>\n","  '''))\n","get_ipython().events.register('pre_run_cell', set_css)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":53},"id":"sPGHTCdlBxux","executionInfo":{"elapsed":25,"status":"ok","timestamp":1637785873188,"user":{"displayName":"zeeshan ahmad","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"15348670853435150367"},"user_tz":480},"outputId":"feddc72e-9c67-42f3-bd9d-b70c04d2d054"},"source":["print(dir(model))"],"execution_count":null,"outputs":[{"data":{"text/html":["\n","  <style>\n","    pre {\n","        white-space: pre-wrap;\n","    }\n","  </style>\n","  "],"text/plain":["<IPython.core.display.HTML object>"]},"metadata":{},"output_type":"display_data"},{"name":"stdout","output_type":"stream","text":["['__class__', '__delattr__', '__dict__', '__dir__', '__doc__', '__eq__', '__format__', '__ge__', '__getattribute__', '__gt__', '__hash__', '__init__', '__init_subclass__', '__le__', '__lt__', '__module__', '__ne__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', '__setattr__', '__sizeof__', '__str__', '__subclasshook__', '__weakref__', 'build_index', 'device', 'encode', 'index', 'is_faiss_index', 'model', 'num_cells', 'num_cells_in_search', 'pooler', 'search', 'similarity', 'tokenizer']\n"]}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":17},"id":"HT1UqJPLCXCQ","executionInfo":{"elapsed":21,"status":"ok","timestamp":1637785873189,"user":{"displayName":"zeeshan ahmad","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"15348670853435150367"},"user_tz":480},"outputId":"11963964-7098-4b66-fd3c-c2b31d69138a"},"source":["tweet_sample = data.sample(1000)['tweet'].to_list()"],"execution_count":null,"outputs":[{"data":{"text/html":["\n","  <style>\n","    pre {\n","        white-space: pre-wrap;\n","    }\n","  </style>\n","  "],"text/plain":["<IPython.core.display.HTML object>"]},"metadata":{},"output_type":"display_data"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":17},"id":"iUEBTVd18yQV","outputId":"58263135-3e6f-418f-afdc-a30fc99a74ca"},"source":["start = time()\n","sim_mat = model.similarity(tweet_sample, tweet_sample)\n","print( time()-start )"],"execution_count":null,"outputs":[{"data":{"text/html":["\n","  <style>\n","    pre {\n","        white-space: pre-wrap;\n","    }\n","  </style>\n","  "],"text/plain":["<IPython.core.display.HTML object>"]},"metadata":{},"output_type":"display_data"}]},{"cell_type":"markdown","metadata":{"id":"jmE0kMXEPas7"},"source":["## Test similarity matrix for 1000 tweets"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":160},"id":"iNTX8Sm7s87w","executionInfo":{"elapsed":31,"status":"ok","timestamp":1637623233078,"user":{"displayName":"zeeshan ahmad","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"15348670853435150367"},"user_tz":480},"outputId":"64a536fb-68d5-4881-829c-16253d4c56a4"},"source":["# Get top 3 similar - the most similar entity will be the entity itself. We consider the 2nd and 3rd indices as the 1st two most similar sentences.\n","\n","import numpy as np\n","import heapq\n","\n","print(tweet_sample[999])\n","# print(sim_mat[999])\n","print(np.argmax(sim_mat[999]))\n","print(tweet_sample[np.argmax(sim_mat[999])])\n","top3 = heapq.nlargest(3, sim_mat[999])\n","print(top3)\n","top3_indices = heapq.nlargest(3, range(len(sim_mat[999])), key=sim_mat[999].__getitem__)\n","print(top3_indices)\n","\n","for idx in top3_indices:\n","  print(tweet_sample[idx])\n"],"execution_count":null,"outputs":[{"data":{"text/html":["\n","  <style>\n","    pre {\n","        white-space: pre-wrap;\n","    }\n","  </style>\n","  "],"text/plain":["<IPython.core.display.HTML object>"]},"metadata":{},"output_type":"display_data"},{"name":"stdout","output_type":"stream","text":["wipe your tears <_url_>\n","999\n","wipe your tears <_url_>\n","[1.0000002, 0.55124474, 0.5318104]\n","[999, 360, 564]\n","wipe your tears <_url_>\n","love you al , thank you 💓 💓 💓 <_url_>\n","<_user_> so sorry for your loss . may she rest in peace and may you all find peace and cherish the memories .\n"]}]},{"cell_type":"markdown","metadata":{"id":"Novhc13jPkuh"},"source":["## Build a similarity matrix for all tweets in the dataset (post processing the dataset)\n","Takes about 1 hour to build the similarity matrix for 600k x 600k (all tweets x all tweets ) and\n","Takes about 15 minutes to build the similarity matrix for 1 x 325k (one tweet row x all bot tweets)"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":197},"id":"EKTym3JLC57o","executionInfo":{"elapsed":894798,"status":"ok","timestamp":1637786768795,"user":{"displayName":"zeeshan ahmad","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"15348670853435150367"},"user_tz":480},"outputId":"2c99efa3-01b9-457d-e79c-7eb9963cd1ae"},"source":["# Not enough memory to pre-compile similarity matrices - even for just all_tweets x bot_tweets\n","\n","sim_mat_time_start = time()\n","# # data['tweet'] = data['tweet'].apply(lambda x: ' '.join(x))\n","\n","all_tweets = data['tweet'].to_list()\n","print(len(all_tweets))\n","bot_tweets = data[data['label']==1]['tweet'].to_list()\n","human_tweets = data[data['label']==0]['tweet'].to_list()\n","print(\"bot_tweets\", len(bot_tweets))\n","print(\"human_tweets\", len(human_tweets))\n","print(len(bot_tweets) + len(human_tweets))\n","\n","# Build a similarity matrix for all tweets with bot tweets\n","# sim_mat_bot = model.similarity(all_tweets, bot_tweets)\n","sim_mat_bot = model.similarity(all_tweets[0], bot_tweets)\n","print(len(sim_mat_bot))\n","\n","# # Build a similarity matrix for all tweets with human tweets\n","\n","\n","\n","print(\"\\nTime taken to build similarity matrix for 1 row\", time() - sim_mat_time_start)"],"execution_count":null,"outputs":[{"data":{"text/html":["\n","  <style>\n","    pre {\n","        white-space: pre-wrap;\n","    }\n","  </style>\n","  "],"text/plain":["<IPython.core.display.HTML object>"]},"metadata":{},"output_type":"display_data"},{"name":"stderr","output_type":"stream","text":["11/24/2021 20:31:13 - INFO - numexpr.utils -   NumExpr defaulting to 4 threads.\n"]},{"name":"stdout","output_type":"stream","text":["623464\n","bot_tweets 325841\n","human_tweets 297623\n","623464\n"]},{"name":"stderr","output_type":"stream","text":["100%|██████████| 1/1 [00:00<00:00,  6.84it/s]\n","100%|██████████| 5092/5092 [14:42<00:00,  5.77it/s]\n"]},{"name":"stdout","output_type":"stream","text":["325841\n","\n","Time taken to build similarity matrix for 1 row 894.5402007102966\n"]}]},{"cell_type":"markdown","metadata":{"id":"iTbqSEUdQFR3"},"source":["## Build similarity matrices for 50k tweets. One sim matrix for tweets x bot_tweets, another for tweets x human_tweets"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":305},"id":"XfCTFBu44KnT","executionInfo":{"elapsed":11,"status":"ok","timestamp":1637625322490,"user":{"displayName":"Zeeshan Ahmad","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13074642537976358258"},"user_tz":480},"outputId":"232ed245-92b3-4501-a649-a153b2354dc7"},"source":["# Include labels while writing to the triplets CSV!\n","\n","sim_mat_time_start = time()\n","# data['tweet'] = data['tweet'].apply(lambda x: ' '.join(x))\n","\n","# As of now, we sample 50000 random rows with seed = 1. If we wanna progressively build the entire dataset, we can take the first 50000 rows, and so on.\n","data_subset = data[['tweet','label']].sample(n=50000, random_state=1)\n","tweet_subset = data_subset['tweet'].to_list()\n","label_subset = data_subset['label'].to_list()\n","print(data_subset[:5])\n","print(data_subset[-5:])\n","bot_tweets = data_subset[data_subset['label']==1]['tweet'].to_list()\n","bot_tweet_labels = data_subset[data_subset['label']==1]['label'].to_list()\n","human_tweets = data_subset[data_subset['label']==0]['tweet'].to_list()\n","human_tweet_labels = data_subset[data_subset['label']==0]['label'].to_list()\n","print(\"bot_tweets\", len(bot_tweets))\n","print(\"human_tweets\", len(human_tweets))\n","print(\"total tweets\", len(bot_tweets) + len(human_tweets))"],"execution_count":null,"outputs":[{"data":{"text/html":["\n","  <style>\n","    pre {\n","        white-space: pre-wrap;\n","    }\n","  </style>\n","  "],"text/plain":["<IPython.core.display.HTML object>"]},"metadata":{},"output_type":"display_data"},{"name":"stderr","output_type":"stream","text":["11/22/2021 23:55:20 - INFO - numexpr.utils -   NumExpr defaulting to 2 threads.\n"]},{"name":"stdout","output_type":"stream","text":["                                                    tweet  label\n","446581  brazilian govt . pretending to protect highly ...      1\n","208653  <_user_> she is the one inviting folks over 😂 ...      0\n","124024  back in lithuania at the end of this month <_u...      0\n","26152   chance the rapper is set to make a movie music...      1\n","450671  <_user_> you are such a savior of the common m...      1\n","                                                    tweet  label\n","513085  <_user_> russians are so sneaky ... they trick...      1\n","210624  <_user_> <_user_> his team does not want him t...      1\n","399449  <_user_> i was completely wrong about him but ...      1\n","227756  <_user_> congrats again . not sure if you got ...      1\n","144317  <_url_> check this short out . my son wrote an...      0\n","bot_tweets 26278\n","human_tweets 23722\n","total tweets 50000\n"]}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":17},"id":"bEflLoF2Q7he","executionInfo":{"elapsed":7,"status":"ok","timestamp":1637623233259,"user":{"displayName":"zeeshan ahmad","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"15348670853435150367"},"user_tz":480},"outputId":"3d1aaecc-6eef-4fba-bc4d-3bae00e39478"},"source":["# Build a similarity matrix for all tweets with bot tweets\n","\n","# sim_mat_bot = model.similarity(tweet_subset, bot_tweets)\n","# print(len(sim_mat_bot))\n","\n","# np.save(data_folder+'similarity_matrix_bot_50k.npy', sim_mat_bot)"],"execution_count":null,"outputs":[{"data":{"text/html":["\n","  <style>\n","    pre {\n","        white-space: pre-wrap;\n","    }\n","  </style>\n","  "],"text/plain":["<IPython.core.display.HTML object>"]},"metadata":{},"output_type":"display_data"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":17},"id":"xf5o22jDOx3E","executionInfo":{"elapsed":8,"status":"ok","timestamp":1637623233260,"user":{"displayName":"zeeshan ahmad","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"15348670853435150367"},"user_tz":480},"outputId":"c2f17ced-e8f9-452f-ba24-9abc3732c554"},"source":["# # Build a similarity matrix for all tweets with human tweets\n","# sim_mat_human = model.similarity(tweet_subset, human_tweets)\n","# print(len(sim_mat_human))\n","# np.save(data_folder+'similarity_matrix_human_50k.npy', sim_mat_human)"],"execution_count":null,"outputs":[{"data":{"text/html":["\n","  <style>\n","    pre {\n","        white-space: pre-wrap;\n","    }\n","  </style>\n","  "],"text/plain":["<IPython.core.display.HTML object>"]},"metadata":{},"output_type":"display_data"}]},{"cell_type":"code","metadata":{"id":"FrYrh-238IW2"},"source":["sim_mat_bot = np.load(data_folder+'similarity_matrix_bot_50k.npy')\n","sim_mat_human = np.load(data_folder+'similarity_matrix_human_50k.npy')"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"dCXRvcnRtP2U"},"source":["## Build triplet CSV (dataset size = 50k right now) \n","# Prev exec : 6622.105570077896 seconds"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":305},"id":"ySaYIeUDOhEy","executionInfo":{"elapsed":6622618,"status":"ok","timestamp":1637632797101,"user":{"displayName":"Zeeshan Ahmad","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"13074642537976358258"},"user_tz":480},"outputId":"2c1d7c3b-7d00-4376-d8fe-c826b37cf7b8"},"source":["\n","triplet_csv_build_start = time()\n","\n","out = pd.DataFrame(columns=['sent0', 'sent1', 'hard_neg', 'sent0_label', 'sent1_label', 'hard_neg_label'])\n","print(\"tweets in tweet_subset\", len(tweet_subset))\n","for idx in range(len(data_subset)):\n","  if idx % 100 == 0:\n","    print(\"{}/{}\".format(idx, len(data_subset)))\n","  # find top n similar bot tweets\n","  top3_bot_indices = heapq.nlargest(3, range(len(sim_mat_bot[idx])), key=sim_mat_bot[idx].__getitem__)\n","  # # find top n similar human tweets\n","  top3_human_indices = heapq.nlargest(3, range(len(sim_mat_human[idx])), key=sim_mat_human[idx].__getitem__)\n","\n","  # combine all the permutations of the most similar bot and human tweets\n","  # ignore the first most similar bot tweet if current tweet is a bot tweet\n","  # else ignore the first most similar human tweet\n","  if label_subset[idx]==1:\n","    for bot_idx in top3_bot_indices[:2]:\n","      for human_idx in top3_human_indices[:2]:\n","        # print(\"<bot>\",tweet_subset[idx], tweet_subset[bot_idx], tweet_subset[human_idx])\n","        out = out.append({'sent0': tweet_subset[idx], 'sent1': bot_tweets[bot_idx], 'hard_neg': human_tweets[human_idx],\\\n","                          'sent0_label': label_subset[idx], 'sent1_label': bot_tweet_labels[bot_idx], 'hard_neg_label': human_tweet_labels[human_idx]}, ignore_index=True)\n","        # print(out_line)\n","        # out.append(out_line)\n","\n","  elif label_subset[idx]==0:\n","    for bot_idx in top3_bot_indices[:2]:\n","      for human_idx in top3_human_indices[1:]:\n","        # print(\"<human>\",tweet_subset[idx], tweet_subset[bot_idx], tweet_subset[human_idx])\n","        out = out.append({'sent0': tweet_subset[idx], 'sent1': human_tweets[human_idx], 'hard_neg': bot_tweets[bot_idx],\\\n","                          'sent0_label': label_subset[idx], 'sent1_label': human_tweet_labels[human_idx], 'hard_neg_label': bot_tweet_labels[bot_idx]}, ignore_index=True)\n","        # print(out_line)\n","        # out.append(out_line)\n","  else:\n","    print(\"something went wrong in the label comparison\")\n","\n","print(out)\n","\n","out.to_csv(data_folder+\"train_triplets_with_similarity.csv\", index=False)\n","\n","print('Time taken to build CSV:', (time() - triplet_csv_build_start))"],"execution_count":null,"outputs":[{"data":{"text/html":["\n","  <style>\n","    pre {\n","        white-space: pre-wrap;\n","    }\n","  </style>\n","  "],"text/plain":["<IPython.core.display.HTML object>"]},"metadata":{},"output_type":"display_data"},{"name":"stdout","output_type":"stream","text":["tweets in tweet_subset 50000\n","                                                    sent0  ... hard_neg_label\n","0       brazilian govt . pretending to protect highly ...  ...              0\n","1       brazilian govt . pretending to protect highly ...  ...              0\n","2       brazilian govt . pretending to protect highly ...  ...              0\n","3       brazilian govt . pretending to protect highly ...  ...              0\n","4       <_user_> she is the one inviting folks over 😂 ...  ...              1\n","...                                                   ...  ...            ...\n","199995  <_user_> congrats again . not sure if you got ...  ...              0\n","199996  <_url_> check this short out . my son wrote an...  ...              1\n","199997  <_url_> check this short out . my son wrote an...  ...              1\n","199998  <_url_> check this short out . my son wrote an...  ...              1\n","199999  <_url_> check this short out . my son wrote an...  ...              1\n","\n","[200000 rows x 6 columns]\n","Time taken to build CSV: 6622.105570077896\n"]}]}]}