birds_download_and_prepare.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"birds_download_and_prepare.ipynb","provenance":[],"collapsed_sections":[],"authorship_tag":"ABX9TyPm9J+hYx9rLk5RaR5tIb7j"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"},"widgets":{"application/vnd.jupyter.widget-state+json":{"444707f1e058424f8b9bc65432649cbb":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_0b7f825619b64c9bb900bd0dedcac550","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_48673c07998d492a9216c0726d127d9e","IPY_MODEL_51f1a053615f419bb113a592a0d53deb","IPY_MODEL_3446e39fb8b948e38a6c1f7db87710ca"]}},"0b7f825619b64c9bb900bd0dedcac550":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"48673c07998d492a9216c0726d127d9e":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_view_name":"HTMLView","style":"IPY_MODEL_8d4ca7c5c2d649b5aa3c345df26a6edc","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":"100%","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_e1950a755b3849e3bee16533ca342dcc"}},"51f1a053615f419bb113a592a0d53deb":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_view_name":"ProgressView","style":"IPY_MODEL_dd6facb4c2dc4d38a7e5e61447ffe2f8","_dom_classes":[],"description":"","_model_name":"FloatProgressModel","bar_style":"success","max":2144,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":2144,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_6caffdfd06364830b7ec6f6cb21134c0"}},"3446e39fb8b948e38a6c1f7db87710ca":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_view_name":"HTMLView","style":"IPY_MODEL_1573c9c73ca34906a88274365509efa5","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 2144/2144 [00:00&lt;00:00, 12296.93it/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_ff18914779f94bf88ff09696f16d04e1"}},"8d4ca7c5c2d649b5aa3c345df26a6edc":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"e1950a755b3849e3bee16533ca342dcc":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"dd6facb4c2dc4d38a7e5e61447ffe2f8":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"6caffdfd06364830b7ec6f6cb21134c0":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"1573c9c73ca34906a88274365509efa5":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"ff18914779f94bf88ff09696f16d04e1":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"459a535f93874a92a4b904b7bbc9b584":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_6be40ec45dec43d1bef829fe8bb0d631","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_fb9f97ad243a43e7bc00284dc976cd52","IPY_MODEL_c3645f2e231f4eed8fd6c2d81b35cb7b","IPY_MODEL_e86b34fb2f754fafaf42f043c7f7f8e3"]}},"6be40ec45dec43d1bef829fe8bb0d631":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"fb9f97ad243a43e7bc00284dc976cd52":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_view_name":"HTMLView","style":"IPY_MODEL_9772bb4762e141a0a11fded2c578d199","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":"100%","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_2ef93ee4c9404c5ea3a458379daaf08d"}},"c3645f2e231f4eed8fd6c2d81b35cb7b":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_view_name":"ProgressView","style":"IPY_MODEL_0dd928867ea842f5b5cff25f2829d83e","_dom_classes":[],"description":"","_model_name":"FloatProgressModel","bar_style":"success","max":4157,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":4157,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_78290f0c05c84bdf89ee215cd60878b3"}},"e86b34fb2f754fafaf42f043c7f7f8e3":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_view_name":"HTMLView","style":"IPY_MODEL_c17f09e55b6d41b48c5ddbac80e58908","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 4157/4157 [35:00&lt;00:00,  1.19it/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_4815bf38cbc04d0db203c4572f0c7063"}},"9772bb4762e141a0a11fded2c578d199":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"2ef93ee4c9404c5ea3a458379daaf08d":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"0dd928867ea842f5b5cff25f2829d83e":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"78290f0c05c84bdf89ee215cd60878b3":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"c17f09e55b6d41b48c5ddbac80e58908":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"4815bf38cbc04d0db203c4572f0c7063":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}}}}},"cells":[{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"S8dehGE5-Hg7","executionInfo":{"status":"ok","timestamp":1639487081950,"user_tz":180,"elapsed":34792,"user":{"displayName":"Fabio Frel","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"14361281977799789269"}},"outputId":"d3d392ea-0064-41a2-b9d7-cb347f1f84d7"},"source":["from google.colab import drive\n","drive.mount('/gdrive')"],"execution_count":1,"outputs":[{"output_type":"stream","name":"stdout","text":["Mounted at /gdrive\n"]}]},{"cell_type":"code","metadata":{"id":"CnOVNxsi-Kgp","executionInfo":{"status":"ok","timestamp":1639487081951,"user_tz":180,"elapsed":12,"user":{"displayName":"Fabio Frel","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"14361281977799789269"}}},"source":["from os import listdir, mkdir\n","import pandas as pd"],"execution_count":2,"outputs":[]},{"cell_type":"code","metadata":{"id":"dk7LoBG-O3SP","executionInfo":{"status":"ok","timestamp":1639487081951,"user_tz":180,"elapsed":9,"user":{"displayName":"Fabio Frel","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"14361281977799789269"}}},"source":[""],"execution_count":2,"outputs":[]},{"cell_type":"code","metadata":{"id":"Oox9Qriy-Rnd","executionInfo":{"status":"ok","timestamp":1639487104940,"user_tz":180,"elapsed":276,"user":{"displayName":"Fabio Frel","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"14361281977799789269"}}},"source":["main_dir = '/gdrive/MyDrive/BirdShazam/bird_data/'"],"execution_count":5,"outputs":[]},{"cell_type":"code","source":["import os\n","os.listdir(main_dir)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"XkxjePukxYnb","executionInfo":{"status":"ok","timestamp":1639487110458,"user_tz":180,"elapsed":237,"user":{"displayName":"Fabio Frel","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"14361281977799789269"}},"outputId":"c48a58ef-88e4-4df7-ef96-3b26f1f70273"},"execution_count":6,"outputs":[{"output_type":"execute_result","data":{"text/plain":["['birds.csv', 'to_download.csv', 'wav_files', 'downloaded.csv']"]},"metadata":{},"execution_count":6}]},{"cell_type":"code","source":["len(os.listdir(main_dir+\"/wav_files/train\"))"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"34lEpdNvC6MM","executionInfo":{"status":"ok","timestamp":1639487169919,"user_tz":180,"elapsed":237,"user":{"displayName":"Fabio Frel","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"14361281977799789269"}},"outputId":"099bfaab-882b-4322-9ccc-e7b8ab156baf"},"execution_count":10,"outputs":[{"output_type":"execute_result","data":{"text/plain":["1923"]},"metadata":{},"execution_count":10}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":200},"id":"MQDVFwoAPBcL","executionInfo":{"status":"error","timestamp":1639487115025,"user_tz":180,"elapsed":260,"user":{"displayName":"Fabio Frel","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"14361281977799789269"}},"outputId":"8f444456-c207-4152-e85c-f721a774a378"},"source":["audio_dir = main_dir+'audio_files/'\n","files = listdir(audio_dir)\n","len(files)"],"execution_count":7,"outputs":[{"output_type":"error","ename":"FileNotFoundError","evalue":"ignored","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)","\u001b[0;32m<ipython-input-7-80e1942abcb9>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0maudio_dir\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmain_dir\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;34m'audio_files/'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mfiles\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlistdir\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maudio_dir\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfiles\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: '/gdrive/MyDrive/BirdShazam/bird_data/audio_files/'"]}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":466},"id":"cdmcRR46-i2e","executionInfo":{"status":"error","timestamp":1639313006815,"user_tz":180,"elapsed":3947,"user":{"displayName":"Fabio Frel","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"14361281977799789269"}},"outputId":"f5eb910c-7519-45cd-f731-9fac9d061415"},"source":["import pandas as pd\n","import requests\n","from tqdm import tqdm\n","\n","country = 'brazil'\n","response = requests.get(f'https://www.xeno-canto.org/api/2/recordings?query=cnt:{country}')\n","js = response.json()\n","\n","df = pd.DataFrame()\n","for n_page in tqdm(range(1, js[\"numPages\"]+1)):\n","    response = requests.get(f'https://www.xeno-canto.org/api/2/recordings?query=cnt:{country}&page={n_page}')\n","    page_json = response.json()\n","    ids, files, file_names, ens, lengths, gens = [], [], [], [], [], []\n","    for recording in page_json[\"recordings\"]:\n","        ids.append(recording[\"id\"])\n","        files.append(recording[\"file\"])\n","        file_names.append(recording[\"file-name\"])\n","        ens.append(recording[\"en\"])\n","        lengths.append(recording[\"length\"])\n","        gens.append(recording[\"gen\"])\n","\n","\n","    df_ = pd.DataFrame.from_records({'id': ids, \"file\": files, \"file-name\": file_names, \"en\": ens, \"gen\": gens, \"length\": lengths})\n","    df = df.append(df_)\n","    \n","df.to_csv(dir+\"birds.csv\", index=False)"],"execution_count":null,"outputs":[{"output_type":"stream","name":"stderr","text":["  0%|          | 0/118 [00:00<?, ?it/s]\n"]},{"output_type":"error","ename":"KeyboardInterrupt","evalue":"ignored","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)","\u001b[0;32m/usr/local/lib/python3.7/dist-packages/urllib3/connectionpool.py\u001b[0m in \u001b[0;36m_make_request\u001b[0;34m(self, conn, method, url, timeout, chunked, **httplib_request_kw)\u001b[0m\n\u001b[1;32m    376\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# Python 2.7, use buffering of HTTP responses\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 377\u001b[0;31m                 \u001b[0mhttplib_response\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mconn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgetresponse\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbuffering\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    378\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# Python 3\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;31mTypeError\u001b[0m: getresponse() got an unexpected keyword argument 'buffering'","\nDuring handling of the above exception, another exception occurred:\n","\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)","\u001b[0;32m<ipython-input-6-827e3a0f92e3>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0mdf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDataFrame\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mn_page\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtqdm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mjs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"numPages\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 11\u001b[0;31m     \u001b[0mresponse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrequests\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf'https://www.xeno-canto.org/api/2/recordings?query=cnt:{country}&page={n_page}'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     12\u001b[0m     \u001b[0mpage_json\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjson\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m     \u001b[0mids\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfiles\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfile_names\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mens\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlengths\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgens\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.7/dist-packages/requests/api.py\u001b[0m in \u001b[0;36mget\u001b[0;34m(url, params, **kwargs)\u001b[0m\n\u001b[1;32m     74\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     75\u001b[0m     \u001b[0mkwargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msetdefault\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'allow_redirects'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 76\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mrequest\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'get'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0murl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     77\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     78\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.7/dist-packages/requests/api.py\u001b[0m in \u001b[0;36mrequest\u001b[0;34m(method, url, **kwargs)\u001b[0m\n\u001b[1;32m     59\u001b[0m     \u001b[0;31m# cases, and look like a memory leak in others.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     60\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0msessions\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSession\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0msession\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 61\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0msession\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmethod\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmethod\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0murl\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0murl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     62\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     63\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.7/dist-packages/requests/sessions.py\u001b[0m in \u001b[0;36mrequest\u001b[0;34m(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)\u001b[0m\n\u001b[1;32m    528\u001b[0m         }\n\u001b[1;32m    529\u001b[0m         \u001b[0msend_kwargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msettings\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 530\u001b[0;31m         \u001b[0mresp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprep\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0msend_kwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    531\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    532\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mresp\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.7/dist-packages/requests/sessions.py\u001b[0m in \u001b[0;36msend\u001b[0;34m(self, request, **kwargs)\u001b[0m\n\u001b[1;32m    641\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    642\u001b[0m         \u001b[0;31m# Send the request\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 643\u001b[0;31m         \u001b[0mr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0madapter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    644\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    645\u001b[0m         \u001b[0;31m# Total elapsed time of the request (approximately)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.7/dist-packages/requests/adapters.py\u001b[0m in \u001b[0;36msend\u001b[0;34m(self, request, stream, timeout, verify, cert, proxies)\u001b[0m\n\u001b[1;32m    447\u001b[0m                     \u001b[0mdecode_content\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    448\u001b[0m                     \u001b[0mretries\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax_retries\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 449\u001b[0;31m                     \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    450\u001b[0m                 )\n\u001b[1;32m    451\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.7/dist-packages/urllib3/connectionpool.py\u001b[0m in \u001b[0;36murlopen\u001b[0;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)\u001b[0m\n\u001b[1;32m    598\u001b[0m                                                   \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtimeout_obj\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    599\u001b[0m                                                   \u001b[0mbody\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbody\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mheaders\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mheaders\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 600\u001b[0;31m                                                   chunked=chunked)\n\u001b[0m\u001b[1;32m    601\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    602\u001b[0m             \u001b[0;31m# If we're going to release the connection in ``finally:``, then\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.7/dist-packages/urllib3/connectionpool.py\u001b[0m in \u001b[0;36m_make_request\u001b[0;34m(self, conn, method, url, timeout, chunked, **httplib_request_kw)\u001b[0m\n\u001b[1;32m    378\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# Python 3\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    379\u001b[0m                 \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 380\u001b[0;31m                     \u001b[0mhttplib_response\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mconn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgetresponse\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    381\u001b[0m                 \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    382\u001b[0m                     \u001b[0;31m# Remove the TypeError from the exception chain in Python 3;\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/lib/python3.7/http/client.py\u001b[0m in \u001b[0;36mgetresponse\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1371\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1372\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1373\u001b[0;31m                 \u001b[0mresponse\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbegin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1374\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mConnectionError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1375\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/lib/python3.7/http/client.py\u001b[0m in \u001b[0;36mbegin\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    317\u001b[0m         \u001b[0;31m# read until we get a non-100 response\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    318\u001b[0m         \u001b[0;32mwhile\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 319\u001b[0;31m             \u001b[0mversion\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstatus\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreason\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_read_status\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    320\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mstatus\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0mCONTINUE\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    321\u001b[0m                 \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/lib/python3.7/http/client.py\u001b[0m in \u001b[0;36m_read_status\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    278\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    279\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_read_status\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 280\u001b[0;31m         \u001b[0mline\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreadline\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_MAXLINE\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"iso-8859-1\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    281\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mline\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0m_MAXLINE\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    282\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mLineTooLong\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"status line\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/lib/python3.7/socket.py\u001b[0m in \u001b[0;36mreadinto\u001b[0;34m(self, b)\u001b[0m\n\u001b[1;32m    587\u001b[0m         \u001b[0;32mwhile\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    588\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 589\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sock\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrecv_into\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    590\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    591\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_timeout_occurred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/lib/python3.7/ssl.py\u001b[0m in \u001b[0;36mrecv_into\u001b[0;34m(self, buffer, nbytes, flags)\u001b[0m\n\u001b[1;32m   1069\u001b[0m                   \u001b[0;34m\"non-zero flags not allowed in calls to recv_into() on %s\"\u001b[0m \u001b[0;34m%\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1070\u001b[0m                   self.__class__)\n\u001b[0;32m-> 1071\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnbytes\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbuffer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1072\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1073\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrecv_into\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbuffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnbytes\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mflags\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/lib/python3.7/ssl.py\u001b[0m in \u001b[0;36mread\u001b[0;34m(self, len, buffer)\u001b[0m\n\u001b[1;32m    927\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    928\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mbuffer\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 929\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sslobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbuffer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    930\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    931\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sslobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;31mKeyboardInterrupt\u001b[0m: "]}]},{"cell_type":"code","metadata":{"id":"mLiX29QV-sRm"},"source":["from pandas import read_csv\n","from urllib.request import urlretrieve\n","from os import listdir, mkdir, path as os_path\n","from tqdm.notebook import tqdm\n","from time import sleep\n","\n","df = read_csv(dir+\"birds.csv\")\n","\n","#Removing recordings that are not classified\n","df = df[df['en'] != 'Identity unknown'].copy()\n","df = df[df['en'] != 'Soundscape'].copy()\n","\n","counts = df['en'].value_counts()\n","chosen = counts.head(25).index\n","df = df[df[\"en\"].isin(chosen)]\n","df.to_csv(dir+\"to_download.csv\", index=False)\n","\n","print(df.shape)\n","print(len(df[\"en\"].unique()), \"different species\")\n","\n","audio_dir = dir+'audio_files/'\n","if not os_path.isdir(audio_dir):\n","    mkdir(audio_dir)\n","\n","nots = []\n","\n","for row in tqdm(df.iterrows(), total=df.shape[0]):\n","    url = row[1][\"file\"]\n","    f_name = str(row[1][\"id\"]) + row[1][\"file-name\"][-4:].lower()\n","    try:\n","        urlretrieve(url, filename=audio_dir + f_name)\n","    except Exception as e:\n","        print(\"\\nRetrying:\", url)\n","        print(e) \n","        sleep(60)\n","        try:\n","            urlretrieve(url, filename=audio_dir + f_name)\n","        except Exception as ee:\n","            print(\"Not downloaded|\", f_name)\n","            print(ee)\n","            nots.append(row[1][\"id\"])\n","            pass\n","\n","if len(nots) > 0:\n","    with open(dir+'not_downloaded.txt', 'w') as f:\n","        for item in nots:\n","            f.write(str(item) + '\\n')\n","    print(str(nots))\n","else:\n","    print('All files were successfully downloaded!')"],"execution_count":null,"outputs":[]},{"cell_type":"code","source":["!pip install pydub\n","from sklearn.model_selection import train_test_split\n","from pydub import AudioSegment\n","from os import listdir, mkdir, path as os_path, remove as os_remove\n","from tqdm.notebook import tqdm\n","\n","#Check downloaded files\n","files = listdir(audio_dir)\n","df = pd.read_csv(dir+'to_download.csv')\n","df['file-name'] = df['file-name'].str.lower()\n","df['filename'] = df['id'].astype(str) + df['file-name'].str[-4:]\n","downloaded = df[df['filename'].isin(files)][['filename', 'en']]\n","downloaded.to_csv(dir+'downloaded.csv', index=False)\n","print('Not downloaded:', len(df) - len(downloaded))\n","\n","train, test = train_test_split(downloaded[['filename', 'en']], stratify=downloaded[\"en\"], test_size=0.1)\n","\n","def to_wav(filename, n_channels, sample_rate, input_dir, output_dir, delmp3 = False):\n","    input_type = filename[-4:].lower()\n","    if input_type == '.mp3':\n","        sound = AudioSegment.from_mp3(f\"{input_dir}/{filename}\")\n","        new_fn = fn[:-4] + '.wav'\n","    elif input_type == '.wav':\n","        sound = AudioSegment.from_wav(f\"{input_dir}/{filename}\")\n","        new_fn = filename\n","    sound = sound.set_channels(n_channels)\n","    sound.set_frame_rate(sample_rate)\n","    sound.export(f\"{output_dir}/{new_fn}\", format=\"wav\")\n","    if delmp3:\n","        if new_fn in listdir(output_dir):\n","            os_remove(f\"{input_dir}/{filename}\")\n","        else:\n","            print(f'Problema: {input_dir}/{filename}')\n","\n","wav_dir = dir+'wav_files/'\n","if not os_path.isdir(wav_dir):\n","    mkdir(wav_dir)\n","    mkdir(wav_dir + 'test/')\n","    mkdir(wav_dir + 'train/')\n","    \n","# for fn in tqdm(test['filename']):\n","#     to_wav(fn, n_channels=1, sample_rate=16000, input_dir=audio_dir, output_dir=wav_dir+'test')\n","\n","for fn in tqdm(train['filename']):\n","    try:\n","        to_wav(fn, n_channels=1, sample_rate=16000, input_dir=audio_dir, output_dir=wav_dir+'train', delmp3=True)\n","    except Exception as e:\n","        print(fn)\n","        print(e)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":450,"referenced_widgets":["459a535f93874a92a4b904b7bbc9b584","6be40ec45dec43d1bef829fe8bb0d631","fb9f97ad243a43e7bc00284dc976cd52","c3645f2e231f4eed8fd6c2d81b35cb7b","e86b34fb2f754fafaf42f043c7f7f8e3","9772bb4762e141a0a11fded2c578d199","2ef93ee4c9404c5ea3a458379daaf08d","0dd928867ea842f5b5cff25f2829d83e","78290f0c05c84bdf89ee215cd60878b3","c17f09e55b6d41b48c5ddbac80e58908","4815bf38cbc04d0db203c4572f0c7063"]},"id":"YNjQx7imRsh4","executionInfo":{"status":"ok","timestamp":1639323917273,"user_tz":180,"elapsed":2106514,"user":{"displayName":"Fabio Frel","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"14361281977799789269"}},"outputId":"9bfc0826-d195-48fd-eae4-686945ed2d14"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Requirement already satisfied: pydub in /usr/local/lib/python3.7/dist-packages (0.25.1)\n","Not downloaded: 0\n"]},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"459a535f93874a92a4b904b7bbc9b584","version_minor":0,"version_major":2},"text/plain":["  0%|          | 0/4157 [00:00<?, ?it/s]"]},"metadata":{}},{"output_type":"stream","name":"stdout","text":["586414.wav\n","Decoding failed. ffmpeg returned error code: 1\n","\n","Output from ffmpeg/avlib:\n","\n","ffmpeg version 3.4.8-0ubuntu0.2 Copyright (c) 2000-2020 the FFmpeg developers\n","  built with gcc 7 (Ubuntu 7.5.0-3ubuntu1~18.04)\n","  configuration: --prefix=/usr --extra-version=0ubuntu0.2 --toolchain=hardened --libdir=/usr/lib/x86_64-linux-gnu --incdir=/usr/include/x86_64-linux-gnu --enable-gpl --disable-stripping --enable-avresample --enable-avisynth --enable-gnutls --enable-ladspa --enable-libass --enable-libbluray --enable-libbs2b --enable-libcaca --enable-libcdio --enable-libflite --enable-libfontconfig --enable-libfreetype --enable-libfribidi --enable-libgme --enable-libgsm --enable-libmp3lame --enable-libmysofa --enable-libopenjpeg --enable-libopenmpt --enable-libopus --enable-libpulse --enable-librubberband --enable-librsvg --enable-libshine --enable-libsnappy --enable-libsoxr --enable-libspeex --enable-libssh --enable-libtheora --enable-libtwolame --enable-libvorbis --enable-libvpx --enable-libwavpack --enable-libwebp --enable-libx265 --enable-libxml2 --enable-libxvid --enable-libzmq --enable-libzvbi --enable-omx --enable-openal --enable-opengl --enable-sdl2 --enable-libdc1394 --enable-libdrm --enable-libiec61883 --enable-chromaprint --enable-frei0r --enable-libopencv --enable-libx264 --enable-shared\n","  libavutil      55. 78.100 / 55. 78.100\n","  libavcodec     57.107.100 / 57.107.100\n","  libavformat    57. 83.100 / 57. 83.100\n","  libavdevice    57. 10.100 / 57. 10.100\n","  libavfilter     6.107.100 /  6.107.100\n","  libavresample   3.  7.  0 /  3.  7.  0\n","  libswscale      4.  8.100 /  4.  8.100\n","  libswresample   2.  9.100 /  2.  9.100\n","  libpostproc    54.  7.100 / 54.  7.100\n","[wav @ 0x559ed55fe000] invalid start code [255][251][212][196] in RIFF header\n","/gdrive/MyDrive/BirdShazam/bird_data/audio_files//586414.wav: Invalid data found when processing input\n","\n"]}]},{"cell_type":"code","source":["len(listdir(wav_dir + 'train/'))"],"metadata":{"id":"VwBEUT8AUtjS","executionInfo":{"status":"ok","timestamp":1639324406740,"user_tz":180,"elapsed":9265,"user":{"displayName":"Fabio Frel","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"14361281977799789269"}},"outputId":"ee6e9282-5859-43f8-de3f-1ca93e7399f7","colab":{"base_uri":"https://localhost:8080/"}},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["1923"]},"metadata":{},"execution_count":12}]},{"cell_type":"code","source":["wav_dir ='/gdrive/MyDrive/BirdShazam/bird_data/wav_files/'"],"metadata":{"id":"6QD2315URso6"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"IaEu4EHgJxTu"},"source":["# Nova seção"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"fIT_S1eA__vj","executionInfo":{"status":"ok","timestamp":1635451525990,"user_tz":180,"elapsed":766,"user":{"displayName":"Fabio Frel","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"14361281977799789269"}},"outputId":"80d38f99-bbd0-4097-d419-5dbb5e5b63ae"},"source":["len(df)"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["2144"]},"metadata":{},"execution_count":44}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"ntXAVnzhMoUL","executionInfo":{"status":"ok","timestamp":1635451525990,"user_tz":180,"elapsed":4,"user":{"displayName":"Fabio Frel","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"14361281977799789269"}},"outputId":"664a80b0-baea-4cad-98a4-e7c7f368d7d1"},"source":["len(downloaded)"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["2144"]},"metadata":{},"execution_count":45}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"ijJf47IvMpHq","executionInfo":{"status":"ok","timestamp":1635451526525,"user_tz":180,"elapsed":537,"user":{"displayName":"Fabio Frel","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"14361281977799789269"}},"outputId":"94c64a69-5985-47a9-d4e8-5bc7002589e7"},"source":["len(files)"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["2144"]},"metadata":{},"execution_count":46}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":111},"id":"lDzfANWjR91e","executionInfo":{"status":"ok","timestamp":1635447784932,"user_tz":180,"elapsed":419,"user":{"displayName":"Fabio Frel","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"14361281977799789269"}},"outputId":"0b89dbee-ee3f-4d00-fb2e-51c04ff6f34b"},"source":["df[~df['filename'].isin(files)][['filename', 'en']]"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>filename</th>\n","      <th>en</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>18632</th>\n","      <td>593914.mp3</td>\n","      <td>Lesser Woodcreeper</td>\n","    </tr>\n","    <tr>\n","      <th>43429</th>\n","      <td>539758.mp3</td>\n","      <td>Blue Manakin</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["         filename                  en\n","18632  593914.mp3  Lesser Woodcreeper\n","43429  539758.mp3        Blue Manakin"]},"metadata":{},"execution_count":24}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"DlqJKXAuS4S4","executionInfo":{"status":"ok","timestamp":1635447985477,"user_tz":180,"elapsed":457,"user":{"displayName":"Fabio Frel","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"14361281977799789269"}},"outputId":"1f4fd419-5929-40a8-b812-ad9ee537f930"},"source":["'678053.mp3' in df['filename'].to_list()"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["True"]},"metadata":{},"execution_count":36}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":83,"referenced_widgets":["444707f1e058424f8b9bc65432649cbb","0b7f825619b64c9bb900bd0dedcac550","48673c07998d492a9216c0726d127d9e","51f1a053615f419bb113a592a0d53deb","3446e39fb8b948e38a6c1f7db87710ca","8d4ca7c5c2d649b5aa3c345df26a6edc","e1950a755b3849e3bee16533ca342dcc","dd6facb4c2dc4d38a7e5e61447ffe2f8","6caffdfd06364830b7ec6f6cb21134c0","1573c9c73ca34906a88274365509efa5","ff18914779f94bf88ff09696f16d04e1"]},"id":"M0zLr3G1SKh5","executionInfo":{"status":"ok","timestamp":1635447992394,"user_tz":180,"elapsed":449,"user":{"displayName":"Fabio Frel","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"14361281977799789269"}},"outputId":"93ef1398-f26a-4e7f-fcf1-df477b8bcf72"},"source":["for file in tqdm(files):\n","    if file not in df['filename'].to_list():\n","        print(file)"],"execution_count":null,"outputs":[{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"444707f1e058424f8b9bc65432649cbb","version_minor":0,"version_major":2},"text/plain":["  0%|          | 0/2144 [00:00<?, ?it/s]"]},"metadata":{}},{"output_type":"stream","name":"stdout","text":["593914.wav\n","539758.wav\n"]}]},{"cell_type":"code","metadata":{"id":"pFgoGVPUSyeF"},"source":[""],"execution_count":null,"outputs":[]}]}