diff --git a/.gitignore b/.gitignore index e1d6c7ed..de07e31c 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ venv/ __pycache__ *.pyc .idea/ +**/.DS_Store predictions/ prescriptions/ diff --git a/covid_xprize/examples/predictors/lstm/data/Additional_Context_Data_Global.csv b/covid_xprize/examples/predictors/lstm/data/Additional_Context_Data_Global.csv index 6e83c2ad..6ea22843 100644 --- a/covid_xprize/examples/predictors/lstm/data/Additional_Context_Data_Global.csv +++ b/covid_xprize/examples/predictors/lstm/data/Additional_Context_Data_Global.csv @@ -8,12 +8,17 @@ Argentina,ARG,45195774,16.2585101,91.87,11.11778888,11683.94962,28.3,1.176868831 Aruba,ABW,106766,,,,,,,,,,, Australia,AUS,25499884,3.249129487,86.012,15.65647523,57373.68668,29,3.042002085,12.62,9.071138937,2.86,15.38876602,665384 Austria,AUT,9006398,107.2069267,58.297,19.00156646,51461.95478,20.1,2.228314728,14.13,4.587225585,4.29,6.86986793,130260 +Azerbaijan,AZE,10127874,,,,,,,,,,, +Bahamas,BHS,385637,,,,,,,,,,, Bahrain,BHR,1701575,2017.2737,89.287,2.426333878,24050.75751,29.8,0.642360539,9.89,18.80903443,12.23,23.45540632,57094 Bangladesh,BGD,164689383,1239.579312,36.632,5.15839064,1698.262802,3.6,0.254187684,10.23,32.49877942,13.83,0.47365761,101383.4657 Barbados,BRB,287375,666.6069767,31.147,15.80269396,17949.28151,23.1,1.502144362,6.05,38.73698576,6.18,4.467476521,0 +Belarus,BLR,9408400,,,,,,,,,,, Belgium,BEL,11589623,377.2149273,98.001,18.78874374,47518.63604,22.1,2.129967521,18.44,19.68641753,4.57,8.328159898,140674 Belize,BLZ,397628,16.79399386,45.724,4.736458777,4884.742421,24.1,0.657790338,6.79,42.90143577,5.83,1.400941234,125681.6909 +Benin,BEN,11733059,,,,,,,,,,, Bermuda,BMU,62278,1184.592572,100,0,85748.06541,,1.540409081,11.83,13.13177427,4.33,8.838314988,0 +Bhutan,BTN,754388,,,,,,,,,,, Bolivia,BOL,11673021,10.48014585,69.425,7.191947392,3548.59014,20.2,0.660451291,6.03,65.03915155,6.4,1.90636432,56592 Bosnia and Herzegovina,BIH,3280819,64.92048828,48.245,16.47031747,6065.672708,17.9,1.280815425,22.92,7.179361467,9.11,6.384938819,0 Botswana,BWA,2351627,3.977424876,69.446,4.223874346,8258.641666,18.9,0.53866266,6.33,58.98480094,5.77,3.367451334,6774 @@ -22,34 +27,51 @@ Brunei,BRN,437479,81.39696395,77.629,4.873147568,31628.32879,14.1,2.287365627,15 Bulgaria,BGR,6948445,,,,,25,1.659590404,18.86,12.95434144,7.14,, Burkina Faso,BFA,20903273,,,,,5.6,0.454025222,1.83,126.8998112,8.33,, Burundi,BDI,11890784,,,,,5.4,0.387557281,3.34,121.5234286,7.96,, +Cambodia,KHM,15288489,,,,,,,,,,, Cameroon,CMR,26545863,53.34398891,56.374,2.728877355,1533.74214,11.4,0.440479926,2.74,116.0201175,6.95,0.308791722,5550 Canada,CAN,37742154,4.075308214,81.411,17.23200668,46232.98962,29.4,4.629727772,17.67,11.57868045,2.8,15.15892722,1475062.586 +Cape Verde,CPV,543767,,,,,,,,,,, +Central African Republic,CAF,4666368,,,,,,,,,,, Chad,TCD,16425864,12.29173364,23.059,2.480518944,728.3432354,6.1,0.378859336,1.13,148.4437073,9.77,0.053407242,0 Chile,CHL,19116201,25.18944607,87.564,11.52980166,15923.35874,28,1.237788506,9.26,19.76227628,5.05,4.6490622,134661 China,CHN,1439323776,148.3488333,59.152,10.92088354,9770.847088,6.2,1.454289027,21.03,12.96889687,11.89,7.543907641,4692008 Colombia,COL,50882891,44.74870212,80.778,8.47804702,6667.7907,22.3,0.875953763,8.3,15.09932772,5.11,1.790356827,358847 +Comoros,COM,850688,,,,,,,,,,, +Congo,COG,5244359,,,,,,,,,,, Costa Rica,CRI,5094118,97.91306306,79.34,9.549847673,12027.36588,25.7,1.336493848,9.36,10.604963,4.43,1.618087849,39730 +Cote d'Ivoire,CIV,26378274,,,,,,,,,,, Croatia,HRV,4105267,73.077198,56.947,20.44543301,14909.69319,24.4,2.153365611,18.24,6.427023522,6.03,3.973804906,27189 Cuba,CUB,11326616,108.9995962,77.037,15.18621142,8821.818891,24.6,1.353519687,18.19,29.07299006,6.15,3.080994246,8733.666 +Cyprus,CYP,1189265,,,,,,,,,,, Czech Republic,CZE,10708981,137.6028879,73.792,19.42087689,23078.57349,26,2.113855018,18.84,15.66182801,5.85,9.165978376,50694 Democratic Republic of Congo,COD,89561403,37.08259236,44.46,3.017840622,561.7771824,6.7,0.373808856,2.59,111.6227305,8.33,0.063330889,18612.19624 Denmark,DNK,5792202,138.0673017,87.874,19.81295264,61350.34791,19.7,2.493679851,21.58,18.23111893,4.36,5.935712482,0 Djibouti,DJI,988000,41.36842105,77.777,4.527579378,3082.543099,13.5,0.520915766,5.51,79.51501977,6.49,0.803830216,0 +Dominica,DMA,71625,,,,,,,,,,, Dominican Republic,DOM,10847910,219.9785759,81.074,7.082817476,8050.631602,27.6,0.753248307,11.52,25.29051513,5.02,2.11899467,0 Ecuador,ECU,17643054,68.78868175,63.821,7.157289727,6344.871979,19.9,0.693830173,5.4,36.41377435,4.41,2.753266363,53033 Egypt,EGY,102334404,98.87346929,42.704,5.229779346,2549.132252,32,0.543731123,13.88,33.5519322,12.12,2.232732005,104473 El Salvador,SLV,6486205,309.8814672,72.023,8.287090005,4058.252439,24.6,0.813825469,5.34,43.20066203,6.39,0.998428944,27601 +Eritrea,ERI,6081196,,,,,,,,,,, Estonia,EST,1326535,30.38610536,68.88,19.62635685,23266.34627,21.2,2.20173988,14.12,9.192635012,2.96,14.84881917,790 Eswatini,SWZ,1160164,66.05761628,23.799,4.014587941,4145.974112,16.5,0.551662124,4.93,97.87967456,5.6,1.098404505,0 +Ethiopia,ETH,109224414,,,,,,,,,,, +Faeroe Islands,FRO,52110,,,,,,,,,,, +Fiji,FJI,926276,,,,,,,,,,, Finland,FIN,5540720,18.15685565,85.382,21.72078755,50152.34014,22.2,2.353037036,10.24,5.057654295,2.05,8.660721243,125804 France,FRA,65273511,122.3383957,80.444,20.03462475,41463.64402,21.6,2.091774092,13.64,10.98052703,3.28,4.573181595,597412 +Gabon,GAB,2119275,,,,,,,,,,, Gambia,GMB,2416668,225.3065217,61.27,2.589980939,716.118489,10.3,0.331867104,4.9,112.0379285,9.39,0.253640982,625 +Georgia,GEO,3716858,,,,,,,,,,, Germany,DEU,83783942,237.3709698,77.312,21.461962,47603.02763,22.3,2.398478606,14.94,11.42272633,4.42,8.889370395,922648.6435 Ghana,GHA,31072940,130.8214292,56.06,3.068898007,2202.312164,10.9,0.465036577,2.9,114.9090784,7.48,0.531371738,14373 Greece,GRC,10423054,83.22473235,79.058,21.65527208,20324.25356,24.9,1.85735654,20.29,14.40514319,5.77,6.180337268,144040 +Greenland,GRL,56081,,,,,,,,,,, Guam,GUM,168775,306.9777778,94.78,9.849910719,35712.56214,,1.013283135,9.25,31.30552592,6.09,0,0 Guatemala,GTM,17915568,160.9537794,51.054,4.812072508,4549.010081,21.2,0.574118049,4.23,81.44076798,8.07,1.151000836,3205 +Guinea,GIN,12414293,,,,,,,,,,, Guyana,GUY,786552,3.957348235,26.606,6.450271243,4979.002188,20.2,0.67575984,7.41,45.84605489,6.49,2.632392779,1931 +Haiti,HTI,11439646,,,,,,,,,,, Honduras,HND,9904607,85.68703191,57.096,4.690617659,2500.113213,21.4,0.577602037,8.95,15.89088323,6.57,1.057648023,14832 Hong Kong,HKG,7496981,7096.190476,100,16.87504833,48675.61646,,,,,,6.393600526,245146 Hungary,HUN,9660351,107.9066055,71.351,19.15772538,16161.98051,26.4,1.7429261,19.67,5.650862974,6.37,4.265574976,183541 @@ -61,6 +83,7 @@ Iraq,IRQ,40222493,88.53057004,70.473,3.323599663,5834.166211,30.4,0.419400518,7. Ireland,IRL,4937786,70.45298302,63.17,13.8658017,78806.432,25.3,2.354530107,18.48,17.5937044,3.61,7.313939808,935961 Israel,ISR,8655535,410.5268022,92.418,11.97698602,41715.02928,26.1,1.452405519,13.34,17.84839215,5.68,7.863181348,53901 Italy,ITA,60461826,205.4507479,70.438,22.7516796,34483.204,19.9,2.277105536,13.64,7.11485387,4.72,5.270866786,256596 +Jamaica,JAM,2726667,,,,,,,,,,, Japan,JPN,126476461,347.0734584,91.616,27.57636991,39289.95843,4.3,2.985185404,14.58,22.15590461,3.61,9.538706103,999594 Jordan,JOR,10203134,112.1424983,90.979,3.846490398,4241.788782,35.5,0.704068038,12.13,18.11428306,7.08,2.965238308,36127 Kazakhstan,KAZ,18776707,6.769825907,57.428,7.391846271,9812.601137,21,0.82055031,14.92,21.92918198,5.94,14.36316974,83287 @@ -69,8 +92,12 @@ Kosovo,RKS,1845000,,,,,,,,,,, Kuwait,KWT,4270571,232.1722222,100,2.550472384,33994.40657,37.9,0.729435743,13.89,28.69543854,10.75,25.84922928,46450 Kyrgyz Republic,KGZ,6316000,32.92909281,36.351,4.494727976,1281.363661,16.6,0.574531,11.21,14.24027696,8.3,1.646395339,16344.37197 Laos,LAO,7275560,30.59578423,35.004,4.075149964,2542.486528,5.3,0.41110246,11.8,69.69277779,11,0.294364883,10172.84292 +Latvia,LVA,1907675,,,,,,,,,,, Lebanon,LBN,6825445,669.4941349,88.593,7.002368134,8269.787676,32,2.200755247,19.08,13.90359598,8.23,3.843598231,25740 Lesotho,LSO,2142249,69.43781291,28.153,4.901087497,1299.153126,16.6,0.522848771,7.35,120.1029033,6.83,1.207715726,0 +Liberia,LBR,5073296,,,,,,,,,,, +Libya,LBY,6871287,,,,,,,,,,, +Lithuania,LTU,2795334,,,,,,,,,,, Luxembourg,LUX,625978,250.0938272,90.981,14.18315418,116639.888,22.6,2.563117583,15.19,13.35703721,3.79,17.36212137,51824 Macao,MAC,649335,20777.50026,100,10.48420293,87208.53591,,,,,,2.174572354,24411 Madagascar,MDG,27691018,45.13985562,37.191,2.986717126,527.501285,5.3,0.414500993,3.15,117.1435165,10.13,0.130420845,10170 @@ -80,11 +107,14 @@ Mali,MLI,20250833,15.63501586,42.356,2.507229758,899.6599081,8.6,0.393821527,1.4 Mauritania,MRT,4649658,4.272163578,53.672,3.141112158,1188.834588,12.7,0.483407624,2.63,83.33593901,8.27,0.689388119,5636.839472 Mauritius,MUS,1271768,623.3019704,40.793,11.47417307,11238.69039,10.8,0.772651479,10.9,22.7451741,7.31,3.353110472,14940 Mexico,MEX,128932753,64.91462641,80.156,7.223684981,9673.443674,28.9,0.894715491,7.25,19.82003555,6.76,3.990446149,590727 +Moldova,MDA,2640438,,,,,,,,,,, +Monaco,MCO,38300,,,,,,,,,,, Mongolia,MNG,3278290,2.040608667,68.445,4.083538787,4121.732435,20.6,0.631546393,13.83,21.81236685,7.84,7.088025678,7777 Morocco,MAR,36910560,80.72851893,62.453,7.012904821,3222.200634,26.1,0.62808944,8.33,21.61185389,8.92,1.750794556,80154 Mozambique,MOZ,31255435,37.50853531,35.988,2.890764457,498.9572202,7.2,0.457598238,3.82,95.97107882,5.64,0.320578017,10479 Myanmar,MMR,54409800,82.23861548,30.579,5.784641933,1325.952924,5.8,0.586053381,16.61,59.3317367,11.07,0.41375859,64397.96449 Namibia,NAM,2540905,2.973745582,50.032,3.636031681,5931.453886,17.2,0.543290319,6.76,74.07970846,6.06,1.651692817,14543 +Nepal,NPL,28095714,,,,,,,,,,, Netherlands,NLD,17134872,511.4579104,91.49,19.19619263,53024.05921,20.4,2.818549509,20.25,16.39636629,4.42,9.920138075,325942.296 New Zealand,NZL,4822233,18.55417569,86.538,15.65242453,41945.33167,30.8,3.211580864,12.89,8.137243446,2.13,7.686575825,187851 Nicaragua,NIC,6624554,53.72704836,58.522,5.247497448,2028.894755,23.7,0.612097552,6.36,16.28074206,6.51,0.791576323,0 @@ -93,6 +123,7 @@ Nigeria,NGA,206139589,215.064989,50.344,2.747377005,2028.18197,8.9,0.573096291,0 Norway,NOR,5421241,14.55491954,82.248,17.04922216,81697.24598,23.1,3.149720455,14.26,15.27950426,2.79,9.270945132,0 Oman,OMN,5106626,15.6041454,84.539,2.392786956,16415.15728,27,0.645021569,8.23,26.64407744,8.07,15.18879511,76672 Pakistan,PAK,220892340,275.2893187,36.666,4.312774311,1482.403063,8.6,0.597205025,9.08,35.0810628,9.04,0.851472804,48542 +Palestine,PSE,4550000,,,,,,,,,,, Panama,PAN,4314767,56.18607748,67.709,8.10473108,15575.07255,22.7,0.982381717,8.12,22.63322706,4.5,2.255854757,119148 Papua New Guinea,PNG,8947024,19.00436338,13.169,3.445269251,2730.274799,21.3,0.596347202,12.45,113.2314881,11.98,0.795074226,41545.17078 Paraguay,PRY,7132538,17.50835892,61.585,6.430215009,5821.814323,20.3,0.682727349,13.12,26.54566891,6.46,0.864029477,4215 @@ -105,36 +136,48 @@ Qatar,QAT,2881053,239.5931955,99.135,1.370070339,68793.78444,35.1,0.998635471,7. Romania,ROU,19237691,84.63984701,53.998,18.33870139,12301.1864,22.5,1.633056952,15.51,21.69072367,5.79,3.516153691,55637 Russia,RUS,145934462,8.822079555,74.433,14.67470832,11288.87845,23.1,1.754436206,16.41,17.73043811,5.42,11.85752777,885094 Rwanda,RWA,12952218,498.6598703,17.211,2.938196078,772.9444599,5.8,0.441838617,5.56,93.22594955,8.1,0.075764224,18439 +San Marino,SMR,34232,,,,,,,,,,, Saudi Arabia,SAU,34813871,15.67665431,83.844,3.314088023,23338.96346,35.4,0.908721275,5.92,28.64011854,9.23,19.44066648,254063.7469 +Senegal,SEN,15854323,,,,,,,,,,, Serbia,SRB,8737371,79.83174022,56.092,18.34579269,7246.731656,21.5,1.761549381,19.99,10.30197527,8.01,5.282521917,28160 Seychelles,SYC,98347,210.3521739,56.691,7.594545604,16433.93529,14,1.287976966,11.54,79.01521626,5.43,5.418677963,18671 Sierra Leone,SLE,7976983,105.987171,42.055,2.966556211,533.9911843,8.7,0.407415789,3.42,138.2487198,8.57,0.186560088,0 Singapore,SGP,5850342,7952.998418,100,11.46338019,64581.94402,6.1,2.174969716,12.1,41.84535578,6.64,10.30633191,208787 Slovak Republic,SVK,5459642,113.2905782,53.726,15.6292465,19442.70576,20.5,2.11595951,15.61,20.22993263,5.96,5.661581328,0 Slovenia,SVN,2078938,102.6398595,54.541,19.60687965,26123.97387,20.2,2.073678982,14.89,12.24346427,4.6,6.213686845,18444 +Solomon Islands,SLB,652857,,,,,,,,,,, +Somalia,SOM,15893219,,,,,,,,,,, South Africa,ZAF,59308690,47.63011978,66.355,5.318005005,6374.028196,28.3,0.556269555,6.3,62.16391298,4.96,8.979062348,180317.186 South Korea,KOR,51269185,529.6521036,81.459,14.41855583,31362.75147,4.7,2.210582796,16.04,17.67501381,5.79,11.57034541,526374 South Sudan,SSD,11193725,,19.615,3.39848288,1119.651332,,0.448355243,2.45,140.1842424,9.33,0.141748232,0 Spain,ESP,46754778,,,,,23.8,2.119853473,15.36,10.16887927,4.11,, Sri Lanka,LKA,21413249,345.558922,18.476,10.47321975,4102.48135,5.2,0.537432573,10.07,24.19874754,6.67,0.885247473,34425 Sudan,SDN,43849260,,34.642,3.581160528,977.2736357,,0.423723785,1.37,30.39750247,8.65,0.299732598,3170 +Suriname,SUR,575990,,,,,,,,,,, Sweden,SWE,10099265,25.00104343,87.431,20.09552493,54608.36025,20.6,2.433869445,14.35,11.20746289,2.45,4.478182178,0 Switzerland,CHE,8654622,,,,,19.5,2.110949955,14.98,7.886846735,3.58,, Syria,SYR,17500658,92.06710777,54.162,4.503124421,2032.623489,27.8,0.572073777,8.77,16.78564351,6.65,1.640539063,167.7535302 Taiwan,TWN,23816775,,,,,,2.715254451,15.45,33.59103582,7.08,, +Tajikistan,TJK,9537645,,,,,,,,,,, Tanzania,TZA,59734218,63.5790788,33.776,2.601299178,1060.994615,8.4,0.448789036,4.43,94.56126331,7.9,0.231426985,51577 Thailand,THA,69799978,135.8972068,49.949,11.90089314,7273.563207,10,0.629747052,12.03,40.42401088,6.55,4.620377146,475886.496 +Timor-Leste,TLS,1183643,,,,,,,,,,, +Togo,TGO,8608444,,,,,,,,,,, Trinidad and Tobago,TTO,1399488,270.9274854,53.184,10.73495352,17129.91309,18.6,0.964836513,8.86,20.19768675,7.94,33.96641445,32770 Tunisia,TUN,11818619,74.44132338,68.945,8.315679079,3447.50791,26.9,0.731108244,16.2,16.03659247,8.68,2.605932406,41446 Turkey,TUR,84339067,106.9601289,75.143,8.48321294,9370.176344,32.1,0.95246871,19.16,13.36273825,9.94,4.479773496,772925.545 +Turkmenistan,TKM,6031187,,,,,,,,,,, Uganda,UGA,45741007,213.0617345,23.774,1.94098692,642.7767443,5.3,0.52120401,2.39,78.81358709,5.85,0.141664527,4832 Ukraine,UKR,43733762,77.02966735,69.352,16.43468644,3095.173581,24.1,1.602203378,16.97,15.16317002,7.27,5.020747064,66712 United Arab Emirates,ARE,9890402,,,,,31.7,0.930793187,9.51,45.83255643,7.85,, United Kingdom,GBR,67886011,,,,,27.8,2.791155517,15.27,24.11005138,4.08,, United States,USA,331002651,35.76608858,82.256,15.80765406,62794.58565,36.2,5.42440701,14.77,15.88386085,3.76,16.50283724,9879630 +United States Virgin Islands,VIR,106405,,,,,,,,,,, Uruguay,URY,3473730,19.70802765,95.334,14.81451953,17277.97011,27.9,1.489823658,14.13,24.11213092,3.56,1.984240835,0 Uzbekistan,UZB,33469203,77.46920545,50.478,4.419137878,1532.371677,16.6,0.587011732,10.19,35.11694115,9.67,3.420722746,21514 +Vanuatu,VUT,307815,,,,,,,,,,, Venezuela,VEN,28435940,32.73079191,88.208,7.26698662,16054.49051,25.6,1.134870218,9.23,19.74587517,5.33,6.164731034,24986 Vietnam,VNM,97338579,308.1252459,35.919,7.274978298,2566.59695,2.1,0.603065672,14.24,22.97805645,8.28,1.819893977,283787 +Yemen,YEM,28498683,,,,,,,,,,, Zambia,ZMB,18383955,23.3414789,43.521,2.099678026,1539.900158,8.1,0.474567911,3.68,93.85563273,6,0.292412222,734 Zimbabwe,ZWE,14862924,37.32459093,32.209,2.939523653,2146.996385,15.5,0.566520697,7.41,137.6330045,7.67,0.884721294,3444 \ No newline at end of file diff --git a/covid_xprize/examples/predictors/lstm/data/data_sources.txt b/covid_xprize/examples/predictors/lstm/data/data_sources.txt new file mode 100644 index 00000000..d31f6943 --- /dev/null +++ b/covid_xprize/examples/predictors/lstm/data/data_sources.txt @@ -0,0 +1,7 @@ +# This file shows where datasets were obtained from. +# Format is: +# dataset_name: source +Additional_Context_Data_Global.csv: unknown, latest update: wikipedia +US_states_populations.csv: unknown +brazil_populations.csv: https://en.wikipedia.org/wiki/States_of_Brazil +uk_populations.csv: Google, queries like "population england" on Sep 17, 2020