Skip to content

Commit

Permalink
fixed merge
Browse files Browse the repository at this point in the history
  • Loading branch information
reneehlozek committed Sep 11, 2019
2 parents 0d5c63b + 86c5141 commit b0109fa
Show file tree
Hide file tree
Showing 31 changed files with 2,640 additions and 1,192 deletions.
220 changes: 220 additions & 0 deletions Plot_Table3.ipynb
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,220 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import numpy as np\n",
"import pandas as pd\n",
"import pickle\n",
"\n",
"import matplotlib as mpl\n",
"# print(mpl.rcParams.items)\n",
"mpl.use('Agg')\n",
"mpl.rcParams['text.usetex'] = False\n",
"mpl.rcParams['mathtext.rm'] = 'serif'\n",
"mpl.rcParams['font.family'] = 'serif'\n",
"mpl.rcParams['font.serif'] = ['Times New Roman']\n",
"# mpl.rcParams['font.family'] = ['Times New Roman']\n",
"mpl.rcParams['axes.titlesize'] = 25\n",
"mpl.rcParams['axes.labelsize'] = 20\n",
"mpl.rcParams['xtick.labelsize'] = 15\n",
"mpl.rcParams['ytick.labelsize'] = 15\n",
"mpl.rcParams['savefig.dpi'] = 250\n",
"mpl.rcParams['figure.dpi'] = 250\n",
"mpl.rcParams['savefig.format'] = 'pdf'\n",
"mpl.rcParams['savefig.bbox'] = 'tight'\n",
"import matplotlib.pyplot as plt\n",
"%matplotlib inline"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"def truncate_colormap(cmap, minval=0.0, maxval=1.0, n=100):\n",
" new_cmap = mpl.colors.LinearSegmentedColormap.from_list(\n",
" 'trunc({n},{a:.2f},{b:.2f})'.format(n=cmap.name, a=minval, b=maxval),\n",
" cmap(np.linspace(minval, maxval, n)))\n",
" return new_cmap\n",
"\n",
"cmap = plt.get_cmap('hot_r')\n",
"fave_cmap = truncate_colormap(cmap, 0.35, 1.0)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"metric_dictionary = {'TBDT':{'FoM': 1, 'LogLoss': 1, 'Brier': 1},\n",
" 'TKNN':{'FoM': 7, 'LogLoss': 6, 'Brier': 7},\n",
" 'TNB':{'FoM': 8, 'LogLoss': 9, 'Brier': 8},\n",
" 'TNN':{'FoM': 5, 'LogLoss': 3, 'Brier': 3},\n",
" 'TSVM':{'FoM': 3, 'LogLoss': 2, 'Brier': 2},\n",
" 'WBDT':{'FoM': 2, 'LogLoss': 5, 'Brier': 4},\n",
" 'WKNN':{'FoM': 9, 'LogLoss': 8, 'Brier': 9},\n",
" 'WNB':{'FoM': 10, 'LogLoss': 10, 'Brier': 10},\n",
" 'WNN':{'FoM': 6, 'LogLoss': 7, 'Brier': 6},\n",
" 'WSVM':{'FoM': 4, 'LogLoss': 4, 'Brier': 5},\n",
" }"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"metric_dictionary['TBDT']"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"symbols = {'TBDT':'o',\n",
" 'TKNN':'d',\n",
" 'TNB':'s',\n",
" 'TNN':'*',\n",
" 'TSVM':'^',\n",
" 'WBDT':'o',\n",
" 'WKNN':'d',\n",
" 'WNB':'s',\n",
" 'WNN':'*',\n",
" 'WSVM':'^',\n",
" }\n",
"\n",
"colors = {'TBDT':fave_cmap(0.05),\n",
" 'TKNN':fave_cmap(0.3),\n",
" 'TNB':fave_cmap(0.55),\n",
" 'TNN':fave_cmap(0.8),\n",
" 'TSVM':fave_cmap(1.0),\n",
" 'WBDT':fave_cmap(0.05),\n",
" 'WKNN':fave_cmap(0.3),\n",
" 'WNB':fave_cmap(0.55),\n",
" 'WNN':fave_cmap(0.75),\n",
" 'WSVM':fave_cmap(1.0),\n",
" }\n",
"\n",
"\n",
"plt.figure()\n",
"for key, value in metric_dictionary.items():\n",
" val = []\n",
" for k, v in value.items():\n",
" val.append(v)\n",
" if 'W' in key:\n",
" plt.plot(val, label=key, marker=symbols[key], ls='--', color=colors[key])\n",
" else:\n",
" plt.plot(val, label=key, marker=symbols[key], color=colors[key])\n",
"\n",
"plt.legend(loc='center left', bbox_to_anchor=(1, 0.5), prop={'size': 12})\n",
"plt.xticks([0, 1, 2], ['FoM', 'LogLoss', 'Brier'])\n",
"plt.yticks(np.arange(1, 11))\n",
"plt.ylabel('Rank')\n",
"\n",
"#plt.savefig('Tables3_option1.pdf')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"\n",
"colors = {'TBDT':fave_cmap(0.05),\n",
" 'TKNN':fave_cmap(0.2375),\n",
" 'TNB':fave_cmap(0.54),\n",
" 'TNN':fave_cmap(0.712499999),\n",
" 'TSVM':fave_cmap(1.0),\n",
" 'WBDT':fave_cmap(0.05),\n",
" 'WKNN':fave_cmap(0.2375),\n",
" 'WNB':fave_cmap(0.54),\n",
" 'WNN':fave_cmap(0.712499999),\n",
" 'WSVM':fave_cmap(1.0),\n",
" }\n",
"\n",
"plt.figure()\n",
"for key, value in metric_dictionary.items():\n",
" val = []\n",
" for k, v in value.items():\n",
" val.append(v)\n",
" if 'W' in key:\n",
" plt.plot(val, label=key, marker=symbols[key], ls='--', color=colors[key], lw=2, ms=7, alpha=0.3)\n",
" else:\n",
" plt.plot(val, label=key, marker=symbols[key], color=colors[key], lw=2, ms=7)\n",
"\n",
"plt.legend(loc='center left', bbox_to_anchor=(1, 0.5), prop={'size': 12})\n",
"plt.xticks([0, 1, 2], ['FoM', 'LogLoss', 'Brier'])\n",
"plt.yticks(np.arange(1, 11))\n",
"plt.ylabel('Rank')\n",
"plt.gca().invert_yaxis()\n",
"\n",
"#plt.savefig('Tables3_option4.pdf')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"plt.figure()\n",
"\n",
"fom = []\n",
"ll = []\n",
"brier = []\n",
"\n",
"for key, value in metric_dictionary.items():\n",
" fom.append(value['FoM'])\n",
" ll.append(value['LogLoss'])\n",
" brier.append(value['Brier'])\n",
"\n",
"plt.plot(fom, label='FoM', marker='o')\n",
"plt.plot(ll, label='LogLoss', marker='D', alpha = 0.5)\n",
"plt.plot(brier, label='Brier', marker='s', alpha=0.23)\n",
"\n",
"plt.legend(loc='center left', bbox_to_anchor=(1, 0.5), prop={'size': 12})\n",
"plt.xticks(np.arange(0, 10), list(metric_dictionary.keys()), rotation=45)\n",
"plt.ylabel('Rank')\n",
"plt.savefig('Tables3_option2.pdf')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 2",
"language": "python",
"name": "python2"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.15"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
42 changes: 33 additions & 9 deletions metrics_evaluation.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
"\n",
"import numpy as np\n",
"import pandas as pd\n",
"from pycm import ConfusionMatrix\n",
"\n",
"import proclam\n",
"from proclam import *"
Expand Down Expand Up @@ -186,7 +187,7 @@
" data_info_dict['dirname'] = dirname + data_info_dict['label'] + '/'\n",
" data_info_dict['classifications'] = ['%s/predicted_prob_%s.csv'%(name, name) for name in names]\n",
" data_info_dict['truth_tables'] = ['%s/truth_table_%s.csv'%(name, name) for name in names]\n",
" print(data_info_dict)\n",
"# print(data_info_dict)\n",
" return data_info_dict"
]
},
Expand Down Expand Up @@ -249,6 +250,13 @@
"scrolled": true
},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"def read_class_pairs(pair, dataset, cc):#loc='', title=''):\n",
" loc=dataset['dirname']\n",
Expand Down Expand Up @@ -362,12 +370,21 @@
" for cc, pair in enumerate(dataset['class_pairs']):\n",
" print(pair)\n",
" probm, truthv = read_class_pairs(pair, dataset, cc)#loc=dataset['dirname'], title=dataset['label']+' '+dataset['names'][cc])\n",
" for count, metric in enumerate(metricslist):\n",
" D = getattr(proclam.metrics, metric)()\n",
" hm = D.evaluate(probm, truthv)\n",
" data[count][cc] = hm\n",
" dataset['results'] = data\n",
" metric_plot(dataset, metricslist, markerlist, colors)"
"# plot_cm(probm, truthv, str(cc), loc='./sandbox/')\n",
" det = proclam.metrics.util.prob_to_det(probm)\n",
" cm = proclam.metrics.util.prob_to_cm(probm, truthv, per_class_norm=False, vb=False)\n",
" rates = proclam.metrics.util.cm_to_rate(cm, vb=True)\n",
"# print(rates)\n",
" compare = ConfusionMatrix(truthv, det)\n",
" printout = proclam.metrics.util.RateMatrix(compare.TPR, compare.FPR, compare.FNR, compare.TNR)\n",
" print('for comparison: ' + str(printout))\n",
"# for count, metric in enumerate(metricslist):\n",
"# D = getattr(proclam.metrics, metric)()\n",
"# hm = D.evaluate(probm, truthv)\n",
"# data[count][cc] = hm\n",
"# dataset['results'] = data\n",
" \n",
"# metric_plot(dataset, metricslist, markerlist, colors)"
]
},
{
Expand All @@ -379,6 +396,13 @@
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
Expand All @@ -390,9 +414,9 @@
"metadata": {
"anaconda-cloud": {},
"kernelspec": {
"display_name": "Python 3",
"display_name": "proclam (Python 3)",
"language": "python",
"name": "python3"
"name": "proclam_3"
},
"language_info": {
"codemirror_mode": {
Expand Down
5 changes: 3 additions & 2 deletions paper/authors.csv
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
Lastname,Firstname,Authorname,AuthorType,Affiliation,Contribution,Email
Malz,Alex,A.I.~Malz,Contact,"German Centre of Cosmological Lensing, Ruhr-Universitaet Bochum, Universitaetsstra{\ss}e 150, 44801 Bochum, Germany","conceptualization, data curation, formal analysis, investigation, methodology, project administration, software, supervision, validation, visualization, writing - editing, writing - original draft",[email protected]
Malz,Alex,A.I.~Malz,Contact,"Center for Cosmology and Particle Physics, New York University, 726 Broadway, New York, NY 10004, USA","conceptualization, data curation, formal analysis, investigation, methodology, project administration, software, supervision, validation, visualization, writing - editing, writing - original draft",[email protected]
Malz,Alex,A.I.~Malz,Contact,"Department of Physics, New York University, 726 Broadway, New York, NY 10004, USA","conceptualization, data curation, formal analysis, investigation, methodology, project administration, software, supervision, validation, visualization, writing - editing, writing - original draft",[email protected]
Hlo\v{z}ek,Ren\'ee,R.~Hlo\v{z}ek,Contributor,"Department of Astronomy and Astrophysics, University of Toronto, 50 St. George St., Toronto, ON M5S 3H4, Canada","data curation, formal analysis, funding acquisition, investigation, project administration, software, supervision, validation, visualization, writing - editing, writing - original draft",[email protected]
Hlo\v{z}ek,Ren\'ee,R.~Hlo\v{z}ek,Contributor,"Dunlap Institute for Astronomy and Astrophysics, University of Toronto, 50 St. George St., Toronto, ON M5S 3H4, Canada","data curation, formal analysis, funding acquisition, investigation, project administration, software, supervision, validation, visualization, writing - editing, writing - original draft",[email protected]
Allam,Tarek,T.~Allam Jr,Contributor,"Mullard Space Science Laboratory, Department of Space and Climate Physics, University College London, Holmbury Hill Rd, Dorking RH5 6NT, UK","investigation, software, validation, writing - original draft",[email]
Bahmanyar,Anita,A.~Bahmanyar,Contributor,"Dunlap Institute for Astronomy and Astrophysics, University of Toronto, 50 St. George St., Toronto, ON M5S 3H4, Canada","formal analysis, investigation, methodology, software, writing - editing, writing - original draft",[email]
Biswas,Rahul,R.~Biswas,Contributor,"The Oskar Klein Centre for Cosmoparticle Physics, Stockholm University, AlbaNova, Stockholm, SE-106 91, Sweden","conceptualization, methodology, software, writing - original draft",[email]
Biswas,Rahul,R.~Biswas,Contributor,"The Oskar Klein Centre for Cosmoparticle Physics, Stockholm University, AlbaNova, Stockholm, SE-106 91, Sweden","conceptualization, methodology, software, supervision, writing - editing, writing - original draft",[email]
Dai,Mi,M.~Dai,Contributor,"Rutgers, the State University of New Jersey, 136 Frelinghuysen Road, Piscataway, NJ 08854 USA","writing - editing",[email]
Galbany,Llu\'is,L.~Galbany,Contributor,"University of Pittsburgh, 300 Allen Hall, 3941 O'Hara St, Pittsburgh, PA 15260","writing - editing",[email]
Ishida,Emille,E.E.O.~Ishida,Contributor,"Universit\'e Clermont Auvergne, CNRS/IN2P3, LPC, F-63000 Clermont-Ferrand, France","conceptualization, project administration, supervision, writing - editing",[email]
Expand All @@ -25,5 +26,5 @@ Narayan,Gautham,G.~Narayan,Contributor,"Space Telescope Science Institute, 3700
Peiris,Hiranya,H.~Peiris,Contributor,"The Oskar Klein Centre for Cosmoparticle Physics, Stockholm University, AlbaNova, Stockholm, SE-106 91, Sweden","conceptualization, funding acquisition, supervision",[email]
Peiris,Hiranya,H.~Peiris,Contributor,"Department of Physics and Astronomy, University College London, Gower Street, London, WC1E 6BT, UK","conceptualization, funding acquisition, supervision",[email]
Peters,Christina~M.,C.M.~Peters,Contributor,"Dunlap Institute for Astronomy and Astrophysics, University of Toronto, 50 St. George St., Toronto, ON M5S 3H4, Canada","writing - editing",[email]
Ponder,Kara,K.~Ponder,Contributor,"Berkeley Center for Cosmological Physics, Campbell Hall 341, University of California Berkeley, Berkeley, CA 94720, USA","writing - editing",[email]
Ponder,Kara,K.~Ponder,Contributor,"Berkeley Center for Cosmological Physics, Campbell Hall 341, University of California Berkeley, Berkeley, CA 94720, USA","visualization, writing - editing",[email]
Setzer,Christian,C.N.~Setzer,Contributor,"The Oskar Klein Centre for Cosmoparticle Physics, Stockholm University, AlbaNova, Stockholm, SE-106 91, Sweden","conceptualization, software",[email protected]
Binary file added paper/fig/Tables3_option4.png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file modified paper/fig/all_sim_cm.png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file modified paper/fig/all_snphotcc_cm.png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file modified paper/fig/combined.png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file modified paper/fig/examples.png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading

0 comments on commit b0109fa

Please sign in to comment.