diff --git a/csep/utils/plots.py b/csep/utils/plots.py index 08ef03ee..78ef1f24 100644 --- a/csep/utils/plots.py +++ b/csep/utils/plots.py @@ -3,7 +3,8 @@ # Third-party imports import numpy import string - +import pandas as pandas +from scipy.integrate import cumtrapz import scipy.stats import matplotlib import matplotlib.lines @@ -2131,7 +2132,7 @@ def plot_pvalues_and_intervals(test_results, ax, var=None): return ax -def plot_ROC(forecast, catalog, axes=None, plot_uniform=True, savepdf=True, +def plot_concentration_ROC_diagram(forecast, catalog, linear=True, axes=None, plot_uniform=True, savepdf=True, savepng=True, show=True, plot_args=None): """ @@ -2154,6 +2155,7 @@ def plot_ROC(forecast, catalog, axes=None, plot_uniform=True, savepdf=True, Args: forecast (:class: `csep.forecast.GriddedForecast`): catalog (:class:`AbstractBaseCatalog`): evaluation catalog + linear: (bool): if true, a linear x-axis is used; if false a logarithmic x-axis is used. axes (:class:`matplotlib.pyplot.ax`): Previously defined ax object savepdf (str): output option of pdf file savepng (str): output option of png file @@ -2183,6 +2185,7 @@ def plot_ROC(forecast, catalog, axes=None, plot_uniform=True, savepdf=True, RuntimeError: throws error if Catalog and Forecast do not have the same region Written by Han Bao, UCLA, March 2021. Modified June 2021. + Modified by Emanuele Biondini, University of Bologna, May 2024. """ if not catalog.region == forecast.region: raise RuntimeError( @@ -2200,7 +2203,7 @@ def plot_ROC(forecast, catalog, axes=None, plot_uniform=True, savepdf=True, title_fontsize = plot_args.get('title_fontsize', 18) label_fontsize = plot_args.get('label_fontsize', 14) filename = plot_args.get('filename', 'roc_figure') - title = plot_args.get('title', 'ROC Curve') + title = plot_args.get('title', 'Concentration ROC Curve') # Plot catalog ordered by forecast rates name = forecast.name @@ -2258,7 +2261,406 @@ def plot_ROC(forecast, catalog, axes=None, plot_uniform=True, savepdf=True, ax.set_ylabel("True Positive Rate", fontsize=label_fontsize) ax.set_xlabel('False Positive Rate (Normalized Area)', fontsize=label_fontsize) - ax.set_xscale('log') + if linear==True: + legend_loc=plot_args.get('legend_loc', 'lower right') + elif linear==False: + ax.set_xscale('log') + + ax.legend(loc=legend_loc, shadow=True, fontsize=legend_fontsize) + ax.set_title(title, fontsize=title_fontsize) + + if filename: + if savepdf: + outFile = "{}.pdf".format(filename) + pyplot.savefig(outFile, format='pdf') + if savepng: + outFile = "{}.png".format(filename) + pyplot.savefig(outFile, format='png') + + if show: + pyplot.show() + return ax + + +def plot_ROC_diagram(forecast, catalog, linear=True, axes=None, plot_uniform=True, savepdf=True, savepng=True, show=True, + plot_args=None): + """ + Plot Receiver operating characteristic (ROC) based on forecast and test catalogs using the contingency table. + The ROC is computed following this procedure: + (1) Obtain spatial rates from GriddedForecast and the observed events from the observed catalog. + (2) Rank the rates in descending order (highest rates first). + (3) Sort forecasted rates by ordering found in (2), and normalize rates so their sum is equals unity. + (4) Obtain binned spatial rates from observed catalog + (5) Sort gridded observed rates by ordering found in (2). + (6) Test each ordered and normalized forecasted rate defined in (3) as a threshold value to obtain the + corresponding contingency table. + (7) Define the H (Success rate) and F (False alarm rate) for each threshold soil using the information provided + by the correspondent contingency table defined in (6). + + Note that: + (1) The testing catalog and forecast should have exactly the same time-window (duration) + (2) Forecasts should be defined over the same region + (3) If calling this function multiple times, update the color in plot_args + (4) The user can choose the x-scale (linear or log), see the Args section below + + Args: + forecast (:class: `csep.forecast.GriddedForecast`): + catalog (:class:`AbstractBaseCatalog`): evaluation catalog + linear: (bool): if true, a linear x-axis is used; if false a logarithmic x-axis is used. + axes (:class:`matplotlib.pyplot.ax`): Previously defined ax object + savepdf (str): output option of pdf file + savepng (str): output option of png file + plot_uniform (bool): if true, include uniform forecast on plot + + Optional plotting arguments: + * figsize: (:class:`list`/:class:`tuple`) - default: [9, 8] + * forecast_linestyle: (:class:`str`) - default: '-' + * legend_fontsize: (:class:`float`) Fontsize of the plot title - default: 16 + * legend_loc: (:class:`str`) - default: 'upper left' + * title_fontsize: (:class:`float`) Fontsize of the plot title - default: 18 + * label_fontsize: (:class:`float`) Fontsize of the plot title - default: 14 + * title: (:class:`str`) - default: ROC Curve from contingency table + * filename: (:class:`str`) - default: contingency_roc_figure. + + Returns: + :class:`matplotlib.pyplot.ax` object + + Raises: + TypeError: throws error if CatalogForecast-like object is provided + RuntimeError: throws error if Catalog and Forecast do not have the same region + + Written by Emanuele Biondini, UNIBO, March 2023. + """ + if not catalog.region == forecast.region: + raise RuntimeError( + "catalog region and forecast region must be identical.") + + # Parse plotting arguments + plot_args = plot_args or {} + figsize = plot_args.get('figsize', (9, 8)) + forecast_linestyle = plot_args.get('forecast_linestyle', '-') + legend_fontsize = plot_args.get('legend_fontsize', 16) + legend_loc = plot_args.get('legend_loc', 'upper left') + title_fontsize = plot_args.get('title_fontsize', 16) + label_fontsize = plot_args.get('label_fontsize', 14) + title = plot_args.get('title', 'ROC Curve from contingency table') + filename = plot_args.get('filename', 'contingency_roc_figure') + + # Initialize figure + if axes is not None: + ax = axes + else: + fig, ax = pyplot.subplots(figsize=figsize) + + name = forecast.name + if not name: + name = '' + else: + name = f'{name}' + + forecast_label = plot_args.get('forecast_label', f'{name}') + observed_label = plot_args.get('observed_label', f'{name}') + + # Obtain forecast rates (or counts) and observed catalog aggregated in spatial cells + rate = forecast.spatial_counts() + obs_counts = catalog.spatial_counts() + # Define the threshold to be analysed to compile the contingency table and draw the ROC curve + + # Get index of rates (descending sort) + I = numpy.argsort(rate) + I = numpy.flip(I) + + # Order forecast and cells rates by highest rate cells first and normalize the rates. + thresholds = (rate[I]) / numpy.sum(rate) + obs_counts = obs_counts[I] + + Table_ROC = pandas.DataFrame({ + 'Threshold': [], + 'Successful_bins': [], + 'Obs_active_bins': [], + 'H': [], + 'F': [] + + }) + + #Each forecasted and normalized rate are tested as a threshold value to define the contingency table. + for threshold in thresholds: + threshold = float(threshold) + + binary_forecast = numpy.where(thresholds >= threshold, 1, 0) + forecastedYes_observedYes = obs_counts[(binary_forecast == 1) & (obs_counts > 0)] + forecastedYes_observedNo=obs_counts[(binary_forecast == 1) & (obs_counts == 0)] + forecastedNo_observedYes=obs_counts[(binary_forecast == 0) & (obs_counts > 0)] + forecastedNo_observedNo = obs_counts[(binary_forecast == 0) & (obs_counts == 0)] + # Creating the DataFrame for the contingency table + data = { + "Observed": [len(forecastedYes_observedYes), len(forecastedNo_observedYes)], + "Not Observed": [len(forecastedYes_observedNo), len(forecastedNo_observedNo)] + } + index = ["Forecasted", "Not Forecasted"] + contingency_df = pandas.DataFrame(data, index=index) + + H = contingency_df.loc['Forecasted', 'Observed'] / ( + contingency_df.loc['Forecasted', 'Observed'] + contingency_df.loc['Not Forecasted', 'Observed']) + F = contingency_df.loc['Forecasted', 'Not Observed'] / ( + contingency_df.loc['Forecasted', 'Not Observed'] + contingency_df.loc[ + 'Not Forecasted', 'Not Observed']) + + threshold_row = { + 'Threshold': threshold, + 'Successful_bins': contingency_df.loc['Forecasted', 'Observed'], + 'Obs_active_bins': contingency_df['Observed'].sum(), + 'H': H, + 'F': F + + } + threshold_row_df = pandas.DataFrame([threshold_row]) + + # Concatena threshold_row_df a Table_molchan + Table_ROC = pandas.concat([Table_ROC, threshold_row_df], ignore_index=True) + + # to start the trajecroy in the poin (0,0) + first_row = pandas.DataFrame({'H': [0], 'F': [0]}) + Table_ROC = pandas.concat([first_row, Table_ROC], ignore_index=True) + + # Plot the ROC curve + ax.plot((Table_ROC['F']), (Table_ROC['H']), + label=forecast_label, + color='black', + linestyle=forecast_linestyle) + + # Plot uniform forecast + if plot_uniform: + x_uniform = numpy.arange(0, 1.001, 0.001) + y_uniform = numpy.arange(0, 1.001, 0.001) + ax.plot(x_uniform, y_uniform, linestyle='--', color='gray', label='SUP') + # Plotting arguments + ax.set_ylabel("Hit Rate", fontsize=label_fontsize) + ax.set_xlabel('Fraction of false alarms', fontsize=label_fontsize) + + if linear==True: + pass + elif linear==False: + ax.set_xscale('log') + + ax.set_yscale('linear') + ax.tick_params(axis='x', labelsize=label_fontsize) + ax.tick_params(axis='y', labelsize=label_fontsize) + ax.legend(loc=legend_loc, shadow=True, fontsize=legend_fontsize) + ax.set_title(title, fontsize=title_fontsize) + + + if filename: + if savepdf: + outFile = "{}.pdf".format(filename) + pyplot.savefig(outFile, format='pdf') + if savepng: + outFile = "{}.png".format(filename) + pyplot.savefig(outFile,format='png') + + if show: + pyplot.show() + return ax + + +def plot_Molchan_diagram(forecast, catalog, linear=True, axes=None, plot_uniform=True, savepdf=True, savepng=True, + show=True, + plot_args=None): + """ + Plot the Molchan Diagram based on forecast and test catalogs using the contingency table. + The Area Skill score and its error are shown in the legend + + The Molchan diagram is computed following this procedure: + (1) Obtain spatial rates from GriddedForecast and the observed events from the observed catalog. + (2) Rank the rates in descending order (highest rates first). + (3) Sort forecasted rates by ordering found in (2), and normalize rates so their sum is equals unity. + (4) Obtain binned spatial rates from observed catalog + (5) Sort gridded observed rates by ordering found in (2). + (6) Test each ordered and normalized forecasted rate defined in (3) as a threshold value to obtain the + corresponding contingency table. + (7) Define the "nu" (Miss rate) and "tau" (Fraction of spatial alarmed cells) for each threshold soil using the + information provided by the correspondent contingency table defined in (6). + + + Note that: + (1) The testing catalog and forecast should have exactly the same time-window (duration) + (2) Forecasts should be defined over the same region + (3) If calling this function multiple times, update the color in plot_args + (4) The user can choose the x-scale (linear or log), see the Args section below + + Args: + forecast (:class: `csep.forecast.GriddedForecast`): + catalog (:class:`AbstractBaseCatalog`): evaluation catalog + linear: (bool): if true, a linear x-axis is used; if false a logarithmic x-axis is used. + axes (:class:`matplotlib.pyplot.ax`): Previously defined ax object + savepdf (str): output option of pdf file + savepng (str): output option of png file + plot_uniform (bool): if true, include uniform forecast on plot + + Optional plotting arguments: + * figsize: (:class:`list`/:class:`tuple`) - default: [9, 8] + * forecast_linestyle: (:class:`str`) - default: '-' + * legend_fontsize: (:class:`float`) Fontsize of the plot title - default: 16 + * legend_loc: (:class:`str`) - default: 'lower left' + * title_fontsize: (:class:`float`) Fontsize of the plot title - default: 18 + * label_fontsize: (:class:`float`) Fontsize of the plot title - default: 14 + * title: (:class:`str`) - default: 'Molchan diagram' + * filename: (:class:`str`) - default: molchan_diagram. + + Returns: + :class:`matplotlib.pyplot.ax` object + + Raises: + TypeError: throws error if CatalogForecast-like object is provided + RuntimeError: throws error if Catalog and Forecast do not have the same region + + Written by Emanuele Biondini, UNIBO, March 2023. + """ + if not catalog.region == forecast.region: + raise RuntimeError( + "catalog region and forecast region must be identical.") + + # Parse plotting arguments + plot_args = plot_args or {} + figsize = plot_args.get('figsize', (9, 8)) + forecast_linestyle = plot_args.get('forecast_linestyle', '-') + legend_fontsize = plot_args.get('legend_fontsize', 16) + legend_loc = plot_args.get('legend_loc', 'lower left') + title_fontsize = plot_args.get('title_fontsize', 16) + label_fontsize = plot_args.get('label_fontsize', 14) + title = plot_args.get('title', '') + filename = plot_args.get('filename', 'molchan_figure') + + # Initialize figure + if axes is not None: + ax = axes + else: + fig, ax = pyplot.subplots(figsize=figsize) + + name = forecast.name + if not name: + name = '' + else: + name = f'{name}' + + forecast_label = plot_args.get('forecast_label', f'{name}') + observed_label = plot_args.get('observed_label', f'{name}') + + # Obtain forecast rates (or counts) and observed catalog aggregated in spatial cells + rate = forecast.spatial_counts() + obs_counts = catalog.spatial_counts() + + # Define the threshold to be analysed tp draw the Molchan diagram + + # Get index of rates (descending sort) + I = numpy.argsort(rate) + I = numpy.flip(I) + + # Order forecast and cells rates by highest rate cells first + thresholds = (rate[I]) / numpy.sum(rate) + obs_counts = obs_counts[I] + + Table_molchan = pandas.DataFrame({ + 'Threshold': [], + 'Successful_bins': [], + 'Obs_active_bins': [], + 'tau': [], + 'nu': [], + 'R_score': [] + }) + + # Each forecasted and normalized rate are tested as a threshold value to define the contingency table. + for threshold in thresholds: + threshold = float(threshold) + + binary_forecast = numpy.where(thresholds >= threshold, 1, 0) + forecastedYes_observedYes = obs_counts[(binary_forecast == 1) & (obs_counts > 0)] + forecastedYes_observedNo = obs_counts[(binary_forecast == 1) & (obs_counts == 0)] + forecastedNo_observedYes = obs_counts[(binary_forecast == 0) & (obs_counts > 0)] + forecastedNo_observedNo = obs_counts[(binary_forecast == 0) & (obs_counts == 0)] + # Creating the DataFrame for the contingency table + data = { + "Observed": [len(forecastedYes_observedYes), len(forecastedNo_observedYes)], + "Not Observed": [len(forecastedYes_observedNo), len(forecastedNo_observedNo)] + } + index = ["Forecasted", "Not Forecasted"] + contingency_df = pandas.DataFrame(data, index=index) + nu = contingency_df.loc['Not Forecasted', 'Observed'] / contingency_df['Observed'].sum() + tau = contingency_df.loc['Forecasted'].sum() / (contingency_df.loc['Forecasted'].sum() + + contingency_df.loc['Not Forecasted'].sum()) + R_score = (contingency_df.loc['Forecasted', 'Observed'] / contingency_df['Observed'].sum()) - \ + (contingency_df.loc['Forecasted', 'Not Observed'] / contingency_df['Not Observed'].sum()) + + threshold_row = { + 'Threshold': threshold, + 'Successful_bins': contingency_df.loc['Forecasted', 'Observed'], + 'Obs_active_bins': contingency_df['Observed'].sum(), + 'tau': tau, + 'nu': nu, + 'R_score': R_score, + + } + threshold_row_df = pandas.DataFrame([threshold_row]) + + Table_molchan = pandas.concat([Table_molchan, threshold_row_df], ignore_index=True) + + bottom_row = {'Threshold': 'Full alarms', 'tau': 1, 'nu': 0, 'Obs_active_bins': contingency_df['Observed'].sum()} + top_row = {'Threshold': 'No alarms', 'tau': 0, 'nu': 1, 'Obs_active_bins': contingency_df['Observed'].sum()} + + Table_molchan = pandas.concat([pandas.DataFrame([top_row]), Table_molchan], ignore_index=True) + + Table_molchan = pandas.concat([Table_molchan, pandas.DataFrame([bottom_row])], ignore_index=True) + + # Computation of Area Skill score (ASS) + Tab_as_score = pandas.DataFrame() + + Tab_as_score['Threshold'] = Table_molchan['Threshold'] + Tab_as_score['tau'] = Table_molchan['tau'] + Tab_as_score['nu'] = Table_molchan['nu'] + + ONE = numpy.ones(len(Tab_as_score)) + Tab_as_score['CUM_BAND'] = cumtrapz(ONE, Tab_as_score['tau'], initial=0) - cumtrapz(Tab_as_score['nu'], + Tab_as_score['tau'], + initial=0) + Tab_as_score['AS_score'] = numpy.divide(Tab_as_score['CUM_BAND'], + cumtrapz(ONE, Tab_as_score['tau'], initial=0) + 1e-10) + Tab_as_score.loc[Tab_as_score.index[-1], 'AS_score'] = max(0.5, Tab_as_score['AS_score'].iloc[-1]) + ASscore = numpy.round(Tab_as_score.loc[Tab_as_score.index[-1], 'AS_score'], 2) + + bin = 0.01 + import math + devstd = numpy.sqrt(1 / (12 * Table_molchan['Obs_active_bins'].iloc[0])) + devstd = devstd * bin ** -1 + devstd = math.ceil(devstd + 0.5) + devstd = devstd / bin ** -1 + Tab_as_score['st_dev'] = devstd + Tab_as_score['st_dev'] = devstd + dev_std = numpy.round(devstd, 2) + + # Plot the Molchan trajectory + + ax.plot(Table_molchan['tau'], Table_molchan['nu'], + label=f"{forecast_label}, ASS={ASscore}±{dev_std} ", + color='black', + linestyle=forecast_linestyle) + + # Plot uniform forecast + if plot_uniform: + x_uniform = numpy.arange(0, 1.001, 0.001) + y_uniform = numpy.arange(1.00, -0.001, -0.001) + ax.plot(x_uniform, y_uniform, linestyle='--', color='gray', label='SUP' + ' ASS=0.50') + + # Plotting arguments + ax.set_ylabel("Miss Rate", fontsize=label_fontsize) + ax.set_xlabel('Fraction of area occupied by alarms', fontsize=label_fontsize) + + if linear == True: + legend_loc = plot_args.get('legend_loc', 'upper right') + elif linear == False: + ax.set_xscale('log') + + ax.tick_params(axis='x', labelsize=label_fontsize) + ax.tick_params(axis='y', labelsize=label_fontsize) ax.legend(loc=legend_loc, shadow=True, fontsize=legend_fontsize) ax.set_title(title, fontsize=title_fontsize) diff --git a/examples/tutorials/contingency_roc_figure.pdf b/examples/tutorials/contingency_roc_figure.pdf new file mode 100644 index 00000000..b17964e3 Binary files /dev/null and b/examples/tutorials/contingency_roc_figure.pdf differ diff --git a/examples/tutorials/contingency_roc_figure.png b/examples/tutorials/contingency_roc_figure.png new file mode 100644 index 00000000..8cdb4d21 Binary files /dev/null and b/examples/tutorials/contingency_roc_figure.png differ diff --git a/examples/tutorials/example_spatial_test.json b/examples/tutorials/example_spatial_test.json new file mode 100644 index 00000000..091bd5ff --- /dev/null +++ b/examples/tutorials/example_spatial_test.json @@ -0,0 +1,1013 @@ +{ + "min_mw": 4.95, + "name": "Poisson S-Test", + "obs_catalog_repr": "", + "obs_name": null, + "observed_statistic": -161.78441260001426, + "quantile": 0.144, + "sim_name": "helmstetter_aftershock", + "status": "normal", + "test_distribution": [ + -142.15912639069947, + -158.03342640374018, + -164.06994363833354, + -156.2118162828313, + -151.1323243503501, + -144.68651018495927, + -156.09208195373162, + -140.96591988410438, + -151.4229310929297, + -145.6222366335313, + -142.562931485923, + -127.9720127172886, + -143.59941519636072, + -153.2935559804288, + -153.07314228151637, + -132.44799584081014, + -151.32734166576273, + -165.64729567106318, + -144.79694713312404, + -149.26336484764687, + -146.9324071605464, + -154.8347706174551, + -137.45728887993445, + -144.32149286286335, + -171.9389300894851, + -149.0680320683137, + -147.78065284497046, + -142.45899987607976, + -156.51790356094176, + -149.99040948672626, + -166.69093762863767, + -135.61651622913666, + -146.4282455462748, + -142.05954199368077, + -142.9820009749025, + -158.13963820682966, + -157.1604506334304, + -146.82532443072841, + -152.41701692302667, + -149.9308883186099, + -144.65276462743887, + -163.39214685184027, + -135.50720652697765, + -150.57477803881824, + -138.47605199374723, + -148.36973586773718, + -154.13761450813277, + -147.2182297359854, + -150.19467260496663, + -160.48284942633254, + -149.5091159897145, + -158.6650361679499, + -134.51425944187872, + -162.07960330698333, + -164.80859782448016, + -145.2595315056542, + -139.10092569324485, + -139.65433683312875, + -153.79604236678045, + -156.52757268042194, + -147.39977208908763, + -145.49554181182324, + -156.83206651069028, + -154.96971319656998, + -158.47501820869115, + -148.25941625500965, + -143.77012170254122, + -156.968641407003, + -138.91207001495317, + -150.5316214210648, + -150.55774226699384, + -158.0656927399125, + -157.1437995280583, + -151.97232670285058, + -170.88745454140846, + -163.24355490752365, + -161.51148383620745, + -151.71276633305172, + -136.9022072054002, + -150.6747197425306, + -150.93077930936366, + -165.77668764009624, + -147.15499737711792, + -153.483081525185, + -145.5008536885861, + -143.9403710948696, + -151.4767624681781, + -143.67312257704197, + -150.69777433930616, + -148.328444595575, + -158.42314277069778, + -150.84249895334867, + -150.6978834989548, + -152.4632750009686, + -164.61329959921008, + -147.81808787705154, + -131.23058238521804, + -167.02316109405223, + -143.20023440412396, + -161.1330817083552, + -147.1623226394725, + -145.78133474954632, + -158.06674885712826, + -137.9137009700574, + -158.20758155754, + -138.30772493370245, + -138.27881888020573, + -149.29593613676715, + -154.59739003563573, + -140.90099916957854, + -157.4496678547531, + -166.06337044865953, + -156.0318248084511, + -140.86174691565864, + -145.26660138350107, + -155.64814956990892, + -167.63866816744957, + -159.43732639359447, + -152.6939082186241, + -163.8025165441094, + -143.21042750930374, + -165.08699432018068, + -160.83112487433237, + -139.65082499821148, + -147.46659130205285, + -137.86493861388726, + -148.11570073969176, + -152.99012859526857, + -147.1122273122803, + -151.62220449699336, + -153.78426276493178, + -149.26562934290718, + -150.62791978078184, + -163.21551241939284, + -144.83243270391077, + -141.19607720525164, + -153.95132313169165, + -174.6303169201641, + -152.20984346193248, + -150.8743507028908, + -143.8787459752769, + -141.85808252176574, + -150.38856485572475, + -145.88285324511884, + -146.14883354083472, + -152.15828261890823, + -157.7798103328196, + -145.0743725303715, + -146.3036348809078, + -149.27847581867525, + -145.92163372505888, + -141.92843582490394, + -163.49760785905787, + -154.8697637427756, + -153.6099183691705, + -149.45634346535962, + -143.452720634022, + -160.3720591818914, + -148.7556427531577, + -145.99725596221498, + -138.74685894830935, + -131.10304743428912, + -150.61655990358616, + -154.26354260754778, + -147.83516659205364, + -143.0656704402159, + -172.5136309001677, + -148.35525668998102, + -147.13540824605198, + -149.53987828660578, + -173.26953196024712, + -158.74855459290904, + -146.28526699157615, + -142.44529666809575, + -152.3557992055471, + -166.0032718360208, + -150.87689037587722, + -139.35094198980894, + -158.92846486077968, + -144.83797855492864, + -148.37282998884348, + -156.24788175815937, + -149.15300747696782, + -154.8723246784437, + -128.9575522475119, + -136.44673046682368, + -124.85765845872473, + -141.0064121102559, + -162.4095497282376, + -146.9110951178423, + -141.43256177407528, + -144.81948036848786, + -152.94173058806427, + -143.1966034451332, + -142.04535290093816, + -134.88268639130018, + -154.1267862639329, + -139.70454979339874, + -162.66197262268295, + -138.9396841470501, + -156.5382930979418, + -129.361613596979, + -142.07458713372745, + -158.47396487694863, + -156.42085755003927, + -139.1707657868423, + -160.64199847786875, + -154.42726567003365, + -149.7648299113532, + -158.86179598942024, + -144.13064326714778, + -154.91964759617707, + -164.38689458681122, + -158.12974767548255, + -154.55864306639364, + -161.3326063582142, + -156.10374733337014, + -155.44514502358646, + -142.62876287670477, + -145.4685722988221, + -143.93637547312866, + -152.82178745528194, + -149.02829692080635, + -162.17824475058083, + -181.4292277821842, + -163.98066865543802, + -135.50129750987412, + -147.56302478765014, + -156.93464529487937, + -153.71233466884712, + -142.6159130917547, + -156.15489290056684, + -136.88173134586845, + -164.1291571034271, + -151.99651893187618, + -157.45066444214007, + -147.48329254168317, + -149.63201596206, + -161.6165314257782, + -166.6942789643197, + -137.5249633258673, + -152.68423720801948, + -140.70800638834487, + -139.45662229884022, + -150.04639748343516, + -145.38291282748958, + -154.22868360488184, + -145.8371266620784, + -130.80557422658555, + -147.17295083715305, + -148.4440187511363, + -166.05644688898488, + -157.09735238474218, + -148.91816463918545, + -162.94082661671493, + -143.64521989743707, + -136.82530236512503, + -149.66515356469006, + -155.34973368958518, + -159.66389762780648, + -133.54235655629984, + -148.045368572169, + -157.15233642745665, + -154.67214633692885, + -158.82120971997676, + -149.340545108675, + -140.04924596944804, + -168.89947761288255, + -143.78003467427897, + -141.5934101658725, + -162.4308235037434, + -142.91943073643205, + -152.2879716855764, + -155.58000187570536, + -149.61438925793163, + -131.03759732647865, + -172.91353027318826, + -155.30121066498495, + -143.40485914697376, + -143.56497526106423, + -161.429737348088, + -151.20313329673144, + -166.57331065072114, + -146.1215675838436, + -153.0754246650369, + -139.31682933075624, + -148.13865659098326, + -147.79700609154852, + -138.8271870565417, + -152.55619668690997, + -149.39513292016562, + -141.8034846953509, + -153.87778362368704, + -136.00629371762295, + -160.6343034171807, + -145.90381154246484, + -159.39913602079213, + -162.89260697723174, + -139.7769673831254, + -155.61264884931165, + -135.43361016655638, + -142.32137848923236, + -149.71733758806732, + -111.79043049641221, + -148.6541396977617, + -135.7191490937662, + -155.27622924669663, + -135.7359527441742, + -149.39901487261608, + -146.40541787895444, + -135.19838270927568, + -151.46183932131316, + -151.67671173068936, + -166.04456579100022, + -148.38847800972917, + -150.7829479836589, + -163.9915111890714, + -149.36778693408888, + -147.49381626302778, + -167.96492060685654, + -171.18816754022944, + -184.77952606027338, + -151.02806883435724, + -153.5129126011361, + -155.40955294409656, + -145.46677025042874, + -141.08243634140834, + -176.80598111737396, + -163.42434876862927, + -150.36008644226752, + -166.94694260069915, + -138.4339276905734, + -147.27677551269807, + -153.41659142073607, + -144.3645017355634, + -153.43397020311744, + -165.06730339653058, + -170.06128577091118, + -155.65525141198606, + -151.5441360182541, + -154.08716297754353, + -129.27332354016278, + -150.93183012724478, + -154.87732438524876, + -154.23618397043094, + -157.34250290811764, + -142.55891320102012, + -136.68094159407843, + -147.87691539902733, + -149.2660019323763, + -160.50915358761281, + -135.54555462119137, + -163.5707485301133, + -136.35634408681273, + -144.77538347218658, + -133.17264462088968, + -165.0601991172571, + -133.59590482876143, + -161.9454437327329, + -150.31405299061075, + -164.9539973042565, + -142.96062945435216, + -156.73142203170636, + -152.61351987289225, + -141.69448172288344, + -146.85357688732185, + -154.6798132372522, + -135.86875447821504, + -155.46919993572715, + -154.7508787031157, + -146.44367593288686, + -158.3523908614643, + -150.9861352254517, + -155.11175674773907, + -149.26530786774825, + -139.758185843853, + -148.07861837553685, + -158.54572559806022, + -153.31009821231947, + -146.43296174425745, + -152.2370354586325, + -162.9348291025676, + -140.62583176914887, + -156.72321663201745, + -144.3339341214519, + -156.75080589917138, + -158.9816069688643, + -142.58311607079838, + -150.9174057799217, + -169.26270528970252, + -167.49174436915018, + -137.2363091702202, + -153.30057465221807, + -161.136761103847, + -145.4541488620123, + -140.73566295607833, + -143.8890803061837, + -148.38904150896943, + -136.4245159933792, + -157.70427707160835, + -166.19155500540714, + -143.4313606501522, + -146.49843792946854, + -166.06138246282495, + -157.9295739805029, + -142.2057992140568, + -140.9087231719445, + -168.44579477896502, + -162.56832795609657, + -145.84798893741765, + -156.71396880056145, + -153.2436327231068, + -156.76870425209648, + -162.96797655067337, + -160.13116923919455, + -141.75693760068913, + -160.52435620363346, + -151.70916610815016, + -152.3755604817635, + -159.60001255193293, + -142.44747479501012, + -146.8460497709746, + -143.279986065512, + -137.74888015515535, + -153.5852307516853, + -149.85497361253704, + -150.15973138482101, + -157.19102254774327, + -145.57350448401792, + -141.84580836794822, + -158.34816212751105, + -155.79802802922777, + -141.34820385290055, + -163.9692923394164, + -159.72711698942214, + -144.93122916408743, + -155.97560127184343, + -160.43979991644153, + -161.7721761821002, + -152.10728081950276, + -144.27397770724718, + -160.7356170225631, + -166.05371132451222, + -160.41830751238422, + -138.2611378316443, + -158.91851943323317, + -139.92894881097817, + -150.60446663782105, + -147.23783960607506, + -139.41585426342698, + -163.86914205259995, + -156.94207958766646, + -148.87754032648596, + -138.91478577449897, + -149.61705809569125, + -156.5109345318093, + -161.67416129950644, + -153.71356282938916, + -152.770433326358, + -150.61642194629866, + -146.74613137504787, + -147.46435642577944, + -160.42440369516743, + -138.7183958557348, + -145.11066670725498, + -165.99663667641258, + -154.46620721806823, + -158.03223601845656, + -158.32865590019486, + -176.68260349277097, + -153.20172826513303, + -129.65399309794287, + -148.72419021805314, + -143.86924957415303, + -147.1827201032632, + -152.51684747302596, + -148.78590804047104, + -149.26404352656903, + -152.71704054676158, + -150.63576498298903, + -143.68001476926509, + -159.2594449317516, + -147.32251316463123, + -149.5545607412921, + -148.80895102398875, + -149.24561625342727, + -143.93338484118442, + -162.71488760694558, + -154.49677038093574, + -137.8107058222747, + -130.9155853063673, + -150.24329376596057, + -141.1471714149635, + -159.0080510906754, + -147.79256744117407, + -140.60924350197124, + -134.0094359111406, + -147.9664204567631, + -152.91739423106014, + -127.93900297073907, + -151.50101994222456, + -152.77891566752837, + -141.6910759218096, + -140.20388945046656, + -139.9296879092004, + -131.31399151718244, + -150.1842137421246, + -151.29416654366486, + -139.12923270118904, + -147.7679817247719, + -145.1958767265594, + -146.27909816506877, + -164.5082932993856, + -154.69598341672327, + -165.1295366961386, + -153.18598833713594, + -162.33258227446635, + -158.70053173092384, + -151.15102586341175, + -139.6587808817423, + -159.23244084747932, + -172.89883887805854, + -153.5983773577908, + -147.40119162611637, + -150.84136701233024, + -148.16634845199175, + -145.73703550855117, + -138.24107883598944, + -139.70987541434232, + -169.49102273078034, + -167.8439853674901, + -136.95079029128246, + -151.90965944788778, + -163.25113811137516, + -143.42642382365796, + -142.12781127664795, + -163.48333283800002, + -140.83465736635435, + -149.71174960063814, + -159.8883456597541, + -152.44823638380504, + -146.20810068722466, + -150.21616566491662, + -155.52174751814033, + -156.89353878761233, + -149.91258617019196, + -153.11990117377707, + -162.49456236665506, + -157.9267761597104, + -134.48378966729885, + -162.5534687949425, + -141.86970291859564, + -151.78486714857922, + -132.64521604240508, + -138.64021386782974, + -150.35520529745514, + -145.79096592044908, + -148.18925057263732, + -134.9749032716414, + -157.13985233384713, + -159.7294836703175, + -135.8344838479037, + -149.23802394512995, + -134.96571863891938, + -156.5715392620005, + -141.2858618787345, + -166.51202059758546, + -158.8926659534237, + -147.94549946113284, + -169.00118166749917, + -165.53114005920438, + -149.73293264579357, + -159.39483843837723, + -136.50195328880386, + -134.24359551862273, + -151.61504546987828, + -161.4758026892111, + -142.4210109354707, + -145.7882489055616, + -140.18748968280346, + -165.7136411068131, + -145.21970494695177, + -156.22283651558348, + -159.11491726036735, + -156.8467704382631, + -148.797859106291, + -154.27916308012234, + -150.5323704272821, + -146.38123870880975, + -156.19897863176197, + -162.59129344800206, + -157.88338192069537, + -155.37940105209728, + -131.4228299461116, + -161.7999606637918, + -178.16452781010642, + -160.79891711680935, + -155.80217105120317, + -124.86496170484017, + -150.421993664622, + -153.2344625327636, + -157.6183954841161, + -169.71439553003043, + -149.28557194327394, + -150.9769587001835, + -157.69326880321182, + -157.67036662327783, + -154.35235317368824, + -137.29590397984953, + -163.89480891708604, + -145.74363829531202, + -144.7005920722935, + -139.75864476492143, + -158.74368587994138, + -152.07535305167454, + -164.1381644319398, + -155.88025550423566, + -149.840993950802, + -145.99219392562114, + -133.21980537015116, + -152.00834204208758, + -142.8859632733065, + -152.8862024597986, + -140.81999517149188, + -155.5281246466449, + -168.23668773354385, + -162.88326907481652, + -139.44429541434664, + -144.1928957858322, + -138.53076721794693, + -146.93351122318828, + -165.52087204703605, + -143.61924535791292, + -154.3762535599551, + -158.33173576676649, + -133.47160480167258, + -160.6463273474228, + -143.01025335010246, + -153.71073435811599, + -138.63031980626448, + -153.69165004330188, + -148.78964229680588, + -140.2949918344928, + -157.80806785330398, + -154.6046252245415, + -156.3201832047376, + -148.98318407654853, + -153.20930406297873, + -132.0567209398775, + -157.38834062932386, + -152.89998351926698, + -158.42315171939893, + -144.22532188848126, + -153.6812538224214, + -157.5003757027294, + -162.6021755753064, + -158.49252063752346, + -130.27000261435367, + -149.42122139955984, + -142.02047021654946, + -170.08226249842502, + -170.8464816442118, + -149.07420567576804, + -147.55643175581127, + -157.3533702288697, + -146.9701313462685, + -136.7766347127696, + -149.36632018347416, + -171.07318622251398, + -143.64522641879392, + -144.5098694445984, + -154.3008207096026, + -154.12653375314864, + -170.95714917397925, + -145.94720566041434, + -147.91240222405418, + -150.926089772471, + -164.61130954784426, + -150.96591726791735, + -137.11549894400258, + -173.15739173515237, + -154.42769253577865, + -148.1827725653889, + -158.6325391092287, + -158.68148328181064, + -147.83946320755945, + -136.11715677376085, + -149.71663988761293, + -159.3345479678042, + -138.0805820995323, + -172.11169752394818, + -139.43294789679305, + -137.42349245140832, + -153.82828283074795, + -160.86668899737697, + -142.10915819967232, + -160.57865320449005, + -156.6468289423171, + -154.73503302164448, + -149.03439731602376, + -170.13901469521062, + -148.53687752877187, + -160.79887213380795, + -135.77873329959635, + -129.85360587875272, + -157.2088992684353, + -145.46756897474938, + -156.23264319231427, + -160.7800177653771, + -166.57623257467094, + -146.78363557164369, + -162.06830173329325, + -147.38658786223928, + -140.50654304937922, + -147.74642054344642, + -153.52570934564451, + -149.2030620352746, + -168.29891691727798, + -155.5567018422138, + -154.4360423826596, + -165.81588048401477, + -148.5513079580687, + -151.31120288591785, + -167.18138906810827, + -146.63027933615598, + -152.1379894830191, + -134.35693326354072, + -165.18971900464126, + -149.10038723011914, + -140.07354725043305, + -136.070715936927, + -171.60483021225488, + -151.438344840263, + -143.54331676276132, + -153.65382565797148, + -153.54375830148797, + -165.22403950628092, + -149.39502509673937, + -136.06876486230934, + -141.40945884441976, + -166.67318878013808, + -134.71746903818203, + -133.71688986383083, + -150.4881577200611, + -155.21472975963218, + -156.77537230547995, + -143.89233241642094, + -131.98825399696554, + -140.75788150830883, + -144.88432311365256, + -154.50567749600032, + -136.6719858130825, + -131.42327465943103, + -134.51237817567272, + -153.46270142468825, + -148.73801622143094, + -136.5247840686459, + -162.67316343037348, + -170.74087287760312, + -154.40752687643118, + -135.01390277190293, + -149.13274424866088, + -163.18342814341779, + -159.55779030792206, + -148.30635222360735, + -149.45190962676517, + -147.99798186263547, + -135.206183142001, + -152.26227446175477, + -161.20300986466432, + -150.09142136693418, + -160.22440072232212, + -161.61946481649642, + -151.68512455532985, + -145.58633892180063, + -147.91500612487374, + -165.62587661008735, + -145.04513455633136, + -163.08410386295014, + -162.31758181593733, + -144.27319711502855, + -144.61549777253293, + -167.62018306863192, + -146.73771459447397, + -144.5814739670925, + -162.34978177197004, + -156.54434264456503, + -142.1506383842688, + -153.35401929378645, + -159.5543076501938, + -144.1753069428766, + -155.91162072983667, + -140.69342869752944, + -144.85455413552108, + -142.5824841789977, + -173.4444721811974, + -142.56416897419797, + -152.18759649358006, + -162.31190211719883, + -148.51277864784234, + -157.41776534213903, + -160.21060915832422, + -144.0459991472361, + -139.98422759766595, + -153.72232543382455, + -169.4084282781118, + -155.56045535698203, + -146.6147321246035, + -163.1186755157077, + -137.21217261885937, + -160.2834821468885, + -160.71141015618434, + -155.0872038302171, + -156.67101271393528, + -151.3673005161189, + -164.64784107745066, + -154.8355413100051, + -144.1904591511808, + -161.61674684237536, + -150.94546236515617, + -151.62530891951135, + -162.17491555869748, + -144.61052303654975, + -147.04522055079846, + -174.98671575394226, + -145.82207838559134, + -146.17473936354386, + -153.54564545981256, + -156.90095368074356, + -150.2360853149536, + -147.6442334778972, + -132.85626851128967, + -145.71032434809877, + -152.22194178549023, + -146.94900398059974, + -170.9994612020659, + -143.47801489750884, + -150.44857422359263, + -149.99779816120872, + -150.54663626862472, + -167.28358627442634, + -153.64027636761256, + -146.21958474913754, + -158.97922430148617, + -148.98629580532395, + -152.72311996225898, + -151.09973007312928, + -135.872712517318, + -143.15774491871872, + -145.0383833244751, + -140.82535470072173, + -140.07088097963344, + -148.72504906345887, + -148.41265839565747, + -156.1404450457094, + -158.53185700633915, + -154.46398997579485, + -137.48535782302082, + -157.91175045831585, + -150.190972532887, + -145.42868619840064, + -157.77204580887968, + -161.37200065076956, + -147.4054069672316, + -148.0787237108371, + -145.9541432578854, + -153.99728979919973, + -140.7877797837341, + -145.14271780806814, + -164.45977329349157, + -156.75240875810965, + -150.65193337230116, + -160.47959863635901, + -167.00900864279296, + -161.8596723806334, + -144.81607392131033, + -151.32415538309965, + -155.91308704330135, + -136.83592541373207, + -164.4699232545354, + -167.9121808494071, + -147.95378472639518, + -137.85203386659293, + -157.01692633555803, + -164.15884843300023, + -146.43090712755566, + -125.28687123823055, + -162.28971667327468, + -130.1362707038406, + -143.69536718632116, + -142.50517571212063, + -162.75306612240365, + -156.12636373923482, + -153.20389500237036, + -157.15500954556273, + -145.49992714302246, + -144.28885217562691, + -151.35754535262345, + -164.28398610981486, + -157.09573494206563, + -157.81388693135847, + -162.22558068949843, + -160.12445041660106, + -153.21414670985936, + -153.85597544237504, + -151.25333203739953, + -142.36618437491046, + -143.51173944729211, + -162.150141957864, + -149.20506333771695, + -149.80150546711513, + -144.05348572718233, + -158.56118904905333, + -153.8676229270514, + -151.16721304443473, + -149.88854073761604, + -133.67610718367564, + -156.79429802409408, + -135.831375861927, + -153.215662960408, + -155.03056854948545, + -143.82621903733258, + -165.4754551601686, + -146.75782203487336, + -151.41498486009752, + -150.73846721754109, + -154.94796637546037, + -156.08348882759182, + -135.04147387829602, + -156.62923768816324, + -135.5268786070539, + -162.05865580187293, + -133.7349710465447, + -154.7458114656379, + -143.32043824396567, + -168.2672832320561, + -138.5361734330478, + -156.4882707841854, + -146.0677493850431, + -149.16038354405444, + -133.85009226249565, + -146.1952965822121, + -149.36689830686646, + -138.48218774305874, + -153.5876156691558, + -146.91073175428005, + -154.92178238913186, + -129.71552153929633, + -154.1978153304223, + -163.66170068256133, + -164.0885578131265, + -151.85758321456086, + -156.21369619167712, + -144.30056169760616, + -160.78747237849743, + -146.41319636041132, + -154.97953491022346, + -161.1349462035746, + -158.08185110041313, + -152.63151340385184, + -152.22562175136503, + -156.5115112807294, + -147.97231012689969, + -158.65330498445726, + -168.25746940527114, + -160.04332201206535, + -161.00008532314132, + -144.22925495266531, + -152.91357933348343, + -143.8992251778478, + -144.80483815823965, + -154.32953993965836, + -136.93619090834542, + -167.19230040184465, + -133.21741164844354, + -146.0520840799864, + -154.66881369738934, + -144.74521840878404, + -153.25640805470408, + -162.09989973034268, + -156.2733136181334, + -158.67310922964157, + -162.09887806246763, + -160.89456912287102, + -155.4050527336256, + -153.15983022366095, + -159.5884134228442, + -151.93228048508414, + -140.77588423166085, + -126.97937092870525, + -152.08144656520034, + -137.12029091549633, + -151.5605568415069, + -144.18020193380858, + -150.60865370599475, + -166.947856656924, + -147.77302343031596, + -158.1381171271884, + -150.9835113126794, + -134.02099180573578 + ], + "type": "EvaluationResult" +} \ No newline at end of file diff --git a/examples/tutorials/gridded_forecast_evaluation.py b/examples/tutorials/gridded_forecast_evaluation.py index eb72827e..4e38ad2d 100644 --- a/examples/tutorials/gridded_forecast_evaluation.py +++ b/examples/tutorials/gridded_forecast_evaluation.py @@ -118,9 +118,38 @@ # When comparing the forecast ROC curve against a catalog, one can evaluate if the forecast is more or less specific (or smooth) at different level or seismic rate. # # Note: This figure just shows an example of plotting an ROC curve with a catalog forecast. +# If "linear=True" the diagram is represented using a linear x-axis. +# If "linear=False" the diagram is represented using a logarithmic x-axis. + + +print("Plotting concentration ROC curve") +_= plots.plot_concentration_ROC_diagram(forecast, catalog, linear=True) + + + + +#################################################################################################################################### +# Plot ROC and Molchan curves using the alarm-based approach +# ----------------------- +#In this script, we generate ROC diagrams and Molchan diagrams using the alarm-based approach to evaluate the predictive +#performance of models. This method exploits contingency table analysis to evaluate the predictive capabilities of +#forecasting models. By analysing the contingency table data, we determine the ROC curve and Molchan trajectory and +#estimate the Area Skill Score to assess the accuracy and reliability of the prediction models. The generated graphs +#visually represent the prediction performance. + +# Note: If "linear=True" the diagram is represented using a linear x-axis. +# If "linear=False" the diagram is represented using a logarithmic x-axis. + +print("Plotting ROC curve from the contingency table") +# Set linear True to obtain a linear x-axis, False to obtain a logical x-axis. +_ = plots.plot_ROC_diagram(forecast, catalog, linear=True) + +print("Plotting Molchan curve from the contingency table and the Area Skill Score") +# Set linear True to obtain a linear x-axis, False to obtain a logical x-axis. +_ = plots.plot_Molchan_diagram(forecast, catalog, linear=True) + + -print("Plotting ROC curve") -_ = plots.plot_ROC(forecast, catalog) #################################################################################################################################### # Calculate Kagan's I_1 score @@ -130,4 +159,4 @@ # (see Kagan, YanY. [2009] Testing long-term earthquake forecasts: likelihood methods and error diagrams, Geophys. J. Int., v.177, pages 532-542). I_1 = get_Kagan_I1_score(forecast, catalog) -print("I_1score is: ", I_1) \ No newline at end of file +print("I_1score is: ", I_1) diff --git a/examples/tutorials/molchan_figure.pdf b/examples/tutorials/molchan_figure.pdf new file mode 100644 index 00000000..85862b7a Binary files /dev/null and b/examples/tutorials/molchan_figure.pdf differ diff --git a/examples/tutorials/molchan_figure.png b/examples/tutorials/molchan_figure.png new file mode 100644 index 00000000..ad7a75ca Binary files /dev/null and b/examples/tutorials/molchan_figure.png differ diff --git a/examples/tutorials/roc_figure.pdf b/examples/tutorials/roc_figure.pdf new file mode 100644 index 00000000..9034b2bc Binary files /dev/null and b/examples/tutorials/roc_figure.pdf differ diff --git a/examples/tutorials/roc_figure.png b/examples/tutorials/roc_figure.png new file mode 100644 index 00000000..0b0b4a8c Binary files /dev/null and b/examples/tutorials/roc_figure.png differ