From 162cf9f18618b42e08737b8dd5d0e00be3b32cee Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Wed, 27 Nov 2024 14:09:21 -0500 Subject: [PATCH 01/12] unit test, fail --- src/test/unit/mcmc/chainset_test.cpp | 16 ++++ .../mcmc/test_csv_files/datagen_output.csv | 82 +++++++++++++++++++ 2 files changed, 98 insertions(+) create mode 100644 src/test/unit/mcmc/test_csv_files/datagen_output.csv diff --git a/src/test/unit/mcmc/chainset_test.cpp b/src/test/unit/mcmc/chainset_test.cpp index 5647652485..01cb18da42 100644 --- a/src/test/unit/mcmc/chainset_test.cpp +++ b/src/test/unit/mcmc/chainset_test.cpp @@ -207,3 +207,19 @@ TEST_F(McmcChains, summary_stats) { EXPECT_NEAR(theta_ac(i), theta_ac_expect(i), 0.0005); } } + +TEST_F(McmcChains, quantile_tests) { + std::ifstream datagen_stream; + datagen_stream.open("src/test/unit/mcmc/test_csv_files/datagen_output.csv", + std::ifstream::in); + stan::io::stan_csv datagen_csv = stan::io::stan_csv_reader::parse(datagen_stream, &out); + datagen_stream.close(); + stan::mcmc::chainset datagen_chains(datagen_csv); + + Eigen::VectorXd probs(6); + probs << 0.0, 0.01, 0.05, 0.95, 0.99, 1.0; + Eigen::VectorXd stepsize_quantiles = datagen_chains.quantiles("stepsize__", probs); + for (size_t i = 0; i < probs.size(); ++i) { + EXPECT_EQ(stepsize_quantiles(i), std::numeric_limits::quiet_NaN()); + } +} diff --git a/src/test/unit/mcmc/test_csv_files/datagen_output.csv b/src/test/unit/mcmc/test_csv_files/datagen_output.csv new file mode 100644 index 0000000000..30349505bf --- /dev/null +++ b/src/test/unit/mcmc/test_csv_files/datagen_output.csv @@ -0,0 +1,82 @@ +# stan_version_major = 2 +# stan_version_minor = 35 +# stan_version_patch = 0 +# model = datagen_poisson_glm_model +# start_datetime = 2024-11-27 17:05:51 UTC +# method = sample (Default) +# sample +# num_samples = 25 +# num_warmup = 1000 (Default) +# save_warmup = false (Default) +# thin = 1 (Default) +# adapt +# engaged = true (Default) +# gamma = 0.05 (Default) +# delta = 0.8 (Default) +# kappa = 0.75 (Default) +# t0 = 10 (Default) +# init_buffer = 75 (Default) +# term_buffer = 50 (Default) +# window = 25 (Default) +# save_metric = false (Default) +# algorithm = hmc (Default) +# hmc +# engine = nuts (Default) +# nuts +# max_depth = 10 (Default) +# metric = diag_e (Default) +# metric_file = (Default) +# stepsize = 1 (Default) +# stepsize_jitter = 0 (Default) +# num_chains = 1 (Default) +# id = 1 (Default) +# data +# file = (Default) +# init = 2 (Default) +# random +# seed = 1975495786 (Default) +# output +# file = output.csv (Default) +# diagnostic_file = (Default) +# refresh = 100 (Default) +# sig_figs = -1 (Default) +# profile_file = profile.csv (Default) +# save_cmdstan_config = false (Default) +# num_threads = 1 (Default) +# stanc_version = stanc3 v2.36.0-rc1 +# stancflags = +lp__,accept_stat__,stepsize__,treedepth__,n_leapfrog__,divergent__,energy__,N,y_sim.1,y_sim.2,y_sim.3,y_sim.4,y_sim.5,y_sim.6,y_sim.7,y_sim.8,y_sim.9,y_sim.10,y_sim.11,y_sim.12,y_sim.13,y_sim.14,y_sim.15,y_sim.16,y_sim.17,y_sim.18,y_sim.19,y_sim.20,x_sim.1,x_sim.2,x_sim.3,x_sim.4,x_sim.5,x_sim.6,x_sim.7,x_sim.8,x_sim.9,x_sim.10,x_sim.11,x_sim.12,x_sim.13,x_sim.14,x_sim.15,x_sim.16,x_sim.17,x_sim.18,x_sim.19,x_sim.20,pop_sim.1,pop_sim.2,pop_sim.3,pop_sim.4,pop_sim.5,pop_sim.6,pop_sim.7,pop_sim.8,pop_sim.9,pop_sim.10,pop_sim.11,pop_sim.12,pop_sim.13,pop_sim.14,pop_sim.15,pop_sim.16,pop_sim.17,pop_sim.18,pop_sim.19,pop_sim.20,alpha_sim,beta_sim,eta.1,eta.2,eta.3,eta.4,eta.5,eta.6,eta.7,eta.8,eta.9,eta.10,eta.11,eta.12,eta.13,eta.14,eta.15,eta.16,eta.17,eta.18,eta.19,eta.20 +# Adaptation terminated +# Step size = nan +# Diagonal elements of inverse mass matrix: +# +0,1,nan,1,1,0,0,20,561,260,798,369,1203,596,334,966,709,309,553,361,359,667,326,1064,687,671,826,332,0.889785,0.869428,0.641095,0.701368,0.276519,0.779494,0.502959,0.357508,0.759545,0.787474,0.988459,0.524494,0.663204,0.362067,0.36927,0.439961,0.150813,0.617442,0.65277,0.406486,1712.33,700.623,1775.35,837.451,2116.05,1482.75,752.832,1966.98,1814.52,819.064,1610.13,789.291,800.894,1285.54,640.934,2237.37,1167.53,1551.64,1963.94,681.831,-0.457442,-0.620075,6.43643,5.55542,6.62679,5.83802,7.0284,6.36087,5.85453,6.90513,6.57516,5.76243,6.31371,5.88847,5.81705,6.47698,5.77651,6.98281,6.51169,6.50677,6.7205,5.81529 +0,1,nan,1,1,0,0,20,1722,1306,754,1528,733,1600,948,1278,1223,1798,693,2037,1719,1212,1288,1378,1592,752,2099,1054,0.668453,0.941061,0.668049,0.30066,0.907355,0.551538,0.169235,0.529835,0.603026,0.212001,0.269008,0.519714,0.348572,0.104479,0.883532,0.685805,0.89391,0.222068,0.101908,0.242857,2066.97,1583.66,830.766,1724.53,924.949,1776.9,1036.71,1407.49,1376.28,1960.11,789.657,2318.4,1841.71,1349.84,1441.71,1627.43,1903.65,815.773,2227.8,1121.83,-0.0495029,-0.132598,7.4957,7.19321,6.58426,7.36334,6.65992,7.35999,6.87186,7.1298,7.09768,7.50314,6.58643,7.63022,7.42273,7.14439,7.10693,7.25432,7.38349,6.62519,7.64575,6.94101 +0,1,nan,1,1,0,0,20,1093,1890,1715,1928,2149,1412,925,464,1152,1150,1215,878,893,2105,658,1281,817,716,1973,1705,0.785469,0.0575787,0.540541,0.818959,0.260517,0.879333,0.149316,0.699759,0.0240241,0.0682071,0.593713,0.270423,0.207794,0.934491,0.36555,0.389162,0.419274,0.549806,0.305225,0.0661502,1038.59,2243.09,1852.39,1788.13,2495.77,1282.29,1078.6,503.433,1540.54,1346.68,1263.11,987.193,1034.43,1970.31,708.613,1455.63,819.307,722.281,2164.94,2089.69,-0.22022,0.312052,6.97051,7.51336,7.47269,7.52427,7.68343,7.21058,6.80979,6.21959,7.12717,7.00646,7.10639,6.75903,6.78623,7.65734,6.45716,7.18441,6.61907,6.53376,7.55517,7.4452 +0,1,nan,1,1,0,0,20,545,727,789,305,179,330,535,79,241,237,396,226,461,545,342,173,837,268,353,327,0.0781223,0.0890694,0.0783259,0.0711005,0.820902,0.250208,0.112536,0.942694,0.876881,0.690707,0.655685,0.49363,0.478303,0.218567,0.398166,0.373951,0.0162069,0.74373,0.422011,0.706077,1512.97,2180.23,2239.16,908.532,1520.5,1139.03,1504.07,858.156,1810.14,1587.6,2337.28,1155.02,2324.08,1869.5,1473.87,757.702,2256.68,1975.91,1657.42,2046.12,-0.908665,-1.45141,6.29978,6.64924,6.69151,5.79997,5.22667,5.76611,6.24393,4.47789,5.31979,5.45882,5.89641,5.42675,6.1482,6.30753,5.80908,5.17887,6.78946,5.60067,5.89185,5.69023 +0,1,nan,1,1,0,0,20,1579,2002,1254,1526,1284,1713,875,1095,1489,851,654,905,1216,506,736,858,1786,604,428,1463,0.318432,0.162465,0.852501,0.236457,0.712322,0.0372841,0.587934,0.419756,0.965406,0.44225,0.370643,0.377072,0.301076,0.667038,0.583629,0.397981,0.0426562,0.937472,0.806566,0.524114,1971.99,2234.81,1953.53,1747.31,1860.38,1777.68,1167.83,1405.08,2441.31,1084.45,796.645,1132.6,1412.56,682.945,982.649,1078.11,1925.83,997.782,678.69,1952.81,-0.027113,-0.493096,7.40267,7.60469,7.12992,7.32213,7.15018,7.43756,6.74589,7.01376,7.29714,6.74364,6.47053,6.81922,7.07759,6.17039,6.57535,6.75961,7.51496,6.41616,6.09534,7.29147 +0,1,nan,1,1,0,0,20,529,175,158,183,933,519,645,422,709,163,232,276,700,357,333,388,698,473,649,527,0.86009,0.41529,0.204179,0.747748,0.918286,0.676058,0.354138,0.864243,0.695624,0.326446,0.530317,0.877253,0.411052,0.899261,0.924517,0.464493,0.920972,0.417376,0.59292,0.395982,1468.54,526.32,596.813,549.33,2446.27,1542.76,2227.9,1118.49,2079.11,569.759,721.775,718.742,2285.92,921.687,864.799,1205.95,1982.1,1400.68,1910.91,1860.77,-1.33,0.358,6.26993,5.08458,5.1347,5.24639,6.80107,6.25336,6.5056,5.99914,6.55873,5.13208,5.44157,5.56156,6.55168,5.81814,5.76347,5.93131,6.59162,6.06413,6.4376,6.34051 +0,1,nan,1,1,0,0,20,4680,6378,3829,4177,5096,2030,1259,2454,9492,4116,3795,4699,6100,2231,6932,5536,1874,4340,8572,6486,0.436024,0.406114,0.814413,0.194004,0.859902,0.19379,0.0602112,0.118848,0.892329,0.55086,0.685525,0.252052,0.31876,0.16974,0.459228,0.80298,0.475591,0.497588,0.954984,0.696509,1438.2,2012.11,762.889,1688.09,969.206,798.637,593.355,1097.51,1742.38,1107.41,874.321,1819.96,2133.24,940.385,2061.06,1111.95,546.09,1214.59,1423.75,1459.74,0.663693,1.16877,8.44445,8.74528,8.25266,8.32179,8.54519,7.57309,7.11986,7.8034,9.16962,8.3173,8.23836,8.46486,8.70164,7.70837,8.8314,8.61606,7.52233,8.34742,9.04089,8.76377 +0,1,nan,1,1,0,0,20,7712,8003,3475,1828,2521,2244,1522,3691,1881,892,4779,5867,1942,5683,928,714,5929,4592,2581,1998,0.920276,0.854027,0.417736,0.291078,0.889187,0.459122,0.109137,0.761823,0.646166,0.33031,0.571085,0.850666,0.350656,0.794782,0.154307,0.286977,0.94255,0.815473,0.902305,0.41887,2139.99,2462.24,2359.98,1485.89,745.913,1359.46,1696.98,1335.99,855.953,660.987,2446.51,1830.17,1435.79,1945.63,989.689,599.201,1585.28,1483.56,719.657,1311.59,-0.32371,1.7492,8.95459,8.97898,8.1734,7.48921,7.84626,7.69423,7.3038,8.2063,7.55878,6.7478,8.47765,8.67644,7.55912,8.63986,6.84359,6.57387,8.69351,8.40491,7.83337,7.58797 +0,1,nan,1,1,0,0,20,3364,2393,2824,2758,3976,2211,1649,1501,3901,3487,2337,3050,1543,900,3217,2959,2198,2033,1750,1141,0.519812,0.539921,0.278301,0.31101,0.403561,0.413657,0.192691,0.140997,0.732162,0.0329133,0.360066,0.127687,0.707244,0.243717,0.892494,0.771718,0.616116,0.382596,0.647362,0.167098,1828.5,1303.74,1691.41,1637.98,2178.93,1272.59,1005.59,936.341,1889.5,2341.32,1327.45,1967.29,749.779,571.902,1464.75,1451.22,1125.32,1115.4,888.773,686.031,0.397394,0.453795,8.14454,7.8154,7.95701,7.93975,8.26712,7.73392,7.39817,7.30336,8.27371,8.1708,7.7518,8.03975,7.33812,6.85696,8.09184,8.02775,7.7028,7.58799,7.48101,7.00414 +0,1,nan,1,1,0,0,20,242,815,460,338,698,1227,996,595,336,235,299,860,790,427,222,927,1130,721,651,1267,0.795344,0.563868,0.832882,0.91279,0.871499,0.324669,0.751731,0.388158,0.293071,0.794457,0.855858,0.558512,0.635328,0.830338,0.74454,0.302512,0.515215,0.114089,0.856256,0.15552,603.143,1848.96,1328.67,945.635,1903.73,2427.79,2474.64,1184.06,660.23,623.633,838.346,1932.32,1890.56,1163.9,527.212,1759.73,2490.51,1229.71,1789.16,2207.75,-0.476085,-0.595748,5.45224,6.71037,6.21966,5.83198,6.55629,7.12523,6.88992,6.36938,5.84191,5.48618,5.74547,6.75766,6.69005,6.08878,5.34796,6.81661,7.03722,6.57048,6.50331,7.13099 +0,1,nan,1,1,0,0,20,2897,3176,3933,1687,1237,2010,5009,3072,1848,3681,4302,1148,4688,3318,4122,2204,4702,2836,4343,5037,0.0475485,0.744016,0.1805,0.918119,0.665334,0.29303,0.054942,0.811896,0.927165,0.780314,0.569677,0.163299,0.672969,0.988158,0.976978,0.715406,0.651474,0.871155,0.684639,0.169713,1261.94,1637.13,1747.92,945.236,650.852,952.699,2240.33,1624.06,995.865,2002.41,2142.27,529.797,2431.9,1833.85,2268.04,1155.98,2472.76,1490.97,2203.45,2266,0.827469,-0.243609,7.95629,8.04692,8.24968,7.45524,7.14367,7.61538,8.52846,8.02237,7.50521,8.23948,8.35831,7.06018,8.45995,8.10092,8.31614,7.70589,8.48185,7.92243,8.35847,8.5119 +0,1,nan,1,1,0,0,20,9914,8758,18384,21779,20278,20582,19120,22618,12212,12764,6956,4759,15338,18786,16820,10793,10224,4567,21159,10105,0.42072,0.0161948,0.397095,0.755409,0.13366,0.0248725,0.690059,0.0497695,0.554931,0.872293,0.15337,0.691976,0.118852,0.0446265,0.903434,0.427333,0.878747,0.885547,0.544709,0.502473,1089.36,928.961,1989.86,2475.8,2109.66,2154.59,2141.66,2355.75,1357.08,1462.92,716.346,539.399,1596.28,1945.09,1960.33,1146.49,1192.32,512.039,2347.83,1128.68,2.26418,-0.111779,9.2105,9.09643,9.81561,9.99406,9.90352,9.93675,9.85638,10.0232,9.41524,9.45486,8.8212,8.47728,9.62632,9.83225,9.74406,9.26087,9.2496,8.40359,9.96453,9.23682 +0,1,nan,1,1,0,0,20,360,647,1224,1327,1275,1278,1471,492,504,1113,1104,627,1057,788,392,721,1594,347,1240,1428,0.288427,0.845872,0.334429,0.0828151,0.443799,0.173286,0.922669,0.78115,0.286532,0.888547,0.419577,0.115097,0.385604,0.372152,0.0780263,0.841237,0.541516,0.239509,0.896918,0.397397,610.447,865.948,1908.5,2222.21,1882.74,2101.67,1920.48,668.876,832.783,1603.44,1589.18,957.632,1628.6,1211.53,651.685,958.262,2357.77,571.964,1720.65,2233.46,-0.516189,0.233062,5.96522,6.44478,7.11583,7.20937,7.12773,7.17469,7.25918,6.17147,6.27536,7.07081,6.95258,6.3751,6.96916,6.67018,5.98156,6.54499,7.37549,5.88871,7.14331,7.28774 +0,1,nan,1,1,0,0,20,347,1674,803,1906,2037,1584,865,1140,1799,809,2028,3549,1567,1486,821,2606,724,3776,923,920,0.955096,0.010191,0.87182,0.271545,0.182519,0.441158,0.793866,0.381061,0.160949,0.254274,0.455679,0.0265811,0.0682471,0.556157,0.204568,0.406464,0.529495,0.0639189,0.726377,0.877094,687.601,1033.02,1327.46,1631.87,1551.73,1565.3,1259.78,1069.53,1330.02,667.164,2027.26,2194.23,1007.99,1706.2,675.335,2481.47,800.371,2402.72,1403.64,1622.78,0.522853,-1.20527,5.90491,7.45081,6.66309,7.59305,7.64999,7.34697,6.70472,7.03855,7.52182,6.71942,7.58807,8.1844,7.35631,7.29455,6.7915,7.84956,6.56974,8.23017,6.8942,6.85761 +0,1,nan,1,1,0,0,20,1611,450,1688,1058,582,1391,1567,968,770,638,978,1605,414,883,336,347,874,1000,1119,1333,0.340141,0.69425,0.281773,0.697115,0.778837,0.643142,0.643251,0.987233,0.71158,0.233096,0.944914,0.192334,0.332537,0.122487,0.36223,0.842542,0.826505,0.555444,0.0661899,0.949055,2294.82,694.872,2444.52,1744.32,981.06,2112.97,2455.03,1675.05,1251.02,814.909,1697.97,2160.65,598.827,1153.63,501.299,571.929,1465.14,1630.17,1495.46,2341.75,-0.252766,-0.331745,7.3728,6.06065,7.45536,6.98009,6.37749,7.18972,7.33973,6.84332,6.64288,6.37298,6.87095,7.36159,6.03189,6.75727,5.84427,5.81674,6.76275,6.95941,7.03547,7.19104 +0,1,nan,1,1,0,0,20,311,744,879,230,444,514,526,789,160,347,503,589,249,708,1447,652,275,1382,460,208,0.965117,0.217942,0.195937,0.868994,0.797547,0.626118,0.601574,0.384657,0.523653,0.250033,0.0300894,0.160337,0.832465,0.11721,0.105928,0.548051,0.369152,0.0956314,0.540776,0.332479,1963.09,1534.02,1674.72,1271.62,2144.57,1953.93,1940.26,1915.3,502.859,752.717,773.804,1074.56,1228.25,1226.47,2443.56,2182.45,681.113,2337.11,1521.16,509.385,-0.38672,-1.48692,5.7605,6.62486,6.74534,5.4692,6.09809,6.25989,6.28937,6.59896,5.05496,5.86519,6.21986,6.35454,5.48882,6.55089,7.25699,6.48658,5.58811,7.22775,6.13642,5.35212 +0,1,nan,1,1,0,0,20,372,426,934,1154,648,224,1334,934,579,1517,1183,597,1729,1134,1161,581,462,755,1342,732,0.647249,0.376471,0.739769,0.367994,0.529283,0.723057,0.473407,0.432584,0.855893,0.0393696,0.572818,0.485885,0.126843,0.404543,0.226162,0.697504,0.903987,0.943243,0.24914,0.96101,786.224,654.624,2238.25,1812.08,1239.08,581.313,2269.26,1509.78,1456.13,1556.7,2362.71,1026.28,1936.89,1713.48,1488.82,1313.57,1373.63,2258.81,1828.07,2173.18,0.0085132,-1.17427,5.91571,6.05049,6.85327,7.07862,6.50912,5.52474,7.17981,6.82026,6.287,7.31261,7.10343,6.37165,7.4284,6.97975,7.04868,6.36996,6.1722,6.62348,7.22697,6.56397 +0,1,nan,1,1,0,0,20,273,989,462,1523,1184,1013,818,771,814,302,556,496,340,737,1312,1100,708,683,1152,1264,0.0200452,0.195202,0.349924,0.865083,0.413215,0.404663,0.883097,0.975909,0.099039,0.589506,0.740706,0.578232,0.539132,0.911407,0.741268,0.701708,0.812422,0.981538,0.265892,0.710299,715.017,2214.76,995.319,2406.03,2359.01,1969.68,1219.32,1130.04,1957.76,525.953,868.025,852.847,665.283,1165.18,2099.6,1877.07,1146.63,1041.07,2449.29,2254.13,-0.909634,0.567649,5.67405,6.90407,6.19206,7.36716,7.09092,6.9057,6.6977,6.67435,6.72614,5.69021,6.27705,6.16718,5.89662,6.66836,7.16065,7.02616,6.59612,6.59554,7.04485,7.21409 +0,1,nan,1,1,0,0,20,1365,3605,3409,3766,1374,3200,1079,3256,3416,2762,4491,4470,3852,3812,2658,3307,1560,3637,2307,3996,0.626009,0.354963,0.234779,0.441017,0.869996,0.983047,0.618899,0.337532,0.903936,0.233736,0.895827,0.688864,0.653846,0.648202,0.520035,0.889846,0.977277,0.496893,0.627897,0.624774,708.816,1776.04,1674.43,1828.62,751.053,1651.74,562.433,1672.08,1719.27,1368.13,2340.32,2366.35,1993.75,1879.23,1360.01,1697.5,793.256,1845.49,1200.8,1991.75,0.724109,-0.0697853,7.24402,8.18148,8.13095,8.20465,7.28487,8.06509,7.01319,8.12238,8.11068,7.929,8.41964,8.44514,8.27625,8.21749,7.90307,8.09892,7.33206,8.20993,7.77104,8.27728 +0,1,nan,1,1,0,0,20,223,367,730,272,782,647,174,354,903,745,341,625,481,554,344,659,511,423,669,360,0.235242,0.318595,0.308517,0.448119,0.302605,0.715798,0.772543,0.266673,0.137946,0.150799,0.115161,0.27319,0.96459,0.959924,0.267,0.578575,0.723343,0.634558,0.0123567,0.430056,667.751,1152.81,2057.49,867.703,2404.87,2430.38,640.794,924.742,2395.8,1970.27,975.718,1754.31,1972.17,2355.63,1049.92,2294.43,1820.67,1364.07,1704.04,1197.17,-0.955496,-0.45132,5.44225,5.95068,6.5345,5.60811,6.69318,6.51725,5.15855,5.75366,6.76372,6.56237,5.8757,6.39104,6.19605,6.37584,5.88047,6.52162,6.22501,5.97635,6.47968,5.93813 +0,1,nan,1,1,0,0,20,520,398,193,573,320,629,696,751,264,756,387,194,618,676,328,582,669,605,282,264,0.642333,0.833253,0.857994,0.305681,0.468622,0.367346,0.710057,0.401176,0.661409,0.367746,0.94235,0.45471,0.647151,0.573432,0.0837689,0.524375,0.10298,0.273974,0.137247,0.286743,1824.21,1568.67,682.327,1878.44,956.974,2056.82,2282.62,2406.95,904.321,2469.24,1316.38,587.644,2063.32,2247.18,1004.92,1852.69,2108.08,1861.72,886.321,884.768,-1.07938,-0.206905,6.29661,6.10619,5.2686,6.39556,5.68743,6.47353,6.50678,6.62373,5.59095,6.65619,5.90828,5.20266,6.41879,6.5194,5.81595,6.33652,6.55284,6.39318,5.6793,5.64661 +0,1,nan,1,1,0,0,20,5065,5430,7191,6283,6414,3632,1735,5402,7859,11936,9803,7821,3470,2083,3321,2263,3216,9885,4563,2509,0.920656,0.158449,0.390712,0.405929,0.331266,0.711116,0.311256,0.355932,0.558372,0.910006,0.768799,0.648421,0.0940245,0.363034,0.518058,0.214666,0.289935,0.922099,0.733832,0.617788,968.55,2081.5,2267.26,1927.55,2100.32,868.207,588.792,1741.79,2099.99,2300.89,2193.62,1978.14,1435.75,674.441,931.018,836.411,1121.44,1885.75,1030.68,663.784,0.791203,0.933171,8.52613,8.57991,8.88213,8.73401,8.75018,8.22123,7.45973,8.58602,8.96195,9.38144,9.20193,8.9862,8.14838,7.64386,8.11092,7.72064,8.08413,9.19376,8.41397,7.86566 +0,1,nan,1,1,0,0,20,2958,2766,4919,3646,3619,5429,2036,5697,3025,4375,4913,1071,3984,5770,5118,3412,4555,2370,1181,2524,0.199729,0.360703,0.332594,0.935477,0.668958,0.344132,0.66818,0.0151425,0.74721,0.185884,0.414354,0.956027,0.673805,0.0430132,0.377004,0.0976181,0.0499394,0.336726,0.360232,0.066569,1295.6,1295.02,2175.96,2277.76,1966.18,2463,1101.97,2208.17,1711.44,1900.26,2326.59,681.441,2229.23,2292.45,2418.69,1376.38,1855.4,1062.45,553.537,989.806,0.941747,-0.505359,8.00754,7.92574,8.45889,8.19994,8.18753,8.57697,7.60893,8.63401,8.00923,8.39755,8.48451,6.98282,8.31065,8.65738,8.54221,8.11962,8.44237,7.73991,7.07603,7.80561 +0,1,nan,1,1,0,0,20,8494,11767,40827,15690,20738,14712,37805,3657,76073,11545,16810,76735,7015,79119,74287,42322,4718,7940,4140,3585,0.148907,0.14529,0.70836,0.210288,0.267326,0.10764,0.60113,0.143739,0.883699,0.0436983,0.503369,0.942939,0.370506,0.778301,0.763357,0.846115,0.0604372,0.216743,0.135714,0.0830512,1244.44,1733.29,1315.54,1957.69,2228.1,2401.29,1653.07,549.871,1526.27,2256.84,941.775,1315.95,557.238,2114.42,2084.74,949.214,887.355,959.625,631.634,620.68,1.5173,2.69985,9.04577,9.36734,10.6118,9.66456,9.94794,9.59167,10.5506,8.21506,11.2337,9.357,9.72409,11.2454,8.8406,11.2751,11.2206,10.6573,8.46872,8.96901,8.33202,8.17234 +0,1,nan,1,1,0,0,20,353,721,1184,2083,1408,940,1196,2050,1604,888,945,840,549,1215,338,1474,842,1113,731,505,0.158258,0.930172,0.765032,0.953583,0.608556,0.632396,0.874398,0.85779,0.379114,0.528139,0.381486,0.853459,0.709751,0.863566,0.308674,0.525423,0.392872,0.89309,0.220937,0.535278,539.495,773.249,1412.43,2331.33,1781.06,1164.44,1387.21,2292.19,2357.2,1180.22,1438.64,1022.14,670.557,1370.7,524.2,2008.28,1246.56,1234.07,1142.53,645.949,-0.549979,0.484815,5.81738,6.55158,7.07398,7.66653,7.23002,6.81661,7.10899,7.60315,7.39905,6.77953,6.90642,6.79344,6.30223,7.09177,5.86154,7.30979,6.76863,7.00107,6.59814,6.18025 +# +# Elapsed Time: 0 seconds (Warm-up) +# 0.001 seconds (Sampling) +# 0.001 seconds (Total) +# From 1c9c66d4b7e8b50787d95a0632692040529750a5 Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Wed, 27 Nov 2024 14:43:00 -0500 Subject: [PATCH 02/12] fix, unit tests passing --- src/stan/mcmc/chainset.hpp | 18 +++++++++++++++--- src/test/unit/mcmc/chainset_test.cpp | 2 +- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/src/stan/mcmc/chainset.hpp b/src/stan/mcmc/chainset.hpp index 9eb9369621..e380504ecc 100644 --- a/src/stan/mcmc/chainset.hpp +++ b/src/stan/mcmc/chainset.hpp @@ -299,10 +299,15 @@ class chainset { * @return parameter value at quantile */ double quantile(const int index, const double prob) const { - // Ensure the probability is within [0, 1] Eigen::MatrixXd draws = samples(index); Eigen::Map map(draws.data(), draws.size()); - return stan::math::quantile(map, prob); + double result; + try { + result = stan::math::quantile(map, prob); + } catch (const std::logic_error& e) { + return std::numeric_limits::quiet_NaN(); + } + return result; } /** @@ -332,7 +337,14 @@ class chainset { Eigen::MatrixXd draws = samples(index); Eigen::Map map(draws.data(), draws.size()); std::vector probs_vec(probs.data(), probs.data() + probs.size()); - std::vector quantiles = stan::math::quantile(map, probs_vec); + std::vector quantiles; + try { + quantiles = stan::math::quantile(map, probs_vec); + } catch (const std::logic_error& e) { + Eigen::VectorXd nans(probs.size()); + nans.setConstant(std::numeric_limits::quiet_NaN()); + return nans; + } return Eigen::Map(quantiles.data(), quantiles.size()); } diff --git a/src/test/unit/mcmc/chainset_test.cpp b/src/test/unit/mcmc/chainset_test.cpp index 01cb18da42..0c26069876 100644 --- a/src/test/unit/mcmc/chainset_test.cpp +++ b/src/test/unit/mcmc/chainset_test.cpp @@ -220,6 +220,6 @@ TEST_F(McmcChains, quantile_tests) { probs << 0.0, 0.01, 0.05, 0.95, 0.99, 1.0; Eigen::VectorXd stepsize_quantiles = datagen_chains.quantiles("stepsize__", probs); for (size_t i = 0; i < probs.size(); ++i) { - EXPECT_EQ(stepsize_quantiles(i), std::numeric_limits::quiet_NaN()); + EXPECT_TRUE(std::isnan(stepsize_quantiles(i))); } } From bb0d6473edfdb26af2477cf9bcd94ef91f13b9cd Mon Sep 17 00:00:00 2001 From: Stan Jenkins Date: Wed, 27 Nov 2024 14:53:36 -0500 Subject: [PATCH 03/12] [Jenkins] auto-formatting by clang-format version 10.0.0-4ubuntu1 --- src/test/unit/mcmc/chainset_test.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/test/unit/mcmc/chainset_test.cpp b/src/test/unit/mcmc/chainset_test.cpp index 0c26069876..32e6f92cf2 100644 --- a/src/test/unit/mcmc/chainset_test.cpp +++ b/src/test/unit/mcmc/chainset_test.cpp @@ -211,14 +211,16 @@ TEST_F(McmcChains, summary_stats) { TEST_F(McmcChains, quantile_tests) { std::ifstream datagen_stream; datagen_stream.open("src/test/unit/mcmc/test_csv_files/datagen_output.csv", - std::ifstream::in); - stan::io::stan_csv datagen_csv = stan::io::stan_csv_reader::parse(datagen_stream, &out); + std::ifstream::in); + stan::io::stan_csv datagen_csv + = stan::io::stan_csv_reader::parse(datagen_stream, &out); datagen_stream.close(); stan::mcmc::chainset datagen_chains(datagen_csv); - + Eigen::VectorXd probs(6); probs << 0.0, 0.01, 0.05, 0.95, 0.99, 1.0; - Eigen::VectorXd stepsize_quantiles = datagen_chains.quantiles("stepsize__", probs); + Eigen::VectorXd stepsize_quantiles + = datagen_chains.quantiles("stepsize__", probs); for (size_t i = 0; i < probs.size(); ++i) { EXPECT_TRUE(std::isnan(stepsize_quantiles(i))); } From d43e4e245571349f2524b5a44840536e1b725b6f Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Wed, 27 Nov 2024 16:30:27 -0500 Subject: [PATCH 04/12] check is_fininte_and_varies before calling quantiles --- src/stan/mcmc/chainset.hpp | 40 +++++++++++++++------------- src/test/unit/mcmc/chainset_test.cpp | 8 ++++++ 2 files changed, 30 insertions(+), 18 deletions(-) diff --git a/src/stan/mcmc/chainset.hpp b/src/stan/mcmc/chainset.hpp index e380504ecc..3d8ac4fcd6 100644 --- a/src/stan/mcmc/chainset.hpp +++ b/src/stan/mcmc/chainset.hpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -288,11 +289,10 @@ class chainset { /** * Compute the quantile value of the specified parameter - * at the specified probability. - * - * Calls stan::math::quantile which throws - * std::invalid_argument If any element of samples_vec is NaN, or size 0. - * and std::domain_error If `p<0` or `p>1`. + * at the specified probability via a call to stan::math::quantile. + * + * If the probability is outside of interval [0, 1], or if the + * sample contains Infs or NaN, will return NaN. * * @param index parameter index * @param prob probability @@ -300,14 +300,11 @@ class chainset { */ double quantile(const int index, const double prob) const { Eigen::MatrixXd draws = samples(index); - Eigen::Map map(draws.data(), draws.size()); - double result; - try { - result = stan::math::quantile(map, prob); - } catch (const std::logic_error& e) { + if (prob < 0 || prob > 1 || !analyze::is_finite_and_varies(draws)) return std::numeric_limits::quiet_NaN(); - } - return result; + + Eigen::Map map(draws.data(), draws.size()); + return stan::math::quantile(map, prob); } /** @@ -335,16 +332,23 @@ class chainset { if (probs.size() == 0) return Eigen::VectorXd::Zero(0); Eigen::MatrixXd draws = samples(index); - Eigen::Map map(draws.data(), draws.size()); - std::vector probs_vec(probs.data(), probs.data() + probs.size()); - std::vector quantiles; - try { - quantiles = stan::math::quantile(map, probs_vec); - } catch (const std::logic_error& e) { + + bool probs_OK = true; + for (int i = 0; i < probs.size(); ++i) { + if (probs(i) < 0 || probs(i) > 1) { + probs_OK = false; + break; + } + } + if (!(probs_OK && analyze::is_finite_and_varies(draws))) { Eigen::VectorXd nans(probs.size()); nans.setConstant(std::numeric_limits::quiet_NaN()); return nans; } + + Eigen::Map map(draws.data(), draws.size()); + std::vector probs_vec(probs.data(), probs.data() + probs.size()); + std::vector quantiles = stan::math::quantile(map, probs_vec); return Eigen::Map(quantiles.data(), quantiles.size()); } diff --git a/src/test/unit/mcmc/chainset_test.cpp b/src/test/unit/mcmc/chainset_test.cpp index 0c26069876..822d98f824 100644 --- a/src/test/unit/mcmc/chainset_test.cpp +++ b/src/test/unit/mcmc/chainset_test.cpp @@ -222,4 +222,12 @@ TEST_F(McmcChains, quantile_tests) { for (size_t i = 0; i < probs.size(); ++i) { EXPECT_TRUE(std::isnan(stepsize_quantiles(i))); } + + Eigen::VectorXd bad_probs(3); + bad_probs << 5, 50, 95; + Eigen::VectorXd y_sim_quantiles = datagen_chains.quantiles("y_sim[1]", bad_probs); + for (size_t i = 0; i < bad_probs.size(); ++i) { + EXPECT_TRUE(std::isnan(y_sim_quantiles(i))); + } + } From 776d2d05b2ef49e66e3497e6e989151e1d4c7981 Mon Sep 17 00:00:00 2001 From: Stan Jenkins Date: Wed, 27 Nov 2024 16:31:02 -0500 Subject: [PATCH 05/12] [Jenkins] auto-formatting by clang-format version 10.0.0-4ubuntu1 --- src/stan/mcmc/chainset.hpp | 6 +++--- src/test/unit/mcmc/chainset_test.cpp | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/stan/mcmc/chainset.hpp b/src/stan/mcmc/chainset.hpp index 3d8ac4fcd6..81c295ecd6 100644 --- a/src/stan/mcmc/chainset.hpp +++ b/src/stan/mcmc/chainset.hpp @@ -290,7 +290,7 @@ class chainset { /** * Compute the quantile value of the specified parameter * at the specified probability via a call to stan::math::quantile. - * + * * If the probability is outside of interval [0, 1], or if the * sample contains Infs or NaN, will return NaN. * @@ -336,8 +336,8 @@ class chainset { bool probs_OK = true; for (int i = 0; i < probs.size(); ++i) { if (probs(i) < 0 || probs(i) > 1) { - probs_OK = false; - break; + probs_OK = false; + break; } } if (!(probs_OK && analyze::is_finite_and_varies(draws))) { diff --git a/src/test/unit/mcmc/chainset_test.cpp b/src/test/unit/mcmc/chainset_test.cpp index 6d26fa1bd8..10cc03fe95 100644 --- a/src/test/unit/mcmc/chainset_test.cpp +++ b/src/test/unit/mcmc/chainset_test.cpp @@ -227,9 +227,9 @@ TEST_F(McmcChains, quantile_tests) { Eigen::VectorXd bad_probs(3); bad_probs << 5, 50, 95; - Eigen::VectorXd y_sim_quantiles = datagen_chains.quantiles("y_sim[1]", bad_probs); + Eigen::VectorXd y_sim_quantiles + = datagen_chains.quantiles("y_sim[1]", bad_probs); for (size_t i = 0; i < bad_probs.size(); ++i) { EXPECT_TRUE(std::isnan(y_sim_quantiles(i))); } - } From 32fd254856eb2493d20f2a0c9e5319daa0ba78d3 Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Wed, 27 Nov 2024 17:27:06 -0500 Subject: [PATCH 06/12] csv_reader more robust --- src/stan/io/stan_csv_reader.hpp | 6 +++++- src/test/unit/io/stan_csv_reader_test.cpp | 11 +++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/stan/io/stan_csv_reader.hpp b/src/stan/io/stan_csv_reader.hpp index 0ca732e32b..4956794f1f 100644 --- a/src/stan/io/stan_csv_reader.hpp +++ b/src/stan/io/stan_csv_reader.hpp @@ -334,7 +334,11 @@ class stan_csv_reader { for (int col = 0; col < cols; col++) { std::getline(ls, line, ','); boost::trim(line); - std::stringstream(line) >> samples(row, col); + try { + samples(row, col) = std::stod(line); + } catch (const std::out_of_range &e) { + samples(row, col) = std::numeric_limits::quiet_NaN(); + } } } } diff --git a/src/test/unit/io/stan_csv_reader_test.cpp b/src/test/unit/io/stan_csv_reader_test.cpp index 9a84ce8ccd..f913615732 100644 --- a/src/test/unit/io/stan_csv_reader_test.cpp +++ b/src/test/unit/io/stan_csv_reader_test.cpp @@ -604,3 +604,14 @@ TEST_F(StanIoStanCsvReader, variational) { ASSERT_EQ(1000, variational.metadata.num_samples); ASSERT_EQ(0, variational.adaptation.metric.size()); } + +TEST_F(StanIoStanCsvReader, read_nans) { + std::ifstream datagen_stream; + datagen_stream.open("src/test/unit/mcmc/test_csv_files/datagen_output.csv", + std::ifstream::in); + std::stringstream out; + stan::io::stan_csv datagen + = stan::io::stan_csv_reader::parse(datagen_stream, &out); + datagen_stream.close(); + ASSERT_TRUE(std::isnan(datagen.samples(0, 2))); +} From b044f2c72a2ff5e8192147d92c3cdabf1b305fc7 Mon Sep 17 00:00:00 2001 From: Stan Jenkins Date: Wed, 27 Nov 2024 17:27:26 -0500 Subject: [PATCH 07/12] [Jenkins] auto-formatting by clang-format version 10.0.0-4ubuntu1 --- src/stan/io/stan_csv_reader.hpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/stan/io/stan_csv_reader.hpp b/src/stan/io/stan_csv_reader.hpp index 4956794f1f..29a813db8b 100644 --- a/src/stan/io/stan_csv_reader.hpp +++ b/src/stan/io/stan_csv_reader.hpp @@ -334,11 +334,11 @@ class stan_csv_reader { for (int col = 0; col < cols; col++) { std::getline(ls, line, ','); boost::trim(line); - try { - samples(row, col) = std::stod(line); - } catch (const std::out_of_range &e) { - samples(row, col) = std::numeric_limits::quiet_NaN(); - } + try { + samples(row, col) = std::stod(line); + } catch (const std::out_of_range& e) { + samples(row, col) = std::numeric_limits::quiet_NaN(); + } } } } From 50d10ed9ddb9575c769da40cb47d21551f856a71 Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Wed, 27 Nov 2024 17:39:52 -0500 Subject: [PATCH 08/12] use stold, downcast --- src/stan/io/stan_csv_reader.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/stan/io/stan_csv_reader.hpp b/src/stan/io/stan_csv_reader.hpp index 4956794f1f..31b33a39bf 100644 --- a/src/stan/io/stan_csv_reader.hpp +++ b/src/stan/io/stan_csv_reader.hpp @@ -335,7 +335,8 @@ class stan_csv_reader { std::getline(ls, line, ','); boost::trim(line); try { - samples(row, col) = std::stod(line); + auto raw = std::stold(line); + samples(row, col) = static_cast(raw); } catch (const std::out_of_range &e) { samples(row, col) = std::numeric_limits::quiet_NaN(); } From 02a20829da3ef8cc589d625497e61e3b93579bdf Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Wed, 27 Nov 2024 17:48:38 -0500 Subject: [PATCH 09/12] changes per code review --- src/stan/mcmc/chainset.hpp | 41 ++++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/src/stan/mcmc/chainset.hpp b/src/stan/mcmc/chainset.hpp index 81c295ecd6..a6793d51ef 100644 --- a/src/stan/mcmc/chainset.hpp +++ b/src/stan/mcmc/chainset.hpp @@ -291,8 +291,10 @@ class chainset { * Compute the quantile value of the specified parameter * at the specified probability via a call to stan::math::quantile. * - * If the probability is outside of interval [0, 1], or if the - * sample contains Infs or NaN, will return NaN. + * Calls stan::math::quantile which throws + * std::invalid_argument If any element of samples_vec is NaN, or size 0. + * and std::domain_error If `p<0` or `p>1`. + * If this happens, error will be caught, quantile value is NaN. * * @param index parameter index * @param prob probability @@ -300,11 +302,14 @@ class chainset { */ double quantile(const int index, const double prob) const { Eigen::MatrixXd draws = samples(index); - if (prob < 0 || prob > 1 || !analyze::is_finite_and_varies(draws)) - return std::numeric_limits::quiet_NaN(); - Eigen::Map map(draws.data(), draws.size()); - return stan::math::quantile(map, prob); + double result; + try { + result = stan::math::quantile(map, prob); + } catch (const std::logic_error& e) { + return std::numeric_limits::quiet_NaN(); + } + return result; } /** @@ -323,6 +328,11 @@ class chainset { * Compute the quantile values of the specified parameter * for a set of specified probabilities. * + * Calls stan::math::quantile which throws + * std::invalid_argument If any element of samples_vec is NaN, or size 0. + * and std::domain_error If `p<0` or `p>1`. + * If this happens, error will be caught, quantile value is NaN. + * * @param index parameter index * @param probs vector of probabilities * @return vector of parameter values for quantiles @@ -332,23 +342,16 @@ class chainset { if (probs.size() == 0) return Eigen::VectorXd::Zero(0); Eigen::MatrixXd draws = samples(index); - - bool probs_OK = true; - for (int i = 0; i < probs.size(); ++i) { - if (probs(i) < 0 || probs(i) > 1) { - probs_OK = false; - break; - } - } - if (!(probs_OK && analyze::is_finite_and_varies(draws))) { + Eigen::Map map(draws.data(), draws.size()); + std::vector probs_vec(probs.data(), probs.data() + probs.size()); + std::vector quantiles; + try { + quantiles = stan::math::quantile(map, probs_vec); + } catch (const std::logic_error& e) { Eigen::VectorXd nans(probs.size()); nans.setConstant(std::numeric_limits::quiet_NaN()); return nans; } - - Eigen::Map map(draws.data(), draws.size()); - std::vector probs_vec(probs.data(), probs.data() + probs.size()); - std::vector quantiles = stan::math::quantile(map, probs_vec); return Eigen::Map(quantiles.data(), quantiles.size()); } From fa3a6fa2d0ebfefdd0e046ccdae921aa539ab007 Mon Sep 17 00:00:00 2001 From: Stan Jenkins Date: Wed, 27 Nov 2024 17:49:02 -0500 Subject: [PATCH 10/12] [Jenkins] auto-formatting by clang-format version 10.0.0-4ubuntu1 --- src/stan/io/stan_csv_reader.hpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/stan/io/stan_csv_reader.hpp b/src/stan/io/stan_csv_reader.hpp index 31b33a39bf..36e5ec551d 100644 --- a/src/stan/io/stan_csv_reader.hpp +++ b/src/stan/io/stan_csv_reader.hpp @@ -334,12 +334,12 @@ class stan_csv_reader { for (int col = 0; col < cols; col++) { std::getline(ls, line, ','); boost::trim(line); - try { - auto raw = std::stold(line); - samples(row, col) = static_cast(raw); - } catch (const std::out_of_range &e) { - samples(row, col) = std::numeric_limits::quiet_NaN(); - } + try { + auto raw = std::stold(line); + samples(row, col) = static_cast(raw); + } catch (const std::out_of_range& e) { + samples(row, col) = std::numeric_limits::quiet_NaN(); + } } } } From 1badfb727e796167d51a93dc5cdf84f23befb46d Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Thu, 28 Nov 2024 13:17:27 -0500 Subject: [PATCH 11/12] changes per code review --- src/stan/io/stan_csv_reader.hpp | 3 +-- src/stan/mcmc/chainset.hpp | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/stan/io/stan_csv_reader.hpp b/src/stan/io/stan_csv_reader.hpp index 31b33a39bf..6cdb1906c0 100644 --- a/src/stan/io/stan_csv_reader.hpp +++ b/src/stan/io/stan_csv_reader.hpp @@ -335,8 +335,7 @@ class stan_csv_reader { std::getline(ls, line, ','); boost::trim(line); try { - auto raw = std::stold(line); - samples(row, col) = static_cast(raw); + samples(row, col) = static_cast(std::stold(line)); } catch (const std::out_of_range &e) { samples(row, col) = std::numeric_limits::quiet_NaN(); } diff --git a/src/stan/mcmc/chainset.hpp b/src/stan/mcmc/chainset.hpp index a6793d51ef..8ff5644693 100644 --- a/src/stan/mcmc/chainset.hpp +++ b/src/stan/mcmc/chainset.hpp @@ -5,7 +5,6 @@ #include #include #include -#include #include #include #include From 5c13934e28eee9616aa36fd92954f17c649fc8dc Mon Sep 17 00:00:00 2001 From: Stan Jenkins Date: Thu, 28 Nov 2024 13:18:30 -0500 Subject: [PATCH 12/12] [Jenkins] auto-formatting by clang-format version 10.0.0-4ubuntu1 --- src/stan/io/stan_csv_reader.hpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/stan/io/stan_csv_reader.hpp b/src/stan/io/stan_csv_reader.hpp index 6cdb1906c0..d02aabcbac 100644 --- a/src/stan/io/stan_csv_reader.hpp +++ b/src/stan/io/stan_csv_reader.hpp @@ -334,11 +334,11 @@ class stan_csv_reader { for (int col = 0; col < cols; col++) { std::getline(ls, line, ','); boost::trim(line); - try { - samples(row, col) = static_cast(std::stold(line)); - } catch (const std::out_of_range &e) { - samples(row, col) = std::numeric_limits::quiet_NaN(); - } + try { + samples(row, col) = static_cast(std::stold(line)); + } catch (const std::out_of_range& e) { + samples(row, col) = std::numeric_limits::quiet_NaN(); + } } } }