From 6a884c3903572287ee1b1638c8b516bca4dfb08a Mon Sep 17 00:00:00 2001 From: jgabry Date: Mon, 22 Apr 2024 14:28:20 -0600 Subject: [PATCH 1/4] debug inits tests --- tests/testthat/test-fit-init-debug.R | 107 +++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 tests/testthat/test-fit-init-debug.R diff --git a/tests/testthat/test-fit-init-debug.R b/tests/testthat/test-fit-init-debug.R new file mode 100644 index 00000000..6dc1947d --- /dev/null +++ b/tests/testthat/test-fit-init-debug.R @@ -0,0 +1,107 @@ +context("fitted-inits") +set_cmdstan_path() + + +data_list_logistic <- testing_data("logistic") + +test_inits <- function(mod, fit_init, data_list = NULL) { + + # override fit_init so that all the code below runs + fit_init <- function() { + list(alpha = rnorm(1), beta = rnorm(3)) + } + + Sys.sleep(1) + fit_sample <- mod$sample(data = data_list, chains = 1, init = fit_init, + iter_sampling = 100, iter_warmup = 100, refresh = 0, seed = 1234) + Sys.sleep(1) + fit_sample_multi <- mod$sample(data = data_list, chains = 5, init = fit_init, + iter_sampling = 100, iter_warmup = 100, refresh = 0, seed = 1234) + Sys.sleep(1) + fit_vb <- mod$variational(data = data_list, refresh = 0, seed = 1234, + init = fit_init, algorithm = "fullrank") + Sys.sleep(1) + fit_path <- mod$pathfinder(data = data_list, seed=1234, refresh = 0, + num_paths = 4, init = fit_init) + Sys.sleep(1) + fit_laplace <- mod$laplace(data = data_list, seed = 1234, refresh=0, + init=fit_init) + Sys.sleep(1) + fit_ml <- mod$optimize(data = data_list, seed = 1234, refresh = 0, + init = fit_init, history_size = 400, algorithm = "lbfgs") + # Sys.sleep(1) + # draws = posterior::as_draws_rvars(fit_init$draws()) + # fit_sample_draws <- mod$sample(data = data_list, chains = 1, init = draws, + # iter_sampling = 100, iter_warmup = 100, refresh = 0, seed = 1234) + # Sys.sleep(1) + return(0) +} + +test_that("Pathfinder works as init", { + mod_logistic <- testing_model("logistic") + utils::capture.output(fit_path_init <- mod_logistic$pathfinder( + seed=1234, data = data_list_logistic, refresh = 0, num_paths = 1)) + expect_no_error(test_inits(mod_logistic, fit_path_init, data_list_logistic)) +}) + +test_that("Multi Pathfinder method works as init", { + set.seed(1234) + mod_logistic <- testing_model("logistic") + utils::capture.output(fit_path_init <- mod_logistic$pathfinder(seed=1234, + data = data_list_logistic, refresh = 0, num_paths = 4)) + expect_no_error(test_inits(mod_logistic, fit_path_init, data_list_logistic)) +}) + +test_that("Pathfinder method with psis_resample as false works as init", { + set.seed(1234) + Sys.sleep(3) + mod_logistic <- testing_model("logistic") + utils::capture.output(fit_path_init <- mod_logistic$pathfinder( + seed=1234, data = data_list_logistic, refresh = 0, num_paths = 1)) + expect_no_error(test_inits(mod_logistic, fit_path_init, data_list_logistic)) +}) + + +test_that("Multi Pathfinder method with psis_resample as false works as init", { + set.seed(1234) + mod_logistic <- testing_model("logistic") + utils::capture.output(fit_path_init <- mod_logistic$pathfinder( + seed=1234, data = data_list_logistic, refresh = 0, num_paths = 4, + psis_resample = FALSE)) + expect_no_error(test_inits(mod_logistic, fit_path_init, data_list_logistic)) +}) + + +test_that("Pathfinder method with calculate_lp as false works as init", { + set.seed(1234) + mod_logistic <- testing_model("logistic") + fit_path_init <- mod_logistic$pathfinder( + seed=1234, data = data_list_logistic, refresh = 0, num_paths = 1, + psis_resample = FALSE, calculate_lp = FALSE) + expect_no_error(test_inits(mod_logistic, fit_path_init, data_list_logistic)) +}) + +test_that("Multi Pathfinder method with calculate_lp as false works as init", { + set.seed(1234) + mod_logistic <- testing_model("logistic") + utils::capture.output(fit_path_init <- mod_logistic$pathfinder( + seed=1234, data = data_list_logistic, refresh = 0, num_paths = 4, + psis_resample = TRUE, calculate_lp = FALSE)) + expect_no_error(test_inits(mod_logistic, fit_path_init, data_list_logistic)) +}) + +test_that("Variational method works as init", { + set.seed(1235) + mod_logistic <- testing_model("logistic") + utils::capture.output(fit_vb_init <- mod_logistic$variational( + data = data_list_logistic, seed=1234, refresh = 0)) + expect_no_error(test_inits(mod_logistic, fit_vb_init, data_list_logistic)) +}) + +test_that("Optimization method works as init", { + set.seed(1234) + mod_logistic <- testing_model("logistic") + utils::capture.output(fit_ml_init <- mod_logistic$optimize( + data = data_list_logistic, seed=1234, refresh = 0)) + expect_no_error(test_inits(mod_logistic, fit_ml_init, data_list_logistic)) +}) From ebe18ce5328ee759a7a46f19aebf5e030b9e9178 Mon Sep 17 00:00:00 2001 From: jgabry Date: Wed, 24 Apr 2024 09:33:00 -0600 Subject: [PATCH 2/4] Update test-fit-init-debug.R --- tests/testthat/test-fit-init-debug.R | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/testthat/test-fit-init-debug.R b/tests/testthat/test-fit-init-debug.R index 6dc1947d..210b68e4 100644 --- a/tests/testthat/test-fit-init-debug.R +++ b/tests/testthat/test-fit-init-debug.R @@ -2,6 +2,7 @@ context("fitted-inits") set_cmdstan_path() + data_list_logistic <- testing_data("logistic") test_inits <- function(mod, fit_init, data_list = NULL) { From d4324ca0b59d32586a9fcb104783d99352bf49e2 Mon Sep 17 00:00:00 2001 From: Steve Bronder Date: Fri, 3 May 2024 15:18:14 -0400 Subject: [PATCH 3/4] MRE for file failure --- tests/testthat/test-fit-init-debug.R | 109 +++------------------------ 1 file changed, 12 insertions(+), 97 deletions(-) diff --git a/tests/testthat/test-fit-init-debug.R b/tests/testthat/test-fit-init-debug.R index 210b68e4..772819de 100644 --- a/tests/testthat/test-fit-init-debug.R +++ b/tests/testthat/test-fit-init-debug.R @@ -3,106 +3,21 @@ set_cmdstan_path() -data_list_logistic <- testing_data("logistic") -test_inits <- function(mod, fit_init, data_list = NULL) { - - # override fit_init so that all the code below runs +test_inits <- function() { fit_init <- function() { list(alpha = rnorm(1), beta = rnorm(3)) } - - Sys.sleep(1) - fit_sample <- mod$sample(data = data_list, chains = 1, init = fit_init, - iter_sampling = 100, iter_warmup = 100, refresh = 0, seed = 1234) - Sys.sleep(1) - fit_sample_multi <- mod$sample(data = data_list, chains = 5, init = fit_init, - iter_sampling = 100, iter_warmup = 100, refresh = 0, seed = 1234) - Sys.sleep(1) - fit_vb <- mod$variational(data = data_list, refresh = 0, seed = 1234, - init = fit_init, algorithm = "fullrank") - Sys.sleep(1) - fit_path <- mod$pathfinder(data = data_list, seed=1234, refresh = 0, - num_paths = 4, init = fit_init) - Sys.sleep(1) - fit_laplace <- mod$laplace(data = data_list, seed = 1234, refresh=0, - init=fit_init) - Sys.sleep(1) - fit_ml <- mod$optimize(data = data_list, seed = 1234, refresh = 0, - init = fit_init, history_size = 400, algorithm = "lbfgs") - # Sys.sleep(1) - # draws = posterior::as_draws_rvars(fit_init$draws()) - # fit_sample_draws <- mod$sample(data = data_list, chains = 1, init = draws, - # iter_sampling = 100, iter_warmup = 100, refresh = 0, seed = 1234) - # Sys.sleep(1) + mod <- testing_model("logistic") + data_list <- testing_data("logistic") + fit_sample_multi <- mod$sample(data = data_list, chains = 2, init = fit_init, + iter_sampling = 200, iter_warmup = 200, refresh = 1, seed = 1234) return(0) } - -test_that("Pathfinder works as init", { - mod_logistic <- testing_model("logistic") - utils::capture.output(fit_path_init <- mod_logistic$pathfinder( - seed=1234, data = data_list_logistic, refresh = 0, num_paths = 1)) - expect_no_error(test_inits(mod_logistic, fit_path_init, data_list_logistic)) -}) - -test_that("Multi Pathfinder method works as init", { - set.seed(1234) - mod_logistic <- testing_model("logistic") - utils::capture.output(fit_path_init <- mod_logistic$pathfinder(seed=1234, - data = data_list_logistic, refresh = 0, num_paths = 4)) - expect_no_error(test_inits(mod_logistic, fit_path_init, data_list_logistic)) -}) - -test_that("Pathfinder method with psis_resample as false works as init", { - set.seed(1234) - Sys.sleep(3) - mod_logistic <- testing_model("logistic") - utils::capture.output(fit_path_init <- mod_logistic$pathfinder( - seed=1234, data = data_list_logistic, refresh = 0, num_paths = 1)) - expect_no_error(test_inits(mod_logistic, fit_path_init, data_list_logistic)) -}) - - -test_that("Multi Pathfinder method with psis_resample as false works as init", { - set.seed(1234) - mod_logistic <- testing_model("logistic") - utils::capture.output(fit_path_init <- mod_logistic$pathfinder( - seed=1234, data = data_list_logistic, refresh = 0, num_paths = 4, - psis_resample = FALSE)) - expect_no_error(test_inits(mod_logistic, fit_path_init, data_list_logistic)) -}) - - -test_that("Pathfinder method with calculate_lp as false works as init", { - set.seed(1234) - mod_logistic <- testing_model("logistic") - fit_path_init <- mod_logistic$pathfinder( - seed=1234, data = data_list_logistic, refresh = 0, num_paths = 1, - psis_resample = FALSE, calculate_lp = FALSE) - expect_no_error(test_inits(mod_logistic, fit_path_init, data_list_logistic)) -}) - -test_that("Multi Pathfinder method with calculate_lp as false works as init", { - set.seed(1234) - mod_logistic <- testing_model("logistic") - utils::capture.output(fit_path_init <- mod_logistic$pathfinder( - seed=1234, data = data_list_logistic, refresh = 0, num_paths = 4, - psis_resample = TRUE, calculate_lp = FALSE)) - expect_no_error(test_inits(mod_logistic, fit_path_init, data_list_logistic)) -}) - -test_that("Variational method works as init", { - set.seed(1235) - mod_logistic <- testing_model("logistic") - utils::capture.output(fit_vb_init <- mod_logistic$variational( - data = data_list_logistic, seed=1234, refresh = 0)) - expect_no_error(test_inits(mod_logistic, fit_vb_init, data_list_logistic)) -}) - -test_that("Optimization method works as init", { - set.seed(1234) - mod_logistic <- testing_model("logistic") - utils::capture.output(fit_ml_init <- mod_logistic$optimize( - data = data_list_logistic, seed=1234, refresh = 0)) - expect_no_error(test_inits(mod_logistic, fit_ml_init, data_list_logistic)) -}) +while (1) { + test_that("Fails when calling sample many times", { + set.seed(1234) + test_inits() + expect_true(TRUE) + }) +} From b8760bda824b887093a147f77166d2ec7502c5c6 Mon Sep 17 00:00:00 2001 From: Steve Bronder Date: Fri, 3 May 2024 15:34:02 -0400 Subject: [PATCH 4/4] remove while around init loop for github workflow testing --- tests/testthat/test-fit-init-debug.R | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/testthat/test-fit-init-debug.R b/tests/testthat/test-fit-init-debug.R index 772819de..68726d07 100644 --- a/tests/testthat/test-fit-init-debug.R +++ b/tests/testthat/test-fit-init-debug.R @@ -14,10 +14,10 @@ test_inits <- function() { iter_sampling = 200, iter_warmup = 200, refresh = 1, seed = 1234) return(0) } -while (1) { +#while (1) { test_that("Fails when calling sample many times", { set.seed(1234) test_inits() expect_true(TRUE) }) -} +#}