From 86b8d44308b421d3afcb0bccaf1594ed8b0d080d Mon Sep 17 00:00:00 2001 From: jgabry Date: Wed, 31 Jul 2024 12:31:22 -0600 Subject: [PATCH 1/3] Document problems with `ppc_stat` with `stat="mean"` closes #321 --- R/ppc-test-statistics.R | 15 ++++++++++----- man/PPC-test-statistics.Rd | 11 ++++++++--- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/R/ppc-test-statistics.R b/R/ppc-test-statistics.R index a9b6da86..11b9827b 100644 --- a/R/ppc-test-statistics.R +++ b/R/ppc-test-statistics.R @@ -1,10 +1,15 @@ #' PPC test statistics #' -#' The distribution of a (test) statistic `T(yrep)`, or a pair of (test) -#' statistics, over the simulated datasets in `yrep`, compared to the -#' observed value `T(y)` computed from the data `y`. See the -#' **Plot Descriptions** and **Details** sections, below, as -#' well as [Gabry et al. (2019)](https://github.com/jgabry/bayes-vis-paper#readme). +#' @description The distribution of a (test) statistic `T(yrep)`, or a pair of +#' (test) statistics, over the simulated datasets in `yrep`, compared to the +#' observed value `T(y)` computed from the data `y`. See the +#' **Plot Descriptions** and **Details** sections, below, as +#' well as Gabry et al. (2019). +#' +#' **NOTE:** Although the default test statistic +#' is the mean, this is unlikely to detect anything interesting in most cases. +#' In general we recommend using some other test statistic as discussed in +#' Section 5 of Gabry et al. (2019). #' #' @name PPC-test-statistics #' @aliases PPC-statistics diff --git a/man/PPC-test-statistics.Rd b/man/PPC-test-statistics.Rd index a96c3ed5..c934e4b3 100644 --- a/man/PPC-test-statistics.Rd +++ b/man/PPC-test-statistics.Rd @@ -118,11 +118,16 @@ customized using the \strong{ggplot2} package. The functions with suffix function. } \description{ -The distribution of a (test) statistic \code{T(yrep)}, or a pair of (test) -statistics, over the simulated datasets in \code{yrep}, compared to the +The distribution of a (test) statistic \code{T(yrep)}, or a pair of +(test) statistics, over the simulated datasets in \code{yrep}, compared to the observed value \code{T(y)} computed from the data \code{y}. See the \strong{Plot Descriptions} and \strong{Details} sections, below, as -well as \href{https://github.com/jgabry/bayes-vis-paper#readme}{Gabry et al. (2019)}. +well as Gabry et al. (2019). + +\strong{NOTE:} Although the default test statistic +is the mean, this is unlikely to detect anything interesting in most cases. +In general we recommend using some other test statistic as discussed in +Section 5 of Gabry et al. (2019). } \details{ For Binomial data, the plots may be more useful if From d103d425b5e77cd94775fe012faf72e5d4c6f286 Mon Sep 17 00:00:00 2001 From: jgabry Date: Thu, 1 Aug 2024 17:21:07 -0600 Subject: [PATCH 2/3] add message if stat="mean" --- R/ppc-test-statistics.R | 19 +++++++++++--- tests/testthat/test-ppc-test-statistics.R | 31 +++++++++++++++++++++++ 2 files changed, 47 insertions(+), 3 deletions(-) diff --git a/R/ppc-test-statistics.R b/R/ppc-test-statistics.R index 11b9827b..a1b8960c 100644 --- a/R/ppc-test-statistics.R +++ b/R/ppc-test-statistics.R @@ -59,7 +59,7 @@ #' @examples #' y <- example_y_data() #' yrep <- example_yrep_draws() -#' ppc_stat(y, yrep) +#' ppc_stat(y, yrep, stat = "median") #' ppc_stat(y, yrep, stat = "sd") + legend_none() #' #' # use your own function for the 'stat' argument @@ -74,8 +74,8 @@ #' # plots by group #' color_scheme_set("teal") #' group <- example_group_data() -#' ppc_stat_grouped(y, yrep, group) -#' ppc_stat_grouped(y, yrep, group) + yaxis_text() +#' ppc_stat_grouped(y, yrep, group, stat = "median") +#' ppc_stat_grouped(y, yrep, group, stat = "mad") + yaxis_text() #' #' # force y-axes to have same scales, allow x axis to vary #' ppc_stat_grouped(y, yrep, group, facet_args = list(scales = "free_x")) + yaxis_text() @@ -111,6 +111,7 @@ ppc_stat <- breaks = NULL, freq = TRUE) { stopifnot(length(stat) == 1) + message_if_using_mean(stat) dots <- list(...) if (!from_grouped(dots)) { check_ignored_arguments(...) @@ -194,6 +195,7 @@ ppc_stat_freqpoly <- bins = NULL, freq = TRUE) { stopifnot(length(stat) == 1) + message_if_using_mean(stat) dots <- list(...) if (!from_grouped(dots)) { check_ignored_arguments(...) @@ -275,6 +277,8 @@ ppc_stat_2d <- function(y, if (length(stat) != 2) { abort("For ppc_stat_2d the 'stat' argument must have length 2.") } + message_if_using_mean(stat[1]) + message_if_using_mean(stat[2]) if (is.character(stat)) { lgnd_title <- bquote(italic(T) == (list(.(stat[1]), .(stat[2])))) @@ -410,3 +414,12 @@ stat_2d_segment_data <- function(data) { Ty_label <- function() expression(italic(T(italic(y)))) Tyrep_label <- function() expression(italic(T)(italic(y)[rep])) + +message_if_using_mean <- function(stat) { + if (is.character(stat) && stat == "mean") { + message( + "Note: in most cases the default test statistic 'mean' is ", + "too weak to detect anything of interest." + ) + } +} diff --git a/tests/testthat/test-ppc-test-statistics.R b/tests/testthat/test-ppc-test-statistics.R index ab8b7166..a0e8e094 100644 --- a/tests/testthat/test-ppc-test-statistics.R +++ b/tests/testthat/test-ppc-test-statistics.R @@ -24,6 +24,37 @@ test_that("ppc_stat throws errors if 'stat' wrong length", { "length(stat) == 1 is not TRUE", fixed = TRUE) }) +test_that("ppc_stat and ppc_stat_freqpoly message if stat='mean'", { + expect_message( + ppc_stat(y, yrep), + "'mean' is too weak to detect anything of interest" + ) + expect_silent( + ppc_stat(y, yrep, stat = "mad") + ) + expect_message( + ppc_stat_grouped(y, yrep, group), + "'mean' is too weak to detect anything of interest" + ) + expect_silent( + ppc_stat_grouped(y, yrep, group, stat = "mad") + ) + expect_message( + ppc_stat_freqpoly(y, yrep), + "'mean' is too weak to detect anything of interest" + ) + expect_silent( + ppc_stat_freqpoly(y, yrep, group, stat = "mad") + ) + expect_message( + ppc_stat_freqpoly_grouped(y, yrep, group), + "'mean' is too weak to detect anything of interest" + ) + expect_silent( + ppc_stat_freqpoly_grouped(y, yrep, group, stat = "mad") + ) +}) + test_that("ppc_stat returns ggplot object", { expect_gg(ppc_stat(y, yrep, binwidth = 0.05)) expect_gg(ppc_stat(y, yrep, stat = "sd", binwidth = 0.05)) From 194daed715d64b83777ae537863d69fb01e044aa Mon Sep 17 00:00:00 2001 From: jgabry Date: Fri, 2 Aug 2024 08:58:46 -0600 Subject: [PATCH 3/3] Update PPC-test-statistics.Rd --- man/PPC-test-statistics.Rd | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/man/PPC-test-statistics.Rd b/man/PPC-test-statistics.Rd index c934e4b3..83883979 100644 --- a/man/PPC-test-statistics.Rd +++ b/man/PPC-test-statistics.Rd @@ -160,7 +160,7 @@ statistics in the observed data is overlaid as large point. \examples{ y <- example_y_data() yrep <- example_yrep_draws() -ppc_stat(y, yrep) +ppc_stat(y, yrep, stat = "median") ppc_stat(y, yrep, stat = "sd") + legend_none() # use your own function for the 'stat' argument @@ -175,8 +175,8 @@ ppc_stat(y, yrep, stat = function(y) quantile(y, 0.25)) # plots by group color_scheme_set("teal") group <- example_group_data() -ppc_stat_grouped(y, yrep, group) -ppc_stat_grouped(y, yrep, group) + yaxis_text() +ppc_stat_grouped(y, yrep, group, stat = "median") +ppc_stat_grouped(y, yrep, group, stat = "mad") + yaxis_text() # force y-axes to have same scales, allow x axis to vary ppc_stat_grouped(y, yrep, group, facet_args = list(scales = "free_x")) + yaxis_text()