From bec0468459293b45dce357fd77139b395935c2c2 Mon Sep 17 00:00:00 2001 From: Henrik Bengtsson Date: Tue, 5 Sep 2023 09:05:12 -0700 Subject: [PATCH 01/36] Bump develop version --- DESCRIPTION | 2 +- NEWS.md | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index aead2cb..fb72e12 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,5 +1,5 @@ Package: future.mirai -Version: 0.1.1 +Version: 0.1.1-9000 Depends: future Imports: diff --git a/NEWS.md b/NEWS.md index 0092052..02c85a5 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,8 @@ +# Version (development version) + + * ... + + # Version 0.1.1 ## Miscelleanous From 9f1d239002858df8b89d1c0b5a50b386dc632602 Mon Sep 17 00:00:00 2001 From: Henrik Bengtsson Date: Fri, 8 Mar 2024 16:40:03 -0800 Subject: [PATCH 02/36] refresh --- .github/workflows/R-CMD-check.yaml | 2 +- .github/workflows/covr.yaml | 4 ++-- .github/workflows/future_tests.yaml | 6 +++--- DESCRIPTION | 6 +++--- R/package.R | 5 +++-- man/future.mirai.Rd | 18 ++++++++++++++++++ 6 files changed, 30 insertions(+), 11 deletions(-) diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml index 701cd6c..4696d7c 100644 --- a/.github/workflows/R-CMD-check.yaml +++ b/.github/workflows/R-CMD-check.yaml @@ -51,7 +51,7 @@ jobs: R_FUTURE_RNG_ONMISUSE: error steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: r-lib/actions/setup-r@v2 with: diff --git a/.github/workflows/covr.yaml b/.github/workflows/covr.yaml index f27ad05..5e909ae 100644 --- a/.github/workflows/covr.yaml +++ b/.github/workflows/covr.yaml @@ -26,7 +26,7 @@ jobs: _R_CHECK_CRAN_INCOMING_: false steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: r-lib/actions/setup-pandoc@v2 @@ -42,7 +42,7 @@ jobs: shell: Rscript {0} - name: Cache R packages - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ${{ env.R_LIBS_USER }} key: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-1-${{ hashFiles('.github/depends.Rds') }} diff --git a/.github/workflows/future_tests.yaml b/.github/workflows/future_tests.yaml index ed92e57..82b70ca 100644 --- a/.github/workflows/future_tests.yaml +++ b/.github/workflows/future_tests.yaml @@ -32,7 +32,7 @@ jobs: R_FUTURE_RNG_ONMISUSE: error steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: r-lib/actions/setup-pandoc@v2 @@ -48,7 +48,7 @@ jobs: shell: Rscript {0} - name: Cache R packages - uses: actions/cache@v2 + uses: actions/cache@v4 with: path: ${{ env.R_LIBS_USER }} key: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-1-${{ hashFiles('.github/depends.Rds') }} @@ -98,7 +98,7 @@ jobs: - name: Upload check results if: failure() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: ${{ runner.os }}-r${{ matrix.future.plan }}-results path: check diff --git a/DESCRIPTION b/DESCRIPTION index fb72e12..f5baf6b 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,9 +1,9 @@ Package: future.mirai -Version: 0.1.1-9000 +Version: 0.1.1-9001 Depends: future Imports: - mirai (>= 0.9.1), + mirai (>= 0.12.1), parallelly, parallel, utils @@ -26,5 +26,5 @@ License: GPL (>= 3) Encoding: UTF-8 URL: https://future.mirai.futureverse.org, https://github.com/HenrikBengtsson/future.mirai BugReports: https://github.com/HenrikBengtsson/future.mirai/issues -RoxygenNote: 7.2.3 +RoxygenNote: 7.3.1 Roxygen: list(markdown = TRUE) diff --git a/R/package.R b/R/package.R index 5e25d91..b33921a 100644 --- a/R/package.R +++ b/R/package.R @@ -8,7 +8,8 @@ #' TRUE #' } #' -#' @docType package #' @aliases future.mirai-package #' @name future.mirai -NULL +"_PACKAGE" + + diff --git a/man/future.mirai.Rd b/man/future.mirai.Rd index 7b5a162..56cd0fc 100644 --- a/man/future.mirai.Rd +++ b/man/future.mirai.Rd @@ -15,3 +15,21 @@ TRUE } } +\seealso{ +Useful links: +\itemize{ + \item \url{https://future.mirai.futureverse.org} + \item \url{https://github.com/HenrikBengtsson/future.mirai} + \item Report bugs at \url{https://github.com/HenrikBengtsson/future.mirai/issues} +} + +} +\author{ +\strong{Maintainer}: Henrik Bengtsson \email{henrikb@braju.com} (\href{https://orcid.org/0000-0002-7579-5165}{ORCID}) [copyright holder] + +Other contributors: +\itemize{ + \item Charlie Gao \email{charlie.gao@shikokuchuo.net} (\href{https://orcid.org/0000-0002-0750-061X}{ORCID}) [contributor] +} + +} From 573f990293b469184bbfc0ef174464c589d8c15d Mon Sep 17 00:00:00 2001 From: Henrik Bengtsson Date: Thu, 4 Apr 2024 08:19:00 -0700 Subject: [PATCH 03/36] GA: Test with more configs --- .github/workflows/R-CMD-check.yaml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml index 4696d7c..e89199c 100644 --- a/.github/workflows/R-CMD-check.yaml +++ b/.github/workflows/R-CMD-check.yaml @@ -16,19 +16,19 @@ jobs: fail-fast: false matrix: config: -# - {os: windows-latest, r: 'devel' } -# - {os: windows-latest, r: 'release' } -# - {os: windows-latest, r: 'oldrel' } -# - {os: macOS-latest, r: 'devel' } -# - {os: macOS-latest, r: 'release' } -# - {os: macOS-latest, r: 'oldrel' } -# - {os: ubuntu-latest, r: 'devel' } -# - {os: ubuntu-latest, r: 'devel' , mirai: 'devel', label: "mirai & nanonext devel" } + - {os: windows-latest, r: 'devel' } + - {os: windows-latest, r: 'release' } + - {os: windows-latest, r: 'oldrel' } + - {os: macOS-latest, r: 'devel' } + - {os: macOS-latest, r: 'release' } + - {os: macOS-latest, r: 'oldrel' } + - {os: ubuntu-latest, r: 'devel' } + - {os: ubuntu-latest, r: 'devel' , mirai: 'devel', label: "mirai & nanonext devel" } - {os: ubuntu-latest, r: 'release' } -# - {os: ubuntu-latest, r: 'oldrel' } + - {os: ubuntu-latest, r: 'oldrel' } # - {os: ubuntu-latest, r: 'oldrel-1' } # - {os: ubuntu-latest, r: 'oldrel-2' } -# - {os: ubuntu-latest, r: '3.6' } + - {os: ubuntu-latest, r: '3.6' } # # - {os: ubuntu-latest, r: 'release' , language: ko, label: ko } # - {os: ubuntu-latest, r: 'release' , language: zh_CN, label: zh_CN } # # - {os: ubuntu-latest, r: 'release' , language: zh_TW, label: zh_TW } From fd54dce1da0b7d2a45c934f857b3044e4ed10319 Mon Sep 17 00:00:00 2001 From: Henrik Bengtsson Date: Thu, 4 Apr 2024 17:01:38 -0700 Subject: [PATCH 04/36] .Rbuildignore: should not use trailing slashed for folders --- .Rbuildignore | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/.Rbuildignore b/.Rbuildignore index dc3284e..52fa684 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -1,12 +1,10 @@ ^[.]gitignore -^[.]github/ -^[.]Rdump/ -^[.]make/ -^docs/ -^pkgdown/ -^incl/ +^[.]github +^[.]make +^docs +^pkgdown +^incl ^Makefile ^CONTRIBUTING.md - ^.*\.Rproj$ ^\.Rproj\.user$ From 56c8c34848d939d3aaa714d4e32668ff6d08e7c4 Mon Sep 17 00:00:00 2001 From: Henrik Bengtsson Date: Thu, 4 Apr 2024 17:08:35 -0700 Subject: [PATCH 05/36] TESTS: Clarify what mirai::daemons(0) does [ci skip] --- tests/mirai_cluster.R | 4 ++-- tests/mirai_multisession.R | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/mirai_cluster.R b/tests/mirai_cluster.R index bb0bfdf..dcae413 100644 --- a/tests/mirai_cluster.R +++ b/tests/mirai_cluster.R @@ -1,5 +1,5 @@ if (requireNamespace("future.tests")) { - mirai::daemons(0) + mirai::daemons(0) ## Reset any daemons running future.tests::check("future.mirai::mirai_cluster", timeout = 30.0, exit_value = FALSE) - mirai::daemons(0) + mirai::daemons(0) ## Reset any daemons running } diff --git a/tests/mirai_multisession.R b/tests/mirai_multisession.R index 146b4bc..f63dd1b 100644 --- a/tests/mirai_multisession.R +++ b/tests/mirai_multisession.R @@ -1,5 +1,5 @@ if (requireNamespace("future.tests")) { - mirai::daemons(0) + mirai::daemons(0) ## Reset any daemons running future.tests::check("future.mirai::mirai_multisession", timeout = 30.0, exit_value = FALSE) - mirai::daemons(0) + mirai::daemons(0) ## Reset any daemons running } From c0f60b01b0c11bc485c92686689dc44ae2a05910 Mon Sep 17 00:00:00 2001 From: Henrik Bengtsson Date: Thu, 4 Apr 2024 21:31:07 -0700 Subject: [PATCH 06/36] GA: error on NOTE --- .github/workflows/R-CMD-check.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml index e89199c..d78db59 100644 --- a/.github/workflows/R-CMD-check.yaml +++ b/.github/workflows/R-CMD-check.yaml @@ -86,3 +86,6 @@ jobs: shell: Rscript {0} - uses: r-lib/actions/check-r-package@v2 + with: + error-on: '"note"' + \ No newline at end of file From 3a829b7f9e901b829f9f8ad22398ef687fcf26b5 Mon Sep 17 00:00:00 2001 From: Henrik Bengtsson Date: Mon, 8 Apr 2024 19:14:40 -0700 Subject: [PATCH 07/36] Prepare code so that mirai's 'dispatcher' argument can be controlled --- .Rbuildignore | 1 + .gitignore | 1 + DESCRIPTION | 2 +- R/MiraiFuture-class.R | 7 +++++-- 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.Rbuildignore b/.Rbuildignore index 52fa684..b6704d3 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -1,6 +1,7 @@ ^[.]gitignore ^[.]github ^[.]make +^[.]local ^docs ^pkgdown ^incl diff --git a/.gitignore b/.gitignore index 8834c02..02b701b 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ *[.]so [.]Rhistory [.]Rdump/ +[.]local/ docs/ .Rproj.user *.Rproj diff --git a/DESCRIPTION b/DESCRIPTION index f5baf6b..013a26c 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,5 +1,5 @@ Package: future.mirai -Version: 0.1.1-9001 +Version: 0.1.1-9002 Depends: future Imports: diff --git a/R/MiraiFuture-class.R b/R/MiraiFuture-class.R index 7d1b8eb..4a9fbc1 100644 --- a/R/MiraiFuture-class.R +++ b/R/MiraiFuture-class.R @@ -42,6 +42,9 @@ MiraiFuture <- function(expr = NULL, if (is.function(workers)) workers <- workers() if (!is.null(workers)) stop_if_not(length(workers) >= 1) + ## FIXME: Expose 'dispatcher' in the API + dispatcher <- TRUE + cluster <- NULL if (is.numeric(workers)) { stop_if_not(length(workers) == 1L, !is.na(workers), workers >= 1) @@ -52,7 +55,7 @@ MiraiFuture <- function(expr = NULL, daemons(n = 0L) } else if (workers != nworkers) { daemons(n = 0L) ## reset is required - daemons(n = workers, dispatcher = TRUE) + daemons(n = workers, dispatcher = dispatcher) } } else if (is.character(workers)) { stop_if_not(length(workers) >= 1L, !anyNA(workers)) @@ -65,7 +68,7 @@ MiraiFuture <- function(expr = NULL, } if (length(workers) != n) { daemons(n = 0L) ## reset is required - daemons(n = length(workers), url = "ws://:0", dispatcher = TRUE) + daemons(n = length(workers), url = "ws://:0", dispatcher = dispatcher) } cluster <- launch_mirai_daemons(workers) } else if (!is.null(workers)) { From 89d566433378e0b186ec779a56d71cae7d1bdff6 Mon Sep 17 00:00:00 2001 From: Henrik Bengtsson Date: Mon, 8 Apr 2024 19:39:42 -0700 Subject: [PATCH 08/36] Use dispatcher=FALSE for mirai_multisession --- DESCRIPTION | 2 +- R/MiraiFuture-class.R | 13 +++++++++---- R/nbrOfWorkers.R | 5 ++++- R/utils,mirai.R | 3 +-- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 013a26c..e5257d3 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,5 +1,5 @@ Package: future.mirai -Version: 0.1.1-9002 +Version: 0.1.1-9003 Depends: future Imports: diff --git a/R/MiraiFuture-class.R b/R/MiraiFuture-class.R index 4a9fbc1..b465dce 100644 --- a/R/MiraiFuture-class.R +++ b/R/MiraiFuture-class.R @@ -18,10 +18,15 @@ MiraiFuture <- function(expr = NULL, packages = NULL, lazy = FALSE, workers = availableCores(), + dispatcher = "auto", ...) { if(isTRUE(substitute)) expr <- substitute(expr) + if (!identical(dispatcher, "auto")) { + stopifnot(is.logical(dispatcher), length(dispatcher) == 1L, !is.na(dispatcher)) + } + ## Record globals if(!isTRUE(attr(globals, "already-done", exact = TRUE))) { gp <- getGlobalsAndPackages(expr, envir = envir, persistent = FALSE, globals = globals) @@ -41,13 +46,11 @@ MiraiFuture <- function(expr = NULL, if (is.function(workers)) workers <- workers() if (!is.null(workers)) stop_if_not(length(workers) >= 1) - - ## FIXME: Expose 'dispatcher' in the API - dispatcher <- TRUE - + cluster <- NULL if (is.numeric(workers)) { stop_if_not(length(workers) == 1L, !is.na(workers), workers >= 1) + if (identical(dispatcher, "auto")) dispatcher <- FALSE ## Do we need to change the number of mirai workers? nworkers <- mirai_daemons_nworkers() @@ -59,6 +62,8 @@ MiraiFuture <- function(expr = NULL, } } else if (is.character(workers)) { stop_if_not(length(workers) >= 1L, !anyNA(workers)) + if (identical(dispatcher, "auto")) dispatcher <- TRUE + dd <- get_mirai_daemons() if (is.data.frame(dd)) { uris <- rownames(dd) diff --git a/R/nbrOfWorkers.R b/R/nbrOfWorkers.R index 8292250..b7b22bf 100644 --- a/R/nbrOfWorkers.R +++ b/R/nbrOfWorkers.R @@ -4,7 +4,10 @@ nbrOfWorkers.mirai <- function(evaluator) { res <- status() workers <- res[["daemons"]] - if (!is.numeric(workers)) { + if (is.character(workers)) { + workers <- res[["connections"]] + stopifnot(is.numeric(workers)) + } else if (!is.numeric(workers)) { stop(FutureError(sprintf("Unknown type of mirai::daemons()$daemons: %s", typeof(workers)))) } diff --git a/R/utils,mirai.R b/R/utils,mirai.R index 6ff178f..27f0a5e 100644 --- a/R/utils,mirai.R +++ b/R/utils,mirai.R @@ -28,7 +28,7 @@ launch_mirai_daemons <- function(hostnames, ..., timeout = 60) { ## Assert that mirai daemons have been configured dd <- get_mirai_daemons() - stopifnot(is.data.frame(dd)) + stopifnot(is.data.frame(dd) || is.numeric(dd)) ## Consider only non-connected daemons dd <- subset(dd, online == 0L) @@ -62,7 +62,6 @@ launch_mirai_daemons <- function(hostnames, ..., timeout = 60) { system2(bin, args = c("-e", shQuote(code)), wait = FALSE) }, chunk.size = 1L) - ## Wait for mirai servers to connect back t0 <- Sys.time() ready <- FALSE From 6e0ee7d2fa7afdd02205ff78704c522cb2ae5c24 Mon Sep 17 00:00:00 2001 From: Henrik Bengtsson Date: Mon, 8 Apr 2024 19:44:35 -0700 Subject: [PATCH 09/36] refresh docs --- man/MiraiFuture.Rd | 1 + 1 file changed, 1 insertion(+) diff --git a/man/MiraiFuture.Rd b/man/MiraiFuture.Rd index ec4c064..e38cd6d 100644 --- a/man/MiraiFuture.Rd +++ b/man/MiraiFuture.Rd @@ -12,6 +12,7 @@ MiraiFuture( packages = NULL, lazy = FALSE, workers = availableCores(), + dispatcher = "auto", ... ) } From e20cba4ef30d44bd7660e3b3df91e4f69006c028 Mon Sep 17 00:00:00 2001 From: Henrik Bengtsson Date: Mon, 8 Apr 2024 19:53:46 -0700 Subject: [PATCH 10/36] TESTS: Skip mirai_cluster() on MS Windows to see if we can avoid 'detritus in the temp directory' NOTE --- DESCRIPTION | 2 +- R/utils,mirai.R | 2 +- tests/mirai_cluster.R | 12 +++++++++++- tests/mirai_multisession.R | 1 + 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index e5257d3..d20a020 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,5 +1,5 @@ Package: future.mirai -Version: 0.1.1-9003 +Version: 0.1.1-9005 Depends: future Imports: diff --git a/R/utils,mirai.R b/R/utils,mirai.R index 27f0a5e..f0f9d53 100644 --- a/R/utils,mirai.R +++ b/R/utils,mirai.R @@ -53,7 +53,7 @@ launch_mirai_daemons <- function(hostnames, ..., timeout = 60) { uris <- sub("//:", sprintf("//%s:", host_ip), uris) ## Launching parallel PSOCK workers - cl <- makeClusterPSOCK(hostnames, ...) + cl <- makeClusterPSOCK(hostnames, ..., autoStop = TRUE) ## Use them to launch mirai daemons to connect back to host void <- parLapply(cl, uris, function(uri) { diff --git a/tests/mirai_cluster.R b/tests/mirai_cluster.R index dcae413..a9c2f95 100644 --- a/tests/mirai_cluster.R +++ b/tests/mirai_cluster.R @@ -1,5 +1,15 @@ if (requireNamespace("future.tests")) { mirai::daemons(0) ## Reset any daemons running - future.tests::check("future.mirai::mirai_cluster", timeout = 30.0, exit_value = FALSE) + + ## FIXME: The following is disable on MS Windows, because it will + ## result in a 'R CMD check' NOTE on "detritus in the temp directory". + ## This happens whenever we use mirai::daemons(..., dispatcher = TRUE) [1], + ## which is what mirai_cluster() uses. + ## [1] https://github.com/shikokuchuo/mirai/discussions/105 + if (.Platform[["OS.type"]] != "windows") { + future.tests::check("future.mirai::mirai_cluster", timeout = 30.0, exit_value = FALSE) + } + mirai::daemons(0) ## Reset any daemons running + gc() } diff --git a/tests/mirai_multisession.R b/tests/mirai_multisession.R index f63dd1b..1b7dda5 100644 --- a/tests/mirai_multisession.R +++ b/tests/mirai_multisession.R @@ -2,4 +2,5 @@ if (requireNamespace("future.tests")) { mirai::daemons(0) ## Reset any daemons running future.tests::check("future.mirai::mirai_multisession", timeout = 30.0, exit_value = FALSE) mirai::daemons(0) ## Reset any daemons running + gc() } From cda4c18a9b805fbf879634ad5fae522d7a38f2d9 Mon Sep 17 00:00:00 2001 From: Henrik Bengtsson Date: Mon, 8 Apr 2024 20:12:04 -0700 Subject: [PATCH 11/36] Try to avoid NOTEs on MS Windows by not testing --- tests/mirai_multisession.R | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/mirai_multisession.R b/tests/mirai_multisession.R index 1b7dda5..5d6563b 100644 --- a/tests/mirai_multisession.R +++ b/tests/mirai_multisession.R @@ -1,6 +1,10 @@ if (requireNamespace("future.tests")) { mirai::daemons(0) ## Reset any daemons running - future.tests::check("future.mirai::mirai_multisession", timeout = 30.0, exit_value = FALSE) + + if (.Platform[["OS.type"]] != "windows") { + future.tests::check("future.mirai::mirai_multisession", timeout = 30.0, exit_value = FALSE) + } + mirai::daemons(0) ## Reset any daemons running gc() } From 3db6ab386b8f82cdf59772479e874aa870709dbc Mon Sep 17 00:00:00 2001 From: Henrik Bengtsson Date: Mon, 8 Apr 2024 20:20:41 -0700 Subject: [PATCH 12/36] Try to avoid NOTEs on MS Windows by disabling also examples --- DESCRIPTION | 2 +- incl/mirai_cluster.R | 4 ++++ incl/mirai_multisession.R | 4 ++++ man/mirai_cluster.Rd | 4 ++++ man/mirai_multisession.Rd | 4 ++++ 5 files changed, 17 insertions(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index d20a020..8f08927 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,5 +1,5 @@ Package: future.mirai -Version: 0.1.1-9005 +Version: 0.1.1-9006 Depends: future Imports: diff --git a/incl/mirai_cluster.R b/incl/mirai_cluster.R index 01d6118..1a53266 100644 --- a/incl/mirai_cluster.R +++ b/incl/mirai_cluster.R @@ -1,3 +1,5 @@ +if (.Platform[["OS.type"]] != "windows") { + mirai::daemons(parallelly::availableCores()) plan(mirai_cluster, workers = NULL) @@ -9,3 +11,5 @@ N <- 1e6 # samples per worker M <- 10 # iterations pi_est <- Reduce(sum, Map(value, replicate(M, f()))) / M print(pi_est) + +} ## if (.Platform[["OS.type"]] != "windows") diff --git a/incl/mirai_multisession.R b/incl/mirai_multisession.R index f039603..e5ff49a 100644 --- a/incl/mirai_multisession.R +++ b/incl/mirai_multisession.R @@ -1,3 +1,5 @@ +if (.Platform[["OS.type"]] != "windows") { + plan(mirai_multisession) # A function that returns a future, note that N uses lexical scoping... @@ -8,3 +10,5 @@ N <- 1e6 # samples per worker M <- 10 # iterations pi_est <- Reduce(sum, Map(value, replicate(M, f()))) / M print(pi_est) + +} ## if (.Platform[["OS.type"]] != "windows") diff --git a/man/mirai_cluster.Rd b/man/mirai_cluster.Rd index 2e06760..86b825f 100644 --- a/man/mirai_cluster.Rd +++ b/man/mirai_cluster.Rd @@ -39,6 +39,8 @@ An object of class \link{MiraiFuture}. Mirai-based cluster futures } \examples{ +if (.Platform[["OS.type"]] != "windows") { + mirai::daemons(parallelly::availableCores()) plan(mirai_cluster, workers = NULL) @@ -50,4 +52,6 @@ N <- 1e6 # samples per worker M <- 10 # iterations pi_est <- Reduce(sum, Map(value, replicate(M, f()))) / M print(pi_est) + +} ## if (.Platform[["OS.type"]] != "windows") } diff --git a/man/mirai_multisession.Rd b/man/mirai_multisession.Rd index 754600f..a1d9f01 100644 --- a/man/mirai_multisession.Rd +++ b/man/mirai_multisession.Rd @@ -34,6 +34,8 @@ An object of class \link{MiraiFuture}. Mirai-based localhost multisession futures } \examples{ +if (.Platform[["OS.type"]] != "windows") { + plan(mirai_multisession) # A function that returns a future, note that N uses lexical scoping... @@ -44,4 +46,6 @@ N <- 1e6 # samples per worker M <- 10 # iterations pi_est <- Reduce(sum, Map(value, replicate(M, f()))) / M print(pi_est) + +} ## if (.Platform[["OS.type"]] != "windows") } From 8d814eb66a6a1fdcf9fc4816d5250100d2dd6ba6 Mon Sep 17 00:00:00 2001 From: Henrik Bengtsson Date: Mon, 8 Apr 2024 20:22:47 -0700 Subject: [PATCH 13/36] Examples: explicitly shut down mirai workers --- DESCRIPTION | 2 +- incl/mirai_cluster.R | 2 ++ incl/mirai_multisession.R | 5 ++--- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 8f08927..d2f51a9 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,5 +1,5 @@ Package: future.mirai -Version: 0.1.1-9006 +Version: 0.1.1-9007 Depends: future Imports: diff --git a/incl/mirai_cluster.R b/incl/mirai_cluster.R index 1a53266..8b6cd04 100644 --- a/incl/mirai_cluster.R +++ b/incl/mirai_cluster.R @@ -12,4 +12,6 @@ M <- 10 # iterations pi_est <- Reduce(sum, Map(value, replicate(M, f()))) / M print(pi_est) +plan(sequential) +mirai::daemons(0) ## Shut down mirai workers } ## if (.Platform[["OS.type"]] != "windows") diff --git a/incl/mirai_multisession.R b/incl/mirai_multisession.R index e5ff49a..c19327a 100644 --- a/incl/mirai_multisession.R +++ b/incl/mirai_multisession.R @@ -1,5 +1,3 @@ -if (.Platform[["OS.type"]] != "windows") { - plan(mirai_multisession) # A function that returns a future, note that N uses lexical scoping... @@ -11,4 +9,5 @@ M <- 10 # iterations pi_est <- Reduce(sum, Map(value, replicate(M, f()))) / M print(pi_est) -} ## if (.Platform[["OS.type"]] != "windows") +plan(sequential) +mirai::daemons(0) ## Shut down mirai workers From a715a67346daa5548b40cd8ad7865c36507758c7 Mon Sep 17 00:00:00 2001 From: Henrik Bengtsson Date: Mon, 8 Apr 2024 20:29:47 -0700 Subject: [PATCH 14/36] Examples: Try to re-enable on example --- DESCRIPTION | 2 +- man/mirai_cluster.Rd | 2 ++ man/mirai_multisession.Rd | 5 ++--- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index d2f51a9..7baa9a5 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,5 +1,5 @@ Package: future.mirai -Version: 0.1.1-9007 +Version: 0.1.1-9008 Depends: future Imports: diff --git a/man/mirai_cluster.Rd b/man/mirai_cluster.Rd index 86b825f..60e9c65 100644 --- a/man/mirai_cluster.Rd +++ b/man/mirai_cluster.Rd @@ -53,5 +53,7 @@ M <- 10 # iterations pi_est <- Reduce(sum, Map(value, replicate(M, f()))) / M print(pi_est) +plan(sequential) +mirai::daemons(0) ## Shut down mirai workers } ## if (.Platform[["OS.type"]] != "windows") } diff --git a/man/mirai_multisession.Rd b/man/mirai_multisession.Rd index a1d9f01..f5c5e4e 100644 --- a/man/mirai_multisession.Rd +++ b/man/mirai_multisession.Rd @@ -34,8 +34,6 @@ An object of class \link{MiraiFuture}. Mirai-based localhost multisession futures } \examples{ -if (.Platform[["OS.type"]] != "windows") { - plan(mirai_multisession) # A function that returns a future, note that N uses lexical scoping... @@ -47,5 +45,6 @@ M <- 10 # iterations pi_est <- Reduce(sum, Map(value, replicate(M, f()))) / M print(pi_est) -} ## if (.Platform[["OS.type"]] != "windows") +plan(sequential) +mirai::daemons(0) ## Shut down mirai workers } From 640fd0061755071bac8b7e7a47f6ec2ab3d2804c Mon Sep 17 00:00:00 2001 From: Henrik Bengtsson Date: Mon, 8 Apr 2024 20:38:03 -0700 Subject: [PATCH 15/36] TESTS: Re-enable mirai_multisession tests on MS Windows --- DESCRIPTION | 2 +- tests/mirai_multisession.R | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 7baa9a5..bf7316d 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,5 +1,5 @@ Package: future.mirai -Version: 0.1.1-9008 +Version: 0.1.1-9009 Depends: future Imports: diff --git a/tests/mirai_multisession.R b/tests/mirai_multisession.R index 5d6563b..1a42564 100644 --- a/tests/mirai_multisession.R +++ b/tests/mirai_multisession.R @@ -1,9 +1,7 @@ if (requireNamespace("future.tests")) { mirai::daemons(0) ## Reset any daemons running - if (.Platform[["OS.type"]] != "windows") { - future.tests::check("future.mirai::mirai_multisession", timeout = 30.0, exit_value = FALSE) - } + future.tests::check("future.mirai::mirai_multisession", timeout = 30.0, exit_value = FALSE) mirai::daemons(0) ## Reset any daemons running gc() From ac000e72bd91419583f4bedb1307c144e8de33ed Mon Sep 17 00:00:00 2001 From: Henrik Bengtsson Date: Mon, 8 Apr 2024 20:54:54 -0700 Subject: [PATCH 16/36] Try to re-enable mirai_cluster example --- incl/mirai_cluster.R | 3 +-- man/mirai_cluster.Rd | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/incl/mirai_cluster.R b/incl/mirai_cluster.R index 8b6cd04..48ef7e0 100644 --- a/incl/mirai_cluster.R +++ b/incl/mirai_cluster.R @@ -1,5 +1,3 @@ -if (.Platform[["OS.type"]] != "windows") { - mirai::daemons(parallelly::availableCores()) plan(mirai_cluster, workers = NULL) @@ -14,4 +12,5 @@ print(pi_est) plan(sequential) mirai::daemons(0) ## Shut down mirai workers +if (.Platform[["OS.type"]] != "windows") { } ## if (.Platform[["OS.type"]] != "windows") diff --git a/man/mirai_cluster.Rd b/man/mirai_cluster.Rd index 60e9c65..5d0ece5 100644 --- a/man/mirai_cluster.Rd +++ b/man/mirai_cluster.Rd @@ -39,8 +39,6 @@ An object of class \link{MiraiFuture}. Mirai-based cluster futures } \examples{ -if (.Platform[["OS.type"]] != "windows") { - mirai::daemons(parallelly::availableCores()) plan(mirai_cluster, workers = NULL) @@ -55,5 +53,6 @@ print(pi_est) plan(sequential) mirai::daemons(0) ## Shut down mirai workers +if (.Platform[["OS.type"]] != "windows") { } ## if (.Platform[["OS.type"]] != "windows") } From bdb7e76a2f20311fee2918643c8cca9aed4d9c9e Mon Sep 17 00:00:00 2001 From: Henrik Bengtsson Date: Mon, 8 Apr 2024 21:42:49 -0700 Subject: [PATCH 17/36] Don't run example(mirai_cluster) on Windows, unless interactive --- DESCRIPTION | 2 +- R/mirai_cluster.R | 8 ++++++++ incl/mirai_cluster.R | 9 +++++---- incl/mirai_multisession.R | 2 +- man/mirai_cluster.Rd | 24 ++++++++++++------------ man/mirai_multisession.Rd | 2 +- 6 files changed, 28 insertions(+), 19 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index bf7316d..53259e5 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,5 +1,5 @@ Package: future.mirai -Version: 0.1.1-9009 +Version: 0.1.1-9010 Depends: future Imports: diff --git a/R/mirai_cluster.R b/R/mirai_cluster.R index 30eecd6..35a65f6 100644 --- a/R/mirai_cluster.R +++ b/R/mirai_cluster.R @@ -3,6 +3,14 @@ #' @inheritParams MiraiFuture #' @inheritParams future::cluster #' +#' @param workers Specifies **mirai** workers to use as parallel workers. +#' If a numeric scalar, then this number of local **mirai** daemons will +#' be launched. +#' If a character vector, then this it specifies the hostnames of where +#' the **mirai** daemons will be launched. +#' If `NULL`, then any **mirai** daemons previously created by +#' [mirai::daemons()] will be used as parallel workers. +#' #' @return An object of class [MiraiFuture]. #' #' @example incl/mirai_cluster.R diff --git a/incl/mirai_cluster.R b/incl/mirai_cluster.R index 48ef7e0..f64ccb7 100644 --- a/incl/mirai_cluster.R +++ b/incl/mirai_cluster.R @@ -1,4 +1,5 @@ -mirai::daemons(parallelly::availableCores()) +\dontshow{if (.Platform[["OS.type"]] != "windows" || interactive()) \{} +mirai::daemons(parallelly::availableCores(), dispatcher = FALSE) plan(mirai_cluster, workers = NULL) # A function that returns a future, note that N uses lexical scoping... @@ -11,6 +12,6 @@ pi_est <- Reduce(sum, Map(value, replicate(M, f()))) / M print(pi_est) plan(sequential) -mirai::daemons(0) ## Shut down mirai workers -if (.Platform[["OS.type"]] != "windows") { -} ## if (.Platform[["OS.type"]] != "windows") +invisible(mirai::daemons(0)) ## Shut down mirai workers +\dontshow{\}} + diff --git a/incl/mirai_multisession.R b/incl/mirai_multisession.R index c19327a..d525316 100644 --- a/incl/mirai_multisession.R +++ b/incl/mirai_multisession.R @@ -10,4 +10,4 @@ pi_est <- Reduce(sum, Map(value, replicate(M, f()))) / M print(pi_est) plan(sequential) -mirai::daemons(0) ## Shut down mirai workers +invisible(mirai::daemons(0)) ## Shut down mirai workers diff --git a/man/mirai_cluster.Rd b/man/mirai_cluster.Rd index 5d0ece5..5ca6be4 100644 --- a/man/mirai_cluster.Rd +++ b/man/mirai_cluster.Rd @@ -23,14 +23,13 @@ identified.} \item{...}{Additional named elements of the future.} -\item{workers}{A \code{\link[parallel:makeCluster]{cluster}} object, -a character vector of host names, a positive numeric scalar, -or a function. -If a character vector or a numeric scalar, a \code{cluster} object -is created using \code{\link[parallelly:makeClusterPSOCK]{makeClusterPSOCK}(workers)}. -If a function, it is called without arguments \emph{when the future -is created} and its value is used to configure the workers. -The function should return any of the above types.} +\item{workers}{Specifies \strong{mirai} workers to use as parallel workers. +If a numeric scalar, then this number of local \strong{mirai} daemons will +be launched. +If a character vector, then this it specifies the hostnames of where +the \strong{mirai} daemons will be launched. +If \code{NULL}, then any \strong{mirai} daemons previously created by +\code{\link[mirai:daemons]{mirai::daemons()}} will be used as parallel workers.} } \value{ An object of class \link{MiraiFuture}. @@ -39,7 +38,8 @@ An object of class \link{MiraiFuture}. Mirai-based cluster futures } \examples{ -mirai::daemons(parallelly::availableCores()) +\dontshow{if (.Platform[["OS.type"]] != "windows" || interactive()) \{} +mirai::daemons(parallelly::availableCores(), dispatcher = FALSE) plan(mirai_cluster, workers = NULL) # A function that returns a future, note that N uses lexical scoping... @@ -52,7 +52,7 @@ pi_est <- Reduce(sum, Map(value, replicate(M, f()))) / M print(pi_est) plan(sequential) -mirai::daemons(0) ## Shut down mirai workers -if (.Platform[["OS.type"]] != "windows") { -} ## if (.Platform[["OS.type"]] != "windows") +invisible(mirai::daemons(0)) ## Shut down mirai workers +\dontshow{\}} + } diff --git a/man/mirai_multisession.Rd b/man/mirai_multisession.Rd index f5c5e4e..9e98687 100644 --- a/man/mirai_multisession.Rd +++ b/man/mirai_multisession.Rd @@ -46,5 +46,5 @@ pi_est <- Reduce(sum, Map(value, replicate(M, f()))) / M print(pi_est) plan(sequential) -mirai::daemons(0) ## Shut down mirai workers +invisible(mirai::daemons(0)) ## Shut down mirai workers } From 2b688539be8732a9b20b94d60f0a5ac69a6c9d1b Mon Sep 17 00:00:00 2001 From: Henrik Bengtsson Date: Thu, 11 Apr 2024 16:17:56 -0700 Subject: [PATCH 18/36] CLEANUP: mirai_cluster() no longer accepts argument 'workers'; it will default to use whatever workers are already set up using mirai::daemons() --- DESCRIPTION | 3 +- NAMESPACE | 4 --- R/MiraiFuture-class.R | 18 +----------- R/mirai_cluster.R | 15 ++-------- R/utils,mirai.R | 66 ------------------------------------------- incl/mirai_cluster.R | 5 +--- man/mirai_cluster.Rd | 21 ++------------ tests/mirai_cluster.R | 14 ++++----- 8 files changed, 14 insertions(+), 132 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 53259e5..121d3fa 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,11 +1,10 @@ Package: future.mirai -Version: 0.1.1-9010 +Version: 0.1.1-9011 Depends: future Imports: mirai (>= 0.12.1), parallelly, - parallel, utils Suggests: future.tests diff --git a/NAMESPACE b/NAMESPACE index 4d8fc4c..a3285d0 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -26,11 +26,7 @@ importFrom(mirai,is_error_value) importFrom(mirai,mirai) importFrom(mirai,status) importFrom(mirai,unresolved) -importFrom(parallel,parLapply) -importFrom(parallel,stopCluster) importFrom(parallelly,availableCores) -importFrom(parallelly,availableWorkers) -importFrom(parallelly,makeClusterPSOCK) importFrom(utils,capture.output) importFrom(utils,packageVersion) importFrom(utils,str) diff --git a/R/MiraiFuture-class.R b/R/MiraiFuture-class.R index b465dce..d2b0eed 100644 --- a/R/MiraiFuture-class.R +++ b/R/MiraiFuture-class.R @@ -60,24 +60,8 @@ MiraiFuture <- function(expr = NULL, daemons(n = 0L) ## reset is required daemons(n = workers, dispatcher = dispatcher) } - } else if (is.character(workers)) { - stop_if_not(length(workers) >= 1L, !anyNA(workers)) - if (identical(dispatcher, "auto")) dispatcher <- TRUE - - dd <- get_mirai_daemons() - if (is.data.frame(dd)) { - uris <- rownames(dd) - n <- length(uris) - } else { - n <- -1L - } - if (length(workers) != n) { - daemons(n = 0L) ## reset is required - daemons(n = length(workers), url = "ws://:0", dispatcher = dispatcher) - } - cluster <- launch_mirai_daemons(workers) } else if (!is.null(workers)) { - stop("Argument 'workers' should be a numeric scalar or a character vector: ", mode(workers)) + stop("Argument 'workers' should be a numeric scalar or NULL: ", mode(workers)) } future <- structure(future, class = c("MiraiFuture", class(future))) diff --git a/R/mirai_cluster.R b/R/mirai_cluster.R index 35a65f6..5e1e527 100644 --- a/R/mirai_cluster.R +++ b/R/mirai_cluster.R @@ -3,31 +3,21 @@ #' @inheritParams MiraiFuture #' @inheritParams future::cluster #' -#' @param workers Specifies **mirai** workers to use as parallel workers. -#' If a numeric scalar, then this number of local **mirai** daemons will -#' be launched. -#' If a character vector, then this it specifies the hostnames of where -#' the **mirai** daemons will be launched. -#' If `NULL`, then any **mirai** daemons previously created by -#' [mirai::daemons()] will be used as parallel workers. -#' #' @return An object of class [MiraiFuture]. #' #' @example incl/mirai_cluster.R #' -#' @importFrom parallelly availableWorkers #' @export mirai_cluster <- function(expr, substitute = TRUE, envir = parent.frame(), - ..., - workers = availableWorkers()) { + ...) { if (substitute) expr <- substitute(expr) future <- MiraiFuture( expr = expr, substitute = FALSE, envir = envir, - workers = workers, + workers = NULL, ... ) if(!isTRUE(future[["lazy"]])) future <- run(future) @@ -35,7 +25,6 @@ mirai_cluster <- function(expr, } class(mirai_cluster) <- c("mirai_cluster", "mirai", "multiprocess", "future", "function") attr(mirai_cluster, "init") <- TRUE -attr(mirai_cluster, "tweakable") <- "workers" #' @importFrom future tweak diff --git a/R/utils,mirai.R b/R/utils,mirai.R index f0f9d53..7417da6 100644 --- a/R/utils,mirai.R +++ b/R/utils,mirai.R @@ -17,69 +17,3 @@ get_mirai_daemons <- function() { as.data.frame(res) } - -#' @importFrom parallelly makeClusterPSOCK -#' @importFrom parallel parLapply stopCluster -launch_mirai_daemons <- function(hostnames, ..., timeout = 60) { - online <- NULL ## To please R CMD check - - stopifnot(is.character(hostnames), !anyNA(hostnames)) - stopifnot(is.numeric(timeout), !is.na(timeout), is.finite(timeout), timeout > 0.0) - - ## Assert that mirai daemons have been configured - dd <- get_mirai_daemons() - stopifnot(is.data.frame(dd) || is.numeric(dd)) - - ## Consider only non-connected daemons - dd <- subset(dd, online == 0L) - - ## Nothing to do? - if (nrow(dd) == 0L) return(NULL) - - ## Launch only a subset? - if (nrow(dd) > length(hostnames)) { - dd <- dd[seq_along(hostnames), ] - } else if (nrow(dd) < length(hostnames)) { - hostnames <- hostnames[seq_len(nrow(dd))] - } - - uris <- rownames(dd) - stopifnot(is.character(uris), !anyNA(uris), anyDuplicated(uris) == 0L) - - ## FIXME: This assumes servers are launched on localhost, or that - ## reverse tunnelling is used when setting up the PSOCK cluster - ## /HB 2023-05-02 - host_ip <- "127.0.0.1" - uris <- sub("//:", sprintf("//%s:", host_ip), uris) - - ## Launching parallel PSOCK workers - cl <- makeClusterPSOCK(hostnames, ..., autoStop = TRUE) - - ## Use them to launch mirai daemons to connect back to host - void <- parLapply(cl, uris, function(uri) { - code <- sprintf('mirai::daemon("%s")', uri) - bin <- file.path(R.home("bin"), "Rscript") - system2(bin, args = c("-e", shQuote(code)), wait = FALSE) - }, chunk.size = 1L) - - ## Wait for mirai servers to connect back - t0 <- Sys.time() - ready <- FALSE - while (!ready) { - dd2 <- get_mirai_daemons() - stopifnot(is.data.frame(dd)) - dd2 <- dd2[rownames(dd), , drop = FALSE] - dd2 <- subset(dd2, online == 0L) - ready <- (nrow(dd2) == 0) - Sys.sleep(1.0) - dt <- Sys.time() - t0 - if (dt > timeout) { - stopCluster(cl) - cl <- NULL - stop(sprintf("%d out of %d mirai servers did not connect back within %g seconds", - nrow(dd2), nrow(dd), timeout)) - } - } - - cl -} ## launch_mirai_daemons() diff --git a/incl/mirai_cluster.R b/incl/mirai_cluster.R index f64ccb7..52a2b4b 100644 --- a/incl/mirai_cluster.R +++ b/incl/mirai_cluster.R @@ -1,6 +1,5 @@ -\dontshow{if (.Platform[["OS.type"]] != "windows" || interactive()) \{} mirai::daemons(parallelly::availableCores(), dispatcher = FALSE) -plan(mirai_cluster, workers = NULL) +plan(mirai_cluster) # A function that returns a future, note that N uses lexical scoping... f <- \() future({4 * sum((runif(N) ^ 2 + runif(N) ^ 2) < 1) / N}, seed = TRUE) @@ -13,5 +12,3 @@ print(pi_est) plan(sequential) invisible(mirai::daemons(0)) ## Shut down mirai workers -\dontshow{\}} - diff --git a/man/mirai_cluster.Rd b/man/mirai_cluster.Rd index 5ca6be4..75a58da 100644 --- a/man/mirai_cluster.Rd +++ b/man/mirai_cluster.Rd @@ -4,13 +4,7 @@ \alias{mirai_cluster} \title{Mirai-based cluster futures} \usage{ -mirai_cluster( - expr, - substitute = TRUE, - envir = parent.frame(), - ..., - workers = availableWorkers() -) +mirai_cluster(expr, substitute = TRUE, envir = parent.frame(), ...) } \arguments{ \item{expr}{An \R \link[base]{expression}.} @@ -22,14 +16,6 @@ mirai_cluster( identified.} \item{...}{Additional named elements of the future.} - -\item{workers}{Specifies \strong{mirai} workers to use as parallel workers. -If a numeric scalar, then this number of local \strong{mirai} daemons will -be launched. -If a character vector, then this it specifies the hostnames of where -the \strong{mirai} daemons will be launched. -If \code{NULL}, then any \strong{mirai} daemons previously created by -\code{\link[mirai:daemons]{mirai::daemons()}} will be used as parallel workers.} } \value{ An object of class \link{MiraiFuture}. @@ -38,9 +24,8 @@ An object of class \link{MiraiFuture}. Mirai-based cluster futures } \examples{ -\dontshow{if (.Platform[["OS.type"]] != "windows" || interactive()) \{} mirai::daemons(parallelly::availableCores(), dispatcher = FALSE) -plan(mirai_cluster, workers = NULL) +plan(mirai_cluster) # A function that returns a future, note that N uses lexical scoping... f <- \() future({4 * sum((runif(N) ^ 2 + runif(N) ^ 2) < 1) / N}, seed = TRUE) @@ -53,6 +38,4 @@ print(pi_est) plan(sequential) invisible(mirai::daemons(0)) ## Shut down mirai workers -\dontshow{\}} - } diff --git a/tests/mirai_cluster.R b/tests/mirai_cluster.R index a9c2f95..1ce8b1f 100644 --- a/tests/mirai_cluster.R +++ b/tests/mirai_cluster.R @@ -1,14 +1,14 @@ if (requireNamespace("future.tests")) { mirai::daemons(0) ## Reset any daemons running - ## FIXME: The following is disable on MS Windows, because it will - ## result in a 'R CMD check' NOTE on "detritus in the temp directory". - ## This happens whenever we use mirai::daemons(..., dispatcher = TRUE) [1], - ## which is what mirai_cluster() uses. + ## FIXME: The following is disabled on MS Windows, because it will + ## result in a 'R CMD check' NOTE on "detritus in the temp directory" [1]. + ## This happens whenever we use mirai::daemons(..., dispatcher = TRUE). ## [1] https://github.com/shikokuchuo/mirai/discussions/105 - if (.Platform[["OS.type"]] != "windows") { - future.tests::check("future.mirai::mirai_cluster", timeout = 30.0, exit_value = FALSE) - } + dispatcher <- (.Platform[["OS.type"]] != "windows") + + mirai::daemons(2, dispatcher = dispatcher) + future.tests::check("future.mirai::mirai_cluster", timeout = 10.0, exit_value = FALSE) mirai::daemons(0) ## Reset any daemons running gc() From 0072d58cfb63f061e13386f0678a7a1e7b8091b8 Mon Sep 17 00:00:00 2001 From: Henrik Bengtsson Date: Thu, 11 Apr 2024 16:45:44 -0700 Subject: [PATCH 19/36] Update README --- NEWS.md | 4 ++-- README.md | 47 ++++++++++++++++++++++++++++++++++------------- incl/OVERVIEW.md | 45 +++++++++++++++++++++++++++++++++------------ inst/WORDLIST | 6 +++++- 4 files changed, 74 insertions(+), 28 deletions(-) diff --git a/NEWS.md b/NEWS.md index 02c85a5..5e61cbb 100644 --- a/NEWS.md +++ b/NEWS.md @@ -5,13 +5,13 @@ # Version 0.1.1 -## Miscelleanous +## Miscellaneous * Align code with **mirai** 0.9.1. # Version 0.1.0 -## Signficant Changes +## Significant Changes * A working, proof-of-concept implementation. diff --git a/README.md b/README.md index a267561..9b41ce6 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ For example, This is obviously a toy example to illustrate what futures look like and how to work with them. For further examples on how to use futures, see the vignettes of the **[future]** package as well as -those of **[future.apply]** and **[doFuture]**. +those of **[future.apply]**, **[furrr]**, and **[doFuture]**. ## Using the future.mirai backend @@ -44,20 +44,30 @@ for **mirai**. | Backend | Description | Alternative in future package |:---------------------|:-----------------------------------------------------------------|:------------------------------ -| `mirai_multisession` | parallel evaluation in a separate R process (on current machine) | `plan(multisession)` +| `mirai_multisession` | parallel evaluation in separate R processes (on current machine) | `plan(multisession)` +| `mirai_cluster` | parallel evaluation in mirai-configured workers | `plan(cluster)` -### Each mirai future uses a fresh R session +### Advantages of mirai futures -When using `mirai` futures, each future is resolved in a fresh -background R session which ends as soon as the value of the future has -been collected. In contrast, `multisession` futures are resolved in -background R worker sessions that serve multiple futures over their -life spans. The advantage with using a new R process for each future -is that it is that the R environment is guaranteed not to be -contaminated by previous futures, e.g. memory allocations, finalizers, -modified options, and loaded and attached packages. The disadvantage, -is an added overhead of launching a new R process. +The **mirai** package provides a low-level future-like mechanism for +evaluating R expression in separate R processes running on the local +machine or on one or more remote machines. Centrally to **mirai** is +its highly-optimized queueing mechanism, which is used to orchestrate +communication between the main R process and parallel workers. A +**mirai** cluster of workers can be configured to communicate securly +via the well-established Transport Layer Security (TLS) protocol. + +Another advantage with `mirai_*` futures, compared to `multisession` +and `cluster` futures, is that we can use more than 125 parallel +workers. The current limit of 125 workers for `multisession` and +`cluster` futures stems from how the underlying **parallel** package +using one R connection per parallel worker and R has a limit of 125 R +connections per session. In R (>= 4.4.0), we can increase this limit +when we launch R, e.g. `R --max-connections=200`. For R (< 4.4.0), R +has to be rebuilt from source after adjusting the source code. The +**mirai** package does not rely on R connections for parallel workers +and does therefore not suffer from this limit. ## Demos @@ -75,17 +85,28 @@ plan(mirai_multisession) demo("mandelbrot", package = "future", ask = FALSE) ``` +and + +```r +library(future.mirai) +mirai::daemons(2) +plan(mirai_cluster) + +demo("mandelbrot", package = "future", ask = FALSE) +``` + [mirai]: https://cran.r-project.org/package=mirai [future]: https://cran.r-project.org/package=future [future.mirai]: https://github.com/HenrikBengtsson/future.mirai [future.apply]: https://cran.r-project.org/package=future.apply +[furrr]: https://cran.r-project.org/package=furrr [doFuture]: https://cran.r-project.org/package=doFuture ## Installation R package future.mirai is only available via [GitHub](https://github.com/HenrikBengtsson/future.mirai) and can be installed in R as: ```r -remotes::install_github("HenrikBengtsson/future.mirai", ref="main") +remotes::install_github("HenrikBengtsson/future.mirai", ref="master") ``` diff --git a/incl/OVERVIEW.md b/incl/OVERVIEW.md index efa2ce3..d8a439c 100644 --- a/incl/OVERVIEW.md +++ b/incl/OVERVIEW.md @@ -25,7 +25,7 @@ For example, This is obviously a toy example to illustrate what futures look like and how to work with them. For further examples on how to use futures, see the vignettes of the **[future]** package as well as -those of **[future.apply]** and **[doFuture]**. +those of **[future.apply]**, **[furrr]**, and **[doFuture]**. ## Using the future.mirai backend @@ -36,20 +36,30 @@ for **mirai**. | Backend | Description | Alternative in future package |:---------------------|:-----------------------------------------------------------------|:------------------------------ -| `mirai_multisession` | parallel evaluation in a separate R process (on current machine) | `plan(multisession)` +| `mirai_multisession` | parallel evaluation in separate R processes (on current machine) | `plan(multisession)` +| `mirai_cluster` | parallel evaluation in mirai-configured workers | `plan(cluster)` -### Each mirai future uses a fresh R session +### Advantages of mirai futures -When using `mirai` futures, each future is resolved in a fresh -background R session which ends as soon as the value of the future has -been collected. In contrast, `multisession` futures are resolved in -background R worker sessions that serve multiple futures over their -life spans. The advantage with using a new R process for each future -is that it is that the R environment is guaranteed not to be -contaminated by previous futures, e.g. memory allocations, finalizers, -modified options, and loaded and attached packages. The disadvantage, -is an added overhead of launching a new R process. +The **mirai** package provides a low-level future-like mechanism for +evaluating R expression in separate R processes running on the local +machine or on one or more remote machines. Centrally to **mirai** is +its highly-optimized queueing mechanism, which is used to orchestrate +communication between the main R process and parallel workers. A +**mirai** cluster of workers can be configured to communicate securly +via the well-established Transport Layer Security (TLS) protocol. + +Another advantage with `mirai_*` futures, compared to `multisession` +and `cluster` futures, is that we can use more than 125 parallel +workers. The current limit of 125 workers for `multisession` and +`cluster` futures stems from how the underlying **parallel** package +using one R connection per parallel worker and R has a limit of 125 R +connections per session. In R (>= 4.4.0), we can increase this limit +when we launch R, e.g. `R --max-connections=200`. For R (< 4.4.0), R +has to be rebuilt from source after adjusting the source code. The +**mirai** package does not rely on R connections for parallel workers +and does therefore not suffer from this limit. ## Demos @@ -67,9 +77,20 @@ plan(mirai_multisession) demo("mandelbrot", package = "future", ask = FALSE) ``` +and + +```r +library(future.mirai) +mirai::daemons(2) +plan(mirai_cluster) + +demo("mandelbrot", package = "future", ask = FALSE) +``` + [mirai]: https://cran.r-project.org/package=mirai [future]: https://cran.r-project.org/package=future [future.mirai]: https://github.com/HenrikBengtsson/future.mirai [future.apply]: https://cran.r-project.org/package=future.apply +[furrr]: https://cran.r-project.org/package=furrr [doFuture]: https://cran.r-project.org/package=doFuture diff --git a/inst/WORDLIST b/inst/WORDLIST index 163250c..fc7b389 100644 --- a/inst/WORDLIST +++ b/inst/WORDLIST @@ -1,9 +1,13 @@ callr doFuture finalizers +furrr globals mirai MiraiFuture multisession +ORCID pre - \ No newline at end of file + + + \ No newline at end of file From 0341a3b6b3e45600bd833467c35ce05de90d6c59 Mon Sep 17 00:00:00 2001 From: Henrik Bengtsson Date: Thu, 11 Apr 2024 17:28:13 -0700 Subject: [PATCH 20/36] Require mirai (>= 0.13.2) so that variables can be passed to the global environment --- .Rbuildignore | 1 + DESCRIPTION | 4 ++-- R/MiraiFuture-class.R | 11 ++++++++++- tests/globals,nested.R | 13 +++++++++++++ 4 files changed, 26 insertions(+), 3 deletions(-) create mode 100644 tests/globals,nested.R diff --git a/.Rbuildignore b/.Rbuildignore index b6704d3..7ecfd19 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -7,5 +7,6 @@ ^incl ^Makefile ^CONTRIBUTING.md +^cran-comments.md ^.*\.Rproj$ ^\.Rproj\.user$ diff --git a/DESCRIPTION b/DESCRIPTION index 121d3fa..7ae5f3d 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,9 +1,9 @@ Package: future.mirai -Version: 0.1.1-9011 +Version: 0.1.1-9012 Depends: future Imports: - mirai (>= 0.12.1), + mirai (>= 0.13.2), parallelly, utils Suggests: diff --git a/R/MiraiFuture-class.R b/R/MiraiFuture-class.R index d2b0eed..ef7bc6d 100644 --- a/R/MiraiFuture-class.R +++ b/R/MiraiFuture-class.R @@ -125,7 +125,16 @@ run.MiraiFuture <- function(future, ...) { expr <- getExpression(future) globals <- future[["globals"]] - mirai <- mirai(expr, .args = globals) + + ## Sanity check + not_allowed <- intersect(names(globals), names(formals(mirai::mirai))) + if (length(not_allowed) > 0) { + stop(FutureError(sprintf("Detected global variables that clash with argument names of mirai::mirai(): %s", paste(sQuote(not_allowed), collapse = ", ")))) + } + + args = list(.expr = expr) + if (length(globals) > 0) args <- c(args, globals) + mirai <- do.call(mirai, args = args) future[["mirai"]] <- mirai future[["state"]] <- "running" diff --git a/tests/globals,nested.R b/tests/globals,nested.R new file mode 100644 index 0000000..7985526 --- /dev/null +++ b/tests/globals,nested.R @@ -0,0 +1,13 @@ +library(future.mirai) +plan(mirai_multisession, workers = I(1)) + +g <- function() 42 +h <- function() g() + +f <- future(h()) +v <- value(f) +print(v) +stopifnot(v == h()) + +plan(sequential) +mirai::daemons(0) From d4c57e49e81dd2276a8ecffc8e3302fe1d12a606 Mon Sep 17 00:00:00 2001 From: Henrik Bengtsson Date: Thu, 11 Apr 2024 17:49:20 -0700 Subject: [PATCH 21/36] Avoid MS Windows check NOTE [#6] --- DESCRIPTION | 2 +- tests/globals,nested.R | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 7ae5f3d..2d18653 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,5 +1,5 @@ Package: future.mirai -Version: 0.1.1-9012 +Version: 0.1.1-9013 Depends: future Imports: diff --git a/tests/globals,nested.R b/tests/globals,nested.R index 7985526..7f13f09 100644 --- a/tests/globals,nested.R +++ b/tests/globals,nested.R @@ -1,5 +1,6 @@ library(future.mirai) -plan(mirai_multisession, workers = I(1)) +mirai::daemons(1, dispatcher = FALSE) +plan(mirai_cluster g <- function() 42 h <- function() g() @@ -10,4 +11,5 @@ print(v) stopifnot(v == h()) plan(sequential) -mirai::daemons(0) +mirai::daemons(0) ## Reset any daemons running +gc() From e87c277eecde5242ab28302728409d23e4f68b4f Mon Sep 17 00:00:00 2001 From: Henrik Bengtsson Date: Thu, 11 Apr 2024 17:50:18 -0700 Subject: [PATCH 22/36] syntax error --- tests/globals,nested.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/globals,nested.R b/tests/globals,nested.R index 7f13f09..7cae9dc 100644 --- a/tests/globals,nested.R +++ b/tests/globals,nested.R @@ -1,6 +1,6 @@ library(future.mirai) mirai::daemons(1, dispatcher = FALSE) -plan(mirai_cluster +plan(mirai_cluster) g <- function() 42 h <- function() g() From 33c62ab61d48655598ff8ff31a361e960897cf63 Mon Sep 17 00:00:00 2001 From: Henrik Bengtsson Date: Thu, 11 Apr 2024 18:22:13 -0700 Subject: [PATCH 23/36] TESTS: Add unit tests from future.callr, just in case we're missing something when checking with future.tests --- DESCRIPTION | 6 +- R/utils.R | 2 + tests/demo.R | 16 +++ tests/dotdotdot.R | 58 ++++++++ tests/future,labels.R | 39 ++++++ tests/future,lazy.R | 27 ++++ tests/globals,formulas.R | 128 +++++++++++++++++ tests/globals,manual.R | 119 ++++++++++++++++ tests/globals,subassignment.R | 129 +++++++++++++++++ tests/globals,tricky.R | 82 +++++++++++ tests/incl/end.R | 4 + tests/incl/start,load-only.R | 33 +++++ tests/incl/start.R | 2 + tests/mirai_cluster,2.R | 115 +++++++++++++++ tests/mirai_cluster,worker-termination.R.HIDE | 27 ++++ tests/nbrOfWorkers.R | 12 ++ tests/plan.R | 49 +++++++ tests/rng.R | 99 +++++++++++++ tests/stdout.R | 65 +++++++++ tests/zzz,future_lapply.R | 131 ++++++++++++++++++ 20 files changed, 1141 insertions(+), 2 deletions(-) create mode 100644 tests/demo.R create mode 100644 tests/dotdotdot.R create mode 100644 tests/future,labels.R create mode 100644 tests/future,lazy.R create mode 100644 tests/globals,formulas.R create mode 100644 tests/globals,manual.R create mode 100644 tests/globals,subassignment.R create mode 100644 tests/globals,tricky.R create mode 100644 tests/incl/end.R create mode 100644 tests/incl/start,load-only.R create mode 100644 tests/incl/start.R create mode 100644 tests/mirai_cluster,2.R create mode 100644 tests/mirai_cluster,worker-termination.R.HIDE create mode 100644 tests/nbrOfWorkers.R create mode 100644 tests/plan.R create mode 100644 tests/rng.R create mode 100644 tests/stdout.R create mode 100644 tests/zzz,future_lapply.R diff --git a/DESCRIPTION b/DESCRIPTION index 2d18653..fc7ba33 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,5 +1,5 @@ Package: future.mirai -Version: 0.1.1-9013 +Version: 0.1.1-9014 Depends: future Imports: @@ -7,7 +7,9 @@ Imports: parallelly, utils Suggests: - future.tests + future.tests, + future.apply, + listenv Title: A Future API for Parallel Processing using 'mirai' Description: A Future API implementation on top of the 'mirai' package. Authors@R: c( diff --git a/R/utils.R b/R/utils.R index f1fd498..66fe538 100644 --- a/R/utils.R +++ b/R/utils.R @@ -30,6 +30,8 @@ mprint <- function(..., prefix = now(), debug = getOption("future.mirai.debug", message(stdout) } +mprintf <- function(...) message(now(), sprintf(...), appendLF = FALSE) + stop_if_not <- function(...) { res <- list(...) for (ii in 1L:length(res)) { diff --git a/tests/demo.R b/tests/demo.R new file mode 100644 index 0000000..5dd9c13 --- /dev/null +++ b/tests/demo.R @@ -0,0 +1,16 @@ +source("incl/start.R") + +options(future.demo.mandelbrot.nrow = 2L) +options(future.demo.mandelbrot.resolution = 50L) +options(future.demo.mandelbrot.delay = FALSE) + +message("*** Demos ...") + +message("*** Mandelbrot demo of the 'future' package ...") + +plan(mirai_multisession, workers = 2) +demo("mandelbrot", package = "future", ask = FALSE) + +message("*** Demos ... DONE") + +source("incl/end.R") diff --git a/tests/dotdotdot.R b/tests/dotdotdot.R new file mode 100644 index 0000000..de295ad --- /dev/null +++ b/tests/dotdotdot.R @@ -0,0 +1,58 @@ +source("incl/start.R") +library("listenv") + +message("*** Global argument '...' in futures ...") + +sum_fcns <- list() + +sum_fcns$A <- function(x, ...) { + message("Arguments '...' exists: ", exists("...", inherits = TRUE)) + y %<-% { sum(x, ...) } + y +} + + +sum_fcns$B <- function(x, ...) { + sumt <- function(x) { + message("Arguments '...' exists: ", exists("...", inherits = TRUE)) + y %<-% { sum(x, ...) } + y + } + sumt(x) +} + +sum_fcns$C <- function(x, y) { + message("Arguments '...' exists: ", exists("...", inherits = TRUE)) + y %<-% { sum(x, y) } + y +} + +sum_fcns$D <- function(x, y) { + message("Arguments '...' exists: ", exists("...", inherits = TRUE)) + y %<-% { sum(x, y, ...) } + y +} + + +for (strategy in c("mirai_cluster", "mirai_multisession")) { + plan(strategy, substitute = FALSE) + + for (name in names(sum_fcns)) { + mprintf("** Sum function '%s' with plan('%s') ...", name, strategy) + sum_fcn <- sum_fcns[[name]] + print(sum_fcn) + y <- tryCatch({ + sum_fcn(1:2, 3) + }, error = identity) + print(y) + if (name %in% c("D")) { + stopifnot(inherits(y, "error")) + } else { + stopifnot(y == 6) + } + } +} + +message("*** Global argument '...' in futures ... DONE") + +source("incl/end.R") diff --git a/tests/future,labels.R b/tests/future,labels.R new file mode 100644 index 0000000..ab2a76c --- /dev/null +++ b/tests/future,labels.R @@ -0,0 +1,39 @@ +source("incl/start.R") + +message("*** Futures - labels ...") + +strategies <- c("mirai_cluster", "mirai_multisession") + +for (strategy in strategies) { + mprintf("- plan('%s') ...", strategy) + plan(strategy) + + for (label in list(NULL, sprintf("strategy_%s", strategy))) { + fcn <- get(strategy, mode = "function") + stopifnot(inherits(fcn, strategy)) + f <- fcn(42, label = label) + print(f) + stopifnot(identical(f$label, label)) + v <- value(f) + stopifnot(v == 42) + + f <- future(42, label = label) + print(f) + stopifnot(identical(f$label, label)) + v <- value(f) + stopifnot(v == 42) + + v %<-% { 42 } %label% label + f <- futureOf(v) + print(f) + stopifnot(identical(f$label, label)) + stopifnot(v == 42) + + } ## for (label ...) + + mprintf("- plan('%s') ... DONE", strategy) +} ## for (strategy ...) + +message("*** Futures - labels ... DONE") + +source("incl/end.R") diff --git a/tests/future,lazy.R b/tests/future,lazy.R new file mode 100644 index 0000000..bce5add --- /dev/null +++ b/tests/future,lazy.R @@ -0,0 +1,27 @@ +source("incl/start.R") + +message("*** Futures - lazy ...") + +strategies <- c("mirai_cluster", "mirai_multisession") + +for (strategy in strategies) { + mprintf("- plan('%s') ...", strategy) + plan(strategy) + + a <- 42 + f <- future(2 * a, lazy = TRUE) + a <- 21 + v <- value(f) + stopifnot(v == 84) + + a <- 42 + v %<-% { 2 * a } %lazy% TRUE + a <- 21 + stopifnot(v == 84) + + mprintf("- plan('%s') ... DONE", strategy) +} ## for (strategy ...) + +message("*** Futures - lazy ... DONE") + +source("incl/end.R") diff --git a/tests/globals,formulas.R b/tests/globals,formulas.R new file mode 100644 index 0000000..e032b15 --- /dev/null +++ b/tests/globals,formulas.R @@ -0,0 +1,128 @@ +source("incl/start.R") + +library("datasets") ## cars data set +library("stats") ## lm(), poly(), xtabs() + +plan(mirai_multisession) + +message("*** Globals - formulas ...") + +message("*** Globals - lm() ...") + +## From example("lm", package = "stats") +ctl <- c(4.17, 5.58, 5.18, 6.11, 4.50, 4.61, 5.17, 4.53, 5.33, 5.14) +trt <- c(4.81, 4.17, 4.41, 3.59, 5.87, 3.83, 6.03, 4.89, 4.32, 4.69) +group <- gl(2, 10, 20, labels = c("Ctl", "Trt")) +weight <- c(ctl, trt) + +## Truth: +fit0 <- lm(weight ~ group - 1) +print(fit0) + +## Explicit future +f <- future({ lm(weight ~ group - 1) }) +fit <- value(f) +print(fit) +stopifnot(all.equal(fit, fit0)) + +## Future assignment +fit %<-% { lm(weight ~ group - 1) } +print(fit) +stopifnot(all.equal(fit, fit0)) + +message("*** Globals - lm() ... DONE") + + +message("*** Globals - one-side formulas, e.g. xtabs(~ x) ...") + +x <- c(1, 1, 2, 2, 2) + +## Truth: +tbl0 <- xtabs(~ x) +print(tbl0) + +## Explicit future +f <- future({ xtabs(~ x) }) +tbl <- value(f) +print(tbl) +stopifnot(all.equal(tbl, tbl0)) + +## Future assignment +tbl %<-% { xtabs(~ x) } +print(tbl) +stopifnot(all.equal(tbl, tbl0)) + +message("*** Globals - one-side formulas, e.g. xtabs(~ x) ... DONE") + + +message("*** Globals - lm(, data = cars) ...") + +exprs <- list( + # "remove-intercept-term" form of no-intercept + a = substitute({ lm(dist ~ . - 1, data = cars) }), + # "make-intercept-zero" form of no-intercept + b = substitute({ lm(dist ~ . + 0, data = cars) }), + # doesn't do what we want here + c = substitute({ lm(dist ~ speed + speed ^ 2, data = cars) }), + # gets us a quadratic term + d = substitute({ lm(dist ~ speed + I(speed ^ 2), data = cars) }), + # avoid potential multicollinearity + e = substitute({ lm(dist ~ poly(speed, 2), data = cars) }) +) + +for (kk in seq_along(exprs)) { + expr <- exprs[[kk]] + name <- names(exprs)[kk] + mprintf("- Globals - lm(, data = cars) ...", + kk, sQuote(name)) + + fit0 <- eval(expr) + print(fit0) + + f <- future(expr, substitute = FALSE) + fit <- value(f) + print(fit) + + stopifnot(all.equal(fit, fit0)) +} ## for (kk ...) + +message("*** Globals - lm(, data = cars) ... DONE") + + +message("*** Globals - map(x, ~ expr) ...") + +## A fake purrr::map() function with limited functionality +map <- function(.x, .f, ...) { + if (inherits(.f, "formula")) { + expr <- .f[[-1]] + .f <- eval(bquote(function(...) { + .(expr) + })) + } + eval(lapply(.x, FUN = .f, ...)) +} + +inner_function <- function(x) { x + 1 } + +outer_function <- function(x) { + map(1:2, ~ inner_function(.x)) +} + +y0 <- outer_function(1L) +str(y0) + +f <- future({ outer_function(1L) }) +y <- value(f) +str(y) +stopifnot(all.equal(y, y0)) + +y %<-% { outer_function(1L) } +str(y) +stopifnot(all.equal(y, y0)) + +message("*** Globals - map(x, ~ expr) ... DONE") + + +message("*** Globals - formulas ... DONE") + +source("incl/end.R") diff --git a/tests/globals,manual.R b/tests/globals,manual.R new file mode 100644 index 0000000..dad7973 --- /dev/null +++ b/tests/globals,manual.R @@ -0,0 +1,119 @@ +source("incl/start.R") + +plan(mirai_multisession) + +message("*** Globals - manually ...") + +message("*** Globals manually specified as named list ...") + +globals <- list( + a = 1, + b = 2, + sumtwo = function(x) x[1] + x[2] +) + +## Assign 'globals' globally +attach_locally(globals) + +## Truth +v0 <- local({ + x <- 1:10 + sumtwo(a + b * x) +}) + + +message("*** Globals - automatic ...") + +attach_locally(globals) +f <- future({ + x <- 1:10 + sumtwo(a + b * x) +}, globals = TRUE) +print(f) +rm(list = names(globals)) +y <- value(f) +print(y) +stopifnot(all.equal(y, v0)) + +attach_locally(globals) +y %<-% { + x <- 1:10 + sumtwo(a + b * x) +} %globals% TRUE +rm(list = names(globals)) +print(y) +stopifnot(all.equal(y, v0)) + +## No need to search for globals +y %<-% { 1 } %globals% FALSE +print(y) +stopifnot(identical(y, 1)) + +## Exception - missing global +attach_locally(globals) +f <- future({ + x <- 1:10 + sumtwo(a + b * x) +}, globals = FALSE) +print(f) +rm(list = names(globals)) +y <- tryCatch(value(f), error = identity) +if (!inherits(f, c("MulticoreFuture"))) { + stopifnot(inherits(y, "error")) +} + +message("*** Globals - automatic ... DONE") + + +message("*** Globals manually specified as named list ...") + +## Make sure globals do not exist +rm(list = names(globals)) + +f <- future({ + x <- 1:10 + sumtwo(a + b * x) +}, globals = globals) +print(f) +v <- value(f) +print(v) +stopifnot(all.equal(v, v0)) + +y %<-% { + x <- 1:10 + sumtwo(a + b * x) +} %globals% globals +print(y) +stopifnot(all.equal(y, v0)) + +message("*** Globals manually specified as named list ... DONE") + + +message("*** Globals manually specified by their names ...") + +attach_locally(globals) +f <- future({ + x <- 1:10 + sumtwo(a + b * x) +}, globals = c("a", "b", "sumtwo")) +print(f) +rm(list = names(globals)) +v <- value(f) +print(v) +stopifnot(all.equal(v, v0)) + +attach_locally(globals) +y %<-% { + x <- 1:10 + sumtwo(a + b * x) +} %globals% c("a", "b", "sumtwo") +rm(list = names(globals)) +print(y) +stopifnot(all.equal(y, v0)) + +message("*** Globals manually specified by their names ... DONE") + + +message("*** Globals - manually ... DONE") + +source("incl/end.R") diff --git a/tests/globals,subassignment.R b/tests/globals,subassignment.R new file mode 100644 index 0000000..f0a84f5 --- /dev/null +++ b/tests/globals,subassignment.R @@ -0,0 +1,129 @@ +source("incl/start.R") + +plan(mirai_multisession) + +oopts <- c(oopts, options( + future.globals.resolve = TRUE, + future.globals.onMissing = "error" +)) + + +message("*** Globals - subassignments ...") + +message("*** Globals - subassignments w/ x$a <- value ...") + +## Truth: +x <- x0 <- list() +y0 <- list(a = 1) +str(list(x = x, y0 = y0)) + +y <- local({ + x$a <- 1 + x +}) +stopifnot(identical(y, y0)) + +y <- local({ + x[["a"]] <- 1 + x +}) +stopifnot(identical(y, y0)) + +y <- local({ + x["a"] <- list(1) + x +}) +stopifnot(identical(y, y0)) + +stopifnot(identical(x, list())) + +## Explicit future +x <- list() +f <- future({ + x$a <- 1 + x +}) +rm(list = "x") +y <- value(f) +print(y) +stopifnot(identical(y, y0)) + +## Future assignment +x <- list() +y %<-% { + x$a <- 1 + x +} +rm(list = "x") +print(y) +stopifnot(identical(y, y0)) + +## 'x' is _not_ a global variable here +x <- list() +y %<-% { + x <- list(b = 2) + x$a <- 1 + x +} +rm(list = "x") +print(y) +stopifnot(identical(y, list(b = 2, a = 1))) + +## Explicit future +x <- list() +f <- future({ + x[["a"]] <- 1 + x +}) +rm(list = "x") +y <- value(f) +print(y) +stopifnot(identical(y, y0)) + +## Future assignment +x <- list() +y %<-% { + x[["a"]] <- 1 + x +} +rm(list = "x") +print(y) +stopifnot(identical(y, y0)) + +## Explicit future +x <- list() +f <- future({ + x["a"] <- list(1) + x +}) +rm(list = "x") +y <- value(f) +print(y) +stopifnot(identical(y, y0)) + +## Future assignment +x <- list() +y %<-% { + x["a"] <- list(1) + x +} +rm(list = "x") +print(y) +stopifnot(identical(y, y0)) + +## Future assignment +x <- list() +name <- "a" +y %<-% { + x[name] <- list(1) + x +} +rm(list = c("x", "name")) +print(y) +stopifnot(identical(y, y0)) + +message("*** Globals - subassignments w/ x$a <- value ... DONE") + +message("*** Globals - subassignments ... DONE") + +source("incl/end.R") diff --git a/tests/globals,tricky.R b/tests/globals,tricky.R new file mode 100644 index 0000000..ae960ca --- /dev/null +++ b/tests/globals,tricky.R @@ -0,0 +1,82 @@ +source("incl/start.R") +library("listenv") + +plan(mirai_multisession) + +message("*** Tricky use cases related to globals ...") + +message("- Globals with the same name as 'base' objects ...") + +## 'col' is masked by 'base::col' (Issue #55) + +col <- 3 +x %<-% { stopifnot(is.numeric(col)); col } +print(x) +stopifnot(x == col) + +message("- flapply(x, FUN = base::vector, ...) ...") + +flapply <- function(x, FUN, ...) { + res <- listenv() + for (ii in seq_along(x)) { + res[[ii]] %<-% FUN(x[[ii]], ...) + } + names(res) <- names(x) + + ## Make sure 'x', 'FUN' and 'ii' are truly + ## exported to the future environment + rm(list = c("x", "FUN", "ii")) + + as.list(res) +} + +x <- list(a = "integer", b = "numeric", c = "character", c = "list") +str(list(x = x)) + +y0 <- lapply(x, FUN = base::vector, length = 2L) +str(list(y0 = y0)) + +y <- flapply(x, FUN = base::vector, length = 2L) +str(list(y = y)) +stopifnot(identical(y, y0)) + + +message("- flapply(x, FUN = future:::hpaste, ...) ...") + +x <- list(a = c("hello", b = 1:100)) +str(list(x = x)) + +y0 <- lapply(x, FUN = future:::hpaste, collapse = "; ", maxHead = 3L) +str(list(y0 = y0)) + +y <- flapply(x, FUN = future:::hpaste, collapse = "; ", maxHead = 3L) +str(list(y = y)) +stopifnot(identical(y, y0)) + + +message("- flapply(x, FUN = listenv::listenv, ...) ...") + +x <- list() + +y <- listenv() +y$A <- 3L +x$a <- y + +y <- listenv() +y$A <- 3L +y$B <- c("hello", b = 1:100) +x$b <- y + +print(x) + +y0 <- lapply(x, FUN = listenv::mapping) +str(list(y0 = y0)) + +y <- flapply(x, FUN = listenv::mapping) +str(list(y = y)) +stopifnot(identical(y, y0)) + + +message("*** Tricky use cases related to globals ... DONE") + +source("incl/end.R") diff --git a/tests/incl/end.R b/tests/incl/end.R new file mode 100644 index 0000000..d64470b --- /dev/null +++ b/tests/incl/end.R @@ -0,0 +1,4 @@ +## Restore original state +options(oopts) +future::plan(oplan) +rm(list = c(setdiff(ls(), ovars))) diff --git a/tests/incl/start,load-only.R b/tests/incl/start,load-only.R new file mode 100644 index 0000000..99c2c5b --- /dev/null +++ b/tests/incl/start,load-only.R @@ -0,0 +1,33 @@ +## Record original state +ovars <- ls() +oopts <- options(warn = 1L, mc.cores = 2L, future.debug = TRUE) +oopts$future.delete <- getOption("future.delete") +oplan <- future::plan() + +## Use local mirai_multisession futures by default +future::plan(future.mirai::mirai_multisession) + +fullTest <- (Sys.getenv("_R_CHECK_FULL_") != "") + +all_strategies <- function() { + strategies <- Sys.getenv("R_FUTURE_TESTS_STRATEGIES") + strategies <- unlist(strsplit(strategies, split = ",")) + strategies <- gsub(" ", "", strategies) + strategies <- strategies[nzchar(strategies)] + strategies <- c(future:::supportedStrategies(), strategies) + unique(strategies) +} + +test_strategy <- function(strategy) { + strategy %in% all_strategies() +} + +attach_locally <- function(x, envir = parent.frame()) { + for (name in names(x)) { + assign(name, value = x[[name]], envir = envir) + } +} + +mprint <- future.mirai:::mprint +mprintf <- future.mirai:::mprintf + diff --git a/tests/incl/start.R b/tests/incl/start.R new file mode 100644 index 0000000..8ee5d3c --- /dev/null +++ b/tests/incl/start.R @@ -0,0 +1,2 @@ +library("future.mirai") +source("incl/start,load-only.R") diff --git a/tests/mirai_cluster,2.R b/tests/mirai_cluster,2.R new file mode 100644 index 0000000..d368f65 --- /dev/null +++ b/tests/mirai_cluster,2.R @@ -0,0 +1,115 @@ +source("incl/start.R") +library("listenv") + +message("*** mirai_multisession() ...") + +for (cores in 1:min(2L, availableCores())) { + ## FIXME: + if (!fullTest && cores > 1) next + + mprintf("Testing with %d cores ...", cores) + options(mc.cores = cores - 1L) + + for (globals in c(FALSE, TRUE)) { + mprintf("*** mirai_multisession(..., globals = %s) without globals", + globals) + + f <- mirai_multisession({ + 42L + }, globals = globals) + stopifnot(inherits(f, "MiraiFuture")) + + print(resolved(f)) + y <- value(f) + print(y) + stopifnot(y == 42L) + + mprintf("*** mirai_multisession(..., globals = %s) with globals", globals) + ## A global variable + a <- 0 + f <- mirai_multisession({ + b <- 3 + c <- 2 + a * b * c + }, globals = globals) + print(f) + + + ## A mirai_multisession future is evaluated in a separated + ## process. Changing the value of a global + ## variable should not affect the result of the + ## future. + a <- 7 ## Make sure globals are frozen + if (globals) { + v <- value(f) + print(v) + stopifnot(v == 0) + } else { + res <- tryCatch({ value(f) }, error = identity) + print(res) + stopifnot(inherits(res, "error")) + } + + + mprintf("*** mirai_multisession(..., globals = %s) with globals and blocking", globals) #nolint + x <- listenv() + for (ii in 1:3) { + mprintf(" - Creating mirai_multisession future #%d ...", ii) + x[[ii]] <- mirai_multisession({ ii }, globals = globals) + } + mprintf(" - Resolving %d mirai_multisession futures", length(x)) + if (globals) { + v <- sapply(x, FUN = value) + stopifnot(all(v == 1:3)) + } else { + v <- lapply(x, FUN = function(f) tryCatch(value(f), error = identity)) + stopifnot(all(sapply(v, FUN = inherits, "error"))) + } + + mprintf("*** mirai_multisession(..., globals = %s) and errors", globals) + f <- mirai_multisession({ + stop("Whoops!") + 1 + }, globals = globals) + print(f) + v <- value(f, signal = FALSE) + print(v) + stopifnot(inherits(v, "error")) + + res <- tryCatch({ + v <- value(f) + }, error = identity) + print(res) + stopifnot(inherits(res, "error")) + + ## Error is repeated + res <- tryCatch({ + v <- value(f) + }, error = identity) + print(res) + stopifnot(inherits(res, "error")) + + } # for (globals ...) + + + message("*** mirai_multisession(..., workers = 1L) ...") + + a <- 2 + b <- 3 + y_truth <- a * b + + f <- mirai_multisession({ a * b }, workers = 1L) + rm(list = c("a", "b")) + + v <- value(f) + print(v) + stopifnot(v == y_truth) + + message("*** mirai_multisession(..., workers = 1L) ... DONE") + + mprintf("Testing with %d cores ... DONE", cores) +} ## for (cores ...) + +message("*** mirai_multisession() ... DONE") + +source("incl/end.R") diff --git a/tests/mirai_cluster,worker-termination.R.HIDE b/tests/mirai_cluster,worker-termination.R.HIDE new file mode 100644 index 0000000..f5bef26 --- /dev/null +++ b/tests/mirai_cluster,worker-termination.R.HIDE @@ -0,0 +1,27 @@ +source("incl/start.R") + +message("*** mirai_multisession() - terminating workers ...") + +plan(mirai_multisession, workers = 2L) + +all <- nbrOfWorkers() +free <- nbrOfFreeWorkers() +stopifnot( + nbrOfWorkers() == 2L, + nbrOfFreeWorkers() == 2L +) + +## Force R worker to quit +f <- future({ tools::pskill(pid = Sys.getpid()) }) +res <- tryCatch(value(f), error = identity) +print(res) +stopifnot(inherits(res, "FutureError")) + +stopifnot( + nbrOfWorkers() == all, + nbrOfFreeWorkers() == free +) + +message("*** mirai_multisession() - terminating workers ... DONE") + +source("incl/end.R") diff --git a/tests/nbrOfWorkers.R b/tests/nbrOfWorkers.R new file mode 100644 index 0000000..53acdb3 --- /dev/null +++ b/tests/nbrOfWorkers.R @@ -0,0 +1,12 @@ +source("incl/start.R") + +message("*** nbrOfWorkers() ...") + +ncores <- availableCores() +n <- nbrOfWorkers(mirai_multisession) +message("Number of workers: ", n) +stopifnot(n == ncores) + +message("*** nbrOfWorkers() ... DONE") + +source("incl/end.R") diff --git a/tests/plan.R b/tests/plan.R new file mode 100644 index 0000000..4ae0f34 --- /dev/null +++ b/tests/plan.R @@ -0,0 +1,49 @@ +source("incl/start,load-only.R") + +message("*** plan() ...") + +message("*** future::plan(future.mirai::mirai_multisession)") +oplan <- future::plan(future.mirai::mirai_multisession) +print(future::plan()) +future::plan(oplan) +print(future::plan()) + + +library("future.mirai") +plan(mirai_multisession) + +for (type in c("mirai_multisession")) { + mprintf("*** plan('%s') ...", type) + + plan(type) + stopifnot(inherits(plan("next"), "mirai_multisession")) + + a <- 0 + f <- future({ + b <- 3 + c <- 2 + a * b * c + }) + a <- 7 ## Make sure globals are frozen + v <- value(f) + print(v) + stopifnot(v == 0) + + mprintf("*** plan('%s') ... DONE", type) +} # for (type ...) + + +message("*** Assert that default backend can be overridden ...") + +mpid <- Sys.getpid() +print(mpid) + +plan(mirai_multisession) +pid %<-% { Sys.getpid() } +print(pid) +stopifnot(pid != mpid) + + +message("*** plan() ... DONE") + +source("incl/end.R") diff --git a/tests/rng.R b/tests/rng.R new file mode 100644 index 0000000..f628398 --- /dev/null +++ b/tests/rng.R @@ -0,0 +1,99 @@ +source("incl/start.R") + +options(future.debug = FALSE) + +message("*** RNG ...") + +plan(mirai_multisession, workers = 2L) + +message("- run() does not update RNG state") + +f1 <- future(1, lazy = TRUE) +f2 <- future(2, lazy = TRUE) + +rng0 <- globalenv()$.Random.seed + +f1 <- run(f1) +stopifnot(identical(globalenv()$.Random.seed, rng0)) ## RNG changed? + +f2 <- run(f2) +stopifnot(identical(globalenv()$.Random.seed, rng0)) ## RNG changed? + +v1 <- value(f1) +stopifnot(identical(v1, 1)) + +v2 <- value(f2) +stopifnot(identical(v2, 2)) + + +message("- future() does not update RNG state") + +rng0 <- globalenv()$.Random.seed + +f1 <- future(1) +stopifnot(identical(globalenv()$.Random.seed, rng0)) ## RNG changed? + +f2 <- future(2) +stopifnot(identical(globalenv()$.Random.seed, rng0)) ## RNG changed? + +v1 <- value(f1) +stopifnot(identical(v1, 1)) + +v2 <- value(f2) +stopifnot(identical(v2, 2)) + + +message("- resolved() does not update RNG state") + +f1 <- future(1) +f2 <- future(2) + +d1 <- resolved(f1) +stopifnot(identical(globalenv()$.Random.seed, rng0)) ## RNG changed? + +d2 <- resolved(f2) +stopifnot(identical(globalenv()$.Random.seed, rng0)) ## RNG changed? + +v1 <- value(f1) +stopifnot(identical(v1, 1)) + +v2 <- value(f2) +stopifnot(identical(v2, 2)) + + +message("- result() does not update RNG state") + +f1 <- future(1) +f2 <- future(2) + +r1 <- result(f1) +stopifnot(identical(r1$value, 1)) +stopifnot(identical(globalenv()$.Random.seed, rng0)) ## RNG changed? + +r2 <- result(f2) +stopifnot(identical(r2$value, 2)) +stopifnot(identical(globalenv()$.Random.seed, rng0)) ## RNG changed? + +v1 <- value(f1) +stopifnot(identical(v1, 1)) + +v2 <- value(f2) +stopifnot(identical(v2, 2)) + + +message("- value() does not update RNG state") + +f1 <- future(1) +f2 <- future(2) + +v1 <- value(f1) +stopifnot(identical(v1, 1)) +stopifnot(identical(globalenv()$.Random.seed, rng0)) ## RNG changed? + +v2 <- value(f2) +stopifnot(identical(v2, 2)) +stopifnot(identical(globalenv()$.Random.seed, rng0)) ## RNG changed? + +message("*** RNG ... DONE") + +source("incl/end.R") diff --git a/tests/stdout.R b/tests/stdout.R new file mode 100644 index 0000000..a73a64d --- /dev/null +++ b/tests/stdout.R @@ -0,0 +1,65 @@ +source("incl/start.R") + +message("*** Standard output ...") + +truth_rows <- utils::capture.output({ + print(1:50) + str(1:50) + cat(letters, sep = "-") + cat(1:6, collapse = "\n") + write.table(datasets::iris[1:10,], sep = "\t") +}) +truth <- paste0(paste(truth_rows, collapse = "\n"), "\n") +print(truth) + +message("mirai_multisession ...") +plan(mirai_multisession) + +for (stdout in c(TRUE, FALSE, NA)) { + message(sprintf("- stdout = %s", stdout)) + + f <- future({ + print(1:50) + str(1:50) + cat(letters, sep = "-") + cat(1:6, collapse = "\n") + write.table(datasets::iris[1:10,], sep = "\t") + 42L + }, stdout = stdout) + r <- result(f) + str(r) + stopifnot(value(f) == 42L) + + if (is.na(stdout)) { + stopifnot(is.null(r$stdout)) + } else if (stdout) { + print(r) + stopifnot(identical(r$stdout, truth)) + } else { + stopifnot(is.null(r$stdout)) + } + + v %<-% { + print(1:50) + str(1:50) + cat(letters, sep = "-") + cat(1:6, collapse = "\n") + write.table(datasets::iris[1:10,], sep = "\t") + 42L + } %stdout% stdout + out <- utils::capture.output(y <- v) + stopifnot(y == 42L) + + if (is.na(stdout) || !stdout) { + stopifnot(out == "") + } else { + print(out) + stopifnot(identical(out, truth_rows)) + } +} ## for (stdout ...) + +message("mirai_multisession ... DONE") + +message("*** Standard output ... DONE") + +source("incl/end.R") diff --git a/tests/zzz,future_lapply.R b/tests/zzz,future_lapply.R new file mode 100644 index 0000000..be8721b --- /dev/null +++ b/tests/zzz,future_lapply.R @@ -0,0 +1,131 @@ +source("incl/start.R") +library("listenv") + +if (requireNamespace("future.apply", quietly = TRUE)) { + future_lapply <- future.apply::future_lapply + + strategies <- c("mirai_multisession") + + message("*** future_lapply() ...") + + message("- future_lapply(x, FUN = vector, ...) ...") + + x <- list(a = "integer", c = "character", c = "list") + str(list(x = x)) + + y0 <- lapply(x, FUN = vector, length = 2L) + str(list(y0 = y0)) + + for (strategy in strategies) { + mprintf("- plan('%s') ...", strategy) + plan(strategy) + stopifnot(nbrOfWorkers() < Inf) + + for (scheduling in list(FALSE, TRUE)) { + y <- future_lapply(x, FUN = vector, length = 2L, + future.scheduling = scheduling) + str(list(y = y)) + stopifnot(identical(y, y0)) + } + } + + + message("- future_lapply(x, FUN = base::vector, ...) ...") + + x <- list(a = "integer", c = "character", c = "list") + str(list(x = x)) + + y0 <- lapply(x, FUN = base::vector, length = 2L) + str(list(y0 = y0)) + + for (strategy in strategies) { + mprintf("- plan('%s') ...", strategy) + plan(strategy) + stopifnot(nbrOfWorkers() < Inf) + + for (scheduling in list(FALSE, TRUE)) { + y <- future_lapply(x, FUN = base::vector, length = 2L, + future.scheduling = scheduling) + str(list(y = y)) + stopifnot(identical(y, y0)) + } + } + + message("- future_lapply(x, FUN = future:::hpaste, ...) ...") + + x <- list(a = c("hello", b = 1:100)) + str(list(x = x)) + + y0 <- lapply(x, FUN = future:::hpaste, collapse = "; ", maxHead = 3L) + str(list(y0 = y0)) + + for (strategy in strategies) { + mprintf("- plan('%s') ...", strategy) + plan(strategy) + stopifnot(nbrOfWorkers() < Inf) + + for (scheduling in list(FALSE, TRUE)) { + y <- future_lapply(x, FUN = future:::hpaste, collapse = "; ", + maxHead = 3L, future.scheduling = scheduling) + str(list(y = y)) + stopifnot(identical(y, y0)) + } + } + + + message("- future_lapply(x, FUN = listenv::listenv, ...) ...") + + x <- list() + + y <- listenv() + y$A <- 3L + x$a <- y + + y <- listenv() + y$A <- 3L + y$B <- c("hello", b = 1:100) + x$b <- y + + print(x) + + y0 <- lapply(x, FUN = listenv::mapping) + str(list(y0 = y0)) + + for (strategy in strategies) { + mprintf("- plan('%s') ...", strategy) + plan(strategy) + stopifnot(nbrOfWorkers() < Inf) + + for (scheduling in list(FALSE, TRUE)) { + y <- future_lapply(x, FUN = listenv::mapping, future.scheduling = scheduling) + str(list(y = y)) + stopifnot(identical(y, y0)) + } + } + + + message("- future_lapply(x, FUN, ...) for large length(x) ...") + a <- 3.14 + x <- 1:1e6 + + y <- future_lapply(x, FUN = function(z) sqrt(z + a)) + y <- unlist(y, use.names = FALSE) + + stopifnot(all.equal(y, sqrt(x + a))) + + + message("- future_lapply() with global in non-attached package ...") + library("tools") + my_ext <- function(x) file_ext(x) + y_truth <- lapply("abc.txt", FUN = my_ext) + + for (strategy in strategies) { + plan(strategy) + y <- future_lapply("abc.txt", FUN = my_ext) + stopifnot(identical(y, y_truth)) + } + + message("*** future_lapply() ... DONE") +} + +source("incl/end.R") From 1cde4b16e70bf2f1497d6668f42c4e021d734ba6 Mon Sep 17 00:00:00 2001 From: Henrik Bengtsson Date: Thu, 11 Apr 2024 18:36:38 -0700 Subject: [PATCH 24/36] Rename files --- tests/future.tests,mirai_cluster.R | 15 ++ ...on.R => future.tests-mirai_multisession.R} | 0 tests/mirai_cluster,2.R | 115 ---------------- tests/mirai_cluster.R | 130 ++++++++++++++++-- 4 files changed, 130 insertions(+), 130 deletions(-) create mode 100644 tests/future.tests,mirai_cluster.R rename tests/{mirai_multisession.R => future.tests-mirai_multisession.R} (100%) delete mode 100644 tests/mirai_cluster,2.R diff --git a/tests/future.tests,mirai_cluster.R b/tests/future.tests,mirai_cluster.R new file mode 100644 index 0000000..1ce8b1f --- /dev/null +++ b/tests/future.tests,mirai_cluster.R @@ -0,0 +1,15 @@ +if (requireNamespace("future.tests")) { + mirai::daemons(0) ## Reset any daemons running + + ## FIXME: The following is disabled on MS Windows, because it will + ## result in a 'R CMD check' NOTE on "detritus in the temp directory" [1]. + ## This happens whenever we use mirai::daemons(..., dispatcher = TRUE). + ## [1] https://github.com/shikokuchuo/mirai/discussions/105 + dispatcher <- (.Platform[["OS.type"]] != "windows") + + mirai::daemons(2, dispatcher = dispatcher) + future.tests::check("future.mirai::mirai_cluster", timeout = 10.0, exit_value = FALSE) + + mirai::daemons(0) ## Reset any daemons running + gc() +} diff --git a/tests/mirai_multisession.R b/tests/future.tests-mirai_multisession.R similarity index 100% rename from tests/mirai_multisession.R rename to tests/future.tests-mirai_multisession.R diff --git a/tests/mirai_cluster,2.R b/tests/mirai_cluster,2.R deleted file mode 100644 index d368f65..0000000 --- a/tests/mirai_cluster,2.R +++ /dev/null @@ -1,115 +0,0 @@ -source("incl/start.R") -library("listenv") - -message("*** mirai_multisession() ...") - -for (cores in 1:min(2L, availableCores())) { - ## FIXME: - if (!fullTest && cores > 1) next - - mprintf("Testing with %d cores ...", cores) - options(mc.cores = cores - 1L) - - for (globals in c(FALSE, TRUE)) { - mprintf("*** mirai_multisession(..., globals = %s) without globals", - globals) - - f <- mirai_multisession({ - 42L - }, globals = globals) - stopifnot(inherits(f, "MiraiFuture")) - - print(resolved(f)) - y <- value(f) - print(y) - stopifnot(y == 42L) - - mprintf("*** mirai_multisession(..., globals = %s) with globals", globals) - ## A global variable - a <- 0 - f <- mirai_multisession({ - b <- 3 - c <- 2 - a * b * c - }, globals = globals) - print(f) - - - ## A mirai_multisession future is evaluated in a separated - ## process. Changing the value of a global - ## variable should not affect the result of the - ## future. - a <- 7 ## Make sure globals are frozen - if (globals) { - v <- value(f) - print(v) - stopifnot(v == 0) - } else { - res <- tryCatch({ value(f) }, error = identity) - print(res) - stopifnot(inherits(res, "error")) - } - - - mprintf("*** mirai_multisession(..., globals = %s) with globals and blocking", globals) #nolint - x <- listenv() - for (ii in 1:3) { - mprintf(" - Creating mirai_multisession future #%d ...", ii) - x[[ii]] <- mirai_multisession({ ii }, globals = globals) - } - mprintf(" - Resolving %d mirai_multisession futures", length(x)) - if (globals) { - v <- sapply(x, FUN = value) - stopifnot(all(v == 1:3)) - } else { - v <- lapply(x, FUN = function(f) tryCatch(value(f), error = identity)) - stopifnot(all(sapply(v, FUN = inherits, "error"))) - } - - mprintf("*** mirai_multisession(..., globals = %s) and errors", globals) - f <- mirai_multisession({ - stop("Whoops!") - 1 - }, globals = globals) - print(f) - v <- value(f, signal = FALSE) - print(v) - stopifnot(inherits(v, "error")) - - res <- tryCatch({ - v <- value(f) - }, error = identity) - print(res) - stopifnot(inherits(res, "error")) - - ## Error is repeated - res <- tryCatch({ - v <- value(f) - }, error = identity) - print(res) - stopifnot(inherits(res, "error")) - - } # for (globals ...) - - - message("*** mirai_multisession(..., workers = 1L) ...") - - a <- 2 - b <- 3 - y_truth <- a * b - - f <- mirai_multisession({ a * b }, workers = 1L) - rm(list = c("a", "b")) - - v <- value(f) - print(v) - stopifnot(v == y_truth) - - message("*** mirai_multisession(..., workers = 1L) ... DONE") - - mprintf("Testing with %d cores ... DONE", cores) -} ## for (cores ...) - -message("*** mirai_multisession() ... DONE") - -source("incl/end.R") diff --git a/tests/mirai_cluster.R b/tests/mirai_cluster.R index 1ce8b1f..d368f65 100644 --- a/tests/mirai_cluster.R +++ b/tests/mirai_cluster.R @@ -1,15 +1,115 @@ -if (requireNamespace("future.tests")) { - mirai::daemons(0) ## Reset any daemons running - - ## FIXME: The following is disabled on MS Windows, because it will - ## result in a 'R CMD check' NOTE on "detritus in the temp directory" [1]. - ## This happens whenever we use mirai::daemons(..., dispatcher = TRUE). - ## [1] https://github.com/shikokuchuo/mirai/discussions/105 - dispatcher <- (.Platform[["OS.type"]] != "windows") - - mirai::daemons(2, dispatcher = dispatcher) - future.tests::check("future.mirai::mirai_cluster", timeout = 10.0, exit_value = FALSE) - - mirai::daemons(0) ## Reset any daemons running - gc() -} +source("incl/start.R") +library("listenv") + +message("*** mirai_multisession() ...") + +for (cores in 1:min(2L, availableCores())) { + ## FIXME: + if (!fullTest && cores > 1) next + + mprintf("Testing with %d cores ...", cores) + options(mc.cores = cores - 1L) + + for (globals in c(FALSE, TRUE)) { + mprintf("*** mirai_multisession(..., globals = %s) without globals", + globals) + + f <- mirai_multisession({ + 42L + }, globals = globals) + stopifnot(inherits(f, "MiraiFuture")) + + print(resolved(f)) + y <- value(f) + print(y) + stopifnot(y == 42L) + + mprintf("*** mirai_multisession(..., globals = %s) with globals", globals) + ## A global variable + a <- 0 + f <- mirai_multisession({ + b <- 3 + c <- 2 + a * b * c + }, globals = globals) + print(f) + + + ## A mirai_multisession future is evaluated in a separated + ## process. Changing the value of a global + ## variable should not affect the result of the + ## future. + a <- 7 ## Make sure globals are frozen + if (globals) { + v <- value(f) + print(v) + stopifnot(v == 0) + } else { + res <- tryCatch({ value(f) }, error = identity) + print(res) + stopifnot(inherits(res, "error")) + } + + + mprintf("*** mirai_multisession(..., globals = %s) with globals and blocking", globals) #nolint + x <- listenv() + for (ii in 1:3) { + mprintf(" - Creating mirai_multisession future #%d ...", ii) + x[[ii]] <- mirai_multisession({ ii }, globals = globals) + } + mprintf(" - Resolving %d mirai_multisession futures", length(x)) + if (globals) { + v <- sapply(x, FUN = value) + stopifnot(all(v == 1:3)) + } else { + v <- lapply(x, FUN = function(f) tryCatch(value(f), error = identity)) + stopifnot(all(sapply(v, FUN = inherits, "error"))) + } + + mprintf("*** mirai_multisession(..., globals = %s) and errors", globals) + f <- mirai_multisession({ + stop("Whoops!") + 1 + }, globals = globals) + print(f) + v <- value(f, signal = FALSE) + print(v) + stopifnot(inherits(v, "error")) + + res <- tryCatch({ + v <- value(f) + }, error = identity) + print(res) + stopifnot(inherits(res, "error")) + + ## Error is repeated + res <- tryCatch({ + v <- value(f) + }, error = identity) + print(res) + stopifnot(inherits(res, "error")) + + } # for (globals ...) + + + message("*** mirai_multisession(..., workers = 1L) ...") + + a <- 2 + b <- 3 + y_truth <- a * b + + f <- mirai_multisession({ a * b }, workers = 1L) + rm(list = c("a", "b")) + + v <- value(f) + print(v) + stopifnot(v == y_truth) + + message("*** mirai_multisession(..., workers = 1L) ... DONE") + + mprintf("Testing with %d cores ... DONE", cores) +} ## for (cores ...) + +message("*** mirai_multisession() ... DONE") + +source("incl/end.R") From 85240fe30f8d49f7d920d1cce0aaea04c5c121fd Mon Sep 17 00:00:00 2001 From: Henrik Bengtsson Date: Fri, 12 Apr 2024 08:56:48 -0700 Subject: [PATCH 25/36] Use mirai::call_mirai_() instead of manual polling --- DESCRIPTION | 2 +- NAMESPACE | 1 + R/MiraiFuture-class.R | 7 ++----- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index fc7ba33..d5202be 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,5 +1,5 @@ Package: future.mirai -Version: 0.1.1-9014 +Version: 0.1.1-9015 Depends: future Imports: diff --git a/NAMESPACE b/NAMESPACE index a3285d0..e582c6f 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -21,6 +21,7 @@ importFrom(future,resolved) importFrom(future,result) importFrom(future,run) importFrom(future,tweak) +importFrom(mirai,call_mirai_) importFrom(mirai,daemons) importFrom(mirai,is_error_value) importFrom(mirai,mirai) diff --git a/R/MiraiFuture-class.R b/R/MiraiFuture-class.R index ef7bc6d..b6dfb5a 100644 --- a/R/MiraiFuture-class.R +++ b/R/MiraiFuture-class.R @@ -152,6 +152,7 @@ mirai_version <- local({ }) #' @importFrom future result +#' @importFrom mirai call_mirai_ #' @export result.MiraiFuture <- function(future, ...) { if(isTRUE(future[["state"]] == "finished")) { @@ -165,11 +166,7 @@ result.MiraiFuture <- function(future, ...) { } mirai <- future[["mirai"]] - while (unresolved(mirai)) { - Sys.sleep(0.1) - } - - result <- mirai$data + result <- call_mirai_(mirai)$data future[["result"]] <- result future[["state"]] <- "finished" From 0d589656269e49f8797994379e77d5bdea10118d Mon Sep 17 00:00:00 2001 From: Henrik Bengtsson Date: Fri, 12 Apr 2024 11:08:51 -0700 Subject: [PATCH 26/36] Handle crashed mirai workers --- DESCRIPTION | 2 +- R/MiraiFuture-class.R | 13 +++++++++++-- R/nbrOfWorkers.R | 5 ++++- ...on.R.HIDE => mirai_cluster,worker-termination.R} | 13 +++++++------ 4 files changed, 23 insertions(+), 10 deletions(-) rename tests/{mirai_cluster,worker-termination.R.HIDE => mirai_cluster,worker-termination.R} (71%) diff --git a/DESCRIPTION b/DESCRIPTION index d5202be..056970b 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,5 +1,5 @@ Package: future.mirai -Version: 0.1.1-9015 +Version: 0.1.1-9016 Depends: future Imports: diff --git a/R/MiraiFuture-class.R b/R/MiraiFuture-class.R index b6dfb5a..38fa3cd 100644 --- a/R/MiraiFuture-class.R +++ b/R/MiraiFuture-class.R @@ -58,7 +58,7 @@ MiraiFuture <- function(expr = NULL, daemons(n = 0L) } else if (workers != nworkers) { daemons(n = 0L) ## reset is required - daemons(n = workers, dispatcher = dispatcher) + daemons(n = workers, dispatcher = dispatcher, resilience = FALSE) } } else if (!is.null(workers)) { stop("Argument 'workers' should be a numeric scalar or NULL: ", mode(workers)) @@ -167,6 +167,14 @@ result.MiraiFuture <- function(future, ...) { mirai <- future[["mirai"]] result <- call_mirai_(mirai)$data + + if (inherits(result, "errorValue")) { + label <- future$label + if (is.null(label)) label <- "" + msg <- sprintf("Failed to retrieve results from %s (%s). The mirai framework reports on error value %s", class(future)[1], label, result) + stop(FutureError(msg)) + } + future[["result"]] <- result future[["state"]] <- "finished" @@ -182,7 +190,8 @@ mirai_daemons_nworkers <- function() { if (is.data.frame(workers)) return(nrow(workers)) if (length(workers) != 1L) { - stop(FutureError(sprintf("Length of mirai::status()$daemons is not one: %d", length(workers)))) + msg <- sprintf("Length of mirai::status()$daemons is not one: %d", length(workers)) + stop(FutureError(msg)) } if (workers == 0L) return(Inf) diff --git a/R/nbrOfWorkers.R b/R/nbrOfWorkers.R index b7b22bf..ab40bbf 100644 --- a/R/nbrOfWorkers.R +++ b/R/nbrOfWorkers.R @@ -33,7 +33,10 @@ nbrOfWorkers.mirai <- function(evaluator) { nbrOfFreeWorkers.mirai <- function(evaluator, background = FALSE, ...) { res <- status() workers <- res[["daemons"]] - if (!is.numeric(workers)) { + if (is.character(workers)) { + workers <- res[["connections"]] + stopifnot(is.numeric(workers)) + } else if (!is.numeric(workers)) { stop(FutureError(sprintf("Unknown type of mirai::daemons()$daemons: %s", typeof(workers)))) } diff --git a/tests/mirai_cluster,worker-termination.R.HIDE b/tests/mirai_cluster,worker-termination.R similarity index 71% rename from tests/mirai_cluster,worker-termination.R.HIDE rename to tests/mirai_cluster,worker-termination.R index f5bef26..e1535a9 100644 --- a/tests/mirai_cluster,worker-termination.R.HIDE +++ b/tests/mirai_cluster,worker-termination.R @@ -5,11 +5,12 @@ message("*** mirai_multisession() - terminating workers ...") plan(mirai_multisession, workers = 2L) all <- nbrOfWorkers() +message("Number of workers: ", all) +stopifnot(all == 2L) free <- nbrOfFreeWorkers() -stopifnot( - nbrOfWorkers() == 2L, - nbrOfFreeWorkers() == 2L -) +message("Number of free workers: ", all) +stopifnot(free == 2L) + ## Force R worker to quit f <- future({ tools::pskill(pid = Sys.getpid()) }) @@ -18,8 +19,8 @@ print(res) stopifnot(inherits(res, "FutureError")) stopifnot( - nbrOfWorkers() == all, - nbrOfFreeWorkers() == free + nbrOfWorkers() == all - 1L, + nbrOfFreeWorkers() == free - 1L ) message("*** mirai_multisession() - terminating workers ... DONE") From 2d1b9194f1114f0d42ff72c507e8a68f4a23dfcf Mon Sep 17 00:00:00 2001 From: Henrik Bengtsson Date: Fri, 12 Apr 2024 11:18:28 -0700 Subject: [PATCH 27/36] Skip worker termination test on MS Windows --- tests/mirai_cluster,worker-termination.R | 26 ++++++++++++++---------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/tests/mirai_cluster,worker-termination.R b/tests/mirai_cluster,worker-termination.R index e1535a9..cf09eb2 100644 --- a/tests/mirai_cluster,worker-termination.R +++ b/tests/mirai_cluster,worker-termination.R @@ -11,17 +11,21 @@ free <- nbrOfFreeWorkers() message("Number of free workers: ", all) stopifnot(free == 2L) - -## Force R worker to quit -f <- future({ tools::pskill(pid = Sys.getpid()) }) -res <- tryCatch(value(f), error = identity) -print(res) -stopifnot(inherits(res, "FutureError")) - -stopifnot( - nbrOfWorkers() == all - 1L, - nbrOfFreeWorkers() == free - 1L -) +## Don't test on MS Windows, because that will leave behind a +## stray Rscript file, which 'R CMD check --as-cran' +## will complain about. /HB 2024-04-12 +if (.Platform$OS.type != "windows") { + ## Force R worker to quit + f <- future({ tools::pskill(pid = Sys.getpid()) }) + res <- tryCatch(value(f), error = identity) + print(res) + stopifnot(inherits(res, "FutureError")) + + stopifnot( + nbrOfWorkers() == all - 1L, + nbrOfFreeWorkers() == free - 1L + ) +} message("*** mirai_multisession() - terminating workers ... DONE") From 8eddd8f1488135fd153b4523aefafa1ebc205b31 Mon Sep 17 00:00:00 2001 From: Henrik Bengtsson Date: Fri, 12 Apr 2024 11:44:02 -0700 Subject: [PATCH 28/36] Relax dependency requirements: All for mirai (>= 0.12.1) such that we can submit to CRAN already now --- DESCRIPTION | 6 +++--- tests/globals,nested.R | 5 +++++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 056970b..2a210e5 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,9 +1,9 @@ Package: future.mirai -Version: 0.1.1-9016 +Version: 0.1.1-9017 Depends: future Imports: - mirai (>= 0.13.2), + mirai (>= 0.12.1), parallelly, utils Suggests: @@ -11,7 +11,7 @@ Suggests: future.apply, listenv Title: A Future API for Parallel Processing using 'mirai' -Description: A Future API implementation on top of the 'mirai' package. +Description: Implementation of the Future API on top of the 'mirai' package. This allows you to process futures, as defined by the 'future' package, in parallel out of the box, on your local (Linux, macOS, Windows, ...) machine. Contrary to backends relying on the 'parallel' package (e.g. 'future::multisession') and socket connections, the 'mirai_cluster' and 'mirai_multisession' backends provided here can run more than 125 parallel R processes. Authors@R: c( person("Henrik", "Bengtsson", role = c("aut", "cre", "cph"), diff --git a/tests/globals,nested.R b/tests/globals,nested.R index 7cae9dc..60c0e46 100644 --- a/tests/globals,nested.R +++ b/tests/globals,nested.R @@ -1,4 +1,7 @@ +## This requires mirai (>= 0.13.2) +if (packageVersion("mirai") >= "0.13.2") { library(future.mirai) + mirai::daemons(1, dispatcher = FALSE) plan(mirai_cluster) @@ -13,3 +16,5 @@ stopifnot(v == h()) plan(sequential) mirai::daemons(0) ## Reset any daemons running gc() + +} From b46a112726139064ccb1619dafc4dbcf9213a609 Mon Sep 17 00:00:00 2001 From: Henrik Bengtsson Date: Fri, 12 Apr 2024 11:54:11 -0700 Subject: [PATCH 29/36] GA: All checks ran the same R version --- .github/workflows/R-CMD-check.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml index d78db59..25ce2ea 100644 --- a/.github/workflows/R-CMD-check.yaml +++ b/.github/workflows/R-CMD-check.yaml @@ -56,6 +56,7 @@ jobs: - uses: r-lib/actions/setup-r@v2 with: use-public-rspm: true + r-version: ${{ matrix.config.r }} - uses: r-lib/actions/setup-r-dependencies@v2 with: From 9a91f382e46441cefc5c46adfe0b24924f587f3b Mon Sep 17 00:00:00 2001 From: Henrik Bengtsson Date: Fri, 12 Apr 2024 11:58:01 -0700 Subject: [PATCH 30/36] TESTS: Another two tests need to be conditioned on mirai (>= 0.13.2) --- DESCRIPTION | 2 +- tests/globals,formulas.R | 19 +++++++++++-------- tests/zzz,future_lapply.R | 5 +++++ 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 2a210e5..0e67637 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,5 +1,5 @@ Package: future.mirai -Version: 0.1.1-9017 +Version: 0.1.1-9018 Depends: future Imports: diff --git a/tests/globals,formulas.R b/tests/globals,formulas.R index e032b15..f61559d 100644 --- a/tests/globals,formulas.R +++ b/tests/globals,formulas.R @@ -111,14 +111,17 @@ outer_function <- function(x) { y0 <- outer_function(1L) str(y0) -f <- future({ outer_function(1L) }) -y <- value(f) -str(y) -stopifnot(all.equal(y, y0)) - -y %<-% { outer_function(1L) } -str(y) -stopifnot(all.equal(y, y0)) +## This requires mirai (>= 0.13.2) +if (packageVersion("mirai") >= "0.13.2") { + f <- future({ outer_function(1L) }) + y <- value(f) + str(y) + stopifnot(all.equal(y, y0)) + + y %<-% { outer_function(1L) } + str(y) + stopifnot(all.equal(y, y0)) +} message("*** Globals - map(x, ~ expr) ... DONE") diff --git a/tests/zzz,future_lapply.R b/tests/zzz,future_lapply.R index be8721b..0fffa61 100644 --- a/tests/zzz,future_lapply.R +++ b/tests/zzz,future_lapply.R @@ -1,3 +1,6 @@ +## This requires mirai (>= 0.13.2) +if (packageVersion("mirai") >= "0.13.2") { + source("incl/start.R") library("listenv") @@ -129,3 +132,5 @@ if (requireNamespace("future.apply", quietly = TRUE)) { } source("incl/end.R") + +} From 0964136fae25cda17ed1aa64199c3d6e40a6362d Mon Sep 17 00:00:00 2001 From: Henrik Bengtsson Date: Fri, 12 Apr 2024 12:12:18 -0700 Subject: [PATCH 31/36] Examples did not work in R (< 4.0.0), because they used the new lambda notation --- DESCRIPTION | 2 +- incl/mirai_cluster.R | 2 +- incl/mirai_multisession.R | 2 +- man/mirai_cluster.Rd | 2 +- man/mirai_multisession.Rd | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 0e67637..decfc51 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,5 +1,5 @@ Package: future.mirai -Version: 0.1.1-9018 +Version: 0.1.1-9019 Depends: future Imports: diff --git a/incl/mirai_cluster.R b/incl/mirai_cluster.R index 52a2b4b..00b6362 100644 --- a/incl/mirai_cluster.R +++ b/incl/mirai_cluster.R @@ -2,7 +2,7 @@ mirai::daemons(parallelly::availableCores(), dispatcher = FALSE) plan(mirai_cluster) # A function that returns a future, note that N uses lexical scoping... -f <- \() future({4 * sum((runif(N) ^ 2 + runif(N) ^ 2) < 1) / N}, seed = TRUE) +f <- function() future({4 * sum((runif(N) ^ 2 + runif(N) ^ 2) < 1) / N}, seed = TRUE) # Run a simple sampling approximation of pi in parallel using M * N points: N <- 1e6 # samples per worker diff --git a/incl/mirai_multisession.R b/incl/mirai_multisession.R index d525316..c3fe910 100644 --- a/incl/mirai_multisession.R +++ b/incl/mirai_multisession.R @@ -1,7 +1,7 @@ plan(mirai_multisession) # A function that returns a future, note that N uses lexical scoping... -f <- \() future({4 * sum((runif(N) ^ 2 + runif(N) ^ 2) < 1) / N}, seed = TRUE) +f <- function() future({4 * sum((runif(N) ^ 2 + runif(N) ^ 2) < 1) / N}, seed = TRUE) # Run a simple sampling approximation of pi in parallel using M * N points: N <- 1e6 # samples per worker diff --git a/man/mirai_cluster.Rd b/man/mirai_cluster.Rd index 75a58da..3623fbe 100644 --- a/man/mirai_cluster.Rd +++ b/man/mirai_cluster.Rd @@ -28,7 +28,7 @@ mirai::daemons(parallelly::availableCores(), dispatcher = FALSE) plan(mirai_cluster) # A function that returns a future, note that N uses lexical scoping... -f <- \() future({4 * sum((runif(N) ^ 2 + runif(N) ^ 2) < 1) / N}, seed = TRUE) +f <- function() future({4 * sum((runif(N) ^ 2 + runif(N) ^ 2) < 1) / N}, seed = TRUE) # Run a simple sampling approximation of pi in parallel using M * N points: N <- 1e6 # samples per worker diff --git a/man/mirai_multisession.Rd b/man/mirai_multisession.Rd index 9e98687..105f4c3 100644 --- a/man/mirai_multisession.Rd +++ b/man/mirai_multisession.Rd @@ -37,7 +37,7 @@ Mirai-based localhost multisession futures plan(mirai_multisession) # A function that returns a future, note that N uses lexical scoping... -f <- \() future({4 * sum((runif(N) ^ 2 + runif(N) ^ 2) < 1) / N}, seed = TRUE) +f <- function() future({4 * sum((runif(N) ^ 2 + runif(N) ^ 2) < 1) / N}, seed = TRUE) # Run a simple sampling approximation of pi in parallel using M * N points: N <- 1e6 # samples per worker From 94b5e0a4da2e4ae58b1cf0ae77080dcad456b95b Mon Sep 17 00:00:00 2001 From: Henrik Bengtsson Date: Fri, 12 Apr 2024 12:15:00 -0700 Subject: [PATCH 32/36] Avoid 'possibly misspelled words' on CRAN incoming --- DESCRIPTION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index decfc51..d7d8a7b 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,5 +1,5 @@ Package: future.mirai -Version: 0.1.1-9019 +Version: 0.1.1-9020 Depends: future Imports: @@ -11,7 +11,7 @@ Suggests: future.apply, listenv Title: A Future API for Parallel Processing using 'mirai' -Description: Implementation of the Future API on top of the 'mirai' package. This allows you to process futures, as defined by the 'future' package, in parallel out of the box, on your local (Linux, macOS, Windows, ...) machine. Contrary to backends relying on the 'parallel' package (e.g. 'future::multisession') and socket connections, the 'mirai_cluster' and 'mirai_multisession' backends provided here can run more than 125 parallel R processes. +Description: Implementation of the Future API on top of the 'mirai' package. This allows you to process futures, as defined by the 'future' package, in parallel out of the box, on your local machine or across remote machines. Contrary to backend frameworks relying on the 'parallel' package (e.g. 'future::multisession') and socket connections, 'mirai_cluster' and 'mirai_multisession', provided here, can run more than 125 parallel R processes. Authors@R: c( person("Henrik", "Bengtsson", role = c("aut", "cre", "cph"), From a24c060fdd6742e46d393b4604715b60077ada28 Mon Sep 17 00:00:00 2001 From: Henrik Bengtsson Date: Fri, 12 Apr 2024 12:15:17 -0700 Subject: [PATCH 33/36] future.mirai 0.2.0 --- DESCRIPTION | 2 +- NEWS.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index d7d8a7b..a5497cb 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,5 +1,5 @@ Package: future.mirai -Version: 0.1.1-9020 +Version: 0.2.0 Depends: future Imports: diff --git a/NEWS.md b/NEWS.md index 5e61cbb..297ef5c 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,6 +1,6 @@ -# Version (development version) +# Version 0.2.0 - * ... + * First public release. # Version 0.1.1 From faa5c17273e55f597231773067d0f60dfd26942d Mon Sep 17 00:00:00 2001 From: Henrik Bengtsson Date: Fri, 12 Apr 2024 11:51:44 -0700 Subject: [PATCH 34/36] Add CRAN comments --- README.md | 35 ++++++++++++++++++++--------------- cran-comments.md | 26 ++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 15 deletions(-) create mode 100644 cran-comments.md diff --git a/README.md b/README.md index 9b41ce6..cddee37 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,3 @@ - -
R CMD check status future.tests checks status Coverage Status
@@ -85,7 +83,7 @@ plan(mirai_multisession) demo("mandelbrot", package = "future", ask = FALSE) ``` -and +To use `mirai_cluster` futures, use: ```r library(future.mirai) @@ -96,27 +94,34 @@ demo("mandelbrot", package = "future", ask = FALSE) ``` -[mirai]: https://cran.r-project.org/package=mirai -[future]: https://cran.r-project.org/package=future -[future.mirai]: https://github.com/HenrikBengtsson/future.mirai -[future.apply]: https://cran.r-project.org/package=future.apply -[furrr]: https://cran.r-project.org/package=furrr -[doFuture]: https://cran.r-project.org/package=doFuture - ## Installation -R package future.mirai is only available via [GitHub](https://github.com/HenrikBengtsson/future.mirai) and can be installed in R as: + +R package **future.mirai** is available on +[CRAN](https://cran.r-project.org/package=future.mirai) and can be +installed in R as: + ```r -remotes::install_github("HenrikBengtsson/future.mirai", ref="master") +install.packages("future.mirai") ``` ### Pre-release version -To install the pre-release version that is available in Git branch `develop` on GitHub, use: +To install the pre-release version that is available in Git branch +`develop` on GitHub, use: + ```r remotes::install_github("HenrikBengtsson/future.mirai", ref="develop") ``` -This will install the package from source. - +This will install the package from source. + +[mirai]: https://cran.r-project.org/package=mirai +[future]: https://cran.r-project.org/package=future +[future.mirai]: https://github.com/HenrikBengtsson/future.mirai +[future.apply]: https://cran.r-project.org/package=future.apply +[furrr]: https://cran.r-project.org/package=furrr +[doFuture]: https://cran.r-project.org/package=doFuture + + diff --git a/cran-comments.md b/cran-comments.md new file mode 100644 index 0000000..d99ec79 --- /dev/null +++ b/cran-comments.md @@ -0,0 +1,26 @@ +# CRAN submission future.mirai 0.2.0 + +on 2024-04-12 + +This is a new package. + +I'm using the same terminology, nomenclature and formatting in the package title and description that I use for other closely related "sibling" packages on CRAN, e.g. future.batchtools and future.callr. + +Thanks in advance + + +## Notes not sent to CRAN + +### R CMD check validation + +The package has been verified using `R CMD check --as-cran` on: + +| R version | GitHub | mac/win-builder | +| --------- | ------ | --------------- | +| 3.6.x | L | | +| 4.2.x | L W | W | +| 4.3.x | L M W | M1 W | +| 4.4.x | | W | +| devel | L M W | | + +*Legend: OS: L = Linux, M = macOS, M1 = macOS M1, W = Windows* From f1a06d2bdac170ac071539868a72b90dec6f8370 Mon Sep 17 00:00:00 2001 From: Henrik Bengtsson Date: Fri, 12 Apr 2024 12:27:00 -0700 Subject: [PATCH 35/36] backends -> back-ends trying to avoid 'possibly misspelled words' on CRAN incoming --- DESCRIPTION | 2 +- cran-comments.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index a5497cb..16f54eb 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -11,7 +11,7 @@ Suggests: future.apply, listenv Title: A Future API for Parallel Processing using 'mirai' -Description: Implementation of the Future API on top of the 'mirai' package. This allows you to process futures, as defined by the 'future' package, in parallel out of the box, on your local machine or across remote machines. Contrary to backend frameworks relying on the 'parallel' package (e.g. 'future::multisession') and socket connections, 'mirai_cluster' and 'mirai_multisession', provided here, can run more than 125 parallel R processes. +Description: Implementation of the Future API on top of the 'mirai' package. This allows you to process futures, as defined by the 'future' package, in parallel out of the box, on your local machine or across remote machines. Contrary to back-ends relying on the 'parallel' package (e.g. 'multisession') and socket connections, 'mirai_cluster' and 'mirai_multisession', provided here, can run more than 125 parallel R processes. Authors@R: c( person("Henrik", "Bengtsson", role = c("aut", "cre", "cph"), diff --git a/cran-comments.md b/cran-comments.md index d99ec79..9225ed8 100644 --- a/cran-comments.md +++ b/cran-comments.md @@ -19,7 +19,7 @@ The package has been verified using `R CMD check --as-cran` on: | --------- | ------ | --------------- | | 3.6.x | L | | | 4.2.x | L W | W | -| 4.3.x | L M W | M1 W | +| 4.3.x | L M W | . W | | 4.4.x | | W | | devel | L M W | | From 2c93986c71c810eb87f91a05bae4998e96ffde30 Mon Sep 17 00:00:00 2001 From: Henrik Bengtsson Date: Thu, 18 Apr 2024 12:14:17 -0700 Subject: [PATCH 36/36] Update package title and description according to CRAN newbie review feedback --- DESCRIPTION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 16f54eb..9bc82d8 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -10,8 +10,8 @@ Suggests: future.tests, future.apply, listenv -Title: A Future API for Parallel Processing using 'mirai' -Description: Implementation of the Future API on top of the 'mirai' package. This allows you to process futures, as defined by the 'future' package, in parallel out of the box, on your local machine or across remote machines. Contrary to back-ends relying on the 'parallel' package (e.g. 'multisession') and socket connections, 'mirai_cluster' and 'mirai_multisession', provided here, can run more than 125 parallel R processes. +Title: A 'Future' API for Parallel Processing using 'mirai' +Description: Implementation of the 'Future' API on top of the 'mirai' package. This allows you to process futures, as defined by the 'future' package, in parallel out of the box, on your local machine or across remote machines. Contrary to back-ends relying on the 'parallel' package (e.g. 'multisession') and socket connections, 'mirai_cluster' and 'mirai_multisession', provided here, can run more than 125 parallel R processes. Authors@R: c( person("Henrik", "Bengtsson", role = c("aut", "cre", "cph"),