From 57f8bbe2fba54c282b14814cada32ef6d5e9a90c Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Mon, 3 Jun 2024 16:35:26 -0500 Subject: [PATCH 1/7] Document and fail fast if tmp directory is noexec --- README.md | 4 ++++ .../bioformats2raw/Converter.java | 19 +++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/README.md b/README.md index f0c3dd3..440ff5c 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,10 @@ If using features that rely on OpenCV (see the [Downsampling type](#downsampling __NOTE:__ If you are setting `jna.library.path` via the `JAVA_OPTS` environment variable, make sure the path is to the folder __containing__ the library not path to the library itself. +If the default temporary directory (usually `/tmp/`) is mounted as `noexec`, conversion will fail. +The easiest solution is to choose a different temporary directory by adding `-Djava.io.tmpdir=/path/to/alternate/tmp` to `JAVA_OPTS`. +If multiple properties need to be set via `JAVA_OPTS`, separate them with a space, e.g. `JAVA_OPTS=-Djava.io.tmpdir/path/to/alternate/tmp -Djna.library.path=/path/to/blosc`. + Installation ============ diff --git a/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java b/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java index ca54f63..0745c0e 100644 --- a/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java +++ b/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java @@ -1172,6 +1172,25 @@ public Integer call() throws Exception { tileWidth, tileHeight); } + // if the tmpdir is mounted as noexec, then any native library + // loading (OpenCV, turbojpeg, etc.) is expected to fail, which + // can cause conversion to fail with an exception that is not user friendly + // + // try to detect this case early and fail before the conversion begins, + // with a more informative message + + // a temp file is created and set as executable + // in the noexec case, setting as executable is expected to silently fail + File tmpdirCheck = File.createTempFile("noexec-test", ".txt"); + // expect 'success' to be true in the noexec case, even though + // the file will not actually be executable + boolean success = tmpdirCheck.setExecutable(true); + tmpdirCheck.deleteOnExit(); + if (!success || !tmpdirCheck.canExecute()) { + throw new RuntimeException(System.getProperty("java.io.tmpdir") + + " is noexec; fix it or specify a different java.io.tmpdir"); + } + OpenCVTools.loadOpenCV(); if (progressBars) { From 9dca8a96d4717a51a81a95d59b37b94c03d3db3b Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Mon, 15 Jul 2024 14:19:05 -0500 Subject: [PATCH 2/7] Delete tmp check file right away --- src/main/java/com/glencoesoftware/bioformats2raw/Converter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java b/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java index 0745c0e..b972eda 100644 --- a/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java +++ b/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java @@ -1185,11 +1185,11 @@ public Integer call() throws Exception { // expect 'success' to be true in the noexec case, even though // the file will not actually be executable boolean success = tmpdirCheck.setExecutable(true); - tmpdirCheck.deleteOnExit(); if (!success || !tmpdirCheck.canExecute()) { throw new RuntimeException(System.getProperty("java.io.tmpdir") + " is noexec; fix it or specify a different java.io.tmpdir"); } + tmpdirCheck.delete(); OpenCVTools.loadOpenCV(); From cc2ec9315a158f22d6015520980b1ecb45f07012 Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Mon, 15 Jul 2024 14:35:20 -0500 Subject: [PATCH 3/7] Add `--warn-no-exec` option to log a warning instead of throwing an exception --- .../bioformats2raw/Converter.java | 37 ++++++++++++++++++- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java b/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java index b972eda..00f7ab9 100644 --- a/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java +++ b/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java @@ -128,6 +128,8 @@ public class Converter implements Callable { private volatile Path inputPath; private volatile String outputLocation; + private volatile boolean warnNoExec = false; + private Map outputOptions; private volatile Integer pyramidResolutions; private volatile List seriesList; @@ -409,6 +411,23 @@ public void setProgressBars(boolean useProgressBars) { progressBars = useProgressBars; } + /** + * Configure whether to warn instead of throwing an exception + * if files created in the temporary directory + * ("java.io.tmpdir" system property) will not be executable. + * + * @param warnOnly true if a warning should be logged instead of an exception + */ + @Option( + names = {"--warn-no-exec"}, + description = "Warn instead of throwing an exception if " + + "java.io.tmpdir is not executable", + defaultValue = "false" + ) + public void setWarnOnNoExec(boolean warnOnly) { + warnNoExec = warnOnly; + } + /** * Configure whether to print version information and exit * without converting. @@ -944,6 +963,14 @@ public boolean getProgressBars() { return progressBars; } + /** + * @return true if a warning is logged instead of an exception when the tmp + * directory is noexec + */ + public boolean getWarnNoExec() { + return warnNoExec; + } + /** * @return true if only version info is displayed */ @@ -1186,8 +1213,14 @@ public Integer call() throws Exception { // the file will not actually be executable boolean success = tmpdirCheck.setExecutable(true); if (!success || !tmpdirCheck.canExecute()) { - throw new RuntimeException(System.getProperty("java.io.tmpdir") + - " is noexec; fix it or specify a different java.io.tmpdir"); + String msg = System.getProperty("java.io.tmpdir") + + " is noexec; fix it or specify a different java.io.tmpdir"; + if (getWarnNoExec()) { + LOGGER.warn(msg); + } + else { + throw new RuntimeException(msg); + } } tmpdirCheck.delete(); From 862e4ac4868b2bd8423a85158e5797104b754e2e Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Mon, 15 Jul 2024 16:48:27 -0500 Subject: [PATCH 4/7] Split tmp directory notes into their own section --- README.md | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 440ff5c..49e3f73 100644 --- a/README.md +++ b/README.md @@ -33,10 +33,29 @@ If using features that rely on OpenCV (see the [Downsampling type](#downsampling __NOTE:__ If you are setting `jna.library.path` via the `JAVA_OPTS` environment variable, make sure the path is to the folder __containing__ the library not path to the library itself. -If the default temporary directory (usually `/tmp/`) is mounted as `noexec`, conversion will fail. -The easiest solution is to choose a different temporary directory by adding `-Djava.io.tmpdir=/path/to/alternate/tmp` to `JAVA_OPTS`. +Temporary directory usage +========================= + +Beginning with 0.10.0, if the default temporary directory (usually `/tmp/`) is mounted as `noexec`, conversion will fail immediately by default. +[CIS security benchmarks](https://www.cisecurity.org/benchmark) recommend that `/tmp/` be mounted as `noexec`; these standards are increasingly +being adopted by major Linux distributions. For certain types of input data (e.g. NDPI, JPEG-XR compression), bioformats2raw needs +to extract a native library from one or more jars to a temporary location. In these cases, the extracted native library must be executable in order +to read image data. + +The recommended solution is to choose a different temporary directory by adding `-Djava.io.tmpdir=/path/to/alternate/tmp` to `JAVA_OPTS`. If multiple properties need to be set via `JAVA_OPTS`, separate them with a space, e.g. `JAVA_OPTS=-Djava.io.tmpdir/path/to/alternate/tmp -Djna.library.path=/path/to/blosc`. +If you know this issue does not affect your input data and wish to warn instead of immediately stopping conversion, the `--warn-no-exec` option can be used. +For input data that relies on native library loading (e.g. NDPI, JPEG-XR compression), using `--warn-no-exec` instead of specifying an alternate +temporary directory will simply cause the conversion to fail at a later point. + +For additional information, please see: + +* https://github.com/glencoesoftware/bioformats2raw/pull/252 +* https://github.com/scijava/native-lib-loader/issues/41 +* https://forum.image.sc/t/after-omero-server-upgrade-hamamatsu-ndpi-files-display-only-in-low-resolution/81868 +* https://forum.image.sc/t/bioformats-unable-to-read-czi-files-with-jpegxr-compression-on-almalinux-os-java-lang-unsatisfiedlinkerror-ome-jxrlib-jxrjni-new-decodecontext-j/82646 + Installation ============ From 80f3ffbc5185a14d76c397d935ec7bf15688b5b5 Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Thu, 18 Jul 2024 08:52:24 -0500 Subject: [PATCH 5/7] Fix JAVA_OPTS syntax MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sébastien Besson --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 49e3f73..5981178 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ to extract a native library from one or more jars to a temporary location. In th to read image data. The recommended solution is to choose a different temporary directory by adding `-Djava.io.tmpdir=/path/to/alternate/tmp` to `JAVA_OPTS`. -If multiple properties need to be set via `JAVA_OPTS`, separate them with a space, e.g. `JAVA_OPTS=-Djava.io.tmpdir/path/to/alternate/tmp -Djna.library.path=/path/to/blosc`. +If multiple properties need to be set via `JAVA_OPTS`, separate them with a space, e.g. `JAVA_OPTS="-Djava.io.tmpdir/path/to/alternate/tmp -Djna.library.path=/path/to/blosc"`. If you know this issue does not affect your input data and wish to warn instead of immediately stopping conversion, the `--warn-no-exec` option can be used. For input data that relies on native library loading (e.g. NDPI, JPEG-XR compression), using `--warn-no-exec` instead of specifying an alternate From 44d031f6b7c265c334c3611596b46c5a72c8104b Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Thu, 18 Jul 2024 09:34:08 -0500 Subject: [PATCH 6/7] One more try at deleting the tmp check file correctly --- .../bioformats2raw/Converter.java | 28 +++++++++++-------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java b/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java index 00f7ab9..c45e87f 100644 --- a/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java +++ b/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java @@ -1209,20 +1209,24 @@ public Integer call() throws Exception { // a temp file is created and set as executable // in the noexec case, setting as executable is expected to silently fail File tmpdirCheck = File.createTempFile("noexec-test", ".txt"); - // expect 'success' to be true in the noexec case, even though - // the file will not actually be executable - boolean success = tmpdirCheck.setExecutable(true); - if (!success || !tmpdirCheck.canExecute()) { - String msg = System.getProperty("java.io.tmpdir") + - " is noexec; fix it or specify a different java.io.tmpdir"; - if (getWarnNoExec()) { - LOGGER.warn(msg); - } - else { - throw new RuntimeException(msg); + try { + // expect 'success' to be true in the noexec case, even though + // the file will not actually be executable + boolean success = tmpdirCheck.setExecutable(true); + if (!success || !tmpdirCheck.canExecute()) { + String msg = System.getProperty("java.io.tmpdir") + + " is noexec; fix it or specify a different java.io.tmpdir"; + if (getWarnNoExec()) { + LOGGER.warn(msg); + } + else { + throw new RuntimeException(msg); + } } } - tmpdirCheck.delete(); + finally { + tmpdirCheck.delete(); + } OpenCVTools.loadOpenCV(); From c772b4612dd6a41ccfec39a2fedfed6f6dc19c12 Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Fri, 19 Jul 2024 08:24:36 -0500 Subject: [PATCH 7/7] Link to README in tmp directory message --- .../java/com/glencoesoftware/bioformats2raw/Converter.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java b/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java index c45e87f..c86a8f6 100644 --- a/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java +++ b/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java @@ -1215,7 +1215,9 @@ public Integer call() throws Exception { boolean success = tmpdirCheck.setExecutable(true); if (!success || !tmpdirCheck.canExecute()) { String msg = System.getProperty("java.io.tmpdir") + - " is noexec; fix it or specify a different java.io.tmpdir"; + " is noexec; fix it or specify a different java.io.tmpdir." + + " See https://github.com/glencoesoftware/bioformats2raw/" + + "blob/master/README.md#temporary-directory-usage for details"; if (getWarnNoExec()) { LOGGER.warn(msg); }