Add --stop-on-error flag for testing (#262)

* Add --stop-on-error flag for testing

* add unit tests

.github/workflows/test.yaml

@@ -78,11 +78,16 @@ jobs:
         --stations=ARV,BAK \
         --xml_path=s3://scedc-pds/FDSNstationXML/CI/ \
         --freq_norm ${{matrix.freq_norm}} \
+        --stop_on_error \
         --format ${{matrix.format}}
-        # noisepy cross_correlate --raw_data_path $RUNNER_TEMP/RAW_DATA --ccf_path $RUNNER_TEMP/CCF --freq_norm ${{matrix.freq_norm}} --format ${{matrix.format}}
     - name: Test Stacking (S2)
       run: |
-        noisepy stack --ccf_path $RUNNER_TEMP/CCF --stack_path $RUNNER_TEMP/STACK --stack_method ${{matrix.method}} --format ${{matrix.format}}
+        noisepy stack \
+        --ccf_path $RUNNER_TEMP/CCF \
+        --stack_path $RUNNER_TEMP/STACK \
+        --stack_method ${{matrix.method}} \
+        --stop_on_error \
+        --format ${{matrix.format}}
   s1_s2_mpi:
     strategy:
       fail-fast: true
@@ -112,10 +117,17 @@ jobs:
         --stations=ARV,BAK \
         --xml_path=s3://scedc-pds/FDSNstationXML/CI/  \
         --freq_norm ${{matrix.freq_norm}} \
+        --stop_on_error \
         --format numpy
     - name: Test Stacking (S2)
       run: |
-        mpiexec -n 3 noisepy stack --mpi --ccf_path $RUNNER_TEMP/CCF --stack_path $RUNNER_TEMP/STACK --stack_method ${{matrix.method}} --format numpy
+        mpiexec -n 3 noisepy stack \
+        --mpi \
+        --ccf_path $RUNNER_TEMP/CCF \
+        --stack_path $RUNNER_TEMP/STACK \
+        --stack_method ${{matrix.method}} \
+        --stop_on_error \
+        --format numpy
   s3_dates:
     strategy:
       fail-fast: true
@@ -135,4 +147,5 @@ jobs:
         --ccf_path $RUNNER_TEMP/CCF_S3 --freq_norm rma \
         --xml_path s3://scedc-pds/FDSNstationXML/CI/ \
         --stations "SBC,RIO" --start_date 2022-02-02 --end_date 2022-02-04 \
+        --stop_on_error \
         --config configs/s3_anon.yaml

README.md

@@ -77,7 +77,7 @@ To run the code on a single core, open the terminal and activate the noisepy env
 We use I/O on disk, so users need root access to the file system. To install rootless docker, see instructions [here](https://docs.docker.com/engine/security/rootless/#install).
 ```bash
 docker pull  ghcr.io/noisepy/noisepy:latest
-docker run -v ~/tmp:/tmp cross_correlate --path /tmp
+docker run -v ~/tmp:/tmp ghcr.io/noisepy/noisepy:latest cross_correlate --path /tmp
 ```
 
 # Tutorials

src/noisepy/seis/main.py

@@ -195,6 +195,13 @@ def makedir(dir: str, storage_options: dict = {}):
     fs.makedirs(dir, exist_ok=True)
 
 
+class ErrorStopHandler(logging.Handler):
+    def handle(self, record):
+        if record.levelno == logging.ERROR:
+            raise RuntimeError(record.getMessage())
+        return record
+
+
 def main(args: typing.Any):
     logger = logging.getLogger(__package__)
     logger.setLevel(args.loglevel.upper())
@@ -260,12 +267,19 @@ def run_stack(params: ConfigParameters):
         scheduler = get_scheduler(args)
         stack_cross_correlations(cc_store, stack_store, params, scheduler)
 
-    if args.cmd == Command.DOWNLOAD:
-        cmd_wrapper(run_download, None, args.raw_data_path)
-    if args.cmd == Command.CROSS_CORRELATE:
-        cmd_wrapper(run_cross_correlation, args.raw_data_path, args.ccf_path)
-    if args.cmd == Command.STACK:
-        cmd_wrapper(run_stack, args.ccf_path, args.stack_path)
+    err_handler = ErrorStopHandler()
+    if args.stop_on_error:
+        logger.addHandler(err_handler)
+    try:
+        if args.cmd == Command.DOWNLOAD:
+            cmd_wrapper(run_download, None, args.raw_data_path)
+        if args.cmd == Command.CROSS_CORRELATE:
+            cmd_wrapper(run_cross_correlation, args.raw_data_path, args.ccf_path)
+        if args.cmd == Command.STACK:
+            cmd_wrapper(run_stack, args.ccf_path, args.stack_path)
+    finally:
+        if args.stop_on_error:
+            logger.removeHandler(err_handler)
 
 
 def add_path(parser, prefix: str):
@@ -299,6 +313,8 @@ def make_step_parser(subparsers: Any, cmd: Command, paths: List[str]) -> Any:
     parser.add_argument(
         "-c", "--config", type=lambda f: _valid_config_file(parser, f), required=False, help="Configuration YAML file"
     )
+    parser.add_argument("--stop_on_error", action="store_true", default=False, help="Stop on any errors")
+
     add_model(parser, ConfigParameters())
 
     if cmd != Command.DOWNLOAD:
@@ -337,13 +353,13 @@ def parse_args(arguments: Iterable[str]) -> argparse.Namespace:
     return args
 
 
-def _enable_s3fs_debug_logs():
-    os.environ["S3FS_LOGGING_LEVEL"] = "DEBUG"
-    for pkg in ["urllib3", "s3fs", "zarr"]:
-        logger.info("Enable debug log for %s", pkg)
-        lgr = logging.getLogger(pkg)
-        lgr.setLevel(logging.DEBUG)
-        lgr.propagate = True
+# def _enable_s3fs_debug_logs():
+#     os.environ["S3FS_LOGGING_LEVEL"] = "DEBUG"
+#     for pkg in ["urllib3", "s3fs", "zarr"]:
+#         logger.info("Enable debug log for %s", pkg)
+#         lgr = logging.getLogger(pkg)
+#         lgr.setLevel(logging.DEBUG)
+#         lgr.propagate = True
 
 
 if __name__ == "__main__":

tests/test_cli.sh

@@ -8,7 +8,7 @@ if [[ "$FORMAT" != "zarr" && "$FORMAT" != "asdf" && "$FORMAT" != "numpy" ]]; the
        exit 1
 fi
 echo "FORMAT is _${FORMAT}_"
-RUNNER_TEMP=~/test_temp_${FORMAT}
+RUNNER_TEMP=~/noisepy_data/${FORMAT}
 
 # RUNNER_TEMP=s3://carlosgjs-noisepy/test_new
 # aws s3 rm --recursive $RUNNER_TEMP
@@ -18,9 +18,10 @@ STACK=$RUNNER_TEMP/STACK
 RAW=~/s3tmp/scedc/
 XML=~/s3tmp/FDSNstationXML
 
-mkdir $RUNNER_TEMP
+mkdir -p $RUNNER_TEMP
 LOGFILE="$HOME/logs/log_${FORMAT}_$(date -j +'%Y%m%d_%H%M%S').txt"
 STATIONS=ARV,BAK
+NETWORKS=CI
 START=2022-02-02
 END=2022-02-04
 CHANNELS=BHE,BHN,BHZ
@@ -36,22 +37,24 @@ rm -rf $CCF
 noisepy cross_correlate  \
 --raw_data_path=$RAW \
 --xml_path=$XML \
+--ccf_path=$CCF \
 --stations=$STATIONS \
 --channels=$CHANNELS \
+--net_list=$NETWORKS \
 --start=$START \
 --end=$END \
---ccf_path=$CCF \
 --loglevel=${LOG_LEVEL} \
 --format=${FORMAT} \
+--stop_on_error \
 --logfile=$LOGFILE \
---loglevel=${LOG_LEVEL}
 
 rm -rf $STACK
 noisepy stack --ccf_path $CCF \
 --stack_path=$STACK \
 --stack_method=all \
 --format=${FORMAT} \
 --logfile=$LOGFILE \
+--stop_on_error \
 --loglevel=${LOG_LEVEL}
 
 du -ch $RUNNER_TEMP

tests/test_main.py

@@ -1,3 +1,4 @@
+import logging
 from datetime import datetime, timezone
 from typing import List
 from unittest import mock
@@ -9,6 +10,7 @@
 from noisepy.seis.datatypes import CCMethod
 from noisepy.seis.main import (
     Command,
+    ErrorStopHandler,
     _valid_config_file,
     initialize_params,
     main,
@@ -50,7 +52,7 @@ def run_cmd_with_empty_dirs(cmd: Command, args: List[str]):
 
 def test_main_cc(tmp_path):
     tmp = str(tmp_path)
-    run_cmd_with_empty_dirs(Command.CROSS_CORRELATE, [empty("raw_data", tmp), empty("xml", tmp)])
+    run_cmd_with_empty_dirs(Command.CROSS_CORRELATE, [empty("raw_data", tmp), empty("xml", tmp), "--stop_on_error"])
 
 
 def test_main_stack(tmp_path):
@@ -76,6 +78,15 @@ def test_main_download(tmp_path):
         )
 
 
+def test_error_handler():
+    handler = ErrorStopHandler()
+
+    # info shpuld not raise
+    handler.handle(logging.LogRecord("name", logging.INFO, "pathname", 0, "msg", None, None))
+    with pytest.raises(RuntimeError):
+        handler.handle(logging.LogRecord("name", logging.ERROR, "pathname", 0, "msg", None, RuntimeError("error")))
+
+
 def test_valid_config(tmp_path):
     cfgfile = tmp_path.joinpath("config.yaml")
     parser = mock.Mock()