From fd6d552241f89b51ce58f43c421acd3410a04307 Mon Sep 17 00:00:00 2001 From: Michel Daab Date: Tue, 12 Dec 2023 11:58:43 +0100 Subject: [PATCH] removed __init__.py in test that pytest didn't need after all update manual for the epsg add a test for epsg --- docs/source/tutorials/make_predictions.md | 3 ++- docs/source/tutorials/prepare_dataset.md | 2 ++ myria3d/pctl/dataset/utils.py | 29 +++++++++++------------ package_metadata.yaml | 2 +- tests/__init__.py | 0 tests/conftest.py | 18 ++++++++++++++ tests/myria3d/__init__.py | 0 tests/myria3d/models/__init__.py | 0 tests/myria3d/pctl/__init__.py | 0 tests/myria3d/pctl/dataset/__init__.py | 0 tests/myria3d/test_train_and_predict.py | 23 ++++++++++++++++++ 11 files changed, 60 insertions(+), 17 deletions(-) delete mode 100644 tests/__init__.py delete mode 100644 tests/myria3d/__init__.py delete mode 100644 tests/myria3d/models/__init__.py delete mode 100644 tests/myria3d/pctl/__init__.py delete mode 100644 tests/myria3d/pctl/dataset/__init__.py diff --git a/docs/source/tutorials/make_predictions.md b/docs/source/tutorials/make_predictions.md index 8341fd5c..bd3fdb34 100644 --- a/docs/source/tutorials/make_predictions.md +++ b/docs/source/tutorials/make_predictions.md @@ -28,7 +28,8 @@ To show you current inference config, simply add a `--help` flag: python run.py task.task_name=predict --help ``` -Note that `predict.src_las` may be any valid glob pattern (e.g. `/path/to/multiple_files/*.las`), in order to **predict on multiple files successively**. +Note that `predict.src_las` may be any valid glob pattern (e.g. `/path/to/multiple_files/*.las`), in order to **predict on multiple files successively**. +If the lidar file doesn't specify an EPSG in its meatadata, it HAS TO BE be specified with `datamodule.epsg=...` ## Run inference from sources diff --git a/docs/source/tutorials/prepare_dataset.md b/docs/source/tutorials/prepare_dataset.md index 1dff18a6..f02a0278 100644 --- a/docs/source/tutorials/prepare_dataset.md +++ b/docs/source/tutorials/prepare_dataset.md @@ -25,6 +25,8 @@ Under the hood, the path of each LAS file will be reconstructed like this: '{dat Large input point clouds need to be divided in smaller clouds that can be digested by segmentation models. We found that a receptive field of 50m x 50m was a good balance between context and memory intensity. The division is performed once, to avoid loading large file in memory multiple times during training. +To be able to read the lidar files, an EPSG is needed. If the files don't all specify an EPSG in their metadata, it should be given as a parameter with `datamodule.epsg=...` + After division, the smaller clouds are preprocessed (i.e. selection of specific LAS dimensions, on-the-fly creation of dimensions) and regrouped into a single HDF5 file whose path is specified via the `datamodule.hdf5_file_path` parameter. The HDF5 dataset is created at training time. It should only happens once. Once this is done, you do not need sources anymore, and simply specifying the path to the HDF5 dataset is enough (there is no need for data_dir or split_csv_path parameters anymore). diff --git a/myria3d/pctl/dataset/utils.py b/myria3d/pctl/dataset/utils.py index 0baa7b66..9eff03d7 100644 --- a/myria3d/pctl/dataset/utils.py +++ b/myria3d/pctl/dataset/utils.py @@ -9,13 +9,10 @@ import pandas as pd import pdal from scipy.spatial import cKDTree -from pyproj import CRS SPLIT_TYPE = Union[Literal["train"], Literal["val"], Literal["test"]] LAS_PATHS_BY_SPLIT_DICT_TYPE = Dict[SPLIT_TYPE, List[str]] -# commons - def find_file_in_dir(data_dir: str, basename: str) -> str: """Query files matching a basename in input_data_dir and its subdirectories. @@ -85,18 +82,20 @@ def get_pdal_reader(las_path: str, epsg: str) -> pdal.Reader.las: pdal.Reader.las: reader to use in a pipeline. """ - if not epsg: - try: - crs = CRS.from_string(get_metadata(las_path)['metadata']['readers.las']['srs']['compoundwkt']) - epsg = str(crs.to_epsg()) - except Exception: - raise Exception("No EPSG provided, neither in the lidar file or as parameter") - - return pdal.Reader.las( - filename=las_path, - nosrs=True, - override_srs=f"EPSG:{epsg}", - ) + + if epsg : + # if an epsg in provided, force pdal to read the lidar file with it + return pdal.Reader.las( + filename=las_path, + nosrs=True, + override_srs=f"EPSG:{epsg}", + ) + + if 'srs' in get_metadata(las_path)['metadata']['readers.las'] and get_metadata(las_path)['metadata']['readers.las']['srs']['compoundwkt']: + # read the lidar file with pdal default + return pdal.Reader.las(filename=las_path) + + raise Exception("No EPSG provided, neither in the lidar file or as parameter") def get_pdal_info_metadata(las_path: str) -> Dict: diff --git a/package_metadata.yaml b/package_metadata.yaml index db6e12d3..e6f0e165 100644 --- a/package_metadata.yaml +++ b/package_metadata.yaml @@ -1,4 +1,4 @@ -__version__: "3.5.2" +__version__: "3.6.0" __name__: "myria3d" __url__: "https://github.com/IGNF/myria3d" __description__: "Deep Learning for the Semantic Segmentation of Aerial Lidar Point Clouds" diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/conftest.py b/tests/conftest.py index 474d199b..322ddc96 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -56,6 +56,24 @@ def run_command(command: List[str]): pytest.fail(reason=msg) +def run_command_with_return_error(command: List[str]): + """Default method for executing shell commands with pytest.""" + msg = None + try: + sh.python(command) + except sh.ErrorReturnCode as e: + msg = e.stderr.decode() + return msg + + +def run_hydra_decorated_command_with_return_error(command: List[str]): + """Default method for executing hydra decorated shell commands with pytest.""" + hydra_specific_paths = [ + "hydra.run.dir=" + os.getcwd(), + ] + return run_command_with_return_error(command + hydra_specific_paths) + + def run_hydra_decorated_command(command: List[str]): """Default method for executing hydra decorated shell commands with pytest.""" hydra_specific_paths = [ diff --git a/tests/myria3d/__init__.py b/tests/myria3d/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/myria3d/models/__init__.py b/tests/myria3d/models/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/myria3d/pctl/__init__.py b/tests/myria3d/pctl/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/myria3d/pctl/dataset/__init__.py b/tests/myria3d/pctl/dataset/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/myria3d/test_train_and_predict.py b/tests/myria3d/test_train_and_predict.py index bc719277..c0d1a7b0 100644 --- a/tests/myria3d/test_train_and_predict.py +++ b/tests/myria3d/test_train_and_predict.py @@ -11,6 +11,7 @@ from tests.conftest import ( make_default_hydra_cfg, run_hydra_decorated_command, + run_hydra_decorated_command_with_return_error, SINGLE_POINT_CLOUD, DEFAULT_EPSG, ) @@ -90,6 +91,28 @@ def test_predict_as_command(one_epoch_trained_RandLaNet_checkpoint, tmpdir): run_hydra_decorated_command(command) +def test_command_without_epsg(one_epoch_trained_RandLaNet_checkpoint, tmpdir): + """Test running inference by CLI for toy LAS. + + Args: + one_epoch_trained_RandLaNet_checkpoint (fixture -> str): path to checkpoint of + a RandLa-Net model that was trained for once epoch at start of test session. + tmpdir (fixture -> str): temporary directory. + + """ + # Hydra changes CWD, and therefore absolute paths are preferred + abs_path_to_toy_LAS = osp.abspath(TOY_LAS_DATA) + command = [ + "run.py", + f"predict.ckpt_path={one_epoch_trained_RandLaNet_checkpoint}", + f"predict.src_las={abs_path_to_toy_LAS}", + f"predict.output_dir={tmpdir}", + "+predict.interpolator.probas_to_save=[building,unclassified]", + "task.task_name=predict", + ] + assert "No EPSG provided, neither in the lidar file or as parameter" in run_hydra_decorated_command_with_return_error(command) + + def test_predict_on_single_point_cloud(one_epoch_trained_RandLaNet_checkpoint, tmpdir): """Test running inference by CLI for cloud with a single point (edge case addressed in V3.4.0)""" # Hydra changes CWD, and therefore absolute paths are preferred