From 2c5a65cd9380858b60adda06782de8d6889ba230 Mon Sep 17 00:00:00 2001 From: sfmig <33267254+sfmig@users.noreply.github.com> Date: Thu, 22 Jun 2023 18:02:22 +0100 Subject: [PATCH 01/33] notebook w typical pipeline --- notebook_det_and_classif_dask.py | 65 ++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 notebook_det_and_classif_dask.py diff --git a/notebook_det_and_classif_dask.py b/notebook_det_and_classif_dask.py new file mode 100644 index 00000000..9ef3cf02 --- /dev/null +++ b/notebook_det_and_classif_dask.py @@ -0,0 +1,65 @@ +# %% +import os +from pathlib import Path +import imlib.IO.cells as cell_io + +from cellfinder_core.main import main as cellfinder_run +from cellfinder_core.tools.IO import read_with_dask + + +def detection_classification_pipeline( + signal_data_dir_str, + background_data_dir_str +): + # Read data + # - dask for ~ TB of data, you pass the directory and it will load all the images as a 3D array + # - tiff can also be a 3D array but no examples in the test data + signal_array = read_with_dask(signal_data_dir_str) # (30, 510, 667) ---> planes , image size (h, w?) + background_array = read_with_dask(background_data_dir_str) + + + # Detection and classification pipeline + # the output is a list of imlib Cell objects w/ centroid coordinate and type + detected_cells = cellfinder_run( + signal_array, + background_array, + voxel_sizes, + ) + + return detected_cells + + +if __name__ == "__main__": + + # Input data + data_dir = Path(os.getcwd()) / "tests" / "data" / "integration" / "detection" + signal_data_dir = data_dir / "crop_planes" / "ch0" + background_data_dir = data_dir / "crop_planes" / "ch1" + voxel_sizes = [5, 2, 2] # microns + + # D+C pipeline + detected_cells = detection_classification_pipeline( + str(signal_data_dir), + str(background_data_dir) + ) + + # Inspect results + print(f'Sample cell type: {type(detected_cells[0])}') + print('Sample cell attributes: ' + f'x={detected_cells[0].x}, ' + f'y={detected_cells[0].y}, ' + f'z={detected_cells[0].z}, ' + f'type={detected_cells[0].type}') # Cell: x: 132, y: 308, z: 10, type: 2 + + num_cells = sum([cell.type == 2 for cell in detected_cells]) # Cell type 2 is a true positive (classified as cell), + num_non_cells = sum([cell.type == 1 for cell in detected_cells]) # Cell type 1 is a false positive (classified as non-cell) + print(f'{num_cells}/{len(detected_cells)} cells classified as cells') + print(f'{num_non_cells}/{len(detected_cells)} cells classified as non-cells') + + + # Save results in the cellfinder XML standard + # it only saves type 1 + cell_io.save_cells(detected_cells, 'output.xml') + + # # to read them + # cell_io.get_cells('output.xml') \ No newline at end of file From 2a4c7293ff23cfc9d02b9b7e59e1c503eea26dd2 Mon Sep 17 00:00:00 2001 From: sfmig <33267254+sfmig@users.noreply.github.com> Date: Mon, 26 Jun 2023 11:52:38 +0100 Subject: [PATCH 02/33] notes on writing benchmarks --- benchmarks/README.md | 12 -- benchmarks/asv.conf.json | 183 ++++++++++++++++++ benchmarks/benchmarks/__init__.py | 1 + benchmarks/benchmarks/basic_benchmarks.py | 19 ++ benchmarks/benchmarks/sample_benchmarks.py | 119 ++++++++++++ benchmarks/mem_benchmarks/README.md | 12 ++ .../detect_and_classify.py | 0 benchmarks/{ => mem_benchmarks}/filter_2d.py | 0 benchmarks/{ => mem_benchmarks}/filter_3d.py | 0 9 files changed, 334 insertions(+), 12 deletions(-) create mode 100644 benchmarks/asv.conf.json create mode 100644 benchmarks/benchmarks/__init__.py create mode 100644 benchmarks/benchmarks/basic_benchmarks.py create mode 100644 benchmarks/benchmarks/sample_benchmarks.py create mode 100644 benchmarks/mem_benchmarks/README.md rename benchmarks/{ => mem_benchmarks}/detect_and_classify.py (100%) rename benchmarks/{ => mem_benchmarks}/filter_2d.py (100%) rename benchmarks/{ => mem_benchmarks}/filter_3d.py (100%) diff --git a/benchmarks/README.md b/benchmarks/README.md index af1352ad..e69de29b 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -1,12 +0,0 @@ -# Benchmarks -`detect_and_classify.py` contains a simple script that runs -detection and classification with the small test dataset. - -## Memory -[memory_profiler](https://github.com/pythonprofilers/memory_profiler) -can be used to profile memory useage. Install, and then run -`mprof run --include-children --multiprocess detect_and_classify.py`. It is **very** -important to use these two flags to capture memory usage by the additional -processes that cellfinder_core uses. - -To show the results of the latest profile run, run `mprof plot`. diff --git a/benchmarks/asv.conf.json b/benchmarks/asv.conf.json new file mode 100644 index 00000000..b6cc00ee --- /dev/null +++ b/benchmarks/asv.conf.json @@ -0,0 +1,183 @@ +{ + // The version of the config file format. Do not change, unless + // you know what you are doing. + "version": 1, + + // The name of the project being benchmarked + "project": "cellfinder-core", + + // The project's homepage + "project_url": "https://brainglobe.info/documentation/cellfinder/index.html", + + // The URL or local path of the source code repository for the + // project being benchmarked + "repo": "..", + + // The Python project's subdirectory in your repo. If missing or + // the empty string, the project is assumed to be located at the root + // of the repository. + // "repo_subdir": "", + + // Customizable commands for building, installing, and + // uninstalling the project. See asv.conf.json documentation. + // + "install_command": ["in-dir={env_dir} python -mpip install {wheel_file}"], + "uninstall_command": ["return-code=any python -mpip uninstall -y {project}"], + "build_command": [ + "PIP_NO_BUILD_ISOLATION=false python -mpip wheel --no-deps --no-index -w {build_cache_dir} {build_dir}" + ], + + // List of branches to benchmark. If not provided, defaults to "master" + // (for git) or "default" (for mercurial). + "branches": ["main"], // for git + // "branches": ["default"], // for mercurial + + // The DVCS being used. If not set, it will be automatically + // determined from "repo" by looking at the protocol in the URL + // (if remote), or by looking for special directories, such as + // ".git" (if local). + // "dvcs": "git", + + // The tool to use to create environments. May be "conda", + // "virtualenv" or other value depending on the plugins in use. + // If missing or the empty string, the tool will be automatically + // determined by looking for tools on the PATH environment + // variable. + "environment_type": "conda", + + // timeout in seconds for installing any dependencies in environment + // defaults to 10 min + //"install_timeout": 600, + + // the base URL to show a commit for the project. + "show_commit_url": "http://github.com/brainglobe/cellfinder-core/commit/", + + // The Pythons you'd like to test against. If not provided, defaults + // to the current version of Python used to run `asv`. + "pythons": ["3.10"], // same as pyproject.toml? ["3.8", "3.9", "3.10"] + + // The list of conda channel names to be searched for benchmark + // dependency packages in the specified order + "conda_channels": ["conda-forge", "defaults"], + + // A conda environment file that is used for environment creation. + // "conda_environment_file": "environment.yml", + + // The matrix of dependencies to test. Each key of the "req" + // requirements dictionary is the name of a package (in PyPI) and + // the values are version numbers. An empty list or empty string + // indicates to just test against the default (latest) + // version. null indicates that the package is to not be + // installed. If the package to be tested is only available from + // PyPi, and the 'environment_type' is conda, then you can preface + // the package name by 'pip+', and the package will be installed + // via pip (with all the conda available packages installed first, + // followed by the pip installed packages). + // + // The ``@env`` and ``@env_nobuild`` keys contain the matrix of + // environment variables to pass to build and benchmark commands. + // An environment will be created for every combination of the + // cartesian product of the "@env" variables in this matrix. + // Variables in "@env_nobuild" will be passed to every environment + // during the benchmark phase, but will not trigger creation of + // new environments. A value of ``null`` means that the variable + // will not be set for the current combination. + // + "matrix": { + "req": {}, + // "napari": ["", null], // test with and without + // // "six": ["", null], // test with and without six installed + // // "pip+emcee": [""] // emcee is only available for install with pip. + // }, + // "env": {"ENV_VAR_1": ["val1", "val2"]}, + // "env_nobuild": {"ENV_VAR_2": ["val3", null]}, + }, + + // Combinations of libraries/python versions can be excluded/included + // from the set to test. Each entry is a dictionary containing additional + // key-value pairs to include/exclude. + // + // An exclude entry excludes entries where all values match. The + // values are regexps that should match the whole string. + // + // An include entry adds an environment. Only the packages listed + // are installed. The 'python' key is required. The exclude rules + // do not apply to includes. + // + // In addition to package names, the following keys are available: + // + // - python + // Python version, as in the *pythons* variable above. + // - environment_type + // Environment type, as above. + // - sys_platform + // Platform, as in sys.platform. Possible values for the common + // cases: 'linux2', 'win32', 'cygwin', 'darwin'. + // - req + // Required packages + // - env + // Environment variables + // - env_nobuild + // Non-build environment variables + // + // "exclude": [ + // {"python": "3.2", "sys_platform": "win32"}, // skip py3.2 on windows + // {"environment_type": "conda", "req": {"six": null}}, // don't run without six on conda + // {"env": {"ENV_VAR_1": "val2"}}, // skip val2 for ENV_VAR_1 + // ], + // + // "include": [ + // // additional env for python2.7 + // {"python": "2.7", "req": {"numpy": "1.8"}, "env_nobuild": {"FOO": "123"}}, + // // additional env if run on windows+conda + // {"platform": "win32", "environment_type": "conda", "python": "2.7", "req": {"libpython": ""}}, + // ], + + // The directory (relative to the current directory) that benchmarks are + // stored in. If not provided, defaults to "benchmarks" + "benchmark_dir": "benchmarks", + + // The directory (relative to the current directory) to cache the Python + // environments in. If not provided, defaults to "env" + "env_dir": "env", + + // The directory (relative to the current directory) that raw benchmark + // results are stored in. If not provided, defaults to "results". + "results_dir": "results", + + // The directory (relative to the current directory) that the html tree + // should be written to. If not provided, defaults to "html". + "html_dir": "html", + + // The number of characters to retain in the commit hashes. + // "hash_length": 8, + + // `asv` will cache results of the recent builds in each + // environment, making them faster to install next time. This is + // the number of builds to keep, per environment. + "build_cache_size": 2, + + // The commits after which the regression search in `asv publish` + // should start looking for regressions. Dictionary whose keys are + // regexps matching to benchmark names, and values corresponding to + // the commit (exclusive) after which to start looking for + // regressions. The default is to start from the first commit + // with results. If the commit is `null`, regression detection is + // skipped for the matching benchmark. + // + // "regressions_first_commits": { + // "some_benchmark": "352cdf", // Consider regressions only after this commit + // "another_benchmark": null, // Skip regression detection altogether + // }, + + // The thresholds for relative change in results, after which `asv + // publish` starts reporting regressions. Dictionary of the same + // form as in ``regressions_first_commits``, with values + // indicating the thresholds. If multiple entries match, the + // maximum is taken. If no entry matches, the default is 5%. + // + // "regressions_thresholds": { + // "some_benchmark": 0.01, // Threshold of 1% + // "another_benchmark": 0.5, // Threshold of 50% + // }, +} diff --git a/benchmarks/benchmarks/__init__.py b/benchmarks/benchmarks/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/benchmarks/benchmarks/__init__.py @@ -0,0 +1 @@ + diff --git a/benchmarks/benchmarks/basic_benchmarks.py b/benchmarks/benchmarks/basic_benchmarks.py new file mode 100644 index 00000000..9230b3b9 --- /dev/null +++ b/benchmarks/benchmarks/basic_benchmarks.py @@ -0,0 +1,19 @@ +class TimeSuiteBasic: + params = [0, 10, 100, 1000] + param_names = ['n'] + + def setup(self, n): + self.d = {} + for x in range(n): + self.d[x] = None + + def time_keys(self, n): # why do I need to pass parameter here? + for ky in self.d.keys(): + pass + + def time_values(self, n): + for val in self.d.values(): + pass + + def teardown(self, n): + pass \ No newline at end of file diff --git a/benchmarks/benchmarks/sample_benchmarks.py b/benchmarks/benchmarks/sample_benchmarks.py new file mode 100644 index 00000000..15cc32d0 --- /dev/null +++ b/benchmarks/benchmarks/sample_benchmarks.py @@ -0,0 +1,119 @@ +# Write the benchmarking functions here. +# See "Writing benchmarks" in the asv docs for more information. +# Notes: +# - benchmarks may be organised into methods of classes if desired +# (or just as functions that start with "time_") + +# ------------------------------------ +# Runtime benchmarks start with 'time' +# (snake case or camelcase) +# ------------------------------------ +class TimeSuite: + """ + An example benchmark that times the performance of various kinds + of iterating over dictionaries in Python. + """ + def setup(self): + """ + Setup includes initialisation bits that should not be included + in the timing of the benchmark. + + It can be added as: + - a method of a class, or + - an attribute of a free function, or + - a module-level setup function (run for every benchmark in the + module, prior to any function-specific setup) + + If setup raises `NotImplementedError`, the benchmark is skipped + """ + self.d = {} + for x in range(500): + self.d[x] = None + + def setup_cache(self): + """ + `Setup_cache` only performs the setup calculation once + (for each benchmark and each repeat) and caches the + result to disk. This may be useful if the setup is + expensive. + + A separate cache is used for each environment and each commit. + The cache is thrown out between benchmark runs. + + There are two options to persist the data for the benchmarks: + - `setup_cache` returns a data structure, which asv pickles to disk, + and then loads and passes as first arg to each benchmark (not + automagically tho), or + - `setup_cache` saves files to the cwd (which is a temp dir managed by + asv), which are then explicitly loaded in each benchmark. Recomm + practice is to actually read the data in a `setup` fn, so that + loading time is not part of the timing + """ + pass + + def teardown(self): + """ + Benchmarks can also have teardown functions that are run after + the benchmark. The behaviour is similar to setup fns. + + Useful for example to clean up changes made to the + filesystem + """ + pass + + def time_keys(self): + # benchmark attributes + timeout = 123 # The amount of time, in seconds, to give the benchmark to run before forcibly killing it. Defaults to 60 seconds. + pretty_name = 'pretty name' + setup = setup + teardown = teardown + rounds = 2 + repeat = (1, 5, 10.0) + warmup_time = 0.1 + # params_names + # params ---> the params attribute allows us to run a single benchmark + # for multiple values of the parameters + + for key in self.d.keys(): + pass + + def time_values(self): + # For best results, the benchmark function should contain + # as little as possible, with as much extraneous setup moved to a setup function: + for value in self.d.values(): + pass + + def time_range(self, n): + d = self.d + for key in range(500): + x = d[key] + +# ----------------------- +# Parametrized benchmarks +# ------------------------- +# - params can be any Python object, but it is recommended +# only strings and number +# - w/ multiple params, the test is run for all combs +class Suite: + params = [0, 10, 20] + param_names = ['n'] + + def setup(self, n): + # Note that setup_cache is not parametrized (can it?) + self.obj = range(n) + + def teardown(self, n): + del self.obj + + def time_range_iter(self, n): + for i in self.obj: + pass + + +# ------------------------------------ +# Memory benchmarks start with 'mem' +# (snake case or camelcase) +# ------------------------------------ +class MemSuite: + def mem_list(self): + return [0] * 256 diff --git a/benchmarks/mem_benchmarks/README.md b/benchmarks/mem_benchmarks/README.md new file mode 100644 index 00000000..af1352ad --- /dev/null +++ b/benchmarks/mem_benchmarks/README.md @@ -0,0 +1,12 @@ +# Benchmarks +`detect_and_classify.py` contains a simple script that runs +detection and classification with the small test dataset. + +## Memory +[memory_profiler](https://github.com/pythonprofilers/memory_profiler) +can be used to profile memory useage. Install, and then run +`mprof run --include-children --multiprocess detect_and_classify.py`. It is **very** +important to use these two flags to capture memory usage by the additional +processes that cellfinder_core uses. + +To show the results of the latest profile run, run `mprof plot`. diff --git a/benchmarks/detect_and_classify.py b/benchmarks/mem_benchmarks/detect_and_classify.py similarity index 100% rename from benchmarks/detect_and_classify.py rename to benchmarks/mem_benchmarks/detect_and_classify.py diff --git a/benchmarks/filter_2d.py b/benchmarks/mem_benchmarks/filter_2d.py similarity index 100% rename from benchmarks/filter_2d.py rename to benchmarks/mem_benchmarks/filter_2d.py diff --git a/benchmarks/filter_3d.py b/benchmarks/mem_benchmarks/filter_3d.py similarity index 100% rename from benchmarks/filter_3d.py rename to benchmarks/mem_benchmarks/filter_3d.py From 9879ca613a63a6eb608c40a01e888d34682a6232 Mon Sep 17 00:00:00 2001 From: sfmig <33267254+sfmig@users.noreply.github.com> Date: Mon, 26 Jun 2023 11:53:02 +0100 Subject: [PATCH 03/33] pipeline notebook --- notebook_det_and_classif_dask.py | 38 ++++++++++++-------------------- 1 file changed, 14 insertions(+), 24 deletions(-) diff --git a/notebook_det_and_classif_dask.py b/notebook_det_and_classif_dask.py index 9ef3cf02..28e673e1 100644 --- a/notebook_det_and_classif_dask.py +++ b/notebook_det_and_classif_dask.py @@ -7,17 +7,24 @@ from cellfinder_core.tools.IO import read_with_dask -def detection_classification_pipeline( - signal_data_dir_str, - background_data_dir_str -): + +# Input data +data_dir = Path(os.getcwd()) / "tests" / "data" / "integration" / "detection" +signal_data_dir = data_dir / "crop_planes" / "ch0" +background_data_dir = data_dir / "crop_planes" / "ch1" +voxel_sizes = [5, 2, 2] # microns + + + +if __name__ == "__main__": + # Read data # - dask for ~ TB of data, you pass the directory and it will load all the images as a 3D array # - tiff can also be a 3D array but no examples in the test data - signal_array = read_with_dask(signal_data_dir_str) # (30, 510, 667) ---> planes , image size (h, w?) - background_array = read_with_dask(background_data_dir_str) - + signal_array = read_with_dask(str(signal_data_dir)) # (30, 510, 667) ---> planes , image size (h, w?) + background_array = read_with_dask(str(background_data_dir)) + # D+C pipeline # Detection and classification pipeline # the output is a list of imlib Cell objects w/ centroid coordinate and type detected_cells = cellfinder_run( @@ -26,23 +33,6 @@ def detection_classification_pipeline( voxel_sizes, ) - return detected_cells - - -if __name__ == "__main__": - - # Input data - data_dir = Path(os.getcwd()) / "tests" / "data" / "integration" / "detection" - signal_data_dir = data_dir / "crop_planes" / "ch0" - background_data_dir = data_dir / "crop_planes" / "ch1" - voxel_sizes = [5, 2, 2] # microns - - # D+C pipeline - detected_cells = detection_classification_pipeline( - str(signal_data_dir), - str(background_data_dir) - ) - # Inspect results print(f'Sample cell type: {type(detected_cells[0])}') print('Sample cell attributes: ' From fad8b8a709b32f9dede93990758e307a829a2c4a Mon Sep 17 00:00:00 2001 From: sfmig <33267254+sfmig@users.noreply.github.com> Date: Mon, 26 Jun 2023 11:53:10 +0100 Subject: [PATCH 04/33] update gitignore --- .gitignore | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitignore b/.gitignore index 84d036c2..a7202bec 100644 --- a/.gitignore +++ b/.gitignore @@ -124,3 +124,9 @@ pip-wheel-metadata/ mprofile*.dat *.DS_Store + +# asv +.asv +benchmarks/results +benchmarks/html +benchmarks/env From 4603bcbbc756479c50e32c4c97683b29ac6aa6ac Mon Sep 17 00:00:00 2001 From: sfmig <33267254+sfmig@users.noreply.github.com> Date: Mon, 26 Jun 2023 15:40:52 +0100 Subject: [PATCH 05/33] add imports benchmarks --- benchmarks/benchmarks/imports.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 benchmarks/benchmarks/imports.py diff --git a/benchmarks/benchmarks/imports.py b/benchmarks/benchmarks/imports.py new file mode 100644 index 00000000..e467f3c1 --- /dev/null +++ b/benchmarks/benchmarks/imports.py @@ -0,0 +1,13 @@ + +# ------------------------------------ +# Runtime benchmarks +# ------------------------------------ +def timeraw_import_main(): + return""" + from cellfinder_core.main import main + """ + +def timeraw_import_io_dask(): + return""" + from cellfinder_core.tools.IO import read_with_dask + """ \ No newline at end of file From 67faa02f849b0dec2be37cdedc30f178bc088ed8 Mon Sep 17 00:00:00 2001 From: sfmig <33267254+sfmig@users.noreply.github.com> Date: Mon, 26 Jun 2023 16:30:32 +0100 Subject: [PATCH 06/33] basic benchmark for reading with dask --- benchmarks/benchmarks/tools.py | 37 ++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 benchmarks/benchmarks/tools.py diff --git a/benchmarks/benchmarks/tools.py b/benchmarks/benchmarks/tools.py new file mode 100644 index 00000000..1cbc01e5 --- /dev/null +++ b/benchmarks/benchmarks/tools.py @@ -0,0 +1,37 @@ +import os +from pathlib import Path + +from cellfinder_core.tools.IO import read_with_dask + + +p = Path(os.path.dirname(__file__)).absolute() +CELLFINDER_CORE_PATH = p.parents[1] + + +class IO: + """ + Benchmarks for IO tools + """ + def setup(self): + # prepare paths for reading data + + # TODO: parametrise these? + self.data_dir = ( + Path(CELLFINDER_CORE_PATH) / "tests" / "data" / + "integration" / "detection" / "crop_planes" + / "ch0" + ) + self.voxel_sizes = [5, 2, 2] # microns + + def time_read_with_dask(self): + self.signal_array = read_with_dask( + str(self.data_dir) + ) + + + # def time_read_with_numpy()? + +# class Tiff: + + +# class Prep: \ No newline at end of file From fb8ebe9ec7ffef2c2e9daf549e042064d729e4d5 Mon Sep 17 00:00:00 2001 From: sfmig <33267254+sfmig@users.noreply.github.com> Date: Mon, 26 Jun 2023 18:10:40 +0100 Subject: [PATCH 07/33] parametrise dask benchmark --- benchmarks/benchmarks/tools.py | 65 +++++++++++++++++++++++++++------- 1 file changed, 52 insertions(+), 13 deletions(-) diff --git a/benchmarks/benchmarks/tools.py b/benchmarks/benchmarks/tools.py index 1cbc01e5..525561b5 100644 --- a/benchmarks/benchmarks/tools.py +++ b/benchmarks/benchmarks/tools.py @@ -4,34 +4,73 @@ from cellfinder_core.tools.IO import read_with_dask + p = Path(os.path.dirname(__file__)).absolute() CELLFINDER_CORE_PATH = p.parents[1] +TESTS_DATA_INTEGRATION_PATH = ( + Path(CELLFINDER_CORE_PATH) + / "tests" / "data" / "integration" +) class IO: """ Benchmarks for IO tools + + # TODO: parametrise these? + # TODO: use cache? """ - def setup(self): - # prepare paths for reading data - - # TODO: parametrise these? - self.data_dir = ( - Path(CELLFINDER_CORE_PATH) / "tests" / "data" / - "integration" / "detection" / "crop_planes" - / "ch0" - ) - self.voxel_sizes = [5, 2, 2] # microns + + # ----------------------------- + # Benchmark parameters + # ----------------------------- + params = ( + [ + Path(*("detection", "crop_planes", "ch0")), + Path(*("detection", "crop_planes", "ch1")), + ], + [ + [5, 2, 2] # microns + ] + ) + param_names = [ + 'tests_data_integration_subdir', + 'voxel_sizes' + ] + + # ----------------------------- + # Setup fn: + # prepare paths for reading data + # ----------------------------- + def setup( + self, + subdir, + voxel_sizes + ): + self.data_dir = TESTS_DATA_INTEGRATION_PATH / subdir - def time_read_with_dask(self): - self.signal_array = read_with_dask( + # ----------------------------- + # Runtime benchmarks + # ----------------------------- + def time_read_with_dask( + self, + subdir, + voxel_sizes + ): + read_with_dask( str(self.data_dir) ) # def time_read_with_numpy()? + + # ----------------------------- + # Peak memory benchmarks? + # ----------------------------- + # class Tiff: -# class Prep: \ No newline at end of file +# class Prep: + From a00131ec3fa569d69bcef5aeaa91602f7db9ee78 Mon Sep 17 00:00:00 2001 From: sfmig <33267254+sfmig@users.noreply.github.com> Date: Wed, 28 Jun 2023 14:50:17 +0100 Subject: [PATCH 08/33] add tiff benchmark and refactor --- benchmarks/benchmarks/tools.py | 76 ----------------------- benchmarks/benchmarks/tools/IO.py | 82 +++++++++++++++++++++++++ benchmarks/benchmarks/tools/__init__.py | 0 3 files changed, 82 insertions(+), 76 deletions(-) delete mode 100644 benchmarks/benchmarks/tools.py create mode 100644 benchmarks/benchmarks/tools/IO.py create mode 100644 benchmarks/benchmarks/tools/__init__.py diff --git a/benchmarks/benchmarks/tools.py b/benchmarks/benchmarks/tools.py deleted file mode 100644 index 525561b5..00000000 --- a/benchmarks/benchmarks/tools.py +++ /dev/null @@ -1,76 +0,0 @@ -import os -from pathlib import Path - -from cellfinder_core.tools.IO import read_with_dask - - - -p = Path(os.path.dirname(__file__)).absolute() -CELLFINDER_CORE_PATH = p.parents[1] -TESTS_DATA_INTEGRATION_PATH = ( - Path(CELLFINDER_CORE_PATH) - / "tests" / "data" / "integration" -) - - -class IO: - """ - Benchmarks for IO tools - - # TODO: parametrise these? - # TODO: use cache? - """ - - # ----------------------------- - # Benchmark parameters - # ----------------------------- - params = ( - [ - Path(*("detection", "crop_planes", "ch0")), - Path(*("detection", "crop_planes", "ch1")), - ], - [ - [5, 2, 2] # microns - ] - ) - param_names = [ - 'tests_data_integration_subdir', - 'voxel_sizes' - ] - - # ----------------------------- - # Setup fn: - # prepare paths for reading data - # ----------------------------- - def setup( - self, - subdir, - voxel_sizes - ): - self.data_dir = TESTS_DATA_INTEGRATION_PATH / subdir - - # ----------------------------- - # Runtime benchmarks - # ----------------------------- - def time_read_with_dask( - self, - subdir, - voxel_sizes - ): - read_with_dask( - str(self.data_dir) - ) - - - # def time_read_with_numpy()? - - - # ----------------------------- - # Peak memory benchmarks? - # ----------------------------- - -# class Tiff: - - -# class Prep: - diff --git a/benchmarks/benchmarks/tools/IO.py b/benchmarks/benchmarks/tools/IO.py new file mode 100644 index 00000000..3c9e39c2 --- /dev/null +++ b/benchmarks/benchmarks/tools/IO.py @@ -0,0 +1,82 @@ +import os +from pathlib import Path + +from cellfinder_core.tools.IO import get_tiff_meta, read_with_dask + + +p = Path(os.path.dirname(__file__)).absolute() +CELLFINDER_CORE_PATH = p.parents[2] +# Q for review: is there a nice way to get cellfinder-core path? +TESTS_DATA_INTEGRATION_PATH = ( + Path(CELLFINDER_CORE_PATH) / "tests" / "data" / "integration" +) + + +# --------------------------------------------- +# Benchmarks for reading 3d arrays with dask +# -------------------------------------------- +class Dask: + param_names = [ + "tests_data_integration_subdir", + "voxel_sizes", # in microns + ] + + params = ( + [ + TESTS_DATA_INTEGRATION_PATH + / Path("detection", "crop_planes", "ch0"), + TESTS_DATA_INTEGRATION_PATH + / Path("detection", "crop_planes", "ch1"), + ], + [[3, 2, 2], [5, 2, 2]], + ) + + def setup(self, subdir, voxel_sizes): + self.data_dir = str(subdir) + + def teardown(self, subdir, voxel_sizes): + del self.data_dir + # Q for review: do I need this? only if it is the parameter we sweep across? + # from https://github.com/astropy/astropy-benchmarks/blob/8758dabf84001903ea00c31a001809708969a3e4/benchmarks/cosmology.py#L24 + # (they only use teardown in that case) + + def time_read_with_dask(self, subdir, voxel_sizes): + read_with_dask(self.data_dir) + + +# ----------------------------------------------- +# Benchmarks for reading metadata from tif files +# ------------------------------------------------- +class Tif: + param_names = [ + "tests_data_integration_tiffile", + ] + + params = ( + [ + *[ + x + for x in Path( + TESTS_DATA_INTEGRATION_PATH, "training", "cells" + ).glob("*.tif") + ], + *[ + x + for x in Path( + TESTS_DATA_INTEGRATION_PATH, "training", "non_cells" + ).glob("*.tif") + ], + ], + ) + + def setup(self, subdir): + self.data_dir = str(subdir) + + def teardown(self, subdir): + del self.data_dir + + def time_get_tiff_meta( + self, + subdir, + ): + get_tiff_meta(self.data_dir) diff --git a/benchmarks/benchmarks/tools/__init__.py b/benchmarks/benchmarks/tools/__init__.py new file mode 100644 index 00000000..e69de29b From 9bc0046c8ef3542dc81756d3d16d3580e6fcfc65 Mon Sep 17 00:00:00 2001 From: sfmig <33267254+sfmig@users.noreply.github.com> Date: Wed, 28 Jun 2023 16:52:02 +0100 Subject: [PATCH 09/33] fix build command to use pyproject correctly --- benchmarks/asv.conf.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/benchmarks/asv.conf.json b/benchmarks/asv.conf.json index b6cc00ee..bc9f2f65 100644 --- a/benchmarks/asv.conf.json +++ b/benchmarks/asv.conf.json @@ -15,7 +15,7 @@ // The Python project's subdirectory in your repo. If missing or // the empty string, the project is assumed to be located at the root - // of the repository. + // of the repository (where setup.py is located) // "repo_subdir": "", // Customizable commands for building, installing, and @@ -24,6 +24,8 @@ "install_command": ["in-dir={env_dir} python -mpip install {wheel_file}"], "uninstall_command": ["return-code=any python -mpip uninstall -y {project}"], "build_command": [ + "python -m pip install build", + "python -m build", "PIP_NO_BUILD_ISOLATION=false python -mpip wheel --no-deps --no-index -w {build_cache_dir} {build_dir}" ], From 2bf4c5a0a23f5e65d43c82466cf96b625e9412e1 Mon Sep 17 00:00:00 2001 From: sfmig <33267254+sfmig@users.noreply.github.com> Date: Thu, 29 Jun 2023 10:17:18 +0100 Subject: [PATCH 10/33] black formatting to IO benchmarks --- benchmarks/benchmarks/tools/IO.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/benchmarks/benchmarks/tools/IO.py b/benchmarks/benchmarks/tools/IO.py index 3c9e39c2..e4915cb1 100644 --- a/benchmarks/benchmarks/tools/IO.py +++ b/benchmarks/benchmarks/tools/IO.py @@ -3,7 +3,6 @@ from cellfinder_core.tools.IO import get_tiff_meta, read_with_dask - p = Path(os.path.dirname(__file__)).absolute() CELLFINDER_CORE_PATH = p.parents[2] # Q for review: is there a nice way to get cellfinder-core path? @@ -36,8 +35,10 @@ def setup(self, subdir, voxel_sizes): def teardown(self, subdir, voxel_sizes): del self.data_dir - # Q for review: do I need this? only if it is the parameter we sweep across? - # from https://github.com/astropy/astropy-benchmarks/blob/8758dabf84001903ea00c31a001809708969a3e4/benchmarks/cosmology.py#L24 + # Q for review: do I need this? only if it is the parameter we sweep + # across? + # from https://github.com/astropy/astropy-benchmarks/blob/ + # 8758dabf84001903ea00c31a001809708969a3e4/benchmarks/cosmology.py#L24 # (they only use teardown in that case) def time_read_with_dask(self, subdir, voxel_sizes): From 336be4c5a0f9d45e3c5e11eda7360356199b7994 Mon Sep 17 00:00:00 2001 From: sfmig <33267254+sfmig@users.noreply.github.com> Date: Thu, 29 Jun 2023 10:18:28 +0100 Subject: [PATCH 11/33] prep benchmarks pending teardown --- benchmarks/benchmarks/tools/prep.py | 92 +++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 benchmarks/benchmarks/tools/prep.py diff --git a/benchmarks/benchmarks/tools/prep.py b/benchmarks/benchmarks/tools/prep.py new file mode 100644 index 00000000..475aed90 --- /dev/null +++ b/benchmarks/benchmarks/tools/prep.py @@ -0,0 +1,92 @@ +import shutil +from pathlib import Path + +# NOTE: imlib to be replaced by brainglobe_utils +from imlib.general.system import get_num_processes + +from cellfinder_core.tools.prep import ( + prep_classification, + prep_models, + prep_tensorflow, + prep_training, +) + + +class Prep: + # common params? + # rounds = 2 # default + # repeat = 1 # default if rounds!= 1 + # ---> (min_repeat, max_repeat, max_time) = (1, 5, 10.0) + # number = 1 # run only once per repeat? + + # common setup + def setup(self): + # print('setup') + + # TODO: how is n_free_cpus and n_process diff? + # n_processes: n of CPUs to use? + # n_free_cpus: n of CPUs to leave free in the machine? + # Determine how many CPU cores to use, based on a minimum number + # of cpu cores + # to leave free, and an optional max number of processes. + self.n_free_cpus = 2 # should this be parametrised?? + self.n_processes = get_num_processes( + min_free_cpu_cores=self.n_free_cpus + ) + self.trained_model = None + self.model_weights = None + self.install_path = Path.home() / ".cellfinder" # default + self.model_name = "resnet50_tv" # resnet50_all + + def teardown(self): + # pass + + # ------------------ + # TODO: not sure why the benchmark is timing out with this teardown? + # remove everything in temp dir? + # Q for review: is this safe? + shutil.rmtree(self.install_path) + # print('teardown') + # ------------------ + + # print([f for f in Path(self.install_path).glob("*")]) + + # for f in Path(self.install_path).glob("*"): + # if f.is_dir(): + # shutil.rmtree(f,) + # return + # # print(f) + # f.unlink(missing_ok=True) + # Path(self.install_path).rmdir() + + def time_prep_tensorfow(self): + prep_tensorflow(self.n_processes) + + def time_prep_models(self): + # downloads model weights to .cellfinder dir + # -should I remove the files after each rep so that it is comparable? + # (overwriting may not be the same as writing from scratch?) + prep_models( + self.trained_model, + self.model_weights, + self.install_path, + self.model_name, + ) + + def time_prep_classification(self): + prep_classification( + self.trained_model, + self.model_weights, + self.install_path, + self.model_name, + self.n_free_cpus, + ) + + def time_prep_training(self): + prep_training( + self.n_free_cpus, + self.trained_model, + self.model_weights, + self.install_path, + self.model_name, + ) From a2b200e9975a2f90f7d038a0832e39ccff4f0583 Mon Sep 17 00:00:00 2001 From: sfmig <33267254+sfmig@users.noreply.github.com> Date: Thu, 29 Jun 2023 14:44:46 +0100 Subject: [PATCH 12/33] remove voxel_size from IO and refactor. change precommit config to skip mypy errors in IO benchmarks --- .pre-commit-config.yaml | 1 + benchmarks/benchmarks/tools/IO.py | 74 ++++++++++++++----------------- 2 files changed, 34 insertions(+), 41 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7f13f9cc..47f0c9d6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -23,6 +23,7 @@ repos: rev: v1.3.0 hooks: - id: mypy + exclude: benchmarks/benchmarks/tools/IO.py additional_dependencies: - types-setuptools - types-requests diff --git a/benchmarks/benchmarks/tools/IO.py b/benchmarks/benchmarks/tools/IO.py index e4915cb1..9a50a3e4 100644 --- a/benchmarks/benchmarks/tools/IO.py +++ b/benchmarks/benchmarks/tools/IO.py @@ -5,35 +5,20 @@ p = Path(os.path.dirname(__file__)).absolute() CELLFINDER_CORE_PATH = p.parents[2] -# Q for review: is there a nice way to get cellfinder-core path? TESTS_DATA_INTEGRATION_PATH = ( Path(CELLFINDER_CORE_PATH) / "tests" / "data" / "integration" ) +# Q for review: is there a nice way to get cellfinder-core path? -# --------------------------------------------- -# Benchmarks for reading 3d arrays with dask -# -------------------------------------------- -class Dask: - param_names = [ - "tests_data_integration_subdir", - "voxel_sizes", # in microns - ] - - params = ( - [ - TESTS_DATA_INTEGRATION_PATH - / Path("detection", "crop_planes", "ch0"), - TESTS_DATA_INTEGRATION_PATH - / Path("detection", "crop_planes", "ch1"), - ], - [[3, 2, 2], [5, 2, 2]], - ) - - def setup(self, subdir, voxel_sizes): +class Read: + # --------------------------------------------- + # Setup & teardown functions + # -------------------------------------------- + def setup(self, subdir): self.data_dir = str(subdir) - def teardown(self, subdir, voxel_sizes): + def teardown(self, subdir): del self.data_dir # Q for review: do I need this? only if it is the parameter we sweep # across? @@ -41,19 +26,38 @@ def teardown(self, subdir, voxel_sizes): # 8758dabf84001903ea00c31a001809708969a3e4/benchmarks/cosmology.py#L24 # (they only use teardown in that case) - def time_read_with_dask(self, subdir, voxel_sizes): + # --------------------------------------------- + # Benchmarks for reading 3d arrays with dask + # -------------------------------------------- + def time_read_with_dask(self, subdir): read_with_dask(self.data_dir) + time_read_with_dask.param_names = [ + "tests_data_integration_subdir", + ] + time_read_with_dask.params = ( + [ + TESTS_DATA_INTEGRATION_PATH + / Path("detection", "crop_planes", "ch0"), + TESTS_DATA_INTEGRATION_PATH + / Path("detection", "crop_planes", "ch1"), + ], + ) + + # ----------------------------------------------- + # Benchmarks for reading metadata from tif files + # ------------------------------------------------- + def time_get_tiff_meta( + self, + subdir, + ): + get_tiff_meta(self.data_dir) -# ----------------------------------------------- -# Benchmarks for reading metadata from tif files -# ------------------------------------------------- -class Tif: - param_names = [ + time_get_tiff_meta.param_names = [ "tests_data_integration_tiffile", ] - params = ( + time_get_tiff_meta.params = ( [ *[ x @@ -69,15 +73,3 @@ class Tif: ], ], ) - - def setup(self, subdir): - self.data_dir = str(subdir) - - def teardown(self, subdir): - del self.data_dir - - def time_get_tiff_meta( - self, - subdir, - ): - get_tiff_meta(self.data_dir) From 3f3a3445e960e3a4d4e09980b289be5506f4f76a Mon Sep 17 00:00:00 2001 From: sfmig <33267254+sfmig@users.noreply.github.com> Date: Thu, 29 Jun 2023 14:59:34 +0100 Subject: [PATCH 13/33] remove list comprehension --- benchmarks/benchmarks/tools/IO.py | 29 +++++++++++------------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/benchmarks/benchmarks/tools/IO.py b/benchmarks/benchmarks/tools/IO.py index 9a50a3e4..9b649c0a 100644 --- a/benchmarks/benchmarks/tools/IO.py +++ b/benchmarks/benchmarks/tools/IO.py @@ -20,11 +20,11 @@ def setup(self, subdir): def teardown(self, subdir): del self.data_dir - # Q for review: do I need this? only if it is the parameter we sweep - # across? + # Q for review: do I need this? + # only if it is the parameter we sweep across? # from https://github.com/astropy/astropy-benchmarks/blob/ # 8758dabf84001903ea00c31a001809708969a3e4/benchmarks/cosmology.py#L24 - # (they only use teardown in that case) + # (they only use teardown function in that case) # --------------------------------------------- # Benchmarks for reading 3d arrays with dask @@ -57,19 +57,12 @@ def time_get_tiff_meta( "tests_data_integration_tiffile", ] - time_get_tiff_meta.params = ( - [ - *[ - x - for x in Path( - TESTS_DATA_INTEGRATION_PATH, "training", "cells" - ).glob("*.tif") - ], - *[ - x - for x in Path( - TESTS_DATA_INTEGRATION_PATH, "training", "non_cells" - ).glob("*.tif") - ], - ], + cells_tif_files = list( + Path(TESTS_DATA_INTEGRATION_PATH, "training", "cells").glob("*.tif") + ) + non_cells_tif_files = list( + Path(TESTS_DATA_INTEGRATION_PATH, "training", "non_cells").glob( + "*.tif" + ) ) + time_get_tiff_meta.params = cells_tif_files + non_cells_tif_files From d20a97bfb4e5e69bc75b3227315775438bf44874 Mon Sep 17 00:00:00 2001 From: sfmig <33267254+sfmig@users.noreply.github.com> Date: Thu, 29 Jun 2023 17:01:08 +0100 Subject: [PATCH 14/33] add benchmarks imports --- benchmarks/benchmarks/imports.py | 40 ++++++++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/benchmarks/benchmarks/imports.py b/benchmarks/benchmarks/imports.py index e467f3c1..d7ef25cc 100644 --- a/benchmarks/benchmarks/imports.py +++ b/benchmarks/benchmarks/imports.py @@ -1,13 +1,43 @@ - # ------------------------------------ -# Runtime benchmarks +# Runtime benchmarks # ------------------------------------ def timeraw_import_main(): - return""" + return """ from cellfinder_core.main import main """ + def timeraw_import_io_dask(): - return""" + return """ from cellfinder_core.tools.IO import read_with_dask - """ \ No newline at end of file + """ + + +def timeraw_import_io_tiff_meta(): + return """ + from cellfinder_core.tools.IO import get_tiff_meta + """ + + +def timeraw_import_prep_tensorflow(): + return """ + from cellfinder_core.tools.prep import prep_tensorflow + """ + + +def timeraw_import_prep_models(): + return """ + from cellfinder_core.tools.prep import prep_models + """ + + +def timeraw_import_prep_classification(): + return """ + from cellfinder_core.tools.prep import prep_classification + """ + + +def timeraw_import_prep_training(): + return """ + from cellfinder_core.tools.prep import prep_training + """ From 80f12a0910c313f11d27f39361d304564599119c Mon Sep 17 00:00:00 2001 From: sfmig <33267254+sfmig@users.noreply.github.com> Date: Thu, 29 Jun 2023 17:01:23 +0100 Subject: [PATCH 15/33] remove initial templates --- benchmarks/benchmarks/basic_benchmarks.py | 19 ---- benchmarks/benchmarks/sample_benchmarks.py | 119 --------------------- 2 files changed, 138 deletions(-) delete mode 100644 benchmarks/benchmarks/basic_benchmarks.py delete mode 100644 benchmarks/benchmarks/sample_benchmarks.py diff --git a/benchmarks/benchmarks/basic_benchmarks.py b/benchmarks/benchmarks/basic_benchmarks.py deleted file mode 100644 index 9230b3b9..00000000 --- a/benchmarks/benchmarks/basic_benchmarks.py +++ /dev/null @@ -1,19 +0,0 @@ -class TimeSuiteBasic: - params = [0, 10, 100, 1000] - param_names = ['n'] - - def setup(self, n): - self.d = {} - for x in range(n): - self.d[x] = None - - def time_keys(self, n): # why do I need to pass parameter here? - for ky in self.d.keys(): - pass - - def time_values(self, n): - for val in self.d.values(): - pass - - def teardown(self, n): - pass \ No newline at end of file diff --git a/benchmarks/benchmarks/sample_benchmarks.py b/benchmarks/benchmarks/sample_benchmarks.py deleted file mode 100644 index 15cc32d0..00000000 --- a/benchmarks/benchmarks/sample_benchmarks.py +++ /dev/null @@ -1,119 +0,0 @@ -# Write the benchmarking functions here. -# See "Writing benchmarks" in the asv docs for more information. -# Notes: -# - benchmarks may be organised into methods of classes if desired -# (or just as functions that start with "time_") - -# ------------------------------------ -# Runtime benchmarks start with 'time' -# (snake case or camelcase) -# ------------------------------------ -class TimeSuite: - """ - An example benchmark that times the performance of various kinds - of iterating over dictionaries in Python. - """ - def setup(self): - """ - Setup includes initialisation bits that should not be included - in the timing of the benchmark. - - It can be added as: - - a method of a class, or - - an attribute of a free function, or - - a module-level setup function (run for every benchmark in the - module, prior to any function-specific setup) - - If setup raises `NotImplementedError`, the benchmark is skipped - """ - self.d = {} - for x in range(500): - self.d[x] = None - - def setup_cache(self): - """ - `Setup_cache` only performs the setup calculation once - (for each benchmark and each repeat) and caches the - result to disk. This may be useful if the setup is - expensive. - - A separate cache is used for each environment and each commit. - The cache is thrown out between benchmark runs. - - There are two options to persist the data for the benchmarks: - - `setup_cache` returns a data structure, which asv pickles to disk, - and then loads and passes as first arg to each benchmark (not - automagically tho), or - - `setup_cache` saves files to the cwd (which is a temp dir managed by - asv), which are then explicitly loaded in each benchmark. Recomm - practice is to actually read the data in a `setup` fn, so that - loading time is not part of the timing - """ - pass - - def teardown(self): - """ - Benchmarks can also have teardown functions that are run after - the benchmark. The behaviour is similar to setup fns. - - Useful for example to clean up changes made to the - filesystem - """ - pass - - def time_keys(self): - # benchmark attributes - timeout = 123 # The amount of time, in seconds, to give the benchmark to run before forcibly killing it. Defaults to 60 seconds. - pretty_name = 'pretty name' - setup = setup - teardown = teardown - rounds = 2 - repeat = (1, 5, 10.0) - warmup_time = 0.1 - # params_names - # params ---> the params attribute allows us to run a single benchmark - # for multiple values of the parameters - - for key in self.d.keys(): - pass - - def time_values(self): - # For best results, the benchmark function should contain - # as little as possible, with as much extraneous setup moved to a setup function: - for value in self.d.values(): - pass - - def time_range(self, n): - d = self.d - for key in range(500): - x = d[key] - -# ----------------------- -# Parametrized benchmarks -# ------------------------- -# - params can be any Python object, but it is recommended -# only strings and number -# - w/ multiple params, the test is run for all combs -class Suite: - params = [0, 10, 20] - param_names = ['n'] - - def setup(self, n): - # Note that setup_cache is not parametrized (can it?) - self.obj = range(n) - - def teardown(self, n): - del self.obj - - def time_range_iter(self, n): - for i in self.obj: - pass - - -# ------------------------------------ -# Memory benchmarks start with 'mem' -# (snake case or camelcase) -# ------------------------------------ -class MemSuite: - def mem_list(self): - return [0] * 256 From bfc7383df816ab72c8e133a471d7a130a120bc6b Mon Sep 17 00:00:00 2001 From: sfmig <33267254+sfmig@users.noreply.github.com> Date: Thu, 29 Jun 2023 17:01:39 +0100 Subject: [PATCH 16/33] add init to benchmarks --- benchmarks/benchmarks/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/benchmarks/benchmarks/__init__.py b/benchmarks/benchmarks/__init__.py index 8b137891..e69de29b 100644 --- a/benchmarks/benchmarks/__init__.py +++ b/benchmarks/benchmarks/__init__.py @@ -1 +0,0 @@ - From 589ffce97bb9e3f7f1c35a89b6f51a0064fecd27 Mon Sep 17 00:00:00 2001 From: sfmig <33267254+sfmig@users.noreply.github.com> Date: Fri, 30 Jun 2023 12:02:53 +0100 Subject: [PATCH 17/33] add readme and comments to asv config --- benchmarks/README.md | 157 +++++++++++++++++++++++++++++++++++++++ benchmarks/asv.conf.json | 5 +- 2 files changed, 161 insertions(+), 1 deletion(-) diff --git a/benchmarks/README.md b/benchmarks/README.md index e69de29b..1e92bb3b 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -0,0 +1,157 @@ +# Benchmarking with asv +[Install asv](https://asv.readthedocs.io/en/stable/installing.html) by running: +``` +pip install asv +``` + +`asv` works roughly as follows: +1. It creates a virtual environment (as defined in the config) +2. It installs the software package version of a specific commit +3. It times the benchmarking tests + +## Running benchmarks +To run benchmarks on a specific commit: +``` +$ asv run 88fbbc33^! +``` + +To run them up to a specific commit: +``` +$ asv run 88fbbc33 +``` + +To run them on a range of commits: +``` +$ asv run 827f322b..729abcf3 +``` + +To collate the benchmarks' results into a viewable website: +``` +$ asv publish +``` +This will create a tree of files in the `html` directory, but this cannot be viewed directly from the local filesystem, so we need to put them in a static site. `asv publish` also detects satistically significant decreases of performance, the results can be inspected in the 'Regression' tab of the static site (more on that on the next section). + +To visualise the results in a static site: +``` +$ asv preview +``` +To share the website on the internet, put the files in the `html` directory on any webserver that can serve static content (e.g. GitHub pages). + +To put the results in the `gh-pages` branch and push them to GitHub: +``` +$asv gh-pages +``` + +## Managing the results + +To remove benchmarks from the database, for example, for a specific commit: + +``` +$ asv rm commit_hash=a802047be +``` +See more options in the [documentation](https://asv.readthedocs.io/en/stable/using.html#managing-the-results-database). + +This will remove the selected results from the files in the `results` directory. To update the results in the static site, remember to run `publish` again! + + +To compare the results of running the benchmarks on two commits: +``` +$ asv compare 88fbbc33 827f322b +``` + +## Automatically detecting performance regressions + + + +## Other handy commands +To update the machine information +``` +$ asv machine +``` + +To display results from previous runs on the command line +``` +$ asv show +``` + +To use binary search to find a commit within the benchmarked range that produced a large regression +``` +$ asv check +$ asv find +``` + +To check the validity of written benchmarks +``` +$ asv check +``` + + +## Development notes: +In development, the following flags to `asv run` are often useful: +- `--bench`: to specify a subset of benchmarks (e.g., `tools.prep.PrepTF`). Regexp can be used. +- `--dry-run`: will not write results to disk +- `--quick`: will only run one repetition, and no results to disk +- `--show-stderr`: will print out stderr +- `--verbose`: provides further info on intermediate steps +- `--python=same`: runs the benchmarks in the same environment that `asv` was launched from + +E.g.: +``` +asv run --bench bench tools.prep.PrepTF --dry-run --show-stderr --quick +``` + +### Running benchmarks against a local commit +To run the benchmarks against a local commit (for example, if you are trying to improve the performance of the code), you need to edit the `repo` field in the asv config file `asv.conf.json`. + +To use the upstream repository, use: +``` +"repo": "https://github.com/brainglobe/cellfinder-core.git", +``` + +To use the local repository, use: +``` +"repo": "..", +``` + +### setup and setup_cache + +Setup includes initialisation bits that should not be included +in the timing of the benchmark. + +It can be added as: + - a method of a class, or + - an attribute of a free function, or + - a module-level setup function (run for every benchmark in the + module, prior to any function-specific setup) + +If setup raises `NotImplementedError`, the benchmark is skipped + +`setup_cache` only performs the setup calculation once +(for each benchmark and each repeat) and caches the +result to disk. This may be useful if the setup is +expensive. + +A separate cache is used for each environment and each commit. +The cache is thrown out between benchmark runs. + +There are two options to persist the data for the benchmarks: +- `setup_cache` returns a data structure, which asv pickles to disk, + and then loads and passes as first arg to each benchmark (not + automagically tho), or +- `setup_cache` saves files to the cwd (which is a temp dir managed by + asv), which are then explicitly loaded in each benchmark. Recomm + practice is to actually read the data in a `setup` fn, so that + loading time is not part of the timing + +### Check benchmarks +To check the validity of written benchmarks +``` +$ asv check +``` + + +---- +## References +- [astropy-benchmarks repository](https://github.com/astropy/astropy-benchmarks/tree/main) +- [numpy benchmarks](https://github.com/numpy/numpy/tree/main/benchmarks/benchmarks) +- [asv documentation](https://asv.readthedocs.io/en/stable/index.html) diff --git a/benchmarks/asv.conf.json b/benchmarks/asv.conf.json index bc9f2f65..1a4e32d9 100644 --- a/benchmarks/asv.conf.json +++ b/benchmarks/asv.conf.json @@ -11,6 +11,9 @@ // The URL or local path of the source code repository for the // project being benchmarked + // To use the upstream repository: uncomment the 1st line (and comment the 2nd) + // To use the local repository: comment the 1st line (and uncomment the 2nd) + //"repo": "https://github.com/brainglobe/cellfinder-core.git", "repo": "..", // The Python project's subdirectory in your repo. If missing or @@ -87,7 +90,7 @@ // "matrix": { "req": {}, - // "napari": ["", null], // test with and without + // "napari": ["", null], // test with and without // // "six": ["", null], // test with and without six installed // // "pip+emcee": [""] // emcee is only available for install with pip. // }, From febb77630f2868aecdac010ccd555b5975dd4782 Mon Sep 17 00:00:00 2001 From: sfmig <33267254+sfmig@users.noreply.github.com> Date: Fri, 30 Jun 2023 12:04:49 +0100 Subject: [PATCH 18/33] add teardown function to prep benchmarks --- benchmarks/benchmarks/tools/prep.py | 85 +++++++++++++---------------- 1 file changed, 37 insertions(+), 48 deletions(-) diff --git a/benchmarks/benchmarks/tools/prep.py b/benchmarks/benchmarks/tools/prep.py index 475aed90..a5c426e3 100644 --- a/benchmarks/benchmarks/tools/prep.py +++ b/benchmarks/benchmarks/tools/prep.py @@ -12,81 +12,70 @@ ) -class Prep: - # common params? - # rounds = 2 # default - # repeat = 1 # default if rounds!= 1 - # ---> (min_repeat, max_repeat, max_time) = (1, 5, 10.0) - # number = 1 # run only once per repeat? +class PrepModels: + param_names = ["model_name"] + params = ["resnet50_tv", "resnet50_all"] - # common setup - def setup(self): - # print('setup') + # increase default timeout to allow for download + timeout = 240 + + # Q for review: + # - should I run only one sample ('number'=1)? + # 'number' as defined here: + # https://asv.readthedocs.io/en/stable/writing_benchmarks.html#timing + # - how are prep_classification and prep_training different? - # TODO: how is n_free_cpus and n_process diff? - # n_processes: n of CPUs to use? - # n_free_cpus: n of CPUs to leave free in the machine? - # Determine how many CPU cores to use, based on a minimum number - # of cpu cores - # to leave free, and an optional max number of processes. - self.n_free_cpus = 2 # should this be parametrised?? + def setup(self, model_name): + self.n_free_cpus = 2 # TODO: should this be parametrised?? self.n_processes = get_num_processes( min_free_cpu_cores=self.n_free_cpus ) self.trained_model = None self.model_weights = None - self.install_path = Path.home() / ".cellfinder" # default - self.model_name = "resnet50_tv" # resnet50_all + self.install_path = Path.home() / ".cellfinder" + self.model_name = model_name - def teardown(self): - # pass + # remove .cellfinder dir if it exists already + shutil.rmtree(self.install_path, ignore_errors=True) + # Q for review: + # - is this safe? + # - should I check if install_path is the expected path? - # ------------------ - # TODO: not sure why the benchmark is timing out with this teardown? - # remove everything in temp dir? - # Q for review: is this safe? + def teardown(self, model_name): + # remove .cellfinder dir after benchmarks shutil.rmtree(self.install_path) - # print('teardown') - # ------------------ - - # print([f for f in Path(self.install_path).glob("*")]) - - # for f in Path(self.install_path).glob("*"): - # if f.is_dir(): - # shutil.rmtree(f,) - # return - # # print(f) - # f.unlink(missing_ok=True) - # Path(self.install_path).rmdir() - - def time_prep_tensorfow(self): - prep_tensorflow(self.n_processes) - def time_prep_models(self): - # downloads model weights to .cellfinder dir - # -should I remove the files after each rep so that it is comparable? - # (overwriting may not be the same as writing from scratch?) + def time_prep_models(self, model_name): prep_models( self.trained_model, self.model_weights, self.install_path, - self.model_name, + model_name, ) - def time_prep_classification(self): + def time_prep_classification(self, model_name): prep_classification( self.trained_model, self.model_weights, self.install_path, - self.model_name, + model_name, self.n_free_cpus, ) - def time_prep_training(self): + def time_prep_training(self, model_name): prep_training( self.n_free_cpus, self.trained_model, self.model_weights, self.install_path, - self.model_name, + model_name, ) + + +class PrepTF: + def setup(self): + n_free_cpus = 2 # TODO: should we parametrise this? + self.n_processes = get_num_processes(min_free_cpu_cores=n_free_cpus) + + def time_prep_tensorflow(self): + prep_tensorflow(self.n_processes) From 2e35a8a03222a3970570d485c73371e66e41a604 Mon Sep 17 00:00:00 2001 From: sfmig <33267254+sfmig@users.noreply.github.com> Date: Fri, 30 Jun 2023 12:05:48 +0100 Subject: [PATCH 19/33] add comment for review --- benchmarks/benchmarks/tools/IO.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/benchmarks/tools/IO.py b/benchmarks/benchmarks/tools/IO.py index 9b649c0a..77fbe9a6 100644 --- a/benchmarks/benchmarks/tools/IO.py +++ b/benchmarks/benchmarks/tools/IO.py @@ -21,7 +21,7 @@ def setup(self, subdir): def teardown(self, subdir): del self.data_dir # Q for review: do I need this? - # only if it is the parameter we sweep across? + # maybe only relevant if it is the parameter we sweep across? # from https://github.com/astropy/astropy-benchmarks/blob/ # 8758dabf84001903ea00c31a001809708969a3e4/benchmarks/cosmology.py#L24 # (they only use teardown function in that case) From 382a2859fd8442dcafd92ca1119f501fe3e0ba70 Mon Sep 17 00:00:00 2001 From: sfmig <33267254+sfmig@users.noreply.github.com> Date: Fri, 30 Jun 2023 13:57:46 +0100 Subject: [PATCH 20/33] add cellfinder_core.tools.prep mypy fix --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 20cc1cff..36647412 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -127,7 +127,8 @@ ignore_missing_imports = true [[tool.mypy.overrides]] module = [ "cellfinder_core.detect.*", - "cellfinder_core.classify.*" + "cellfinder_core.classify.*", + "cellfinder_core.tools.prep.*" ] disallow_untyped_defs = true disallow_incomplete_defs = true From 357725fb499d6c9e506e4649a5d32247d4c5698e Mon Sep 17 00:00:00 2001 From: sfmig <33267254+sfmig@users.noreply.github.com> Date: Fri, 30 Jun 2023 13:57:55 +0100 Subject: [PATCH 21/33] replace imlib by brainglobe_utils --- benchmarks/benchmarks/tools/prep.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/benchmarks/benchmarks/tools/prep.py b/benchmarks/benchmarks/tools/prep.py index a5c426e3..95aa7f35 100644 --- a/benchmarks/benchmarks/tools/prep.py +++ b/benchmarks/benchmarks/tools/prep.py @@ -1,8 +1,7 @@ import shutil from pathlib import Path -# NOTE: imlib to be replaced by brainglobe_utils -from imlib.general.system import get_num_processes +from brainglobe_utils.general.system import get_num_processes from cellfinder_core.tools.prep import ( prep_classification, From 43fe3982c60a3933a43c231d79a27ebb14ef1cb8 Mon Sep 17 00:00:00 2001 From: sfmig <33267254+sfmig@users.noreply.github.com> Date: Fri, 30 Jun 2023 13:58:33 +0100 Subject: [PATCH 22/33] small additions to readme --- benchmarks/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/benchmarks/README.md b/benchmarks/README.md index 1e92bb3b..7296dce3 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -7,7 +7,9 @@ pip install asv `asv` works roughly as follows: 1. It creates a virtual environment (as defined in the config) 2. It installs the software package version of a specific commit -3. It times the benchmarking tests +3. It times the benchmarking tests and saves the results to json files +4. The json files are 'published' into an html dir +5. The html dir can be visualised in a static website ## Running benchmarks To run benchmarks on a specific commit: From f6b8f08961217f1778a0d9ec91a0c65eed96e0d0 Mon Sep 17 00:00:00 2001 From: sfmig <33267254+sfmig@users.noreply.github.com> Date: Fri, 30 Jun 2023 14:04:03 +0100 Subject: [PATCH 23/33] move cellfinder_core.tool.prep to ignore imports section --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 36647412..5a1e296a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -121,6 +121,7 @@ module = [ "scipy.*", "skimage.*", "sklearn.*", + "cellfinder_core.tools.prep.*", ] ignore_missing_imports = true @@ -128,7 +129,6 @@ ignore_missing_imports = true module = [ "cellfinder_core.detect.*", "cellfinder_core.classify.*", - "cellfinder_core.tools.prep.*" ] disallow_untyped_defs = true disallow_incomplete_defs = true From 2051c4a6758ab47d04268c52222399437f90fb85 Mon Sep 17 00:00:00 2001 From: sfmig <33267254+sfmig@users.noreply.github.com> Date: Fri, 30 Jun 2023 14:08:35 +0100 Subject: [PATCH 24/33] remove notebook --- notebook_det_and_classif_dask.py | 55 -------------------------------- 1 file changed, 55 deletions(-) delete mode 100644 notebook_det_and_classif_dask.py diff --git a/notebook_det_and_classif_dask.py b/notebook_det_and_classif_dask.py deleted file mode 100644 index 28e673e1..00000000 --- a/notebook_det_and_classif_dask.py +++ /dev/null @@ -1,55 +0,0 @@ -# %% -import os -from pathlib import Path -import imlib.IO.cells as cell_io - -from cellfinder_core.main import main as cellfinder_run -from cellfinder_core.tools.IO import read_with_dask - - - -# Input data -data_dir = Path(os.getcwd()) / "tests" / "data" / "integration" / "detection" -signal_data_dir = data_dir / "crop_planes" / "ch0" -background_data_dir = data_dir / "crop_planes" / "ch1" -voxel_sizes = [5, 2, 2] # microns - - - -if __name__ == "__main__": - - # Read data - # - dask for ~ TB of data, you pass the directory and it will load all the images as a 3D array - # - tiff can also be a 3D array but no examples in the test data - signal_array = read_with_dask(str(signal_data_dir)) # (30, 510, 667) ---> planes , image size (h, w?) - background_array = read_with_dask(str(background_data_dir)) - - # D+C pipeline - # Detection and classification pipeline - # the output is a list of imlib Cell objects w/ centroid coordinate and type - detected_cells = cellfinder_run( - signal_array, - background_array, - voxel_sizes, - ) - - # Inspect results - print(f'Sample cell type: {type(detected_cells[0])}') - print('Sample cell attributes: ' - f'x={detected_cells[0].x}, ' - f'y={detected_cells[0].y}, ' - f'z={detected_cells[0].z}, ' - f'type={detected_cells[0].type}') # Cell: x: 132, y: 308, z: 10, type: 2 - - num_cells = sum([cell.type == 2 for cell in detected_cells]) # Cell type 2 is a true positive (classified as cell), - num_non_cells = sum([cell.type == 1 for cell in detected_cells]) # Cell type 1 is a false positive (classified as non-cell) - print(f'{num_cells}/{len(detected_cells)} cells classified as cells') - print(f'{num_non_cells}/{len(detected_cells)} cells classified as non-cells') - - - # Save results in the cellfinder XML standard - # it only saves type 1 - cell_io.save_cells(detected_cells, 'output.xml') - - # # to read them - # cell_io.get_cells('output.xml') \ No newline at end of file From d0ce5399660ae5ba2d2411d4f750d9ccf9300822 Mon Sep 17 00:00:00 2001 From: sfmig <33267254+sfmig@users.noreply.github.com> Date: Fri, 30 Jun 2023 16:12:46 +0100 Subject: [PATCH 25/33] increase timeout --- benchmarks/benchmarks/tools/prep.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/benchmarks/tools/prep.py b/benchmarks/benchmarks/tools/prep.py index 95aa7f35..97c52f4c 100644 --- a/benchmarks/benchmarks/tools/prep.py +++ b/benchmarks/benchmarks/tools/prep.py @@ -16,7 +16,7 @@ class PrepModels: params = ["resnet50_tv", "resnet50_all"] # increase default timeout to allow for download - timeout = 240 + timeout = 480 # Q for review: # - should I run only one sample ('number'=1)? From 70dffb32da9a1df99736a20d6a3e2c5fd2e01671 Mon Sep 17 00:00:00 2001 From: sfmig <33267254+sfmig@users.noreply.github.com> Date: Thu, 20 Jul 2023 10:41:11 +0200 Subject: [PATCH 26/33] small additions and format edits to the readme --- benchmarks/README.md | 37 +++++++++++++------------------------ 1 file changed, 13 insertions(+), 24 deletions(-) diff --git a/benchmarks/README.md b/benchmarks/README.md index 7296dce3..adb5ec8e 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -6,7 +6,7 @@ pip install asv `asv` works roughly as follows: 1. It creates a virtual environment (as defined in the config) -2. It installs the software package version of a specific commit +2. It installs the software package version of a specific commit (or of a local commit) 3. It times the benchmarking tests and saves the results to json files 4. The json files are 'published' into an html dir 5. The html dir can be visualised in a static website @@ -78,7 +78,6 @@ $ asv show To use binary search to find a commit within the benchmarked range that produced a large regression ``` -$ asv check $ asv find ``` @@ -117,39 +116,29 @@ To use the local repository, use: ### setup and setup_cache -Setup includes initialisation bits that should not be included -in the timing of the benchmark. - -It can be added as: +- `setup` includes initialisation bits that should not be included +in the timing of the benchmark. It can be added as: - a method of a class, or - an attribute of a free function, or - a module-level setup function (run for every benchmark in the module, prior to any function-specific setup) -If setup raises `NotImplementedError`, the benchmark is skipped + If `setup` raises `NotImplementedError`, the benchmark is skipped -`setup_cache` only performs the setup calculation once +- `setup_cache` only performs the setup calculation once (for each benchmark and each repeat) and caches the -result to disk. This may be useful if the setup is +result to disk. This may be useful if the setup is computationally expensive. -A separate cache is used for each environment and each commit. -The cache is thrown out between benchmark runs. + A separate cache is used for each environment and each commit. The cache is thrown out between benchmark runs. -There are two options to persist the data for the benchmarks: -- `setup_cache` returns a data structure, which asv pickles to disk, - and then loads and passes as first arg to each benchmark (not - automagically tho), or -- `setup_cache` saves files to the cwd (which is a temp dir managed by - asv), which are then explicitly loaded in each benchmark. Recomm - practice is to actually read the data in a `setup` fn, so that - loading time is not part of the timing + There are two options to persist the data for the benchmarks: + - `setup_cache` returns a data structure, which asv pickles to disk, + and then loads and passes as the first argument to each benchmark (not + automagically though), or + - `setup_cache` saves files to the cwd (which is a temp dir managed by + asv), which are then explicitly loaded in each benchmark. The recommended practice is to actually read the data in a `setup` function, so that loading time is not part of the benchmark timing. -### Check benchmarks -To check the validity of written benchmarks -``` -$ asv check -``` ---- From 701a13515a494fe047ac5e4dc9b83206ac3ef895 Mon Sep 17 00:00:00 2001 From: sfmig <33267254+sfmig@users.noreply.github.com> Date: Thu, 20 Jul 2023 16:36:09 +0200 Subject: [PATCH 27/33] exclude benchmarks from manifest --- MANIFEST.in | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index aaf57952..7ef5b1b5 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1,15 @@ -prune tests/data +include README.md +include LICENSE +include pyproject.toml + +exclude *.yml +exclude *.yaml +exclude tox.ini +exclude CHANGELOG.md + +graft src + +prune benchmarks +prune tests + +exclude cellfinder-core/benchmarks/* From 8d3ed4b77d9b3bcbafb92b1616f27a5e3e30eee6 Mon Sep 17 00:00:00 2001 From: sfmig <33267254+sfmig@users.noreply.github.com> Date: Thu, 20 Jul 2023 16:42:25 +0200 Subject: [PATCH 28/33] small additions to the readme --- benchmarks/README.md | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/benchmarks/README.md b/benchmarks/README.md index adb5ec8e..8d98c111 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -31,7 +31,7 @@ To collate the benchmarks' results into a viewable website: ``` $ asv publish ``` -This will create a tree of files in the `html` directory, but this cannot be viewed directly from the local filesystem, so we need to put them in a static site. `asv publish` also detects satistically significant decreases of performance, the results can be inspected in the 'Regression' tab of the static site (more on that on the next section). +This will create a tree of files in the `html` directory, but this cannot be viewed directly from the local filesystem, so we need to put them in a static site. `asv publish` also detects satistically significant decreases of performance, the results can be inspected in the 'Regression' tab of the static site. To visualise the results in a static site: ``` @@ -41,7 +41,7 @@ To share the website on the internet, put the files in the `html` directory on a To put the results in the `gh-pages` branch and push them to GitHub: ``` -$asv gh-pages +$ asv gh-pages ``` ## Managing the results @@ -51,19 +51,16 @@ To remove benchmarks from the database, for example, for a specific commit: ``` $ asv rm commit_hash=a802047be ``` -See more options in the [documentation](https://asv.readthedocs.io/en/stable/using.html#managing-the-results-database). This will remove the selected results from the files in the `results` directory. To update the results in the static site, remember to run `publish` again! +See more options for `asv rm` in the [asv documentation](https://asv.readthedocs.io/en/stable/using.html#managing-the-results-database). To compare the results of running the benchmarks on two commits: ``` $ asv compare 88fbbc33 827f322b ``` -## Automatically detecting performance regressions - - ## Other handy commands To update the machine information @@ -80,12 +77,18 @@ To use binary search to find a commit within the benchmarked range that produced ``` $ asv find ``` +Note this will only find the global maximum if runtimes over the range are more-or-less monotonic. See the [asv docs](https://asv.readthedocs.io/en/stable/using.html#finding-a-commit-that-produces-a-large-regression) for further details. To check the validity of written benchmarks ``` $ asv check ``` +`asv` has features to run a given benchmark in the Python standard profiler `cProfile`, and then visualise the results in the tool of your choice. For example: +``` +$ asv profile time_units.time_very_simple_unit_parse 10fc29cb +``` +See the [asv docs on profiling](https://asv.readthedocs.io/en/stable/using.html#running-a-benchmark-in-the-profiler) for further details ## Development notes: In development, the following flags to `asv run` are often useful: From 150eda7c1d98a134def926ba7e4c322b3b9b7155 Mon Sep 17 00:00:00 2001 From: sfmig <33267254+sfmig@users.noreply.github.com> Date: Thu, 20 Jul 2023 16:47:31 +0200 Subject: [PATCH 29/33] reduce readme to basic commands --- benchmarks/README.md | 112 ------------------------------------------- 1 file changed, 112 deletions(-) diff --git a/benchmarks/README.md b/benchmarks/README.md index 8d98c111..dde2fe03 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -37,115 +37,3 @@ To visualise the results in a static site: ``` $ asv preview ``` -To share the website on the internet, put the files in the `html` directory on any webserver that can serve static content (e.g. GitHub pages). - -To put the results in the `gh-pages` branch and push them to GitHub: -``` -$ asv gh-pages -``` - -## Managing the results - -To remove benchmarks from the database, for example, for a specific commit: - -``` -$ asv rm commit_hash=a802047be -``` - -This will remove the selected results from the files in the `results` directory. To update the results in the static site, remember to run `publish` again! - -See more options for `asv rm` in the [asv documentation](https://asv.readthedocs.io/en/stable/using.html#managing-the-results-database). - -To compare the results of running the benchmarks on two commits: -``` -$ asv compare 88fbbc33 827f322b -``` - - -## Other handy commands -To update the machine information -``` -$ asv machine -``` - -To display results from previous runs on the command line -``` -$ asv show -``` - -To use binary search to find a commit within the benchmarked range that produced a large regression -``` -$ asv find -``` -Note this will only find the global maximum if runtimes over the range are more-or-less monotonic. See the [asv docs](https://asv.readthedocs.io/en/stable/using.html#finding-a-commit-that-produces-a-large-regression) for further details. - -To check the validity of written benchmarks -``` -$ asv check -``` - -`asv` has features to run a given benchmark in the Python standard profiler `cProfile`, and then visualise the results in the tool of your choice. For example: -``` -$ asv profile time_units.time_very_simple_unit_parse 10fc29cb -``` -See the [asv docs on profiling](https://asv.readthedocs.io/en/stable/using.html#running-a-benchmark-in-the-profiler) for further details - -## Development notes: -In development, the following flags to `asv run` are often useful: -- `--bench`: to specify a subset of benchmarks (e.g., `tools.prep.PrepTF`). Regexp can be used. -- `--dry-run`: will not write results to disk -- `--quick`: will only run one repetition, and no results to disk -- `--show-stderr`: will print out stderr -- `--verbose`: provides further info on intermediate steps -- `--python=same`: runs the benchmarks in the same environment that `asv` was launched from - -E.g.: -``` -asv run --bench bench tools.prep.PrepTF --dry-run --show-stderr --quick -``` - -### Running benchmarks against a local commit -To run the benchmarks against a local commit (for example, if you are trying to improve the performance of the code), you need to edit the `repo` field in the asv config file `asv.conf.json`. - -To use the upstream repository, use: -``` -"repo": "https://github.com/brainglobe/cellfinder-core.git", -``` - -To use the local repository, use: -``` -"repo": "..", -``` - -### setup and setup_cache - -- `setup` includes initialisation bits that should not be included -in the timing of the benchmark. It can be added as: - - a method of a class, or - - an attribute of a free function, or - - a module-level setup function (run for every benchmark in the - module, prior to any function-specific setup) - - If `setup` raises `NotImplementedError`, the benchmark is skipped - -- `setup_cache` only performs the setup calculation once -(for each benchmark and each repeat) and caches the -result to disk. This may be useful if the setup is computationally -expensive. - - A separate cache is used for each environment and each commit. The cache is thrown out between benchmark runs. - - There are two options to persist the data for the benchmarks: - - `setup_cache` returns a data structure, which asv pickles to disk, - and then loads and passes as the first argument to each benchmark (not - automagically though), or - - `setup_cache` saves files to the cwd (which is a temp dir managed by - asv), which are then explicitly loaded in each benchmark. The recommended practice is to actually read the data in a `setup` function, so that loading time is not part of the benchmark timing. - - - ----- -## References -- [astropy-benchmarks repository](https://github.com/astropy/astropy-benchmarks/tree/main) -- [numpy benchmarks](https://github.com/numpy/numpy/tree/main/benchmarks/benchmarks) -- [asv documentation](https://asv.readthedocs.io/en/stable/index.html) From 333155e2ce15126d6ae3128ad8a30d16c4979533 Mon Sep 17 00:00:00 2001 From: sfmig <33267254+sfmig@users.noreply.github.com> Date: Fri, 21 Jul 2023 10:54:22 +0200 Subject: [PATCH 30/33] fixes to IO benchmarks from review discussions --- benchmarks/benchmarks/tools/IO.py | 54 ++++++++++++++----------------- 1 file changed, 25 insertions(+), 29 deletions(-) diff --git a/benchmarks/benchmarks/tools/IO.py b/benchmarks/benchmarks/tools/IO.py index 77fbe9a6..6bc56057 100644 --- a/benchmarks/benchmarks/tools/IO.py +++ b/benchmarks/benchmarks/tools/IO.py @@ -1,51 +1,54 @@ -import os from pathlib import Path from cellfinder_core.tools.IO import get_tiff_meta, read_with_dask -p = Path(os.path.dirname(__file__)).absolute() -CELLFINDER_CORE_PATH = p.parents[2] +CELLFINDER_CORE_PATH = Path(__file__).parents[3] TESTS_DATA_INTEGRATION_PATH = ( Path(CELLFINDER_CORE_PATH) / "tests" / "data" / "integration" ) -# Q for review: is there a nice way to get cellfinder-core path? class Read: + # ------------------------------------ + # Data + # ------------------------------ + detection_crop_planes_ch0 = TESTS_DATA_INTEGRATION_PATH / Path( + "detection", "crop_planes", "ch0" + ) + detection_crop_planes_ch1 = TESTS_DATA_INTEGRATION_PATH / Path( + "detection", "crop_planes", "ch1" + ) + cells_tif_files = list( + Path(TESTS_DATA_INTEGRATION_PATH, "training", "cells").glob("*.tif") + ) + non_cells_tif_files = list( + Path(TESTS_DATA_INTEGRATION_PATH, "training", "non_cells").glob( + "*.tif" + ) + ) + # --------------------------------------------- - # Setup & teardown functions + # Setup function # -------------------------------------------- def setup(self, subdir): self.data_dir = str(subdir) - def teardown(self, subdir): - del self.data_dir - # Q for review: do I need this? - # maybe only relevant if it is the parameter we sweep across? - # from https://github.com/astropy/astropy-benchmarks/blob/ - # 8758dabf84001903ea00c31a001809708969a3e4/benchmarks/cosmology.py#L24 - # (they only use teardown function in that case) - # --------------------------------------------- - # Benchmarks for reading 3d arrays with dask + # Reading 3d arrays with dask # -------------------------------------------- def time_read_with_dask(self, subdir): read_with_dask(self.data_dir) + # parameters to sweep across time_read_with_dask.param_names = [ "tests_data_integration_subdir", ] time_read_with_dask.params = ( - [ - TESTS_DATA_INTEGRATION_PATH - / Path("detection", "crop_planes", "ch0"), - TESTS_DATA_INTEGRATION_PATH - / Path("detection", "crop_planes", "ch1"), - ], + [detection_crop_planes_ch0, detection_crop_planes_ch1], ) # ----------------------------------------------- - # Benchmarks for reading metadata from tif files + # Reading metadata from tif files # ------------------------------------------------- def time_get_tiff_meta( self, @@ -53,16 +56,9 @@ def time_get_tiff_meta( ): get_tiff_meta(self.data_dir) + # parameters to sweep across time_get_tiff_meta.param_names = [ "tests_data_integration_tiffile", ] - cells_tif_files = list( - Path(TESTS_DATA_INTEGRATION_PATH, "training", "cells").glob("*.tif") - ) - non_cells_tif_files = list( - Path(TESTS_DATA_INTEGRATION_PATH, "training", "non_cells").glob( - "*.tif" - ) - ) time_get_tiff_meta.params = cells_tif_files + non_cells_tif_files From 45510da7ba6702a707b554215606612924436fce Mon Sep 17 00:00:00 2001 From: sfmig <33267254+sfmig@users.noreply.github.com> Date: Fri, 21 Jul 2023 11:38:09 +0200 Subject: [PATCH 31/33] fix typo --- benchmarks/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/README.md b/benchmarks/README.md index dde2fe03..04a355ed 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -31,7 +31,7 @@ To collate the benchmarks' results into a viewable website: ``` $ asv publish ``` -This will create a tree of files in the `html` directory, but this cannot be viewed directly from the local filesystem, so we need to put them in a static site. `asv publish` also detects satistically significant decreases of performance, the results can be inspected in the 'Regression' tab of the static site. +This will create a tree of files in the `html` directory, but this cannot be viewed directly from the local filesystem, so we need to put them in a static site. `asv publish` also detects statistically significant decreases of performance, the results can be inspected in the 'Regression' tab of the static site. To visualise the results in a static site: ``` From 397aeb3ee9d8ca564861dee312d203e13a9855a3 Mon Sep 17 00:00:00 2001 From: sfmig <33267254+sfmig@users.noreply.github.com> Date: Fri, 21 Jul 2023 11:43:01 +0200 Subject: [PATCH 32/33] Apply Will's suggestions from code review replace exclude by prune in manifest file Co-authored-by: Will Graham <32364977+willGraham01@users.noreply.github.com> --- MANIFEST.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index 7ef5b1b5..06e3e5e0 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -12,4 +12,4 @@ graft src prune benchmarks prune tests -exclude cellfinder-core/benchmarks/* +prune cellfinder-core/benchmarks/ From a5fcee180878e2b089c37042b452ec0259156b64 Mon Sep 17 00:00:00 2001 From: sfmig <33267254+sfmig@users.noreply.github.com> Date: Fri, 21 Jul 2023 11:47:35 +0200 Subject: [PATCH 33/33] change install path. remove TODOs. increase default timeout further --- benchmarks/benchmarks/tools/prep.py | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/benchmarks/benchmarks/tools/prep.py b/benchmarks/benchmarks/tools/prep.py index 97c52f4c..09e1e755 100644 --- a/benchmarks/benchmarks/tools/prep.py +++ b/benchmarks/benchmarks/tools/prep.py @@ -12,36 +12,33 @@ class PrepModels: + # parameters to sweep across param_names = ["model_name"] params = ["resnet50_tv", "resnet50_all"] # increase default timeout to allow for download - timeout = 480 + timeout = 600 - # Q for review: - # - should I run only one sample ('number'=1)? - # 'number' as defined here: - # https://asv.readthedocs.io/en/stable/writing_benchmarks.html#timing - # - how are prep_classification and prep_training different? + # install path + def benchmark_install_path(self): + # also allow to run as "user" on GH actions? + return Path(Path.home() / ".cellfinder-benchmarks") def setup(self, model_name): - self.n_free_cpus = 2 # TODO: should this be parametrised?? + self.n_free_cpus = 2 self.n_processes = get_num_processes( min_free_cpu_cores=self.n_free_cpus ) self.trained_model = None self.model_weights = None - self.install_path = Path.home() / ".cellfinder" + self.install_path = self.benchmark_install_path() self.model_name = model_name - # remove .cellfinder dir if it exists already + # remove .cellfinder-benchmarks dir if it exists shutil.rmtree(self.install_path, ignore_errors=True) - # Q for review: - # - is this safe? - # - should I check if install_path is the expected path? def teardown(self, model_name): - # remove .cellfinder dir after benchmarks + # remove .cellfinder-benchmarks dir after benchmarks shutil.rmtree(self.install_path) def time_prep_models(self, model_name): @@ -73,7 +70,7 @@ def time_prep_training(self, model_name): class PrepTF: def setup(self): - n_free_cpus = 2 # TODO: should we parametrise this? + n_free_cpus = 2 self.n_processes = get_num_processes(min_free_cpu_cores=n_free_cpus) def time_prep_tensorflow(self):