diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d7ee351 --- /dev/null +++ b/.gitignore @@ -0,0 +1,147 @@ +# Personal deploy folders +personal-deploy/ +personal_deploy/ +microns-personal-deploy/ +microns_personal_deploy/ + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# Linter cfg +.flake8 +.hadolint.yaml +.isort.cfg +.markdownlint.yaml +.shellcheckrc +.trunk/ +.pylintrc + +# vscode cfg +.vscode/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..4880140 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,18 @@ +FROM at-docker.ad.bcm.edu:5000/microns-base + +LABEL mantainer="Stelios Papadopoulos " + +# Install additional packages + +# Update and rebuild jupyterlab +RUN python -m pip install --no-cache-dir --upgrade jupyterlab + +WORKDIR /root +ARG CLOUDVOLUME_TOKEN +RUN mkdir -p .cloudvolume/secrets +RUN echo "{\"token\": \"${CLOUDVOLUME_TOKEN:-}\"}" > .cloudvolume/secrets/cave-secret.json + +# CURRENT PACKAGE +COPY . /src/microns-proximities +RUN python -m pip install --no-cache-dir /src/microns-proximities/python/microns-proximities +RUN python -m pip install --no-cache-dir /src/microns-proximities/python/microns-proximities-api diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml new file mode 100644 index 0000000..330e15c --- /dev/null +++ b/deploy/docker-compose.yml @@ -0,0 +1,26 @@ +version: '3.4' +x-build: &build + context: ../ + dockerfile: Dockerfile + +x-shared: &common + ipc: host + build: + context: ../ + dockerfile: Dockerfile + volumes: + - ../:/src/microns-proximities + - /mnt:/mnt + env_file: + - .env + container_name: "microns-proximities" + +services: + notebook: + <<: *common + ports: + - "${JUPYTER_HOST:-0.0.0.0}:${JUPYTER_PORT_CONTAINER:-8888}:8888" + entrypoint: /src/microns-proximities/deploy/notebook.sh + bin: + <<: *common + entrypoint: /bin/bash \ No newline at end of file diff --git a/deploy/notebook.sh b/deploy/notebook.sh new file mode 100755 index 0000000..0e5dcac --- /dev/null +++ b/deploy/notebook.sh @@ -0,0 +1,3 @@ +#!/bin/bash +cd /notebooks +jupyter lab --ip=0.0.0.0 --allow-root --NotebookApp.token=${JUPYTER_PASSWORD:-} --no-browser \ No newline at end of file diff --git a/python/microns-proximities-api/microns_proximities_api/__init__.py b/python/microns-proximities-api/microns_proximities_api/__init__.py new file mode 100644 index 0000000..63b89c0 --- /dev/null +++ b/python/microns-proximities-api/microns_proximities_api/__init__.py @@ -0,0 +1,13 @@ +from microns_utils import version_utils + +__version__ = version_utils.check_package_version( + package='microns-proximities-api', + check_if_latest=True, + check_if_latest_kwargs=dict( + owner='cajal', + repo='microns-proximities', + source='tag', + ) +) + +check_latest_version_from_github = version_utils.latest_github_version_checker(owner='cajal', repo='microns-proximities') diff --git a/python/microns-proximities-api/microns_proximities_api/config/__init__.py b/python/microns-proximities-api/microns_proximities_api/config/__init__.py new file mode 100644 index 0000000..e978623 --- /dev/null +++ b/python/microns-proximities-api/microns_proximities_api/config/__init__.py @@ -0,0 +1,17 @@ +""" +Configuration package/module for microns-proximities. +""" +import datajoint_plus as djp +from microns_utils.config_utils import SchemaConfig + +from . import adapters, externals + +djp.enable_datajoint_flags() + +minnie_proximities_config = SchemaConfig( + module_name="minnie_proximities", + schema_name="microns_minnie_proximities", + externals=externals.minnie_proximities, + adapters=adapters.minnie_proximities, +) + diff --git a/python/microns-proximities-api/microns_proximities_api/config/adapters.py b/python/microns-proximities-api/microns_proximities_api/config/adapters.py new file mode 100644 index 0000000..eaeccc0 --- /dev/null +++ b/python/microns-proximities-api/microns_proximities_api/config/adapters.py @@ -0,0 +1,2 @@ +minnie_proximities = {} + diff --git a/python/microns-proximities-api/microns_proximities_api/config/externals.py b/python/microns-proximities-api/microns_proximities_api/config/externals.py new file mode 100644 index 0000000..05b1b27 --- /dev/null +++ b/python/microns-proximities-api/microns_proximities_api/config/externals.py @@ -0,0 +1,10 @@ +""" +Externals for DataJoint tables. +""" + +from pathlib import Path +import datajoint_plus as djp + +base_path = Path() / "/mnt" / "dj-stor01" / "microns" / "minnie" / "proximities" + +minnie_proximities = {} diff --git a/python/microns-proximities-api/microns_proximities_api/methods/__init__.py b/python/microns-proximities-api/microns_proximities_api/methods/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/python/microns-proximities-api/microns_proximities_api/schemas/__init__.py b/python/microns-proximities-api/microns_proximities_api/schemas/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/python/microns-proximities-api/microns_proximities_api/schemas/__init__.py @@ -0,0 +1 @@ + diff --git a/python/microns-proximities-api/microns_proximities_api/schemas/minnie_proximities.py b/python/microns-proximities-api/microns_proximities_api/schemas/minnie_proximities.py new file mode 100644 index 0000000..904776e --- /dev/null +++ b/python/microns-proximities-api/microns_proximities_api/schemas/minnie_proximities.py @@ -0,0 +1,12 @@ +""" +DataJoint tables for proximities. +""" +import datajoint as dj +import datajoint_plus as djp + +from ..config import minnie_proximities_config as config + +config.register_externals() +config.register_adapters(context=locals()) + +schema = djp.schema(config.schema_name, create_schema=True) \ No newline at end of file diff --git a/python/microns-proximities-api/microns_proximities_api/utils/__init__.py b/python/microns-proximities-api/microns_proximities_api/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/python/microns-proximities-api/microns_proximities_api/utils/numpy_utils.py b/python/microns-proximities-api/microns_proximities_api/utils/numpy_utils.py new file mode 100644 index 0000000..f20386f --- /dev/null +++ b/python/microns-proximities-api/microns_proximities_api/utils/numpy_utils.py @@ -0,0 +1,47 @@ +import numpy as np + + +def intersect2d(ar1, ar2, assume_unique=False, return_indices=False): + """ + Use numpy intersect1d with 2-dimensional arrays + + https://stackoverflow.com/a/8317403 + """ + def get_dtype_for_view(arr): + _, ncols = arr.shape + return { + 'names':['f{}'.format(i) for i in range(ncols)], + 'formats': ncols * [arr.dtype] + } + + arr1 = np.array(ar1) + arr2 = np.array(ar2) + + return np.intersect1d( + arr1.view(get_dtype_for_view(arr1)), + arr2.view(get_dtype_for_view(arr2)), + assume_unique=assume_unique, + return_indices=return_indices + ) + + +def unique_row_view(data, return_inverse=False): + """ + Faster implementation of np.unique with axis=0 argument. + + Credit to nschloe: + https://github.com/numpy/numpy/issues/11136 + """ + if data.shape[0] == 0: + return data + + b = np.ascontiguousarray(data).view( + np.dtype((np.void, data.dtype.itemsize * data.shape[1])) + ) + + u = np.unique(b, return_inverse=return_inverse) + if return_inverse: + u, inverse_idx = u + return u.view(data.dtype).reshape(-1, data.shape[1]), inverse_idx + else: + return u.view(data.dtype).reshape(-1, data.shape[1]) \ No newline at end of file diff --git a/python/microns-proximities-api/microns_proximities_api/utils/skeleton_utils.py b/python/microns-proximities-api/microns_proximities_api/utils/skeleton_utils.py new file mode 100644 index 0000000..ef31a23 --- /dev/null +++ b/python/microns-proximities-api/microns_proximities_api/utils/skeleton_utils.py @@ -0,0 +1,42 @@ +import numpy as np +from . import numpy_utils as npu + + +def discretize_skeleton(full_edges, maximum_length, return_as_int=True, reshape=True): + """ + from Christos + """ + if full_edges.shape[0] == 0: + return full_edges + + p0s = full_edges[:, 0] + p1s = full_edges[:, 1] + + diffs = p1s - p0s + distances = np.linalg.norm(diffs, axis=1) + inc_nums = np.ceil(distances / maximum_length).astype(int) + inc_nums[inc_nums<2] = 2 + diffs_inc = np.repeat(diffs / inc_nums[:, None], inc_nums, axis=0) + + p0s_stack = np.repeat(p0s, inc_nums, axis=0) + max_arange = np.arange(inc_nums.max()) + multiplicative_incrementer = np.hstack([max_arange[0:i] for i in inc_nums.tolist()]) + evenly_spaced = p0s_stack + (multiplicative_incrementer[:, None] * diffs_inc) + + total = 0 + incremented_edges = list() + for increment, p1 in zip(inc_nums, p1s): + temp_total = total + increment + inc_edge = evenly_spaced[total:temp_total] + inc_edge = np.concatenate((inc_edge, p1[None])) + incremented_edges.append(inc_edge) + total = temp_total + new_full_edges = np.vstack([np.array((inc_edge[:-1], inc_edge[1:])).transpose(1, 0, 2) for inc_edge in incremented_edges]) + + if return_as_int: + new_full_edges = np.round(new_full_edges).astype(int) + + if not reshape: + return new_full_edges + else: + return npu.unique_row_view(new_full_edges.reshape(-1, 3)) \ No newline at end of file diff --git a/python/microns-proximities-api/requirements.txt b/python/microns-proximities-api/requirements.txt new file mode 100644 index 0000000..196e73b --- /dev/null +++ b/python/microns-proximities-api/requirements.txt @@ -0,0 +1,3 @@ +datajoint-plus +microns-utils +torch>1.0 \ No newline at end of file diff --git a/python/microns-proximities-api/setup.py b/python/microns-proximities-api/setup.py new file mode 100644 index 0000000..f4bee02 --- /dev/null +++ b/python/microns-proximities-api/setup.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python +from setuptools import setup, find_packages +from os import path + +here = path.abspath(path.dirname(__file__)) + +with open(path.join(here, '..', 'version.py')) as f: + exec(f.read()) + +with open(path.join(here, 'requirements.txt')) as f: + requirements = f.read().split() + +setup( + name="microns-proximities-api", + version=__version__, + description="EM/Functional proximities API for MICrONS", + author="Stelios Papadopoulos, Christos Papadopoulos", + packages=find_packages(), + install_requires=requirements +) \ No newline at end of file diff --git a/python/microns-proximities/microns_proximities/__init__.py b/python/microns-proximities/microns_proximities/__init__.py new file mode 100644 index 0000000..9b3aff7 --- /dev/null +++ b/python/microns-proximities/microns_proximities/__init__.py @@ -0,0 +1,14 @@ +from microns_utils import version_utils + +__version__ = version_utils.check_package_version( + package='microns-proximities', + prefix='microns-proximities/python', + check_if_latest=True, + check_if_latest_kwargs=dict( + owner='cajal', + repo='microns-proximities', + source='tag', + ) +) + +check_latest_version_from_github = version_utils.latest_github_version_checker(owner='cajal', repo='microns-proximities') diff --git a/python/microns-proximities/requirements.txt b/python/microns-proximities/requirements.txt new file mode 100644 index 0000000..e69de29 diff --git a/python/microns-proximities/setup.py b/python/microns-proximities/setup.py new file mode 100644 index 0000000..9e69583 --- /dev/null +++ b/python/microns-proximities/setup.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python +from setuptools import setup, find_packages +from os import path + +def find_api(name): + return f"{name} @ file://localhost/{here}/../{name}#egg={name}" + +here = path.abspath(path.dirname(__file__)) + +with open(path.join(here, '..', 'version.py')) as f: + exec(f.read()) + +with open(path.join(here, 'requirements.txt')) as f: + requirements = f.read().split() + +requirements += [find_api('microns-proximities-api')] + +setup( + name='microns-proximities', + version=__version__, + description='EM/Functional proximities for MICrONS', + author='Andreas Tolias Lab', + author_email='astolias@bcm.edu', + packages=find_packages(exclude=[]), + install_requires=requirements +) \ No newline at end of file diff --git a/python/version.py b/python/version.py new file mode 100644 index 0000000..b8023d8 --- /dev/null +++ b/python/version.py @@ -0,0 +1 @@ +__version__ = '0.0.1'