Merge pull request #1 from spapa013/main

v0.0.1 - initialize microns-proximities

.gitignore

@@ -0,0 +1,147 @@
+# Personal deploy folders
+personal-deploy/
+personal_deploy/
+microns-personal-deploy/
+microns_personal_deploy/
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# Linter cfg
+.flake8
+.hadolint.yaml
+.isort.cfg
+.markdownlint.yaml
+.shellcheckrc
+.trunk/
+.pylintrc
+
+# vscode cfg
+.vscode/

Dockerfile

@@ -0,0 +1,18 @@
+FROM at-docker.ad.bcm.edu:5000/microns-base 
+
+LABEL mantainer="Stelios Papadopoulos <[email protected]>"
+
+# Install additional packages
+
+# Update and rebuild jupyterlab 
+RUN python -m pip install --no-cache-dir --upgrade jupyterlab
+
+WORKDIR /root
+ARG CLOUDVOLUME_TOKEN
+RUN mkdir -p .cloudvolume/secrets
+RUN echo "{\"token\": \"${CLOUDVOLUME_TOKEN:-}\"}" > .cloudvolume/secrets/cave-secret.json
+
+# CURRENT PACKAGE
+COPY . /src/microns-proximities
+RUN python -m pip install --no-cache-dir /src/microns-proximities/python/microns-proximities
+RUN python -m pip install --no-cache-dir /src/microns-proximities/python/microns-proximities-api

deploy/docker-compose.yml

@@ -0,0 +1,26 @@
+version: '3.4'
+x-build: &build
+  context: ../
+  dockerfile: Dockerfile
+
+x-shared: &common
+  ipc: host
+  build:
+    context: ../
+    dockerfile: Dockerfile
+  volumes:
+    - ../:/src/microns-proximities
+    - /mnt:/mnt
+  env_file:
+    - .env
+  container_name: "microns-proximities"
+
+services:
+  notebook:
+    <<: *common
+    ports:
+        - "${JUPYTER_HOST:-0.0.0.0}:${JUPYTER_PORT_CONTAINER:-8888}:8888"
+    entrypoint: /src/microns-proximities/deploy/notebook.sh
+  bin:
+    <<: *common
+    entrypoint: /bin/bash

deploy/notebook.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+cd /notebooks
+jupyter lab --ip=0.0.0.0 --allow-root --NotebookApp.token=${JUPYTER_PASSWORD:-} --no-browser

python/microns-proximities-api/microns_proximities_api/__init__.py

@@ -0,0 +1,13 @@
+from microns_utils import version_utils
+
+__version__ = version_utils.check_package_version(
+    package='microns-proximities-api', 
+    check_if_latest=True, 
+    check_if_latest_kwargs=dict(
+        owner='cajal', 
+        repo='microns-proximities', 
+        source='tag', 
+    )
+)
+
+check_latest_version_from_github = version_utils.latest_github_version_checker(owner='cajal', repo='microns-proximities')

python/microns-proximities-api/microns_proximities_api/config/__init__.py

@@ -0,0 +1,17 @@
+"""
+Configuration package/module for microns-proximities.
+"""
+import datajoint_plus as djp
+from microns_utils.config_utils import SchemaConfig
+
+from . import adapters, externals
+
+djp.enable_datajoint_flags()
+
+minnie_proximities_config = SchemaConfig(
+    module_name="minnie_proximities",
+    schema_name="microns_minnie_proximities",
+    externals=externals.minnie_proximities,
+    adapters=adapters.minnie_proximities,
+)
+

python/microns-proximities-api/microns_proximities_api/config/adapters.py

@@ -0,0 +1,2 @@
+minnie_proximities = {}
+

python/microns-proximities-api/microns_proximities_api/config/externals.py

@@ -0,0 +1,10 @@
+"""
+Externals for DataJoint tables.
+"""
+
+from pathlib import Path
+import datajoint_plus as djp
+
+base_path = Path() / "/mnt" / "dj-stor01" / "microns" / "minnie" / "proximities"
+
+minnie_proximities = {}

python/microns-proximities-api/microns_proximities_api/schemas/__init__.py

@@ -0,0 +1 @@
+

python/microns-proximities-api/microns_proximities_api/schemas/minnie_proximities.py

@@ -0,0 +1,12 @@
+"""
+DataJoint tables for proximities.
+"""
+import datajoint as dj
+import datajoint_plus as djp
+
+from ..config import minnie_proximities_config as config
+
+config.register_externals()
+config.register_adapters(context=locals())
+
+schema = djp.schema(config.schema_name, create_schema=True)

python/microns-proximities-api/microns_proximities_api/utils/numpy_utils.py

@@ -0,0 +1,47 @@
+import numpy as np
+
+
+def intersect2d(ar1, ar2, assume_unique=False, return_indices=False):
+    """
+    Use numpy intersect1d with 2-dimensional arrays
+    
+    https://stackoverflow.com/a/8317403
+    """
+    def get_dtype_for_view(arr):
+        _, ncols = arr.shape
+        return {
+            'names':['f{}'.format(i) for i in range(ncols)],
+            'formats': ncols * [arr.dtype]
+        }
+
+    arr1 = np.array(ar1)
+    arr2 = np.array(ar2)
+
+    return np.intersect1d(
+        arr1.view(get_dtype_for_view(arr1)), 
+        arr2.view(get_dtype_for_view(arr2)), 
+        assume_unique=assume_unique, 
+        return_indices=return_indices
+    )
+
+
+def unique_row_view(data, return_inverse=False):
+    """
+    Faster implementation of np.unique with axis=0 argument.
+
+    Credit to nschloe:
+    https://github.com/numpy/numpy/issues/11136
+    """
+    if data.shape[0] == 0:
+        return data
+
+    b = np.ascontiguousarray(data).view(
+        np.dtype((np.void, data.dtype.itemsize * data.shape[1]))
+        )
+
+    u = np.unique(b, return_inverse=return_inverse)
+    if return_inverse:
+        u, inverse_idx = u
+        return u.view(data.dtype).reshape(-1, data.shape[1]), inverse_idx
+    else:
+        return u.view(data.dtype).reshape(-1, data.shape[1])

python/microns-proximities-api/microns_proximities_api/utils/skeleton_utils.py

@@ -0,0 +1,42 @@
+import numpy as np
+from . import numpy_utils as npu
+
+
+def discretize_skeleton(full_edges, maximum_length, return_as_int=True, reshape=True):
+    """
+    from Christos
+    """
+    if full_edges.shape[0] == 0:
+        return full_edges
+
+    p0s = full_edges[:, 0]
+    p1s = full_edges[:, 1]
+
+    diffs = p1s - p0s
+    distances = np.linalg.norm(diffs, axis=1)
+    inc_nums = np.ceil(distances / maximum_length).astype(int)
+    inc_nums[inc_nums<2] = 2
+    diffs_inc = np.repeat(diffs / inc_nums[:, None], inc_nums, axis=0)
+
+    p0s_stack = np.repeat(p0s, inc_nums, axis=0)
+    max_arange = np.arange(inc_nums.max())
+    multiplicative_incrementer = np.hstack([max_arange[0:i] for i in inc_nums.tolist()])
+    evenly_spaced = p0s_stack + (multiplicative_incrementer[:, None] * diffs_inc)
+
+    total = 0
+    incremented_edges = list()
+    for increment, p1 in zip(inc_nums, p1s):
+        temp_total = total + increment
+        inc_edge = evenly_spaced[total:temp_total]
+        inc_edge = np.concatenate((inc_edge, p1[None]))
+        incremented_edges.append(inc_edge)
+        total = temp_total
+    new_full_edges = np.vstack([np.array((inc_edge[:-1], inc_edge[1:])).transpose(1, 0, 2) for inc_edge in incremented_edges])
+
+    if return_as_int:
+        new_full_edges = np.round(new_full_edges).astype(int)
+
+    if not reshape:
+        return new_full_edges
+    else:
+        return npu.unique_row_view(new_full_edges.reshape(-1, 3))

python/microns-proximities-api/requirements.txt

@@ -0,0 +1,3 @@
+datajoint-plus
+microns-utils
+torch>1.0

python/microns-proximities-api/setup.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env python
+from setuptools import setup, find_packages
+from os import path
+
+here = path.abspath(path.dirname(__file__))
+
+with open(path.join(here, '..', 'version.py')) as f:
+    exec(f.read())
+
+with open(path.join(here, 'requirements.txt')) as f:
+    requirements = f.read().split()
+
+setup(
+    name="microns-proximities-api",
+    version=__version__,
+    description="EM/Functional proximities API for MICrONS",
+    author="Stelios Papadopoulos, Christos Papadopoulos",
+    packages=find_packages(),
+    install_requires=requirements
+)

python/microns-proximities/microns_proximities/__init__.py

@@ -0,0 +1,14 @@
+from microns_utils import version_utils
+
+__version__ = version_utils.check_package_version(
+    package='microns-proximities', 
+    prefix='microns-proximities/python',
+    check_if_latest=True, 
+    check_if_latest_kwargs=dict(
+        owner='cajal', 
+        repo='microns-proximities', 
+        source='tag', 
+    )
+)
+
+check_latest_version_from_github = version_utils.latest_github_version_checker(owner='cajal', repo='microns-proximities')