Update to permit multiple root dirs

.gitignore

@@ -107,8 +107,7 @@ ENV/
 .mypy_cache/
 
 # datajoint
-dj_local_conf.json
-dj_local_conf_old.json
+dj_local_con*.json
 
 # emacs
 **/*~
@@ -122,3 +121,6 @@ Diagram.ipynb
 
 # vscode
 .vscode/settings.json
+
+# notes
+temp*

Dockerfile.dev

@@ -26,7 +26,7 @@ RUN pip install -e /main/element-session
 RUN pip install -e /main/element-array-ephys
 RUN pip install -e /main/workflow-array-ephys
 RUN pip install -r /main/workflow-array-ephys/requirements_test.txt
-     
+
 WORKDIR /main/workflow-array-ephys
 
-ENTRYPOINT ["tail", "-f", "/dev/null"]
+ENTRYPOINT ["tail", "-f", "/dev/null"]

Dockerfile.test

@@ -8,20 +8,31 @@ USER anaconda
 WORKDIR /main/workflow-array-ephys
 
 # Option 1 - Install DataJoint's remote fork of the workflow and elements
-RUN git clone https://github.com/datajoint/workflow-array-ephys.git /main/workflow-array-ephys
+# RUN git clone https://github.com/datajoint/workflow-array-ephys.git /main/
 
-# Option 2 - Install user's remote fork of element and workflow 
+# Option 2 - Install user's remote fork of element and workflow
 #            or an unreleased version of the element
+# RUN pip install git+https://github.com/<user>element-lab.git
+# RUN pip install git+https://github.com/<user>/element-animal.git
+# RUN pip install git+https://github.com/<user>/element-session.git
 # RUN pip install git+https://github.com/<user>/element-array-ephys.git
-# RUN git clone https://github.com/<user>/workflow-array-ephys.git /main/workflow-array-ephys
 
 # Option 3 - Install user's local fork of element and workflow
-# RUN mkdir /main/element-array-ephys
-# COPY --chown=anaconda:anaconda ./element-array-ephys /main/element-array-ephys
-# RUN pip install /main/element-array-ephys
-# COPY --chown=anaconda:anaconda ./workflow-array-ephys /main/workflow-array-ephys
+RUN mkdir /main/element-lab
+COPY --chown=anaconda:anaconda ./element-lab /main/element-lab
+RUN pip install -e /main/element-lab
+RUN mkdir /main/element-animal
+COPY --chown=anaconda:anaconda ./element-animal /main/element-animal
+RUN pip install -e /main/element-animal
+RUN mkdir /main/element-session
+COPY --chown=anaconda:anaconda ./element-session /main/element-session
+RUN pip install -e /main/element-session
+RUN mkdir /main/element-array-ephys
+COPY --chown=anaconda:anaconda ./element-array-ephys /main/element-array-ephys
+RUN pip install -e /main/element-array-ephys
+COPY --chown=anaconda:anaconda ./workflow-array-ephys /main/workflow-array-ephys
 # RUN rm -f /main/workflow-array-ephys/dj_local_conf.json
 
 # Install the workflow
 RUN pip install /main/workflow-array-ephys
-RUN pip install -r /main/workflow-array-ephys/requirements_test.txt
+RUN pip install -r /main/workflow-array-ephys/requirements_test.txt

README.md

@@ -1,7 +1,7 @@
 # DataJoint Workflow - Array Electrophysiology
 
-Workflow for extracellular array electrophysiology data acquired with a polytrode probe (e.g. 
-Neuropixels, Neuralynx) using the `SpikeGLX` or `OpenEphys` acquisition software and processed 
+Workflow for extracellular array electrophysiology data acquired with a polytrode probe (e.g.
+Neuropixels, Neuralynx) using the `SpikeGLX` or `OpenEphys` acquisition software and processed
 with MATLAB- or python-based `Kilosort` spike sorting software.
 
 A complete electrophysiology workflow can be built using the DataJoint Elements.
@@ -11,17 +11,17 @@ A complete electrophysiology workflow can be built using the DataJoint Elements.
 + [element-array-ephys](https://github.com/datajoint/element-array-ephys)
 
 This repository provides demonstrations for:
-1. Set up a workflow using DataJoint Elements (see 
+1. Set up a workflow using DataJoint Elements (see
 [workflow_array_ephys/pipeline.py](workflow_array_ephys/pipeline.py))
-2. Ingestion of data/metadata based on a predefined file structure, file naming 
-convention, and directory lookup methods (see 
+2. Ingestion of data/metadata based on a predefined file structure, file naming
+convention, and directory lookup methods (see
 [workflow_array_ephys/paths.py](workflow_array_ephys/paths.py)).
 3. Ingestion of clustering results.
 
 ## Workflow architecture
 
 The electrophysiology workflow presented here uses components from 4 DataJoint 
-Elements (`element-lab`, `element-animal`, `element-session`, 
+Elements (`element-lab`, `element-animal`, `element-session`,
 `element-array-ephys`) assembled together to form a fully functional workflow.
 
 ### element-lab
@@ -46,6 +46,6 @@ https://github.com/datajoint/element-animal/blob/main/images/subject_diagram.svg
 ## Interacting with the DataJoint workflow
 
 + Please refer to the following workflow-specific 
-[Jupyter notebooks](/notebooks) for an in-depth explanation of how to run the 
-workflow ([03-process.ipynb](notebooks/03-process.ipynb)) and explore the data 
-([05-explore.ipynb](notebooks/05-explore.ipynb)).
+ [Jupyter notebooks](/notebooks) for an in-depth explanation of how to run the 
+ workflow ([03-process.ipynb](notebooks/03-process.ipynb)) and explore the data 
+ ([05-explore.ipynb](notebooks/05-explore.ipynb)).

docker-compose-dev.yaml

@@ -30,4 +30,4 @@ services:
       db:
         condition: service_healthy
 networks:
-  main:
+  main:

docker-compose-test.yaml

@@ -1,4 +1,7 @@
+# export COMPOSE_DOCKER_CLI_BUILD=0 # some machines need for smooth --build
+# .env file: TEST_DATA_DIR=<local parent directory of workflow_ephys_data{1,2}>
 # docker-compose -f docker-compose-test.yaml up --build
+# docker exec -it workflow-session_workflow_1 /bin/bash
 # docker-compose -f docker-compose-test.yaml down
 
 version: "2.4"
@@ -22,20 +25,22 @@ services:
       - DJ_HOST=db
       - DJ_USER=root
       - DJ_PASS=simple
-      - EPHYS_ROOT_DATA_DIR=/main/test_data
+      - EPHYS_ROOT_DATA_DIR=/main/test_data/workflow_ephys_data1/,/main/test_data/workflow_ephys_data2/
       - DATABASE_PREFIX=test_
     command:
       - bash
       - -c
       - |
         echo "------ INTEGRATION TESTS ------"
-        pytest -sv --cov-report term-missing --cov=workflow-array-ephys -p no:warnings
+        pytest -sv --cov-report term-missing --cov=workflow_array_ephys -p no:warnings tests/
         tail -f /dev/null
     volumes:
       - ${TEST_DATA_DIR}:/main/test_data
       - ./apt_requirements.txt:/tmp/apt_requirements.txt
+      - ../element-array-ephys:/main/element-array-ephys
+      - .:/main/workflow-array-ephys
     depends_on:
       db:
         condition: service_healthy
 networks:
-  main:
+  main:

requirements.txt

@@ -1,7 +1,7 @@
 datajoint>=0.13.0
 element-array-ephys==0.1.0b0
-element-lab==0.1.0b0
+element-lab>=0.1.0b0
 element-animal==0.1.0b0
 element-session==0.1.0b0
 element-interface @ git+https://github.com/datajoint/element-interface.git
-ipykernel==6.0.1
+ipykernel==6.0.1