Merge branch 'main' of https://github.com/datajoint/workflow-array-ephys

.devcontainer/Dockerfile

@@ -1,15 +1,34 @@
-FROM mcr.microsoft.com/devcontainers/python:3.9-bullseye
+FROM python:3.9-slim
+
+RUN \
+    adduser --system --disabled-password --shell /bin/bash vscode && \
+    # install docker
+    apt-get update && \
+    apt-get install ca-certificates curl gnupg lsb-release -y && \
+    mkdir -m 0755 -p /etc/apt/keyrings && \
+    curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg && \
+    echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null && \
+    apt-get update && \
+    apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin -y && \
+    usermod -aG docker vscode && \
+    apt-get clean
 
-COPY ./requirements.txt /tmp/
 RUN \
     # dev setup
     apt update && \
-    apt-get install bash-completion graphviz default-mysql-client s3fs procps -y && \
-    pip install --no-cache-dir --upgrade pip && \
+    apt-get install sudo git bash-completion graphviz default-mysql-client s3fs procps -y && \
+    usermod -aG sudo vscode && \
+    echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers && \
+    pip install --no-cache-dir --upgrade black pip && \
+    echo '. /etc/bash_completion' >> /home/vscode/.bashrc && \
+    echo 'export PS1="\[\e[32;1m\]\u\[\e[m\]@\[\e[34;1m\]\H\[\e[m\]:\[\e[33;1m\]\w\[\e[m\]$ "' >> /home/vscode/.bashrc && \
+    # dircolors -b >> /home/vscode/.bashrc && \  # somehow fix colors
+    apt-get clean
+COPY ./requirements.txt /tmp/
+RUN \
     # workflow dependencies
     apt-get install gcc ffmpeg libsm6 libxext6 -y && \
     pip install --no-cache-dir -r /tmp/requirements.txt && \
-    # clean up
     rm /tmp/requirements.txt && \
     apt-get clean
 
@@ -21,4 +40,7 @@ ENV EPHYS_MODE acute
 ENV DJ_PUBLIC_S3_MOUNT_PATH /workspaces/workflow-array-ephys/example_data
 ENV RAW_ROOT_DATA_DIR /workspaces/workflow-array-ephys/example_data/raw
 ENV PROCESSED_ROOT_DATA_DIR /workspaces/workflow-array-ephys/example_data/processed
-ENV DATABASE_PREFIX neuro_
+ENV DATABASE_PREFIX neuro_
+
+USER vscode
+CMD bash -c "sudo rm /var/run/docker.pid; sudo dockerd"

.devcontainer/devcontainer.json

@@ -9,15 +9,12 @@
 		"LOCAL_WORKSPACE_FOLDER": "${localWorkspaceFolder}"
 	},
 	// https://containers.dev/features
-	"features": {
-		"ghcr.io/devcontainers/features/docker-in-docker:2": {},
-		"ghcr.io/devcontainers/features/git:1": {},
-	},
-	"onCreateCommand": "mkdir -p ${DJ_PUBLIC_S3_MOUNT_PATH} && pip install -e .",
-	"postStartCommand": "docker volume prune -f && s3fs ${DJ_PUBLIC_S3_LOCATION} ${DJ_PUBLIC_S3_MOUNT_PATH} -o nonempty,multipart_size=530,endpoint=us-east-1,url=http://s3.amazonaws.com,public_bucket=1 && MYSQL_VER=8.0 docker compose down && MYSQL_VER=8.0 docker compose up --build --wait",
+	"onCreateCommand": "mkdir -p ${DJ_PUBLIC_S3_MOUNT_PATH} && pip install -e . && MYSQL_VER=8.0 docker compose down && MYSQL_VER=8.0 docker compose up --build --wait",
+	"postStartCommand": "docker volume prune -f && s3fs ${DJ_PUBLIC_S3_LOCATION} ${DJ_PUBLIC_S3_MOUNT_PATH} -o nonempty,multipart_size=530,endpoint=us-east-1,url=http://s3.amazonaws.com,public_bucket=1",
 	"hostRequirements": {
 		"cpus": 4,
-		"memory": "8gb"
+		"memory": "8gb",
+		"storage": "32gb"
 	},
 	"forwardPorts": [
 		3306

.devcontainer/docker-compose.yaml

@@ -11,4 +11,6 @@ services:
     cap_add:
       - SYS_ADMIN
     security_opt:
-      - apparmor:unconfined
+      - apparmor:unconfined
+volumes:
+  docker_data:

.devcontainer/local/devcontainer.json

@@ -9,15 +9,12 @@
 		"LOCAL_WORKSPACE_FOLDER": "${localWorkspaceFolder}"
 	},
 	// https://containers.dev/features
-	"features": {
-		"ghcr.io/devcontainers/features/docker-in-docker:2": {},
-		"ghcr.io/devcontainers/features/git:1": {},
-	},
-	"onCreateCommand": "pip install -e .",
-	"postStartCommand": "docker volume prune -f && MYSQL_VER=8.0 docker compose down && MYSQL_VER=8.0 docker compose up --build --wait",
+	"onCreateCommand": "pip install -e . && MYSQL_VER=8.0 docker compose down && MYSQL_VER=8.0 docker compose up --build --wait",
+	"postStartCommand": "docker volume prune -f",
 	"hostRequirements": {
 		"cpus": 4,
-		"memory": "8gb"
+		"memory": "8gb",
+		"storage": "32gb"
 	},
 	"forwardPorts": [
 		3306

.devcontainer/local/docker-compose.yaml

@@ -10,5 +10,7 @@ services:
       - fakeservices.datajoint.io:127.0.0.1
     volumes:
       - ../../..:/workspaces:cached
-    entrypoint: /usr/local/share/docker-init.sh
-    command: tail -f /dev/null
+      - docker_data:/var/lib/docker  # persist docker images
+    privileged: true # only because of dind
+volumes:
+  docker_data:

CHANGELOG.md

@@ -3,9 +3,11 @@
 Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and
 [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) convention.
 
-## [0.4.0] - 2023-03-20
+## [0.3.1] - 2023-04-12
 
-+ Add - 60 minute tutorial notebook using GitHub Codespaces
++ Add - pytest for new `QCmetric` tables.
++ Add - `graphviz` in `requirements.txt`.
++ Update - visualization notebook.
 
 ## [0.3.0] - 2023-03-09
 
@@ -85,9 +87,8 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and
 
 + Add - Version
 
-[0.4.0]: https://github.com/datajoint/workflow-array-ephys/releases/tag/0.4.0
+[0.3.1]: https://github.com/datajoint/workflow-array-ephys/releases/tag/0.3.1
 [0.3.0]: https://github.com/datajoint/workflow-array-ephys/releases/tag/0.3.0
-[0.2.7]: https://github.com/datajoint/workflow-array-ephys/releases/tag/0.2.7
 [0.2.6]: https://github.com/datajoint/workflow-array-ephys/releases/tag/0.2.6
 [0.2.5]: https://github.com/datajoint/workflow-array-ephys/releases/tag/0.2.5
 [0.2.4]: https://github.com/datajoint/workflow-array-ephys/releases/tag/0.2.4

README.md

@@ -18,28 +18,34 @@ To get started, see below for an [interactive tutorial](#interactive-tutorial) o
 
 ![datajoint](https://raw.githubusercontent.com/datajoint/workflow-array-ephys/main/images/attached_array_ephys_element.svg)
 
-## Interactive tutorial
+## Interactive Tutorial
 
-The easiest way to learn about DataJoint Elements is to use the tutorial notebook within a [GitHub Codespace](https://docs.github.com/en/codespaces/overview). Please follow the steps below for the best experience:
+The easiest way to learn about DataJoint Elements is to use the tutorial notebooks within the included interactive environment configured using [DevContainer](https://containers.dev/).
 
-1. Fork this repository to your GitHub account.
+### Launch Environment
 
-2. Select the green `Code` button.
+Here are some options that provide a great experience:
 
-3. Within the dropdown menu, select the `Codespaces` tab.
+- **Cloud-based IDE**: (*recommended*)
+  - Launch using [GitHub Codespaces](https://github.com/features/codespaces) using the `+` option which will `Create codespace on main` in the codebase repository on your fork with default options. For more control, see the `...` where you may create `New with options...`.
+  - Build time for a codespace is **~7m**. This is done infrequently and cached for convenience.
+  - Start time for a codespace is **~30s**. This will pull the built codespace from cache when you need it.
+  - *Tip*: Each month, GitHub renews a [free-tier](https://docs.github.com/en/billing/managing-billing-for-github-codespaces/about-billing-for-github-codespaces#monthly-included-storage-and-core-hours-for-personal-accounts) quota of compute and storage. Typically we run into the storage limits before anything else since Codespaces consume storage while stopped. It is best to delete Codespaces when not actively in use and recreate when needed. We'll soon be creating prebuilds to avoid larger build times. Once any portion of your quota is reached, you will need to wait for it to be reset at the end of your cycle or add billing info to your GitHub account to handle overages.
+  - *Tip*: GitHub auto names the codespace but you can rename the codespace so that it is easier to identify later.
+- **Local IDE**:
+  - Ensure you have [Git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git)
+  - Ensure you have [Docker](https://docs.docker.com/get-docker/)
+  - Ensure you have [VSCode](https://code.visualstudio.com/)
+  - Install the [Dev Containers extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers)
+  - `git clone` the codebase repository and open it in VSCode
+  - Use the `Dev Containers extension` to `Reopen in Container` (More info in the `Getting started` included with the extension)
 
-4. Select the green `Create codespace on main` button.
+You will know your environment has finished loading once you either see a terminal open related to `Running postStartCommand` with a final message: `Done` or the `README.md` is opened in `Preview`.
 
-5. The environment is ready when a Visual Studio Code window is rendered within your browser.  This takes ~5 minutes the first time being launched, and ~1 minute if you revisit this Codespace.
+### Instructions
 
-6. Navigate to the `notebooks` directory on the left panel and open the `tutorial.ipynb` Jupyter notebook. Execute the cells in this notebook to begin your walk through the tutorial.
+1. We recommend you start by navigating to the `notebooks` directory on the left panel and go through the `demo_prepare.ipynb` and `demo_run.ipynb` Jupyter notebooks. Execute the cells in the notebooks to begin your walk through of the tutorial.
 
-7. Once you are done, GitHub will automatically stop the Codespace after 30 minutes of inactivity or you can manually stop the Codespace.
+1. Once you are done, see the options available to you in the menu in the bottom-left corner. For example, in Codespace you will have an option to `Stop Current Codespace` but when running DevContainer on your own machine the equivalent option is `Reopen folder locally`. By default, GitHub will also automatically stop the Codespace after 30 minutes of inactivity.
 
-8. After stopping the Codespace, we recommend deleting the Codespace to save on storage costs, which are free for the first 15 GB-month.
-
-+ If you are new to GitHub and run into any errors, please contact us via email at [email protected]. If you are experienced with GitHub, please create an issue on the upstream repository or issue a pull request with a thorough explanantion of the error and proposed solution.
-
-**Please Note:**
-
-+ GitHub Codespaces are limited to 120 core-hours per month and 15 GB-month for free users. Once you exceed this limit, you will have to wait for the hours to reset or pay to use Codespaces.
+If you are new to GitHub and run into any errors, please contact us via email at [email protected]. If you are experienced with GitHub, please create an issue on the upstream repository or if you'd like help contribute, feel free to create a pull request. Please include a thorough explanantion of the error and/or proposed solution.