From 0c8f39890e6682c6661682dc76215d569fe1561c Mon Sep 17 00:00:00 2001
From: Ben Dichter <ben.dichter@gmail.com>
Date: Mon, 1 Jul 2024 16:41:02 -0400
Subject: [PATCH] add action and scripts for displaying Dandisets with
 associated notebooks

---
 .github/scripts/metadata-collector-script.py | 108 +++++++++++++++++++
 .github/templates/index.html                 |  36 +++++++
 .github/workflows/index_workflow.yml         |  30 ++++++
 3 files changed, 174 insertions(+)
 create mode 100644 .github/scripts/metadata-collector-script.py
 create mode 100644 .github/templates/index.html
 create mode 100644 .github/workflows/index_workflow.yml

diff --git a/.github/scripts/metadata-collector-script.py b/.github/scripts/metadata-collector-script.py
new file mode 100644
index 0000000..59876ee
--- /dev/null
+++ b/.github/scripts/metadata-collector-script.py
@@ -0,0 +1,108 @@
+import os
+from typing import List, Dict, Any, Optional
+from jinja2 import Environment, FileSystemLoader
+from dandi.dandiapi import DandiAPIClient
+
+
+def get_dandiset_metadata(dandiset_id: str) -> Optional[Dict[str, Any]]:
+    """
+    Fetch metadata for a given dandiset ID.
+
+    Parameters
+    ----------
+    dandiset_id : str
+        The ID of the dandiset to fetch metadata for.
+
+    Returns
+    -------
+    Optional[Dict[str, Any]]
+        A dictionary containing the dandiset metadata if successful, None otherwise.
+    """
+    with DandiAPIClient() as client:
+        try:
+            dandiset = client.get_dandiset(dandiset_id)
+            metadata = dandiset.get_raw_metadata()
+            return metadata
+        except Exception as e:
+            print(f"Error fetching metadata for dandiset {dandiset_id}: {str(e)}")
+            return None
+
+
+def find_notebooks(folder: str) -> List[str]:
+    """
+    Recursively find all Jupyter notebooks in a given folder.
+
+    Parameters
+    ----------
+    folder : str
+        The path to the folder to search in.
+
+    Returns
+    -------
+    List[str]
+        A list of relative paths to the found notebooks.
+    """
+    notebooks = []
+    for root, _, files in os.walk(folder):
+        for file in files:
+            if file.endswith('.ipynb'):
+                rel_path = os.path.relpath(os.path.join(root, file), folder)
+                notebooks.append(rel_path)
+    return notebooks
+
+
+def collect_metadata() -> List[Dict[str, Any]]:
+    """
+    Collect metadata and notebook information for all dandisets in the current directory.
+
+    Returns
+    -------
+    List[Dict[str, Any]]
+        A list of dictionaries, each containing information about a dandiset,
+        sorted by dandiset ID.
+    """
+    dandisets = []
+    for folder in os.listdir('.'):
+        if os.path.isdir(folder) and folder.isdigit():
+            metadata = get_dandiset_metadata(folder)
+            if metadata:
+                notebooks = find_notebooks(folder)
+                dandisets.append({
+                    'id': folder,
+                    'metadata': metadata,
+                    'notebooks': notebooks
+                })
+
+    dandisets.sort(key=lambda x: x['id'])
+    return dandisets
+
+
+def render_webpage(dandisets: List[Dict[str, Any]]) -> None:
+    """
+    Render the webpage using the collected dandiset information.
+
+    Parameters
+    ----------
+    dandisets : List[Dict[str, Any]]
+        A list of dictionaries containing information about each dandiset.
+
+    Returns
+    -------
+    None
+    """
+    current_dir = os.path.dirname(os.path.abspath(__file__))
+    template_dir = os.path.join(current_dir, '..', 'templates')
+
+    env = Environment(loader=FileSystemLoader(template_dir))
+    template = env.get_template('index.html')
+
+    output = template.render(dandisets=dandisets)
+
+    output_dir = os.path.join(current_dir, '..', '..', 'output')
+    os.makedirs(output_dir, exist_ok=True)
+    with open(os.path.join(output_dir, 'index.html'), 'w') as f:
+        f.write(output)
+
+if __name__ == "__main__":
+    dandisets = collect_metadata()
+    render_webpage(dandisets)
\ No newline at end of file
diff --git a/.github/templates/index.html b/.github/templates/index.html
new file mode 100644
index 0000000..6cae1f8
--- /dev/null
+++ b/.github/templates/index.html
@@ -0,0 +1,36 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>DANDI Datasets</title>
+    <style>
+        body { font-family: Arial, sans-serif; line-height: 1.6; padding: 20px; }
+        h1 { color: #333; }
+        .dandiset { margin-bottom: 30px; border: 1px solid #ddd; padding: 15px; }
+        .dandiset h2 { margin-top: 0; }
+        .notebooks { margin-top: 10px; }
+    </style>
+</head>
+<body>
+    <h1>DANDI Datasets with Example Notebooks</h1>
+    {% for dandiset in dandisets %}
+    <div class="dandiset">
+        <h2>{{ dandiset.metadata.name }}</h2>
+        <p><strong>ID:</strong> {{ dandiset.id }}</p>
+        <p><strong>Description:</strong> {{ dandiset.metadata.description }}</p>
+        <p><a href="https://dandiarchive.org/dandiset/{{ dandiset.id }}">View on DANDI Archive</a></p>
+        {% if dandiset.notebooks %}
+        <div class="notebooks">
+            <h3>Notebooks:</h3>
+            <ul>
+            {% for notebook in dandiset.notebooks %}
+                <li><a href="https://github.com/dandi/example-notebooks/blob/master/{{ dandiset.id }}/{{ notebook }}">{{ notebook }}</a></li>
+            {% endfor %}
+            </ul>
+        </div>
+        {% endif %}
+    </div>
+    {% endfor %}
+</body>
+</html>
diff --git a/.github/workflows/index_workflow.yml b/.github/workflows/index_workflow.yml
new file mode 100644
index 0000000..5fde1e9
--- /dev/null
+++ b/.github/workflows/index_workflow.yml
@@ -0,0 +1,30 @@
+name: DANDI Metadata Collector
+
+on:
+  push:
+    branches: [ master ]
+
+jobs:
+  collect-and-render:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.9'
+    
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install requests jinja2 dandi
+    
+    - name: Run metadata collector and renderer
+      run: python .github/scripts/collect_and_render.py
+    
+    - name: Deploy to GitHub Pages
+      uses: peaceiris/actions-gh-pages@v3
+      with:
+        github_token: ${{ secrets.GITHUB_TOKEN }}
+        publish_dir: ./output