Skip to content

Commit

Permalink
add action and scripts for displaying Dandisets with associated noteb…
Browse files Browse the repository at this point in the history
…ooks
  • Loading branch information
bendichter committed Jul 1, 2024
1 parent 5dfbea3 commit 0c8f398
Show file tree
Hide file tree
Showing 3 changed files with 174 additions and 0 deletions.
108 changes: 108 additions & 0 deletions .github/scripts/metadata-collector-script.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
import os
from typing import List, Dict, Any, Optional
from jinja2 import Environment, FileSystemLoader
from dandi.dandiapi import DandiAPIClient


def get_dandiset_metadata(dandiset_id: str) -> Optional[Dict[str, Any]]:
"""
Fetch metadata for a given dandiset ID.
Parameters
----------
dandiset_id : str
The ID of the dandiset to fetch metadata for.
Returns
-------
Optional[Dict[str, Any]]
A dictionary containing the dandiset metadata if successful, None otherwise.
"""
with DandiAPIClient() as client:
try:
dandiset = client.get_dandiset(dandiset_id)
metadata = dandiset.get_raw_metadata()
return metadata
except Exception as e:
print(f"Error fetching metadata for dandiset {dandiset_id}: {str(e)}")
return None


def find_notebooks(folder: str) -> List[str]:
"""
Recursively find all Jupyter notebooks in a given folder.
Parameters
----------
folder : str
The path to the folder to search in.
Returns
-------
List[str]
A list of relative paths to the found notebooks.
"""
notebooks = []
for root, _, files in os.walk(folder):
for file in files:
if file.endswith('.ipynb'):
rel_path = os.path.relpath(os.path.join(root, file), folder)
notebooks.append(rel_path)
return notebooks


def collect_metadata() -> List[Dict[str, Any]]:
"""
Collect metadata and notebook information for all dandisets in the current directory.
Returns
-------
List[Dict[str, Any]]
A list of dictionaries, each containing information about a dandiset,
sorted by dandiset ID.
"""
dandisets = []
for folder in os.listdir('.'):
if os.path.isdir(folder) and folder.isdigit():
metadata = get_dandiset_metadata(folder)
if metadata:
notebooks = find_notebooks(folder)
dandisets.append({
'id': folder,
'metadata': metadata,
'notebooks': notebooks
})

dandisets.sort(key=lambda x: x['id'])
return dandisets


def render_webpage(dandisets: List[Dict[str, Any]]) -> None:
"""
Render the webpage using the collected dandiset information.
Parameters
----------
dandisets : List[Dict[str, Any]]
A list of dictionaries containing information about each dandiset.
Returns
-------
None
"""
current_dir = os.path.dirname(os.path.abspath(__file__))
template_dir = os.path.join(current_dir, '..', 'templates')

env = Environment(loader=FileSystemLoader(template_dir))
template = env.get_template('index.html')

output = template.render(dandisets=dandisets)

output_dir = os.path.join(current_dir, '..', '..', 'output')
os.makedirs(output_dir, exist_ok=True)
with open(os.path.join(output_dir, 'index.html'), 'w') as f:
f.write(output)

if __name__ == "__main__":
dandisets = collect_metadata()
render_webpage(dandisets)
36 changes: 36 additions & 0 deletions .github/templates/index.html
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>DANDI Datasets</title>
<style>
body { font-family: Arial, sans-serif; line-height: 1.6; padding: 20px; }
h1 { color: #333; }
.dandiset { margin-bottom: 30px; border: 1px solid #ddd; padding: 15px; }
.dandiset h2 { margin-top: 0; }
.notebooks { margin-top: 10px; }
</style>
</head>
<body>
<h1>DANDI Datasets with Example Notebooks</h1>
{% for dandiset in dandisets %}
<div class="dandiset">
<h2>{{ dandiset.metadata.name }}</h2>
<p><strong>ID:</strong> {{ dandiset.id }}</p>
<p><strong>Description:</strong> {{ dandiset.metadata.description }}</p>
<p><a href="https://dandiarchive.org/dandiset/{{ dandiset.id }}">View on DANDI Archive</a></p>
{% if dandiset.notebooks %}
<div class="notebooks">
<h3>Notebooks:</h3>
<ul>
{% for notebook in dandiset.notebooks %}
<li><a href="https://github.com/dandi/example-notebooks/blob/master/{{ dandiset.id }}/{{ notebook }}">{{ notebook }}</a></li>
{% endfor %}
</ul>
</div>
{% endif %}
</div>
{% endfor %}
</body>
</html>
30 changes: 30 additions & 0 deletions .github/workflows/index_workflow.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
name: DANDI Metadata Collector

on:
push:
branches: [ master ]

jobs:
collect-and-render:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2

- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: '3.9'

- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install requests jinja2 dandi
- name: Run metadata collector and renderer
run: python .github/scripts/collect_and_render.py

- name: Deploy to GitHub Pages
uses: peaceiris/actions-gh-pages@v3
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
publish_dir: ./output

0 comments on commit 0c8f398

Please sign in to comment.