Offload API to static files + Fix search bug (#17)

Co-authored-by: Ryan Ly <[email protected]>
Co-authored-by: github-actions[bot] <runner@fv-az618-841.ogsaqtoyx52uhcfy3b1k3wjvzc.jx.internal.cloudapp.net>
Co-authored-by: github-actions[bot] <runner@fv-az1230-275.pnbcmxnyn2burpnk1ig1bplgja.jx.internal.cloudapp.net>
Co-authored-by: github-actions[bot] <runner@fv-az167-507.10vc1fp43w4ejfym3u1nh0imhc.jx.internal.cloudapp.net>
Co-authored-by: github-actions[bot] <runner@fv-az247-115.pyoqamc1z0nuzhu5fvcxfugxug.jx.internal.cloudapp.net>
Co-authored-by: github-actions[bot] <runner@fv-az1442-795.gtdgxysclnuenlnkab13ghtgjb.cx.internal.cloudapp.net>

.github/workflows/data.yml

@@ -0,0 +1,52 @@
+name: Download records & Push to repository
+
+on:
+  push:
+    branches: ["main"]
+  schedule:
+    # Run at 8:00 on working days
+    - cron: 0 8 * * 0-5
+  workflow_dispatch:
+
+
+jobs:
+  download:
+    runs-on: ubuntu-latest
+    if: ${{ !startsWith(github.event.head_commit.message, '[bot]') }}
+    environment:
+      name: download-data
+    env:
+      CI_COMMIT_USER: 'github-actions[bot]'
+      CI_COMMIT_EMAIL: 'github-actions[bot]@users.noreply.github.com'
+      CI_COMMIT_MESSAGE: '[bot] update records'
+      CI_PUSH_REMOTE: [email protected]:$GITHUB_REPOSITORY.git
+      CI_PUSH_BRANCH: 'main'
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        token: ${{ secrets.NWB_EXTENSIONS_BOT_GITHUB_TOKEN }}
+        persist-credentials: true
+    - uses: actions/setup-python@v5
+      with:
+          python-version: '3.12'
+          cache: 'pip'
+    - run: |
+        pip install -r utils/requirements.txt
+    - name: Download records data
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # in future, consider replace with PAT
+      run: |
+        python utils/collect-records.py
+    - name: Commit & Push to repository
+      run: |
+        git config user.name "${{ env.CI_COMMIT_USER }}"
+        git config user.mail "${{ env.CI_COMMIT_EMAIL }}"
+
+        if [[ `git diff data` ]] ; then
+          git add data
+          git commit -m "${{ env.CI_COMMIT_MESSAGE }}"
+          git push "${{ env.CI_PUSH_REMOTE }}" "HEAD:${{ env.CI_PUSH_BRANCH }}"
+        else
+          echo "No changes were found in data"
+          exit 0
+        fi

.github/workflows/jekyll.yml

@@ -0,0 +1,64 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+# Sample workflow for building and deploying a Jekyll site to GitHub Pages
+name: Deploy Jekyll site to Pages
+
+on:
+  # Runs on pushes targeting the default branch
+  push:
+    branches: ["main"]
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
+# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
+concurrency:
+  group: "pages"
+  cancel-in-progress: false
+
+jobs:
+  # Build job
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Setup Ruby
+        uses: ruby/setup-ruby@55283cc23133118229fd3f97f9336ee23a179fcf # v1.146.0
+        with:
+          ruby-version: '3.1' # Not needed with a .ruby-version file
+          bundler-cache: true # runs 'bundle install' and caches installed gems automatically
+          cache-version: 0 # Increment this number if you need to re-download cached gems
+      - name: Setup Pages
+        id: pages
+        uses: actions/configure-pages@v3
+      - name: Build with Jekyll
+        # Outputs to the './_site' directory by default
+        run: bundle exec jekyll build --baseurl "${{ steps.pages.outputs.base_path }}"
+        env:
+          JEKYLL_ENV: production
+      - name: Upload artifact
+        # Automatically uploads an artifact from the './_site' directory by default
+        uses: actions/upload-pages-artifact@v2
+
+  # Deployment job
+  deploy:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v2

.gitignore

@@ -4,6 +4,8 @@ _site
 .vagrant
 .jekyll-cache
 Gemfile.lock
+.jekyll-metadata
+.venv
 
 # general
 .DS_Store

Gemfile

@@ -2,3 +2,4 @@ source 'https://rubygems.org'
 
 gem 'jekyll'
 gem 'jekyll-scholar'
+gem 'kramdown-parser-gfm'

Makefile

@@ -0,0 +1,12 @@
+get-records: ## Get records data
+	python utils/collect-records.py
+
+serve: ## Serve and allow tracking changes to _config.yml with
+	npx watchy -w _config.yml -- bundle exec jekyll serve --incremental --watch --livereload
+
+cache-clean: ## Clean cache
+	bundle exec jekyll clean
+
+clean: # Clean
+	rm -rf _site
+	make cache-clean

_config.yml

@@ -10,11 +10,16 @@ title: Neuro Data Extensions (NDX) Catalog
 email: [email protected]
 description: > # this means to ignore newlines until "baseurl:"
   Web page for technical content for HDMF
-exclude: ["README.md", "Gemfile" ]
+
 url: ""
 
 include:
   - _pages
+exclude:
+  - "README.md"
+  - "Gemfile"
+  - utils
+  - .github
 
 sass:
   sass_dir: _sass

_includes/header.html

@@ -7,7 +7,7 @@
         <span class="icon-bar"></span>
         <span class="icon-bar"></span>
       </button>
-      <a class="navbar-brand" href="{{ site.url }}{{ site.baseurl }}/"><img alt="NDX Catalog" src="/images/ndx-logo-text-long.png" height="42px"></a>
+      <a class="navbar-brand" href="{{ site.url }}{{ site.baseurl }}/"><img alt="NDX Catalog" src="{{ site.baseurl }}/images/ndx-logo-text-long.png" height="42px"></a>
     </div>
     <div class="collapse navbar-collapse" id="navbar-collapse-1">
       <ul class="nav navbar-nav navbar-right">

_pages/home.md

@@ -19,65 +19,34 @@ The [Neurodata Extensions Catalog (NDX Catalog)](https://github.com/nwb-extensio
 <p><span id="search-process" style="display: none">Showing <span id="search-results-count"></span> results for "<strong id="search-query"></strong>"</span><span id="show-all" style="display: none">Showing <span id="show-all-count"></span> extensions</span></p>
 <ul id="search-results"></ul>
 
-<script src="{{ site.url }}/js/lunr.min.js"></script>
-<script src="{{ site.url }}/js/jquery-3.5.0.min.js" ></script>
-<script src="{{ site.url }}/js/js-yaml.min.js" ></script>
+<script src="{{ site.url }}{{ site.baseurl }}/js/lunr.min.js"></script>
+<script src="{{ site.url }}{{ site.baseurl }}/js/jquery-3.5.0.min.js" ></script>
+<script src="{{ site.url }}{{ site.baseurl }}/js/js-yaml.min.js" ></script>
 <script src="https://cdn.jsdelivr.net/npm/marked/marked.min.js"></script>
+<script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/purify.min.js"></script>
 
 <script>
 (function() {
-window.data = {}
 
-function loadSearch() {
-  var fileref = document.createElement('script')
-  fileref.setAttribute("type", "text/javascript")
-  fileref.setAttribute("src", "{{ site.baseurl }}/js/search.js")
-  document.getElementsByTagName("head")[0].appendChild(fileref)
-}
-
-$.getJSON("https://api.github.com/orgs/{{ site.github_username }}/repos").done(function(data) {
-  countTotal = 0
-  $.each(data, function(key, recordJson) {
-    if (recordJson.name.startsWith("{{ site.prefix }}") && recordJson.name.endsWith("{{ site.suffix }}") ) {
-      countTotal += 1
-    }
+  function loadSearch() {
+    var fileref = document.createElement('script')
+    fileref.setAttribute("type", "text/javascript")
+    fileref.setAttribute("src", "{{ site.baseurl }}/js/search.js")
+    document.getElementsByTagName("head")[0].appendChild(fileref)
+  }
+
+  $.getJSON('{{ site.baseurl }}/data/records.json').done(function(data) {
+    Object.keys(data).forEach(key=>{
+      // add DOMPurify to sanitize parsed HTML, because marked doesn't sanitize
+      data[key].readme = DOMPurify.sanitize(
+        marked.parse(data[key].readme)
+      );
+      // data[key].pypi_version_badge = "https://img.shields.io/pypi/v/" + data[key].name;
+    });
+    window.data = data;
+    loadSearch();
   });
-  countLoaded = 0
-  $.each(data, function(key, recordJson) {
-    if (recordJson.name.startsWith("{{ site.prefix }}") && recordJson.name.endsWith("{{ site.suffix }}") ) {
-      var resultJson = {}
-      resultJson.ref = recordJson.name
-      resultJson.record_url = recordJson.html_url
-      resultJson.last_updated = recordJson.pushed_at
-      window.data[recordJson.name] = resultJson
-
-      var metaUrl = "https://api.github.com/repos/{{ site.github_username }}/" + recordJson.name + "/contents/ndx-meta.yaml"  
-      $.getJSON(metaUrl).done(function(res, status) {
-        if (status == "success") {
-          metaJson = jsyaml.load(atob(res.content))
-          for (k in metaJson) {
-            window.data[recordJson.name][k] = metaJson[k]
-          }
-        }
-        var readmeUrl = "https://api.github.com/repos/{{ site.github_username }}/" + recordJson.name + "/contents/README.md"
-        $.getJSON(readmeUrl).done(function(res, status) {
-          if (status == "success") {
-            readmeText = atob(res.content)
-            var readmeHtml = marked.parse(readmeText)
-            window.data[recordJson.name].readme = readmeHtml
-
-            countLoaded += 1
-            if (countLoaded == countTotal) {
-              loadSearch()
-            }
-          }
-        });
-      });
-    }
-  });
-});
 })();
-
 </script>
 
 </div>

css/main.scss

@@ -215,3 +215,19 @@ table {
   position: relative;
   padding-right: 20px;
 }
+
+.readme-preview h1 {
+  font-size: 24px;
+}
+
+.readme-preview h2 {
+  font-size: 20px;
+}
+
+.readme-preview h3 {
+  font-size: 16px;
+}
+
+.readme-preview h4 h5 h6 {
+  font-size: 12px;
+}

js/search.js

@@ -75,14 +75,14 @@ excluded_in_search: true
           "<p>" +
           "<span class='ndx-stats'>Version: " + item.version + "</span>" +
           "<span class='ndx-stats'><a href=\"" + item.pip + "\">PyPI project page</a></span>" +
+        //   "<span class='ndx-stats'><img alt='" + item.name + "' on PyPi' src='" + item.pypi_version_badge + "'></img></span>" +
           "<span class='ndx-stats'><a href=\"" + item.record_url + "\">Record repo</a></span>" +
           "<span class='ndx-stats'>License: " + item.license + "</span>" +
           "</p>" +
           "<p>Maintainers: " + maintainers + "</p>" +
-          "<div class='readme-preview'><p>" + readmeHtml + "</p></div></li>";
+          "<details><div class='readme-preview'><p>" + readmeHtml + "</p></div></details></li>";
         return resultHtml;
       }
-
     function displaySearchResults(results, query) {
         var searchResultsEl = document.getElementById("search-results");
         var searchProcessEl = document.getElementById("search-process");
@@ -131,5 +131,10 @@ excluded_in_search: true
     var searchQueryEl = document.getElementById("search-query");
     searchQueryEl.innerText = query;
 
-    displaySearchResults(window.index.search('*' + query + '*'), query); // Hand the results off to be displayed
+    var results = window.index.search('*' + query + '*'); // start with glob search
+    if (results.length == 0) { // use fuzzy if can't find
+        var idx_query = (!query) ? '*' : (query + '~1'); // fuzzy match, edit distance = 1
+        results = window.index.search(idx_query);
+    }
+    displaySearchResults(results, query); // Hand the results off to be displayed
 })();