add plugin manager support

.github/workflows/deploy.yml

@@ -0,0 +1,50 @@
+name: Deploy repository to Github Pages
+
+on:
+  push:
+    branches: [ master, develop ]
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+jobs:
+  build:
+    runs-on: ubuntu-22.04
+    steps:
+    - name: Checkout master
+      uses: actions/checkout@v2
+      with:
+        path: master
+        ref: master
+        fetch-depth: '0'
+    - run: |
+        cd master
+        ./build_site.sh ../_site/master
+    - name: Checkout develop
+      uses: actions/checkout@v2
+      with:
+        path: develop
+        # change this ref to whatever dev branch/tag we need when necessary
+        ref: develop
+        fetch-depth: '0'
+    - run: |
+        cd develop
+        ../master/build_site.sh ../_site/develop
+    - uses: actions/upload-pages-artifact@v2
+
+  deploy:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-22.04
+    needs: build
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v2

build_site.sh

@@ -0,0 +1,75 @@
+#!/bin/bash
+
+# builds a repository of scrapers
+# outputs to _site with the following structure:
+# index.yml
+# <scraper_id>.zip
+# Each zip file contains the scraper.yml file and any other files in the same directory
+
+outdir="$1"
+if [ -z "$outdir" ]; then
+    outdir="_site"
+fi
+
+rm -rf "$outdir"
+mkdir -p "$outdir"
+
+buildPlugin() 
+{
+    f=$1
+
+    if grep -q "^#pkgignore" "$f"; then
+        return
+    fi
+
+    # get the scraper id from the directory
+    dir=$(dirname "$f")
+    plugin_id=$(basename "$f" .yml)
+
+    echo "Processing $plugin_id"
+
+    # create a directory for the version
+    version=$(git log -n 1 --pretty=format:%h -- "$dir"/*)
+    updated=$(TZ=UTC0 git log -n 1 --date="format-local:%F %T" --pretty=format:%ad -- "$dir"/*)
+
+    # create the zip file
+    # copy other files
+    zipfile=$(realpath "$outdir/$plugin_id.zip")
+
+    pushd "$dir" > /dev/null
+    zip -r "$zipfile" . > /dev/null
+    popd > /dev/null
+
+    name=$(grep "^name:" "$f" | head -n 1 | cut -d' ' -f2- | sed -e 's/\r//' -e 's/^"\(.*\)"$/\1/')
+    description=$(grep "^description:" "$f" | head -n 1 | cut -d' ' -f2- | sed -e 's/\r//' -e 's/^"\(.*\)"$/\1/')
+    ymlVersion=$(grep "^version:" "$f" | head -n 1 | cut -d' ' -f2- | sed -e 's/\r//' -e 's/^"\(.*\)"$/\1/')
+    version="$ymlVersion-$version"
+    dep=$(grep "^# requires:" "$f" | cut -c 12- | sed -e 's/\r//')
+
+    # write to spec index
+    echo "- id: $plugin_id
+  name: $name
+  metadata:
+    description: $description
+  version: $version
+  date: $updated
+  path: $plugin_id.zip
+  sha256: $(sha256sum "$zipfile" | cut -d' ' -f1)" >> "$outdir"/index.yml
+
+    # handle dependencies
+    if [ ! -z "$dep" ]; then
+        echo "  requires:" >> "$outdir"/index.yml
+        for d in ${dep//,/ }; do
+            echo "    - $d" >> "$outdir"/index.yml
+        done
+    fi
+
+    echo "" >> "$outdir"/index.yml
+}
+
+find ./plugins -mindepth 1 -name *.yml | while read file; do
+    buildPlugin "$file"
+done
+find ./themes -mindepth 1 -name *.yml | while read file; do
+    buildPlugin "$file"
+done