diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml new file mode 100644 index 00000000..1bb00f85 --- /dev/null +++ b/.github/workflows/python-app.yml @@ -0,0 +1,81 @@ +name: Daily Plugin Processing + +on: + schedule: + - cron: '0 15 * * 2,5' # Run every Tuesday and Friday at 15:00 UTC + push: + branches: + - main + pull_request: + branches: + - main + create: + tags: + - '*' + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Clean repository + run: | + find . -maxdepth 1 -mindepth 1 ! -name '_parse_plugins' ! -name '.*' -exec rm -rf {} + + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.12' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install requests jinja2 + + - name: Install pandoc + run: | + sudo apt-get update + sudo apt-get install -y pandoc + + - name: Clone and Install Pyxu repository + run: | + git clone https://github.com/pyxu-org/pyxu.git + python -m pip install pyxu/.[dev] + + - name: Clean pyxu/doc/fair directory + run: | + rm -rf pyxu/doc/fair/*.rst + rm -rf pyxu/doc/fair/plugins/*.rst + + - name: Run parse_pypi.py + run: python _parse_plugins/parse_pypi.py + + - name: Run make_pages.py + run: python _parse_plugins/make_pages.py + + - name: Build HTML documentation + run: | + export TZ=UTC + sphinx-build -b html -j auto -w build/html/WARNINGS.log pyxu/doc/ ./ + + - name: Remove Pyxu repository + run: rm -rf pyxu + + - name: Commit changes + run: | + git config --global user.name 'github-actions[bot]' + git config --global user.email 'github-actions[bot]@users.noreply.github.com' + git add -A + git commit -m "Update generated HTML files [skip ci]" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Push changes + run: | + git push "https://${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}.git" HEAD:main + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/_parse_plugins/make_pages.py b/_parse_plugins/make_pages.py new file mode 100644 index 00000000..d16948ce --- /dev/null +++ b/_parse_plugins/make_pages.py @@ -0,0 +1,119 @@ +import os +from collections import defaultdict +from jinja2 import Environment, FileSystemLoader +import sqlite3 +import json + +DATABASE_FILE = '_parse_plugins/plugins.db' +TEMPLATES_DIR = "_parse_plugins/templates" +OUTPUT_DIR = "pyxu/doc/fair/plugins" +RST_DIR = "rst" + +entrypoint_metainfo = { + "pyxu.operator": {"shortname": "Operator", "colorclass": "blue"}, + "pyxu.opt.solver": {"shortname": "Solver", "colorclass": "brown"}, + "pyxu.opt.stop": {"shortname": "Stop", "colorclass": "purple"}, + "pyxu.math": {"shortname": "Math", "colorclass": "green"}, + "pyxu.contrib": {"shortname": "Contrib", "colorclass": "orange"}, +} + +status_dict = { + "1": ["Planning: Not yet ready to use. Developers welcome!", "status-planning-d9644d.svg"], + "2": ["Pre-alpha: Not yet ready to use. Developers welcome!", "status-planning-d9644d.svg"], + "3": ["Alpha: Adds new functionality, not yet ready for production. Testing welcome!", "status-alpha-d6af23.svg"], + "4": ["Beta: Adds new functionality, not yet ready for production. Testing welcome!", "status-beta-d6af23.svg"], + "5": ["Production/Stable: Ready for production calculations. Bug reports welcome!", "status-stable-4cc61e.svg"], + "6": ["Mature: Ready for production calculations. Bug reports welcome!", "status-stable-4cc61e.svg"], + "7": ["Inactive: No longer maintained.", "status-inactive-bbbbbb.svg"], +} + +entrypoints_count = defaultdict(list) + +def get_summary_info(entry_points): + summary_info = [] + ep = json.loads(entry_points) + + for entrypoint_name, meta in entrypoint_metainfo.items(): + num = len(ep.get(entrypoint_name, {})) + if num > 0: + summary_info.append({"colorclass": meta["colorclass"], "text": meta["shortname"], "count": num}) + entrypoints_count[entrypoint_name].append(num) + + return summary_info + +def render_plugin_pages(plugins, env): + if os.path.exists(RST_DIR): + for f in os.listdir(RST_DIR): + if f.endswith(".rst"): + os.remove(os.path.join(RST_DIR, f)) + else: + os.mkdir(RST_DIR) + + for plugin in plugins: + summary_info = get_summary_info(plugin["entrypoints"]) + dev_status = status_dict[plugin["development_status"]] + entry_points = json.loads(plugin["entrypoints"]) + rst_plugin_template = env.get_template("plugin.rst") + rst_content = rst_plugin_template.render(plugin=plugin, summary_info=summary_info, dev_status=dev_status, entrypointtypes=entrypoint_metainfo, entry_points=entry_points) + + with open(os.path.join(OUTPUT_DIR, f'{plugin["name"]}.rst'), 'w') as f: + f.write(rst_content) + +def render_catalogue_page(plugins, plugins_info, env): + rst_catalogue_template = env.get_template("catalogue.rst") + rst_content = rst_catalogue_template.render( + plugins=plugins, + summary_info=plugins_info["summary_info"], + dev_status=plugins_info["dev_status"], + dev_status_count=plugins_info["dev_status_count"], + summary_info_count=plugins_info["summary_info_count"].values(), + ) + + with open(os.path.join(OUTPUT_DIR, 'index.rst'), 'w') as f: + f.write(rst_content) + +def main(): + env = Environment(loader=FileSystemLoader(TEMPLATES_DIR)) + + conn = sqlite3.connect(DATABASE_FILE) + c = conn.cursor() + c.execute("SELECT name, pyxu_version, version, author, author_email, home_page, short_description, license, development_status, entrypoints, score FROM plugins ORDER BY name COLLATE NOCASE ASC") + + plugins = [{ + 'name': row[0], + 'pyxu_version': row[1], + 'version': row[2], + 'author': row[3], + 'author_email': row[4], + 'home_page': row[5], + 'short_description': row[6], + 'license': row[7], + 'development_status': row[8], + 'entrypoints': row[9], + 'score': row[10], + } for row in c.fetchall()] + conn.close() + + plugins_info = { + "summary_info": {}, + "dev_status": {}, + "summary_info_count": {epm["shortname"]: {"colorclass": epm["colorclass"], "num_entries": 0, "name": epm["shortname"], "total_num": 0} for epm in entrypoint_metainfo.values()}, + "dev_status_count": {k: {"badge": v[1], "num_entries": 0} for k, v in status_dict.items()} + } + + for plugin in plugins: + summary_info = get_summary_info(plugin["entrypoints"]) + dev_status = status_dict[plugin["development_status"]] + plugins_info["summary_info"].update({plugin["name"]: summary_info}) + plugins_info["dev_status"].update({plugin["name"]: dev_status}) + + for entry in summary_info: + plugins_info["summary_info_count"][entry["text"]]["num_entries"] += 1 + plugins_info["summary_info_count"][entry["text"]]["total_num"] += entry["count"] + plugins_info["dev_status_count"][plugin["development_status"]]["num_entries"] += 1 + + render_plugin_pages(plugins, env) + render_catalogue_page(plugins, plugins_info, env) + +if __name__ == "__main__": + main() diff --git a/_parse_plugins/parse_pypi.py b/_parse_plugins/parse_pypi.py new file mode 100644 index 00000000..7b835747 --- /dev/null +++ b/_parse_plugins/parse_pypi.py @@ -0,0 +1,152 @@ +import sqlite3 +import re +import warnings +import requests +from requests.utils import requote_uri +import configparser +import tempfile +import zipfile +import json +from pathlib import Path +from typing import Dict, Tuple, Any + +DATABASE_FILE = "_parse_plugins/plugins.db" +TROVE_CLASSIFIER = "Framework :: Pycsou" + + +def query_pypi() -> Dict[str, str]: + """ + Query PyPI to get all plugins matching the specified classifier. + :return: Dictionary with plugin names and their latest versions. + """ + packages = {} + name_pattern = re.compile('class="package-snippet__name">(.+?)') + version_pattern = re.compile('class="package-snippet__version">(.+?)') + url = requote_uri(f"https://pypi.org/search/?q=&o=-created&c={TROVE_CLASSIFIER}") + + response = requests.get(url) + response.raise_for_status() + + html = response.text + names = name_pattern.findall(html) + versions = version_pattern.findall(html) + + if len(names) != len(versions): + return {} + + packages = dict(zip(names, versions)) + return packages + + +class CaseSensitiveConfigParser(configparser.ConfigParser): + """Case-sensitive config parser.""" + optionxform = staticmethod(str) + + +def parse_entrypoints(plugin_data: Dict[str, Any]) -> Tuple[str, Dict[str, str]]: + build_types = {data.get("packagetype"): data.get("url") for data in plugin_data.get("urls") if data.get("packagetype")} + + if "bdist_wheel" not in build_types: + warnings.warn("No bdist_wheel available for PyPI release") + return "{}", {} + + wheel_url = build_types.get("bdist_wheel") + if not wheel_url: + return "{}", {} + + try: + with requests.get(wheel_url, stream=True, timeout=120) as download: + download.raise_for_status() + with tempfile.TemporaryDirectory() as tmpdirname: + wheel_path = Path(tmpdirname) / "wheel.whl" + with wheel_path.open("wb") as handle: + for chunk in download.iter_content(chunk_size=8192): + handle.write(chunk) + with zipfile.ZipFile(wheel_path) as whl: + entry_points_content = whl.read(next(name for name in whl.namelist() if name.endswith(".dist-info/entry_points.txt"))).decode("utf-8") + metadata_content = whl.read(next(name for name in whl.namelist() if name.endswith(".dist-info/METADATA"))).decode("utf-8") + + # Parse entry points + parser = CaseSensitiveConfigParser() + parser.read_string(entry_points_content) + entry_points = {section: dict(parser.items(section)) for section in parser.sections()} + + # Parse metadata + metadata = parse_metadata(metadata_content) + + except Exception as err: + warnings.warn(f"Unable to read wheel file from PyPI release of package {plugin_data['info']['name']}: {err}") + return "{}", {} + + return json.dumps(entry_points), metadata + + +def parse_metadata(metadata_content): + metadata = {} + for line in metadata_content.splitlines(): + if line.startswith("Name: "): + metadata["name"] = line.split("Name: ")[1] + elif line.startswith("Version: "): + metadata["version"] = line.split("Version: ")[1] + elif line.startswith("Author-email: "): + metadata["author"] = line.split('Author-email: ')[1].split(" <")[0] + elif line.startswith("Author-email: "): + metadata["author_email"] = line.split('<')[1][:-1] + elif line.startswith("Summary: "): + metadata["short_description"] = line.split("Summary: ")[1] + elif line.startswith("Project-URL: download, "): + metadata["home_page"] = line.split("Project-URL: download, ")[1] + elif line.startswith("License-Expression: "): + metadata["license"] = line.split("License-Expression: ")[1] + elif line.startswith("Classifier: Development Status :: "): + metadata["development_status"] = line.split(":: ")[-1][0] + + + return metadata + + + +def store_plugin_data(plugin_data: Dict[str, Any], conn: sqlite3.Connection, c: sqlite3.Cursor): + entrypoints, metadata = parse_entrypoints(plugin_data) + if not entrypoints or not metadata: + return + name = metadata.get("name", "") + pyxu_version = "2" # TODO: Change according to min, max pyxu versions + version = metadata.get("version", "") + author = metadata.get("author", "") + author_email = metadata.get("author_email", "") + home_page = metadata.get("home_page", "") + short_description = metadata.get("short_description", "") + license = metadata.get("license", "") + development_status = metadata.get("development_status", "1") + score = 100 + + c.execute(""" + INSERT INTO plugins + (name, pyxu_version, version, author, author_email, home_page, short_description, license, development_status, entrypoints, score) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, (name, pyxu_version, version, author, author_email, home_page, short_description, license, development_status, entrypoints, score)) + conn.commit() + + +def main(): + conn = sqlite3.connect(DATABASE_FILE) + c = conn.cursor() + c.execute('''DROP TABLE IF EXISTS plugins''') + c.execute('''CREATE TABLE plugins + (name TEXT, pyxu_version TEXT, version TEXT, author TEXT, author_email TEXT, home_page TEXT, + short_description TEXT, license TEXT, development_status TEXT, entrypoints TEXT, score INTEGER)''') + + plugin_names = query_pypi() + + for plugin_name, plugin_version in plugin_names.items(): + url = f"https://pypi.org/pypi/{plugin_name}/json" + response = requests.get(url) + response.raise_for_status() + plugin_data = response.json() + store_plugin_data(plugin_data, conn, c) + + conn.close() + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/_parse_plugins/plugins.db b/_parse_plugins/plugins.db new file mode 100644 index 00000000..9dbf3b6c Binary files /dev/null and b/_parse_plugins/plugins.db differ diff --git a/_parse_plugins/static/images/pyxu-fair.png b/_parse_plugins/static/images/pyxu-fair.png new file mode 100644 index 00000000..12f7bb38 Binary files /dev/null and b/_parse_plugins/static/images/pyxu-fair.png differ diff --git a/_parse_plugins/static/images/pyxu.png b/_parse_plugins/static/images/pyxu.png new file mode 100644 index 00000000..632a77be Binary files /dev/null and b/_parse_plugins/static/images/pyxu.png differ diff --git a/_parse_plugins/static/images/status-alpha-d6af23.svg b/_parse_plugins/static/images/status-alpha-d6af23.svg new file mode 100644 index 00000000..8d8df548 --- /dev/null +++ b/_parse_plugins/static/images/status-alpha-d6af23.svg @@ -0,0 +1 @@ + statusstatusalphaalpha diff --git a/_parse_plugins/static/images/status-beta-d6af23.svg b/_parse_plugins/static/images/status-beta-d6af23.svg new file mode 100644 index 00000000..e7c31fe5 --- /dev/null +++ b/_parse_plugins/static/images/status-beta-d6af23.svg @@ -0,0 +1 @@ + statusstatusbetabeta diff --git a/_parse_plugins/static/images/status-inactive-bbbbbb.svg b/_parse_plugins/static/images/status-inactive-bbbbbb.svg new file mode 100644 index 00000000..f893172e --- /dev/null +++ b/_parse_plugins/static/images/status-inactive-bbbbbb.svg @@ -0,0 +1 @@ +status: inactivestatusinactive diff --git a/_parse_plugins/static/images/status-planning-d9644d.svg b/_parse_plugins/static/images/status-planning-d9644d.svg new file mode 100644 index 00000000..9c282792 --- /dev/null +++ b/_parse_plugins/static/images/status-planning-d9644d.svg @@ -0,0 +1 @@ + statusstatusplanningplanning diff --git a/_parse_plugins/static/images/status-stable-4cc61e.svg b/_parse_plugins/static/images/status-stable-4cc61e.svg new file mode 100644 index 00000000..980341bc --- /dev/null +++ b/_parse_plugins/static/images/status-stable-4cc61e.svg @@ -0,0 +1 @@ + statusstatusstablestable diff --git a/_parse_plugins/templates/catalogue.rst b/_parse_plugins/templates/catalogue.rst new file mode 100644 index 00000000..bbdbf7aa --- /dev/null +++ b/_parse_plugins/templates/catalogue.rst @@ -0,0 +1,264 @@ +:html_theme.sidebar_secondary.remove: +:sd_hide_title: true + +.. |br| raw:: html + +
+ +.. raw:: html + + + + +********* +Catalogue +********* + + +.. raw:: html + +

Catalogue

+ + +**Total Registered Plugin Packages: {{ plugins | length }}** + +.. raw:: html + +
+
+ {% for summaryentry in summary_info_count %} + + {{summaryentry.name}}{% if summaryentry.tooltip %}{{ summaryentry.tooltip}}{% endif %} + {{ summaryentry.total_num }} plugin{% if summaryentry.total_num != 1 %}s{% endif %} in {{ summaryentry.num_entries }} package{% if summaryentry.num_entries != 1 %}s{% endif %} + + {% endfor %} +
+
+ +**Plugin list in alphabetical order** + + +.. grid:: 2 2 3 3 + :gutter: 3 + + {% for plugin in plugins %} + .. grid-item-card:: + + .. raw:: html + +

{{ plugin.name }}

+ +

+ {% if dev_status %} + + {%- endif -%} + {%- if plugin.pyxu_version -%} +   + {%- endif -%} +

+ + + +

+ {% for summaryinfoelem in summary_info[plugin.name] %} + + {{summaryinfoelem.text}} + {{summaryinfoelem.count}} + + {% endfor %} +

+ + {% endfor %} + + + +.. toctree:: + :maxdepth: 1 + :hidden: + + {% for plugin in plugins %}{{ plugin.name }} + {% endfor %} \ No newline at end of file diff --git a/_parse_plugins/templates/index.rst b/_parse_plugins/templates/index.rst new file mode 100644 index 00000000..5f7d5c10 --- /dev/null +++ b/_parse_plugins/templates/index.rst @@ -0,0 +1,284 @@ +:html_theme.sidebar_secondary.remove: +:sd_hide_title: true + +.. |br| raw:: html + +
+ +.. raw:: html + + + + +********* +Pyxu FAIR +********* + + +.. raw:: html + +

Pyxu FAIR

+

A marketplace for Pyxu-based plugins

+

While Pyxu offers flexibility and portability across various imaging domains, its general-purpose design might + not cater to the specific needs of certain imaging communities. The Pyxu FAIR addresses this by + offering a platform that allows for the development, sharing, and integration of specialized plugins to enhance the + framework. +

+ +

Want to contribute?

+ +

You can contribute contribute and share new image processing tools based on Pyxu with the Pyxu cookiecutter. The cookiecutter aids developers by providing a structured template of classes and functions for plugin creation. Developers can then seamlessly introduce new features in line with Pyxu's established principles and structure. Once you have uploaded your package to PyPi, you can register your plugin package in the Pyxu FAIR by submitting a pull request to the Pyxu repository. See the Contributing to Pyxu-FAIR tutorial for more details. +

+ + + +

Catalogue (alphabetical order)

+ + +**Total Registered Plugin Packages: {{ plugins | length }}** + +.. raw:: html + +
+
+ {% for summaryentry in summary_info_count %} + + {{summaryentry.name}}{% if summaryentry.tooltip %}{{ summaryentry.tooltip}}{% endif %} + {{ summaryentry.total_num }} plugin{% if summaryentry.total_num != 1 %}s{% endif %} in {{ summaryentry.num_entries }} package{% if summaryentry.num_entries != 1 %}s{% endif %} + + {% endfor %} +
+
+ + +.. grid:: 2 2 3 3 + :gutter: 3 + + {% for plugin in plugins %} + .. grid-item-card:: + + .. raw:: html + +

{{ plugin.name }}

+ +

+ {% if dev_status %} + + {%- endif -%} + {%- if plugin.pyxu_version -%} +   + {%- endif -%} +

+ + + +

+ {% for summaryinfoelem in summary_info[plugin.name] %} + + {{summaryinfoelem.text}} + {{summaryinfoelem.count}} + + {% endfor %} +

+ + {% endfor %} + + + +.. toctree:: + :maxdepth: 2 + :hidden: + + howto + contribute + plugins/index \ No newline at end of file diff --git a/_parse_plugins/templates/plugin.rst b/_parse_plugins/templates/plugin.rst new file mode 100644 index 00000000..ee98de3b --- /dev/null +++ b/_parse_plugins/templates/plugin.rst @@ -0,0 +1,249 @@ +.. _plugins-page: + +.. |br| raw:: html + +
+ +.. raw:: html + + + + +{{ '*' * (plugin.name | length) }} +{{plugin.name}} +{{ '*' * (plugin.name | length) }} + +{% if plugin.description %} +Description +=========== + +.. raw:: html + {{ plugin.description | indent(4) }} +{% endif %} + +General information +=================== + +.. raw:: html + + {% if plugin.short_description %} +

+ Short description: {{ plugin.short_description }} +

+ {% endif %} + +

+ {% if dev_status %} + + {%- endif -%} +

+ {% if plugin.pip_install_cmd %} +

+ How to install: {{ plugin.pip_install_cmd }} +

+ {% endif %} + +

+ Source code: Go to the source code repository +

+ + +Detailed information +==================== + +.. raw:: html + {% if plugin.author %} +

+ Author(s): {{ plugin.author }} +

+ {% endif %} + {% if plugin.author_email %} +

+ Contact: {{ plugin.author_email }} +

+ {% endif %} + {% if plugin.version %} +

+ Most recent version: {{ plugin.version }} +

+ {% endif %} + {%- if plugin.pyxu_version -%} +

+ Compatibility: + +

+ {% endif %} + + +Components contributed +====================== + +.. raw:: html +

+ {% for summaryinfoelem in summary_info %} + + {{summaryinfoelem.text}} + {{summaryinfoelem.count}} + + {% endfor %} +

+ {% if entry_points %} + {% for entrypointtype, entrypointlist in entry_points.items() %} + {% if entrypointtype in entrypointtypes.keys() %} + {{ entrypointtypes[entrypointtype].shortname}} ({{ entrypointtype }}) + {% else %} + {{ entrypointtype }} + {% endif %} + + {% endfor %} + {% else %} +

No entry points defined for this plugin.

+ {% endif %}