From 00ca224d3527cb34443d570fdc0daee17bcd1cf9 Mon Sep 17 00:00:00 2001 From: Dmitry Marakasov Date: Mon, 23 Sep 2024 20:51:15 +0300 Subject: [PATCH] Tweak Open VSX entry - Fix parser to use standard json processing facility - Fix handling optional fields (download, description - Fix package link types and add another link (recipe/manifest) - Improbe name handling, handle display name properly - Move repository entry into modules/ fix type, update minpackages, enable shadow --- repology/packagemaker/names.py | 14 ++++++++++++++ repology/parsers/parsers/openvsx.py | 25 +++++++++++++++---------- repos.d/modules/openvsx.yaml | 28 ++++++++++++++++++++++++++++ repos.d/openvsx.yaml | 22 ---------------------- 4 files changed, 57 insertions(+), 32 deletions(-) create mode 100644 repos.d/modules/openvsx.yaml delete mode 100644 repos.d/openvsx.yaml diff --git a/repology/packagemaker/names.py b/repology/packagemaker/names.py index 0ce4a8a6..9ec4ca99 100644 --- a/repology/packagemaker/names.py +++ b/repology/packagemaker/names.py @@ -137,6 +137,10 @@ class NameType: CHROMEBREW_NAME: ClassVar[int] = 330 + OPENVSX_DISPLAYNAME: ClassVar[int] = 340 + OPENVSX_NAMESPACE_SLASH_NAME: ClassVar[int] = 341 + OPENVSX_NAMESPACE_DOT_NAME: ClassVar[int] = 342 + # Note that packages reside in subdirs such as packages/, # root-packages/, x11-packages/ and it may be that srcname # should include directory as well. However these are names @@ -547,6 +551,16 @@ class MappedNames: visiblename=NameType.CHROMEBREW_NAME, projectname_seed=NameType.CHROMEBREW_NAME, ), + # OpenVSX + _NameMapping( + # assuming srcname according to URLs on open-vsx.org + srcname=NameType.OPENVSX_NAMESPACE_SLASH_NAME, + # assuming binname according to https://stackoverflow.com/questions/34286515/how-to-install-visual-studio-code-extensions-from-command-line, + binname=NameType.OPENVSX_NAMESPACE_DOT_NAME, + trackname=NameType.OPENVSX_NAMESPACE_DOT_NAME, + visiblename=NameType.OPENVSX_DISPLAYNAME, + projectname_seed=NameType.OPENVSX_NAMESPACE_DOT_NAME, + ), ] diff --git a/repology/parsers/parsers/openvsx.py b/repology/parsers/parsers/openvsx.py index 4a3cc295..e0cece96 100644 --- a/repology/parsers/parsers/openvsx.py +++ b/repology/parsers/parsers/openvsx.py @@ -22,22 +22,27 @@ from repology.package import LinkType from repology.packagemaker import NameType, PackageFactory, PackageMaker from repology.parsers import Parser +from repology.parsers.json import iter_json_list class OpenVSXParser(Parser): def iter_parse(self, path: str, factory: PackageFactory) -> Iterable[PackageMaker]: - with open(path, 'r') as extdatafile: - extension_data = json.load(extdatafile) - raw_extensions = extension_data['extensions'] - - for extension in raw_extensions: + for extension in iter_json_list(path, ('extensions', None)): with factory.begin() as pkg: # TODO: More metadata is available, it's just harder to fetch and will require its own fetcher, in all likelihood - pkg.add_name('vscode-extension:{namespace}-{name}'.format(**extension), NameType.GENERIC_SRC_NAME) + namespace = extension['namespace'] + name = extension['name'] + pkg.add_name(f'{namespace}.{name}', NameType.OPENVSX_NAMESPACE_DOT_NAME) + pkg.add_name(f'{namespace}/{name}', NameType.OPENVSX_NAMESPACE_SLASH_NAME) + pkg.add_name(extension.get('displayName', name), NameType.OPENVSX_DISPLAYNAME) pkg.set_version(extension['version']) - pkg.set_summary(extension['description']) - pkg.add_maintainers('{namespace}@openvsx'.format(**extension)) - pkg.add_links(LinkType.UPSTREAM_HOMEPAGE, 'https://open-vsx.org/extension/{namespace}/{name}'.format(**extension)) - pkg.add_links(LinkType.UPSTREAM_DOWNLOAD, extension['files']['download']) + pkg.set_summary(extension.get('description')) + pkg.add_maintainers(f'{namespace}@openvsx') + + if not extension['files']: + continue + + pkg.add_links(LinkType.PROJECT_DOWNLOAD, extension['files']['download']) + pkg.add_links(LinkType.PACKAGE_RECIPE_RAW, extension['files']['download'].rsplit('/', 1)[0] + '/package.json') yield pkg diff --git a/repos.d/modules/openvsx.yaml b/repos.d/modules/openvsx.yaml new file mode 100644 index 00000000..539eb32a --- /dev/null +++ b/repos.d/modules/openvsx.yaml @@ -0,0 +1,28 @@ +########################################################################### +# OpenVSX +########################################################################### +# NOTE: API returns more data for individual packages, including upstream +# url. Unfortunately, we cannot fetch individual API output for every package +# Also if ?size=10000 breaks at some point, we'll have to disable this +- name: openvsx + type: modules + desc: Open VSX + family: openvsx + ruleset: openvsx + minpackages: 3500 + sources: + - name: search.json + fetcher: + class: FileFetcher + url: https://open-vsx.org/api/-/search?size=10000 + allow_zero_size: false + parser: + class: OpenVSXParser + shadow: true + repolinks: + - desc: Open VSX registry + url: https://open-vsx.org/ + packagelinks: + - type: PROJECT_HOMEPAGE + url: 'https://open-vsx.org/extension/{srcname}' + groups: [ all, production ] diff --git a/repos.d/openvsx.yaml b/repos.d/openvsx.yaml deleted file mode 100644 index 61b88387..00000000 --- a/repos.d/openvsx.yaml +++ /dev/null @@ -1,22 +0,0 @@ -########################################################################### -# OpenVSX -########################################################################### -- name: openvsx - type: repository - desc: OpenVSX - family: openvsx - ruleset: openvsx - minpackages: 3000 - sources: - - name: packages - fetcher: - class: FileFetcher - url: https://open-vsx.org/api/-/search?size=10000 # TODO: Do proper pagination - allow_zero_size: false - parser: - class: OpenVSXParser - repolinks: - - desc: OpenVSX registry - url: https://open-vsx.org/ - packagelinks: [] - groups: [ all, production ]