From c998c366a2a017d88d4ba4ad0f680e5155da2105 Mon Sep 17 00:00:00 2001 From: Gavin John Date: Wed, 11 Sep 2024 12:45:21 -0700 Subject: [PATCH] Add VSC Marketplace --- repology-schemacheck.py | 1 + repology/fetchers/fetchers/vscmarketplace.py | 118 +++++++++++++++++++ repology/parsers/parsers/vscmarketplace.py | 60 ++++++++++ repos.d/vscmarketplace.yaml | 20 ++++ 4 files changed, 199 insertions(+) create mode 100644 repology/fetchers/fetchers/vscmarketplace.py create mode 100644 repology/parsers/parsers/vscmarketplace.py create mode 100644 repos.d/vscmarketplace.yaml diff --git a/repology-schemacheck.py b/repology-schemacheck.py index 91bd0c14a..566d949f2 100755 --- a/repology-schemacheck.py +++ b/repology-schemacheck.py @@ -125,6 +125,7 @@ 'termux', 'ubi', 'vcpkg', + 'vscmarketplace', 'void', 'wakemeops', 'wikidata', diff --git a/repology/fetchers/fetchers/vscmarketplace.py b/repology/fetchers/fetchers/vscmarketplace.py new file mode 100644 index 000000000..392399d8b --- /dev/null +++ b/repology/fetchers/fetchers/vscmarketplace.py @@ -0,0 +1,118 @@ +# Copyright (C) 2024 Gavin John +# +# This file is part of repology +# +# repology is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# repology is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with repology. If not, see . + +import json + +from repology.atomic_fs import AtomicFile +from repology.fetchers import PersistentData, ScratchFileFetcher +from repology.fetchers.http import PoliteHTTP +from repology.logger import Logger + + +class VSCMarketplaceFetcher(ScratchFileFetcher): + def __init__(self, page_size=100, fetch_timeout: int = 5, fetch_delay: int | None = None) -> None: + self.url = url + self.do_http = PoliteHTTP(timeout=fetch_timeout, delay=fetch_delay) + + # Constants + self.include_versions=True + self.include_files=True + self.include_category_and_tags=False + self.include_shared_accounts=True + self.include_version_properties=True + self.exclude_non_validated=False + self.include_installation_targets=False + self.include_asset_uri=True + self.include_statistics=False + self.include_latest_version_only=True + self.unpublished=False + self.include_name_conflict_info=True + self.api_version='7.2-preview.1', + + def _do_fetch(self, statefile: AtomicFile, persdata: PersistentData, logger: Logger) -> bool: + extensions = [] + + flags = 0 + if self.include_versions: + flags |= 0x1 + + if self.include_files: + flags |= 0x2 + + if self.include_category_and_tags: + flags |= 0x4 + + if self.include_shared_accounts: + flags |= 0x8 + + if self.include_version_properties: + flags |= 0x10 + + if self.exclude_non_validated: + flags |= 0x20 + + if self.include_installation_targets: + flags |= 0x40 + + if self.include_asset_uri: + flags |= 0x80 + + if self.include_statistics: + flags |= 0x100 + + if self.include_latest_version_only: + flags |= 0x200 + + if self.unpublished: + flags |= 0x1000 + + if self.include_name_conflict_info: + flags |= 0x8000 + + page = 1 + while True: + body = { + "filters": [ + { + "criteria": [ + { + "filterType": 8, + "value": "Microsoft.VisualStudio.Code" + } + ], + "pageNumber": page, + "pageSize": page_size, + "sortBy": 0, + "sortOrder": 0 + } + ], + "assetTypes": [], + "flags": flags + } + + r = self.do_http("https://marketplace.visualstudio.com/_apis/public/gallery/extensionquery?api-version={version}".format(version=self.api_version), "POST", json=body) + response = r.json() + + for extension in response['results'][0]['extensions']: + extensions.append(extension) + + page += 1 + + with open(statedir.get_path(), 'w', encoding='utf-8') as extdata: + json.dump(extensions, extdata) + + return True diff --git a/repology/parsers/parsers/vscmarketplace.py b/repology/parsers/parsers/vscmarketplace.py new file mode 100644 index 000000000..7d76c583a --- /dev/null +++ b/repology/parsers/parsers/vscmarketplace.py @@ -0,0 +1,60 @@ +# Copyright (C) 2024 Gavin John +# +# This file is part of repology +# +# repology is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# repology is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with repology. If not, see . + +import json + +from typing import Iterable + +from repology.package import PackageFlags +from repology.packagemaker import NameType, PackageFactory, PackageMaker +from repology.parsers import Parser + +class VSCMarketplaceParser(Parser): + def iter_parse(self, path: str, factory: PackageFactory) -> Iterable[PackageMaker]: + with open(path, "r"): + extension_data = json.load() + raw_extensions = extension_data["extensions"] + + for extension in raw_extensions: + with factory.begin() as pkg: + version_idx = 0 + while True: + for package_property in extension["versions"][version_idx]["properties"]: + if package_property["key"] == "Microsoft.VisualStudio.Code.PreRelease" and package_property["value"] == "true": + version_idx += 1 + continue + break + + pkg.add_name("vscode-extension:{publisherName}-{extensionName}".format(publisherName=extension["publisher"]["publisherName"], extensionName=extension["extensionName"]), NameType.GENERIC_SRC_NAME) + pkg.set_version(extension["versions"][version_idx]["version"]) + pkg.set_summary(extension["shortDescription"]) + pkg.add_maintainers("{publisherName}@vscmarketplace".format(**extension["publisher"])) + pkg.add_homepages("https://marketplace.visualstudio.com/items?itemName={publisherName}.{extensionName}".format(publisherName=extension["publisher"]["publisherName"], extensionName=extension["extensionName"])) + + for file_meta in extension["versions"][version_idx]["files"]: + match file_meta["assetType"]: + case "Microsoft.VisualStudio.Services.Content.Changelog": + pkg.set_extra_field("changelog", file_meta["source"]) + case "Microsoft.VisualStudio.Services.Content.License": + pkg.add_licenses(file_meta["source"]) + + for package_property in extension["versions"][version_idx]["properties"]: + match package_property["key"]: + case "Microsoft.VisualStudio.Services.Links.Source": + pkg.add_homepages(package_property["value"]) + + yield pkg diff --git a/repos.d/vscmarketplace.yaml b/repos.d/vscmarketplace.yaml new file mode 100644 index 000000000..183dd73a0 --- /dev/null +++ b/repos.d/vscmarketplace.yaml @@ -0,0 +1,20 @@ +########################################################################### +# VSC Marketplace +########################################################################### +- name: vscmarketplace + type: repository + desc: Visual Studio Code Marketplace + family: vscmarketplace + ruleset: vscmarketplace + minpackages: 3000 + sources: + - name: packages + fetcher: + class: VSCMarketplaceFetcher + parser: + class: VSCMarketplaceParser + repolinks: + - desc: Visual Studio Code Marketplace + url: https://marketplace.visualstudio.com/vscode + packagelinks: [] + groups: [ all, production ]