Add VSC Marketplace

repology-schemacheck.py

@@ -125,6 +125,7 @@
     'termux',
     'ubi',
     'vcpkg',
+    'vscmarketplace',
     'void',
     'wakemeops',
     'wikidata',

repology/fetchers/fetchers/vscmarketplace.py

@@ -0,0 +1,118 @@
+# Copyright (C) 2024 Gavin John <[email protected]>
+#
+# This file is part of repology
+#
+# repology is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# repology is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with repology.  If not, see <http://www.gnu.org/licenses/>.
+
+import json
+
+from repology.atomic_fs import AtomicFile
+from repology.fetchers import PersistentData, ScratchFileFetcher
+from repology.fetchers.http import PoliteHTTP
+from repology.logger import Logger
+
+
+class VSCMarketplaceFetcher(ScratchFileFetcher):
+    def __init__(self, page_size=100, fetch_timeout: int = 5, fetch_delay: int | None = None) -> None:
+        self.url = url
+        self.do_http = PoliteHTTP(timeout=fetch_timeout, delay=fetch_delay)
+
+        # Constants
+        self.include_versions=True
+        self.include_files=True
+        self.include_category_and_tags=False
+        self.include_shared_accounts=True
+        self.include_version_properties=True
+        self.exclude_non_validated=False
+        self.include_installation_targets=False
+        self.include_asset_uri=True
+        self.include_statistics=False
+        self.include_latest_version_only=True
+        self.unpublished=False
+        self.include_name_conflict_info=True
+        self.api_version='7.2-preview.1',
+
+    def _do_fetch(self, statefile: AtomicFile, persdata: PersistentData, logger: Logger) -> bool:
+        extensions = []
+
+        flags = 0
+        if self.include_versions:
+            flags |= 0x1
+
+        if self.include_files:
+            flags |= 0x2
+
+        if self.include_category_and_tags:
+            flags |= 0x4
+
+        if self.include_shared_accounts:
+            flags |= 0x8
+
+        if self.include_version_properties:
+            flags |= 0x10
+
+        if self.exclude_non_validated:
+            flags |= 0x20
+
+        if self.include_installation_targets:
+            flags |= 0x40
+
+        if self.include_asset_uri:
+            flags |= 0x80
+
+        if self.include_statistics:
+            flags |= 0x100
+
+        if self.include_latest_version_only:
+            flags |= 0x200
+
+        if self.unpublished:
+            flags |= 0x1000
+
+        if self.include_name_conflict_info:
+            flags |= 0x8000
+
+        page = 1
+        while True:
+            body = {
+                "filters": [
+                    {
+                        "criteria": [
+                            {
+                                "filterType": 8,
+                                "value": "Microsoft.VisualStudio.Code"
+                            }
+                        ],
+                        "pageNumber": page,
+                        "pageSize": page_size,
+                        "sortBy": 0,
+                        "sortOrder": 0
+                    }
+                ],
+                "assetTypes": [],
+                "flags": flags
+            }
+
+            r = self.do_http("https://marketplace.visualstudio.com/_apis/public/gallery/extensionquery?api-version={version}".format(version=self.api_version), "POST", json=body)
+            response = r.json()
+
+            for extension in response['results'][0]['extensions']:
+                extensions.append(extension)
+
+            page += 1
+
+        with open(statedir.get_path(), 'w', encoding='utf-8') as extdata:
+            json.dump(extensions, extdata)
+
+        return True

repology/parsers/parsers/vscmarketplace.py

@@ -0,0 +1,60 @@
+# Copyright (C) 2024 Gavin John <[email protected]>
+#
+# This file is part of repology
+#
+# repology is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# repology is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with repology.  If not, see <http://www.gnu.org/licenses/>.
+
+import json
+
+from typing import Iterable
+
+from repology.package import PackageFlags
+from repology.packagemaker import NameType, PackageFactory, PackageMaker
+from repology.parsers import Parser
+
+class VSCMarketplaceParser(Parser):
+    def iter_parse(self, path: str, factory: PackageFactory) -> Iterable[PackageMaker]:
+        with open(path, "r"):
+            extension_data = json.load()
+            raw_extensions = extension_data["extensions"]
+
+        for extension in raw_extensions:
+            with factory.begin() as pkg:
+                version_idx = 0
+                while True:
+                    for package_property in extension["versions"][version_idx]["properties"]:
+                        if package_property["key"] == "Microsoft.VisualStudio.Code.PreRelease" and package_property["value"] == "true":
+                            version_idx += 1
+                            continue
+                    break
+
+                pkg.add_name("vscode-extension:{publisherName}-{extensionName}".format(publisherName=extension["publisher"]["publisherName"], extensionName=extension["extensionName"]), NameType.GENERIC_SRC_NAME)
+                pkg.set_version(extension["versions"][version_idx]["version"])
+                pkg.set_summary(extension["shortDescription"])
+                pkg.add_maintainers("{publisherName}@vscmarketplace".format(**extension["publisher"]))
+                pkg.add_homepages("https://marketplace.visualstudio.com/items?itemName={publisherName}.{extensionName}".format(publisherName=extension["publisher"]["publisherName"], extensionName=extension["extensionName"]))
+
+                for file_meta in extension["versions"][version_idx]["files"]:
+                    match file_meta["assetType"]:
+                        case "Microsoft.VisualStudio.Services.Content.Changelog":
+                            pkg.set_extra_field("changelog", file_meta["source"])
+                        case "Microsoft.VisualStudio.Services.Content.License":
+                            pkg.add_licenses(file_meta["source"])
+
+                for package_property in extension["versions"][version_idx]["properties"]:
+                    match package_property["key"]:
+                        case "Microsoft.VisualStudio.Services.Links.Source":
+                            pkg.add_homepages(package_property["value"])
+
+                yield pkg

repos.d/vscmarketplace.yaml

@@ -0,0 +1,20 @@
+###########################################################################
+# VSC Marketplace
+###########################################################################
+- name: vscmarketplace
+  type: repository
+  desc: Visual Studio Code Marketplace
+  family: vscmarketplace
+  ruleset: vscmarketplace
+  minpackages: 3000
+  sources:
+    - name: packages
+      fetcher:
+        class: VSCMarketplaceFetcher
+      parser:
+        class: VSCMarketplaceParser
+  repolinks:
+    - desc: Visual Studio Code Marketplace
+      url: https://marketplace.visualstudio.com/vscode
+  packagelinks: []
+  groups: [ all, production ]