Simplify vendoring and declare dependencies optionally

.github/workflows/main.yml

@@ -129,7 +129,6 @@ jobs:
         job:
         - diffcov
         - docs
-        - check-extern
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4

conftest.py

@@ -32,11 +32,8 @@ def pytest_configure(config):
     'setuptools/tests/mod_with_constant.py',
     'setuptools/_distutils',
     '_distutils_hack',
-    'setuptools/extern',
-    'pkg_resources/extern',
     'pkg_resources/tests/data',
     'setuptools/_vendor',
-    'pkg_resources/_vendor',
     'setuptools/config/_validate_pyproject',
     'setuptools/modified.py',
     'setuptools/tests/bdist_wheel_testdata',

mypy.ini

@@ -12,7 +12,7 @@ exclude = (?x)(
 	| ^.tox/
 	| ^.eggs/
 	| ^pkg_resources/tests/data/my-test-package-source/setup.py$ # Duplicate module name
-	| ^.+?/(_vendor|extern)/ # Vendored
+	| ^setuptools/_vendor/ # Vendored
 	| ^setuptools/_distutils/ # Vendored
 	| ^setuptools/config/_validate_pyproject/ # Auto-generated
     | ^setuptools/tests/bdist_wheel_testdata/  # Duplicate module name
@@ -31,15 +31,14 @@ disable_error_code = attr-defined
 [mypy-pkg_resources.tests.*]
 disable_error_code = import-not-found
 
-# - Avoid raising issues when importing from "extern" modules, as those are added to path dynamically.
-#   https://github.com/pypa/setuptools/pull/3979#discussion_r1367968993
 # - distutils._modified has different errors on Python 3.8 [import-untyped], on Python 3.9+ [import-not-found]
 # - All jaraco modules are still untyped
 # - _validate_project sometimes complains about trove_classifiers (#4296)
-[mypy-pkg_resources.extern.*,setuptools.extern.*,distutils._modified,jaraco.*,trove_classifiers]
+# - wheel appears to be untyped
+[mypy-distutils._modified,jaraco.*,trove_classifiers,wheel.*]
 ignore_missing_imports = True
 
-# Even when excluding vendored/generated modules, there might be problems: https://github.com/python/mypy/issues/11936#issuecomment-1466764006
-[mypy-setuptools._vendor.packaging._manylinux,setuptools.config._validate_pyproject.*]
+# Even when excluding generated modules, there might be problems: https://github.com/python/mypy/issues/11936#issuecomment-1466764006
+[mypy-setuptools.config._validate_pyproject.*]
 follow_imports = silent
 # silent => ignore errors when following imports

newsfragments/2825.removal.rst

@@ -0,0 +1 @@
+Now setuptools declares its own dependencies in the ``core`` extra. Dependencies are still vendored for bootstrapping purposes, but setuptools will prefer installed dependencies if present. The ``core`` extra is used for informational purposes and should *not* be declared in package metadata (e.g. ``build-requires``). Downstream packagers can de-vendor by simply removing the ``setuptools/_vendor`` directory.

pkg_resources/__init__.py

@@ -74,6 +74,8 @@
 
 import _imp
 
+sys.path.extend(((vendor_path := os.path.join(os.path.dirname(os.path.dirname(__file__)), 'setuptools', '_vendor')) not in sys.path) * [vendor_path])  # fmt: skip
+
 # capture these to bypass sandboxing
 from os import utime
 from os import open as os_open
@@ -87,16 +89,17 @@
     # no write support, probably under GAE
     WRITE_SUPPORT = False
 
-from pkg_resources.extern.jaraco.text import (
+import packaging.specifiers
+from jaraco.text import (
     yield_lines,
     drop_comment,
     join_continuation,
 )
-from pkg_resources.extern.packaging import markers as _packaging_markers
-from pkg_resources.extern.packaging import requirements as _packaging_requirements
-from pkg_resources.extern.packaging import utils as _packaging_utils
-from pkg_resources.extern.packaging import version as _packaging_version
-from pkg_resources.extern.platformdirs import user_cache_dir as _user_cache_dir
+from packaging import markers as _packaging_markers
+from packaging import requirements as _packaging_requirements
+from packaging import utils as _packaging_utils
+from packaging import version as _packaging_version
+from platformdirs import user_cache_dir as _user_cache_dir
 
 if TYPE_CHECKING:
     from _typeshed import BytesPath, StrPath, StrOrBytesPath
@@ -538,8 +541,7 @@ def get_distribution(dist: Distribution | _PkgReqType) -> Distribution:
     if isinstance(dist, str):
         dist = Requirement.parse(dist)
     if isinstance(dist, Requirement):
-        # Bad type narrowing, dist has to be a Requirement here, so get_provider has to return Distribution
-        dist = get_provider(dist)  # type: ignore[assignment]
+        dist = get_provider(dist)
     if not isinstance(dist, Distribution):
         raise TypeError("Expected str, Requirement, or Distribution", dist)
     return dist
@@ -1117,11 +1119,10 @@ def markers_pass(self, req: Requirement, extras: tuple[str, ...] | None = None):
         Return False if the req has a marker and fails
         evaluation. Otherwise, return True.
         """
-        extra_evals = (
+        return not req.marker or any(
             req.marker.evaluate({'extra': extra})
-            for extra in self.get(req, ()) + (extras or (None,))
+            for extra in self.get(req, ()) + (extras or ("",))
         )
-        return not req.marker or any(extra_evals)
 
 
 class Environment:
@@ -3430,15 +3431,18 @@ class RequirementParseError(_packaging_requirements.InvalidRequirement):
 
 
 class Requirement(_packaging_requirements.Requirement):
+    # prefer variable length tuple to set (as found in
+    # packaging.requirements.Requirement)
+    extras: tuple[str, ...]  # type: ignore[assignment]
+
     def __init__(self, requirement_string: str):
         """DO NOT CALL THIS UNDOCUMENTED METHOD; use Requirement.parse()!"""
         super().__init__(requirement_string)
         self.unsafe_name = self.name
         project_name = safe_name(self.name)
         self.project_name, self.key = project_name, project_name.lower()
         self.specs = [(spec.operator, spec.version) for spec in self.specifier]
-        # packaging.requirements.Requirement uses a set for its extras. We use a variable-length tuple
-        self.extras: tuple[str] = tuple(map(safe_extra, self.extras))
+        self.extras = tuple(map(safe_extra, self.extras))
         self.hashCmp = (
             self.key,
             self.url,
@@ -3454,17 +3458,24 @@ def __eq__(self, other: object):
     def __ne__(self, other):
         return not self == other
 
-    def __contains__(self, item: Distribution | str | tuple[str, ...]) -> bool:
+    def __contains__(
+        self, item: Distribution | packaging.specifiers.UnparsedVersion
+    ) -> bool:
         if isinstance(item, Distribution):
             if item.key != self.key:
                 return False
 
-            item = item.version
+            version = item.version
+        else:
+            version = item
 
         # Allow prereleases always in order to match the previous behavior of
         # this method. In the future this should be smarter and follow PEP 440
         # more accurately.
-        return self.specifier.contains(item, prereleases=True)
+        return self.specifier.contains(
+            version,
+            prereleases=True,
+        )
 
     def __hash__(self):
         return self.__hash