copy 3p stubs example from fwingerter-Ocient/bazel-mypy-pypi-example

.github/workflows/continuous-integration.yml

@@ -13,12 +13,8 @@ jobs:
     steps:
     # Checks-out the repository under $GITHUB_WORKSPACE, so the job can access it
     - uses: actions/checkout@v2
-
-    - name: Setup Bazel
-      uses: abhinavsingh/setup-bazel@v3
-      with:
-        # Must match version in repo's .bazelversion file.
-        version: 5.1.0
-
     - name: Run tests
       run: ./test.sh
+    - name: Check examples
+      working-directory: examples
+      run: bazel test //...

BUILD

@@ -1,4 +1,5 @@
 load("@buildifier_prebuilt//:rules.bzl", "buildifier")
+load("@rules_python//python/pip_install:requirements.bzl", "compile_pip_requirements")
 
 package(default_visibility = ["//visibility:private"])
 
@@ -20,3 +21,9 @@ exports_files([
     "LICENSE",
     "version.bzl",
 ])
+
+compile_pip_requirements(
+    name = "requirements",
+    requirements_in = "current_mypy_version.txt",
+    requirements_txt = "requirements-locked.txt",
+)

WORKSPACE

@@ -12,9 +12,21 @@ load("//:config.bzl", "mypy_configuration")
 
 mypy_configuration()
 
+load("@rules_python//python:repositories.bzl", "python_register_toolchains")
+
+python_register_toolchains(
+    name = "python3_8",
+    python_version = "3.8",
+)
+
+load("@python3_8//:defs.bzl", "interpreter")
+
 load("//repositories:deps.bzl", mypy_integration_deps = "deps")
 
-mypy_integration_deps("//:current_mypy_version.txt")
+mypy_integration_deps(
+    "//:current_mypy_version.txt",
+    python_interpreter = interpreter,
+)
 
 http_archive(
     name = "buildifier_prebuilt",
@@ -56,3 +68,11 @@ http_archive(
     strip_prefix = "buildtools-master",
     url = "https://github.com/bazelbuild/buildtools/archive/master.zip",
 )
+
+load("@aspect_rules_py//py:repositories.bzl", "rules_py_dependencies")
+
+rules_py_dependencies()
+
+load("@mypy_integration_pip_deps//:requirements.bzl", "install_deps")
+
+install_deps()

config.bzl

@@ -3,18 +3,18 @@ Provides functions to support (optionally) passing the MyPy configuration file
 to this integration.
 """
 
-def _create_config_impl(ctx):
-    if ctx.attr.config_filepath:
-        user_mypy_config_contents = ctx.read(ctx.attr.config_filepath)
+def _create_config_impl(rctx):
+    if rctx.attr.config_filepath:
+        user_mypy_config_contents = rctx.read(rctx.attr.config_filepath)
     else:
         user_mypy_config_contents = "[mypy]"
 
-    ctx.file(
+    rctx.file(
         "mypy.ini",
         content = user_mypy_config_contents,
         executable = False,
     )
-    ctx.file(
+    rctx.file(
         "BUILD",
         content = "exports_files(['mypy.ini'])",
         executable = False,

examples/.bazelrc

@@ -1,2 +1 @@
-build --aspects //tools/linting:aspect.bzl%lint
-build --output_groups=+report
+test --test_output=errors

examples/third_party/BUILD

@@ -0,0 +1,21 @@
+load("@my_deps//:requirements.bzl", "requirement")
+load("@mypy_integration//:mypy.bzl", "mypy_test")
+
+py_binary(
+    name = "uses-deps",
+    srcs = ["uses-deps.py"],
+    deps = [
+        requirement("types_python_dateutil"),
+    ],
+)
+
+mypy_test(
+    name = "uses_deps_mypy",
+    deps = [
+        ":uses-deps",
+        requirement("types_python_dateutil"),
+    ],
+    # Confirmed the launcher script contains
+    # export MYPYPATH="$(pwd):../my_deps/pypi__types_python_dateutil"
+    include_imports = True,
+)

examples/third_party/requirements.txt

@@ -1 +1,3 @@
-parse==1.12.1
+parse==1.12.1
+types-python-dateutil==0.1.4
+

examples/third_party/uses-deps.py

@@ -0,0 +1,4 @@
+import dateutil
+from dateutil import parser
+
+dateutil.parser.parse("1970")

examples/tools/typing/mypy_version.txt

@@ -1 +1,2 @@
-mypy==0.750
+mypy==0.910
+typed_ast

mypy.bzl

@@ -65,12 +65,21 @@ def _extract_srcs(srcs):
                 direct_src_files.append(f)
     return direct_src_files
 
-def _extract_transitive_deps(deps):
+def _extract_transitive_deps(deps, include_imports):
     transitive_deps = []
+    transitive_imports = []
+    seen_imports = {}  # No sets in Starlark, so use a dict.
     for dep in deps:
-        if MyPyStubsInfo not in dep and PyInfo in dep and not _is_external_dep(dep):
-            transitive_deps.append(dep[PyInfo].transitive_sources)
-    return transitive_deps
+        if MyPyStubsInfo not in dep and PyInfo in dep:
+            if include_imports:
+                transitive_deps.append(dep[PyInfo].transitive_sources)
+                for imp in dep[PyInfo].imports.to_list():
+                    if imp not in seen_imports:
+                        seen_imports[imp] = None
+                        transitive_imports.append(imp)
+            elif not _is_external_dep(dep):
+                transitive_deps.append(dep[PyInfo].transitive_sources)
+    return transitive_deps, transitive_imports
 
 def _extract_stub_deps(deps):
     # Need to add the .py files AND the .pyi files that are
@@ -110,22 +119,37 @@ def _mypy_rule_impl(ctx, is_aspect = False):
     transitive_srcs_depsets = []
     stub_files = []
 
+    include_imports = hasattr(base_rule.attr, "include_imports") and base_rule.attr.include_imports
+
     if hasattr(base_rule.attr, "srcs"):
         direct_src_files = _extract_srcs(base_rule.attr.srcs)
 
     if hasattr(base_rule.attr, "deps"):
-        transitive_srcs_depsets = _extract_transitive_deps(base_rule.attr.deps)
+        transitive_srcs_depsets, transitive_imports = _extract_transitive_deps(base_rule.attr.deps, include_imports)
         stub_files = _extract_stub_deps(base_rule.attr.deps)
+        if transitive_imports:
+            rel_workspace_root = ''
+            # If in a package, imports need to be made relative to the
+            # workspace root.
+            if ctx.label.package:
+                rel_workspace_root = '../' * (ctx.label.package.count('/') + 1)
+            mypypath_parts += [rel_workspace_root + x for x in transitive_imports]
 
     if hasattr(base_rule.attr, "imports"):
-        mypypath_parts = _extract_imports(base_rule.attr.imports, ctx.label)
+        mypypath_parts += _extract_imports(base_rule.attr.imports, ctx.label)
 
     final_srcs_depset = depset(transitive = transitive_srcs_depsets +
                                             [depset(direct = direct_src_files)])
-    src_files = [f for f in final_srcs_depset.to_list() if not _is_external_src(f)]
-    if not src_files:
+    input_src_files = final_srcs_depset.to_list()
+    target_src_files = [f for f in input_src_files if not _is_external_src(f)]
+    if not target_src_files:
         return None
 
+    # If imports aren't being included, the input src files are restricted to
+    # only the direct targets.
+    if not include_imports:
+        input_src_files = target_src_files
+
     mypypath_parts += [src_f.dirname for src_f in stub_files]
     mypypath = ":".join(mypypath_parts)
 
@@ -149,34 +173,43 @@ def _mypy_rule_impl(ctx, is_aspect = False):
     # Compose a list of the files needed for use. Note that aspect rules can use
     # the project version of mypy however, other rules should fall back on their
     # relative runfiles.
-    runfiles = ctx.runfiles(files = src_files + stub_files + [mypy_config_file])
+    runfiles = ctx.runfiles(files = input_src_files + stub_files + [mypy_config_file])
     if not is_aspect:
         runfiles = runfiles.merge(ctx.attr._mypy_cli.default_runfiles)
 
     src_root_paths = sets.to_list(
-        sets.make([f.root.path for f in src_files]),
+        sets.make([f.root.path for f in input_src_files]),
     )
 
+    follow_imports = ""
+    if include_imports:
+        # --follow-imports=silent is passed in order to suppress errors on
+        # non-target (imported) libraries.
+        # 0.810 has a --exclude flag which may work better:
+        # https://github.com/python/mypy/pull/9992
+        follow_imports = "--follow-imports=silent"
+
     ctx.actions.expand_template(
         template = ctx.file._template,
         output = exe,
         substitutions = {
             "{MYPY_EXE}": ctx.executable._mypy_cli.path,
             "{MYPY_ROOT}": ctx.executable._mypy_cli.root.path,
-            "{CACHE_MAP_TRIPLES}": " ".join(_sources_to_cache_map_triples(src_files, is_aspect)),
+            "{CACHE_MAP_TRIPLES}": " ".join(_sources_to_cache_map_triples(input_src_files, is_aspect)),
             "{PACKAGE_ROOTS}": " ".join([
                 "--package-root " + shell.quote(path or ".")
                 for path in src_root_paths
             ]),
             "{SRCS}": " ".join([
                 shell.quote(f.path) if is_aspect else shell.quote(f.short_path)
-                for f in src_files
+                for f in target_src_files
             ]),
             "{VERBOSE_OPT}": "--verbose" if DEBUG else "",
             "{VERBOSE_BASH}": "set -x" if DEBUG else "",
             "{OUTPUT}": out.path if out else "",
             "{MYPYPATH_PATH}": mypypath if mypypath else "",
             "{MYPY_INI_PATH}": mypy_config_file.path,
+            "{FOLLOW_IMPORTS}": follow_imports,
         },
         is_executable = True,
     )
@@ -234,5 +267,9 @@ mypy_test = rule(
     implementation = _mypy_test_impl,
     test = True,
     attrs = dict(DEFAULT_ATTRS.items() +
-                 [("deps", attr.label_list(aspects = [mypy_aspect]))]),
+                 [("deps", attr.label_list(aspects = [mypy_aspect])),
+                  ("include_imports",
+                   attr.bool(doc = "Set to true to include imported Python files for mypy. This is required for use with pip `requirement()` rules.")),
+                 ]
+    ),
 )

mypy/BUILD

@@ -1,12 +1,12 @@
 load("@mypy_integration_pip_deps//:requirements.bzl", "requirement")
+load("@aspect_rules_py//py:defs.bzl", "py_binary")
 
 py_binary(
     name = "mypy",
     srcs = ["main.py"],
     main = "main.py",
-    python_version = "PY3",
     visibility = ["//visibility:public"],
     deps = [
-        requirement("mypy"),
+        "@mypy_integration_pip_deps_mypy//:wheel",
     ],
 )

repositories/deps.bzl

@@ -4,7 +4,7 @@ Provides functions to pull all external package dependencies of this
 repository.
 """
 
-load(":py_repositories.bzl", "py_deps")
+load(":py_repositories.bzl", "py_deps", "mypy_deps")
 
 def deps(mypy_requirements_file, python_interpreter = None, python_interpreter_target = None, extra_pip_args = None):
     """Pull in external dependencies needed by rules in this repo.
@@ -16,4 +16,5 @@ def deps(mypy_requirements_file, python_interpreter = None, python_interpreter_t
     'repositories' in //repositories:repositories.bzl have been imported
     already.
     """
-    py_deps(mypy_requirements_file, python_interpreter, python_interpreter_target, extra_pip_args)
+    # py_deps(mypy_requirements_file, python_interpreter, python_interpreter_target, extra_pip_args)
+    mypy_deps(python_interpreter)

repositories/py_repositories.bzl

@@ -1,9 +1,31 @@
-# NOTE: Once recursive workspaces are implemented in Bazel, this file should cease to exist.
+# WORKSPACE support, to be replaced in the future with bzlmod.
 """
 Provides functions to pull the external Mypy package dependency.
 """
 
-load("@rules_python//python:pip.bzl", "pip_install")
+load("@rules_python//python:pip.bzl", "pip_install", "package_annotation", "pip_parse")
+
+PY_WHEEL_RULE_CONTENT = """\
+load("@aspect_rules_py//py:defs.bzl", "py_wheel")
+py_wheel(
+    name = "wheel",
+    src = ":whl",
+)
+"""
+
+def mypy_deps(interpreter):
+    PACKAGES = ["mypy", "typed-ast"]
+    ANNOTATIONS = {
+        pkg: package_annotation(additive_build_content = PY_WHEEL_RULE_CONTENT)
+        for pkg in PACKAGES
+    }
+
+    pip_parse(
+        name = "mypy_integration_pip_deps",
+        annotations = ANNOTATIONS,
+        python_interpreter_target = interpreter,
+        requirements_lock = "//:requirements-locked.txt",
+    )
 
 # buildifier: disable=function-docstring-args
 def py_deps(mypy_requirements_file, python_interpreter, python_interpreter_target, extra_pip_args):

repositories/repositories.bzl

@@ -1,27 +1,37 @@
 """Rules to load all dependencies of this project."""
 
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")
+
+versions = struct(
+    bazel_skylib = "1.0.2",
+    rules_python = "0.8.1",
+)
 
 # buildifier: disable=function-docstring
 def repositories():
-    excludes = native.existing_rules().keys()
-
-    rules_python_version = "0.1.0"
+    maybe(
+        http_archive,
+        name = "rules_python",
+        url = "https://github.com/bazelbuild/rules_python/archive/{}.tar.gz".format(versions.rules_python),
+        strip_prefix = "rules_python-{}".format(versions.rules_python),
+        sha256 = "cdf6b84084aad8f10bf20b46b77cb48d83c319ebe6458a18e9d2cebf57807cdd",
+    )
 
-    if "rules_python" not in excludes:
-        http_archive(
-            name = "rules_python",
-            url = "https://github.com/bazelbuild/rules_python/archive/{version}.tar.gz".format(version = rules_python_version),
-            strip_prefix = "rules_python-{version}".format(version = rules_python_version),
-            sha256 = "48f7e716f4098b85296ad93f5a133baf712968c13fbc2fdf3a6136158fe86eac",
-        )
+    maybe(
+        http_archive,
+        name = "bazel_skylib",
+        urls = [
+            "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/{0}/bazel-skylib-{0}.tar.gz".format(versions.bazel_skylib),
+            "https://github.com/bazelbuild/bazel-skylib/releases/download/{0}/bazel-skylib-{0}.tar.gz".format(versions.bazel_skylib),
+        ],
+        sha256 = "97e70364e9249702246c0e9444bccdc4b847bed1eb03c5a3ece4f83dfe6abc44",
+    )
 
-    if "bazel_skylib" not in excludes:
-        http_archive(
-            name = "bazel_skylib",
-            urls = [
-                "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.0.2/bazel-skylib-1.0.2.tar.gz",
-                "https://github.com/bazelbuild/bazel-skylib/releases/download/1.0.2/bazel-skylib-1.0.2.tar.gz",
-            ],
-            sha256 = "97e70364e9249702246c0e9444bccdc4b847bed1eb03c5a3ece4f83dfe6abc44",
-        )
+    maybe(
+        http_archive,
+        name = "aspect_rules_py",
+        sha256 = "874c5ae0e763e0326c6f45f3a5d12d00a751fc7c658e02271b6fa89f15f38862",
+        strip_prefix = "rules_py-main",
+        urls = ["https://github.com/aspect-build/rules_py/archive/refs/heads/main.zip"],
+    )