rename possibly identical modules to prevent test collisions

.github/workflows/ci.yaml

@@ -298,8 +298,10 @@ jobs:
         CIBW_MUSLLINUX_AARCH64_IMAGE: ${{ matrix.musllinux_img || 'musllinux_1_1' }}
         CIBW_PRERELEASE_PYTHONS: 'True'
         CIBW_FREE_THREADED_SUPPORT: 'True'
-        CIBW_TEST_REQUIRES: pytest setuptools  # 3.12+ no longer includes distutils, just always ensure setuptools is present
-        CIBW_TEST_COMMAND: PYTHONUNBUFFERED=1 python -m pytest ${{ matrix.test_args || '{project}' }}  # default to test all
+        # 3.12+ no longer includes distutils, just always ensure setuptools is present
+        CIBW_TEST_REQUIRES: pytest setuptools pytest-xdist filelock
+        # default to test all
+        CIBW_TEST_COMMAND: PYTHONUNBUFFERED=1 python -m pytest ${{ matrix.test_args || '{project}' }}  -n 4
       run: |
         set -eux
 
@@ -417,8 +419,8 @@ jobs:
       env:
         CIBW_BUILD: ${{ matrix.spec }}
         CIBW_PRERELEASE_PYTHONS: 'True'
-        CIBW_TEST_REQUIRES: pytest setuptools
-        CIBW_TEST_COMMAND: pip install pip --upgrade; cd {project}; PYTHONUNBUFFERED=1 pytest
+        CIBW_TEST_REQUIRES: pytest setuptools pytest-xdist filelock
+        CIBW_TEST_COMMAND: pip install pip --upgrade; cd {project}; PYTHONUNBUFFERED=1 pytest -n 4
         MACOSX_DEPLOYMENT_TARGET: ${{ matrix.deployment_target || '10.9' }}
         SDKROOT: ${{ matrix.sdkroot || 'macosx' }}
       run: |
@@ -508,10 +510,8 @@ jobs:
       env:
         CIBW_BUILD: ${{ matrix.spec }}
         CIBW_PRERELEASE_PYTHONS: 'True'
-        CIBW_TEST_REQUIRES: pytest setuptools
-        CIBW_TEST_COMMAND: 'python -m pytest {package}/src/c'
-        # FIXME: /testing takes ~45min on Windows and has some failures...
-        # CIBW_TEST_COMMAND='python -m pytest {package}/src/c {project}/testing'
+        CIBW_TEST_REQUIRES: pytest setuptools pytest-xdist filelock
+        CIBW_TEST_COMMAND: 'python -m pytest {package} -n 4'
       run: |
         set -eux
 

doc/source/installation.rst

@@ -42,9 +42,10 @@ Requirements:
 * pycparser >= 2.06: https://github.com/eliben/pycparser (automatically
   tracked by ``pip install cffi``).
 
-* `pytest`_ is needed to run the tests of CFFI itself.
+* `pytest`_ and `filelock` are needed to run the tests of CFFI itself.
 
 .. _`pytest`: http://pypi.python.org/pypi/pytest
+.. _`filelock`: http://pypi.python.org/pypi/filelock
 
 Download and Installation:
 
@@ -54,8 +55,7 @@ Download and Installation:
   ``git clone https://github.com/python-cffi/cffi``
 
 * running the tests: ``pytest  c/  testing/`` (if you didn't
-  install cffi yet, you need first ``python setup_base.py build_ext -f
-  -i``)
+  install cffi yet, you need first ``python -m pip install -e .``)
 
 .. _`GitHub`: https://github.com/python-cffi/cffi
 

src/cffi/verifier.py

@@ -1,7 +1,7 @@
 #
 # DEPRECATED: implementation for ffi.verify()
 #
-import sys, os, binascii, shutil, io
+import sys, os, binascii, shutil, io, threading
 from . import __version_verifier_modules__
 from . import ffiplatform
 from .error import VerificationError
@@ -197,8 +197,20 @@ def _write_source(self, file=None):
 
     def _compile_module(self):
         # compile this C source
+        # Note: compilation will create artifacts in tmpdir + sourcefilename
+        # This can exceed the windows MAXPATH quite easily. To make it shorter,
+        # cd into tmpdir and make the sourcefilename relative to tmdir
         tmpdir = os.path.dirname(self.sourcefilename)
-        outputfilename = ffiplatform.compile(tmpdir, self.get_extension())
+        olddir = os.getcwd()
+        os.chdir(tmpdir)
+        self.sourcefilename_orig = self.sourcefilename
+        try:
+            self.sourcefilename = os.path.relpath(self.sourcefilename)
+            output_rel_filename = ffiplatform.compile(tmpdir, self.get_extension())
+            outputfilename = os.path.join(tmpdir, output_rel_filename)
+        finally:
+            os.chdir(olddir)
+            self.sourcefilename = self.sourcefilename_orig
         try:
             same = ffiplatform.samefile(outputfilename, self.modulefilename)
         except OSError:
@@ -215,12 +227,13 @@ def _load_library(self):
         else:
             return self._vengine.load_library()
 
+local = threading.local()
 # ____________________________________________________________
 
-_FORCE_GENERIC_ENGINE = False      # for tests
+local._FORCE_GENERIC_ENGINE = False      # for tests
 
 def _locate_engine_class(ffi, force_generic_engine):
-    if _FORCE_GENERIC_ENGINE:
+    if local._FORCE_GENERIC_ENGINE:
         force_generic_engine = True
     if not force_generic_engine:
         if '__pypy__' in sys.builtin_module_names:
@@ -241,11 +254,11 @@ def _locate_engine_class(ffi, force_generic_engine):
 
 # ____________________________________________________________
 
-_TMPDIR = None
+local._TMPDIR = None
 
 def _caller_dir_pycache():
-    if _TMPDIR:
-        return _TMPDIR
+    if local._TMPDIR:
+        return local._TMPDIR
     result = os.environ.get('CFFI_TMPDIR')
     if result:
         return result
@@ -255,8 +268,7 @@ def _caller_dir_pycache():
 
 def set_tmpdir(dirname):
     """Set the temporary directory to use instead of __pycache__."""
-    global _TMPDIR
-    _TMPDIR = dirname
+    local._TMPDIR = dirname
 
 def cleanup_tmpdir(tmpdir=None, keep_so=False):
     """Clean up the temporary directory by removing all files in it

testing/cffi0/test_ownlib.py

@@ -141,10 +141,15 @@ def setup_class(cls):
                 return
             # try (not too hard) to find the version used to compile this python
             # no mingw
-            from distutils.msvc9compiler import get_build_version
-            version = get_build_version()
-            toolskey = "VS%0.f0COMNTOOLS" % version
-            toolsdir = os.environ.get(toolskey, None)
+            toolsdir = None
+            try:
+                # This will always fail on setuptools>73 which removes msvc9compiler
+                from distutils.msvc9compiler import get_build_version
+                version = get_build_version()
+                toolskey = "VS%0.f0COMNTOOLS" % version
+                toolsdir = os.environ.get(toolskey, None)
+            except Exception:
+                pass
             if toolsdir is None:
                 return
             productdir = os.path.join(toolsdir, os.pardir, os.pardir, "VC")

testing/cffi0/test_verify.py

@@ -1,6 +1,6 @@
 import re
 import pytest
-import sys, os, math, weakref
+import sys, os, math, weakref, random
 from cffi import FFI, VerificationError, VerificationMissing, model, FFIError
 from testing.support import *
 from testing.support import extra_compile_args, is_musl
@@ -18,15 +18,24 @@
     if distutils.ccompiler.get_default_compiler() == 'msvc':
         lib_m = ['msvcrt']
     pass      # no obvious -Werror equivalent on MSVC
+    class FFI(FFI):
+        def verify(self, *args, **kwds):
+            modulename = kwds.get("modulename", "_cffi_test%d_%d" % (
+                random.randint(0,1000000000), random.randint(0,1000000000)))
+            kwds["modulename"] = modulename
+            return super(FFI, self).verify(*args, **kwds)
+
 else:
     class FFI(FFI):
         def verify(self, *args, **kwds):
+            modulename = kwds.get("modulename", "_cffi_test%d_%d" % (
+                random.randint(0,1000000000), random.randint(0,1000000000)))
+            kwds["modulename"] = modulename
             return super(FFI, self).verify(
                 *args, extra_compile_args=extra_compile_args, **kwds)
 
 def setup_module():
     import cffi.verifier
-    cffi.verifier.cleanup_tmpdir()
     #
     # check that no $ sign is produced in the C file; it used to be the
     # case that anonymous enums would produce '$enum_$1', which was
@@ -52,10 +61,12 @@ def test_module_type():
     ffi = FFI()
     lib = ffi.verify()
     if hasattr(lib, '_cffi_python_module'):
-        print('verify got a PYTHON module')
+        pass
+        # print('verify got a PYTHON module')
     if hasattr(lib, '_cffi_generic_module'):
-        print('verify got a GENERIC module')
-    expected_generic = (cffi.verifier._FORCE_GENERIC_ENGINE or
+        pass
+        # print('verify got a GENERIC module')
+    expected_generic = (cffi.verifier.local._FORCE_GENERIC_ENGINE or
                         '__pypy__' in sys.builtin_module_names)
     assert hasattr(lib, '_cffi_python_module') == (not expected_generic)
     assert hasattr(lib, '_cffi_generic_module') == expected_generic
@@ -2164,21 +2175,23 @@ def test_verify_dlopen_flags():
     ffi1 = FFI()
     ffi1.cdef("extern int foo_verify_dlopen_flags;")
 
+    modulename = "_cffi_test_verify_dlopen_flags_%d" % random.randint(0, 1000000000)
     lib1 = ffi1.verify("int foo_verify_dlopen_flags;",
-                       flags=ffi1.RTLD_GLOBAL | ffi1.RTLD_LAZY)
-    lib2 = get_second_lib()
+                       flags=ffi1.RTLD_GLOBAL | ffi1.RTLD_LAZY,
+                       modulename=modulename)
+    lib2 = get_second_lib(modulename)
 
     lib1.foo_verify_dlopen_flags = 42
     assert lib2.foo_verify_dlopen_flags == 42
     lib2.foo_verify_dlopen_flags += 1
     assert lib1.foo_verify_dlopen_flags == 43
 
-def get_second_lib():
-    # Hack, using modulename makes the test fail
+def get_second_lib(modulename):
     ffi2 = FFI()
     ffi2.cdef("extern int foo_verify_dlopen_flags;")
     lib2 = ffi2.verify("int foo_verify_dlopen_flags;",
-                       flags=ffi2.RTLD_GLOBAL | ffi2.RTLD_LAZY)
+                       flags=ffi2.RTLD_GLOBAL | ffi2.RTLD_LAZY,
+                       modulename=modulename)
     return lib2
 
 def test_consider_not_implemented_function_type():
@@ -2246,7 +2259,7 @@ def test_implicit_unicode_on_windows():
 
 def test_use_local_dir():
     ffi = FFI()
-    lib = ffi.verify("", modulename="test_use_local_dir")
+    lib = ffi.verify("", modulename="_cffi_test_use_local_dir")
     this_dir = os.path.dirname(__file__)
     pycache_files = os.listdir(os.path.join(this_dir, '__pycache__'))
     assert any('test_use_local_dir' in s for s in pycache_files)
@@ -2585,3 +2598,20 @@ def test_passing_large_list():
     arg = list(range(20000000))
     lib.passing_large_list(arg)
     # assert did not segfault
+
+def test_no_regen():
+    from cffi.verifier import Verifier, _caller_dir_pycache
+    import os
+    ffi = FFI()
+    modulename = "_cffi_test_no_regen_%d" % random.randint(0, 1000000000)
+    ffi.cdef("double sin(double x);")
+    lib = ffi.verify('#include <math.h>', libraries=lib_m, modulename=modulename)
+    assert lib.sin(1.23) == math.sin(1.23)
+    # Make sure that recompiling the same code does not rebuild the C file
+    cfile = os.path.join(ffi.verifier.tmpdir, f"{modulename}.c")
+    assert os.path.exists(cfile)
+    os.unlink(cfile)
+    assert not os.path.exists(cfile)
+    lib = ffi.verify('#include <math.h>', libraries=lib_m, modulename=modulename)
+    assert lib.sin(1.23) == math.sin(1.23)
+    assert not os.path.exists(cfile)

testing/cffi0/test_vgen.py

@@ -3,10 +3,9 @@
 
 
 def setup_module():
-    cffi.verifier.cleanup_tmpdir()
-    cffi.verifier._FORCE_GENERIC_ENGINE = True
+    cffi.verifier.local._FORCE_GENERIC_ENGINE = True
     # Runs all tests with _FORCE_GENERIC_ENGINE = True, to make sure we
     # also test vengine_gen.py.
 
 def teardown_module():
-    cffi.verifier._FORCE_GENERIC_ENGINE = False
+    cffi.verifier.local._FORCE_GENERIC_ENGINE = False

testing/cffi0/test_zdistutils.py

@@ -90,7 +90,7 @@ def test_compile_module_explicit_filename(self):
         csrc = '/*hi there %s!2*/\n#include <math.h>\n' % self
         v = Verifier(ffi, csrc, force_generic_engine=self.generic,
                      libraries=[self.lib_m])
-        basename = self.__class__.__name__[:10] + '_test_compile_module'
+        basename = self.__class__.__name__[:20] + '_test_compile_module'
         v.modulefilename = filename = str(udir.join(basename + '.so'))
         v.compile_module()
         assert filename == v.modulefilename

testing/cffi1/test_parse_c_type.py

@@ -2,6 +2,7 @@
 import pytest
 import cffi
 from cffi import cffi_opcode
+from filelock import FileLock
 from pathlib import Path
 
 if '__pypy__' in sys.builtin_module_names:
@@ -26,14 +27,21 @@
 ffi = cffi.FFI()
 ffi.cdef(header)
 
-with open(os.path.join(cffi_dir, '..', 'c', 'parse_c_type.c')) as _body:
-    body = _body.read()
-lib = ffi.verify(
-    body + """
-static const char *get_common_type(const char *search, size_t search_len) {
-    return NULL;
-}
-""",    include_dirs=[cffi_dir])
+
+def build_lib():
+    sourcename = os.path.join(cffi_dir, '..', 'c', 'parse_c_type.c')
+    with FileLock(sourcename + ".lock"):
+        with open(sourcename) as _body:
+            body = _body.read()
+        lib = ffi.verify(
+            body + """
+        static const char *get_common_type(const char *search, size_t search_len) {
+            return NULL;
+        }
+        """,    include_dirs=[cffi_dir])
+    return lib
+
+lib = build_lib()
 
 class ParseError(Exception):
     pass

testing/embedding/__init__.py

@@ -0,0 +1,5 @@
+import sys
+import pytest
+
+if sys.platform == "win32":
+    pytest.skip("XXX fixme", allow_module_level=True)