Jules - added tests for #3813 and #3448, minor changes to build diagnostics.

pipcl.py

@@ -1899,7 +1899,16 @@ def git_items( directory, submodules=False):
     return ret
 
 
-def run( command, capture=False, check=1, verbose=1, env_extra=None, caller=1):
+def run(
+        command,
+        *,
+        capture=False,
+        check=1,
+        verbose=1,
+        env_extra=None,
+        timeout=None,
+        caller=1,
+        ):
     '''
     Runs a command using `subprocess.run()`.
 
@@ -1923,6 +1932,10 @@ def run( command, capture=False, check=1, verbose=1, env_extra=None, caller=1):
             If true we show the command.
         env_extra:
             None or dict to add to environ.
+        timeout:
+            If not None, timeout in seconds; passed directly to
+            subprocess.run(). Note that on MacOS subprocess.run() seems to
+            leave processes running if timeout expires.
     Returns:
         check capture   Return
         --------------------------
@@ -1949,6 +1962,7 @@ def run( command, capture=False, check=1, verbose=1, env_extra=None, caller=1):
             check=check,
             encoding='utf8',
             env=env,
+            timeout=timeout,
             )
     if check:
         return cp.stdout if capture else None

scripts/gh_release.py

@@ -89,6 +89,14 @@
 
 pymupdf_dir = os.path.abspath( f'{__file__}/../..')
 
+sys.path.insert(0, pymupdf_dir)
+import pipcl
+del sys.path[0]
+
+log = pipcl.log0
+run = pipcl.run
+
+
 def main():
 
     log( '### main():')
@@ -586,49 +594,6 @@ def relpath(path, start=None):
         return os.path.relpath(path, start)
 
 
-def log(text, caller=0):
-    '''
-    Writes `text` to stdout with prefix showing caller path relative to
-    pymupdf_dir and fn name.
-    '''
-    frame_record = inspect.stack( context=0)[ caller+1]
-    filename    = frame_record.filename
-    line        = frame_record.lineno
-    function    = frame_record.function
-    prefix = f'{relpath(filename, pymupdf_dir)}:{line}:{function}(): '
-    print(textwrap.indent(text, prefix), flush=1)
-
-
-def run(command, env_extra=None, check=1, timeout=None):
-    '''
-    Runs a command using subprocess.run().
-    Args:
-        command:
-            The command to run.
-        env_extra:
-            None or dict containing extra environment variable settings to add
-            to os.environ.
-        check:
-            Whether to raise exception if command fails.
-        timeout:
-            If not None, timeout in seconds; passed directory to
-            subprocess.run(). Note that on MacOS subprocess.run() seems to
-            leave processes running if timeout expires.
-    '''
-    env = None
-    message = 'Running: '
-    if env_extra:
-        env = os.environ.copy()
-        env.update(env_extra)
-        message += '\n[environment:\n'
-        for n, v in env_extra.items():
-            message += f'    {n}={shlex.quote(v)}\n'
-        message += ']\n'
-    message += f'{command}'
-    log(message, caller=1)
-    return subprocess.run(command, check=check, shell=1, env=env, timeout=timeout)
-
-
 def platform_tag():
     bits = cpu_bits()
     if platform.system() == 'Windows':

scripts/sysinstall.py

@@ -86,6 +86,14 @@
 
 import test as test_py
 
+pymupdf_dir = os.path.abspath( f'{__file__}/../..')
+
+sys.path.insert(0, pymupdf_dir)
+import pipcl
+del sys.path[0]
+
+log = pipcl.log0
+
 # Requirements for a system build and install:
 #
 # system packages (Debian names):
@@ -106,12 +114,12 @@
 def main():
 
     if 1:
-        print(f'## {__file__}: Starting.')
-        print(f'{sys.executable=}')
-        print(f'{platform.python_version()=}')
-        print(f'{__file__=}')
-        print(f'{sys.argv=}')
-        print(f'{sysconfig.get_path("platlib")=}')
+        log(f'## {__file__}: Starting.')
+        log(f'{sys.executable=}')
+        log(f'{platform.python_version()=}')
+        log(f'{__file__=}')
+        log(f'{sys.argv=}')
+        log(f'{sysconfig.get_path("platlib")=}')
         run_command(f'python -V', check=0)
         run_command(f'python3 -V', check=0)
         run_command(f'sudo python -V', check=0)
@@ -132,7 +140,6 @@ def main():
     packages = True
     prefix = '/usr/local'
     pymupdf_do = True
-    pymupdf_dir = os.path.abspath( f'{__file__}/../..')
     root = 'sysinstall_test'
     tesseract5 = True
     pytest_args = None
@@ -152,7 +159,7 @@ def main():
         except StopIteration:
             break
         if arg in ('-h', '--help'):
-            print(__doc__)
+            log(__doc__)
             return
         elif arg == '--mupdf-do':       mupdf_do = int(next(args))
         elif arg == '--mupdf-dir':      mupdf_dir = next(args)
@@ -161,7 +168,6 @@ def main():
         elif arg == '--packages':       packages = int(next(args))
         elif arg == '--prefix':         prefix = next(args)
         elif arg == '--pymupdf-do':     pymupdf_do = int(next(args))
-        elif arg == '--pymupdf-dir':    pymupdf_dir = next(args)
         elif arg == '--root':           root = next(args)
         elif arg == '--tesseract5':     tesseract5 = int(next(args))
         elif arg == '--pytest-do':      pytest_do = int(next(args))
@@ -191,23 +197,23 @@ def run(command):
     if mupdf_git:
         # Update existing checkout or do `git clone`.
         if os.path.exists(mupdf_dir):
-            print(f'## Update MuPDF checkout {mupdf_dir}.')
+            log(f'## Update MuPDF checkout {mupdf_dir}.')
             run(f'cd {mupdf_dir} && git pull && git submodule update --init')
         else:
             # No existing git checkout, so do a fresh clone.
-            print(f'## Clone MuPDF into {mupdf_dir}.')
+            log(f'## Clone MuPDF into {mupdf_dir}.')
             run(f'git clone --recursive --depth 1 --shallow-submodules {mupdf_git} {mupdf_dir}')
 
     if packages:
         # Install required system packages. We assume a Debian package system.
         #
-        print('## Install system packages required by MuPDF.')
+        log('## Install system packages required by MuPDF.')
         run(f'sudo apt update')
         run(f'sudo apt install {" ".join(g_sys_packages)}')
         # Ubuntu-22.04 has freeglut3-dev, not libglut-dev.
         run(f'sudo apt install libglut-dev | sudo apt install freeglut3-dev')
         if tesseract5:
-            print(f'## Force installation of libtesseract-dev version 5.')
+            log(f'## Force installation of libtesseract-dev version 5.')
             # https://stackoverflow.com/questions/76834972/how-can-i-run-pytesseract-python-library-in-ubuntu-22-04
             #
             run('sudo apt install -y software-properties-common')
@@ -220,12 +226,12 @@ def run(command):
     # Build+install MuPDF. We use mupd:Makefile's install-shared-python target.
     #
     if pip == 'sudo':
-        print('## Installing Python packages required for building MuPDF and PyMuPDF.')
+        log('## Installing Python packages required for building MuPDF and PyMuPDF.')
         run(f'sudo pip install --upgrade pip')
         names = test_py.wrap_get_requires_for_build_wheel(f'{__file__}/../..')
         run(f'sudo pip install {names}')
 
-    print('## Build and install MuPDF.')
+    log('## Build and install MuPDF.')
     command = f'cd {mupdf_dir}'
     command += f' && {sudo}make'
     command += f' -j {multiprocessing.cpu_count()}'
@@ -246,10 +252,10 @@ def run(command):
 
     # Build+install PyMuPDF.
     #
-    print('## Build and install PyMuPDF.')
+    log('## Build and install PyMuPDF.')
     def run(command):
         return run_command(command, doit=pymupdf_do)
-    flags_freetype2 = run_command('pkg-config --cflags freetype2', capture_output=1).stdout.strip()
+    flags_freetype2 = run_command('pkg-config --cflags freetype2', capture=1)
     compile_flags = f'-I {root_prefix}/include {flags_freetype2}'
     link_flags = f'-L {root_prefix}/lib'
     env = ''
@@ -258,7 +264,7 @@ def run(command):
     env += f'LDFLAGS="-L {root}/{prefix}/lib" '
     env += f'PYMUPDF_SETUP_MUPDF_BUILD= '       # Use system MuPDF.
     if use_installer:
-        print(f'## Building wheel.')
+        log(f'## Building wheel.')
         if pip == 'venv':
             venv_name = 'venv-pymupdf-sysinstall'
         run(f'pwd')
@@ -277,7 +283,7 @@ def run(command):
         wheel = glob.glob(f'dist/*')
         assert len(wheel) == 1, f'{wheel=}'
         wheel = wheel[0]
-        print(f'## Installing wheel using `installer`.')
+        log(f'## Installing wheel using `installer`.')
         pv = '.'.join(platform.python_version_tuple()[:2])
         p = f'{root_prefix}/lib/python{pv}'
         # `python -m installer` fails to overwrite existing files.
@@ -315,13 +321,10 @@ def run(command):
                     if leaf in dirnames:
                         pythonpath.append(os.path.join(dirpath, leaf))
         pythonpath = ':'.join(pythonpath)
-        print(f'{pythonpath=}')
+        log(f'{pythonpath=}')
     else:
         command = f'{env} pip install -vv --root {root} {os.path.abspath(pymupdf_dir)}'
         run( command)
-        sys.path.insert(0, pymupdf_dir)
-        import pipcl
-        del sys.path[0]
         pythonpath = pipcl.install_dir(root)
 
     # Show contents of installation directory. This is very slow on github,
@@ -330,7 +333,7 @@ def run(command):
 
     # Run pytest tests.
     #
-    print('## Run PyMuPDF pytest tests.')
+    log('## Run PyMuPDF pytest tests.')
     def run(command):
         return run_command(command, doit=pytest_do)
     import gh_release
@@ -387,13 +390,11 @@ def run(command):
     run(command)
 
 
-def run_command(command, capture_output=False, check=True, doit=True):
+def run_command(command, capture=False, check=True, doit=True):
     if doit:
-        print(f'## Running: {command}')
-        sys.stdout.flush()
-        return subprocess.run(command, shell=1, check=check, text=1, capture_output=capture_output)
+        return pipcl.run(command, capture=capture, check=check, caller=2)
     else:
-        print(f'## Would have run: {command}')
+        log(f'## Would have run: {command}', caller=2)
 
 
 if __name__ == '__main__':

scripts/test.py

@@ -135,6 +135,13 @@
 
 pymupdf_dir = os.path.abspath( f'{__file__}/../..')
 
+sys.path.insert(0, pymupdf_dir)
+import pipcl
+del sys.path[0]
+
+log = pipcl.log0
+run = pipcl.run
+
 
 def main(argv):
 
@@ -402,7 +409,7 @@ def build(
             if venv_quick:
                 log(f'{venv_quick=}: Not installing packages with pip: {names}')
             else:
-                gh_release.run( f'python -m pip install --upgrade {names}')
+                run( f'python -m pip install --upgrade {names}')
         build_isolation_text = ' --no-build-isolation'
 
     env_extra = dict()
@@ -413,9 +420,9 @@ def build(
     if build_flavour:
         env_extra['PYMUPDF_SETUP_FLAVOUR'] = build_flavour
     if wheel:
-        gh_release.run(f'pip wheel{build_isolation_text} -v {pymupdf_dir}', env_extra=env_extra)
+        run(f'pip wheel{build_isolation_text} -v {pymupdf_dir}', env_extra=env_extra)
     else:
-        gh_release.run(f'pip install{build_isolation_text} -v {pymupdf_dir}', env_extra=env_extra)
+        run(f'pip install{build_isolation_text} -v {pymupdf_dir}', env_extra=env_extra)
 
 
 def build_pyodide_wheel(pyodide_build_version=None):
@@ -452,14 +459,14 @@ def build_pyodide_wheel(pyodide_build_version=None):
     env_extra['PYMUPDF_SETUP_MUPDF_TESSERACT'] = '0'
     setup = pyodide_setup(pymupdf_dir, pyodide_build_version=pyodide_build_version)
     command = f'{setup} && pyodide build --exports pyinit'
-    gh_release.run(command, env_extra=env_extra)
+    run(command, env_extra=env_extra)
 
     # Copy wheel into `wheelhouse/` so it is picked up as a workflow
     # artifact.
     #
-    gh_release.run(f'ls -l {pymupdf_dir}/dist/')
-    gh_release.run(f'mkdir -p {pymupdf_dir}/wheelhouse && cp -p {pymupdf_dir}/dist/* {pymupdf_dir}/wheelhouse/')
-    gh_release.run(f'ls -l {pymupdf_dir}/wheelhouse/')    
+    run(f'ls -l {pymupdf_dir}/dist/')
+    run(f'mkdir -p {pymupdf_dir}/wheelhouse && cp -p {pymupdf_dir}/dist/* {pymupdf_dir}/wheelhouse/')
+    run(f'ls -l {pymupdf_dir}/wheelhouse/')    
 
 
 def pyodide_setup(
@@ -655,7 +662,7 @@ def test(
         if venv_quick:
             log(f'{venv_quick=}: Not installing test packages: {gh_release.test_packages}')
         else:
-            gh_release.run(f'pip install --upgrade {gh_release.test_packages}')
+            run(f'pip install --upgrade {gh_release.test_packages}')
         run_compound_args = ''
         if implementations:
             run_compound_args += f' -i {implementations}'
@@ -664,9 +671,9 @@ def test(
         env_extra = None
         if valgrind:
             log('Installing valgrind.')
-            gh_release.run(f'sudo apt update')
-            gh_release.run(f'sudo apt install --upgrade valgrind')
-            gh_release.run(f'valgrind --version')
+            run(f'sudo apt update')
+            run(f'sudo apt install --upgrade valgrind')
+            run(f'valgrind --version')
 
             log('Running PyMuPDF tests under valgrind.')
             command = (
@@ -714,7 +721,7 @@ def test(
                     f.write(text2)
 
         log(f'Running tests with tests/run_compound.py and pytest.')
-        gh_release.run(command, env_extra=env_extra, timeout=timeout)
+        run(command, env_extra=env_extra, timeout=timeout)
 
     except subprocess.TimeoutExpired as e:
          log(f'Timeout when running tests.')
@@ -768,10 +775,6 @@ def wrap_get_requires_for_build_wheel(dir_):
     return ' '.join(ret)
 
 
-def log(text):
-    gh_release.log(text, caller=1)
-
-
 if __name__ == '__main__':
     try:
         sys.exit(main(sys.argv))