From c623ec6660301c52f4778b6561772d4717eb1341 Mon Sep 17 00:00:00 2001 From: Julian Smith Date: Mon, 28 Aug 2023 17:38:19 +0100 Subject: [PATCH] scripts/gh_release.py: added support for building Pyodide wheels. --- scripts/gh_release.py | 205 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 180 insertions(+), 25 deletions(-) diff --git a/scripts/gh_release.py b/scripts/gh_release.py index bdbb2fe62..ac10f9ca3 100755 --- a/scripts/gh_release.py +++ b/scripts/gh_release.py @@ -36,13 +36,21 @@ inputs_skeleton Build minimal wheel; for testing only. inputs_sdist - inputs_wheels + inputs_wheels_default inputs_wheels_linux_aarch64 inputs_wheels_linux_auto + inputs_wheels_linux_pyodide inputs_wheels_macos_arm64 inputs_wheels_macos_auto inputs_wheels_windows_auto +Buiding for Pyodide + + If `inputs_wheels_linux_pyodide` is true and we are on Linux, we clone + `emsdk.git`, set it up, and run `pyodide build`. This runs our setup.py + with CC etc set up to create Pyodide binaries in a wheel called, for + example, `PyMuPDF-1.23.2-cp311-none-emscripten_3_1_32_wasm32.whl`. + Example usage: PYMUPDF_SETUP_MUPDF_BUILD=../mupdf-master py -3.9-32 PyMuPDF/scripts/gh_release.py venv build-devel @@ -56,6 +64,7 @@ import shlex import sys import subprocess +import textwrap def main(): @@ -147,37 +156,34 @@ def get_bool(name, default=0): inputs_flavours = get_bool('inputs_flavours', 1) inputs_sdist = get_bool('inputs_sdist') inputs_skeleton = os.environ.get('inputs_skeleton') - inputs_wheels = get_bool('inputs_wheels') - inputs_wheels_linux_aarch64 = get_bool('inputs_wheels_linux_aarch64') - inputs_wheels_linux_auto = get_bool('inputs_wheels_linux_auto', 1) - inputs_wheels_macos_arm64 = get_bool('inputs_wheels_macos_arm64') - inputs_wheels_macos_auto = get_bool('inputs_wheels_macos_auto', 1) - inputs_wheels_windows_auto = get_bool('inputs_wheels_windows_auto', 1) + inputs_wheels_default = get_bool('inputs_wheels_default', 1) + inputs_wheels_linux_aarch64 = get_bool('inputs_wheels_linux_aarch64', inputs_wheels_default) + inputs_wheels_linux_auto = get_bool('inputs_wheels_linux_auto', inputs_wheels_default) + inputs_wheels_linux_pyodide = get_bool('inputs_wheels_linux_pyodide', 0) + inputs_wheels_macos_arm64 = get_bool('inputs_wheels_macos_arm64', inputs_wheels_default) + inputs_wheels_macos_auto = get_bool('inputs_wheels_macos_auto', inputs_wheels_default) + inputs_wheels_windows_auto = get_bool('inputs_wheels_windows_auto', inputs_wheels_default) inputs_wheels_cps = os.environ.get('inputs_wheels_cps') - #inputs_wheels_cp38 = get_bool('inputs_wheels_cp38', 1) - #inputs_wheels_cp39 = get_bool('inputs_wheels_cp39', 1) - #inputs_wheels_cp310 = get_bool('inputs_wheels_cp310', 1) - #inputs_wheels_cp311 = get_bool('inputs_wheels_cp311', 1) - log( f'{inputs_flavours=}') log( f'{inputs_sdist=}') log( f'{inputs_skeleton=}') - log( f'{inputs_wheels=}') + log( f'{inputs_wheels_default=}') log( f'{inputs_wheels_linux_aarch64=}') log( f'{inputs_wheels_linux_auto=}') + log( f'{inputs_wheels_linux_pyodide=}') log( f'{inputs_wheels_macos_arm64=}') log( f'{inputs_wheels_macos_auto=}') log( f'{inputs_wheels_windows_auto=}') log( f'{inputs_wheels_cps=}') - #log( f'{inputs_wheels_cp38=}') - #log( f'{inputs_wheels_cp39=}') - #log( f'{inputs_wheels_cp310=}') - #log( f'{inputs_wheels_cp311=}') - - # Build + # Build Pyodide wheel if specified. + # + if platform.system() == 'Linux' and inputs_wheels_linux_pyodide: + build_pyodide_wheel() + # Build + # env_extra = dict() def set_if_unset(name, value): @@ -269,12 +275,6 @@ def set_cibuild_test(): else: env_set('CIBW_TEST_REQUIRES', 'fontTools pytest') env_set('CIBW_TEST_COMMAND', 'python {project}/tests/run_compound.py pytest -s {project}/tests') - - # Don't attempt to run tests on cross-built ARM wheels. - # - # https://cibuildwheel.readthedocs.io/en/stable/options/#test-skip - # - #env_set('CIBW_TEST_SKIP', 'manylinux*_aarch64 macos*_arm64') pymupdf_dir = os.path.abspath( f'{__file__}/../..') if pymupdf_dir != os.path.abspath( os.getcwd()): @@ -331,9 +331,47 @@ def set_cibuild_test(): env_set( 'PYMUPDF_SETUP_FLAVOUR', 'pb', pass_=1) run( f'cibuildwheel{platform_arg}', env_extra=env_extra) + run( 'ls -lt wheelhouse') +def build_pyodide_wheel(): + ''' + Build Pyodide wheel. + + This does not use cibuildwheel but instead runs `pyodide build` inside + the PyMuPDF directory, which in turn runs setup.py in a Pyodide build + environment. + ''' + log(f'## Building Pyodide wheel.') + + # Our setup.py does not know anything about Pyodide; we set a few + # required environmental variables here. + # + env_extra = dict() + + # Disable libcrypto because not available in Pyodide. + env_extra['HAVE_LIBCRYPTO'] = 'no' + + # Set OS=wasm-mt for MuPDF build. + env_extra['OS'] = 'wasm-mt' + + # Build PyMuPDF as a single wheel without a separate PyMuPDFb + # wheel. + env_extra['PYMUPDF_SETUP_IMPLEMENTATIONS'] = 'a' + + command = pyodide_setup() + command += ' && pyodide build --exports pyinit' + run(command, env_extra=env_extra) + + # Copy wheel into `wheelhouse/` so it is picked up as a workflow + # artifact. + # + run('ls -l dist/') + run('mkdir -p wheelhouse && cp -p dist/* wheelhouse/') + run('ls -l wheelhouse/') + + def venv( command=None, packages=None): ''' Runs remaining args, or the specified command if present, in a venv. @@ -403,6 +441,123 @@ def test( project, package): #run( f'pip install {wheel_p}') +def pyodide_setup(clean=False): + ''' + Returns a command that will set things up for a pyodide build. + + Args: + clean: + If true we create an entirely new environment. Otherwise + we reuse any existing emsdk repository and venv. + + * Clone emsdk repository to `pipcl_emsdk` if not already present. + * Create and activate a venv `pipcl_venv_pyodide` if not already present. + * Install/upgrade package `pyodide-build`. + * Run emsdk install scripts and enter emsdk environment. + * Replace emsdk/upstream/bin/wasm-opt + (https://github.com/pyodide/pyodide/issues/4048). + + Example usage in a build function: + + command = pipcl_wasm.pyodide_setup() + command += ' && pyodide build --exports pyinit' + subprocess.run(command, shell=1, check=1) + ''' + command = 'true' + + # Clone emsdk. + # + dir_emsdk = 'emsdk' + if clean and os.path.exists(dir_emsdk): + shutil.rmtree( dir_emsdk, ignore_errors=1) + if not os.path.exists(dir_emsdk): + command += f' && echo "### cloning emsdk.git"' + command += f' && git clone https://github.com/emscripten-core/emsdk.git {dir_emsdk}' + + # Create and enter Python venv. + # + venv_pyodide = 'venv_pyodide' + if not os.path.exists( venv_pyodide): + command += f' && echo "### creating venv {venv_pyodide}"' + command += f' && {sys.executable} -m venv {venv_pyodide}' + command += f' && . {venv_pyodide}/bin/activate' + command += f' && echo "### running pip install ..."' + command += f' && python -m pip install --upgrade pip wheel pyodide-build' + + # Run emsdk install scripts and enter emsdk environment. + # + command += f' && cd {dir_emsdk}' + command += ' && PYODIDE_EMSCRIPTEN_VERSION=$(pyodide config get emscripten_version)' + command += ' && echo "### running ./emsdk install"' + command += ' && ./emsdk install ${PYODIDE_EMSCRIPTEN_VERSION}' + command += ' && echo "### running ./emsdk activate"' + command += ' && ./emsdk activate ${PYODIDE_EMSCRIPTEN_VERSION}' + command += ' && echo "### running ./emsdk_env.sh"' + command += ' && . ./emsdk_env.sh' # Need leading `./` otherwise weird 'Not found' error. + + if 1: + # Make our returned command replace emsdk/upstream/bin/wasm-opt + # with a script that does nothing, otherwise the linker + # command fails after it has created the output file. See: + # https://github.com/pyodide/pyodide/issues/4048 + # + + def write( text, path): + with open( path, 'w') as f: + f.write( text) + os.chmod( path, 0o755) + + # Create a script that our command runs, that overwrites + # `emsdk/upstream/bin/wasm-opt`, hopefully in a way that is + # idempotent. + # + # The script moves the original wasm-opt to wasm-opt-0. + # + write( + textwrap.dedent(''' + #! /usr/bin/env python3 + import os + p = 'upstream/bin/wasm-opt' + p0 = 'upstream/bin/wasm-opt-0' + p1 = '../wasm-opt-1' + if os.path.exists( p0): + print(f'### {__file__}: {p0!r} already exists so not overwriting from {p!r}.') + else: + s = os.stat( p) + assert s.st_size > 15000000, f'File smaller ({s.st_size}) than expected: {p!r}' + print(f'### {__file__}: Moving {p!r} -> {p0!r}.') + os.rename( p, p0) + print(f'### {__file__}: Moving {p1!r} -> {p!r}.') + os.rename( p1, p) + ''' + ).strip(), + 'wasm-opt-replace.py', + ) + + # Create a wasm-opt script that basically does nothing, except + # defers to the original script when run with `--version`. + # + write( + textwrap.dedent(''' + #!/usr/bin/env python3 + import os + import sys + import subprocess + if sys.argv[1:] == ['--version']: + root = os.path.dirname(__file__) + subprocess.run(f'{root}/wasm-opt-0 --version', shell=1, check=1) + else: + print(f'{__file__}: Doing nothing. {sys.argv=}') + ''' + ).strip(), + 'wasm-opt-1', + ) + command += ' && ../wasm-opt-replace.py' + + command += ' && cd ..' + + return command + def log(text): print(f'{__file__}: {text}')