From bc7f14a463887a2dfd1196490096d33ca2b45217 Mon Sep 17 00:00:00 2001 From: Lukas Tenbrink Date: Tue, 29 Oct 2024 14:34:18 +0100 Subject: [PATCH] Add config options for indygreg standalone python build version, as well as python version. Move build files. Unlock windows and linux build configs for arm64. --- .github/workflows/build.yml | 23 +++-- SConstruct | 131 ++++++++++++++------------ src/extension/extension.cpp | 3 +- tools/build/prepare_python.py | 169 ++++++++++++++++------------------ 4 files changed, 167 insertions(+), 159 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b07cf1c..5f94b74 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -20,15 +20,23 @@ jobs: - platform: linux arch: x86_64 os: ubuntu-latest + python_version: 3.12.0 + python_standalone_build: 20231002 - platform: windows arch: x86_64 os: windows-latest + python_version: 3.12.0 + python_standalone_build: 20231002 - platform: macos arch: x86_64 os: macos-latest + python_version: 3.12.0 + python_standalone_build: 20231002 - platform: macos arch: arm64 os: macos-latest + python_version: 3.12.0 + python_standalone_build: 20231002 steps: - name: Checkout repository @@ -39,7 +47,8 @@ jobs: - name: Setup Python uses: actions/setup-python@v5 with: - python-version: '3.12' + # Note that this is for scons, not for the python version we are building. + python-version: 3.12 - name: Setup SCons shell: bash @@ -51,24 +60,24 @@ jobs: - name: Build extension (Linux) if: matrix.os == 'ubuntu-latest' run: | - scons platform=${{ matrix.platform }} arch=${{ matrix.arch }} single_source=true + scons platform=${{ matrix.platform }} arch=${{ matrix.arch }} python_standalone_build=${{ matrix.python_standalone_build }} python_version=${{ matrix.python_version }} single_source=true - name: Build extension (Windows) if: matrix.os == 'windows-latest' shell: pwsh run: | - scons platform=${{ matrix.platform }} arch=${{ matrix.arch }} single_source=true + scons platform=${{ matrix.platform }} arch=${{ matrix.arch }} python_standalone_build=${{ matrix.python_standalone_build }} python_version=${{ matrix.python_version }} single_source=true - name: Build extension (macOS) if: matrix.os == 'macos-latest' run: | - scons platform=${{ matrix.platform }} arch=${{ matrix.arch }} single_source=true + scons platform=${{ matrix.platform }} arch=${{ matrix.arch }} python_standalone_build=${{ matrix.python_standalone_build }} python_version=${{ matrix.python_version }} single_source=true - name: Upload artifacts (Linux) if: matrix.os == 'ubuntu-latest' uses: actions/upload-artifact@v4 with: - name: godot-python-${{ matrix.platform }}-${{ matrix.arch }} + name: godot-python-${{ matrix.platform }}-${{ matrix.arch }}.${{ matrix.python_standalone_build }}.${{ matrix.python_version }} path: bin/**/* retention-days: 30 @@ -76,7 +85,7 @@ jobs: if: matrix.os == 'windows-latest' uses: actions/upload-artifact@v4 with: - name: godot-python-${{ matrix.platform }}-${{ matrix.arch }} + name: godot-python-${{ matrix.platform }}.${{ matrix.arch }}.${{ matrix.python_standalone_build }}.${{ matrix.python_version }} path: | bin/**/* !bin/**/*.lib @@ -87,7 +96,7 @@ jobs: if: matrix.os == 'macos-latest' uses: actions/upload-artifact@v4 with: - name: godot-python-${{ matrix.platform }}-${{ matrix.arch }} + name: godot-python-${{ matrix.platform }}.${{ matrix.arch }}.${{ matrix.python_standalone_build }}.${{ matrix.python_version }} path: bin/**/* retention-days: 30 diff --git a/SConstruct b/SConstruct index cafc9c2..93a4140 100644 --- a/SConstruct +++ b/SConstruct @@ -86,13 +86,15 @@ opts.Add( # godot python options opts.Add( - PathVariable( - key="python", - help="Path to the `python` to build against.", - default='python3', - validator=(lambda key, val, env: build_utils.validate_executable(key, val, env) - if not env.get('python_lib_dir') else None), - ) + key="python_version", + help="Version of python to use. Must be available in the python_standalone_build.", + default='3.12.0', +) + +opts.Add( + key="python_standalone_build", + help="Build ID to use for the python version. You can find the list of builds at https://github.com/indygreg/python-build-standalone/releases.", + default="20231002", ) opts.Add( @@ -291,44 +293,52 @@ env.Alias("godot_module_archive", [ from tools.build import prepare_python -prepared_python_config = prepare_python.platform_configs[(env['platform'], env['arch'])] +prepared_python_config = prepare_python.configure( + platform=env['platform'], + arch=env['arch'], + python_version=env['python_version'], + build=env['python_standalone_build'], +) def _fetch_python(target, source, env): - dest = pathlib.Path(target[0].path).parent - dest.mkdir(parents=True, exist_ok=True) - prepare_python.fetch_python_for_platform(env['platform'], env['arch'], dest) - -fetch_python_alias = env.Alias("fetch_python", [ - Builder(action = env.Action(_fetch_python, "Fetching Python"))( - env, - target = os.fspath(generated_path / 'python' - / prepared_python_config.name / pathlib.Path(prepared_python_config.source_url).name), - source = [ - ], - ) -]) + try: + prepare_python.fetch_python_for_config(prepared_python_config, target[0]) + except Exception as e: + print(f"Error while fetching python: {e}") + return 1 + +fetched_python_files = env.Command( + target = os.fspath( + generated_path / 'python' / prepared_python_config.name / pathlib.Path(prepared_python_config.source_url).name + ), + source = [ + ], + action=_fetch_python +) def _prepare_python(target, source, env): - dest = pathlib.Path(target[0].path).parent.resolve() - dest.mkdir(parents=True, exist_ok=True) - - src = pathlib.Path(source[0].path).parent.resolve() - - env['python'] = prepare_python.prepare_for_platform(env['platform'], env['arch'], - src_dir = src, dest_dir = dest) - -prepare_python_alias = env.Alias("prepare_python", [ - Builder(action = Action(_prepare_python, "Preparing Python"))( - env, - target = f'bin/{prepared_python_config.name}/python312.zip', # XXX: version - source = [ - fetch_python_alias[0].children(), - prepare_python.__file__, - ], - ) -]) + try: + dest = pathlib.Path(target[0].path).parent.resolve() + dest.mkdir(parents=True, exist_ok=True) + + src = pathlib.Path(source[0].path).parent.resolve() + + env['python'] = prepare_python.prepare_for_platform(prepared_python_config, + src_dir = src, dest_dir = dest) + except Exception as e: + print(f"Error while preparing python: {e}") + return 1 + +prepared_python_files = env.Command( + target = f'bin/{prepared_python_config.name}/python{prepared_python_config.python_version_minor}-lib.zip', + source = [ + *fetched_python_files, + prepare_python.__file__, + ], + action=_prepare_python +) @@ -366,31 +376,36 @@ env.Append(CPPDEFINES = [f'PYGODOT_ARCH=\\"{env["arch"]}\\"']) def _append_python_config(env, target, **kwargs): - src_dir = generated_path / 'python' / prepared_python_config.name - env['python'] = os.fspath(prepare_python.get_python_for_platform(env['platform'], env['arch'], src_dir)) + try: + src_dir = generated_path / 'python' / prepared_python_config.name + env['python'] = os.fspath(prepare_python.get_python_for_platform(prepared_python_config, src_dir)) - from tools.build import python_config - _config_vars = python_config.get_python_config_vars(env) + from tools.build import python_config + _config_vars = python_config.get_python_config_vars(env) - env.Append(LIBPATH = _config_vars.lib_paths) - env.Append(LINKFLAGS = _config_vars.link_flags) - env.Append(LIBS = _config_vars.link_libs) - env.Append(CPPPATH = _config_vars.include_flags) + env.Append(LIBPATH = _config_vars.lib_paths) + env.Append(LINKFLAGS = _config_vars.link_flags) + env.Append(LIBS = _config_vars.link_libs) + env.Append(CPPPATH = _config_vars.include_flags) - if env['platform'] != 'windows': - env.Append(CPPDEFINES = [f'PYTHON_LIBRARY_PATH=\\"{_config_vars.ldlibrary or ""}\\"']) + if env['platform'] != 'windows': + env.Append(CPPDEFINES = [f'PYTHON_LIBRARY_PATH=\\"{_config_vars.ldlibrary or ""}\\"']) - dest = pathlib.Path(target[0].path) - dest.write_text(repr(_config_vars)) + dest = pathlib.Path(target[0].path) + dest.write_text(repr(_config_vars)) + except Exception as e: + print(f"Error while appending python config: {e}") + return 1 -append_python_config = Builder(action = Action(_append_python_config, None))( - env, target='src/.generated/.append_python_config', source=None) - -env.Depends(append_python_config, prepare_python_alias) -env.AlwaysBuild(append_python_config) +append_python_config_files = env.Command( + source=prepared_python_files, + target='src/.generated/.append_python_config', + action=_append_python_config +) -env.Depends(sources, append_python_config) +env.AlwaysBuild(append_python_config_files) +env.Depends(sources, append_python_config_files) # library name @@ -410,7 +425,7 @@ env["OBJSUFFIX"] = suffix + env["OBJSUFFIX"] library_name = "libgodot-python{}{}".format(env["suffix"], env["SHLIBSUFFIX"]) library = env.SharedLibrary( - target = f"bin/{env['platform']}-{env['arch']}/{library_name}", + target = f"bin/{prepared_python_config.name}/{library_name}", source = sources, ) diff --git a/src/extension/extension.cpp b/src/extension/extension.cpp index 3b0b24a..584b10b 100644 --- a/src/extension/extension.cpp +++ b/src/extension/extension.cpp @@ -169,8 +169,7 @@ static bool init_python_isolated() { auto py_major = std::to_string(PY_MAJOR_VERSION); auto py_minor = std::to_string(PY_MINOR_VERSION); auto py_version = py_major + "." + py_minor; - auto py_version_no_dot = py_major + py_minor; - auto python_zip_name = "python" + py_version_no_dot + ".zip"; + auto python_zip_name = "python" + py_version + "-lib.zip"; auto python_lib_name = "python" + py_version; add_module_search_path((runtime_config.python_home_path / python_zip_name).string()); diff --git a/tools/build/prepare_python.py b/tools/build/prepare_python.py index 651f36f..b28280a 100644 --- a/tools/build/prepare_python.py +++ b/tools/build/prepare_python.py @@ -12,6 +12,10 @@ class PlatformConfig: platform: str arch: str + python_version_major: str + python_version_minor: str + python_version_patch: str + build: str source_url: str so_suffixes: list[str] ext_suffixes: list[str] @@ -22,85 +26,81 @@ class PlatformConfig: @property def name(self): - return f'{self.platform}-{self.arch}' - - -platform_configs = {} - -def add_platform_config(*args, **kwargs): - config = PlatformConfig(*args, **kwargs) - key = (config.platform, config.arch) - platform_configs[key] = config - - -add_platform_config( - platform = 'linux', - arch = 'x86_64', - source_url = 'https://github.com/indygreg/python-build-standalone/releases/download/' - '20231002/cpython-3.12.0+20231002-x86_64-unknown-linux-gnu-install_only.tar.gz', - so_suffixes = ['.so'], - ext_suffixes = ['.so'], - so_path = 'lib/libpython3.12.so.1.0', - python_lib_dir = 'lib/python3.12', - python_ext_dir = 'lib/python3.12/lib-dynload', - executable = 'bin/python3.12', -) - -add_platform_config( - platform = 'windows', - arch = 'x86_64', - source_url = 'https://github.com/indygreg/python-build-standalone/releases/download/' - '20231002/cpython-3.12.0+20231002-x86_64-pc-windows-msvc-shared-install_only.tar.gz', - so_suffixes = ['.dll'], - ext_suffixes = ['.dll', '.pyd'], - so_path = 'python312.dll', - python_lib_dir = 'Lib', - python_ext_dir = 'DLLs', - executable = 'python.exe', -) - -add_platform_config( - platform = 'macos', - arch = 'x86_64', - source_url = 'https://github.com/indygreg/python-build-standalone/releases/download/' - '20231002/cpython-3.12.0+20231002-x86_64-apple-darwin-install_only.tar.gz', - so_suffixes = ['.so', '.dylib'], - ext_suffixes = ['.so'], - so_path = 'lib/libpython3.12.dylib', - python_lib_dir = 'lib/python3.12', - python_ext_dir = 'lib/python3.12/lib-dynload', - executable = 'bin/python3.12', -) - -add_platform_config( - platform = 'macos', - arch = 'arm64', - source_url = 'https://github.com/indygreg/python-build-standalone/releases/download/' - '20231002/cpython-3.12.0+20231002-aarch64-apple-darwin-install_only.tar.gz', - so_suffixes = ['.so', '.dylib'], - ext_suffixes = ['.so'], - so_path = 'lib/libpython3.12.dylib', - python_lib_dir = 'lib/python3.12', - python_ext_dir = 'lib/python3.12/lib-dynload', - executable = 'bin/python3.12', -) - - -def fetch_python_for_platform(platform: str, arch: str, dest_dir: pathlib.Path): - config = platform_configs[(platform, arch)] + return f'{self.platform}-{self.arch}-{self.build}-{self.python_version_patch}' + + +def configure(platform: str, arch: str, python_version: str, build: str): + python_version_major = python_version.split(".")[0] + python_version_minor = ".".join(python_version.split(".")[:2]) + + shared_args = dict( + platform = platform, + arch = arch, + python_version_major = python_version_major, + python_version_minor = python_version_minor, + python_version_patch = python_version, + build = build, + ) + + arch_indygreg = { + 'x86_64': 'x86_64', + 'arm64': 'aarch64', + }[arch] + + if platform == 'linux': + return PlatformConfig( + **shared_args, + source_url = 'https://github.com/indygreg/python-build-standalone/releases/download/' + f'{build}/cpython-{python_version}+{build}-{arch_indygreg}-unknown-linux-gnu-install_only.tar.gz', + so_suffixes = ['.so'], + ext_suffixes = ['.so'], + so_path = f'lib/libpython{python_version_minor}.so.1.0', + python_lib_dir = f'lib/python{python_version_minor}', + python_ext_dir = f'lib/python{python_version_minor}/lib-dynload', + executable = f'bin/python{python_version_minor}', + ) + + if platform == 'windows': + return PlatformConfig( + **shared_args, + source_url = 'https://github.com/indygreg/python-build-standalone/releases/download/' + f'{build}/cpython-{python_version}+{build}-{arch_indygreg}-pc-windows-msvc-shared-install_only.tar.gz', + so_suffixes = ['.dll'], + ext_suffixes = ['.dll', '.pyd'], + so_path = f'python{python_version_minor.replace(".", "")}.dll', + python_lib_dir = 'Lib', + python_ext_dir = 'DLLs', + executable = 'python.exe', + ) + if platform == 'macos': + return PlatformConfig( + **shared_args, + source_url = 'https://github.com/indygreg/python-build-standalone/releases/download/' + f'{build}/cpython-{python_version}+{build}-{arch_indygreg}-apple-darwin-install_only.tar.gz', + so_suffixes = ['.so', '.dylib'], + ext_suffixes = ['.so'], + so_path = f'lib/libpython{python_version_minor}.dylib', + python_lib_dir = f'lib/python{python_version_minor}', + python_ext_dir = f'lib/python{python_version_minor}/lib-dynload', + executable = f'bin/python{python_version_minor}', + ) + + raise ValueError("Unsupported platform.") + + +def fetch_python_for_config(config: PlatformConfig, target): print(f'fetching python for {config.name}') print(f' {config.source_url}') + target_path = pathlib.Path(target.path) with urllib.request.urlopen(config.source_url) as response: - with (dest_dir / pathlib.Path(config.source_url).name).open('wb') as dest: + target_path.parent.mkdir(parents=True, exist_ok=True) + with target_path.open('wb') as dest: shutil.copyfileobj(response, dest) -def prepare_for_platform(platform: str, arch: str, - src_dir: pathlib.Path, dest_dir: pathlib.Path) -> pathlib.Path: - config = platform_configs[(platform, arch)] - +def prepare_for_platform(config: PlatformConfig, src_dir: pathlib.Path, dest_dir: pathlib.Path) -> pathlib.Path: print(f'preparing for {config.name}') shutil.unpack_archive(src_dir / pathlib.Path(config.source_url).name, extract_dir = src_dir) @@ -109,7 +109,7 @@ def prepare_for_platform(platform: str, arch: str, src_lib_path = src / config.so_path lib_filename = pathlib.Path(config.so_path).name - if platform == 'macos': + if config.platform == 'macos': # Rename the library id (which we depend on) to be in @rpath. # (it defaults to /install/lib/) subprocess.run(['install_name_tool', '-id', f'@rpath/{lib_filename}', src_lib_path], check=True) @@ -117,36 +117,21 @@ def prepare_for_platform(platform: str, arch: str, dest_dir.mkdir(parents=True, exist_ok=True) shutil.copy2(src_lib_path, dest_dir) - if platform == 'macos': + if config.platform == 'macos': subprocess.run(['strip', '-x', dest_dir / lib_filename], check=True) else: subprocess.run(['strip', '-s', dest_dir / lib_filename], check=True) if (src / config.python_ext_dir).exists(): - dest_ext_dir = dest_dir / 'python3.12' / 'lib-dynload' + dest_ext_dir = dest_dir / f'python{config.python_version_minor}' / 'lib-dynload' dest_ext_dir.mkdir(parents=True, exist_ok=True) for path in (src / config.python_ext_dir).iterdir(): if any(suffix in path.suffixes for suffix in config.ext_suffixes): shutil.copy2(path, dest_ext_dir) - shutil.make_archive(dest_dir / 'python312', 'zip', root_dir=src / config.python_lib_dir, base_dir='') - - -def get_python_for_platform(platform: str, arch: str, src_dir: pathlib.Path) -> pathlib.Path: - config = platform_configs[(platform, arch)] - - src = src_dir / 'python' - - return src / config.executable - - - -def main(): - for platform in platform_configs: - prepare_for_platform(*platform) - + shutil.make_archive(dest_dir / f'python{config.python_version_minor}-lib', 'zip', root_dir=src / config.python_lib_dir, base_dir='') -if __name__ == '__main__': - main() +def get_python_for_platform(config: PlatformConfig, src_dir: pathlib.Path) -> pathlib.Path: + return src_dir / 'python' / config.executable