From 4c8105a232cd987951c494ce290c95f8d0d32e2b Mon Sep 17 00:00:00 2001 From: Ankith Date: Tue, 1 Oct 2024 23:42:08 +0530 Subject: [PATCH] Build windows deps ourselves --- .github/workflows/build-windows.yml | 61 ++++++++++++++ .../docker_base/gettext/build-gettext.sh | 7 +- .../docker_base/libtiff/build-tiff.sh | 2 +- .../docker_base/ogg/build-ogg.sh | 7 ++ .../docker_base/portmidi/build-portmidi.sh | 5 +- .../docker_base/portmidi/portmidi.sha512 | 2 +- .../docker_base/sdl_libs/build-sdl2-libs.sh | 16 +++- buildconfig/windependencies/build_win_deps.sh | 84 +++++++++++++++++++ .../windependencies/clean_windows_deps.py | 58 +++++++++++++ 9 files changed, 230 insertions(+), 12 deletions(-) create mode 100644 buildconfig/windependencies/build_win_deps.sh create mode 100644 buildconfig/windependencies/clean_windows_deps.py diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml index 6fc13d8203..a1b9003971 100644 --- a/.github/workflows/build-windows.yml +++ b/.github/workflows/build-windows.yml @@ -36,8 +36,69 @@ concurrency: cancel-in-progress: true jobs: + deps: + name: ${{ matrix.name }} deps + runs-on: windows-latest + strategy: + matrix: + # make i686 deps and x86_64 deps + include: + - { + name: "64-bit", + winarch: x86_64, + msystem: MINGW64 + } + - { + name: "32-bit", + winarch: i686, + msystem: MINGW32 + } + + steps: + - run: git config --global core.autocrlf input # do not introduce carriage returns + - uses: actions/checkout@v4.1.0 + + - name: Test for Win Deps cache hit + id: windep-cache + uses: actions/cache@v3.3.2 + with: + path: ${{ github.workspace }}/pygame_win_deps_${{ matrix.winarch }} + # The hash of all files in buildconfig manylinux-build and windependencies is + # the key to the cache. If anything changes here, the deps are built again + key: windep-${{ hashFiles('buildconfig/manylinux-build/**') }}-${{ hashFiles('buildconfig/windependencies/*.sh') }}-${{ matrix.winarch }} + lookup-only: true + + - uses: msys2/setup-msys2@v2 + with: + msystem: ${{ matrix.msystem }} + install: >- + mingw-w64-${{ matrix.winarch }}-gcc + mingw-w64-${{ matrix.winarch }}-cmake + mingw-w64-${{ matrix.winarch }}-meson + mingw-w64-${{ matrix.winarch }}-autotools + mingw-w64-${{ matrix.winarch }}-nasm + mingw-w64-${{ matrix.winarch }}-python + mingw-w64-${{ matrix.winarch }}-python-setuptools + + # build win deps on cache miss + - name: Build Win Deps + if: steps.windep-cache.outputs.cache-hit != 'true' + shell: msys2 {0} + run: | + export WIN_ARCH="${{ matrix.winarch }}" + cd buildconfig/windependencies + bash ./build_win_deps.sh + + # Uncomment when you want to manually verify the deps by downloading them + - name: Upload win deps + uses: actions/upload-artifact@v3 + with: + name: pygame-win-deps-${{ matrix.winarch }} + path: ${{ github.workspace }}/pygame_win_deps_${{ matrix.winarch }} + build: name: ${{ matrix.winarch }} + needs: deps runs-on: windows-latest strategy: fail-fast: false # if a particular matrix build fails, don't skip the rest diff --git a/buildconfig/manylinux-build/docker_base/gettext/build-gettext.sh b/buildconfig/manylinux-build/docker_base/gettext/build-gettext.sh index 7026dfc0b1..dde5b3e773 100644 --- a/buildconfig/manylinux-build/docker_base/gettext/build-gettext.sh +++ b/buildconfig/manylinux-build/docker_base/gettext/build-gettext.sh @@ -9,12 +9,11 @@ GETTEXT=gettext-0.21 curl -sL --retry 10 https://ftp.gnu.org/gnu/gettext/${GETTEXT}.tar.gz > ${GETTEXT}.tar.gz sha512sum -c gettext.sha512 -if [[ "$OSTYPE" == "linux-gnu"* ]]; then - # linux - export GETTEXT_CONFIGURE= -elif [[ "$OSTYPE" == "darwin"* ]]; then +if [[ "$OSTYPE" == "darwin"* ]]; then # Mac OSX, ship libintl.h on mac. export GETTEXT_CONFIGURE=--with-included-gettext +else + export GETTEXT_CONFIGURE= fi tar xzf ${GETTEXT}.tar.gz diff --git a/buildconfig/manylinux-build/docker_base/libtiff/build-tiff.sh b/buildconfig/manylinux-build/docker_base/libtiff/build-tiff.sh index 4c22cb5bd0..ea04360f35 100644 --- a/buildconfig/manylinux-build/docker_base/libtiff/build-tiff.sh +++ b/buildconfig/manylinux-build/docker_base/libtiff/build-tiff.sh @@ -13,7 +13,7 @@ cd $TIFF if [[ "$OSTYPE" == "linux-gnu"* ]]; then ./configure $PG_BASE_CONFIGURE_FLAGS --disable-lzma --disable-webp --disable-zstd -elif [[ "$OSTYPE" == "darwin"* ]]; then +else # Use CMake on macOS because arm64 builds fail with weird errors in ./configure cmake . $PG_BASE_CMAKE_FLAGS -Dlzma=OFF -Dwebp=OFF -Dzstd=OFF fi diff --git a/buildconfig/manylinux-build/docker_base/ogg/build-ogg.sh b/buildconfig/manylinux-build/docker_base/ogg/build-ogg.sh index 871056019c..49ae6790d0 100644 --- a/buildconfig/manylinux-build/docker_base/ogg/build-ogg.sh +++ b/buildconfig/manylinux-build/docker_base/ogg/build-ogg.sh @@ -22,6 +22,13 @@ cd .. tar xzf ${VORBIS}.tar.gz cd $VORBIS +# some hackery needed to make libvorbis build under mingw +case "$OSTYPE" in + msys|mingw32|mingw64) + sed -i '/LIBRARY/d' win32/*.def + ;; +esac + cmake . $PG_BASE_CMAKE_FLAGS make make install diff --git a/buildconfig/manylinux-build/docker_base/portmidi/build-portmidi.sh b/buildconfig/manylinux-build/docker_base/portmidi/build-portmidi.sh index a44e131f8e..ab7225832d 100644 --- a/buildconfig/manylinux-build/docker_base/portmidi/build-portmidi.sh +++ b/buildconfig/manylinux-build/docker_base/portmidi/build-portmidi.sh @@ -3,10 +3,11 @@ set -e -x cd $(dirname `readlink -f "$0"`) -PORTMIDI_VER="2.0.4" +# 2.0.4 has compilation issues on windows, so pin to latest commit on github +PORTMIDI_VER="7a5de5b7597c46f963d72a83defe7592f901e5f1" PORTMIDI="portmidi-${PORTMIDI_VER}" -curl -sL --retry 10 https://github.com/PortMidi/portmidi/archive/refs/tags/v${PORTMIDI_VER}.tar.gz> ${PORTMIDI}.tar.gz +curl -sL --retry 10 https://github.com/PortMidi/portmidi/archive/$PORTMIDI_VER.tar.gz > ${PORTMIDI}.tar.gz sha512sum -c portmidi.sha512 tar xzf ${PORTMIDI}.tar.gz diff --git a/buildconfig/manylinux-build/docker_base/portmidi/portmidi.sha512 b/buildconfig/manylinux-build/docker_base/portmidi/portmidi.sha512 index 3791893524..f43040dd19 100644 --- a/buildconfig/manylinux-build/docker_base/portmidi/portmidi.sha512 +++ b/buildconfig/manylinux-build/docker_base/portmidi/portmidi.sha512 @@ -1 +1 @@ -d9f22d161e1dd9a4bde1971bb2b6e5352da51545f4fe5ecad11c55e7a535f0d88efce18d1c8fd91e93b70a7926150f86a0f53972ad92370e86556a8dd72dc194 portmidi-2.0.4.tar.gz +b8ccc72dc7e199266e4bfbaa777b1979c5fc1d97e7dcb10707ec0c2325abe79c036e6939b79f97fea6c9602620d41e6659cd60b0bea4c59d8fb95e478ba0bd75 portmidi-7a5de5b7597c46f963d72a83defe7592f901e5f1.tar.gz diff --git a/buildconfig/manylinux-build/docker_base/sdl_libs/build-sdl2-libs.sh b/buildconfig/manylinux-build/docker_base/sdl_libs/build-sdl2-libs.sh index f096177fec..7d373b32e1 100644 --- a/buildconfig/manylinux-build/docker_base/sdl_libs/build-sdl2-libs.sh +++ b/buildconfig/manylinux-build/docker_base/sdl_libs/build-sdl2-libs.sh @@ -20,10 +20,18 @@ curl -sL --retry 10 https://github.com/libsdl-org/SDL_mixer/releases/download/re curl -sL --retry 10 https://github.com/libsdl-org/SDL_ttf/releases/download/release-$TTF2_VER/$TTF2.tar.gz > ${TTF2}.tar.gz sha512sum -c sdl2.sha512 -# On mac/manylinux we have to make use of standard dynamic linking rather than -# dlopen-ing the library itself. This is important for when auditwheel/delocate -# moves libraries into the wheel. -PG_DEPS_SHARED=0 +case "$OSTYPE" in + msys|mingw32|mingw64) + # on windows, do SDL-style shared deps (SDLs default strategy) + PG_DEPS_SHARED=1 + ;; + *) + # On mac/manylinux we have to make use of standard dynamic linking rather than + # dlopen-ing the library itself. This is important for when auditwheel/delocate + # moves libraries into the wheel. + PG_DEPS_SHARED=0 + ;; +esac # Build SDL tar xzf ${SDL2}.tar.gz diff --git a/buildconfig/windependencies/build_win_deps.sh b/buildconfig/windependencies/build_win_deps.sh new file mode 100644 index 0000000000..adca5859fe --- /dev/null +++ b/buildconfig/windependencies/build_win_deps.sh @@ -0,0 +1,84 @@ +# This uses manylinux build scripts to build dependencies on windows. + +set -e -x + +# This is needed for tar to work in some places +export MSYS=winsymlinks:lnk + +# The below three lines convert something like D:\path\goes\here to /d/path/goes/here +export BASE_DIR=$(echo "$GITHUB_WORKSPACE" | tr '[:upper:]' '[:lower:]') +export BASE_DIR="${BASE_DIR//\\//}" # //\\// replaces all \ with / in the variable +export BASE_DIR="/${BASE_DIR//:/}" # remove colon from drive part, add leading / + +export PG_DEP_PREFIX="$BASE_DIR/pygame_win_deps_$WIN_ARCH" + +export PKG_CONFIG_PATH="$PG_DEP_PREFIX/lib/pkgconfig:$PKG_CONFIG_PATH" + +mkdir $PG_DEP_PREFIX + +# for great speed. +export MAKEFLAGS="-j 4" + +# for scripts using ./configure +export CC="gcc" +export CXX="g++" + +# With this we +# 1) Force install prefix to $PG_DEP_PREFIX +# 2) use lib directory within $PG_DEP_PREFIX (and not lib64) +# 3) make release binaries +# 4) build shared libraries +# 5) make cmake use gcc/g++/make +export PG_BASE_CMAKE_FLAGS="-DCMAKE_INSTALL_PREFIX=$PG_DEP_PREFIX \ + -DCMAKE_INSTALL_LIBDIR:PATH=lib \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_SHARED_LIBS=true \ + -DCMAKE_C_COMPILER=$CC -DCMAKE_CXX_COMPILER=$CXX -DCMAKE_MAKE_PROGRAM=make" + +export CMAKE_GENERATOR="MSYS Makefiles" + +export PG_BASE_CONFIGURE_FLAGS="--prefix=$PG_DEP_PREFIX" + +export PG_BASE_MESON_FLAGS="--prefix=$PG_DEP_PREFIX \ + -Dlibdir=lib \ + -Dbuildtype=release \ + -Ddefault_library=shared" + +cd ../manylinux-build/docker_base + +# Now start installing dependencies +# --------------------------------- + +# install some buildtools +# bash buildtools/install.sh + +# sdl_image deps +bash zlib-ng/build-zlib-ng.sh +bash libpng/build-png.sh # depends on zlib +bash libjpegturbo/build-jpeg-turbo.sh +bash libtiff/build-tiff.sh +bash libwebp/build-webp.sh + +# freetype (also sdl_ttf dep) +bash brotli/build-brotli.sh +bash bzip2/build-bzip2.sh +bash freetype/build-freetype.sh + +# sdl_mixer deps +bash libxmp/build-libxmp.sh +bash ogg/build-ogg.sh +bash flac/build-flac.sh +bash mpg123/build-mpg123.sh +bash opus/build-opus.sh # needs libogg (which is a container format) +bash wavpack/build-wavpack.sh + +# fluidsynth (for sdl_mixer) +# bash gettext/build-gettext.sh +bash glib/build-glib.sh +bash sndfile/build-sndfile.sh +bash fluidsynth/build-fluidsynth.sh + +bash sdl_libs/build-sdl2-libs.sh + +# for pygame.midi +bash portmidi/build-portmidi.sh diff --git a/buildconfig/windependencies/clean_windows_deps.py b/buildconfig/windependencies/clean_windows_deps.py new file mode 100644 index 0000000000..4aa3522d82 --- /dev/null +++ b/buildconfig/windependencies/clean_windows_deps.py @@ -0,0 +1,58 @@ +""" +Cleans self-built windows deps, to only retain what's needed for the build +""" + +import sys +import pathlib +import shutil + +base_dir = pathlib.Path(sys.argv[1]) + +# delete 'share' and 'man' +shutil.rmtree(base_dir / "share") +shutil.rmtree(base_dir / "man") + + +# delete all headers and import lib in 'include' and 'lib' except the ones we +# need +def clean_dir(dir: pathlib.Path, allowlist: set[str]): + for path in dir.iterdir(): + if path.name in allowlist: + continue + + if path.is_dir(): + shutil.rmtree(path) + else: + path.unlink() + + +clean_dir( + base_dir / "include", + { + "SDL2", + "freetype2", + "portmidi.h", + "porttime.h", + "pmutil.h", + }, +) + +clean_dir( + base_dir / "lib", + { + "libSDL2.dll.a", + "libSDL2_ttf.dll.a", + "libSDL2_image.dll.a", + "libSDL2_mixer.dll.a", + "libportmidi.dll.a", + "libfreetype.dll.a", + }, +) + + +# copy all dlls from 'bin' into 'lib', and then delete 'bin' +bindir = base_dir / "bin" +for dll in bindir.glob("*.dll"): + shutil.copy(dll, base_dir / "lib") + +shutil.rmtree(bindir)