Skip to content

Commit

Permalink
Update dependency bundling
Browse files Browse the repository at this point in the history
  • Loading branch information
gerlero committed Dec 27, 2023
1 parent b77358a commit d563248
Show file tree
Hide file tree
Showing 5 changed files with 111 additions and 100 deletions.
68 changes: 9 additions & 59 deletions .github/workflows/build-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -78,64 +78,21 @@ env:
OPENFOAM: ${{ inputs.openfoam-version || inputs.openfoam-git-branch }}

jobs:
deps:
build:
runs-on: ${{ inputs.build-os || 'macos-12' }}
env:
BUILD_OS: ${{ inputs.build-os || 'macos-12' }}
outputs:
deps-restore-key: ${{ steps.caching.outputs.DEPS_RESTORE_KEY }}
build-restore-key: ${{ steps.caching.outputs.BUILD_RESTORE_KEY }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Get Make recipes for caching
run: |
make deps --dry-run ${{ env.MAKE_VARS }} > make_deps.txt
make build --dry-run ${{ env.MAKE_VARS }} > make_build.txt
- name: Generate cache restore keys
id: caching
run: |
DEPS_RESTORE_KEY="deps-${{ env.OPENFOAM }}-${{ env.BUILD_OS }}-${{ hashFiles('make_deps.txt', 'Brewfile') }}-"
BUILD_RESTORE_KEY="build-${{ env.OPENFOAM }}-${{ env.BUILD_OS }}-${{ hashFiles('make_build.txt', 'Brewfile', format('OpenFOAM-v${0}.tgz.sha256', inputs.openfoam-version), 'configure.sh') }}-"
echo "DEPS_RESTORE_KEY=$DEPS_RESTORE_KEY" >> "$GITHUB_OUTPUT"
BUILD_RESTORE_KEY="build-${{ env.OPENFOAM }}-${{ env.BUILD_OS }}-${{ hashFiles('make_build.txt', 'Brewfile', 'bundle_deps.py', format('OpenFOAM-v${0}.tgz.sha256', inputs.openfoam-version), 'configure.sh') }}-"
echo "BUILD_RESTORE_KEY=$BUILD_RESTORE_KEY" >> "$GITHUB_OUTPUT"
- name: Look up cached build
if: inputs.use-cached
id: cache_build
uses: actions/cache/restore@v3
with:
path: build/*-build.sparsebundle
key: ignore
restore-keys:
${{ steps.caching.outputs.BUILD_RESTORE_KEY }}
lookup-only: true
- name: Look up cached deps
if: inputs.use-cached && steps.cache_build.outputs.cache-matched-key == ''
id: cache_deps
uses: actions/cache/restore@v3
with:
path: build/*-deps.sparsebundle
key: ignore
restore-keys:
${{ steps.caching.outputs.DEPS_RESTORE_KEY }}
lookup-only: true
- name: Make deps
if: steps.cache_build.outputs.cache-matched-key == '' && steps.cache_deps.outputs.cache-matched-key == ''
run: |
make deps ${{ env.MAKE_VARS }}
- name: Save deps to cache
if: steps.cache_build.outputs.cache-matched-key == '' && steps.cache_deps.outputs.cache-matched-key == ''
uses: actions/cache/save@v3
with:
path: build/*-deps.sparsebundle
key: ${{ steps.caching.outputs.DEPS_RESTORE_KEY }}${{ github.run_id }}

build:
needs: deps
runs-on: ${{ inputs.build-os || 'macos-12' }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Restore cached build if available
if: inputs.use-cached
id: cache_build
Expand All @@ -144,20 +101,10 @@ jobs:
path: build/*-build.sparsebundle
key: ignore
restore-keys:
${{ needs.deps.outputs.build-restore-key }}
- name: Restore cached deps
if: steps.cache_build.outputs.cache-matched-key == ''
id: cache_deps
uses: actions/cache/restore@v3
with:
path: build/*-deps.sparsebundle
key: ignore
restore-keys:
${{ needs.deps.outputs.deps-restore-key }}
fail-on-cache-miss: true
- name: Reuse cached build or deps
${{ steps.caching.outputs.build-restore-key }}
- name: Reuse cached build
if: steps.cache_build.outputs.cache-matched-key != ''
run: |
touch -c build/*-deps.sparsebundle
touch -c build/*-build.sparsebundle
- name: Build
if: steps.cache_build.outputs.cache-matched-key == ''
Expand All @@ -168,7 +115,7 @@ jobs:
uses: actions/cache/save@v3
with:
path: build/*-build.sparsebundle
key: ${{ needs.deps.outputs.build-restore-key }}${{ github.run_id }}
key: ${{ steps.caching.outputs.build-restore-key }}${{ github.run_id }}
- name: Make app
run: |
make zip ${{ env.MAKE_VARS }}
Expand Down Expand Up @@ -198,6 +145,9 @@ jobs:
run: |
unzip *-app-*.zip
working-directory: build
- name: Uninstall all Homebrew formulae
run: |
brew uninstall $(brew list --formulae)
- name: Test
run: |
make test ${{ env.MAKE_VARS }}
Expand Down
58 changes: 21 additions & 37 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ endif
# Build targets
app: build/$(APP_NAME).app
build: build/$(APP_NAME)-build.sparsebundle
deps: build/$(APP_NAME)-deps.sparsebundle
deps: Brewfile.lock.json
fetch-source: $(OPENFOAM_TARBALL)

ifeq ($(DEPENDENCIES_KIND),both)
Expand Down Expand Up @@ -128,15 +128,9 @@ build/$(APP_NAME).app/Contents/Resources/$(APP_NAME).dmg: build/$(APP_NAME)-buil
rm -rf $(VOLUME)/build
rm -rf -- $(VOLUME)/**/.git(N)
rm -f -- $(VOLUME)/**/.DS_Store(N)
ifeq ($(DEPENDENCIES_KIND),standalone)
rm $(VOLUME)/usr/bin/brew
rm $(VOLUME)/Brewfile
rm $(VOLUME)/Brewfile.lock.json
else ifeq ($(DEPENDENCIES_KIND),homebrew)
ifeq ($(DEPENDENCIES_KIND),homebrew)
rm -rf $(VOLUME)/usr
ln -s $(shell brew --prefix) $(VOLUME)/usr
else
$(error Invalid value for DEPENDENCIES_KIND)
endif
rm -rf $(VOLUME)/.fseventsd || true
mkdir -p build/$(APP_NAME).app/Contents/Resources
Expand All @@ -150,10 +144,26 @@ endif
hdiutil detach $(VOLUME)
rm build/$(APP_NAME)-build.sparsebundle.shadow

build/$(APP_NAME)-build.sparsebundle: build/$(APP_NAME)-deps.sparsebundle $(OPENFOAM_TARBALL) configure.sh
build/$(APP_NAME)-build.sparsebundle: Brewfile Brewfile.lock.json $(if $(filter standalone,$(DEPENDENCIES_KIND)),bundle_deps.py) $(OPENFOAM_TARBALL) configure.sh
[ ! -d $(VOLUME) ] || hdiutil detach $(VOLUME)
mv build/$(APP_NAME)-deps.sparsebundle build/$(APP_NAME)-build.sparsebundle
hdiutil attach build/$(APP_NAME)-build.sparsebundle
mkdir -p build
hdiutil create \
-size 50g \
-fs $(VOLUME_FILESYSTEM) \
-volname $(APP_NAME) \
build/$(APP_NAME)-build.sparsebundle \
-ov -attach
brew bundle check --verbose --no-upgrade
ifeq ($(DEPENDENCIES_KIND),standalone)
cd $(VOLUME) \
&& HOMEBREW_BUNDLE_FILE="$(CURDIR)/Brewfile" "$(CURDIR)/bundle_deps.py"
else ifeq ($(DEPENDENCIES_KIND),homebrew)
cp Brewfile $(VOLUME)/
cp Brewfile.lock.json $(VOLUME)/
ln -s $(shell brew --prefix) $(VOLUME)/usr
else
$(error Invalid value for DEPENDENCIES_KIND)
endif
ifdef OPENFOAM_TARBALL
tar -xzf $(OPENFOAM_TARBALL) --strip-components 1 -C $(VOLUME)
else ifdef OPENFOAM_GIT_BRANCH
Expand All @@ -170,32 +180,6 @@ endif
&& ./Allwmake -j $(WMAKE_NJOBS) -s
hdiutil detach $(VOLUME)

build/$(APP_NAME)-deps.sparsebundle: Brewfile $(if $(filter homebrew,$(DEPENDENCIES_KIND)),Brewfile.lock.json)
[ ! -d $(VOLUME) ] || hdiutil detach $(VOLUME)
mkdir -p build
hdiutil create \
-size 50g \
-fs $(VOLUME_FILESYSTEM) \
-volname $(APP_NAME) \
build/$(APP_NAME)-deps.sparsebundle \
-ov -attach
cp Brewfile $(VOLUME)/
ifeq ($(DEPENDENCIES_KIND),standalone)
git clone https://github.com/Homebrew/brew $(VOLUME)/homebrew
mkdir -p $(VOLUME)/usr/bin
ln -s ../../homebrew/bin/brew $(VOLUME)/usr/bin/
HOMEBREW_RELOCATABLE_INSTALL_NAMES=1 $(VOLUME)/usr/bin/brew bundle --file $(VOLUME)/Brewfile --verbose
$(VOLUME)/usr/bin/brew autoremove
$(VOLUME)/usr/bin/brew list --versions
else ifeq ($(DEPENDENCIES_KIND),homebrew)
brew bundle check --verbose --no-upgrade
cp Brewfile.lock.json $(VOLUME)/
ln -s $(shell brew --prefix) $(VOLUME)/usr
else
$(error Invalid value for DEPENDENCIES_KIND)
endif
hdiutil detach $(VOLUME)

$(OPENFOAM_TARBALL): $(or $(wildcard $(OPENFOAM_TARBALL).sha256), \
$(warning No checksum file found for $(OPENFOAM_TARBALL); will skip verification))
curl -L -o $(OPENFOAM_TARBALL) $(OPENFOAM_TARBALL_URL)
Expand Down
62 changes: 62 additions & 0 deletions bundle_deps.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
#!/usr/bin/env python3
"""
Bundle Homebrew dependencies from a installed with Homebrew Bundle
"""

import subprocess
import shutil
import os

from pathlib import Path

import macho

SRC_PREFIX = Path(subprocess.run(["brew", "--prefix"], stdout=subprocess.PIPE, check=True).stdout.decode().strip())
DST_PREFIX = Path("usr")

def change_lib_id(lib, *, id):
subprocess.run(["install_name_tool", "-id", id, lib], check=True)
subprocess.run(["codesign", "--force", "--preserve-metadata=entitlements,requirements,flags,runtime", "--sign", "-", lib], check=True)

def copy_installed_formula(formula):
print(f"Bundling {formula}")

src_prefix = SRC_PREFIX / "opt" / formula.name
dst_prefix = DST_PREFIX / "opt" / formula.name

src_cellar = SRC_PREFIX / "Cellar" / formula.name
dst_cellar = DST_PREFIX / "Cellar" / formula.name

shutil.copytree(src_cellar, dst_cellar)

dst_prefix.parent.mkdir(exist_ok=True)
shutil.copy(src_prefix, dst_prefix, follow_symlinks=False)

# Replace library IDs and references to other libraries (install_names)
for file in dst_cellar.rglob("*"):
if not file.is_file():
continue
if (file.suffix == ".dylib" or file.suffix == ".so"):
macho.change_lib_id(file, id=dst_prefix.absolute() / Path(*file.relative_to(dst_cellar).parts[1:]))
if (file.suffix == "" or file.suffix == ".bin" or file.suffix == ".dylib" or file.suffix == ".so"):
for install_name in macho.get_install_names(file):
if install_name.is_absolute() and install_name.is_relative_to(SRC_PREFIX):
if install_name.is_relative_to(SRC_PREFIX):
new_install_name = DST_PREFIX.absolute() / install_name.relative_to(SRC_PREFIX)
relative_install_name = Path("@loader_path") / os.path.relpath(new_install_name, start=file.parent)
macho.change_install_name(file, install_name, relative_install_name)

def get_deps(*, recursive=True):
if not recursive:
return {Path(formula) for formula in subprocess.run(["brew", "bundle", "list"], stdout=subprocess.PIPE, check=True).stdout.decode().splitlines()}

deps = get_deps(recursive=False)
for dep in list(deps):
recursive_deps = {Path(formula) for formula in subprocess.run(["brew", "deps", dep], stdout=subprocess.PIPE, check=True).stdout.decode().splitlines()}
deps.update(recursive_deps)

return deps


for formula in get_deps():
copy_installed_formula(formula)
15 changes: 15 additions & 0 deletions macho.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
import subprocess

from pathlib import Path

def change_lib_id(lib, *, id):
subprocess.run(["install_name_tool", "-id", id, lib], check=True)
subprocess.run(["codesign", "--force", "--preserve-metadata=entitlements,requirements,flags,runtime", "--sign", "-", lib], check=True)

def get_install_names(file):
otool_stdout = subprocess.run(["otool", "-L", file], stdout=subprocess.PIPE, check=True).stdout.decode()
install_names = [Path(line.split(" (compatibility version ")[0].strip()) for line in otool_stdout.splitlines()[1:]]
return install_names

def change_install_name(file, old_install_name, new_install_name):
subprocess.run(["install_name_tool", "-change", old_install_name, new_install_name, file], check=True)
8 changes: 4 additions & 4 deletions relativize_install_names.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,17 +5,17 @@

from pathlib import Path

import macho

def relativize_install_names(file, lib_dirs):
otool_stdout = subprocess.run(["otool", "-L", file], stdout=subprocess.PIPE, check=True).stdout.decode()
install_names = [Path(line.split(" (compatibility version ")[0].strip()) for line in otool_stdout.splitlines()[1:]]
for install_name in install_names:
for install_name in macho.get_install_names(file):
if install_name.is_absolute():
for lib_dir,new_lib_dir in lib_dirs.items():
lib_dir = lib_dir.absolute()
if install_name.is_relative_to(lib_dir):
new_install_name = new_lib_dir.absolute() / install_name.relative_to(lib_dir)
relative_install_name = Path("@loader_path") / os.path.relpath(new_install_name, start=file.parent)
subprocess.run(["install_name_tool", "-change", install_name, relative_install_name, file])
macho.change_install_name(file, install_name, relative_install_name)
break

# Replace references to dependencies
Expand Down

0 comments on commit d563248

Please sign in to comment.