From a1484533c44e18002821bdac485a0f2fb3cdb7fe Mon Sep 17 00:00:00 2001 From: Jaap <33715902+Jaapel@users.noreply.github.com> Date: Fri, 23 Aug 2024 18:17:45 +0200 Subject: [PATCH 1/3] update pixi env and hooks --- .build/build.spec | 37 ++++++++++++++++++++++++++--------- .build/pyi_rth_gdal_driver.py | 29 +++++++++++++++++++++++++++ pixi.toml | 4 ++++ 3 files changed, 61 insertions(+), 9 deletions(-) create mode 100644 .build/pyi_rth_gdal_driver.py diff --git a/.build/build.spec b/.build/build.spec index 76219d3d..31eb65df 100644 --- a/.build/build.spec +++ b/.build/build.spec @@ -3,34 +3,53 @@ from fiat.util import generic_folder_check import inspect import os import sys -from pathlib import Path +from pathlib import Path, PurePath #Pre build event setup app_name = "fiat" sys.setrecursionlimit(5000) _file = Path(inspect.getfile(lambda: None)) -cwd = _file.parent -env_path = os.path.dirname(sys.executable) -generic_folder_check(Path(cwd, "../bin")) + +# Get project root directory +project_root = _file.parents[1].absolute() + +build_dir = project_root / ".build" +env_path = Path(os.getenv("CONDA_PREFIX")) + +# check if bin folder is in build dir. +generic_folder_check(build_dir / "bin") mode = "Release" -proj = Path(os.environ["PROJ_LIB"]) +# Find proj folder +proj = Path(os.environ.get("PROJ_LIB", env_path / "share" / "proj")) +# Find gdal plugins folder +is_win = sys.platform.startswith("win") +if is_win: + gdal_plugins_default_directory = PurePath("Library") / "lib" / "gdalplugins" +else: + gdal_plugins_default_directory = PurePath("lib") / "gdalplugins" + +gdal_plugins = Path(os.environ.get("GDAL_DRIVER_PATH", Path(env_path) / gdal_plugins_default_directory)) binaries = [ - (Path(proj, 'proj.db'), './share'), + (Path(proj, 'proj.db'), "./share"), + (Path(gdal_plugins), str(gdal_plugins_default_directory)), ] # Build event a = Analysis( - [Path(cwd, "../src/fiat/cli/main.py")], - pathex=[Path(cwd, "../src"), Path(env_path, "lib/site-packages")], + [Path(project_root, "src/fiat/cli/main.py")], + pathex=[Path(project_root, "src")], binaries=binaries, datas=[], hiddenimports=[], hookspath=[], hooksconfig={}, - runtime_hooks=[Path(cwd, 'runtime_hooks.py')], + runtime_hooks=[ + Path(build_dir, "runtime_hooks.py"), + Path(build_dir, "pyi_rth_gdal_driver.py"), + ], excludes=[], win_no_prefer_redirects=False, win_private_assemblies=False, diff --git a/.build/pyi_rth_gdal_driver.py b/.build/pyi_rth_gdal_driver.py new file mode 100644 index 00000000..c53a7b2d --- /dev/null +++ b/.build/pyi_rth_gdal_driver.py @@ -0,0 +1,29 @@ +""" +Pyinstaller hook that sets mandatory env var. + +Based on https://github.com/conda-forge/gdal-feedstock/blob/main/recipe/scripts/activate.sh +""" + +import os +import sys + +# Installing `osgeo` Conda packages with plugins requires to set `GDAL_DRIVER_PATH` + +is_win = sys.platform.startswith("win") +if is_win: + gdal_plugins = os.path.join(sys._MEIPASS, "share", "gdalplugins") + if not os.path.exists(gdal_plugins): + gdal_plugins = os.path.join(sys._MEIPASS, "Library", "lib", "gdalplugins") + # last attempt, check if one of the required file is in the generic folder Library/data + if not os.path.exists(os.path.join(gdal_plugins, "gcs.csv")): + gdal_plugins = os.path.join(sys._MEIPASS, "Library", "lib") + +else: + gdal_plugins = os.path.join(sys._MEIPASS, "lib", "gdalplugins") + +if os.path.exists(gdal_plugins): + os.environ["GDAL_DRIVER_PATH"] = gdal_plugins + +print(f"GDAL_DRIVER_PATH = {gdal_plugins}") +print("contents:") +print(os.listdir(gdal_plugins)) diff --git a/pixi.toml b/pixi.toml index 0264ef88..d628aee8 100644 --- a/pixi.toml +++ b/pixi.toml @@ -33,6 +33,9 @@ quarto-quick = { cmd = ["quarto", "render", "docs"] } quarto-render = { cmd = ["quarto", "render", "docs", "--execute"] } serve = { cmd = ["python", "-m", "http.server", "8000", "-d", "docs/_site"] } +# Build +build = { cmd = "pyinstaller .build/build.spec --distpath .build/bin --workpath .build/intermediates", env = { "PROJ_LIB" = "$CONDA_PREFIX/share/proj" } } + # Testing test = { cmd = ["pytest"] } test-lf = { cmd = ["pytest", "--lf", "--tb=short"] } @@ -55,6 +58,7 @@ clean-docs = { cmd = ["rm", "-rf", "docs/_site"] } ## Dependencies [dependencies] gdal = ">=3.5" +libgdal = "*" numpy = "*" regex = "*" tomli = "*" From 1d715dd4e5964b7409c78888c72b23387a672d6a Mon Sep 17 00:00:00 2001 From: Jaap <33715902+Jaapel@users.noreply.github.com> Date: Fri, 23 Aug 2024 18:37:34 +0200 Subject: [PATCH 2/3] remove debug statements --- .build/pyi_rth_gdal_driver.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.build/pyi_rth_gdal_driver.py b/.build/pyi_rth_gdal_driver.py index c53a7b2d..76c76daf 100644 --- a/.build/pyi_rth_gdal_driver.py +++ b/.build/pyi_rth_gdal_driver.py @@ -23,7 +23,3 @@ if os.path.exists(gdal_plugins): os.environ["GDAL_DRIVER_PATH"] = gdal_plugins - -print(f"GDAL_DRIVER_PATH = {gdal_plugins}") -print("contents:") -print(os.listdir(gdal_plugins)) From 8900352ea9fbf05559ab44f92acd8952d0ada11d Mon Sep 17 00:00:00 2001 From: Jaap <33715902+Jaapel@users.noreply.github.com> Date: Wed, 28 Aug 2024 17:18:26 +0200 Subject: [PATCH 3/3] even more robust --- .build/build.spec | 24 ++------- .build/hook-fiat.py | 50 +++++++++++++++++++ ...pyi_rth_gdal_driver.py => pyi_rth_fiat.py} | 13 ++++- .build/runtime_hooks.py | 10 ---- 4 files changed, 64 insertions(+), 33 deletions(-) create mode 100644 .build/hook-fiat.py rename .build/{pyi_rth_gdal_driver.py => pyi_rth_fiat.py} (74%) delete mode 100644 .build/runtime_hooks.py diff --git a/.build/build.spec b/.build/build.spec index 31eb65df..c5d74d08 100644 --- a/.build/build.spec +++ b/.build/build.spec @@ -15,40 +15,22 @@ _file = Path(inspect.getfile(lambda: None)) project_root = _file.parents[1].absolute() build_dir = project_root / ".build" -env_path = Path(os.getenv("CONDA_PREFIX")) # check if bin folder is in build dir. generic_folder_check(build_dir / "bin") mode = "Release" -# Find proj folder -proj = Path(os.environ.get("PROJ_LIB", env_path / "share" / "proj")) -# Find gdal plugins folder -is_win = sys.platform.startswith("win") -if is_win: - gdal_plugins_default_directory = PurePath("Library") / "lib" / "gdalplugins" -else: - gdal_plugins_default_directory = PurePath("lib") / "gdalplugins" - -gdal_plugins = Path(os.environ.get("GDAL_DRIVER_PATH", Path(env_path) / gdal_plugins_default_directory)) - -binaries = [ - (Path(proj, 'proj.db'), "./share"), - (Path(gdal_plugins), str(gdal_plugins_default_directory)), -] - # Build event a = Analysis( [Path(project_root, "src/fiat/cli/main.py")], pathex=[Path(project_root, "src")], - binaries=binaries, + binaries=[], datas=[], hiddenimports=[], - hookspath=[], + hookspath=[".build"], hooksconfig={}, runtime_hooks=[ - Path(build_dir, "runtime_hooks.py"), - Path(build_dir, "pyi_rth_gdal_driver.py"), + Path(build_dir, "pyi_rth_fiat.py"), ], excludes=[], win_no_prefer_redirects=False, diff --git a/.build/hook-fiat.py b/.build/hook-fiat.py new file mode 100644 index 00000000..780c3ed7 --- /dev/null +++ b/.build/hook-fiat.py @@ -0,0 +1,50 @@ +""" +Analysis hook for fiat installation. + +Helps finding different dependencies for PyInstaller. +""" + +import os +import sys + +from PyInstaller.compat import is_conda, is_win +from PyInstaller.utils.hooks.conda import ( + Distribution, + distribution, +) + +datas = [] + +if hasattr(sys, "real_prefix"): # check if in a virtual environment + root_path = sys.real_prefix +else: + root_path = sys.prefix + +if is_conda: + netcdf_dist: Distribution = distribution("libgdal-netcdf") + # append netcdf files to datas + datas += list( + map( + lambda path: (os.path.join(root_path, path), str(path.parent)), + netcdf_dist.files, + ) + ) + # a runtime hook defines the path for `GDAL_DRIVER_PATH` + +# - conda-specific +if is_win: + tgt_proj_data = os.path.join("Library", "share", "proj") + src_proj_data = os.path.join(root_path, "Library", "share", "proj") + +else: # both linux and darwin + tgt_proj_data = os.path.join("share", "proj") + src_proj_data = os.path.join(root_path, "share", "proj") + +if is_conda: + if os.path.exists(src_proj_data): + datas.append((src_proj_data, tgt_proj_data)) + else: + from PyInstaller.utils.hooks import logger + + logger.warning("Datas for proj not found at:\n{}".format(src_proj_data)) + # A runtime hook defines the path for `PROJ_LIB` diff --git a/.build/pyi_rth_gdal_driver.py b/.build/pyi_rth_fiat.py similarity index 74% rename from .build/pyi_rth_gdal_driver.py rename to .build/pyi_rth_fiat.py index 76c76daf..b38b566b 100644 --- a/.build/pyi_rth_gdal_driver.py +++ b/.build/pyi_rth_fiat.py @@ -1,15 +1,16 @@ """ -Pyinstaller hook that sets mandatory env var. +Pyinstaller hook that sets mandatory env_vars. Based on https://github.com/conda-forge/gdal-feedstock/blob/main/recipe/scripts/activate.sh +and pyproj hooks. """ import os import sys # Installing `osgeo` Conda packages with plugins requires to set `GDAL_DRIVER_PATH` - is_win = sys.platform.startswith("win") + if is_win: gdal_plugins = os.path.join(sys._MEIPASS, "share", "gdalplugins") if not os.path.exists(gdal_plugins): @@ -23,3 +24,11 @@ if os.path.exists(gdal_plugins): os.environ["GDAL_DRIVER_PATH"] = gdal_plugins + +if is_win: + proj_data = os.path.join(sys._MEIPASS, "Library", "share", "proj") +else: + proj_data = os.path.join(sys._MEIPASS, "share", "proj") + +if os.path.exists(proj_data): + os.environ["PROJ_LIB"] = proj_data diff --git a/.build/runtime_hooks.py b/.build/runtime_hooks.py deleted file mode 100644 index 7838bb97..00000000 --- a/.build/runtime_hooks.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Runtime hooks for pyinstaller.""" - -import os -import sys -from pathlib import Path - -cwd = Path(sys.executable).parent - -os.environ["PROJ_LIB"] = str(Path(cwd, "bin", "share")) -sys.path.append(str(Path(cwd, "bin", "share")))