Fix build issues

.build/build.spec

@@ -1,8 +1,6 @@
 """Spec file for building fiat."""
 
-import importlib
 import inspect
-import os
 import sys
 import time
 from pathlib import Path
@@ -11,38 +9,32 @@ from fiat.util import generic_folder_check
 
 # Pre build event setup
 app_name = "fiat"
+mode = "Release"
 sys.setrecursionlimit(5000)
 
 # Some general information
 _file = Path(inspect.getfile(lambda: None))
-cwd = _file.parent
-env_path =  os.path.dirname(sys.executable)
-generic_folder_check(Path(cwd, "../bin"))
-mode = "Release"
+project_root = _file.parents[1]
+build_dir = Path(project_root, ".build")
+
+generic_folder_check(Path(project_root, "bin"))
+
 # Set the build time for '--version' usage
 now = time.localtime(time.time())
 FIAT_BUILD_TIME = time.strftime('%Y-%m-%dT%H:%M:%S UTC%z', now)
-with open(Path(cwd, "fiat_build_time.py"), "w") as _w:
+with open(Path(build_dir, "fiat_build_time.py"), "w") as _w:
     _w.write(f'BUILD_TIME = "{FIAT_BUILD_TIME}"')
 
-# Get the location of the proj database
-proj = Path(os.environ["PROJ_LIB"])
-
-# Add to the list of binaries (although its a database)
-binaries = [
-    (Path(proj, 'proj.db'), './share'),
-]
-
 # Build event
 a = Analysis(
-    [Path(cwd, "../src/fiat/cli/main.py")],
-    pathex=[Path(cwd), Path(cwd, "../src")],
-    binaries=binaries,
+    [Path(project_root, "src/fiat/cli/main.py")],
+    pathex=[Path(build_dir), Path(project_root, "src")],
+    binaries=[],
     datas=[],
     hiddenimports=["fiat_build_time"],
-    hookspath=[],
+    hookspath=[build_dir.as_posix()],
     hooksconfig={},
-    runtime_hooks=[Path(cwd, 'runtime_hooks.py')],
+    runtime_hooks=[Path(build_dir, 'runtime_hooks.py')],
     excludes=[],
     win_no_prefer_redirects=False,
     win_private_assemblies=False,

.build/hook-fiat.py

@@ -0,0 +1,43 @@
+"""Build hook for FIAT."""
+
+import glob
+import os
+import sys
+from pathlib import Path
+
+from osgeo.gdal import __version__ as gdal_version
+from packaging.version import Version
+from PyInstaller.compat import is_conda
+from PyInstaller.utils.hooks import logger
+from PyInstaller.utils.hooks.conda import (
+    distribution,
+)
+
+datas = []
+
+if hasattr(sys, "real_prefix"):  # check if in a virtual environment
+    root_path = sys.real_prefix
+else:
+    root_path = sys.prefix
+
+if is_conda and Version(gdal_version) >= Version("3.9.1"):
+    plugin = distribution("libgdal-netcdf")
+
+    # Look for all the plugins
+    plugin_dir = Path(root_path, plugin.files[0].parent)
+    all_plugins = glob.glob(Path(plugin_dir, "*").as_posix())
+
+    # Append the data
+    datas += list(map(lambda path: (path, "./gdalplugins"), all_plugins))
+
+# Sort out the proj database
+src_proj = None
+if "PROJ_DATA" in os.environ:
+    src_proj = os.environ["PROJ_DATA"]
+elif "PROJ_LIB" in os.environ:
+    src_proj = os.environ["PROJ_LIB"]
+else:
+    logger.warning("Proj data was not found.")
+
+if src_proj is not None:
+    datas.append((src_proj, "./share"))

.build/linux64.sh

@@ -1,32 +1,107 @@
-echo "Setup paths relative to this script"
 #!/usr/bin/bash
 # Absolute path to this script, e.g. /home/user/bin/foo.sh
 SCRIPT=$(readlink -f "$0")
-# Absolute path this script is in, thus /home/user/bin
 SCRIPTPATH=$(dirname "$SCRIPT")
+PROJECTPATH=$(dirname "$SCRIPTPATH")
+PIXIPATH=$PROJECTPATH/.pixi
 
-# Setting up..
-echo "Locating conda.."
-paths=$(which -a conda)
-conda_executable=$(echo "$paths" | grep "^$HOME")
-
-if [ -z "$conda_executable" ]
-then
-  # If home_conda is empty, grep with "/home/share"
-  conda_executable=$(echo "$paths" | grep "^/usr/share")
+bin_var=conda
+shell_var=bash
+
+# Help message
+function help_message {
+  echo "Building FIAT on linux systems."
+  echo "Usage: $0 [-b value] [-h | --help]"
+  echo ""
+  echo "Options:"
+  echo $'\t'"-b"$'\t'"Binary name for python environment creation (default: $bin_var)"
+  echo $'\t'"-s"$'\t'"Shell type (default: $shell_var)"
+  echo $'\t'"-h,"$'\t'"Display this help message"$'\n\t'"--help"
+}
+
+# # Check if no arguments are provided
+# if [[ $# -eq 0 ]]; then
+#   help_message
+#   exit 0
+# fi
+
+# Parsing the cli input
+while [[ "$1" != "" ]]; do
+  case $1 in
+    -b ) shift
+         bin_var=$1
+         ;;
+    -s ) shift
+         shell_var=$1
+         ;;
+    -h | --help ) help_message
+                  exit 0
+                  ;;
+    * ) echo "Invalid option: $1"
+        help_message
+        exit 1
+  esac
+  shift
+done
+
+# Valid bin values
+valid_values=("conda" "pixi")
+
+# Check if value for binary is valid
+is_valid=false
+for value in "${valid_values[@]}"; do
+  if [[ "$bin_var" == "$value" ]]; then
+    is_valid=true
+    break
+  fi
+done
+
+if [ $is_valid == false ]; then
+  echo "Not a valid python env system: $bin_var"
+  exit 1
 fi
 
-if [ -z "$conda_executable" ]
-then
-  conda_executable="/home/runner/miniconda3/condabin/conda"
+# Setting up..
+echo "INFO: Locating $bin_var"
+paths=$(which -a $bin_var)
+executable=$(echo "$paths" | grep "^$HOME")
+
+if [ -z "$executable" ] && [ $bin_var != "conda" ]; then
+  echo "Cannot find binary for: $bin_var"
+  exit 1
+elif [ -z "$executable" ]; then
+  executable="/home/runner/miniconda3/condabin/conda"
+  if [ ! -e $executable ]; then
+    echo "Cannot find binary for: $bin_var"
+    exit 1
+  fi
 fi
 
-conda_base_dir=$(dirname $(dirname $conda_executable))
-source $conda_base_dir/etc/profile.d/conda.sh
+# if [ -z "$executable" ]
+# then
+#   # If home_conda is empty, grep with "/home/share"
+#   executable=$(echo "$paths" | grep "^/usr/share")
+# fi
+
+# if [ -z "$conda_executable" ]
+# then
+#   conda_executable="/home/runner/miniconda3/condabin/conda"
+# fi
+
+echo "INFO: Executable found here: $executable"
+
+bin_dir=$(dirname $(dirname $executable))
+
+if [ $bin_var == "conda" ]; then
+  source $bin_dir/etc/profile.d/conda.sh
+  conda activate fiat_build
+  export PROJ_LIB=$bin_dir/envs/fiat_build/share/proj
+elif [ $bin_var == "pixi" ]; then
+  eval $(pixi shell-hook --manifest-path $PROJECTPATH/pixi.toml -s $shell_var -e build)
+  export PROJ_LIB=$PIXIPATH/envs/build/share/proj
+fi
 
 # Do the thing!
-echo "Build stuff.."
-conda activate fiat_build
-export PROJ_LIB=$conda_base_dir/envs/fiat_build/share/proj
+echo "INFO: Building binary.."
 pip install -e "$SCRIPTPATH/.."
 pyinstaller "$SCRIPTPATH/build.spec" --distpath $SCRIPTPATH/../bin --workpath $SCRIPTPATH/../bin/intermediates

.build/runtime_hooks.py

@@ -8,5 +8,6 @@
 cwd = Path(sys.executable).parent
 
 # Paths to libaries
+os.environ["GDAL_DRIVER_PATH"] = Path(cwd, "bin", "gdalplugins").as_posix()
 os.environ["PROJ_LIB"] = str(Path(cwd, "bin", "share"))
 sys.path.append(str(Path(cwd, "bin", "share")))

.github/workflows/sonar.yml

@@ -4,7 +4,13 @@ on:
   push:
     branches:
       - master
+    paths:
+      - src/fiat/*
+      - test/*
   pull_request:
+    paths:
+      - src/fiat/*
+      - test/*
     types: [opened, synchronize, reopened]
 
 jobs:

Dockerfile

@@ -1,7 +1,7 @@
-FROM debian:bookworm-slim as base
+FROM debian:bookworm-slim AS base
 ARG PIXIENV
 ARG UID=1000
-RUN apt-get update && apt-get install -y curl
+RUN apt-get update && apt-get install -y curl && apt-get install -y vim && apt-get install -y binutils
 
 RUN useradd deltares
 RUN usermod -u ${UID} deltares
@@ -12,6 +12,8 @@ RUN curl -fsSL https://pixi.sh/install.sh | bash
 ENV PATH=/home/deltares/.pixi/bin:$PATH
 COPY pixi.toml pyproject.toml README.md ./
 COPY --chown=deltares:deltares src/fiat ./src/fiat
+# Unquote for building inside the docker
+# COPY --chown=deltares:deltares .build ./.build
 
 RUN chmod u+x src/ \
   && pixi run -e ${PIXIENV} install-fiat \
@@ -24,5 +26,5 @@ ENV RUNENV="${PIXIENV}"
 RUN echo "pixi run --locked -e ${RUNENV} \$@" > run_pixi.sh \
   && chown deltares:deltares run_pixi.sh \
   && chmod u+x run_pixi.sh
-ENTRYPOINT ["sh", "run_pixi.sh"]
-CMD ["fiat","info"]
+ENTRYPOINT ["bash", "run_pixi.sh"]
+CMD ["fiat"]

docs/changelog.qmd

@@ -18,9 +18,11 @@ This contains the unreleased changes to Delft-FIAT.
 - Numpy >= 2.0.0 support
 - Python 3.12 support
 - Setting return period as a variable in hazard map bands (risk)
+- Support for using pixi for binary creation (properly)
 
 ### Changed
 - Better version of `BufferHandler`
+- Fixed binary creation in general, but also specifically for `GDAL >= v3.9.1`
 - Made read methods of `BaseModel`, `GeomModel` and `GridModel` public (removed underscore)
 - Testing of workers (not properly caught due to using `multiprocessing`)
 - Testing only based on integers