GeoscienceAustralia · Ariana-B · Nov 16, 2023 · Nov 22, 2023 · Nov 22, 2023 · Nov 23, 2023
diff --git a/.github/workflows/dive.yml b/.github/workflows/dive.yml
@@ -83,11 +83,11 @@ jobs:
           sudo rm -rf ${GITHUB_WORKSPACE}/.git
 
       - name: Dive
-        run: |
-          docker pull ${DIVE}
-          docker run -e CI=true -e DOCKER_API_VERSION=${DOCKER_API} --rm -v /var/run/docker.sock:/var/run/docker.sock \
-          --mount type=bind,source=${{ github.workspace }}/dive-ci.yml,target=/.dive-ci \
-          ${DIVE}  --ci-config /.dive-ci ${ORG}/${IMAGE}:_build
+        uses: MaxymVlasov/[email protected]
+        with:
+          image: ${{ env.ORG }}/${{ env.IMAGE}}:_build
+          config-file: ${{ github.workspace }}/dive-ci.yml
+          github-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Docker image size check
         uses: wemake-services/docker-image-size-limit@master

diff --git a/docker/env.yaml b/docker/env.yaml
@@ -57,6 +57,7 @@ dependencies:
   - distributed>=2023.11
   - docutils
   - ephem
+  - eodatasets3
   - fiona
   - Flask
   - Flask-Babel
@@ -94,6 +95,10 @@ dependencies:
   - nodejs
   - numexpr
   - numpy
+  - odc-dscache
+  - odc-geo
+  - odc-io
+  - odc-stac
   - ordered-set
   - packaging
   - pandas

diff --git a/docker/requirements.txt b/docker/requirements.txt
@@ -12,7 +12,6 @@ jupyterlab-logout==0.5.0
 
 # ODC/DEA: these are installed in builder stage
 otps
-eodatasets3
 
 # Dale's s2cloudmask
 #  https://github.com/daleroberts/s2cloudmask
@@ -24,12 +23,8 @@ opencv-contrib-python-headless
 datacube[performance,s3] >= 1.8.19
 odc-algo @ git+https://github.com/opendatacube/odc-algo@b8dcfce
 odc-cloud[ASYNC]
-odc-dscache
-odc-io
-odc-stac
 odc-stats[ows]
 odc-ui
-odc-geo
 dea-tools
 
 thredds-crawler

diff --git a/nci_environment/README.md b/nci_environment/README.md
@@ -2,25 +2,22 @@
 
 These scripts are used to update and deploy DEA modules on NCI.
 
-There are two modules, with date-based version numbers:
+The *dea* module, with a date-based version number, contains third party dependencies
+of all of the DEA code, installed via a `conda` environment. This `conda` environment is the same
+as the one for dea-sandbox.
 
- 1. The Python Environment module *dea-env*
-
-    * This contains third party dependencies of all of the DEA code, installed via
-      a `conda` environment.
-
- 2. A *dea* module, which depends on the _environment module_:
+Additionally, the environment includes:
 
     * [Open Data Cube Core](https://github.com/opendatacube/datacube-core/)
 
-    * [EO Datasets](https://github.com/GeoscienceAustralia/eo-datasets/)
-
     * [Digital Earth AU](https://github.com/GeoscienceAustralia/digitalearthau/)
 
     * [Data Cube Stats](https://github.com/GeoscienceAustralia/datacube-stats/)
 
     * [Fractional Cover](https://github.com/GeoscienceAustralia/fc/)
 
+    * [Water Observation From Space](https://github.com/GeoscienceAustralia/wofs)
+
     * Creates users accounts in the Production Database the first time it is
       loaded by a user.
 
@@ -34,14 +31,11 @@ There are two modules, with date-based version numbers:
 
 This will load the latest version of `dea/<build_date>` module.
 
-It will also load `dea-env/<build_date>` which contains all of the software
-dependencies for using DEA.
-
 ## Notes
 
-Loading these module might conflict with other python modules you have loaded.
+Loading this module might conflict with other python modules you have loaded.
 
-The `dea-env` module will prevent conflicts with locally installed python packages by
+The `dea` module will prevent conflicts with locally installed python packages by
 changing `PYTHONUSERBASE` for each release;
 
     pip install --user <package_name>
@@ -54,33 +48,23 @@ It includes a config file, which it specifies by setting the
 
 # Maintainer Instructions
 
-Only run these scripts from Raijin. We've seen filesystem sync issues when
+Only run these scripts from Gadi. We've seen filesystem sync issues when
 run from VDI.
 
-    module load python3/3.8.5
+    module load python3/3.10.4
     pip3 install --user pyyaml jinja2
 
-## Building a new _Environment Module_
-
-It requires python 3.8+ and pyyaml. Run the following on raijin at the NCI:
-
-      $ module use /g/data/v10/public/modules/modulefiles/
-      $ module load python3/3.8.5
-      $ ./build_environment_module.py dea-env/modulespec.yaml
-
-This will build a new environment module for today.
-
-The module version number is the current date in format YYYYMMDD, as it is a snapshot
-of all of our pip/conda dependencies on that date.
-
 ## Building a new _DEA Module_
 
-A DEA module will specify one exact environment module.
+It requires python 3.10+ and pyyaml. Run the following on gadi at the NCI:
 
     $ module use /g/data/v10/public/modules/modulefiles/
-    $ module load python3/3.8.5
+    $ module load python3/3.10.4
     $ ./build_environment_module.py dea/modulespec.yaml
 
+The module version number is the current date in format YYYYMMDD, as it is a snapshot
+of all of our pip/conda dependencies on that date.
+
 ## Updating the Default Version
 
 Once a module has been tested and approved, it can be made the default.
@@ -89,14 +73,6 @@ Edit the `.version` file in the modulefiles directory.
 
 Eg. For `dea` this is: `/g/data/v10/public/modules/modulefiles/dea/.version`
 
-
-## Re-Building _dea-unstable_
-
-    module load python3/3.6.2
-    rm -rf /g/data/v10/public/modules/dea/unstable
-    python3 build_environment_module.py dea_unstable/modulespec.yaml
-
-
 ## Archiving an old module
 
 [TO DO]...
@@ -108,14 +84,13 @@ Eg. For `dea` this is: `/g/data/v10/public/modules/modulefiles/dea/.version`
 ## Setup
 
     Copy the 3 lines below and modify the VERSION value
-    to the dea and dea-env module version you would
+    to the dea module version you would
     like the tests to be run on. Paste them in a brand
     new shell session/terminal
 
         VERSION="20230710"
         module use /g/data/v10/public/modules/modulefiles
         module load dea/$VERSION
-        module load dea-env/$VERSION
 
 ## Execution
 On gadi, just run the tests with in this fashion:

diff --git a/nci_environment/build_environment_module.py b/nci_environment/build_environment_module.py
@@ -9,21 +9,13 @@
  - (opt) Conda environment to create
  - (opt) Pip style requirements.txt to install to a directory
 
-It requires python 3.8+ and pyyaml.
+It requires python 3.10+ and pyyaml.
 Use a qsub interactive copyq job on raijin with sufficient memory to run the following commands at the NCI:
-New DEA-Env Module
   $ module use /g/data/v10/public/modules/modulefiles/
-  $ module load python3/3.8.5
+  $ module load python3/3.10.4
   # if pyyaml is not installed in gadi
   $ pip install PyYAML --user
 
-  $ # Building a new Environment Module:
-  $ python3 build_environment_module.py dea-env/modulespec.yaml
-
-New DEA Module
-  $ module use /g/data/v10/public/modules/modulefiles/
-  $ module load python3/3.8.5
-
   $ # Building a new DEA Module
   $ python3 build_environment_module.py dea/modulespec.yaml
 
@@ -262,8 +254,8 @@ def copy_and_fill_templates(template_tasks, variables):
 
         src = Path(task["src"])
         dest = Path(task["dest"])
-        LOG.info("Copy and fill dea-env modulefile %s in %s", src, dest)
-        # Write the module file template to modulefiles/dea-env directory
+        LOG.info("Copy and fill dea modulefile %s in %s", src, dest)
+        # Write the module file template to modulefiles/dea directory
         write_template(src, variables, dest)
 
         if "chmod" in task:
@@ -361,20 +353,6 @@ def run_final_commands_on_module(commands, module_path):
         run_command(cmd)
 
 
-def include_stable_module_dep_versions(config):
-    """
-    Include stable module dependency versions
-
-    :param config: Dictionary of configuration variables
-    :return: None
-    """
-    stable_module_deps = config.get("stable_module_deps", [])
-    for dep in stable_module_deps:
-        default_version = find_default_version(dep)
-        dep = dep.replace("-", "_")
-        config["variables"][f"fixed_{dep}"] = default_version
-
-
 def main(config_path):
     """
     Build new environment module
@@ -394,7 +372,6 @@ def main(config_path):
     if "module_version" not in variables:
         variables["module_version"] = date()
     include_templated_vars(config)
-    include_stable_module_dep_versions(config)
 
     pre_check(config)
     prep(config_path)

diff --git a/nci_environment/dea-env/modulefile.template b/nci_environment/dea-env/modulefile.template
diff --git a/nci_environment/dea-env/modulespec.yaml b/nci_environment/dea-env/modulespec.yaml
diff --git a/nci_environment/dea/modulefile.template b/nci_environment/dea/modulefile.template
@@ -16,29 +16,53 @@ set base          ${module_path}
 
 module-whatis   "${module_description} ${module_version}"
 
-module load ${fixed_dea_env}
+# Enable segfault tracebacks in py3. https://docs.python.org/3/library/faulthandler.html
+setenv PYTHONFAULTHANDLER 1
+
+# Set PYTHONUSERBASE based on the version of dea module.
+# This allows users to install python packages with "pip install --user <package>",
+setenv PYTHONUSERBASE ~/.dea-sandbox/${module_name}/${module_version}/local
+setenv GDAL_DATA ${module_path}/share/gdal
+setenv PROJ_LIB ${module_path}/share/proj
+setenv PROJ_NETWORK ON
+setenv UDUNITS2_XML_PATH ${module_path}/share/udunits/udunits2.xml
+setenv GSETTINGS_SCHEMA_DIR ${module_path}/share/glib-2.0/schemas
+
+if {[module-info mode load] && [info exists env(PYTHONPATH)] && $$env(PYTHONPATH) != ""} {
+        puts stderr "Warning: ${module_name}/${module_version} exists in the python env ($$env(PYTHONPATH))"
+        puts stderr "Unload all python modules, if you experience any issues."
+}
+
+prepend-path PYTHONPATH ${module_path}/share/qgis/python
+
+# Remove duplicate entries for module path dir and prepend again
+remove-path PATH ${module_path}/bin
+prepend-path PATH ${module_path}/bin
+
+# Remove duplicate entries for python path and prepend again
+remove-path PYTHONPATH ${python_path}
+prepend-path PYTHONPATH ${python_path}
+
+# To avoid user packages conflicting with Environment Module packages, point the PYTHONUSERBASE and PATH
+# variables to point to a directory based on the Environment Module version which is loaded so that extra
+# packages must be re-installed when a new dea module is released
+# Remove duplicate entries for HOME dir and prepend again
+remove-path PATH ~/.dea-sandbox/${module_name}/${module_version}/local/bin
+prepend-path PATH ~/.dea-sandbox/${module_name}/${module_version}/local/bin
 
 setenv DATACUBE_CONFIG_PATH ${module_path}/datacube.conf
 setenv LC_ALL en_AU.utf8
 setenv LANG C.UTF-8
 
+setenv DEA_TOOLS_TIDE_MODELS /g/data/v10/tide_models
+
 if {[module-info mode load] && [is-loaded $$name/$$version]} {
         puts stderr "Warning: $$name/$$version exists in the python env ($$env(PYTHONPATH))"
 }
 
-# To avoid user packages conflicting with Environment Module packages, point the PYTHONUSERBASE and PATH
-# variables to point to a directory based on the Environment Module version which is loaded so that extra
-# packages must be re-installed when a new dea-env module is released
-prepend-path PATH ${module_path}/bin
-prepend-path PYTHONPATH ${python_path}
-
-# Remove duplicate entries for HOME dir and prepend at the top
-remove-path PATH ~/.dea-sandbox/${fixed_dea_env}/local/bin
-prepend-path PATH ~/.dea-sandbox/${fixed_dea_env}/local/bin
-
 # Remove duplicate entries for HOME dir and prepend at the top
-remove-path PYTHONPATH ~/.dea-sandbox/${fixed_dea_env}/local/lib/python3.6/site-packages
-prepend-path PYTHONPATH ~/.dea-sandbox/${fixed_dea_env}/local/lib/python3.6/site-packages
+remove-path PYTHONPATH ~/.dea-sandbox/${module_name}/${module_version}/local/lib/python3.10/site-packages
+prepend-path PYTHONPATH ~/.dea-sandbox/${module_name}/${module_version}/local/lib/python3.10/site-packages
 
 
 #############################################################