diff --git a/.github/workflows/dive.yml b/.github/workflows/dive.yml index 83372dc8..483b7303 100644 --- a/.github/workflows/dive.yml +++ b/.github/workflows/dive.yml @@ -83,11 +83,11 @@ jobs: sudo rm -rf ${GITHUB_WORKSPACE}/.git - name: Dive - run: | - docker pull ${DIVE} - docker run -e CI=true -e DOCKER_API_VERSION=${DOCKER_API} --rm -v /var/run/docker.sock:/var/run/docker.sock \ - --mount type=bind,source=${{ github.workspace }}/dive-ci.yml,target=/.dive-ci \ - ${DIVE} --ci-config /.dive-ci ${ORG}/${IMAGE}:_build + uses: MaxymVlasov/dive-action@v1.0.1 + with: + image: ${{ env.ORG }}/${{ env.IMAGE}}:_build + config-file: ${{ github.workspace }}/dive-ci.yml + github-token: ${{ secrets.GITHUB_TOKEN }} - name: Docker image size check uses: wemake-services/docker-image-size-limit@master diff --git a/docker/env.yaml b/docker/env.yaml index 182be01e..538082ff 100644 --- a/docker/env.yaml +++ b/docker/env.yaml @@ -57,6 +57,7 @@ dependencies: - distributed>=2023.11 - docutils - ephem + - eodatasets3 - fiona - Flask - Flask-Babel @@ -94,6 +95,10 @@ dependencies: - nodejs - numexpr - numpy + - odc-dscache + - odc-geo + - odc-io + - odc-stac - ordered-set - packaging - pandas diff --git a/docker/requirements.txt b/docker/requirements.txt index cc82e74b..0f4a2e84 100644 --- a/docker/requirements.txt +++ b/docker/requirements.txt @@ -12,7 +12,6 @@ jupyterlab-logout==0.5.0 # ODC/DEA: these are installed in builder stage otps -eodatasets3 # Dale's s2cloudmask # https://github.com/daleroberts/s2cloudmask @@ -24,12 +23,8 @@ opencv-contrib-python-headless datacube[performance,s3] >= 1.8.19 odc-algo @ git+https://github.com/opendatacube/odc-algo@b8dcfce odc-cloud[ASYNC] -odc-dscache -odc-io -odc-stac odc-stats[ows] odc-ui -odc-geo dea-tools thredds-crawler diff --git a/nci_environment/README.md b/nci_environment/README.md index 743e859f..c0f5bb86 100644 --- a/nci_environment/README.md +++ b/nci_environment/README.md @@ -2,25 +2,22 @@ These scripts are used to update and deploy DEA modules on NCI. -There are two modules, with date-based version numbers: +The *dea* module, with a date-based version number, contains third party dependencies +of all of the DEA code, installed via a `conda` environment. This `conda` environment is the same +as the one for dea-sandbox. - 1. The Python Environment module *dea-env* - - * This contains third party dependencies of all of the DEA code, installed via - a `conda` environment. - - 2. A *dea* module, which depends on the _environment module_: +Additionally, the environment includes: * [Open Data Cube Core](https://github.com/opendatacube/datacube-core/) - * [EO Datasets](https://github.com/GeoscienceAustralia/eo-datasets/) - * [Digital Earth AU](https://github.com/GeoscienceAustralia/digitalearthau/) * [Data Cube Stats](https://github.com/GeoscienceAustralia/datacube-stats/) * [Fractional Cover](https://github.com/GeoscienceAustralia/fc/) + * [Water Observation From Space](https://github.com/GeoscienceAustralia/wofs) + * Creates users accounts in the Production Database the first time it is loaded by a user. @@ -34,14 +31,11 @@ There are two modules, with date-based version numbers: This will load the latest version of `dea/` module. -It will also load `dea-env/` which contains all of the software -dependencies for using DEA. - ## Notes -Loading these module might conflict with other python modules you have loaded. +Loading this module might conflict with other python modules you have loaded. -The `dea-env` module will prevent conflicts with locally installed python packages by +The `dea` module will prevent conflicts with locally installed python packages by changing `PYTHONUSERBASE` for each release; pip install --user @@ -54,33 +48,23 @@ It includes a config file, which it specifies by setting the # Maintainer Instructions -Only run these scripts from Raijin. We've seen filesystem sync issues when +Only run these scripts from Gadi. We've seen filesystem sync issues when run from VDI. - module load python3/3.8.5 + module load python3/3.10.4 pip3 install --user pyyaml jinja2 -## Building a new _Environment Module_ - -It requires python 3.8+ and pyyaml. Run the following on raijin at the NCI: - - $ module use /g/data/v10/public/modules/modulefiles/ - $ module load python3/3.8.5 - $ ./build_environment_module.py dea-env/modulespec.yaml - -This will build a new environment module for today. - -The module version number is the current date in format YYYYMMDD, as it is a snapshot -of all of our pip/conda dependencies on that date. - ## Building a new _DEA Module_ -A DEA module will specify one exact environment module. +It requires python 3.10+ and pyyaml. Run the following on gadi at the NCI: $ module use /g/data/v10/public/modules/modulefiles/ - $ module load python3/3.8.5 + $ module load python3/3.10.4 $ ./build_environment_module.py dea/modulespec.yaml +The module version number is the current date in format YYYYMMDD, as it is a snapshot +of all of our pip/conda dependencies on that date. + ## Updating the Default Version Once a module has been tested and approved, it can be made the default. @@ -89,14 +73,6 @@ Edit the `.version` file in the modulefiles directory. Eg. For `dea` this is: `/g/data/v10/public/modules/modulefiles/dea/.version` - -## Re-Building _dea-unstable_ - - module load python3/3.6.2 - rm -rf /g/data/v10/public/modules/dea/unstable - python3 build_environment_module.py dea_unstable/modulespec.yaml - - ## Archiving an old module [TO DO]... @@ -108,14 +84,13 @@ Eg. For `dea` this is: `/g/data/v10/public/modules/modulefiles/dea/.version` ## Setup Copy the 3 lines below and modify the VERSION value - to the dea and dea-env module version you would + to the dea module version you would like the tests to be run on. Paste them in a brand new shell session/terminal VERSION="20230710" module use /g/data/v10/public/modules/modulefiles module load dea/$VERSION - module load dea-env/$VERSION ## Execution On gadi, just run the tests with in this fashion: diff --git a/nci_environment/build_environment_module.py b/nci_environment/build_environment_module.py index 98750fe0..c70cafc8 100755 --- a/nci_environment/build_environment_module.py +++ b/nci_environment/build_environment_module.py @@ -9,21 +9,13 @@ - (opt) Conda environment to create - (opt) Pip style requirements.txt to install to a directory -It requires python 3.8+ and pyyaml. +It requires python 3.10+ and pyyaml. Use a qsub interactive copyq job on raijin with sufficient memory to run the following commands at the NCI: -New DEA-Env Module $ module use /g/data/v10/public/modules/modulefiles/ - $ module load python3/3.8.5 + $ module load python3/3.10.4 # if pyyaml is not installed in gadi $ pip install PyYAML --user - $ # Building a new Environment Module: - $ python3 build_environment_module.py dea-env/modulespec.yaml - -New DEA Module - $ module use /g/data/v10/public/modules/modulefiles/ - $ module load python3/3.8.5 - $ # Building a new DEA Module $ python3 build_environment_module.py dea/modulespec.yaml @@ -262,8 +254,8 @@ def copy_and_fill_templates(template_tasks, variables): src = Path(task["src"]) dest = Path(task["dest"]) - LOG.info("Copy and fill dea-env modulefile %s in %s", src, dest) - # Write the module file template to modulefiles/dea-env directory + LOG.info("Copy and fill dea modulefile %s in %s", src, dest) + # Write the module file template to modulefiles/dea directory write_template(src, variables, dest) if "chmod" in task: @@ -361,20 +353,6 @@ def run_final_commands_on_module(commands, module_path): run_command(cmd) -def include_stable_module_dep_versions(config): - """ - Include stable module dependency versions - - :param config: Dictionary of configuration variables - :return: None - """ - stable_module_deps = config.get("stable_module_deps", []) - for dep in stable_module_deps: - default_version = find_default_version(dep) - dep = dep.replace("-", "_") - config["variables"][f"fixed_{dep}"] = default_version - - def main(config_path): """ Build new environment module @@ -394,7 +372,6 @@ def main(config_path): if "module_version" not in variables: variables["module_version"] = date() include_templated_vars(config) - include_stable_module_dep_versions(config) pre_check(config) prep(config_path) diff --git a/nci_environment/dea-env/modulefile.template b/nci_environment/dea-env/modulefile.template deleted file mode 100644 index 81b03495..00000000 --- a/nci_environment/dea-env/modulefile.template +++ /dev/null @@ -1,47 +0,0 @@ -#%Module######################################################################## -## -## ${module_name} modulefile -## - -proc ModulesHelp { } { - global version - - puts stderr " ${module_description}" - puts stderr " Version ${module_version}" -} - -set name ${module_name} -set version ${module_version} -set base ${module_path} - -module-whatis "${module_description} ${module_version}" - -# Enable segfault tracebacks in py3. https://docs.python.org/3/library/faulthandler.html -setenv PYTHONFAULTHANDLER 1 - -# Set PYTHONUSERBASE based on the version of dea-env module. -# This allows users to install python packages with "pip install --user ", -setenv PYTHONUSERBASE ~/.dea-sandbox/${module_name}/${module_version}/local -setenv GDAL_DATA ${module_path}/share/gdal -setenv PROJ_LIB ${module_path}/share/proj -setenv PROJ_NETWORK ON -setenv UDUNITS2_XML_PATH ${module_path}/share/udunits/udunits2.xml -setenv GSETTINGS_SCHEMA_DIR ${module_path}/share/glib-2.0/schemas - -if {[module-info mode load] && [info exists env(PYTHONPATH)] && $$env(PYTHONPATH) != ""} { - puts stderr "Warning: ${module_name}/${module_version} exists in the python env ($$env(PYTHONPATH))" - puts stderr "Unload all python modules, if you experience any issues." -} - -prepend-path PYTHONPATH ${module_path}/share/qgis/python - -# Remove duplicate entries for module path dir and prepend again -remove-path PATH ${module_path}/bin -prepend-path PATH ${module_path}/bin - -# To avoid user packages conflicting with Environment Module packages, point the PYTHONUSERBASE and PATH -# variables to point to a directory based on the Environment Module version which is loaded so that extra -# packages must be re-installed when a new dea-env module is released -# Remove duplicate entries for HOME dir and prepend again -remove-path PATH ~/.dea-sandbox/${module_name}/${module_version}/local/bin -prepend-path PATH ~/.dea-sandbox/${module_name}/${module_version}/local/bin diff --git a/nci_environment/dea-env/modulespec.yaml b/nci_environment/dea-env/modulespec.yaml deleted file mode 100644 index a0c5c474..00000000 --- a/nci_environment/dea-env/modulespec.yaml +++ /dev/null @@ -1,23 +0,0 @@ -variables: - module_name: dea-env - module_description: DEA Environment Module - modules_dir: "/g/data/v10/public/modules" - conda_path: "/g/data/v10/private/mambaforge/bin/mamba" - python_version: 3.8 - -# These templated variables are filled and included in the available variables used -# in template files and configuration sections below -templated_variables: - module_path: "{modules_dir}/{module_name}/{module_version}" - python_path: "{modules_dir}/{module_name}/{module_version}/lib/python{python_version}/site-packages/" - -install_conda_packages: ../../docker/env.yaml - -copy_files: -- src: ../../docker/env.yaml - dest: "{modules_dir}/{module_name}/{module_version}/environment.yaml" - -template_files: -- src: modulefile.template - dest: "{modules_dir}/modulefiles/{module_name}/{module_version}" - chmod: 0o444 diff --git a/nci_environment/dea/modulefile.template b/nci_environment/dea/modulefile.template index d451542e..b8d86eec 100644 --- a/nci_environment/dea/modulefile.template +++ b/nci_environment/dea/modulefile.template @@ -16,29 +16,53 @@ set base ${module_path} module-whatis "${module_description} ${module_version}" -module load ${fixed_dea_env} +# Enable segfault tracebacks in py3. https://docs.python.org/3/library/faulthandler.html +setenv PYTHONFAULTHANDLER 1 + +# Set PYTHONUSERBASE based on the version of dea module. +# This allows users to install python packages with "pip install --user ", +setenv PYTHONUSERBASE ~/.dea-sandbox/${module_name}/${module_version}/local +setenv GDAL_DATA ${module_path}/share/gdal +setenv PROJ_LIB ${module_path}/share/proj +setenv PROJ_NETWORK ON +setenv UDUNITS2_XML_PATH ${module_path}/share/udunits/udunits2.xml +setenv GSETTINGS_SCHEMA_DIR ${module_path}/share/glib-2.0/schemas + +if {[module-info mode load] && [info exists env(PYTHONPATH)] && $$env(PYTHONPATH) != ""} { + puts stderr "Warning: ${module_name}/${module_version} exists in the python env ($$env(PYTHONPATH))" + puts stderr "Unload all python modules, if you experience any issues." +} + +prepend-path PYTHONPATH ${module_path}/share/qgis/python + +# Remove duplicate entries for module path dir and prepend again +remove-path PATH ${module_path}/bin +prepend-path PATH ${module_path}/bin + +# Remove duplicate entries for python path and prepend again +remove-path PYTHONPATH ${python_path} +prepend-path PYTHONPATH ${python_path} + +# To avoid user packages conflicting with Environment Module packages, point the PYTHONUSERBASE and PATH +# variables to point to a directory based on the Environment Module version which is loaded so that extra +# packages must be re-installed when a new dea module is released +# Remove duplicate entries for HOME dir and prepend again +remove-path PATH ~/.dea-sandbox/${module_name}/${module_version}/local/bin +prepend-path PATH ~/.dea-sandbox/${module_name}/${module_version}/local/bin setenv DATACUBE_CONFIG_PATH ${module_path}/datacube.conf setenv LC_ALL en_AU.utf8 setenv LANG C.UTF-8 +setenv DEA_TOOLS_TIDE_MODELS /g/data/v10/tide_models + if {[module-info mode load] && [is-loaded $$name/$$version]} { puts stderr "Warning: $$name/$$version exists in the python env ($$env(PYTHONPATH))" } -# To avoid user packages conflicting with Environment Module packages, point the PYTHONUSERBASE and PATH -# variables to point to a directory based on the Environment Module version which is loaded so that extra -# packages must be re-installed when a new dea-env module is released -prepend-path PATH ${module_path}/bin -prepend-path PYTHONPATH ${python_path} - -# Remove duplicate entries for HOME dir and prepend at the top -remove-path PATH ~/.dea-sandbox/${fixed_dea_env}/local/bin -prepend-path PATH ~/.dea-sandbox/${fixed_dea_env}/local/bin - # Remove duplicate entries for HOME dir and prepend at the top -remove-path PYTHONPATH ~/.dea-sandbox/${fixed_dea_env}/local/lib/python3.6/site-packages -prepend-path PYTHONPATH ~/.dea-sandbox/${fixed_dea_env}/local/lib/python3.6/site-packages +remove-path PYTHONPATH ~/.dea-sandbox/${module_name}/${module_version}/local/lib/python3.10/site-packages +prepend-path PYTHONPATH ~/.dea-sandbox/${module_name}/${module_version}/local/lib/python3.10/site-packages ############################################################# diff --git a/nci_environment/dea/modulespec.yaml b/nci_environment/dea/modulespec.yaml index 42919bab..f7f4b379 100644 --- a/nci_environment/dea/modulespec.yaml +++ b/nci_environment/dea/modulespec.yaml @@ -1,23 +1,27 @@ variables: module_name: dea module_description: DEA tools for the NCI - modules_base: "/g/data/v10/public/modules" + modules_dir: "/g/data/v10/public/modules" + conda_path: "/g/data/v10/private/mambaforge/bin/mamba" dbhost: dea-db.nci.org.au dbport: 6432 - python_version: 3.8 + python_version: "3.10" templated_variables: - module_path: "{modules_base}/{module_name}/{module_version}" - python_path: "{modules_base}/{module_name}/{module_version}/lib/python{python_version}/site-packages/" + module_path: "{modules_dir}/{module_name}/{module_version}" + python_path: "{modules_dir}/{module_name}/{module_version}/lib/python{python_version}/site-packages/" dea_module: "{module_name}/{module_version}" + pip_path: "{modules_dir}/{module_name}/{module_version}/bin/pip3" -stable_module_deps: -- dea-env +install_conda_packages: ../../docker/env.yaml install_pip_packages: - pip_cmd: "module load {fixed_dea_env}; pip install --no-warn-script-location --prefix {module_path} --requirement requirements.txt; pip install --no-warn-script-location --prefix {module_path} --requirement requirements-private.txt" + # need to specify which python/pip to use otherwise it defaults to the incorrect one + pip_cmd: "{pip_path} install --no-warn-script-location --prefix {module_path} --requirement requirements.txt; {pip_path} install --no-warn-script-location --prefix {module_path} --requirement requirements-private.txt" copy_files: +- src: ../../docker/env.yaml + dest: "{modules_dir}/{module_name}/{module_version}/environment.yaml" - src: requirements.txt dest: "{module_path}/requirements.txt" - src: datacube.conf @@ -39,4 +43,5 @@ wget_files: template_files: - src: modulefile.template - dest: "{modules_base}/modulefiles/{module_name}/{module_version}" + dest: "{modules_dir}/modulefiles/{module_name}/{module_version}" + chmod: 0o444 \ No newline at end of file diff --git a/nci_environment/dea/requirements.txt b/nci_environment/dea/requirements.txt index bad0c985..f3992ffd 100644 --- a/nci_environment/dea/requirements.txt +++ b/nci_environment/dea/requirements.txt @@ -1,8 +1,6 @@ -datacube -eodatasets3 -odc-algo +datacube >= 1.8.18 +pytest-odc +odc-algo @ git+https://github.com/opendatacube/odc-algo@b8dcfce odc-apps-dc-tools -odc-dscache odc-geom -odc-io odc-ui diff --git a/nci_environment/dea_unstable/modulespec.yaml b/nci_environment/dea_unstable/modulespec.yaml deleted file mode 100644 index 05c189f6..00000000 --- a/nci_environment/dea_unstable/modulespec.yaml +++ /dev/null @@ -1,43 +0,0 @@ -variables: - module_name: dea - module_description: DEA tools for the NCI - modules_base: "/g/data/v10/public/modules" - dbhost: dea-db.nci.org.au - dbport: 6432 - python_version: 3.6 - module_version: unstable - -templated_variables: - module_path: "{modules_base}/{module_name}/{module_version}" - python_path: "{modules_base}/{module_name}/{module_version}/lib/python{python_version}/site-packages/" - dea_module: "{module_name}/{module_version}" - -stable_module_deps: -- dea-env - -install_pip_packages: - pip_cmd: "module load {fixed_dea_env}; pip install --no-warn-script-location --prefix {module_path} --requirement requirements.txt" - -copy_files: -- src: ../dea/requirements.txt - dest: "{module_path}/requirements.txt" -- src: ../dea/datacube.conf - dest: "{module_path}/datacube.conf" -- src: ../dea/scripts/datacube-ensure-user.py - dest: "{module_path}/bin/datacube-ensure-user.py" - chmod: 0o755 -- src: ../dea/scripts/add-aws-vault - dest: "{module_path}/bin/add-aws-vault" - chmod: 0o755 -- src: ../dea/scripts/login-aws-vault - dest: "{module_path}/bin/login-aws-vault" - chmod: 0o755 - -wget_files: -- src: "https://github.com/99designs/aws-vault/releases/download/v6.3.1/aws-vault-linux-amd64" - dest: "{module_path}/bin/aws-vault" - chmod: 0o755 - -template_files: -- src: ../dea/modulefile.template - dest: "{modules_base}/modulefiles/{module_name}/{module_version}" diff --git a/nci_environment/dea_unstable/requirements.txt b/nci_environment/dea_unstable/requirements.txt deleted file mode 100644 index 5ed66a4a..00000000 --- a/nci_environment/dea_unstable/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ ---pre --r ../dea/requirements.txt diff --git a/nci_environment/test-module.sh b/nci_environment/test-module.sh index caaf74e5..fb33a1c2 100644 --- a/nci_environment/test-module.sh +++ b/nci_environment/test-module.sh @@ -2,6 +2,10 @@ # Module to test +if [[ -z $1 ]]; then + echo "ERROR: no module provided" + exit 1 +fi module_to_test="$1" echo Testing module: "$module_to_test" @@ -10,7 +14,7 @@ echo Testing module: "$module_to_test" export MODULEPATH=/apps/.mf:/opt/Modules/modulefiles:/apps/Modules/modulefiles: # Ensure that all files are world readable -if [[ $(find "$module_to_test" ! -perm /004) ]]; then +if [[ $(find /g/data/v10/public/modules/modulefiles/"$module_to_test" ! -perm /004) ]]; then echo "ERROR: Some files in $module_to_test are not world readable." exit 1 fi @@ -26,6 +30,12 @@ module load "$module_to_test" # TODO Check the output from: datacube system check +echo Testing notebooks +cd ../.. +if [[ ! -d dea-notebooks ]]; then + git clone --depth 1 https://github.com/GeoscienceAustralia/dea-notebooks.git +fi +./dea-notebooks/Tests/test_notebooks.sh # Run the initialise Test Database Script