From 736b36383a967f56d08d47575525e6cc9ce3c9b6 Mon Sep 17 00:00:00 2001 From: Robbi Bishop-Taylor Date: Tue, 16 Jul 2024 14:04:35 +1000 Subject: [PATCH 1/7] Add minimum package versions for ESRI/GDAL nodata fix --- docker/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/requirements.txt b/docker/requirements.txt index cc82e74..ca6ed2b 100644 --- a/docker/requirements.txt +++ b/docker/requirements.txt @@ -12,7 +12,7 @@ jupyterlab-logout==0.5.0 # ODC/DEA: these are installed in builder stage otps -eodatasets3 +eodatasets3 => 0.30.6 # required for ESRI/GDAL nodata fix # Dale's s2cloudmask # https://github.com/daleroberts/s2cloudmask From 3919844e3aab7a64f35583c16e81adff27ce59c4 Mon Sep 17 00:00:00 2001 From: Robbi Bishop-Taylor Date: Wed, 17 Jul 2024 10:16:40 +1000 Subject: [PATCH 2/7] Use latest versions of odc-geo and eodatasets --- docker/requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/requirements.txt b/docker/requirements.txt index ca6ed2b..348af0e 100644 --- a/docker/requirements.txt +++ b/docker/requirements.txt @@ -12,7 +12,7 @@ jupyterlab-logout==0.5.0 # ODC/DEA: these are installed in builder stage otps -eodatasets3 => 0.30.6 # required for ESRI/GDAL nodata fix +eodatasets3 >= 0.30.6 # min version required for ESRI/GDAL nodata fix # Dale's s2cloudmask # https://github.com/daleroberts/s2cloudmask @@ -21,7 +21,7 @@ s2cloudmask opencv-python-headless opencv-contrib-python-headless -datacube[performance,s3] >= 1.8.19 +datacube[performance,s3] >= 1.8.19 # min version required for ESRI/GDAL nodata fix odc-algo @ git+https://github.com/opendatacube/odc-algo@b8dcfce odc-cloud[ASYNC] odc-dscache @@ -29,7 +29,7 @@ odc-io odc-stac odc-stats[ows] odc-ui -odc-geo +odc-geo >= 0.4.8 # min version required for ESRI/GDAL nodata fix dea-tools thredds-crawler From 0affe9099db33434422a4c6837d48688ad015cb2 Mon Sep 17 00:00:00 2001 From: Robbi Bishop-Taylor Date: Wed, 17 Jul 2024 13:29:17 +1000 Subject: [PATCH 3/7] Download only subset of tide models --- .github/workflows/integration-test.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index a83426b..e5c8efc 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -45,8 +45,10 @@ jobs: role-to-assume: arn:aws:iam::538673716275:role/github-actions-role-readonly aws-region: ap-southeast-2 - - name: Copy tide modelling files with the AWS CLI - run: aws s3 sync s3://dea-non-public-data/tide_models/tide_models tide_models + - name: Copy tide modelling files with the AWS CLI + run: | + aws s3 sync s3://dea-non-public-data/tide_models/tide_models/fes2014 tide_models/fes2014 + aws s3 sync s3://dea-non-public-data/tide_models/tide_models/hamtide tide_models/hamtide - name: Start docker-compose run: | From c71dd278ff0da12e59c1e3594d0f05526cdf7734 Mon Sep 17 00:00:00 2001 From: Emma Ai Date: Thu, 18 Jul 2024 04:46:02 +0000 Subject: [PATCH 4/7] install awscliv2 from conda --- docker/Dockerfile | 9 --------- docker/env.yaml | 1 + 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index d1d613c..468a8f3 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -48,15 +48,6 @@ RUN echo "Enable server extensions" \ && jupyter server extension list \ && echo "...done" -RUN echo "Install AWS CLI v2" \ - && curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "/tmp/awscliv2.zip" \ - && unzip /tmp/awscliv2.zip -d /tmp \ - && /tmp/aws/install -i /env/bin/aws-cli -b /env/bin \ - && aws --version \ - && rm -f /tmp/awscliv2.zip \ - && rm -fr /tmp/aws \ - && echo "...done" - COPY assets/sync_repo assets/jupyterhub-singleuser /usr/local/bin/ COPY assets/overrides.json /env/share/jupyter/lab/settings/ diff --git a/docker/env.yaml b/docker/env.yaml index 182be01..874f061 100644 --- a/docker/env.yaml +++ b/docker/env.yaml @@ -21,6 +21,7 @@ dependencies: - rio-cogeo - access - aiobotocore + - awscliv2 - boto3 - s5cmd - affine From f802bf77f6ce5c4aa9104a29c6974b98980273fa Mon Sep 17 00:00:00 2001 From: Emma Ai Date: Thu, 18 Jul 2024 04:52:00 +0000 Subject: [PATCH 5/7] fix indentation of workflow yaml --- .github/workflows/integration-test.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index e5c8efc..fb9191d 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -45,10 +45,10 @@ jobs: role-to-assume: arn:aws:iam::538673716275:role/github-actions-role-readonly aws-region: ap-southeast-2 - - name: Copy tide modelling files with the AWS CLI - run: | - aws s3 sync s3://dea-non-public-data/tide_models/tide_models/fes2014 tide_models/fes2014 - aws s3 sync s3://dea-non-public-data/tide_models/tide_models/hamtide tide_models/hamtide + - name: Copy tide modelling files with the AWS CLI + run: | + aws s3 sync s3://dea-non-public-data/tide_models/tide_models/fes2014 tide_models/fes2014 + aws s3 sync s3://dea-non-public-data/tide_models/tide_models/hamtide tide_models/hamtide - name: Start docker-compose run: | From 95a632cef6ba2e61a7ad5f9530b5b8c1230676b4 Mon Sep 17 00:00:00 2001 From: Emma Ai Date: Thu, 18 Jul 2024 06:21:11 +0000 Subject: [PATCH 6/7] output python package version --- docker/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index 468a8f3..0a7ac52 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -51,6 +51,8 @@ RUN echo "Enable server extensions" \ COPY assets/sync_repo assets/jupyterhub-singleuser /usr/local/bin/ COPY assets/overrides.json /env/share/jupyter/lab/settings/ +RUN pip list --format=freeze + WORKDIR "/home/$nb_user" ENTRYPOINT ["/env/bin/tini", "--"] CMD ["jupyter", "lab", \ From d707b22537639b2c3fb19cb2599fe10c59ad5919 Mon Sep 17 00:00:00 2001 From: Robbi Bishop-Taylor Date: Fri, 19 Jul 2024 15:00:33 +1000 Subject: [PATCH 7/7] Remove pyTMD restriction --- docker/env.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/env.yaml b/docker/env.yaml index 874f061..2570e42 100644 --- a/docker/env.yaml +++ b/docker/env.yaml @@ -205,7 +205,7 @@ dependencies: - spyndex - urbanaccess - contextily - - pyTMD<=2.1.0 + - pyTMD - xarray-spatial # jupyter things - autopep8