upgrade to jupyterlab > 4 (#291)

* change awscli install back to unzip

* pin jupyterlab_code_formatter for compatibility

* upgrade jupyterlab>4

* note the gotchas

* Attempt to fix Docker compose

* add spark

* update numexpr with fix

* Pin pyTMD 2.1.6

* Fix syntax

* pin numpy < 2.0

* update fc

* adapt to new naming of fc

* Pin latest DEA Tools

* Fix syntax

---------

Co-authored-by: Emma Ai <[email protected]>
Co-authored-by: Robbi Bishop-Taylor <[email protected]>

.github/workflows/integration-test.yml

@@ -51,16 +51,16 @@ jobs:
           aws s3 sync s3://dea-non-public-data/tide_models/tide_models/fes2014 tide_models/fes2014
           aws s3 sync s3://dea-non-public-data/tide_models/tide_models/hamtide tide_models/hamtide
 
-      - name: Start docker-compose
+      - name: Start docker compose
         run: |
           sudo chown -R 1000:100 ./dea-notebooks
           cd ./dea-sandbox/integration-testing
-          CURRENT_UID=1000:100 docker-compose up -d
+          CURRENT_UID=1000:100 docker compose up -d
 
       - name: Extract package versions
         run: |
           cd ./dea-sandbox/integration-testing
-          docker-compose exec -T sandbox cat /tmp/requirements.txt > requirements.txt
+          docker compose exec -T sandbox cat /tmp/requirements.txt > requirements.txt
 
       - name: Read package versions
         id: versions
@@ -85,5 +85,5 @@ jobs:
       - name: Set up Datacube and Test
         run: |
           cd ./dea-sandbox/integration-testing
-          docker-compose exec -T sandbox ./dea-notebooks/Tests/setup_test_datacube.sh
-          docker-compose exec -T sandbox ./dea-notebooks/Tests/test_notebooks.sh
+          docker compose exec -T sandbox ./dea-notebooks/Tests/setup_test_datacube.sh
+          docker compose exec -T sandbox ./dea-notebooks/Tests/test_notebooks.sh

docker/Dockerfile

@@ -12,6 +12,7 @@ COPY requirements.txt /conf/
 # required to build hdmedians
 # or any --no-binary
 ENV CC=/env/bin/x86_64-conda_cos6-linux-gnu-gcc \
+    CXX=/env/bin/x86_64-conda_cos6-linux-gnu-g++ \
     LDSHARED="/env/bin/x86_64-conda_cos6-linux-gnu-gcc -pthread -shared -B /env/compiler_compat -L/env/lib -Wl,-rpath=/env/lib -Wl,--no-as-needed"
 RUN micromamba run -p /env pip install --no-cache-dir \
     --no-build-isolation -r /conf/requirements.txt
@@ -36,16 +37,28 @@ ENV LC_ALL=C.UTF-8 \
     PATH=/env/bin:$PATH
 
 USER $nb_user
-# NOTE - JupyterLab extensions can now be distributed as prebuilt extensions and available to PyPI
+
+# NOTE - Install lab extension from pypi NOT here. 
+# Leave the layer here for potential disable/enable and non-working check
 RUN echo "Enable jupyter lab extensions" \
-  && jupyter labextension install jupyterlab-topbar-extension jupyterlab-theme-toggle \
   && jupyter labextension list \
   && echo "...done"
 
 RUN echo "Enable server extensions" \
    && jupyter server extension enable  --py jupyterlab_iframe --sys-prefix \
    && jupyter server extension enable  --py jupyter_resource_usage --sys-prefix \
    && jupyter server extension list \
+   && jupyter lab build \
+   && echo "...done"
+
+# Install from unzip instead of conda as conda version is not the tool but an "invoker" 
+RUN echo "Install AWS CLI v2" \
+   && curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "/tmp/awscliv2.zip" \
+   && unzip /tmp/awscliv2.zip -d /tmp > /dev/null \
+   && /tmp/aws/install -i /env/bin/aws-cli -b /env/bin \
+   && aws --version \
+   && rm -f /tmp/awscliv2.zip \
+   && rm -fr /tmp/aws \
    && echo "...done"
 
 COPY assets/sync_repo assets/jupyterhub-singleuser /usr/local/bin/

docker/env.yaml

@@ -21,7 +21,6 @@ dependencies:
   - rio-cogeo
   - access
   - aiobotocore
-  - awscliv2
   - boto3
   - s5cmd
   - affine
@@ -93,8 +92,11 @@ dependencies:
   - netCDF4
   - networkx
   - nodejs
-  - numexpr
-  - numpy
+  - numpy<2.0
+  # required by spark
+  - openjdk
+  - apache-sedona
+  - pyspark
   - ordered-set
   - packaging
   - pandas
@@ -169,6 +171,7 @@ dependencies:
 # Scientific Stack
   - gsl
   - cgal-cpp
+  - cgal
   - boost
   - openmp
   - muparser
@@ -179,7 +182,7 @@ dependencies:
   - tensorflow>=2.16
   - xgboost
   - zarr
-  - bokeh=3.2.2
+  - bokeh
   - descartes
   - matplotlib
   - seaborn
@@ -205,29 +208,24 @@ dependencies:
   - spyndex
   - urbanaccess
   - contextily
-  - pyTMD
+  - pyTMD=2.1.6
   - xarray-spatial
 # jupyter things
   - autopep8
   - black
   - python-jose
   - ecdsa
-  - jupyter
-  - jupyterlab
-  - jupyterlab_code_formatter
+  - jupyterlab>=4.0
+  - jupyterlab_code_formatter<3.0 # Non-caught error in >= 3.0
   - jupyterlab-spellchecker
-  - jupyterlab-code-snippets
-  - jupyterlab-dash
   - jupyterlab-geojson
   - jupyterlab-git
   - jupyterlab-github
   - jupyterlab_iframe
   - jupyterlab_widgets
-  - jupyterlab-topbar
-  - jupyterlab-system-monitor
-  - jupyterhub
-  - jupytext<1.16
-  - jupyterlab_pygments<0.3.0
+  - jupyterhub=4.1.5 # Match hub version
+  - jupytext
+  - jupyterlab_pygments
   - jupyter-resource-usage
   - jupyter_server
   - jupyter-server-proxy
@@ -242,6 +240,7 @@ dependencies:
   - isort
   - nbdime
   - nbgitpuller
+  - notebook<7.0 # Compatible with nbconvert
   - mypy
   - prompt-toolkit # because of line_profiler/ipython
   - nbconvert # because of jupyter_contrib_nbextensions

docker/requirements.txt

@@ -1,14 +1,15 @@
 cognitojwt
 nbval
-cgal
 line-profiler
 jupyterlab-logout
 jupyter-contrib-core
 jupyter-contrib-nbextensions
 jupyter-nbextensions-configurator
-# pin for jupyterlab=3.x
-jupyterlab-topbar-text==0.6.2
-jupyterlab-logout==0.5.0
+jupyterlab-code-snippets
+jupyterlab-topbar-text
+jupyterlab-logout
+jupyterlab-theme-toggler
+numexpr @ git+https://github.com/pydata/numexpr@a99412e
 
 # ODC/DEA: these are installed in builder stage
 otps
@@ -30,11 +31,12 @@ odc-stac
 odc-stats[ows]
 odc-ui
 odc-geo >= 0.4.8  # min version required for ESRI/GDAL nodata fix
-dea-tools
+dea-tools >= 0.3.5
 
 thredds-crawler
 hdstats==0.1.8.post1
 hdmedians
 datacube-stats
-fc
+fractional_cover>=1.3.10
 --extra-index-url="https://packages.dea.ga.gov.au"
+--find-links="https://packages.dea.ga.gov.au/fc"