diff --git a/jupyterhub/gfts-hub/Chart.lock b/jupyterhub/gfts-hub/Chart.lock index 9be7dac..33b1b43 100644 --- a/jupyterhub/gfts-hub/Chart.lock +++ b/jupyterhub/gfts-hub/Chart.lock @@ -1,7 +1,7 @@ dependencies: - name: jupyterhub repository: https://jupyterhub.github.io/helm-chart - version: 3.3.2 + version: 3.3.6 - name: dask-gateway repository: https://helm.dask.org/ version: 2024.1.0 @@ -14,5 +14,5 @@ dependencies: - name: grafana repository: https://grafana.github.io/helm-charts version: 7.0.14 -digest: sha256:0e76398b9b637b1a642450e916c5b13b1a259d5c1bdb746fb06a13b528177c83 -generated: "2024-03-21T13:17:37.982155+01:00" +digest: sha256:f65f939aed209d2ebec4ea4534deef0d14d8b99ee1c5ee20ddee11d0a7115b20 +generated: "2024-04-03T14:18:11.909303+02:00" diff --git a/jupyterhub/gfts-hub/Chart.yaml b/jupyterhub/gfts-hub/Chart.yaml index 442bd88..a515cc3 100644 --- a/jupyterhub/gfts-hub/Chart.yaml +++ b/jupyterhub/gfts-hub/Chart.yaml @@ -6,7 +6,7 @@ kubeVersion: ">= 1.24.0-0" dependencies: # jupyterhub - name: jupyterhub - version: "3.3.2" + version: "3.3.6" repository: https://jupyterhub.github.io/helm-chart # dask-gateway diff --git a/jupyterhub/images/user/Dockerfile b/jupyterhub/images/user/Dockerfile index 02da10f..eb56095 100644 --- a/jupyterhub/images/user/Dockerfile +++ b/jupyterhub/images/user/Dockerfile @@ -1,4 +1,4 @@ -FROM quay.io/pangeo/pangeo-notebook:2024.01.23 +FROM quay.io/pangeo/pangeo-notebook:2024.03.22 # install some extra things with requirements.txt COPY requirements.txt /tmp/requirements.txt diff --git a/jupyterhub/images/user/jupyter_server_config.py b/jupyterhub/images/user/jupyter_server_config.py index f1f153e..9a01e88 100644 --- a/jupyterhub/images/user/jupyter_server_config.py +++ b/jupyterhub/images/user/jupyter_server_config.py @@ -1,4 +1,12 @@ import os +from configparser import ConfigParser +from pathlib import Path +from urllib.parse import urlparse + +import fs.errors +import fs.opener +from fs_s3fs import S3FS +from fs_s3fs.opener import S3FSOpener c = get_config() # noqa @@ -20,3 +28,74 @@ c.MappingKernelManager.cull_connected = True c.ContentsManager.hide_globs.extend(["lost+found"]) + + +# workaround https://github.com/PyFilesystem/s3fs/issues/70 +# because our files weren't created with S3FS (aka fs-s3fs) +# they were created with s3fs. Ha! +class EnsureDirS3FS(S3FS): + def getinfo(self, path, *args, **kwargs): + try: + return super().getinfo(path, *args, **kwargs) + except fs.errors.ResourceNotFound as e: + # check if getinfo failed becuase it's a directory + # without an empty Object + # if so, create it + # S3FS.scandir and S3FS.getinfo don't work on missing directories + # but S3FS.listdir does + try: + self.listdir(path) + except fs.errors.ResourceNotFound: + # it actually doesn't exist, raise original error + raise e from None + else: + # it's a directory but the empty directory object doesn't exist + # create it then call getinfo + print(f"Making empty directory {path}") + self.makedir(path) + return super().getinfo(path, *args, **kwargs) + + +# define custom opener for GFTS +# loads GFTS S3 credentials from .aws/credentials [gfts] profile + + +class GFTSOpener(S3FSOpener): + protocols = ["gfts-s3"] + + def open_fs(self, fs_url, parse_result, *args, **kwargs): + bucket_name, _, dir_path = parse_result.resource.partition("/") + creds = ConfigParser() + creds.read(Path.home() / ".aws/credentials") + endpoint_url = creds.get( + "gfts", + "aws_endpoint_url", + fallback="https://s3.gra.perf.cloud.ovh.net", + ) + # get 'gra' from 's3.gra.perf....' + region_name = urlparse(endpoint_url).hostname.split(".")[1] + return EnsureDirS3FS( + bucket_name, + dir_path=dir_path or "/", + aws_access_key_id=creds.get("gfts", "aws_access_key_id", fallback=None), + aws_secret_access_key=creds.get( + "gfts", "aws_secret_access_key", fallback=None + ), + endpoint_url=endpoint_url, + region=region_name, + acl=parse_result.params.get("acl", None), + cache_control=parse_result.params.get("cache_control", None), + ) + + +fs.opener.registry.install(GFTSOpener) + + +c.JupyterFs.resources = [ + { + "name": "gfts-data-lake", + "url": "gfts-s3://destine-gfts-data-lake/", + }, +] + +c.ServerApp.contents_manager_class = "jupyterfs.metamanager.MetaManager" diff --git a/jupyterhub/images/user/requirements.txt b/jupyterhub/images/user/requirements.txt index 5b6125c..36f400a 100644 --- a/jupyterhub/images/user/requirements.txt +++ b/jupyterhub/images/user/requirements.txt @@ -1 +1,2 @@ -jupyterhub==4.1.0 +jupyter-fs +jupyterhub==4.1.4