enable browsing s3 via jupyter-fs

jupyterhub/gfts-hub/Chart.lock

@@ -1,7 +1,7 @@
 dependencies:
 - name: jupyterhub
  repository: https://jupyterhub.github.io/helm-chart
- version: 3.3.2
+ version: 3.3.6
 - name: dask-gateway
  repository: https://helm.dask.org/
  version: 2024.1.0
@@ -14,5 +14,5 @@ dependencies:
 - name: grafana
  repository: https://grafana.github.io/helm-charts
  version: 7.0.14
-digest: sha256:0e76398b9b637b1a642450e916c5b13b1a259d5c1bdb746fb06a13b528177c83
-generated: "2024-03-21T13:17:37.982155+01:00"
+digest: sha256:f65f939aed209d2ebec4ea4534deef0d14d8b99ee1c5ee20ddee11d0a7115b20
+generated: "2024-04-03T14:18:11.909303+02:00"

jupyterhub/gfts-hub/Chart.yaml

@@ -6,7 +6,7 @@ kubeVersion: ">= 1.24.0-0"
 dependencies:
  # jupyterhub
  - name: jupyterhub
- version: "3.3.2"
+ version: "3.3.6"
  repository: https://jupyterhub.github.io/helm-chart
 
  # dask-gateway

jupyterhub/images/user/Dockerfile

@@ -1,4 +1,4 @@
-FROM quay.io/pangeo/pangeo-notebook:2024.01.23
+FROM quay.io/pangeo/pangeo-notebook:2024.03.22
 
 # install some extra things with requirements.txt
 COPY requirements.txt /tmp/requirements.txt

jupyterhub/images/user/jupyter_server_config.py

@@ -1,4 +1,12 @@
 import os
+from configparser import ConfigParser
+from pathlib import Path
+from urllib.parse import urlparse
+
+import fs.errors
+import fs.opener
+from fs_s3fs import S3FS
+from fs_s3fs.opener import S3FSOpener
 
 c = get_config() # noqa
 
@@ -20,3 +28,74 @@
  c.MappingKernelManager.cull_connected = True
 
 c.ContentsManager.hide_globs.extend(["lost+found"])
+
+
+# workaround https://github.com/PyFilesystem/s3fs/issues/70
+# because our files weren't created with S3FS (aka fs-s3fs)
+# they were created with s3fs. Ha!
+class EnsureDirS3FS(S3FS):
+ def getinfo(self, path, *args, **kwargs):
+ try:
+ return super().getinfo(path, *args, **kwargs)
+ except fs.errors.ResourceNotFound as e:
+ # check if getinfo failed becuase it's a directory
+ # without an empty Object
+ # if so, create it
+ # S3FS.scandir and S3FS.getinfo don't work on missing directories
+ # but S3FS.listdir does
+ try:
+ self.listdir(path)
+ except fs.errors.ResourceNotFound:
+ # it actually doesn't exist, raise original error
+ raise e from None
+ else:
+ # it's a directory but the empty directory object doesn't exist
+ # create it then call getinfo
+ print(f"Making empty directory {path}")
+ self.makedir(path)
+ return super().getinfo(path, *args, **kwargs)
+
+
+# define custom opener for GFTS
+# loads GFTS S3 credentials from .aws/credentials [gfts] profile
+
+
+class GFTSOpener(S3FSOpener):
+ protocols = ["gfts-s3"]
+
+ def open_fs(self, fs_url, parse_result, *args, **kwargs):
+ bucket_name, _, dir_path = parse_result.resource.partition("/")
+ creds = ConfigParser()
+ creds.read(Path.home() / ".aws/credentials")
+ endpoint_url = creds.get(
+ "gfts",
+ "aws_endpoint_url",
+ fallback="https://s3.gra.perf.cloud.ovh.net",
+ )
+ # get 'gra' from 's3.gra.perf....'
+ region_name = urlparse(endpoint_url).hostname.split(".")[1]
+ return EnsureDirS3FS(
+ bucket_name,
+ dir_path=dir_path or "/",
+ aws_access_key_id=creds.get("gfts", "aws_access_key_id", fallback=None),
+ aws_secret_access_key=creds.get(
+ "gfts", "aws_secret_access_key", fallback=None
+ ),
+ endpoint_url=endpoint_url,
+ region=region_name,
+ acl=parse_result.params.get("acl", None),
+ cache_control=parse_result.params.get("cache_control", None),
+ )
+
+
+fs.opener.registry.install(GFTSOpener)
+
+
+c.JupyterFs.resources = [
+ {
+ "name": "gfts-data-lake",
+ "url": "gfts-s3://destine-gfts-data-lake/",
+ },
+]
+
+c.ServerApp.contents_manager_class = "jupyterfs.metamanager.MetaManager"

jupyterhub/images/user/requirements.txt

@@ -1 +1,2 @@
-jupyterhub==4.1.0
+jupyter-fs
+jupyterhub==4.1.4