From 983e04ec0330245d7516b0f006125a156e39e95f Mon Sep 17 00:00:00 2001
From: "2i2c-token-generator-bot[bot]"
 <106546794+2i2c-token-generator-bot[bot]@users.noreply.github.com>
Date: Wed, 8 Feb 2023 14:55:40 +0000
Subject: [PATCH 01/44] Bump charts ['binderhub'] to versions
 ['1.0.0-0.dev.git.3018.he00ec49'], respectively

---
 helm-charts/binderhub/Chart.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/helm-charts/binderhub/Chart.yaml b/helm-charts/binderhub/Chart.yaml
index 71e5a8fafe..d48c4d6bae 100644
--- a/helm-charts/binderhub/Chart.yaml
+++ b/helm-charts/binderhub/Chart.yaml
@@ -5,7 +5,7 @@ name: binderhub
 version: "0.1.0"
 dependencies:
   - name: binderhub
-    version: "1.0.0-0.dev.git.3009.h9046454"
+    version: "1.0.0-0.dev.git.3018.he00ec49"
     repository: "https://jupyterhub.github.io/helm-chart/"
   - name: dask-gateway
     version: "2022.10.0"

From 55db4547da22bc6131e09797288d518806f67075 Mon Sep 17 00:00:00 2001
From: "2i2c-token-generator-bot[bot]"
 <106546794+2i2c-token-generator-bot[bot]@users.noreply.github.com>
Date: Mon, 27 Feb 2023 00:02:23 +0000
Subject: [PATCH 02/44] Bump charts ['binderhub'] to versions
 ['1.0.0-0.dev.git.3020.h694f8cd'], respectively

---
 helm-charts/binderhub/Chart.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/helm-charts/binderhub/Chart.yaml b/helm-charts/binderhub/Chart.yaml
index d48c4d6bae..e9172d1a30 100644
--- a/helm-charts/binderhub/Chart.yaml
+++ b/helm-charts/binderhub/Chart.yaml
@@ -5,7 +5,7 @@ name: binderhub
 version: "0.1.0"
 dependencies:
   - name: binderhub
-    version: "1.0.0-0.dev.git.3018.he00ec49"
+    version: "1.0.0-0.dev.git.3020.h694f8cd"
     repository: "https://jupyterhub.github.io/helm-chart/"
   - name: dask-gateway
     version: "2022.10.0"

From abcd4f2681ccde4178d05925b739ab78898531a1 Mon Sep 17 00:00:00 2001
From: "2i2c-token-generator-bot[bot]"
 <106546794+2i2c-token-generator-bot[bot]@users.noreply.github.com>
Date: Mon, 6 Mar 2023 00:02:24 +0000
Subject: [PATCH 03/44] Bump charts ['binderhub'] to versions
 ['1.0.0-0.dev.git.3025.h276be90'], respectively

---
 helm-charts/binderhub/Chart.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/helm-charts/binderhub/Chart.yaml b/helm-charts/binderhub/Chart.yaml
index e9172d1a30..e91becde15 100644
--- a/helm-charts/binderhub/Chart.yaml
+++ b/helm-charts/binderhub/Chart.yaml
@@ -5,7 +5,7 @@ name: binderhub
 version: "0.1.0"
 dependencies:
   - name: binderhub
-    version: "1.0.0-0.dev.git.3020.h694f8cd"
+    version: "1.0.0-0.dev.git.3025.h276be90"
     repository: "https://jupyterhub.github.io/helm-chart/"
   - name: dask-gateway
     version: "2022.10.0"

From a52a0d14b685f2b17a07efc80b160ba310337f33 Mon Sep 17 00:00:00 2001
From: "2i2c-token-generator-bot[bot]"
 <106546794+2i2c-token-generator-bot[bot]@users.noreply.github.com>
Date: Mon, 27 Mar 2023 00:02:47 +0000
Subject: [PATCH 04/44] Bump charts ['binderhub'] to versions
 ['1.0.0-0.dev.git.3031.h8cac969'], respectively

---
 helm-charts/binderhub/Chart.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/helm-charts/binderhub/Chart.yaml b/helm-charts/binderhub/Chart.yaml
index e91becde15..73ca182841 100644
--- a/helm-charts/binderhub/Chart.yaml
+++ b/helm-charts/binderhub/Chart.yaml
@@ -5,7 +5,7 @@ name: binderhub
 version: "0.1.0"
 dependencies:
   - name: binderhub
-    version: "1.0.0-0.dev.git.3025.h276be90"
+    version: "1.0.0-0.dev.git.3031.h8cac969"
     repository: "https://jupyterhub.github.io/helm-chart/"
   - name: dask-gateway
     version: "2022.10.0"

From c19dd3bd97f900dd1fffb7fcb82ef3f7b61ccdab Mon Sep 17 00:00:00 2001
From: "2i2c-token-generator-bot[bot]"
 <106546794+2i2c-token-generator-bot[bot]@users.noreply.github.com>
Date: Mon, 3 Apr 2023 00:02:50 +0000
Subject: [PATCH 05/44] Bump charts ['binderhub'] to versions
 ['1.0.0-0.dev.git.3035.h32213a4'], respectively

---
 helm-charts/binderhub/Chart.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/helm-charts/binderhub/Chart.yaml b/helm-charts/binderhub/Chart.yaml
index 73ca182841..67d4539d98 100644
--- a/helm-charts/binderhub/Chart.yaml
+++ b/helm-charts/binderhub/Chart.yaml
@@ -5,7 +5,7 @@ name: binderhub
 version: "0.1.0"
 dependencies:
   - name: binderhub
-    version: "1.0.0-0.dev.git.3031.h8cac969"
+    version: "1.0.0-0.dev.git.3035.h32213a4"
     repository: "https://jupyterhub.github.io/helm-chart/"
   - name: dask-gateway
     version: "2022.10.0"

From 0b7df577170ccdd8eade47b4d53d5a318b80870a Mon Sep 17 00:00:00 2001
From: "2i2c-token-generator-bot[bot]"
 <106546794+2i2c-token-generator-bot[bot]@users.noreply.github.com>
Date: Wed, 19 Apr 2023 16:40:35 +0000
Subject: [PATCH 06/44] Bump charts ['binderhub'] to versions
 ['1.0.0-0.dev.git.3056.h8738813'], respectively

---
 helm-charts/binderhub/Chart.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/helm-charts/binderhub/Chart.yaml b/helm-charts/binderhub/Chart.yaml
index 67d4539d98..f7b5f592f3 100644
--- a/helm-charts/binderhub/Chart.yaml
+++ b/helm-charts/binderhub/Chart.yaml
@@ -5,7 +5,7 @@ name: binderhub
 version: "0.1.0"
 dependencies:
   - name: binderhub
-    version: "1.0.0-0.dev.git.3035.h32213a4"
+    version: "1.0.0-0.dev.git.3056.h8738813"
     repository: "https://jupyterhub.github.io/helm-chart/"
   - name: dask-gateway
     version: "2022.10.0"

From 90d7466e9a4810d9966f93ccf2d14e57772b1f8a Mon Sep 17 00:00:00 2001
From: "2i2c-token-generator-bot[bot]"
 <106546794+2i2c-token-generator-bot[bot]@users.noreply.github.com>
Date: Mon, 1 May 2023 00:09:22 +0000
Subject: [PATCH 07/44] Bump charts ['binderhub'] to versions
 ['1.0.0-0.dev.git.3080.h8f9a1dc'], respectively

---
 helm-charts/binderhub/Chart.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/helm-charts/binderhub/Chart.yaml b/helm-charts/binderhub/Chart.yaml
index f7b5f592f3..bf4f046c76 100644
--- a/helm-charts/binderhub/Chart.yaml
+++ b/helm-charts/binderhub/Chart.yaml
@@ -5,7 +5,7 @@ name: binderhub
 version: "0.1.0"
 dependencies:
   - name: binderhub
-    version: "1.0.0-0.dev.git.3056.h8738813"
+    version: "1.0.0-0.dev.git.3080.h8f9a1dc"
     repository: "https://jupyterhub.github.io/helm-chart/"
   - name: dask-gateway
     version: "2022.10.0"

From 7301f5a05c52fd99cc4789185ec43ac2c8ccd8e3 Mon Sep 17 00:00:00 2001
From: "2i2c-token-generator-bot[bot]"
 <106546794+2i2c-token-generator-bot[bot]@users.noreply.github.com>
Date: Thu, 1 Jun 2023 00:11:36 +0000
Subject: [PATCH 08/44] Bump charts ['binderhub'] to versions
 ['1.0.0-0.dev.git.3115.h29473bf'], respectively

---
 helm-charts/binderhub/Chart.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/helm-charts/binderhub/Chart.yaml b/helm-charts/binderhub/Chart.yaml
index bf4f046c76..6efcdd8234 100644
--- a/helm-charts/binderhub/Chart.yaml
+++ b/helm-charts/binderhub/Chart.yaml
@@ -5,7 +5,7 @@ name: binderhub
 version: "0.1.0"
 dependencies:
   - name: binderhub
-    version: "1.0.0-0.dev.git.3080.h8f9a1dc"
+    version: "1.0.0-0.dev.git.3115.h29473bf"
     repository: "https://jupyterhub.github.io/helm-chart/"
   - name: dask-gateway
     version: "2022.10.0"

From ded8df86422920ed6c00a3afb5043b90d95142a9 Mon Sep 17 00:00:00 2001
From: "2i2c-token-generator-bot[bot]"
 <106546794+2i2c-token-generator-bot[bot]@users.noreply.github.com>
Date: Wed, 28 Jun 2023 07:42:11 +0000
Subject: [PATCH 09/44] Bump charts ['binderhub'] to versions
 ['1.0.0-0.dev.git.3145.h62334a7'], respectively

---
 helm-charts/binderhub/Chart.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/helm-charts/binderhub/Chart.yaml b/helm-charts/binderhub/Chart.yaml
index 6efcdd8234..7aaad416bf 100644
--- a/helm-charts/binderhub/Chart.yaml
+++ b/helm-charts/binderhub/Chart.yaml
@@ -5,7 +5,7 @@ name: binderhub
 version: "0.1.0"
 dependencies:
   - name: binderhub
-    version: "1.0.0-0.dev.git.3115.h29473bf"
+    version: "1.0.0-0.dev.git.3145.h62334a7"
     repository: "https://jupyterhub.github.io/helm-chart/"
   - name: dask-gateway
     version: "2022.10.0"

From ff83ffccf77a5111f15bc3335ca27ce1c903aca3 Mon Sep 17 00:00:00 2001
From: "2i2c-token-generator-bot[bot]"
 <106546794+2i2c-token-generator-bot[bot]@users.noreply.github.com>
Date: Tue, 1 Aug 2023 00:10:44 +0000
Subject: [PATCH 10/44] Bump charts ['binderhub'] to versions
 ['1.0.0-0.dev.git.3161.hfcbd1e1'], respectively

---
 helm-charts/binderhub/Chart.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/helm-charts/binderhub/Chart.yaml b/helm-charts/binderhub/Chart.yaml
index 7aaad416bf..a31bf86062 100644
--- a/helm-charts/binderhub/Chart.yaml
+++ b/helm-charts/binderhub/Chart.yaml
@@ -5,7 +5,7 @@ name: binderhub
 version: "0.1.0"
 dependencies:
   - name: binderhub
-    version: "1.0.0-0.dev.git.3145.h62334a7"
+    version: "1.0.0-0.dev.git.3161.hfcbd1e1"
     repository: "https://jupyterhub.github.io/helm-chart/"
   - name: dask-gateway
     version: "2022.10.0"

From bd44c7f9a2382162a62ef91874e7970f7c06b774 Mon Sep 17 00:00:00 2001
From: "2i2c-token-generator-bot[bot]"
 <106546794+2i2c-token-generator-bot[bot]@users.noreply.github.com>
Date: Fri, 1 Sep 2023 00:09:25 +0000
Subject: [PATCH 11/44] Bump charts ['binderhub'] to versions
 ['1.0.0-0.dev.git.3170.h84b1db9'], respectively

---
 helm-charts/binderhub/Chart.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/helm-charts/binderhub/Chart.yaml b/helm-charts/binderhub/Chart.yaml
index a31bf86062..8b12d56b79 100644
--- a/helm-charts/binderhub/Chart.yaml
+++ b/helm-charts/binderhub/Chart.yaml
@@ -5,7 +5,7 @@ name: binderhub
 version: "0.1.0"
 dependencies:
   - name: binderhub
-    version: "1.0.0-0.dev.git.3161.hfcbd1e1"
+    version: "1.0.0-0.dev.git.3170.h84b1db9"
     repository: "https://jupyterhub.github.io/helm-chart/"
   - name: dask-gateway
     version: "2022.10.0"

From a5b03b70a0845f1542a421568180e838b715ee9b Mon Sep 17 00:00:00 2001
From: YuviPanda <yuvipanda@gmail.com>
Date: Thu, 28 Sep 2023 13:19:49 -0700
Subject: [PATCH 12/44] Setup scratch bucket for openscapes

Fixes https://github.com/2i2c-org/infrastructure/issues/3157
---
 config/clusters/openscapes/prod.values.yaml   |  5 ++++
 .../clusters/openscapes/staging.values.yaml   |  5 ++++
 terraform/aws/buckets.tf                      | 11 ++++++++
 terraform/aws/projects/openscapes.tfvars      | 25 ++++++++++++++++++-
 4 files changed, 45 insertions(+), 1 deletion(-)

diff --git a/config/clusters/openscapes/prod.values.yaml b/config/clusters/openscapes/prod.values.yaml
index a937472f8a..191b335c7b 100644
--- a/config/clusters/openscapes/prod.values.yaml
+++ b/config/clusters/openscapes/prod.values.yaml
@@ -1,4 +1,7 @@
 basehub:
+  userServiceAccount:
+    annotations:
+      eks.amazonaws.com/role-arn: arn:aws:iam::783616723547:role/openscapeshub-prod
   jupyterhub:
     ingress:
       hosts: [openscapes.2i2c.cloud]
@@ -6,6 +9,8 @@ basehub:
         - hosts: [openscapes.2i2c.cloud]
           secretName: https-auto-tls
     singleuser:
+      extraEnv:
+        SCRATCH_BUCKET: s3://openscapeshub-scratch/$(JUPYTERHUB_USER)
       profileList:
         - display_name: Python
           description: Python datascience environment
diff --git a/config/clusters/openscapes/staging.values.yaml b/config/clusters/openscapes/staging.values.yaml
index db4aaf4ab7..0fd1e653f0 100644
--- a/config/clusters/openscapes/staging.values.yaml
+++ b/config/clusters/openscapes/staging.values.yaml
@@ -1,4 +1,7 @@
 basehub:
+  userServiceAccount:
+    annotations:
+      eks.amazonaws.com/role-arn: arn:aws:iam::783616723547:role/openscapeshub-staging
   jupyterhub:
     ingress:
       hosts: [staging.openscapes.2i2c.cloud]
@@ -6,6 +9,8 @@ basehub:
         - hosts: [staging.openscapes.2i2c.cloud]
           secretName: https-auto-tls
     singleuser:
+      extraEnv:
+        SCRATCH_BUCKET: s3://openscapeshub-scratch-staging/$(JUPYTERHUB_USER)
       profileList:
         - display_name: Python
           description: Python datascience environment
diff --git a/terraform/aws/buckets.tf b/terraform/aws/buckets.tf
index 79fd3876b4..195a5083e1 100644
--- a/terraform/aws/buckets.tf
+++ b/terraform/aws/buckets.tf
@@ -55,3 +55,14 @@ resource "aws_s3_bucket_policy" "user_bucket_access" {
   bucket   = aws_s3_bucket.user_buckets[each.value.bucket_name].id
   policy   = data.aws_iam_policy_document.bucket_access[each.key].json
 }
+
+output "buckets" {
+  value       = { for b, _ in var.user_buckets : b => aws_s3_bucket.user_buckets[b].id }
+  description = <<-EOT
+  List of S3 buckets created for this cluster
+
+  Since S3 bucket names need to be globally unique, we prefix each item in
+  the user_buckets variable with the prefix variable. This output displays
+  the full name of all S3 buckets created conveniently.
+  EOT
+}
diff --git a/terraform/aws/projects/openscapes.tfvars b/terraform/aws/projects/openscapes.tfvars
index 4f5186cc39..27c2207d82 100644
--- a/terraform/aws/projects/openscapes.tfvars
+++ b/terraform/aws/projects/openscapes.tfvars
@@ -2,4 +2,27 @@ region = "us-west-2"
 
 cluster_name = "openscapeshub"
 
-cluster_nodes_location = "us-west-2b"
\ No newline at end of file
+cluster_nodes_location = "us-west-2b"
+
+user_buckets = {
+  "scratch-staging" : {
+    "delete_after" : 7
+  },
+  "scratch" : {
+    "delete_after" : 7
+  },
+}
+
+
+hub_cloud_permissions = {
+  "staging" : {
+    requestor_pays : true,
+    bucket_admin_access : ["scratch-staging"],
+    extra_iam_policy : ""
+  },
+  "prod" : {
+    requestor_pays : true,
+    bucket_admin_access : ["scratch"],
+    extra_iam_policy : ""
+  },
+}

From 60d885ff407b52433c7ec7e1f326b445b88f308e Mon Sep 17 00:00:00 2001
From: Georgiana Dolocan <georgiana.dolocan@gmail.com>
Date: Thu, 28 Sep 2023 19:35:28 +0300
Subject: [PATCH 13/44] Add command to transfer files from one homedir to
 another

---
 deployer/debug.py | 152 ++++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 140 insertions(+), 12 deletions(-)

diff --git a/deployer/debug.py b/deployer/debug.py
index aa29d077f4..77046539d6 100644
--- a/deployer/debug.py
+++ b/deployer/debug.py
@@ -13,6 +13,7 @@
 from .cli_app import app
 from .cluster import Cluster
 from .file_acquisition import find_absolute_path_to_cluster_file
+from .utils import print_colour
 
 # Without `pure=True`, I get an exception about str / byte issues
 yaml = YAML(typ="safe", pure=True)
@@ -120,18 +121,8 @@ def user_logs(
         subprocess.check_call(cmd)
 
 
-@app.command()
-def exec_homes_shell(
-    cluster_name: str = typer.Argument(..., help="Name of cluster to operate on"),
-    hub_name: str = typer.Argument(..., help="Name of hub to operate on"),
-):
-    """
-    Pop an interactive shell with the home directories of the given hub mounted on /home
-    """
-    # Name pod to include hub name so it is displayed as part of the default prompt
-    # This makes sure we don't end up deleting the wrong thing
-    pod_name = f"{cluster_name}-{hub_name}-shell"
-    pod = {
+def generate_pod_definition_with_home_mount(pod_name):
+    return {
         "apiVersion": "v1",
         "kind": "Pod",
         "spec": {
@@ -160,6 +151,20 @@ def exec_homes_shell(
         },
     }
 
+
+@app.command()
+def exec_homes_shell(
+    cluster_name: str = typer.Argument(..., help="Name of cluster to operate on"),
+    hub_name: str = typer.Argument(..., help="Name of hub to operate on"),
+):
+    """
+    Pop an interactive shell with the home directories of the given hub mounted on /home
+    """
+    # Name pod to include hub name so it is displayed as part of the default prompt
+    # This makes sure we don't end up deleting the wrong thing
+    pod_name = f"{cluster_name}-{hub_name}-shell"
+    pod = generate_pod_definition_with_home_mount(pod_name)
+
     cmd = [
         "kubectl",
         "-n",
@@ -259,5 +264,128 @@ def start_docker_proxy(
         subprocess.check_call(cmd)
 
 
+@app.command()
+def transfer_old_home_dir_to_new_location(
+    cluster_name: str = typer.Argument(..., help="Name of cluster to operate on"),
+    hub_name: str = typer.Argument(..., help="Name of hub to operate on"),
+    source_user: str = typer.Option(
+        prompt=True,
+        confirmation_prompt=True,
+        help="Name of source user who's home dir files we are transferring",
+    ),
+    destination_user: str = typer.Option(
+        prompt=True,
+        confirmation_prompt=True,
+        help="Name of destination user in who's home dir we will be moving the files",
+    ),
+):
+    """
+    Copy all the files from a user's old home dir to the new home directory
+    """
+    # Name pod to include hub name so it is displayed as part of the default prompt
+    # This makes sure we don't end up deleting the wrong thing
+    pod_name = f"{cluster_name}-{hub_name}-transfer-shell"
+    pod = generate_pod_definition_with_home_mount(pod_name)
+
+    # Command that creates the pod described above
+    create_pod_cmd = [
+        "kubectl",
+        "-n",
+        hub_name,
+        "run",
+        pod_name,
+        "--overrides",
+        json.dumps(pod),
+        "--image",
+        # Use ubuntu image so we get GNU rm and other tools
+        # Should match what we have in our pod definition
+        "ubuntu:jammy",
+        pod_name,
+    ]
+
+    # Command that waits for the pod above to be ready
+    wait_for_pod_cmd = [
+        "kubectl",
+        "-n",
+        hub_name,
+        "wait",
+        "pods",
+        "-l",
+        f"run={pod_name}",
+        "--for",
+        "condition=Ready",
+        "--timeout=90s",
+    ]
+
+    # Command that lists the contents of the source and destination home dirs
+    ls_dirs_cmd = [
+        "kubectl",
+        "-n",
+        hub_name,
+        "exec",
+        pod_name,
+        "--",
+        "ls",
+        "-lat",
+        f"/home/{source_user}",
+        f"/home/{destination_user}",
+    ]
+
+    # Command that copies the home directory of the source user
+    # into the home directory of the destination user
+    # under a directory located at `/home/{destination_user}/{source_user}-homedir``
+    copy_cmd = [
+        "kubectl",
+        "-n",
+        hub_name,
+        "exec",
+        pod_name,
+        "--",
+        "cp",
+        "-r",
+        f"/home/{source_user}",
+        f"/home/{destination_user}/{source_user}-homedir",
+    ]
+
+    # Command to delete the pod we created
+    delete_pod_cmd = [
+        "kubectl",
+        "-n",
+        hub_name,
+        "delete",
+        "pod",
+        pod_name,
+    ]
+
+    config_file_path = find_absolute_path_to_cluster_file(cluster_name)
+    with open(config_file_path) as f:
+        cluster = Cluster(yaml.load(f), config_file_path.parent)
+    with cluster.auth():
+        try:
+            print_colour(
+                f"Creating a pod with the home directories of {cluster_name} - {hub_name} hub mounted on /home...",
+                "yellow",
+            )
+            subprocess.check_call(create_pod_cmd)
+            subprocess.check_call(wait_for_pod_cmd)
+
+            print_colour(
+                f"Attention! You are about to start transferring the home dir of {source_user} into the home dir of {destination_user}/{source_user}-homedir",
+                "yellow",
+            )
+            confirm = typer.confirm("Are you sure you want to continue?")
+            if not confirm:
+                raise typer.Abort()
+
+            print_colour("The contents of the source and destination home dirs is:")
+            subprocess.check_call(ls_dirs_cmd)
+
+            print_colour("Starting the transfer...")
+            subprocess.check_call(copy_cmd)
+        finally:
+            print_colour("Deleting the pod...", "red")
+            subprocess.check_call(delete_pod_cmd)
+
+
 if __name__ == "__main__":
     app()

From d867e5ebeada8e0d3a3f27a8a726ecbd9f0ac553 Mon Sep 17 00:00:00 2001
From: Georgiana Dolocan <georgiana.dolocan@gmail.com>
Date: Fri, 29 Sep 2023 11:29:34 +0300
Subject: [PATCH 14/44] create a pod and container that can only be ran by uid
 1000 and fix a few comments

---
 deployer/debug.py | 67 +++++++++++++++++++++++++++++++++--------------
 1 file changed, 47 insertions(+), 20 deletions(-)

diff --git a/deployer/debug.py b/deployer/debug.py
index 77046539d6..a77c96ceaa 100644
--- a/deployer/debug.py
+++ b/deployer/debug.py
@@ -121,8 +121,18 @@ def user_logs(
         subprocess.check_call(cmd)
 
 
-def generate_pod_definition_with_home_mount(pod_name):
-    return {
+@app.command()
+def exec_homes_shell(
+    cluster_name: str = typer.Argument(..., help="Name of cluster to operate on"),
+    hub_name: str = typer.Argument(..., help="Name of hub to operate on"),
+):
+    """
+    Pop an interactive shell with the home directories of the given hub mounted on /home
+    """
+    # Name pod to include hub name so it is displayed as part of the default prompt
+    # This makes sure we don't end up deleting the wrong thing
+    pod_name = f"{cluster_name}-{hub_name}-shell"
+    pod = {
         "apiVersion": "v1",
         "kind": "Pod",
         "spec": {
@@ -151,20 +161,6 @@ def generate_pod_definition_with_home_mount(pod_name):
         },
     }
 
-
-@app.command()
-def exec_homes_shell(
-    cluster_name: str = typer.Argument(..., help="Name of cluster to operate on"),
-    hub_name: str = typer.Argument(..., help="Name of hub to operate on"),
-):
-    """
-    Pop an interactive shell with the home directories of the given hub mounted on /home
-    """
-    # Name pod to include hub name so it is displayed as part of the default prompt
-    # This makes sure we don't end up deleting the wrong thing
-    pod_name = f"{cluster_name}-{hub_name}-shell"
-    pod = generate_pod_definition_with_home_mount(pod_name)
-
     cmd = [
         "kubectl",
         "-n",
@@ -282,10 +278,41 @@ def transfer_old_home_dir_to_new_location(
     """
     Copy all the files from a user's old home dir to the new home directory
     """
-    # Name pod to include hub name so it is displayed as part of the default prompt
-    # This makes sure we don't end up deleting the wrong thing
+    # Name the pod so we know what to delete when the transfer is done
     pod_name = f"{cluster_name}-{hub_name}-transfer-shell"
-    pod = generate_pod_definition_with_home_mount(pod_name)
+    pod = {
+        "apiVersion": "v1",
+        "kind": "Pod",
+        "spec": {
+            "terminationGracePeriodSeconds": 1,
+            "automountServiceAccountToken": False,
+            # Make sure the pod and containers can only be ran as the jupyter user
+            # Otherwise the files we copy might end up be owned by another user
+            "securityContext": {"runAsUser": 1000, "runAsGroup": 1000},
+            "volumes": [
+                # This PVC is created by basehub
+                {"name": "home", "persistentVolumeClaim": {"claimName": "home-nfs"}}
+            ],
+            "containers": [
+                {
+                    "name": pod_name,
+                    # Use ubuntu image so we get better GNU cp
+                    "image": "ubuntu:jammy",
+                    "stdin": True,
+                    "stdinOnce": True,
+                    "tty": True,
+                    "user": 1000,
+                    "volumeMounts": [
+                        {
+                            "name": "home",
+                            "mountPath": "/home",
+                        }
+                    ],
+                    "securityContext": {"allowPrivilegeEscalation": False},
+                }
+            ],
+        },
+    }
 
     # Command that creates the pod described above
     create_pod_cmd = [
@@ -297,7 +324,7 @@ def transfer_old_home_dir_to_new_location(
         "--overrides",
         json.dumps(pod),
         "--image",
-        # Use ubuntu image so we get GNU rm and other tools
+        # Use ubuntu image so we get GNU cp and other tools
         # Should match what we have in our pod definition
         "ubuntu:jammy",
         pod_name,

From 92c6ce2b5fa2aff442c8ae111bce0774a0fd5f7b Mon Sep 17 00:00:00 2001
From: Georgiana Dolocan <georgiana.dolocan@gmail.com>
Date: Fri, 29 Sep 2023 11:40:32 +0300
Subject: [PATCH 15/44] Add more flags to the cp command to avoid messing up
 anything at the destination and add more logs

---
 deployer/debug.py | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/deployer/debug.py b/deployer/debug.py
index a77c96ceaa..81519d3ebd 100644
--- a/deployer/debug.py
+++ b/deployer/debug.py
@@ -301,7 +301,6 @@ def transfer_old_home_dir_to_new_location(
                     "stdin": True,
                     "stdinOnce": True,
                     "tty": True,
-                    "user": 1000,
                     "volumeMounts": [
                         {
                             "name": "home",
@@ -369,7 +368,12 @@ def transfer_old_home_dir_to_new_location(
         pod_name,
         "--",
         "cp",
-        "-r",
+        # preserve all attributes of the directory being copied
+        "--archive",
+        # do not overwrite any existing files at the destination
+        "--no-clobber",
+        # copy directories recursively
+        "--recursive",
         f"/home/{source_user}",
         f"/home/{destination_user}/{source_user}-homedir",
     ]
@@ -404,11 +408,15 @@ def transfer_old_home_dir_to_new_location(
             if not confirm:
                 raise typer.Abort()
 
-            print_colour("The contents of the source and destination home dirs is:")
+            print_colour("The content of the source and destination home dirs is:")
             subprocess.check_call(ls_dirs_cmd)
 
-            print_colour("Starting the transfer...")
+            print_colour("Starting the transfer...", "yellow")
             subprocess.check_call(copy_cmd)
+            print_colour(
+                "Done! The content of the source and destination home dirs now is:"
+            )
+            subprocess.check_call(ls_dirs_cmd)
         finally:
             print_colour("Deleting the pod...", "red")
             subprocess.check_call(delete_pod_cmd)

From f2108b8dc99db4befaea0244e31a17cba4d840a0 Mon Sep 17 00:00:00 2001
From: "2i2c-token-generator-bot[bot]"
 <106546794+2i2c-token-generator-bot[bot]@users.noreply.github.com>
Date: Sun, 1 Oct 2023 00:10:20 +0000
Subject: [PATCH 16/44] Bump charts ['binderhub'] to versions
 ['1.0.0-0.dev.git.3182.h5312a70'], respectively

---
 helm-charts/binderhub/Chart.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/helm-charts/binderhub/Chart.yaml b/helm-charts/binderhub/Chart.yaml
index 8b12d56b79..e35fd9dff7 100644
--- a/helm-charts/binderhub/Chart.yaml
+++ b/helm-charts/binderhub/Chart.yaml
@@ -5,7 +5,7 @@ name: binderhub
 version: "0.1.0"
 dependencies:
   - name: binderhub
-    version: "1.0.0-0.dev.git.3170.h84b1db9"
+    version: "1.0.0-0.dev.git.3182.h5312a70"
     repository: "https://jupyterhub.github.io/helm-chart/"
   - name: dask-gateway
     version: "2022.10.0"

From dc4455095ebc67e02d38133460671501e4790990 Mon Sep 17 00:00:00 2001
From: "2i2c-token-generator-bot[bot]"
 <106546794+2i2c-token-generator-bot[bot]@users.noreply.github.com>
Date: Sun, 1 Oct 2023 00:11:02 +0000
Subject: [PATCH 17/44] Bump charts ['prometheus', 'grafana', 'ingress-nginx']
 to versions ['25.0.0', '6.60.1', '4.8.0'], respectively

---
 helm-charts/support/Chart.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/helm-charts/support/Chart.yaml b/helm-charts/support/Chart.yaml
index 705ecef8ad..4d9b651192 100644
--- a/helm-charts/support/Chart.yaml
+++ b/helm-charts/support/Chart.yaml
@@ -15,13 +15,13 @@ dependencies:
   - name: prometheus
     # NOTE: CHECK INSTRUCTIONS UNDER prometheus.server.command IN support/values.yaml
     # EACH TIME THIS VERSION IS BUMPED!
-    version: 24.1.0
+    version: 25.0.0
     repository: https://prometheus-community.github.io/helm-charts
 
   # Grafana for dashboarding of metrics.
   # https://github.com/grafana/helm-charts/tree/main/charts/grafana
   - name: grafana
-    version: 6.59.0
+    version: 6.60.1
     repository: https://grafana.github.io/helm-charts
 
   # ingress-nginx for a k8s Ingress resource controller that routes traffic from
@@ -29,7 +29,7 @@ dependencies:
   # that references this controller.
   # https://github.com/kubernetes/ingress-nginx/tree/main/charts/ingress-nginx
   - name: ingress-nginx
-    version: 4.7.1
+    version: 4.8.0
     repository: https://kubernetes.github.io/ingress-nginx
 
   # cluster-autoscaler for k8s clusters where it doesn't come out of the box (EKS)

From afd3963836a7b16eab43abd990c45d9b025373bc Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sun, 1 Oct 2023 01:07:58 +0000
Subject: [PATCH 18/44] Update gspread requirement from ==5.10.* to ==5.11.*

Updates the requirements on [gspread](https://github.com/burnash/gspread) to permit the latest version.
- [Release notes](https://github.com/burnash/gspread/releases)
- [Changelog](https://github.com/burnash/gspread/blob/master/HISTORY.rst)
- [Commits](https://github.com/burnash/gspread/compare/v5.10.0...v5.11.3)

---
updated-dependencies:
- dependency-name: gspread
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 314f676a24..134218453c 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -33,7 +33,7 @@ prometheus_pandas
 pytest-mock
 google-cloud-bigquery==3.11.*
 google-cloud-bigquery[pandas]==3.11.*
-gspread==5.10.*
+gspread==5.11.*
 
 # requests is used by deployer/cilogon_app.py
 requests==2.*

From 17bd5fa78a758caf59d2a692e30521e2268d8704 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sun, 1 Oct 2023 01:23:25 +0000
Subject: [PATCH 19/44] Bump actions/checkout from 3 to 4

Bumps [actions/checkout](https://github.com/actions/checkout) from 3 to 4.
- [Release notes](https://github.com/actions/checkout/releases)
- [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md)
- [Commits](https://github.com/actions/checkout/compare/v3...v4)

---
updated-dependencies:
- dependency-name: actions/checkout
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/comment-deployment-plan-pr.yaml      | 2 +-
 .github/workflows/comment-pending-deployment-info.yaml | 2 +-
 .github/workflows/comment-test-link-merged-pr.yaml     | 2 +-
 .github/workflows/deploy-grafana-dashboards.yaml       | 2 +-
 .github/workflows/deploy-hubs.yaml                     | 6 +++---
 .github/workflows/ensure-uptime-checks.yaml            | 2 +-
 .github/workflows/terraform-lint.yaml                  | 2 +-
 .github/workflows/test-deployer-code.yaml              | 2 +-
 .github/workflows/validate-clusters.yaml               | 4 ++--
 9 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/comment-deployment-plan-pr.yaml b/.github/workflows/comment-deployment-plan-pr.yaml
index 2306e73403..14fba55501 100644
--- a/.github/workflows/comment-deployment-plan-pr.yaml
+++ b/.github/workflows/comment-deployment-plan-pr.yaml
@@ -32,7 +32,7 @@ jobs:
       issues: read
       pull-requests: write
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - run: pip install requests
       - run: python extra-scripts/comment-deployment-plan-pr.py
         env:
diff --git a/.github/workflows/comment-pending-deployment-info.yaml b/.github/workflows/comment-pending-deployment-info.yaml
index 4393e3cc84..664f376813 100644
--- a/.github/workflows/comment-pending-deployment-info.yaml
+++ b/.github/workflows/comment-pending-deployment-info.yaml
@@ -11,7 +11,7 @@ jobs:
     permissions:
       issues: write
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Parse new hub request issue template form
         uses: stefanbuck/github-issue-praser@v3
         id: issue-parser
diff --git a/.github/workflows/comment-test-link-merged-pr.yaml b/.github/workflows/comment-test-link-merged-pr.yaml
index f0d0c24d50..6eb5609e57 100644
--- a/.github/workflows/comment-test-link-merged-pr.yaml
+++ b/.github/workflows/comment-test-link-merged-pr.yaml
@@ -20,7 +20,7 @@ jobs:
       actions: read
       pull-requests: write
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - run: pip install requests
       - run: python extra-scripts/comment-test-link-merged-pr.py
         env:
diff --git a/.github/workflows/deploy-grafana-dashboards.yaml b/.github/workflows/deploy-grafana-dashboards.yaml
index 48ec7e9ea4..e214731452 100644
--- a/.github/workflows/deploy-grafana-dashboards.yaml
+++ b/.github/workflows/deploy-grafana-dashboards.yaml
@@ -39,7 +39,7 @@ jobs:
 
     steps:
       - name: Checkout repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - uses: actions/setup-python@v4
         with:
diff --git a/.github/workflows/deploy-hubs.yaml b/.github/workflows/deploy-hubs.yaml
index fbd26935d2..5c951d4c73 100644
--- a/.github/workflows/deploy-hubs.yaml
+++ b/.github/workflows/deploy-hubs.yaml
@@ -64,7 +64,7 @@ jobs:
 
     steps:
       - name: Checkout repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install Python 3.9
         uses: actions/setup-python@v4
@@ -214,7 +214,7 @@ jobs:
 
     steps:
       - name: Checkout repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Setup deploy for ${{ matrix.jobs.cluster_name }}
         uses: ./.github/actions/setup-deploy
@@ -387,7 +387,7 @@ jobs:
 
     steps:
       - name: Checkout repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Setup deploy for ${{ matrix.jobs.cluster_name }} cluster
         uses: ./.github/actions/setup-deploy
diff --git a/.github/workflows/ensure-uptime-checks.yaml b/.github/workflows/ensure-uptime-checks.yaml
index 71de00b094..a4a1901408 100644
--- a/.github/workflows/ensure-uptime-checks.yaml
+++ b/.github/workflows/ensure-uptime-checks.yaml
@@ -30,7 +30,7 @@ jobs:
 
     steps:
       - name: Checkout repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       # Uptime checks are set up and managed via terraform
       - uses: hashicorp/setup-terraform@v2
diff --git a/.github/workflows/terraform-lint.yaml b/.github/workflows/terraform-lint.yaml
index bcadfa5ff7..1eab9b743f 100644
--- a/.github/workflows/terraform-lint.yaml
+++ b/.github/workflows/terraform-lint.yaml
@@ -9,7 +9,7 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: hashicorp/setup-terraform@v2
       - uses: terraform-linters/setup-tflint@v3
       - name: Run terraform linter
diff --git a/.github/workflows/test-deployer-code.yaml b/.github/workflows/test-deployer-code.yaml
index 3464073f5c..e2582e8596 100644
--- a/.github/workflows/test-deployer-code.yaml
+++ b/.github/workflows/test-deployer-code.yaml
@@ -25,7 +25,7 @@ jobs:
   test-deployer:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
         with:
           python-version: "3.9"
diff --git a/.github/workflows/validate-clusters.yaml b/.github/workflows/validate-clusters.yaml
index b89b47b736..736d79d5a5 100644
--- a/.github/workflows/validate-clusters.yaml
+++ b/.github/workflows/validate-clusters.yaml
@@ -76,7 +76,7 @@ jobs:
       (github.event_name == 'pull_request' && contains(github.head_ref, fromJson('["dependabot", "pre-commit"]')) == false)
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
         with:
           python-version: "3.10"
@@ -201,7 +201,7 @@ jobs:
         jobs: ${{ fromJson(needs.generate-clusters-to-validate.outputs.cluster_matrix) }}
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
         with:
           python-version: "3.10"

From 17cf399b9b8b82663a335c00a6eab736abed10b9 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sun, 1 Oct 2023 01:23:28 +0000
Subject: [PATCH 20/44] Bump mdgreenwald/mozilla-sops-action from 1.4.1 to
 1.5.0

Bumps [mdgreenwald/mozilla-sops-action](https://github.com/mdgreenwald/mozilla-sops-action) from 1.4.1 to 1.5.0.
- [Release notes](https://github.com/mdgreenwald/mozilla-sops-action/releases)
- [Commits](https://github.com/mdgreenwald/mozilla-sops-action/compare/v1.4.1...v1.5.0)

---
updated-dependencies:
- dependency-name: mdgreenwald/mozilla-sops-action
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/deploy-grafana-dashboards.yaml | 2 +-
 .github/workflows/ensure-uptime-checks.yaml      | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/deploy-grafana-dashboards.yaml b/.github/workflows/deploy-grafana-dashboards.yaml
index 48ec7e9ea4..221a1cb89d 100644
--- a/.github/workflows/deploy-grafana-dashboards.yaml
+++ b/.github/workflows/deploy-grafana-dashboards.yaml
@@ -51,7 +51,7 @@ jobs:
           sudo apt install jsonnet
 
       - name: Install sops
-        uses: mdgreenwald/mozilla-sops-action@v1.4.1
+        uses: mdgreenwald/mozilla-sops-action@v1.5.0
 
       - name: Setup sops credentials to decrypt repo secrets
         uses: google-github-actions/auth@v1
diff --git a/.github/workflows/ensure-uptime-checks.yaml b/.github/workflows/ensure-uptime-checks.yaml
index 71de00b094..aa09735ed9 100644
--- a/.github/workflows/ensure-uptime-checks.yaml
+++ b/.github/workflows/ensure-uptime-checks.yaml
@@ -38,7 +38,7 @@ jobs:
       # We use sops to store encrypted GCP ServiceAccount Key that terraform uses
       # to run, as well as PagerDuty config terraform uses
       - name: Install sops
-        uses: mdgreenwald/mozilla-sops-action@v1.4.1
+        uses: mdgreenwald/mozilla-sops-action@v1.5.0
 
       # Authenticate with the correct KMS key that sops will use.
       - name: Setup sops credentials to decrypt repo secrets

From 11aecc24d8c1e12f7e6a4a50868e4e02fc54e82e Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sun, 1 Oct 2023 01:23:37 +0000
Subject: [PATCH 21/44] Bump tibdex/github-app-token from 1.8 to 2.1

Bumps [tibdex/github-app-token](https://github.com/tibdex/github-app-token) from 1.8 to 2.1.
- [Release notes](https://github.com/tibdex/github-app-token/releases)
- [Commits](https://github.com/tibdex/github-app-token/compare/v1.8...v2.1)

---
updated-dependencies:
- dependency-name: tibdex/github-app-token
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/bump-helm-versions.yaml | 2 +-
 .github/workflows/bump-image-tags.yaml    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/bump-helm-versions.yaml b/.github/workflows/bump-helm-versions.yaml
index dfb65c5ca2..4359edfef8 100644
--- a/.github/workflows/bump-helm-versions.yaml
+++ b/.github/workflows/bump-helm-versions.yaml
@@ -44,7 +44,7 @@ jobs:
       # Action: https://github.com/marketplace/actions/github-app-token
       - name: Fetch a token from GitHub App
         id: generate_token
-        uses: tibdex/github-app-token@v1.8
+        uses: tibdex/github-app-token@v2.1
         with:
           app_id: ${{ secrets.APP_ID }}
           private_key: ${{ secrets.PRIVATE_KEY }}
diff --git a/.github/workflows/bump-image-tags.yaml b/.github/workflows/bump-image-tags.yaml
index fb8f42ef6b..d069a3484d 100644
--- a/.github/workflows/bump-image-tags.yaml
+++ b/.github/workflows/bump-image-tags.yaml
@@ -47,7 +47,7 @@ jobs:
       # Action: https://github.com/marketplace/actions/github-app-token
       - name: Fetch a token from GitHub App
         id: generate_token
-        uses: tibdex/github-app-token@v1.8
+        uses: tibdex/github-app-token@v2.1
         with:
           app_id: ${{ secrets.APP_ID }}
           private_key: ${{ secrets.PRIVATE_KEY }}

From 49835b1485b9a3401feb93a3c03d66da998317fa Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sun, 1 Oct 2023 01:23:40 +0000
Subject: [PATCH 22/44] Bump toshimaru/auto-author-assign from 1.6.2 to 2.0.1

Bumps [toshimaru/auto-author-assign](https://github.com/toshimaru/auto-author-assign) from 1.6.2 to 2.0.1.
- [Release notes](https://github.com/toshimaru/auto-author-assign/releases)
- [Changelog](https://github.com/toshimaru/auto-author-assign/blob/main/CHANGELOG.md)
- [Commits](https://github.com/toshimaru/auto-author-assign/compare/v1.6.2...v2.0.1)

---
updated-dependencies:
- dependency-name: toshimaru/auto-author-assign
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/auto-author-assign-pull-request.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/auto-author-assign-pull-request.yaml b/.github/workflows/auto-author-assign-pull-request.yaml
index c84c0894f3..c00899ae21 100644
--- a/.github/workflows/auto-author-assign-pull-request.yaml
+++ b/.github/workflows/auto-author-assign-pull-request.yaml
@@ -16,4 +16,4 @@ jobs:
     steps:
       - name: Automatically assign PR author
         id: assignation
-        uses: toshimaru/auto-author-assign@v1.6.2
+        uses: toshimaru/auto-author-assign@v2.0.1

From 07eb402da55a15b50290a331b3c6009a5c9e4e91 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sun, 1 Oct 2023 09:00:48 +0000
Subject: [PATCH 23/44] Bump terraform-linters/setup-tflint from 3 to 4

Bumps [terraform-linters/setup-tflint](https://github.com/terraform-linters/setup-tflint) from 3 to 4.
- [Release notes](https://github.com/terraform-linters/setup-tflint/releases)
- [Commits](https://github.com/terraform-linters/setup-tflint/compare/v3...v4)

---
updated-dependencies:
- dependency-name: terraform-linters/setup-tflint
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/terraform-lint.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/terraform-lint.yaml b/.github/workflows/terraform-lint.yaml
index 1eab9b743f..77776e8ff9 100644
--- a/.github/workflows/terraform-lint.yaml
+++ b/.github/workflows/terraform-lint.yaml
@@ -11,7 +11,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - uses: hashicorp/setup-terraform@v2
-      - uses: terraform-linters/setup-tflint@v3
+      - uses: terraform-linters/setup-tflint@v4
       - name: Run terraform linter
         run: |
           cd terraform

From 578ad789ad6916365e0c078d6a8c7b1d2be7a965 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sun, 1 Oct 2023 09:07:09 +0000
Subject: [PATCH 24/44] Bump mdgreenwald/mozilla-sops-action in
 /.github/actions/setup-deploy

Bumps [mdgreenwald/mozilla-sops-action](https://github.com/mdgreenwald/mozilla-sops-action) from 1.4.1 to 1.5.0.
- [Release notes](https://github.com/mdgreenwald/mozilla-sops-action/releases)
- [Commits](https://github.com/mdgreenwald/mozilla-sops-action/compare/v1.4.1...v1.5.0)

---
updated-dependencies:
- dependency-name: mdgreenwald/mozilla-sops-action
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/actions/setup-deploy/action.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/actions/setup-deploy/action.yaml b/.github/actions/setup-deploy/action.yaml
index 99a717aac7..5b461ac572 100644
--- a/.github/actions/setup-deploy/action.yaml
+++ b/.github/actions/setup-deploy/action.yaml
@@ -109,7 +109,7 @@ runs:
 
     # This action use the github official cache mechanism internally
     - name: Install sops
-      uses: mdgreenwald/mozilla-sops-action@v1.4.1
+      uses: mdgreenwald/mozilla-sops-action@v1.5.0
 
     # Install pre-requisite for "gcloud container clusters get-credentials"
     # command with a modern k8s client.

From f878a11d9b7081e3ca5ee5ffcd731fd14032584b Mon Sep 17 00:00:00 2001
From: Erik Sundell <erik.i.sundell@gmail.com>
Date: Mon, 2 Oct 2023 10:37:49 +0200
Subject: [PATCH 25/44] ubc-eoas: add image choice for class EOS212

---
 config/clusters/ubc-eoas/common.values.yaml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/config/clusters/ubc-eoas/common.values.yaml b/config/clusters/ubc-eoas/common.values.yaml
index 7a3f50350c..188c54389e 100644
--- a/config/clusters/ubc-eoas/common.values.yaml
+++ b/config/clusters/ubc-eoas/common.values.yaml
@@ -79,6 +79,12 @@ jupyterhub:
           # Using 'latest' for now so updates do not require 2i2c
           # involvement.
           image: quay.io/henrykmodzelewski/2i2c-eosc211:latest
+      - display_name: EOSC212
+        description: "For class EOSC212, provides upto 1 CPU and ~4G RAM"
+        kubespawner_override:
+          # Using 'latest' for now so updates do not require 2i2c
+          # involvement.
+          image: quay.io/henrykmodzelewski/2i2c-eosc212:latest
       - display_name: EOSC350
         description: "For class EOSC350, provides upto 1 CPU and ~4G RAM"
         kubespawner_override:

From 8049584687b9527f7096175b694b08002fafdebe Mon Sep 17 00:00:00 2001
From: Erik Sundell <erik.i.sundell@gmail.com>
Date: Mon, 2 Oct 2023 10:55:43 +0200
Subject: [PATCH 26/44] 2i2c, temple: update memory requests/limit on request

---
 config/clusters/2i2c/temple.values.yaml | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/config/clusters/2i2c/temple.values.yaml b/config/clusters/2i2c/temple.values.yaml
index a0bd3d02a8..abbeb5ac05 100644
--- a/config/clusters/2i2c/temple.values.yaml
+++ b/config/clusters/2i2c/temple.values.yaml
@@ -33,6 +33,16 @@ jupyterhub:
         operator: "Equal"
         value: "temple"
         effect: "NoSchedule"
+    memory:
+      # Memory defaults are 256MB to 1G in basehub. These are bumped based on a
+      # request to bump these (https://2i2c.freshdesk.com/a/tickets/1003) at
+      # least during October 2-15.
+      #
+      # A previous request (https://2i2c.freshdesk.com/a/tickets/643) included
+      # notes on the new memory request and limits to adopt once again.
+      #
+      guarantee: 512M
+      limit: 2G
   hub:
     config:
       Authenticator:

From 629ce96add71d5630f763e2a04259c4bd392b51f Mon Sep 17 00:00:00 2001
From: Erik Sundell <erik.i.sundell@gmail.com>
Date: Mon, 2 Oct 2023 11:23:17 +0200
Subject: [PATCH 27/44] refactor: cleanup redundant config about dask-gateway
 idle_timeout

---
 config/clusters/carbonplan/common.values.yaml              | 5 -----
 config/clusters/jupyter-meets-the-earth/common.values.yaml | 3 ---
 config/clusters/meom-ige/common.values.yaml                | 6 ------
 config/clusters/openscapes/common.values.yaml              | 6 ------
 4 files changed, 20 deletions(-)

diff --git a/config/clusters/carbonplan/common.values.yaml b/config/clusters/carbonplan/common.values.yaml
index 28a0dd8685..3c6e40c791 100644
--- a/config/clusters/carbonplan/common.values.yaml
+++ b/config/clusters/carbonplan/common.values.yaml
@@ -237,8 +237,3 @@ dask-gateway:
       limits:
         cpu: 2
         memory: 4Gi
-      # TODO: figure out a replacement for userLimits.
-    extraConfig:
-      idle: |
-        # timeout after 30 minutes of inactivity
-        c.KubeClusterConfig.idle_timeout = 1800
diff --git a/config/clusters/jupyter-meets-the-earth/common.values.yaml b/config/clusters/jupyter-meets-the-earth/common.values.yaml
index 0d6f5b1daa..2bf0a95e29 100644
--- a/config/clusters/jupyter-meets-the-earth/common.values.yaml
+++ b/config/clusters/jupyter-meets-the-earth/common.values.yaml
@@ -337,6 +337,3 @@ dask-gateway:
                 handler=option_handler,
             )
         c.Backend.cluster_options = cluster_options
-      idle: |
-        # timeout after 30 minutes of inactivity
-        c.KubeClusterConfig.idle_timeout = 1800
diff --git a/config/clusters/meom-ige/common.values.yaml b/config/clusters/meom-ige/common.values.yaml
index 1454f1ef43..c9ee011f5c 100644
--- a/config/clusters/meom-ige/common.values.yaml
+++ b/config/clusters/meom-ige/common.values.yaml
@@ -158,9 +158,3 @@ basehub:
             - roxyboy
             - lesommer
             - auraoupa
-dask-gateway:
-  gateway:
-    extraConfig:
-      idle: |
-        # timeout after 30 minutes of inactivity
-        c.KubeClusterConfig.idle_timeout = 1800
diff --git a/config/clusters/openscapes/common.values.yaml b/config/clusters/openscapes/common.values.yaml
index cb4feca425..b4807a4557 100644
--- a/config/clusters/openscapes/common.values.yaml
+++ b/config/clusters/openscapes/common.values.yaml
@@ -73,9 +73,3 @@ basehub:
           #        configured explicitly.
           #
           allowed_users: *users
-dask-gateway:
-  gateway:
-    extraConfig:
-      idle: |-
-        # timeout after 30 minutes of inactivity
-        c.KubeClusterConfig.idle_timeout = 1800

From 71c80ec05135add9faeb58f9df7d552132e7075b Mon Sep 17 00:00:00 2001
From: Erik Sundell <erik.i.sundell@gmail.com>
Date: Mon, 2 Oct 2023 11:25:06 +0200
Subject: [PATCH 28/44] refactor: cleanup redundant config about
 matchNodePurpose

---
 config/clusters/openscapes/common.values.yaml | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/config/clusters/openscapes/common.values.yaml b/config/clusters/openscapes/common.values.yaml
index b4807a4557..02505c43bc 100644
--- a/config/clusters/openscapes/common.values.yaml
+++ b/config/clusters/openscapes/common.values.yaml
@@ -43,9 +43,6 @@ basehub:
     scheduling:
       userScheduler:
         enabled: true
-      userPods:
-        nodeAffinity:
-          matchNodePurpose: require
     hub:
       allowNamedServers: true
       readinessProbe:

From 75745379c16bbb772138ccda3c5c1af13d12fbee Mon Sep 17 00:00:00 2001
From: Erik Sundell <erik.i.sundell@gmail.com>
Date: Mon, 2 Oct 2023 11:37:31 +0200
Subject: [PATCH 29/44] docs: fix indentation about tolerations

---
 docs/howto/features/dedicated-nodepool.md | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/docs/howto/features/dedicated-nodepool.md b/docs/howto/features/dedicated-nodepool.md
index 3689b3ee91..79bc5b82b2 100644
--- a/docs/howto/features/dedicated-nodepool.md
+++ b/docs/howto/features/dedicated-nodepool.md
@@ -54,14 +54,14 @@ Some hubs on shared clusters require dedicated nodepools, for a few reasons:
 
    ```yaml
    jupyterhub:
-      singleuser:
-         nodeSelector:
-            2i2c.org/community: <community-name>
-         extraTolerations:
-            - key: "2i2c.org/community"
-              operator: "Equal"
-              value: "<community-name>"
-              effect: "NoSchedule"
+     singleuser:
+       nodeSelector:
+         2i2c.org/community: <community-name>
+       extraTolerations:
+         - key: 2i2c.org/community
+           operator: Equal
+           value: <community-name>
+           effect: NoSchedule
    ```
 
    ```{note}

From c02ea8493d62934fdbf568b938d65891358e0e70 Mon Sep 17 00:00:00 2001
From: Erik Sundell <erik.i.sundell@gmail.com>
Date: Mon, 2 Oct 2023 11:38:10 +0200
Subject: [PATCH 30/44] support, node-exporter: tolerate 2i2c.org/community
 tainted nodes

---
 helm-charts/support/values.yaml | 15 ---------------
 1 file changed, 15 deletions(-)

diff --git a/helm-charts/support/values.yaml b/helm-charts/support/values.yaml
index 686deaeec0..87f50ea338 100644
--- a/helm-charts/support/values.yaml
+++ b/helm-charts/support/values.yaml
@@ -85,21 +85,6 @@ prometheus:
   # values ref: https://github.com/prometheus-community/helm-charts/blob/main/charts/prometheus-node-exporter/values.yaml
   #
   prometheus-node-exporter:
-    tolerations:
-      # Tolerate tainted jupyterhub user nodes
-      - key: hub.jupyter.org_dedicated
-        value: user
-        effect: NoSchedule
-      - key: hub.jupyter.org/dedicated
-        value: user
-        effect: NoSchedule
-      # Tolerate tainted dask worker nodes
-      - key: k8s.dask.org_dedicated
-        value: worker
-        effect: NoSchedule
-      - key: k8s.dask.org/dedicated
-        value: worker
-        effect: NoSchedule
     # resources for the node-exporter was set after inspecting cpu and memory
     # use via prometheus and grafana.
     #

From c1914c7abe55328aa91d96609102744b3cb8006d Mon Sep 17 00:00:00 2001
From: Erik Sundell <erik.i.sundell@gmail.com>
Date: Mon, 2 Oct 2023 14:11:08 +0200
Subject: [PATCH 31/44] 2i2c, terraform: transition user nodes from
 n1-highmem-4 to n2-highmem-4

---
 .../clusters/2i2c/imagebuilding-demo.values.yaml | 16 ++++++++++------
 terraform/gcp/projects/pilot-hubs.tfvars         |  2 +-
 2 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/config/clusters/2i2c/imagebuilding-demo.values.yaml b/config/clusters/2i2c/imagebuilding-demo.values.yaml
index 766c8facac..cce5335087 100644
--- a/config/clusters/2i2c/imagebuilding-demo.values.yaml
+++ b/config/clusters/2i2c/imagebuilding-demo.values.yaml
@@ -80,14 +80,15 @@ jupyterhub:
             choices:
               mem_2_7:
                 display_name: 2.7 GB RAM, upto 3.479 CPUs
-                description: "Use this for the workshop on 2023 September"
+                description: Use this for the workshop on 2023 September
                 kubespawner_override:
                   mem_guarantee: 2904451072
                   mem_limit: 2904451072
                   cpu_guarantee: 0.434875
                   cpu_limit: 3.479
                   node_selector:
-                    node.kubernetes.io/instance-type: n1-highmem-4
+                    # FIXME: guarantee/limits initialized for n1-highmem-4, not n2-
+                    node.kubernetes.io/instance-type: n2-highmem-4
                 default: true
               mem_5_4:
                 display_name: 5.4 GB RAM, upto 3.479 CPUs
@@ -97,7 +98,8 @@ jupyterhub:
                   cpu_guarantee: 0.86975
                   cpu_limit: 3.479
                   node_selector:
-                    node.kubernetes.io/instance-type: n1-highmem-4
+                    # FIXME: guarantee/limits initialized for n1-highmem-4, not n2-
+                    node.kubernetes.io/instance-type: n2-highmem-4
               mem_10_8:
                 display_name: 10.8 GB RAM, upto 3.479 CPUs
                 kubespawner_override:
@@ -106,17 +108,19 @@ jupyterhub:
                   cpu_guarantee: 1.7395
                   cpu_limit: 3.479
                   node_selector:
-                    node.kubernetes.io/instance-type: n1-highmem-4
+                    # FIXME: guarantee/limits initialized for n1-highmem-4, not n2-
+                    node.kubernetes.io/instance-type: n2-highmem-4
               mem_21_6:
                 display_name: 21.6 GB RAM, upto 3.479 CPUs
-                description: "Largest amount of RAM, might take a few minutes to start"
+                description: Largest amount of RAM, might take a few minutes to start
                 kubespawner_override:
                   mem_guarantee: 23235608576
                   mem_limit: 23235608576
                   cpu_guarantee: 3.479
                   cpu_limit: 3.479
                 node_selector:
-                  node.kubernetes.io/instance-type: n1-highmem-4
+                  # FIXME: guarantee/limits initialized for n1-highmem-4, not n2-
+                  node.kubernetes.io/instance-type: n2-highmem-4
   hub:
     services:
       binder:
diff --git a/terraform/gcp/projects/pilot-hubs.tfvars b/terraform/gcp/projects/pilot-hubs.tfvars
index 345f76931d..2a8396173c 100644
--- a/terraform/gcp/projects/pilot-hubs.tfvars
+++ b/terraform/gcp/projects/pilot-hubs.tfvars
@@ -22,7 +22,7 @@ notebook_nodes = {
   "user" : {
     min : 0,
     max : 20,
-    machine_type : "n1-highmem-4",
+    machine_type : "n2-highmem-4",
   },
   "climatematch" : {
     min : 0,

From d76f104d0b2ce5a3e496bcbe257a073dbebc393e Mon Sep 17 00:00:00 2001
From: Erik Sundell <erik.i.sundell@gmail.com>
Date: Mon, 2 Oct 2023 14:22:11 +0200
Subject: [PATCH 32/44] 2i2c, terraform: transition from n1- to n2-
 neurohackademy nodes

---
 config/clusters/2i2c/neurohackademy.values.yaml | 2 +-
 terraform/gcp/projects/pilot-hubs.tfvars        | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/config/clusters/2i2c/neurohackademy.values.yaml b/config/clusters/2i2c/neurohackademy.values.yaml
index f5fba70b7f..17f703e810 100644
--- a/config/clusters/2i2c/neurohackademy.values.yaml
+++ b/config/clusters/2i2c/neurohackademy.values.yaml
@@ -46,7 +46,7 @@ jupyterhub:
         effect: "NoSchedule"
     cpu:
       guarantee: 0.5
-      # We're on n1-highmem-16 machines
+      # We're on n2-highmem-16 machines
       limit: 14
     memory:
       guarantee: 4G
diff --git a/terraform/gcp/projects/pilot-hubs.tfvars b/terraform/gcp/projects/pilot-hubs.tfvars
index 2a8396173c..572da56d4a 100644
--- a/terraform/gcp/projects/pilot-hubs.tfvars
+++ b/terraform/gcp/projects/pilot-hubs.tfvars
@@ -45,7 +45,7 @@ notebook_nodes = {
     # We expect around 120 users
     min : 0,
     max : 100,
-    machine_type : "n1-highmem-16",
+    machine_type : "n2-highmem-16",
     labels : {
       "2i2c.org/community" : "neurohackademy"
     },

From 226233726bd5feaf86a5d08a54cc522e72e331a9 Mon Sep 17 00:00:00 2001
From: Erik Sundell <erik.i.sundell@gmail.com>
Date: Mon, 2 Oct 2023 14:23:02 +0200
Subject: [PATCH 33/44] docs: reduce config example to avoid mentioning n1-

---
 .../configure-auth/github-orgs.md              | 18 +++++++-----------
 1 file changed, 7 insertions(+), 11 deletions(-)

diff --git a/docs/hub-deployment-guide/configure-auth/github-orgs.md b/docs/hub-deployment-guide/configure-auth/github-orgs.md
index c6be13a064..b8f38f835b 100644
--- a/docs/hub-deployment-guide/configure-auth/github-orgs.md
+++ b/docs/hub-deployment-guide/configure-auth/github-orgs.md
@@ -208,27 +208,23 @@ To enable this access,
     jupyterhub:
       singleuser:
         profileList:
-          - display_name: "Small"
-            description: 5GB RAM, 2 CPUs
+          - display_name: Small
+            description: 1.0 GB RAM
             default: true
             allowed_teams:
               - <org-name>:<team-name>
               - 2i2c-org:hub-access-for-2i2c-staff
             kubespawner_override:
-              mem_limit: 7G
-              mem_guarantee: 4.5G
-              node_selector:
-                node.kubernetes.io/instance-type: n1-standard-2
+              mem_guarantee: 1G
+              mem_limit: 1G
           - display_name: Medium
-            description: 11GB RAM, 4 CPUs
+            description: 4.0 GB RAM
             allowed_teams:
               - <org-name>:<team-name>
               - 2i2c-org:hub-access-for-2i2c-staff
             kubespawner_override:
-              mem_limit: 15G
-              mem_guarantee: 11G
-              node_selector:
-                node.kubernetes.io/instance-type: n1-standard-4
+              mem_guarantee: 4G
+              mem_limit: 4G
     ```
 
     Users who are a part of *any* of the listed teams will be able to access that profile.

From d16e17b8ac0e803bdc2764547b22b448c6a13251 Mon Sep 17 00:00:00 2001
From: Erik Sundell <erik.i.sundell@gmail.com>
Date: Mon, 2 Oct 2023 21:52:49 +0200
Subject: [PATCH 34/44] jmte: reduce memory request to ensure cryptonono can
 fit

---
 config/clusters/jupyter-meets-the-earth/common.values.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/config/clusters/jupyter-meets-the-earth/common.values.yaml b/config/clusters/jupyter-meets-the-earth/common.values.yaml
index 2bf0a95e29..5ca62e82d4 100644
--- a/config/clusters/jupyter-meets-the-earth/common.values.yaml
+++ b/config/clusters/jupyter-meets-the-earth/common.values.yaml
@@ -151,7 +151,7 @@ basehub:
           profile_options: *profile_options
           kubespawner_override:
             cpu_guarantee: 3.5
-            mem_guarantee: 14G
+            mem_guarantee: 13G
             mem_limit: null
             node_selector:
               node.kubernetes.io/instance-type: m5.xlarge
@@ -184,7 +184,7 @@ basehub:
           profile_options: *profile_options
           kubespawner_override:
             cpu_guarantee: 3.5
-            mem_guarantee: 14G
+            mem_guarantee: 13G
             mem_limit: null
             environment:
               NVIDIA_DRIVER_CAPABILITIES: compute,utility

From 11bc4263154e41ac00a3a779302bbe0ca1a4c80c Mon Sep 17 00:00:00 2001
From: Erik Sundell <erik.i.sundell@gmail.com>
Date: Mon, 2 Oct 2023 22:57:14 +0200
Subject: [PATCH 35/44] Pin github action tibdex/github-app-token to v2 instead
 of v2.1

---
 .github/workflows/bump-helm-versions.yaml | 2 +-
 .github/workflows/bump-image-tags.yaml    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/bump-helm-versions.yaml b/.github/workflows/bump-helm-versions.yaml
index 4359edfef8..2c11de3929 100644
--- a/.github/workflows/bump-helm-versions.yaml
+++ b/.github/workflows/bump-helm-versions.yaml
@@ -44,7 +44,7 @@ jobs:
       # Action: https://github.com/marketplace/actions/github-app-token
       - name: Fetch a token from GitHub App
         id: generate_token
-        uses: tibdex/github-app-token@v2.1
+        uses: tibdex/github-app-token@v2
         with:
           app_id: ${{ secrets.APP_ID }}
           private_key: ${{ secrets.PRIVATE_KEY }}
diff --git a/.github/workflows/bump-image-tags.yaml b/.github/workflows/bump-image-tags.yaml
index d069a3484d..17e303c759 100644
--- a/.github/workflows/bump-image-tags.yaml
+++ b/.github/workflows/bump-image-tags.yaml
@@ -47,7 +47,7 @@ jobs:
       # Action: https://github.com/marketplace/actions/github-app-token
       - name: Fetch a token from GitHub App
         id: generate_token
-        uses: tibdex/github-app-token@v2.1
+        uses: tibdex/github-app-token@v2
         with:
           app_id: ${{ secrets.APP_ID }}
           private_key: ${{ secrets.PRIVATE_KEY }}

From a62fcf7289a4e4a08e36becd8c068e609c15e784 Mon Sep 17 00:00:00 2001
From: Erik Sundell <erik.i.sundell@gmail.com>
Date: Tue, 3 Oct 2023 01:31:28 +0200
Subject: [PATCH 36/44] daskhub: update to use dask-gateway 2023.9.0

---
 deployer/deployer.py           | 2 +-
 helm-charts/daskhub/Chart.yaml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/deployer/deployer.py b/deployer/deployer.py
index 18b157f3f0..677b8db6de 100644
--- a/deployer/deployer.py
+++ b/deployer/deployer.py
@@ -157,7 +157,7 @@ def deploy(
         help="Name of hub to operate deploy. Omit to deploy all hubs on the cluster",
     ),
     dask_gateway_version: str = typer.Option(
-        "2023.1.0", help="Version of dask-gateway to install CRDs for"
+        "2023.9.0", help="Version of dask-gateway to install CRDs for"
     ),
     debug: bool = typer.Option(
         False,
diff --git a/helm-charts/daskhub/Chart.yaml b/helm-charts/daskhub/Chart.yaml
index 41316307d0..37445c078e 100644
--- a/helm-charts/daskhub/Chart.yaml
+++ b/helm-charts/daskhub/Chart.yaml
@@ -11,5 +11,5 @@ dependencies:
     # in the deployer's CLI
     # https://github.com/2i2c-org/infrastructure/blob/HEAD/deployer/deployer.py#L195
   - name: dask-gateway
-    version: "2023.1.0"
+    version: "2023.9.0"
     repository: "https://helm.dask.org/"

From cf44ff8a007c20ea254a51010725e883357bd149 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Tue, 3 Oct 2023 01:36:40 +0000
Subject: [PATCH 37/44] [pre-commit.ci] pre-commit autoupdate
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

updates:
- [github.com/asottile/pyupgrade: v3.10.1 → v3.14.0](https://github.com/asottile/pyupgrade/compare/v3.10.1...v3.14.0)
- [github.com/psf/black: 23.7.0 → 23.9.1](https://github.com/psf/black/compare/23.7.0...23.9.1)
---
 .pre-commit-config.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 6af250d603..69fe24c12a 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -17,7 +17,7 @@ repos:
 
   # Autoformat: Python code, syntax patterns are modernized
   - repo: https://github.com/asottile/pyupgrade
-    rev: v3.10.1
+    rev: v3.14.0
     hooks:
       - id: pyupgrade
         args:
@@ -31,7 +31,7 @@ repos:
 
   # Autoformat: Python code
   - repo: https://github.com/psf/black
-    rev: "23.7.0"
+    rev: "23.9.1"
     hooks:
       - id: black
 

From fb013398f044d30782e6c027b527e41e911d25db Mon Sep 17 00:00:00 2001
From: Georgiana <georgiana.dolocan@gmail.com>
Date: Tue, 3 Oct 2023 12:49:02 +0300
Subject: [PATCH 38/44] Typo

Co-authored-by: Yuvi Panda <yuvipanda@gmail.com>
---
 deployer/debug.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deployer/debug.py b/deployer/debug.py
index 81519d3ebd..f1706ab8e9 100644
--- a/deployer/debug.py
+++ b/deployer/debug.py
@@ -272,7 +272,7 @@ def transfer_old_home_dir_to_new_location(
     destination_user: str = typer.Option(
         prompt=True,
         confirmation_prompt=True,
-        help="Name of destination user in who's home dir we will be moving the files",
+        help="Name of destination user to whose home dir we will be moving the files",
     ),
 ):
     """

From 14efd76767588a7f93c9b267b8c48fad9bc9de52 Mon Sep 17 00:00:00 2001
From: Georgiana <georgiana.dolocan@gmail.com>
Date: Tue, 3 Oct 2023 12:49:40 +0300
Subject: [PATCH 39/44] Rephrase for clarity

Co-authored-by: Yuvi Panda <yuvipanda@gmail.com>
---
 deployer/debug.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deployer/debug.py b/deployer/debug.py
index f1706ab8e9..00db5c0f90 100644
--- a/deployer/debug.py
+++ b/deployer/debug.py
@@ -276,7 +276,7 @@ def transfer_old_home_dir_to_new_location(
     ),
 ):
     """
-    Copy all the files from a user's old home dir to the new home directory
+    Copy all the files from a user's home dir to a subdir in destination user's home dir
     """
     # Name the pod so we know what to delete when the transfer is done
     pod_name = f"{cluster_name}-{hub_name}-transfer-shell"

From d4e8f012cd1af5f57ac075691e5d6e6f508c8344 Mon Sep 17 00:00:00 2001
From: Georgiana <georgiana.dolocan@gmail.com>
Date: Tue, 3 Oct 2023 12:50:46 +0300
Subject: [PATCH 40/44] Use -h flag of ls to improve readability

Co-authored-by: Erik Sundell <erik.i.sundell@gmail.com>
---
 deployer/debug.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deployer/debug.py b/deployer/debug.py
index 00db5c0f90..2c4ea9dfb4 100644
--- a/deployer/debug.py
+++ b/deployer/debug.py
@@ -352,7 +352,7 @@ def transfer_old_home_dir_to_new_location(
         pod_name,
         "--",
         "ls",
-        "-lat",
+        "-lath",
         f"/home/{source_user}",
         f"/home/{destination_user}",
     ]

From 8d6f20290e569f5315e8905206ebb579474cedde Mon Sep 17 00:00:00 2001
From: Georgiana Dolocan <georgiana.dolocan@gmail.com>
Date: Tue, 3 Oct 2023 14:30:05 +0300
Subject: [PATCH 41/44] Define the ubuntu image used by pods as number and in
 one central place

Co-authored-by: Erik Sundell <erik.i.sundell@gmail.com>
---
 deployer/debug.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/deployer/debug.py b/deployer/debug.py
index 2c4ea9dfb4..26339b4b39 100644
--- a/deployer/debug.py
+++ b/deployer/debug.py
@@ -17,6 +17,10 @@
 
 # Without `pure=True`, I get an exception about str / byte issues
 yaml = YAML(typ="safe", pure=True)
+# Define the numerical version of the ubuntu image
+# used by the helper pods created by this script
+# in one central place to update it more easily
+UBUNTU_IMAGE = "ubuntu:22.04"
 
 
 class InfraComponents(Enum):
@@ -146,7 +150,7 @@ def exec_homes_shell(
                 {
                     "name": pod_name,
                     # Use ubuntu image so we get better gnu rm
-                    "image": "ubuntu:jammy",
+                    "image": UBUNTU_IMAGE,
                     "stdin": True,
                     "stdinOnce": True,
                     "tty": True,
@@ -173,7 +177,7 @@ def exec_homes_shell(
         "--image",
         # Use ubuntu image so we get GNU rm and other tools
         # Should match what we have in our pod definition
-        "ubuntu:jammy",
+        UBUNTU_IMAGE,
         pod_name,
         "--",
         "/bin/bash",
@@ -297,7 +301,7 @@ def transfer_old_home_dir_to_new_location(
                 {
                     "name": pod_name,
                     # Use ubuntu image so we get better GNU cp
-                    "image": "ubuntu:jammy",
+                    "image": UBUNTU_IMAGE,
                     "stdin": True,
                     "stdinOnce": True,
                     "tty": True,
@@ -325,7 +329,7 @@ def transfer_old_home_dir_to_new_location(
         "--image",
         # Use ubuntu image so we get GNU cp and other tools
         # Should match what we have in our pod definition
-        "ubuntu:jammy",
+        UBUNTU_IMAGE,
         pod_name,
     ]
 

From aab580426b5441c8017e293af562eddf6bfb5c6b Mon Sep 17 00:00:00 2001
From: Georgiana Dolocan <georgiana.dolocan@gmail.com>
Date: Tue, 3 Oct 2023 14:45:37 +0300
Subject: [PATCH 42/44] Rename command

---
 deployer/debug.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deployer/debug.py b/deployer/debug.py
index 26339b4b39..5afdd4adb9 100644
--- a/deployer/debug.py
+++ b/deployer/debug.py
@@ -265,7 +265,7 @@ def start_docker_proxy(
 
 
 @app.command()
-def transfer_old_home_dir_to_new_location(
+def copy_homedir_into_another(
     cluster_name: str = typer.Argument(..., help="Name of cluster to operate on"),
     hub_name: str = typer.Argument(..., help="Name of hub to operate on"),
     source_user: str = typer.Option(

From 5993b16913c16e1d9b94983fdf47ffddb8e41e98 Mon Sep 17 00:00:00 2001
From: Georgiana Dolocan <georgiana.dolocan@gmail.com>
Date: Tue, 3 Oct 2023 16:56:54 +0300
Subject: [PATCH 43/44] Update the copy command with more logging and and extra
 prompt

---
 deployer/debug.py | 177 +++++++++++++++++++++++++++++++++++-----------
 1 file changed, 134 insertions(+), 43 deletions(-)

diff --git a/deployer/debug.py b/deployer/debug.py
index 5afdd4adb9..3eafd93882 100644
--- a/deployer/debug.py
+++ b/deployer/debug.py
@@ -264,26 +264,13 @@ def start_docker_proxy(
         subprocess.check_call(cmd)
 
 
-@app.command()
-def copy_homedir_into_another(
-    cluster_name: str = typer.Argument(..., help="Name of cluster to operate on"),
-    hub_name: str = typer.Argument(..., help="Name of hub to operate on"),
-    source_user: str = typer.Option(
-        prompt=True,
-        confirmation_prompt=True,
-        help="Name of source user who's home dir files we are transferring",
-    ),
-    destination_user: str = typer.Option(
-        prompt=True,
-        confirmation_prompt=True,
-        help="Name of destination user to whose home dir we will be moving the files",
-    ),
-):
+def create_ready_home_pod_jupyter_user(pod_name, cluster_name, hub_name):
     """
-    Copy all the files from a user's home dir to a subdir in destination user's home dir
+    Function that:
+        - creates a pod with with the home directories of <cluster_name>-<hub_name> hub mounted on /home
+        - starts a container for the jupyter user, uuid 1000 in this pod
+        - waits for the pod  to be ready
     """
-    # Name the pod so we know what to delete when the transfer is done
-    pod_name = f"{cluster_name}-{hub_name}-transfer-shell"
     pod = {
         "apiVersion": "v1",
         "kind": "Pod",
@@ -347,8 +334,64 @@ def copy_homedir_into_another(
         "--timeout=90s",
     ]
 
+    print_colour(
+        f"Creating a pod with the home directories of {cluster_name} - {hub_name} hub mounted on /home...",
+        "yellow",
+    )
+    subprocess.check_call(create_pod_cmd)
+    subprocess.check_call(wait_for_pod_cmd)
+
+
+def ask_for_dirname_again():
+    """
+    Function that asks the user to provide the name of the source and dest directories using typer prompts.
+
+    Returns the name of the source and dest directories as a touple if they were provided by the user
+    or the None, None touple.
+    """
+    print_colour("Asking for the dirs again...", "yellow")
+    continue_with_dir_names_confirmation = typer.confirm(
+        "Can you now provide the names of the source and destination home directories?"
+    )
+    if not continue_with_dir_names_confirmation:
+        return None, None
+
+    source_homedir = typer.prompt(
+        "What is the (escaped) name of source homedir that will be copied?"
+    )
+    destination_homedir = typer.prompt(
+        "What is the (escaped) name of destination homedir where the files will be copied?"
+    )
+
+    return source_homedir, destination_homedir
+
+
+def ls_home_dir(hub_name, pod_name):
+    """
+    List the contents of /home in <pod_name>
+    """
+    print_colour("Listing the content of the /home directory instead...", "yellow")
+    # Command that lists the contents of /home directory
+    ls_home_cmd = [
+        "kubectl",
+        "-n",
+        hub_name,
+        "exec",
+        pod_name,
+        "--",
+        "ls",
+        "-lath",
+        "/home",
+    ]
+    subprocess.check_call(ls_home_cmd)
+
+
+def ls_source_and_dest_dirs(source_dir, dest_dir, hub_name, pod_name):
+    """
+    Lists the content of <source_dir> and <dest_dir> in <pod_name>
+    """
     # Command that lists the contents of the source and destination home dirs
-    ls_dirs_cmd = [
+    ls_source_and_dest_dirs_cmd = [
         "kubectl",
         "-n",
         hub_name,
@@ -357,13 +400,20 @@ def copy_homedir_into_another(
         "--",
         "ls",
         "-lath",
-        f"/home/{source_user}",
-        f"/home/{destination_user}",
+        f"/home/{source_dir}",
+        f"/home/{dest_dir}",
     ]
+    print_colour("The content of the source and destination home dirs is:")
+    subprocess.check_call(ls_source_and_dest_dirs_cmd)
 
-    # Command that copies the home directory of the source user
-    # into the home directory of the destination user
-    # under a directory located at `/home/{destination_user}/{source_user}-homedir``
+
+def copy_into_subdir(source_dir, dest_dir, hub_name, pod_name):
+    """
+    Copy the source home directory <source_dir>,
+    into the home directory of the destination directory <dest_dir>,
+    under a directory located at `/home/<dest_dir>/<source_dir>-homedir`
+    """
+    print_colour("Starting the copying process...", "yellow")
     copy_cmd = [
         "kubectl",
         "-n",
@@ -378,10 +428,18 @@ def copy_homedir_into_another(
         "--no-clobber",
         # copy directories recursively
         "--recursive",
-        f"/home/{source_user}",
-        f"/home/{destination_user}/{source_user}-homedir",
+        f"/home/{source_dir}",
+        f"/home/{dest_dir}/{source_dir}-homedir",
     ]
+    subprocess.check_call(copy_cmd)
+    print_colour("Done!")
+
 
+def delete_pod(pod_name, hub_name):
+    """
+    Delete the <pod_name> from <hub_name> namespace
+    """
+    print_colour("Deleting the pod...", "red")
     # Command to delete the pod we created
     delete_pod_cmd = [
         "kubectl",
@@ -391,39 +449,72 @@ def copy_homedir_into_another(
         "pod",
         pod_name,
     ]
+    subprocess.check_call(delete_pod_cmd)
+
+
+@app.command()
+def copy_homedir_into_another(
+    cluster_name: str = typer.Argument(..., help="Name of cluster to operate on"),
+    hub_name: str = typer.Argument(..., help="Name of hub to operate on"),
+    source_homedir: str = typer.Option(
+        None,
+        help="Name of source homedir that we are transferring. "
+        "This is usually the escaped username, eg. user-402i2c-2eorg."
+        "If none is provided, the command will just list all of the homedirs available on the hub.",
+    ),
+    destination_homedir: str = typer.Option(
+        None,
+        help="Name of destination homedir where we will be copying over the files. "
+        "This is usually the escaped username, eg. user-402i2c-2eorg. "
+        "If none is provided, the command will just list all of the homedirs available on the hub.",
+    ),
+):
+    """
+    Copy all the files from a user's home dir to a subdir in destination user's home dir
+    """
+    # Name the pod so we know what to delete when the transfer is done
+    pod_name = f"{cluster_name}-{hub_name}-transfer-shell"
 
     config_file_path = find_absolute_path_to_cluster_file(cluster_name)
     with open(config_file_path) as f:
         cluster = Cluster(yaml.load(f), config_file_path.parent)
     with cluster.auth():
         try:
-            print_colour(
-                f"Creating a pod with the home directories of {cluster_name} - {hub_name} hub mounted on /home...",
-                "yellow",
-            )
-            subprocess.check_call(create_pod_cmd)
-            subprocess.check_call(wait_for_pod_cmd)
+            # Create the pod that mounts the /home directory and wait for it to be ready
+            create_ready_home_pod_jupyter_user(pod_name, cluster_name, hub_name)
+
+            # Check if a source and destination home directory names have been provided as cmd flags
+            # If not, then ls the contents of /home and ask for them again
+            if not source_homedir or not destination_homedir:
+                print_colour(
+                    "The names of the source and destination home directories were not provided. ",
+                    "red",
+                )
+                ls_home_dir(hub_name, pod_name)
+                source_homedir, destination_homedir = ask_for_dirname_again()
+                if not source_homedir or not destination_homedir:
+                    raise typer.Abort()
 
             print_colour(
-                f"Attention! You are about to start transferring the home dir of {source_user} into the home dir of {destination_user}/{source_user}-homedir",
+                f"Attention! You are about to start copying the home dir of {source_homedir} into the home dir of {destination_homedir}/{source_homedir}-homedir",
                 "yellow",
             )
             confirm = typer.confirm("Are you sure you want to continue?")
             if not confirm:
                 raise typer.Abort()
 
-            print_colour("The content of the source and destination home dirs is:")
-            subprocess.check_call(ls_dirs_cmd)
-
-            print_colour("Starting the transfer...", "yellow")
-            subprocess.check_call(copy_cmd)
-            print_colour(
-                "Done! The content of the source and destination home dirs now is:"
+            # First list the contents of the source and dest dirs
+            ls_source_and_dest_dirs(
+                source_homedir, destination_homedir, hub_name, pod_name
+            )
+            # Copy the actual data from /home/<source_homedir> to /home/<destination_homedir>/<source-homedir>-homedir
+            copy_into_subdir(source_homedir, destination_homedir, hub_name, pod_name)
+            # List the dirs again to double check everything happened as it should
+            ls_source_and_dest_dirs(
+                source_homedir, destination_homedir, hub_name, pod_name
             )
-            subprocess.check_call(ls_dirs_cmd)
         finally:
-            print_colour("Deleting the pod...", "red")
-            subprocess.check_call(delete_pod_cmd)
+            delete_pod(pod_name, hub_name)
 
 
 if __name__ == "__main__":

From cbf628375fdd062fbbf773e62dd3076cb5d1a6de Mon Sep 17 00:00:00 2001
From: Georgiana Dolocan <georgiana.dolocan@gmail.com>
Date: Tue, 3 Oct 2023 17:26:40 +0300
Subject: [PATCH 44/44] Update the cmd comment

---
 deployer/debug.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/deployer/debug.py b/deployer/debug.py
index 3eafd93882..f8ed161b48 100644
--- a/deployer/debug.py
+++ b/deployer/debug.py
@@ -470,7 +470,10 @@ def copy_homedir_into_another(
     ),
 ):
     """
-    Copy all the files from a user's home dir to a subdir in destination user's home dir
+    Start a pod with the home directories of the given hub mounted on /home and run a container as the jupyter user (uuid 1000) in this pod.
+    Use this container to run commands to copy the source home directory <source_dir>, into the home directory of the destination directory <dest_dir>,
+    under a directory located at `/home/<dest_dir>/<source_dir>-homedir`.
+    Delete the pod when the copying is done or if any exception is raised.
     """
     # Name the pod so we know what to delete when the transfer is done
     pod_name = f"{cluster_name}-{hub_name}-transfer-shell"