Merge branch 'master' into pr/upgrade-oauth16

2i2c-org · Oct 4, 2023 · 79414b3 · 79414b3
2 parents ff04404 + 97a3ec9
commit 79414b3
Show file tree

Hide file tree

Showing 36 changed files with 382 additions and 100 deletions.
diff --git a/.github/actions/setup-deploy/action.yaml b/.github/actions/setup-deploy/action.yaml
@@ -109,7 +109,7 @@ runs:
 
     # This action use the github official cache mechanism internally
     - name: Install sops
-      uses: mdgreenwald/mozilla-sops-action@v1.4.1
+      uses: mdgreenwald/mozilla-sops-action@v1.5.0
 
     # Install pre-requisite for "gcloud container clusters get-credentials"
     # command with a modern k8s client.

diff --git a/.github/workflows/auto-author-assign-pull-request.yaml b/.github/workflows/auto-author-assign-pull-request.yaml
@@ -16,4 +16,4 @@ jobs:
     steps:
       - name: Automatically assign PR author
         id: assignation
-        uses: toshimaru/auto-author-assign@v1.6.2
+        uses: toshimaru/auto-author-assign@v2.0.1
diff --git a/.github/workflows/bump-helm-versions.yaml b/.github/workflows/bump-helm-versions.yaml
@@ -44,7 +44,7 @@ jobs:
       # Action: https://github.com/marketplace/actions/github-app-token
       - name: Fetch a token from GitHub App
         id: generate_token
-        uses: tibdex/github-app-token@v1.8
+        uses: tibdex/github-app-token@v2
         with:
           app_id: ${{ secrets.APP_ID }}
           private_key: ${{ secrets.PRIVATE_KEY }}

diff --git a/.github/workflows/bump-image-tags.yaml b/.github/workflows/bump-image-tags.yaml
@@ -47,7 +47,7 @@ jobs:
       # Action: https://github.com/marketplace/actions/github-app-token
       - name: Fetch a token from GitHub App
         id: generate_token
-        uses: tibdex/github-app-token@v1.8
+        uses: tibdex/github-app-token@v2
         with:
           app_id: ${{ secrets.APP_ID }}
           private_key: ${{ secrets.PRIVATE_KEY }}

diff --git a/.github/workflows/comment-deployment-plan-pr.yaml b/.github/workflows/comment-deployment-plan-pr.yaml
@@ -32,7 +32,7 @@ jobs:
       issues: read
       pull-requests: write
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - run: pip install requests
       - run: python extra-scripts/comment-deployment-plan-pr.py
         env:

diff --git a/.github/workflows/comment-pending-deployment-info.yaml b/.github/workflows/comment-pending-deployment-info.yaml
@@ -11,7 +11,7 @@ jobs:
     permissions:
       issues: write
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Parse new hub request issue template form
         uses: stefanbuck/github-issue-praser@v3
         id: issue-parser

diff --git a/.github/workflows/comment-test-link-merged-pr.yaml b/.github/workflows/comment-test-link-merged-pr.yaml
@@ -20,7 +20,7 @@ jobs:
       actions: read
       pull-requests: write
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - run: pip install requests
       - run: python extra-scripts/comment-test-link-merged-pr.py
         env:

diff --git a/.github/workflows/deploy-grafana-dashboards.yaml b/.github/workflows/deploy-grafana-dashboards.yaml
@@ -39,7 +39,7 @@ jobs:
 
     steps:
       - name: Checkout repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - uses: actions/setup-python@v4
         with:
@@ -51,7 +51,7 @@ jobs:
           sudo apt install jsonnet
 
       - name: Install sops
-        uses: mdgreenwald/mozilla-sops-action@v1.4.1
+        uses: mdgreenwald/mozilla-sops-action@v1.5.0
 
       - name: Setup sops credentials to decrypt repo secrets
         uses: google-github-actions/auth@v1

diff --git a/.github/workflows/deploy-hubs.yaml b/.github/workflows/deploy-hubs.yaml
@@ -64,7 +64,7 @@ jobs:
 
     steps:
       - name: Checkout repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install Python 3.9
         uses: actions/setup-python@v4
@@ -214,7 +214,7 @@ jobs:
 
     steps:
       - name: Checkout repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Setup deploy for ${{ matrix.jobs.cluster_name }}
         uses: ./.github/actions/setup-deploy
@@ -387,7 +387,7 @@ jobs:
 
     steps:
       - name: Checkout repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Setup deploy for ${{ matrix.jobs.cluster_name }} cluster
         uses: ./.github/actions/setup-deploy

diff --git a/.github/workflows/ensure-uptime-checks.yaml b/.github/workflows/ensure-uptime-checks.yaml
@@ -30,15 +30,15 @@ jobs:
 
     steps:
       - name: Checkout repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       # Uptime checks are set up and managed via terraform
       - uses: hashicorp/setup-terraform@v2
 
       # We use sops to store encrypted GCP ServiceAccount Key that terraform uses
       # to run, as well as PagerDuty config terraform uses
       - name: Install sops
-        uses: mdgreenwald/mozilla-sops-action@v1.4.1
+        uses: mdgreenwald/mozilla-sops-action@v1.5.0
 
       # Authenticate with the correct KMS key that sops will use.
       - name: Setup sops credentials to decrypt repo secrets

diff --git a/.github/workflows/terraform-lint.yaml b/.github/workflows/terraform-lint.yaml
@@ -9,9 +9,9 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: hashicorp/setup-terraform@v2
-      - uses: terraform-linters/setup-tflint@v3
+      - uses: terraform-linters/setup-tflint@v4
       - name: Run terraform linter
         run: |
           cd terraform

diff --git a/.github/workflows/test-deployer-code.yaml b/.github/workflows/test-deployer-code.yaml
@@ -25,7 +25,7 @@ jobs:
   test-deployer:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
         with:
           python-version: "3.9"

diff --git a/.github/workflows/validate-clusters.yaml b/.github/workflows/validate-clusters.yaml
@@ -76,7 +76,7 @@ jobs:
       (github.event_name == 'pull_request' && contains(github.head_ref, fromJson('["dependabot", "pre-commit"]')) == false)
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
         with:
           python-version: "3.10"
@@ -201,7 +201,7 @@ jobs:
         jobs: ${{ fromJson(needs.generate-clusters-to-validate.outputs.cluster_matrix) }}
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
         with:
           python-version: "3.10"

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -17,7 +17,7 @@ repos:
 
   # Autoformat: Python code, syntax patterns are modernized
   - repo: https://github.com/asottile/pyupgrade
-    rev: v3.10.1
+    rev: v3.14.0
     hooks:
       - id: pyupgrade
         args:
@@ -31,7 +31,7 @@ repos:
 
   # Autoformat: Python code
   - repo: https://github.com/psf/black
-    rev: "23.7.0"
+    rev: "23.9.1"
     hooks:
       - id: black
 

diff --git a/config/clusters/2i2c/imagebuilding-demo.values.yaml b/config/clusters/2i2c/imagebuilding-demo.values.yaml
@@ -80,14 +80,15 @@ jupyterhub:
             choices:
               mem_2_7:
                 display_name: 2.7 GB RAM, upto 3.479 CPUs
-                description: "Use this for the workshop on 2023 September"
+                description: Use this for the workshop on 2023 September
                 kubespawner_override:
                   mem_guarantee: 2904451072
                   mem_limit: 2904451072
                   cpu_guarantee: 0.434875
                   cpu_limit: 3.479
                   node_selector:
-                    node.kubernetes.io/instance-type: n1-highmem-4
+                    # FIXME: guarantee/limits initialized for n1-highmem-4, not n2-
+                    node.kubernetes.io/instance-type: n2-highmem-4
                 default: true
               mem_5_4:
                 display_name: 5.4 GB RAM, upto 3.479 CPUs
@@ -97,7 +98,8 @@ jupyterhub:
                   cpu_guarantee: 0.86975
                   cpu_limit: 3.479
                   node_selector:
-                    node.kubernetes.io/instance-type: n1-highmem-4
+                    # FIXME: guarantee/limits initialized for n1-highmem-4, not n2-
+                    node.kubernetes.io/instance-type: n2-highmem-4
               mem_10_8:
                 display_name: 10.8 GB RAM, upto 3.479 CPUs
                 kubespawner_override:
@@ -106,17 +108,19 @@ jupyterhub:
                   cpu_guarantee: 1.7395
                   cpu_limit: 3.479
                   node_selector:
-                    node.kubernetes.io/instance-type: n1-highmem-4
+                    # FIXME: guarantee/limits initialized for n1-highmem-4, not n2-
+                    node.kubernetes.io/instance-type: n2-highmem-4
               mem_21_6:
                 display_name: 21.6 GB RAM, upto 3.479 CPUs
-                description: "Largest amount of RAM, might take a few minutes to start"
+                description: Largest amount of RAM, might take a few minutes to start
                 kubespawner_override:
                   mem_guarantee: 23235608576
                   mem_limit: 23235608576
                   cpu_guarantee: 3.479
                   cpu_limit: 3.479
                 node_selector:
-                  node.kubernetes.io/instance-type: n1-highmem-4
+                  # FIXME: guarantee/limits initialized for n1-highmem-4, not n2-
+                  node.kubernetes.io/instance-type: n2-highmem-4
   hub:
     services:
       binder:

diff --git a/config/clusters/2i2c/neurohackademy.values.yaml b/config/clusters/2i2c/neurohackademy.values.yaml
@@ -46,7 +46,7 @@ jupyterhub:
         effect: "NoSchedule"
     cpu:
       guarantee: 0.5
-      # We're on n1-highmem-16 machines
+      # We're on n2-highmem-16 machines
       limit: 14
     memory:
       guarantee: 4G

diff --git a/config/clusters/2i2c/temple.values.yaml b/config/clusters/2i2c/temple.values.yaml
@@ -33,6 +33,16 @@ jupyterhub:
         operator: "Equal"
         value: "temple"
         effect: "NoSchedule"
+    memory:
+      # Memory defaults are 256MB to 1G in basehub. These are bumped based on a
+      # request to bump these (https://2i2c.freshdesk.com/a/tickets/1003) at
+      # least during October 2-15.
+      #
+      # A previous request (https://2i2c.freshdesk.com/a/tickets/643) included
+      # notes on the new memory request and limits to adopt once again.
+      #
+      guarantee: 512M
+      limit: 2G
   hub:
     config:
       JupyterHub:

diff --git a/config/clusters/carbonplan/common.values.yaml b/config/clusters/carbonplan/common.values.yaml
@@ -256,8 +256,3 @@ dask-gateway:
       limits:
         cpu: 2
         memory: 4Gi
-      # TODO: figure out a replacement for userLimits.
-    extraConfig:
-      idle: |
-        # timeout after 30 minutes of inactivity
-        c.KubeClusterConfig.idle_timeout = 1800
diff --git a/config/clusters/jupyter-meets-the-earth/common.values.yaml b/config/clusters/jupyter-meets-the-earth/common.values.yaml
@@ -149,7 +149,7 @@ basehub:
           profile_options: *profile_options
           kubespawner_override:
             cpu_guarantee: 3.5
-            mem_guarantee: 14G
+            mem_guarantee: 13G
             mem_limit: null
             node_selector:
               node.kubernetes.io/instance-type: m5.xlarge
@@ -182,7 +182,7 @@ basehub:
           profile_options: *profile_options
           kubespawner_override:
             cpu_guarantee: 3.5
-            mem_guarantee: 14G
+            mem_guarantee: 13G
             mem_limit: null
             environment:
               NVIDIA_DRIVER_CAPABILITIES: compute,utility
@@ -354,6 +354,3 @@ dask-gateway:
                 handler=option_handler,
             )
         c.Backend.cluster_options = cluster_options
-      idle: |
-        # timeout after 30 minutes of inactivity
-        c.KubeClusterConfig.idle_timeout = 1800
diff --git a/config/clusters/meom-ige/common.values.yaml b/config/clusters/meom-ige/common.values.yaml
@@ -158,10 +158,3 @@ basehub:
             - roxyboy
             - lesommer
             - auraoupa
-
-dask-gateway:
-  gateway:
-    extraConfig:
-      idle: |
-        # timeout after 30 minutes of inactivity
-        c.KubeClusterConfig.idle_timeout = 1800
diff --git a/config/clusters/openscapes/common.values.yaml b/config/clusters/openscapes/common.values.yaml
@@ -43,9 +43,6 @@ basehub:
     scheduling:
       userScheduler:
         enabled: true
-      userPods:
-        nodeAffinity:
-          matchNodePurpose: require
     hub:
       allowNamedServers: true
       readinessProbe:
@@ -92,9 +89,3 @@ basehub:
             - jules32
             - erinmr
             - betolink
-dask-gateway:
-  gateway:
-    extraConfig:
-      idle: |-
-        # timeout after 30 minutes of inactivity
-        c.KubeClusterConfig.idle_timeout = 1800
diff --git a/config/clusters/openscapes/prod.values.yaml b/config/clusters/openscapes/prod.values.yaml
@@ -1,11 +1,16 @@
 basehub:
+  userServiceAccount:
+    annotations:
+      eks.amazonaws.com/role-arn: arn:aws:iam::783616723547:role/openscapeshub-prod
   jupyterhub:
     ingress:
       hosts: [openscapes.2i2c.cloud]
       tls:
         - hosts: [openscapes.2i2c.cloud]
           secretName: https-auto-tls
     singleuser:
+      extraEnv:
+        SCRATCH_BUCKET: s3://openscapeshub-scratch/$(JUPYTERHUB_USER)
       profileList:
         - display_name: Python
           description: Python datascience environment

diff --git a/config/clusters/openscapes/staging.values.yaml b/config/clusters/openscapes/staging.values.yaml
@@ -1,11 +1,16 @@
 basehub:
+  userServiceAccount:
+    annotations:
+      eks.amazonaws.com/role-arn: arn:aws:iam::783616723547:role/openscapeshub-staging
   jupyterhub:
     ingress:
       hosts: [staging.openscapes.2i2c.cloud]
       tls:
         - hosts: [staging.openscapes.2i2c.cloud]
           secretName: https-auto-tls
     singleuser:
+      extraEnv:
+        SCRATCH_BUCKET: s3://openscapeshub-scratch-staging/$(JUPYTERHUB_USER)
       profileList:
         - display_name: Python
           description: Python datascience environment

diff --git a/config/clusters/ubc-eoas/common.values.yaml b/config/clusters/ubc-eoas/common.values.yaml
@@ -77,6 +77,12 @@ jupyterhub:
           # Using 'latest' for now so updates do not require 2i2c
           # involvement.
           image: quay.io/henrykmodzelewski/2i2c-eosc211:latest
+      - display_name: EOSC212
+        description: "For class EOSC212, provides upto 1 CPU and ~4G RAM"
+        kubespawner_override:
+          # Using 'latest' for now so updates do not require 2i2c
+          # involvement.
+          image: quay.io/henrykmodzelewski/2i2c-eosc212:latest
       - display_name: EOSC350
         description: "For class EOSC350, provides upto 1 CPU and ~4G RAM"
         kubespawner_override: