From 804ffb11c1c99dbec0ed69411c71f9a828669263 Mon Sep 17 00:00:00 2001 From: Guy Elsmore-Paddock Date: Fri, 11 Nov 2022 00:46:02 -0500 Subject: [PATCH 1/4] [ITSA-1478] Prefer Burstable Nodes for Burstable Workloads These pods already had tolerations to be scheduled on burstable nodes, but Kubernetes would only do that as a last resort. This conveys to the scheduler that these pods actually prefer that. Unfortunately, there is not a way to use a taint as a matching criterion, so we had to duplicate the taint as a label on the node pool. This was added with a command like the following: ``` az aks nodepool update \ --resource-group RESOURCE-GROUP \ --cluster-name CLUSTER-NAME \ --name NODE-POOL-NAME \ --labels inveniem.com/workload-type=burstable ``` --- .../manifests/app-nextcloud.apache.yaml | 15 +++++++++++++++ .../manifests/app-nextcloud.nginx-fpm.yaml | 15 +++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/components/http-apache/manifests/app-nextcloud.apache.yaml b/components/http-apache/manifests/app-nextcloud.apache.yaml index ad605cf..79e738c 100644 --- a/components/http-apache/manifests/app-nextcloud.apache.yaml +++ b/components/http-apache/manifests/app-nextcloud.apache.yaml @@ -22,6 +22,21 @@ spec: role: backend spec: affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + # Prefer to be scheduled on a burstable node. + # + # NOTE: "inveniem.com/workload-type" is a *label* that has the same + # name and value as the *taint* on the same nodes. Kubernetes + # doesn't have a way to use the taint for affinity, so we duplicate + # it as a label. + - weight: 100 + preference: + matchExpressions: + - key: inveniem.com/workload-type + operator: In + values: + - burstable podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: # Prevent multiple replicas from being on the same node. diff --git a/components/http-nginx-fpm/manifests/app-nextcloud.nginx-fpm.yaml b/components/http-nginx-fpm/manifests/app-nextcloud.nginx-fpm.yaml index fcbefe6..f740ac2 100644 --- a/components/http-nginx-fpm/manifests/app-nextcloud.nginx-fpm.yaml +++ b/components/http-nginx-fpm/manifests/app-nextcloud.nginx-fpm.yaml @@ -23,6 +23,21 @@ spec: role: backend spec: affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + # Prefer to be scheduled on a burstable node. + # + # NOTE: "inveniem.com/workload-type" is a *label* that has the same + # name and value as the *taint* on the same nodes. Kubernetes + # doesn't have a way to use the taint for affinity, so we duplicate + # it as a label. + - weight: 100 + preference: + matchExpressions: + - key: inveniem.com/workload-type + operator: In + values: + - burstable podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: # Prevent multiple replicas from being on the same node. From 22a23f188f04d37bc707dad0d31a1bf479e94f68 Mon Sep 17 00:00:00 2001 From: Guy Elsmore-Paddock Date: Fri, 11 Nov 2022 00:53:59 -0500 Subject: [PATCH 2/4] [ITSA-1478] Allow ClamAV to Run on Burstable Nodes ClamAV really only has something to do when it's scanning a file that uploaded. Otherwise, it's idle. So, a burtstable host node might be a better fit than dedicated. --- base/manifests/app-clamav.yaml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/base/manifests/app-clamav.yaml b/base/manifests/app-clamav.yaml index f96eb89..75c79bb 100644 --- a/base/manifests/app-clamav.yaml +++ b/base/manifests/app-clamav.yaml @@ -27,6 +27,28 @@ spec: app: backend-clamav role: backend spec: + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + # Prefer to be scheduled on a burstable node. + # + # NOTE: "inveniem.com/workload-type" is a *label* that has the same + # name and value as the *taint* on the same nodes. Kubernetes + # doesn't have a way to use the taint for affinity, so we duplicate + # it as a label. + - weight: 100 + preference: + matchExpressions: + - key: inveniem.com/workload-type + operator: In + values: + - burstable + tolerations: + # Allow scheduling this job on burstable nodes. + - key: inveniem.com/workload-type + operator: Equal + value: burstable + effect: NoSchedule containers: - name: backend-clamav image: "mkodockx/docker-clamav:latest" From 4a9d3002f131526ce7473ab89eed80249f7d371c Mon Sep 17 00:00:00 2001 From: Guy Elsmore-Paddock Date: Fri, 11 Nov 2022 00:56:23 -0500 Subject: [PATCH 3/4] [ITSA-1478] Run Regular CronJob on Burstable Nodes This CronJob does light maintenance tasks for Nextcloud every few mins, so we can schedule it on lower-cost burstable nodes. The other CronJobs, which include the Nextcloud file scan and the temporary file cleanup, tend to have continuous file I/O, so they're not likely a good fit for a burstable host node because they'll exhaust all the I/O credits. --- base/manifests/cronjob-nextcloud-cron.yaml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/base/manifests/cronjob-nextcloud-cron.yaml b/base/manifests/cronjob-nextcloud-cron.yaml index ea8e2e0..af51b5d 100644 --- a/base/manifests/cronjob-nextcloud-cron.yaml +++ b/base/manifests/cronjob-nextcloud-cron.yaml @@ -17,6 +17,28 @@ spec: template: spec: restartPolicy: Never + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + # Prefer to be scheduled on a burstable node. + # + # NOTE: "inveniem.com/workload-type" is a *label* that has the same + # name and value as the *taint* on the same nodes. Kubernetes + # doesn't have a way to use the taint for affinity, so we duplicate + # it as a label. + - weight: 100 + preference: + matchExpressions: + - key: inveniem.com/workload-type + operator: In + values: + - burstable + tolerations: + # Allow scheduling this job on burstable nodes. + - key: inveniem.com/workload-type + operator: Equal + value: burstable + effect: NoSchedule containers: - name: cron-nextcloud image: "inveniem/nextcloud-cron:latest" From f3def42f798845da2eb110ed7dd2c073f07bdfaf Mon Sep 17 00:00:00 2001 From: Guy Elsmore-Paddock Date: Fri, 11 Nov 2022 01:12:34 -0500 Subject: [PATCH 4/4] [10.2.0] Bump Version to 10.2.0 in Sample --- overlays/00-sample/kustomization.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/overlays/00-sample/kustomization.yaml b/overlays/00-sample/kustomization.yaml index c6206df..4938e4d 100644 --- a/overlays/00-sample/kustomization.yaml +++ b/overlays/00-sample/kustomization.yaml @@ -96,20 +96,20 @@ images: - name: inveniem/nextcloud-cron newName: your-acr-instance.azurecr.io/inveniem/nextcloud-cron - newTag: 10.1.1 + newTag: 10.2.0 - name: inveniem/nextcloud-apache newName: your-acr-instance.azurecr.io/inveniem/nextcloud-apache - newTag: 10.1.1 + newTag: 10.2.0 - name: inveniem/nextcloud-fpm newName: your-acr-instance.azurecr.io/inveniem/nextcloud-fpm - newTag: 10.1.1 + newTag: 10.2.0 - name: inveniem/nextcloud-nginx-middleware newName: your-acr-instance.azurecr.io/inveniem/nextcloud-nginx-middleware - newTag: 10.1.1 + newTag: 10.2.0 - name: inveniem/sftp-ws-server newName: your-acr-instance.azurecr.io/inveniem/sftp-ws-server - newTag: 10.1.1 + newTag: 10.2.0