From 58cc2781da58ea6cd9347448be8f00287c16194e Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 1 May 2018 12:54:57 +0200 Subject: [PATCH 1/8] Enables volume expansion for GKE --- configure/gke-storageclass-broker-pd.yml | 1 + configure/gke-storageclass-zookeeper-ssd.yml | 1 + kafka/50kafka.yml | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/configure/gke-storageclass-broker-pd.yml b/configure/gke-storageclass-broker-pd.yml index dbb7203f..25c7ed5b 100644 --- a/configure/gke-storageclass-broker-pd.yml +++ b/configure/gke-storageclass-broker-pd.yml @@ -4,5 +4,6 @@ metadata: name: kafka-broker provisioner: kubernetes.io/gce-pd reclaimPolicy: Retain +allowVolumeExpansion: true parameters: type: pd-standard diff --git a/configure/gke-storageclass-zookeeper-ssd.yml b/configure/gke-storageclass-zookeeper-ssd.yml index 5d6673a1..2223d7b2 100644 --- a/configure/gke-storageclass-zookeeper-ssd.yml +++ b/configure/gke-storageclass-zookeeper-ssd.yml @@ -4,5 +4,6 @@ metadata: name: kafka-zookeeper provisioner: kubernetes.io/gce-pd reclaimPolicy: Retain +allowVolumeExpansion: true parameters: type: pd-ssd diff --git a/kafka/50kafka.yml b/kafka/50kafka.yml index 659c9d4d..26cf2422 100644 --- a/kafka/50kafka.yml +++ b/kafka/50kafka.yml @@ -85,4 +85,4 @@ spec: storageClassName: kafka-broker resources: requests: - storage: 200Gi + storage: 10Gi From a6fc309dda4e3f3a5954e180742f29be6c9deb68 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 1 May 2018 12:56:08 +0200 Subject: [PATCH 2/8] Volume expansion should be supported for AWS too --- configure/aws-storageclass-broker-gp2.yml | 1 + configure/aws-storageclass-zookeeper-gp2.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/configure/aws-storageclass-broker-gp2.yml b/configure/aws-storageclass-broker-gp2.yml index 94996b54..16289b51 100644 --- a/configure/aws-storageclass-broker-gp2.yml +++ b/configure/aws-storageclass-broker-gp2.yml @@ -6,5 +6,6 @@ metadata: k8s-addon: storage-aws.addons.k8s.io provisioner: kubernetes.io/aws-ebs reclaimPolicy: Retain +allowVolumeExpansion: true parameters: type: gp2 diff --git a/configure/aws-storageclass-zookeeper-gp2.yml b/configure/aws-storageclass-zookeeper-gp2.yml index 806e455e..1e651df1 100644 --- a/configure/aws-storageclass-zookeeper-gp2.yml +++ b/configure/aws-storageclass-zookeeper-gp2.yml @@ -6,5 +6,6 @@ metadata: k8s-addon: storage-aws.addons.k8s.io provisioner: kubernetes.io/aws-ebs reclaimPolicy: Retain +allowVolumeExpansion: true parameters: type: gp2 From 3e9ff803bca6c80dbf43ab679431f42543c01fcd Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sat, 21 Jul 2018 16:24:36 +0200 Subject: [PATCH 3/8] =?UTF-8?q?Uses=20GKE=20"Regional\=E2=80=9D=20persiste?= =?UTF-8?q?nt=20volumes=20-=20fixed=20zones?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../gke-storageclass-zookeeper-ssd-regional1.yml | 11 +++++++++++ zookeeper/51zoo.yml | 15 +++++++++++---- 2 files changed, 22 insertions(+), 4 deletions(-) create mode 100644 configure/gke-storageclass-zookeeper-ssd-regional1.yml diff --git a/configure/gke-storageclass-zookeeper-ssd-regional1.yml b/configure/gke-storageclass-zookeeper-ssd-regional1.yml new file mode 100644 index 00000000..bc371def --- /dev/null +++ b/configure/gke-storageclass-zookeeper-ssd-regional1.yml @@ -0,0 +1,11 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: kafka-zookeeper +provisioner: kubernetes.io/gce-pd +reclaimPolicy: Retain +allowVolumeExpansion: true +parameters: + type: pd-ssd + replication-type: regional-pd + zones: europe-west1-b, europe-west1-c diff --git a/zookeeper/51zoo.yml b/zookeeper/51zoo.yml index 88b781af..80af89ee 100644 --- a/zookeeper/51zoo.yml +++ b/zookeeper/51zoo.yml @@ -7,7 +7,7 @@ spec: selector: matchLabels: app: zookeeper - storage: ephemeral + storage: persistent-regional serviceName: "zoo" replicas: 2 updateStrategy: @@ -16,7 +16,7 @@ spec: metadata: labels: app: zookeeper - storage: ephemeral + storage: persistent-regional annotations: spec: terminationGracePeriodSeconds: 10 @@ -71,5 +71,12 @@ spec: name: zookeeper-config - name: config emptyDir: {} - - name: data - emptyDir: {} + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: [ "ReadWriteOnce" ] + storageClassName: kafka-zookeeper-regional1 + resources: + requests: + storage: 1Gi From 971d9315e7ad365debe40f9c24986c667b0eb4b5 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Mon, 23 Jul 2018 06:04:38 +0200 Subject: [PATCH 4/8] We'd have to use storage class naming to inidicate the choice of zones, and everyone would have to customize it, and if all zk pods used this we'd be spanning only two zones, not all. --- ...regional1.yml => gke-regional-euwest1cd-zookeeper-ssd.yml} | 4 ++-- zookeeper/51zoo.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) rename configure/{gke-storageclass-zookeeper-ssd-regional1.yml => gke-regional-euwest1cd-zookeeper-ssd.yml} (70%) diff --git a/configure/gke-storageclass-zookeeper-ssd-regional1.yml b/configure/gke-regional-euwest1cd-zookeeper-ssd.yml similarity index 70% rename from configure/gke-storageclass-zookeeper-ssd-regional1.yml rename to configure/gke-regional-euwest1cd-zookeeper-ssd.yml index bc371def..f15fc4cb 100644 --- a/configure/gke-storageclass-zookeeper-ssd-regional1.yml +++ b/configure/gke-regional-euwest1cd-zookeeper-ssd.yml @@ -1,11 +1,11 @@ kind: StorageClass apiVersion: storage.k8s.io/v1 metadata: - name: kafka-zookeeper + name: kafka-zookeeper-regional-euwest1cd provisioner: kubernetes.io/gce-pd reclaimPolicy: Retain allowVolumeExpansion: true parameters: type: pd-ssd replication-type: regional-pd - zones: europe-west1-b, europe-west1-c + zones: europe-west1-c, europe-west1-d diff --git a/zookeeper/51zoo.yml b/zookeeper/51zoo.yml index 80af89ee..b9d4f6b8 100644 --- a/zookeeper/51zoo.yml +++ b/zookeeper/51zoo.yml @@ -76,7 +76,7 @@ spec: name: data spec: accessModes: [ "ReadWriteOnce" ] - storageClassName: kafka-zookeeper-regional1 + storageClassName: kafka-zookeeper-regional-euwest1cd resources: requests: storage: 1Gi From cfbbf94d9b01a9c530c7828051842ad3722f4f05 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Wed, 28 Nov 2018 07:58:12 +0100 Subject: [PATCH 5/8] Clusters in different regions should use the same storage class name, but apply different definitions. --- configure/gke-regional-euwest1cd-zookeeper-ssd.yml | 2 +- zookeeper/51zoo.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/configure/gke-regional-euwest1cd-zookeeper-ssd.yml b/configure/gke-regional-euwest1cd-zookeeper-ssd.yml index f15fc4cb..22d9602f 100644 --- a/configure/gke-regional-euwest1cd-zookeeper-ssd.yml +++ b/configure/gke-regional-euwest1cd-zookeeper-ssd.yml @@ -1,7 +1,7 @@ kind: StorageClass apiVersion: storage.k8s.io/v1 metadata: - name: kafka-zookeeper-regional-euwest1cd + name: kafka-zookeeper-regional provisioner: kubernetes.io/gce-pd reclaimPolicy: Retain allowVolumeExpansion: true diff --git a/zookeeper/51zoo.yml b/zookeeper/51zoo.yml index b9d4f6b8..31eb1c8b 100644 --- a/zookeeper/51zoo.yml +++ b/zookeeper/51zoo.yml @@ -76,7 +76,7 @@ spec: name: data spec: accessModes: [ "ReadWriteOnce" ] - storageClassName: kafka-zookeeper-regional-euwest1cd + storageClassName: kafka-zookeeper-regional resources: requests: storage: 1Gi From c217c6507c9b568ed2b60178bcf8cd54e78e15c0 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Wed, 28 Nov 2018 07:58:50 +0100 Subject: [PATCH 6/8] Fixes line endings --- configure/aks-storageclass-broker-managed.yml | 2 +- configure/aks-storageclass-zookeeper-managed.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/configure/aks-storageclass-broker-managed.yml b/configure/aks-storageclass-broker-managed.yml index 1c9b3c65..50ae160b 100644 --- a/configure/aks-storageclass-broker-managed.yml +++ b/configure/aks-storageclass-broker-managed.yml @@ -6,4 +6,4 @@ provisioner: kubernetes.io/azure-disk reclaimPolicy: Retain parameters: kind: "Managed" - storageaccounttype: Premium_LRS \ No newline at end of file + storageaccounttype: Premium_LRS diff --git a/configure/aks-storageclass-zookeeper-managed.yml b/configure/aks-storageclass-zookeeper-managed.yml index 6963dcca..2c4e0ff1 100644 --- a/configure/aks-storageclass-zookeeper-managed.yml +++ b/configure/aks-storageclass-zookeeper-managed.yml @@ -6,4 +6,4 @@ provisioner: kubernetes.io/azure-disk reclaimPolicy: Retain parameters: kind: "Managed" - storageaccounttype: Premium_LRS \ No newline at end of file + storageaccounttype: Premium_LRS From 5df7477f21dab07cf9ab28975ff6adca4ceb5ba8 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Wed, 28 Nov 2018 10:24:01 +0100 Subject: [PATCH 7/8] Adds the second storageclass for Minikube --- configure/minikube-storageclass-zookeeper.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/configure/minikube-storageclass-zookeeper.yml b/configure/minikube-storageclass-zookeeper.yml index 48c0f35b..73aa4cab 100644 --- a/configure/minikube-storageclass-zookeeper.yml +++ b/configure/minikube-storageclass-zookeeper.yml @@ -1,6 +1,14 @@ +--- kind: StorageClass apiVersion: storage.k8s.io/v1 metadata: name: kafka-zookeeper provisioner: k8s.io/minikube-hostpath reclaimPolicy: Retain +--- +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: kafka-zookeeper-regional +provisioner: k8s.io/minikube-hostpath +reclaimPolicy: Retain From 37d64ec1392780dd752161fb2b6c0162ea50a5ad Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Wed, 28 Nov 2018 19:56:07 +0100 Subject: [PATCH 8/8] Joins minikube storage classes into one yaml, as variations are unlikely --- configure/minikube-storageclass-broker.yml | 6 ------ ...rageclass-zookeeper.yml => minikube-storageclasses.yml} | 7 +++++++ 2 files changed, 7 insertions(+), 6 deletions(-) delete mode 100644 configure/minikube-storageclass-broker.yml rename configure/{minikube-storageclass-zookeeper.yml => minikube-storageclasses.yml} (67%) diff --git a/configure/minikube-storageclass-broker.yml b/configure/minikube-storageclass-broker.yml deleted file mode 100644 index ae930b4d..00000000 --- a/configure/minikube-storageclass-broker.yml +++ /dev/null @@ -1,6 +0,0 @@ -kind: StorageClass -apiVersion: storage.k8s.io/v1 -metadata: - name: kafka-broker -provisioner: k8s.io/minikube-hostpath -reclaimPolicy: Retain diff --git a/configure/minikube-storageclass-zookeeper.yml b/configure/minikube-storageclasses.yml similarity index 67% rename from configure/minikube-storageclass-zookeeper.yml rename to configure/minikube-storageclasses.yml index 73aa4cab..045699f1 100644 --- a/configure/minikube-storageclass-zookeeper.yml +++ b/configure/minikube-storageclasses.yml @@ -1,6 +1,13 @@ --- kind: StorageClass apiVersion: storage.k8s.io/v1 +metadata: + name: kafka-broker +provisioner: k8s.io/minikube-hostpath +reclaimPolicy: Retain +--- +kind: StorageClass +apiVersion: storage.k8s.io/v1 metadata: name: kafka-zookeeper provisioner: k8s.io/minikube-hostpath