From c858818f2be21f6f8ed20248033597bd9d61776d Mon Sep 17 00:00:00 2001 From: Daniel Lavoie Date: Tue, 8 Oct 2019 09:00:06 -0500 Subject: [PATCH] Added minio s3 cli blob --- README.md | 8 +- config/blobs.yml | 4 + doc/gcp-instructions.md | 72 ++++++---- doc/state-of-security.md | 129 ++++++------------ jobs/confluent-connect/spec | 43 +++++- jobs/confluent-connect/templates/bin/ctl | 8 ++ .../templates/bin/download-connectors.sh | 22 +++ .../templates/bin/pre-start.erb | 23 +++- .../templates/config/connect-jaas.conf | 4 + .../templates/config/connect-login.conf.erb | 1 + .../templates/config/connect.properties.erb | 19 +-- .../templates/config/log4j.properties | 20 +++ jobs/confluent-control-center/spec | 17 ++- .../templates/bin/ctl.erb | 2 +- .../templates/config/control-center-jaas.conf | 4 + .../config/control-center-login.conf.erb | 2 + .../config/control-center.properties.erb | 78 +++++------ jobs/confluent-ksql/spec | 29 +++- jobs/confluent-ksql/templates/bin/ctl | 2 + .../templates/config/ksql-server-jaas.conf | 4 + .../config/ksql-server-login.conf.erb | 9 ++ .../config/ksql-server.properties.erb | 21 +-- jobs/confluent-schema-registry/spec | 20 ++- .../templates/bin/ctl | 2 + .../config/schema-registry-jaas.conf | 4 + .../config/schema-registry-login.conf.erb | 8 ++ .../config/schema-registry.properties.erb | 18 ++- .../templates/config/server.properties.erb | 7 +- manifests/confluent-platform-solo.yml | 90 ------------ manifests/confluent-platform.yml | 127 ++++++++++++++--- packages/minio-mc/packaging | 5 + packages/minio-mc/spec | 5 + 32 files changed, 492 insertions(+), 315 deletions(-) create mode 100755 jobs/confluent-connect/templates/bin/download-connectors.sh create mode 100644 jobs/confluent-connect/templates/config/connect-jaas.conf create mode 100644 jobs/confluent-connect/templates/config/connect-login.conf.erb create mode 100644 jobs/confluent-connect/templates/config/log4j.properties create mode 100644 jobs/confluent-control-center/templates/config/control-center-jaas.conf create mode 100644 jobs/confluent-control-center/templates/config/control-center-login.conf.erb create mode 100644 jobs/confluent-ksql/templates/config/ksql-server-jaas.conf create mode 100644 jobs/confluent-ksql/templates/config/ksql-server-login.conf.erb create mode 100644 jobs/confluent-schema-registry/templates/config/schema-registry-jaas.conf create mode 100644 jobs/confluent-schema-registry/templates/config/schema-registry-login.conf.erb delete mode 100644 manifests/confluent-platform-solo.yml create mode 100644 packages/minio-mc/packaging create mode 100644 packages/minio-mc/spec diff --git a/README.md b/README.md index 1c6b260..ac1600d 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ Long story short, Bosh let you declare a desired state of your software and the ## TL;DR - I just want to deploy * [AWS deployment instructions](doc/aws-instructions.md) -* GCP Deployment instructions - sooooon +* [GCP Deployment instructions](gcp-instructions.md) * vSphere Deployment instructions - sooooon * Virtual Box deployment instructions - sooooon @@ -37,12 +37,6 @@ A lot of security features are to be implemented. For a complete state of the bi This current iteration was successully tested on AWS and GCP cpis. -## Deploy single collocated VM - -```plain -bosh deploy confluent-platform-bosh-release/manifests/confluent-platform-solo.yml -o confluent-platform-bosh-release/manifests/operators/create.yml -``` - ## Deploy Confluent Platform Cluster ``` diff --git a/config/blobs.yml b/config/blobs.yml index 76bc5db..16aa842 100644 --- a/config/blobs.yml +++ b/config/blobs.yml @@ -5,3 +5,7 @@ java/jdk8u192-b03.tar.gz: size: 45670457 object_id: 5a70262f-3127-4a35-6685-b271ba939661 sha: sha256:5d8203117cad2ed7ef1e20d951f3c1b1515f725484e35cc10c61307e66018efe +minio/mc: + size: 16605184 + object_id: 0fb6f283-7aea-4c8a-5157-c6d3a509680f + sha: sha256:67280ce05acdd656156ca39b266f2931889ed2b58b703300639b1ccba645a6b3 \ No newline at end of file diff --git a/doc/gcp-instructions.md b/doc/gcp-instructions.md index 065bc1f..a004088 100644 --- a/doc/gcp-instructions.md +++ b/doc/gcp-instructions.md @@ -10,17 +10,10 @@ Example CIDR : 10.0.10.0/16 ### Create subnets -#### Infrastructure - -* Example subnet 1 name : infrastructure -* Example subnet 1 CIDR : 10.0.10.0/24 -* Example subnet 1 region : northamerica-northeast1 - -#### Confluent Platform - -* Example subnet 2 name : confluent-platform -* Example subnet 2 CIDR : 10.0.20.0/24 -* Example subnet 2 region : northamerica-northeast1 +| Name | CIDR | Region | +|---|---|---| +| instrastructure | 10.0.10.0/24 | northamerica-northeast1 | +| confluent-platform | 10.0.20.0/24 | northamerica-northeast1 | ### Create Firewall rules @@ -28,44 +21,63 @@ Example CIDR : 10.0.10.0/16 | ------------- | ------------- | ------------- | ------------- | ------------- | | bosh-allow-ssh | allow-ssh | IP ranges: 0.0.0.0/0 | tcp:22 | cp-bosh | | bosh-unrestricted | confluent-platform | Tags: confluent-platform | all | cp-bosh | -| bosh-allow-control-center | allow-control-center | IP ranges: 0.0.0.0/0 | tcp:9021 | cp-bosh | - -### Create a TCP Load Balancer for Confluent Server - -TODO +| bosh-allow-control-center | allow-control-center | IP ranges: 0.0.0.0/0 | tcp:9021 | cp-bosh | +| bosh-allow-ksql | allow-ksql | IP ranges: 0.0.0.0/0 | tcp:8088 | cp-bosh | ### Create unmanaged instance groups for Control Center -Instance Group 1 Name : cp-control-center -Instance Group 1 Zone : northamerica-northeast1-a -Instance Group 1 Network : cp-bosh -Instance Group 1 Subnet : confluent-platform - - -Instance Group 2 Name : cp-control-center -Instance Group 2 Zone : northamerica-northeast1-b -Instance Group 2 Network : cp-bosh -Instance Group 2 Subnet : confluent-platform +| Number | Zone | Name | Network | Subnet | +|---|---|---|---|---| +| 1 | northamerica-northeast1-a | cp-control-center | cp-bosh | confluent-platform | +| 2 | northamerica-northeast1-b | cp-control-center | cp-bosh | confluent-platform | +| 3 | northamerica-northeast1-c | cp-control-center | cp-bosh | confluent-platform | +### Create unmanaged instance groups for KSQL -Instance Group 3 Name : cp-control-center -Instance Group 3 Zone : northamerica-northeast1-c -Instance Group 3 Network : cp-bosh -Instance Group 3 Subnet : confluent-platform +| Number | Zone | Name | Network | Subnet | +|---|---|---|---|---| +| 1 | northamerica-northeast1-a | cp-ksql | cp-bosh | confluent-platform | +| 2 | northamerica-northeast1-b | cp-ksql | cp-bosh | confluent-platform | +| 3 | northamerica-northeast1-c | cp-ksql | cp-bosh | confluent-platform | ### Create an Http Load Balancer for Control Center #### Backend services Instance Group : cp-control-center + Port number : 9021 + Health check : HTTP on :9021/ + Backend Services : cp-control-center #### Frontend protocol : http + port : 80 + +ip : Reserved ipv4 + +### Create an Http Load Balancer for KSQL + +#### Backend services + +Instance Group : cp-ksql + +Port number : 8088 + +Health check : HTTP on :8088/ + +Backend Services : cp-ksql + +#### Frontend + +protocol : https + +port : 443 + ip : Reserved ipv4 ### Create a jumpbox to run Bosh CLI Commands diff --git a/doc/state-of-security.md b/doc/state-of-security.md index 88630b9..b1d6f53 100644 --- a/doc/state-of-security.md +++ b/doc/state-of-security.md @@ -1,91 +1,44 @@ # State of security implementation -- [ ] Broker - - [X] Brokers to brokers - - [X] Encryption - - [X] Authentication - - [ ] Metric reporter - - [X] Encryption - - [X] Authentication - - [ ] ACL - - [ ] RBAC - -- [ ] Connect - - [ ] Workers to Brokers - - [X] Encryption - - [X] Authentication - - [ ] ACL - - [ ] RBAC - - [ ] Rest API - - [ ] Encryption - - [ ] Authentication - - [ ] RBAC - - [ ] Interceptors - - [ ] Encryption - - [ ] Authentication - - [ ] ACL - - [ ] RBAC - -- [ ] KSQL - - [ ] KSQL nodes to Brokers - - [X] Encryption - - [X] Authentication - - [ ] ACL - - [ ] Rest API - - [ ] Encryption - - [ ] Authentication - - [ ] Schema Registry - - [ ] Encryption - - [ ] Authentication - - [ ] Interceptors - - [ ] Encryption - - [ ] Authentication - - [ ] ACL - -- [ ] Schema Registry - - [ ] Schema registry to Brokers - - [X] Encryption - - [X] Authentication - - [ ] ACL - - [ ] RBAC - - [ ] Rest API - - [ ] Encryption - - [ ] Authentication - - [ ] RBAC - - [ ] Interceptors - - [ ] SSL - - [ ] Authentication - - [ ] ACL - - [ ] RBAC - -- [ ] Control Center - - [ ] Rest API - - [ ] Encryption - - [ ] Authentication - - [ ] ACL - - [ ] RBAC - - [ ] Brokers - - [X] Encryption - - [X] Authentication - - [ ] ACL - - [ ] RBAC - - [ ] Connect - - [ ] Encryption - - [ ] Authentication - - [ ] ACL - - [ ] RBAC - - [ ] KSQL - - [ ] Encryption - - [ ] Authentication - - [ ] ACL - - [ ] RBAC +- [X] Encryption + - [X] Kafka Broker + - [X] mTLS for broker intercommunication + - [X] mTLS between Metric Reporters and Kafka Cluster + - [X] Kafka Connect + - [X] mTLS with Kafka cluster + - [X] Https for Connect REST endpoints + - [X] Schema Registry + - [X] mTLS with Kafka cluster + - [X] Https for REST endpoints + - [X] KSQL + - [X] mTLS with Kafka cluster + - [X] Https for REST endpoints + - [X] Control Center + - [X] mTLS with Kafka cluster + - [X] Https for REST endpoints +- [ ] Authentication + - [X] Kafka Broker + - [X] SASL for broker intercommunication + - [X] SASL between Metric Reporters and Kafka Cluster (to test) + - [X] Kafka Connect + - [X] SASL with Kafka cluster + - [X] REST endpoints - [ ] Schema Registry - - [ ] Encryption - - [ ] Authentication - - [ ] ACL - - [ ] RBAC - - [ ] Zookeeper - - [ ] Authentication - -- [ ] Zookeeper - - [ ] Authentication + - [X] SASL with Kafka cluster + - [ ] REST endpoints + - [X] KSQL + - [X] SASL for with Kafka cluster + - [X] REST endpoints + - [X] Control Center + - [X] SASL with Kafka cluster + - [X] Basic Auth for REST endpoints +- [ ] Kafka Topics ACL + - [ ] Kafka Connect + - [ ] Schema Regisry + - [ ] KSQL + - [ ] Control Center +- [ ] RBAC + - [ ] Kafka Connect + - [ ] Schema Regisry + - [ ] KSQL + - [ ] Control Center \ No newline at end of file diff --git a/jobs/confluent-connect/spec b/jobs/confluent-connect/spec index b435476..cabd451 100644 --- a/jobs/confluent-connect/spec +++ b/jobs/confluent-connect/spec @@ -3,16 +3,21 @@ name: confluent-connect templates: bin/ctl: bin/ctl + bin/download-connectors.sh: bin/download-connectors.sh bin/pre-start.erb: bin/pre-start config/bpm.yml: config/bpm.yml config/ca_certs.pem.erb: config/ca_certs.pem config/cert.pem.erb: config/cert.pem + config/connect-jaas.conf: config/connect-jaas.conf + config/connect-login.conf.erb: config/connect-login.conf config/connect.properties.erb: config/connect.properties config/key.pem.erb: config/key.pem + config/log4j.properties: config/log4j.properties packages: - openjdk-8 - confluent-platform +- minio-mc consumes: - name: confluent-server @@ -29,8 +34,8 @@ provides: properties: listen_port: - description: The port to listen for client connections - default: 8083 + description: "Https port for Confluent Connect REST endpoints" + default: 8443 group_id: description: Unique identifier for the set of workers that form the Kafka Connect cluster default: connect-cluster @@ -66,8 +71,34 @@ properties: description: "Keystore password" default: notasecret - jaas.username: - description: "Username used in JAAS configuration" + kafka.jaas.username: + description: "Username used for Kafka Broker" - jaas.password: - description: "Password used in JAAS configuration" \ No newline at end of file + kafka.jaas.password: + description: "Password used for Kafka Broker" + + basic.jaas.username: + description: "Username used for Basic Auth" + + basic.jaas.password: + description: "Password used for Basic Auth" + + schema_registry.basic.username: + description: Username for Basic Auth on Schema Registry + + schema_registry.basic.password: + description: Password for Basic Auth on Schema Registry + + connectors.s3.endpoint: + description: "S3 endpoint to lookup for connectors" + + connectors.s3.access_key: + description: "S3 Access key to lookup for connectors" + default: "" + + connectors.s3.secret_key: + description: "S3 Secret key to lookup for connectors" + default: "" + + connectors.s3.bucket: + description: "Bucket to lookup for connectors" \ No newline at end of file diff --git a/jobs/confluent-connect/templates/bin/ctl b/jobs/confluent-connect/templates/bin/ctl index 8d291ae..4375bd4 100755 --- a/jobs/confluent-connect/templates/bin/ctl +++ b/jobs/confluent-connect/templates/bin/ctl @@ -4,6 +4,14 @@ set -e source /var/vcap/packages/openjdk-8/bosh/runtime.env +export KAFKA_OPTS="-Djava.security.auth.login.config=/var/vcap/jobs/confluent-connect/config/connect-jaas.conf" + +#export KAFKA_LOG4J_OPTS="-Dlog4j.configuration=file:/var/vcap/jobs/confluent-connect/config/log4j.properties" + +export SCHEMA_REGISTRY_OPTS="-Dssl.truststore.location=/var/vcap/jobs/confluent-connect/config/generated.truststore.jks -Dssl.truststore.password=<%= p("keystore_password") %> -Dssl.keystore.location=/var/vcap/jobs/confluent-connect/config/generated.keystore.jks -Dssl.keystore.password=<%= p("keystore_password") %> -Dssl.key.password=<%= p("keystore_password") %>" + +export LOG_DIR=/var/vcap/sys/log/confluent-connect + case $1 in start) diff --git a/jobs/confluent-connect/templates/bin/download-connectors.sh b/jobs/confluent-connect/templates/bin/download-connectors.sh new file mode 100755 index 0000000..62f2fa5 --- /dev/null +++ b/jobs/confluent-connect/templates/bin/download-connectors.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +function downloadConnectors() { + CONNECTORS_FOLDER=$1 + S3_ENDPOINT=$2 + S3_ACCESS_KEY=$3 + S3_SECRET_KEY=$4 + S3_BUCKET=$5 + + rm -rf $CONNECTORS_FOLDER + + mkdir $CONNECTORS_FOLDER + + /var/vcap/packages/minio-mc/mc config host add connectors $S3_ENDPOINT $S3_ACCESS_KEY $S3_SECRET_KEY + + /var/vcap/packages/minio-mc/mc cp --recursive connectors/$S3_BUCKET/ $CONNECTORS_FOLDER + + for i in $CONNECTORS_FOLDER/*.zip; do + newdir="${i:0:-4}" && mkdir "$newdir" + unzip "$i" -d "$newdir" + done +} \ No newline at end of file diff --git a/jobs/confluent-connect/templates/bin/pre-start.erb b/jobs/confluent-connect/templates/bin/pre-start.erb index 40ed245..acc0108 100644 --- a/jobs/confluent-connect/templates/bin/pre-start.erb +++ b/jobs/confluent-connect/templates/bin/pre-start.erb @@ -4,6 +4,8 @@ set -eux set -o pipefail source /var/vcap/packages/openjdk-8/bosh/runtime.env +source /var/vcap/jobs/confluent-connect/bin/download-connectors.sh + export PATH=$PATH:/var/vcap/packages/confluent-platform/bin:$PATH CONFIG_DIR=/var/vcap/jobs/confluent-connect/config @@ -47,4 +49,23 @@ $KEY_TOOL -importkeystore \ -srcstorepass $KEYSTORE_PASSWORD \ -srckeypass $KEYSTORE_PASSWORD \ -alias localhost -<% end %> \ No newline at end of file +<% end %> + +CONNECTORS_FOLDER=/var/vcap/packages/confluent-platform/share/java/custom-connectors + +if [ ! -d "/var/log" ]; then + mkdir -p /var/log +fi + +if [ ! -d "$CONNECTORS_FOLDER" ]; then + mkdir -p $CONNECTORS_FOLDER +fi + +if [ -z "$(ls -A $CONNECTORS_FOLDER)" ]; then + downloadConnectors \ + $CONNECTORS_FOLDER \ + <%= p("connectors.s3.endpoint") %> \ + <%= p("connectors.s3.access_key") %> \ + <%= p("connectors.s3.secret_key") %> \ + <%= p("connectors.s3.bucket") %> +fi \ No newline at end of file diff --git a/jobs/confluent-connect/templates/config/connect-jaas.conf b/jobs/confluent-connect/templates/config/connect-jaas.conf new file mode 100644 index 0000000..37a11af --- /dev/null +++ b/jobs/confluent-connect/templates/config/connect-jaas.conf @@ -0,0 +1,4 @@ +KafkaConnect { + org.apache.kafka.connect.rest.basic.auth.extension.PropertyFileLoginModule required + file="/var/vcap/jobs/confluent-connect/config/connect-login.conf"; +}; \ No newline at end of file diff --git a/jobs/confluent-connect/templates/config/connect-login.conf.erb b/jobs/confluent-connect/templates/config/connect-login.conf.erb new file mode 100644 index 0000000..6988d74 --- /dev/null +++ b/jobs/confluent-connect/templates/config/connect-login.conf.erb @@ -0,0 +1 @@ +<%= p("basic.jaas.username") %>: <%= p("basic.jaas.password") %> \ No newline at end of file diff --git a/jobs/confluent-connect/templates/config/connect.properties.erb b/jobs/confluent-connect/templates/config/connect.properties.erb index 40b223d..d356252 100644 --- a/jobs/confluent-connect/templates/config/connect.properties.erb +++ b/jobs/confluent-connect/templates/config/connect.properties.erb @@ -11,8 +11,8 @@ security.protocol=SASL_SSL sasl.mechanism=SCRAM-SHA-512 sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required \ -username="<%= p("jaas.username") %>" \ -password="<%= p("jaas.password") %>"; +username="<%= p("kafka.jaas.username") %>" \ +password="<%= p("kafka.jaas.password") %>"; ssl.truststore.location=/var/vcap/jobs/confluent-connect/config/generated.truststore.jks ssl.truststore.password=<%= p("keystore_password") %> @@ -27,11 +27,15 @@ group.id=<%= p("group_id") %> # The converters specify the format of data in Kafka and how to translate it into Connect data. # Every Connect user will need to configure these based on the format they want their data in # when loaded from or stored into Kafka -<% schema_registry_url = "http://" + schemaRegistries.address + ":" + schemaRegistries.p("listen_port").to_s %> +<% schema_registry_url = schemaRegistries.instances.map { |instance| "https://#{instance.address}:#{schemaRegistries.p('listen_port')}" }.join(",") %> key.converter=io.confluent.connect.avro.AvroConverter key.converter.schema.registry.url=<%= schema_registry_url %> +key.converter.schema.registry.basic.auth.credentials.source=USER_INFO +key.converter.schema.registry.basic.auth.user.info=<%= p("schema_registry.basic.username") %>:<%= p("schema_registry.basic.password") %> value.converter=io.confluent.connect.avro.AvroConverter value.converter.schema.registry.url=<%= schema_registry_url %> +value.converter.schema.registry.basic.auth.credentials.source=USER_INFO +value.converter.schema.registry.basic.auth.user.info=<%= p("schema_registry.basic.username") %>:<%= p("schema_registry.basic.password") %> # Internal Storage Topics. # @@ -78,17 +82,16 @@ internal.value.converter.schemas.enable=false # producer.interceptor.classes=io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor # consumer.interceptor.classes=io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor -# These are provided to inform the user about the presence of the REST host and port configs -# Hostname & Port for the REST API to listen on. If this is set, it will bind to the interface used to listen to requests. -#rest.host.name=0.0.0.0 -#rest.port=<%= p("listen_port") %> -listeners=https://<%= spec.address %>:8443 +listeners=https://<%= spec.address %>:<%= p("listen_port") %> listeners.https.ssl.truststore.location=/var/vcap/jobs/confluent-connect/config/generated.truststore.jks listeners.https.ssl.truststore.password=<%= p("keystore_password") %> listeners.https.ssl.keystore.location=/var/vcap/jobs/confluent-connect/config/generated.keystore.jks listeners.https.ssl.keystore.password=<%= p("keystore_password") %> listeners.https.ssl.key.password=<%= p("keystore_password") %> +rest.extension.classes=org.apache.kafka.connect.rest.basic.auth.extension.BasicAuthSecurityRestExtension + + # The Hostname & Port that will be given out to other workers to connect to i.e. URLs that are routable from other servers. #rest.advertised.host.name=0.0.0.0 #rest.advertised.port=8083 diff --git a/jobs/confluent-connect/templates/config/log4j.properties b/jobs/confluent-connect/templates/config/log4j.properties new file mode 100644 index 0000000..3ce7503 --- /dev/null +++ b/jobs/confluent-connect/templates/config/log4j.properties @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +log4j.rootLogger=INFO, stdout + +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%d %p --- [%15.15t] (%c) : %m%n \ No newline at end of file diff --git a/jobs/confluent-control-center/spec b/jobs/confluent-control-center/spec index 9bbf33a..e8f662a 100644 --- a/jobs/confluent-control-center/spec +++ b/jobs/confluent-control-center/spec @@ -7,6 +7,8 @@ templates: config/bpm.yml: config/bpm.yml config/ca_certs.pem.erb: config/ca_certs.pem config/cert.pem.erb: config/cert.pem + config/control-center-jaas.conf: config/control-center-jaas.conf + config/control-center-login.conf.erb: config/control-center-login.conf config/control-center.properties.erb: config/control-center.properties config/key.pem.erb: config/key.pem config/log4j.properties: config/log4j.properties @@ -54,8 +56,15 @@ properties: description: "Keystore password" default: notasecret - jaas.username: - description: "Username used in JAAS configuration" + kafka.jaas.username: + description: "Username used for Kafka broker" - jaas.password: - description: "Password used in JAAS configuration" + kafka.jaas.password: + description: "Password used for Kafka broker" + + basic.jaas.username: + description: "Username used for BASIC auth." + + basic.jaas.password: + description: "Password used for BASIC auth." + diff --git a/jobs/confluent-control-center/templates/bin/ctl.erb b/jobs/confluent-control-center/templates/bin/ctl.erb index e55a21c..2f7d6b8 100755 --- a/jobs/confluent-control-center/templates/bin/ctl.erb +++ b/jobs/confluent-control-center/templates/bin/ctl.erb @@ -4,7 +4,7 @@ set -e source /var/vcap/packages/openjdk-8/bosh/runtime.env -export CONTROL_CENTER_OPTS="-Djava.io.tmpdir=/var/vcap/data/tmp -Djavax.net.ssl.trustStore=/var/vcap/jobs/confluent-control-center/config/generated.truststore.jks -Djavax.net.ssl.trustStorePassword=<%= p("keystore_password") %> -Djavax.net.ssl.keyStore=/var/vcap/jobs/confluent-control-center/config/generated.keystore.jks -Djavax.net.ssl.keyStorePassword=<%= p("keystore_password") %>" +export CONTROL_CENTER_OPTS="-Djava.security.auth.login.config=/var/vcap/jobs/confluent-control-center/config/control-center-jaas.conf -Djava.io.tmpdir=/var/vcap/data/tmp -Djavax.net.ssl.trustStore=/var/vcap/jobs/confluent-control-center/config/generated.truststore.jks -Djavax.net.ssl.trustStorePassword=<%= p("keystore_password") %> -Djavax.net.ssl.keyStore=/var/vcap/jobs/confluent-control-center/config/generated.keystore.jks -Djavax.net.ssl.keyStorePassword=<%= p("keystore_password") %>" export CONTROL_CENTER_LOG4J_OPTS="-Dlog4j.configuration=file:/var/vcap/jobs/confluent-control-center/config/log4j.properties" case $1 in diff --git a/jobs/confluent-control-center/templates/config/control-center-jaas.conf b/jobs/confluent-control-center/templates/config/control-center-jaas.conf new file mode 100644 index 0000000..7198928 --- /dev/null +++ b/jobs/confluent-control-center/templates/config/control-center-jaas.conf @@ -0,0 +1,4 @@ +c3 { + org.eclipse.jetty.jaas.spi.PropertyFileLoginModule required + file="/var/vcap/jobs/confluent-control-center/config/control-center-login.conf"; +}; \ No newline at end of file diff --git a/jobs/confluent-control-center/templates/config/control-center-login.conf.erb b/jobs/confluent-control-center/templates/config/control-center-login.conf.erb new file mode 100644 index 0000000..e179a42 --- /dev/null +++ b/jobs/confluent-control-center/templates/config/control-center-login.conf.erb @@ -0,0 +1,2 @@ +<%= p("basic.jaas.username") %>: <%= p("basic.jaas.password") %>,Administrators +disallowed: no_access \ No newline at end of file diff --git a/jobs/confluent-control-center/templates/config/control-center.properties.erb b/jobs/confluent-control-center/templates/config/control-center.properties.erb index 0b73916..8ff8158 100644 --- a/jobs/confluent-control-center/templates/config/control-center.properties.erb +++ b/jobs/confluent-control-center/templates/config/control-center.properties.erb @@ -1,61 +1,63 @@ -# host/port pairs to use for establishing the initial connection to the Kafka cluster -<% servers = link('confluent-server') %> -<% zks = link('confluent-zookeeper') %> -<% connect = link('confluent-connect') %> -<% ksql = link('confluent-ksql') %> -<% registry = link('confluent-schema-registry') %> +<% servers = link('confluent-server') + zks = link('confluent-zookeeper') + connect = link('confluent-connect') + ksql = link('confluent-ksql') + registry = link('confluent-schema-registry') + replication = servers.instances.size > 2 ? 3 : 1 %> +bootstrap.servers=<%= servers.instances.map { |instance| "#{instance.address}:9093" }.join(",") %> + +confluent.controlcenter.auth.restricted.roles=Restricted +confluent.controlcenter.command.topic.replication=<%= replication %> +confluent.controlcenter.connect.<%= connect.p('group_id') %>.cluster=<%= connect.instances.map { |instance| "https://#{instance.address}:#{connect.p('listen_port')}" }.join(",") %> +confluent.controlcenter.data.dir=/var/vcap/store/confluent-control-center +confluent.controlcenter.internal.topics.replication=<%= replication %> + +confluent.controlcenter.ksql.url=<%= ksql.instances.map { |instance| "https://#{instance.address}:#{ksql.p('listen_port')}" }.join(",") %> +confluent.controlcenter.ksql.advertised.url=https://<%= p("basic.jaas.username") %>:<%= p("basic.jaas.password") %><%= ksql.p("external_hostname") %> + +confluent.controlcenter.rest.authentication.method=BASIC +confluent.controlcenter.rest.authentication.realm=c3 +confluent.controlcenter.rest.authentication.roles=Administrators,Restricted confluent.controlcenter.rest.listeners=https://0.0.0.0:<%= p("listen_port") %> +confluent.controlcenter.rest.ssl.key.password=<%= p("keystore_password") %> confluent.controlcenter.rest.ssl.keystore.location=/var/vcap/jobs/confluent-control-center/config/generated.keystore.jks confluent.controlcenter.rest.ssl.keystore.password=<%= p("keystore_password") %> -confluent.controlcenter.rest.ssl.key.password=<%= p("keystore_password") %> confluent.controlcenter.rest.ssl.truststore.location=/var/vcap/jobs/confluent-control-center/config/generated.truststore.jks confluent.controlcenter.rest.ssl.truststore.password=<%= p("keystore_password") %> -bootstrap.servers=<%= servers.instances.map { |instance| "#{instance.address}:9093" }.join(",") %> +confluent.controlcenter.schema.registry.url=<%= registry.instances.map { |instance| "https://#{instance.address}:#{registry.p('listen_port')}" }.join(",") %> +confluent.controlcenter.schema.registry.basic.auth.credentials.source=USER_INFO +confluent.controlcenter.schema.registry.basic.auth.user.info=<%= p("basic.jaas.username") %>:<%= p("basic.jaas.password") %> -# location for Control Center data -confluent.controlcenter.data.dir=/var/vcap/store/confluent-control-center +confluent.controlcenter.streams.security.protocol=SASL_SSL +confluent.controlcenter.streams.sasl.mechanism=SCRAM-SHA-512 +confluent.controlcenter.streams.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required \ + username="<%= p("kafka.jaas.username") %>" \ + password="<%= p("kafka.jaas.password") %>"; +confluent.controlcenter.streams.ssl.truststore.location=/var/vcap/jobs/confluent-control-center/config/generated.truststore.jks +confluent.controlcenter.streams.ssl.truststore.password=<%= p("keystore_password") %> +confluent.controlcenter.streams.ssl.keystore.location=/var/vcap/jobs/confluent-control-center/config/generated.keystore.jks +confluent.controlcenter.streams.ssl.keystore.password=<%= p("keystore_password") %> +confluent.controlcenter.streams.ssl.key.password=<%= p("keystore_password") %> # the Confluent license <% if !p("confluent.license", nil).nil? %> confluent.license=<%= p("confluent.license") %> <% end %> +confluent.metrics.topic.replication=<%= replication %> + +confluent.monitoring.interceptor.topic.replication=<%= replication %> + sasl.mechanism.inter.broker.protocol=SSL + ssl.keystore.location=/var/vcap/jobs/confluent-control-center/config/generated.truststore.jks ssl.keystore.password=<%= p("keystore_password") %> ssl.key.password=<%= p("keystore_password") %> ssl.truststore.location=/var/vcap/jobs/confluent-control-center/config/generated.truststore.jks ssl.truststore.password=<%= p("keystore_password") %> -confluent.controlcenter.streams.security.protocol=SASL_SSL -confluent.controlcenter.streams.sasl.mechanism=SCRAM-SHA-512 -confluent.controlcenter.streams.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="<%= p("jaas.username") %>" password="<%= p("jaas.password") %>"; -confluent.controlcenter.streams.ssl.truststore.location=/var/vcap/jobs/confluent-control-center/config/generated.truststore.jks -confluent.controlcenter.streams.ssl.truststore.password=<%= p("keystore_password") %> -confluent.controlcenter.streams.ssl.keystore.location=/var/vcap/jobs/confluent-control-center/config/generated.keystore.jks -confluent.controlcenter.streams.ssl.keystore.password=<%= p("keystore_password") %> -confluent.controlcenter.streams.ssl.key.password=<%= p("keystore_password") %> - # ZooKeeper connection string with host and port of a ZooKeeper servers <% zk_port = zks.p('client_port') %> -zookeeper.connect=<%= zks.instances.map { |instance| "#{instance.address}:#{zk_port}" }.join(",") %> - -<% replication = servers.instances.size > 2 ? 3 : 1 %> -confluent.metrics.topic.replication=<%= replication %> -confluent.monitoring.interceptor.topic.replication=<%= replication %> -confluent.controlcenter.command.topic.replication=<%= replication %> -confluent.controlcenter.internal.topics.replication=<%= replication %> - -# A comma separated list of Connect host names -<% connect_listen_port = connect.p('listen_port') %> -confluent.controlcenter.connect.<%= connect.p('group_id') %>.cluster=<%= connect.instances.map { |instance| "https://#{instance.address}:8443" }.join(",") %> - -# KSQL cluster URL -<% ksql_listen_port = ksql.p('listen_port') %> -confluent.controlcenter.ksql.url=<%= ksql.instances.map { |instance| "http://#{instance.address}:#{ksql_listen_port}" }.join(",") %> - -# Schema Registry cluster URL -<% registry_listen_port = registry.p('listen_port') %> -confluent.controlcenter.schema.registry.url=<%= registry.instances.map { |instance| "http://#{instance.address}:#{registry_listen_port}" }.join(",") %> +zookeeper.connect=<%= zks.instances.map { |instance| "#{instance.address}:#{zk_port}" }.join(",") %> \ No newline at end of file diff --git a/jobs/confluent-ksql/spec b/jobs/confluent-ksql/spec index e49588d..b28fd7a 100644 --- a/jobs/confluent-ksql/spec +++ b/jobs/confluent-ksql/spec @@ -8,6 +8,8 @@ templates: config/ca_certs.pem.erb: config/ca_certs.pem config/cert.pem.erb: config/cert.pem config/key.pem.erb: config/key.pem + config/ksql-server-jaas.conf: config/ksql-server-jaas.conf + config/ksql-server-login.conf.erb: config/ksql-server-login.conf config/ksql-server.properties.erb: config/ksql-server.properties packages: @@ -25,12 +27,21 @@ provides: type: ksql-conn properties: - listen_port + - external_hostname + - cluster_name properties: listen_port: description: The port the server listens on. default: 8088 + external_hostname: + description: Hostname used by KSQL clients + + cluster_name: + description: KSQL cluster name + default: KSQL + tls.ca_certs: description: | List of CA certs used to verify the brokers certificates @@ -44,8 +55,18 @@ properties: description: "Keystore password" default: notasecret - jaas.username: - description: "Username used in JAAS configuration" + kafka.jaas.username: + description: "Username used for Kafka Broker" + + kafka.jaas.password: + description: "Password used for Kafka Broker" - jaas.password: - description: "Password used in JAAS configuration" \ No newline at end of file + users: + description: |- + List of KSQL users + admin: + username: admin + password: password + user: + username: user + password: password \ No newline at end of file diff --git a/jobs/confluent-ksql/templates/bin/ctl b/jobs/confluent-ksql/templates/bin/ctl index 3ee3b8e..8a3b88b 100755 --- a/jobs/confluent-ksql/templates/bin/ctl +++ b/jobs/confluent-ksql/templates/bin/ctl @@ -4,6 +4,8 @@ set -e source /var/vcap/packages/openjdk-8/bosh/runtime.env +export KSQL_OPTS="-Djava.security.auth.login.config=/var/vcap/jobs/confluent-ksql/config/ksql-server-jaas.conf" + case $1 in start) diff --git a/jobs/confluent-ksql/templates/config/ksql-server-jaas.conf b/jobs/confluent-ksql/templates/config/ksql-server-jaas.conf new file mode 100644 index 0000000..a5a31c7 --- /dev/null +++ b/jobs/confluent-ksql/templates/config/ksql-server-jaas.conf @@ -0,0 +1,4 @@ +KsqlServer { + org.eclipse.jetty.jaas.spi.PropertyFileLoginModule required + file="/var/vcap/jobs/confluent-ksql/config/ksql-server-login.conf"; +}; \ No newline at end of file diff --git a/jobs/confluent-ksql/templates/config/ksql-server-login.conf.erb b/jobs/confluent-ksql/templates/config/ksql-server-login.conf.erb new file mode 100644 index 0000000..5d5a87c --- /dev/null +++ b/jobs/confluent-ksql/templates/config/ksql-server-login.conf.erb @@ -0,0 +1,9 @@ + +<% p("users").each_value do |user| + if user["roles"] != nil + roles = user["roles"].map { |role| "#{role}" }.join(",") + if roles != "" + roles = "," + roles + end + end %><%= user["username"] %>: <%= user["password"] %><%= roles %> +<% end %> \ No newline at end of file diff --git a/jobs/confluent-ksql/templates/config/ksql-server.properties.erb b/jobs/confluent-ksql/templates/config/ksql-server.properties.erb index 5098042..3f0d459 100644 --- a/jobs/confluent-ksql/templates/config/ksql-server.properties.erb +++ b/jobs/confluent-ksql/templates/config/ksql-server.properties.erb @@ -15,21 +15,26 @@ # specific language governing permissions and limitations under the License. # +authentication.method=BASIC +authentication.roles=admin,developer,user,ksq-user +authentication.realm=KsqlServer + #------ Endpoint config ------- ### HTTP ### # The URL the KSQL server will listen on: -listeners=http://<%= spec.address %>:<%= p("listen_port") %> +listeners=https://0.0.0.0:<%= p("listen_port") %> ### HTTPS ### # To switch KSQL over to communicating using HTTPS comment out the 'listeners' line above # uncomment and complete the properties below. # See: https://docs.confluent.io/current/ksql/docs/installation/server-config/security.html#configuring-ksql-cli-for-https # -# listeners=https://localhost:8088 -# ssl.keystore.location=? -# ssl.keystore.password=? -# ssl.key.password=? +ssl.truststore.location=/var/vcap/jobs/confluent-ksql/config/generated.truststore.jks +ssl.truststore.password=<%= p("keystore_password") %> +ssl.keystore.location=/var/vcap/jobs/confluent-ksql/config/generated.keystore.jks +ssl.keystore.password=<%= p("keystore_password") %> +ssl.key.password=<%= p("keystore_password") %> #------ Logging config ------- @@ -54,8 +59,8 @@ security.protocol=SASL_SSL sasl.mechanism=SCRAM-SHA-512 sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required \ - username="<%= p("jaas.username") %>" \ - password="<%= p("jaas.password") %>"; + username="<%= p("kafka.jaas.username") %>" \ + password="<%= p("kafka.jaas.password") %>"; ssl.truststore.location=/var/vcap/jobs/confluent-ksql/config/generated.truststore.jks ssl.truststore.password=<%= p("keystore_password") %> @@ -64,5 +69,5 @@ ssl.keystore.password=<%= p("keystore_password") %> ssl.key.password=<%= p("keystore_password") %> # Uncomment and complete the following to enable KSQL's integration to the Confluent Schema Registry: -<% schema_registry_url = "http://" + schemaRegistries.address + ":" + schemaRegistries.p("listen_port").to_s %> +<% schema_registry_url = "https://" + schemaRegistries.address + ":" + schemaRegistries.p("listen_port").to_s %> ksql.schema.registry.url=<%= schema_registry_url %> \ No newline at end of file diff --git a/jobs/confluent-schema-registry/spec b/jobs/confluent-schema-registry/spec index d2fcabd..5a5a772 100644 --- a/jobs/confluent-schema-registry/spec +++ b/jobs/confluent-schema-registry/spec @@ -8,6 +8,8 @@ templates: config/ca_certs.pem.erb: config/ca_certs.pem config/cert.pem.erb: config/cert.pem config/key.pem.erb: config/key.pem + config/schema-registry-jaas.conf: config/schema-registry-jaas.conf + config/schema-registry-login.conf.erb: config/schema-registry-login.conf config/schema-registry.properties.erb: config/schema-registry.properties consumes: @@ -50,8 +52,18 @@ properties: description: "Keystore password" default: notasecret - jaas.username: - description: "Username used in JAAS configuration" + kafka.jaas.username: + description: "Username used for Kafka Broker" - jaas.password: - description: "Password used in JAAS configuration" \ No newline at end of file + kafka.jaas.password: + description: "Password used for Kafka Broker" + + users: + description: |- + List of Schema Registry users + admin: + username: admin + password: password + user: + username: user + password: password \ No newline at end of file diff --git a/jobs/confluent-schema-registry/templates/bin/ctl b/jobs/confluent-schema-registry/templates/bin/ctl index 3e26c89..f80d3fd 100755 --- a/jobs/confluent-schema-registry/templates/bin/ctl +++ b/jobs/confluent-schema-registry/templates/bin/ctl @@ -4,6 +4,8 @@ set -e source /var/vcap/packages/openjdk-8/bosh/runtime.env +#export SCHEMA_REGISTRY_OPTS="-Djava.security.auth.login.config=/var/vcap/jobs/confluent-schema-registry/config/schema-registry-jaas.conf" + case $1 in start) diff --git a/jobs/confluent-schema-registry/templates/config/schema-registry-jaas.conf b/jobs/confluent-schema-registry/templates/config/schema-registry-jaas.conf new file mode 100644 index 0000000..c260eac --- /dev/null +++ b/jobs/confluent-schema-registry/templates/config/schema-registry-jaas.conf @@ -0,0 +1,4 @@ +SchemaRegistry { + org.eclipse.jetty.jaas.spi.PropertyFileLoginModule required + file="/var/vcap/jobs/confluent-schema-registry/config/schema-registry-login.conf"; +} \ No newline at end of file diff --git a/jobs/confluent-schema-registry/templates/config/schema-registry-login.conf.erb b/jobs/confluent-schema-registry/templates/config/schema-registry-login.conf.erb new file mode 100644 index 0000000..09749e0 --- /dev/null +++ b/jobs/confluent-schema-registry/templates/config/schema-registry-login.conf.erb @@ -0,0 +1,8 @@ +<% p("users").each_value do |user| + if user["roles"] != nil + roles = user["roles"].map { |role| "#{role}" }.join(",") + if roles != "" + roles = "," + roles + end + end %><%= user["username"] %>: <%= user["password"] %><%= roles %> +<% end %> \ No newline at end of file diff --git a/jobs/confluent-schema-registry/templates/config/schema-registry.properties.erb b/jobs/confluent-schema-registry/templates/config/schema-registry.properties.erb index a0b0128..3ce35df 100644 --- a/jobs/confluent-schema-registry/templates/config/schema-registry.properties.erb +++ b/jobs/confluent-schema-registry/templates/config/schema-registry.properties.erb @@ -15,12 +15,24 @@ # limitations under the License. # +#authentication.method=BASIC +#authentication.roles=admin,app +#authentication.realm=SchemaRegistry + # The address the socket server listens on. # FORMAT: # listeners = listener_name://host_name:port # EXAMPLE: # listeners = PLAINTEXT://your.host.name:9092 -listeners=http://<%= spec.address %>:<%= p("listen_port") %> +listeners=https://<%= spec.address %>:<%= p("listen_port") %> + +inter.instance.protocol=https + +ssl.truststore.location=/var/vcap/jobs/confluent-schema-registry/config/generated.truststore.jks +ssl.truststore.password=<%= p("keystore_password") %> +ssl.keystore.location=/var/vcap/jobs/confluent-schema-registry/config/generated.keystore.jks +ssl.keystore.password=<%= p("keystore_password") %> +ssl.key.password=<%= p("keystore_password") %> # Zookeeper connection string for the Zookeeper cluster used by your Kafka cluster # (see zookeeper docs for details). @@ -41,8 +53,8 @@ kafkastore.security.protocol=SASL_SSL kafkastore.sasl.mechanism=SCRAM-SHA-512 kafkastore.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required \ - username="<%= p("jaas.username") %>" \ - password="<%= p("jaas.password") %>"; + username="<%= p("kafka.jaas.username") %>" \ + password="<%= p("kafka.jaas.password") %>"; kafkastore.ssl.truststore.location=/var/vcap/jobs/confluent-schema-registry/config/generated.truststore.jks kafkastore.ssl.truststore.password=<%= p("keystore_password") %> diff --git a/jobs/confluent-server/templates/config/server.properties.erb b/jobs/confluent-server/templates/config/server.properties.erb index fb02d66..4317600 100644 --- a/jobs/confluent-server/templates/config/server.properties.erb +++ b/jobs/confluent-server/templates/config/server.properties.erb @@ -40,8 +40,8 @@ sasl.enabled.mechanisms=SCRAM-SHA-512 sasl.mechanism=SCRAM-SHA-512 sasl.mechanism.inter.broker.protocol=SCRAM-SHA-512 sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required \ -username="<%= p("jaas.username") %>" \ -password="<%= p("jaas.password") %>"; + username="<%= p("jaas.username") %>" \ + password="<%= p("jaas.password") %>"; ssl.keystore.location=/var/vcap/jobs/confluent-server/config/generated.keystore.jks ssl.keystore.password=<%= p("keystore_password") %> @@ -179,4 +179,7 @@ confluent.metrics.reporter.ssl.keystore.location=/var/vcap/jobs/confluent-server confluent.metrics.reporter.ssl.keystore.password=<%= p("keystore_password") %> confluent.metrics.reporter.ssl.keystore.type=PKCS12 +confluent.metrics.reporter.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required \ + username="<%= p("metric.jaas.username") %>" \ + password="<%= p("metric.jaas.password") %>"; <% end %> \ No newline at end of file diff --git a/manifests/confluent-platform-solo.yml b/manifests/confluent-platform-solo.yml deleted file mode 100644 index 7cb6597..0000000 --- a/manifests/confluent-platform-solo.yml +++ /dev/null @@ -1,90 +0,0 @@ ---- -name: confluent-platform-dev -addons: -- name: bpm - jobs: - - name: bpm - release: bpm - -instance_groups: -- name: confluent-server - azs: [z1, z2, z3] - instances: 1 - vm_resources: - cpu: 4 - ram: 8192 - ephemeral_disk_size: 10 - vm_extensions: - - control-center - stemcell: default - persistent_disk: 10240 - networks: - - name: default - jobs: - - name: confluent-zookeeper - release: confluent-platform - - name: confluent-server - release: confluent-platform - properties: - offsets: - topic: - replication: - factor: 1 - transaction: - state: - log: - replication: - factor: 1 - min: - isr: 1 - metric: - replicas: 1 - jaas: - username: metric - password: ((metric-jaas-password)) - - name: confluent-control-center - release: confluent-platform - - name: confluent-schema-registry - release: confluent-platform - - name: confluent-connect - release: confluent-platform - properties: - config: - storage: - replication_factor: 1 - offset: - storage: - replication_factor: 1 - status: - storage: - replication_factor: 1 - - name: confluent-ksql - release: confluent-platform - -stemcells: -- alias: default - os: ubuntu-xenial - version: 315.latest - -update: - canaries: 1 - canary_watch_time: 1000-60000 - update_watch_time: 1000-60000 - max_in_flight: 1 - serial: false - -releases: -- name: bpm - sha1: 12142ca9437e48694374876fe0236938e252d1e2 - stemcell: - os: ubuntu-xenial - version: "315.61" - url: https://confluent-platform-bosh-release.s3.amazonaws.com/bpm/bpm-release-1.1.0.tgz - version: 1.1.0 -- name: confluent-platform - sha1: a1531cd6410a4b9dda2014906cbee633b485a3c7 - stemcell: - os: ubuntu-xenial - version: "315.61" - url: https://s3.amazonaws.com/kafka-boshrelease/compiled-releases/kafka/kafka-2.2.3-ubuntu-xenial-250.23-20190323-023747-677211024-20190323023753.tgz - version: 5.2.1 \ No newline at end of file diff --git a/manifests/confluent-platform.yml b/manifests/confluent-platform.yml index 9fe4a16..406da8f 100644 --- a/manifests/confluent-platform.yml +++ b/manifests/confluent-platform.yml @@ -24,23 +24,43 @@ variables: options: ca: ca common_name: "*.((confluent-connect-external-host))" +- name: confluent-schema-registry-tls + type: certificate + options: + ca: ca + common_name: "*.((confluent-schema-registry-external-host))" - name: confluent-control-center-tls type: certificate options: ca: ca common_name: "*.((confluent-control-center-external-host))" +- name: confluent-ksql-tls + type: certificate + options: + ca: ca + common_name: "*.((confluent-ksql-external-host))" - name: server-jaas-password type: password - name: metric-jaas-password type: password -- name: connect-jaas-password +- name: connect-kafka-jaas-password + type: password +- name: control-center-kafka-jaas-password type: password -- name: control-center-jaas-password +- name: basic-jaas-password type: password -- name: ksql-jaas-password +- name: schema-registry-users-connect-password + type: password +- name: schema-registry-users-ksql-password + type: password +- name: schema-registry-users-app-password + type: password +- name: ksql-kafka-jaas-password type: password - name: schema-registry-jaas-password type: password +- name: ksql-users-developer-password + type: password instance_groups: - name: confluent-zookeeper @@ -65,13 +85,13 @@ instance_groups: password: ((metric-jaas-password)) connect: username: connect - password: ((connect-jaas-password)) + password: ((connect-kafka-jaas-password)) control-center: username: control-center - password: ((control-center-jaas-password)) + password: ((control-center-kafka-jaas-password)) ksql: username: ksql - password: ((ksql-jaas-password)) + password: ((ksql-kafka-jaas-password)) schema-registry: username: schema-registry password: ((schema-registry-jaas-password)) @@ -80,7 +100,7 @@ instance_groups: instances: 3 vm_type: default stemcell: default - persistent_disk: 10240 + persistent_disk: 200_000 networks: - name: default jobs: @@ -118,9 +138,14 @@ instance_groups: - ((ca.certificate)) certificate: ((confluent-control-center-tls)) keystore_password: ((keystore-password)) - jaas: - username: control-center - password: ((control-center-jaas-password)) + kafka: + jaas: + username: control-center + password: ((control-center-kafka-jaas-password)) + basic: + jaas: + username: admin + password: ((basic-jaas-password)) - name: confluent-schema-registry azs: [z1, z2, z3] instances: 1 @@ -133,14 +158,40 @@ instance_groups: - name: confluent-schema-registry release: confluent-platform properties: + debug: true tls: ca_certs: - ((ca.certificate)) - certificate: ((confluent-server-tls)) + certificate: ((confluent-schema-registry-tls)) keystore_password: ((keystore-password)) - jaas: - username: schema-registry - password: ((schema-registry-jaas-password)) + kafka: + jaas: + username: schema-registry + password: ((schema-registry-jaas-password)) + users: + admin: + username: admin + password: ((basic-jaas-password)) + roles: + - admin + - app + connect: + username: connect + password: ((schema-registry-users-connect-password)) + roles: + - admin + - app + ksql: + username: ksql + password: ((schema-registry-users-ksql-password)) + roles: + - admin + - app + app: + username: user + password: ((schema-registry-users-app-password)) + roles: + - app - name: confluent-connect azs: [z1, z2, z3] instances: 1 @@ -158,29 +209,63 @@ instance_groups: - ((ca.certificate)) certificate: ((confluent-connect-tls)) keystore_password: ((keystore-password)) - jaas: - username: connect - password: ((connect-jaas-password)) + basic: + jaas: + username: admin + password: ((basic-jaas-password)) + kafka: + jaas: + username: connect + password: ((connect-kafka-jaas-password)) + schema_registry: + basic: + username: connect + password: ((schema-registry-users-connect-password)) + connectors: + s3: + endpoint: ((connectors-s3-endpoint)) + access_key: ((connectors-s3-access-key)) + secret_key: ((connectors-s3-secret-key)) + bucket: ((connectors-s3-bucket)) - name: confluent-ksql azs: [z1, z2, z3] instances: 1 vm_type: default stemcell: default persistent_disk: 10240 + vm_extensions: + - ksql networks: - name: default jobs: - name: confluent-ksql release: confluent-platform properties: + external_hostname: ((ksql-external-hostname)) tls: ca_certs: - ((ca.certificate)) - certificate: ((confluent-server-tls)) + certificate: ((confluent-ksql-tls)) keystore_password: ((keystore-password)) - jaas: - username: ksql - password: ((ksql-jaas-password)) + kafka: + jaas: + username: ksql + password: ((ksql-kafka-jaas-password)) + users: + admin: + username: admin + password: ((basic-jaas-password)) + roles: + - admin + - developer + - user + - ksq-user + developer: + username: developer + password: ((ksql-users-developer-password)) + roles: + - developer + - user stemcells: - alias: default diff --git a/packages/minio-mc/packaging b/packages/minio-mc/packaging new file mode 100644 index 0000000..34df423 --- /dev/null +++ b/packages/minio-mc/packaging @@ -0,0 +1,5 @@ +set -ex + +chmod +x ${BOSH_COMPILE_TARGET}/minio/mc + +mv ${BOSH_COMPILE_TARGET}/minio/mc ${BOSH_INSTALL_TARGET} \ No newline at end of file diff --git a/packages/minio-mc/spec b/packages/minio-mc/spec new file mode 100644 index 0000000..3bddc49 --- /dev/null +++ b/packages/minio-mc/spec @@ -0,0 +1,5 @@ +--- +name: minio-mc +dependencies: [] +files: +- minio/mc \ No newline at end of file