Added minio s3 cli blob

daniellavoie · Oct 25, 2019 · c858818 · c858818
1 parent a08b1d0
commit c858818
Show file tree

Hide file tree

Showing 32 changed files with 492 additions and 315 deletions.
diff --git a/README.md b/README.md
@@ -19,7 +19,7 @@ Long story short, Bosh let you declare a desired state of your software and the
 ## TL;DR - I just want to deploy
 
 * [AWS deployment instructions](doc/aws-instructions.md)
-* GCP Deployment instructions - sooooon
+* [GCP Deployment instructions](gcp-instructions.md)
 * vSphere Deployment instructions - sooooon
 * Virtual Box deployment instructions - sooooon
 
@@ -37,12 +37,6 @@ A lot of security features are to be implemented. For a complete state of the bi
 
 This current iteration was successully tested on AWS and GCP cpis.
 
-## Deploy single collocated VM
-
-```plain
-bosh deploy confluent-platform-bosh-release/manifests/confluent-platform-solo.yml -o confluent-platform-bosh-release/manifests/operators/create.yml
-```
-
 ## Deploy Confluent Platform Cluster
 
 ```

diff --git a/config/blobs.yml b/config/blobs.yml
@@ -5,3 +5,7 @@ java/jdk8u192-b03.tar.gz:
   size: 45670457
   object_id: 5a70262f-3127-4a35-6685-b271ba939661
   sha: sha256:5d8203117cad2ed7ef1e20d951f3c1b1515f725484e35cc10c61307e66018efe
+minio/mc:
+  size: 16605184
+  object_id: 0fb6f283-7aea-4c8a-5157-c6d3a509680f
+  sha: sha256:67280ce05acdd656156ca39b266f2931889ed2b58b703300639b1ccba645a6b3
diff --git a/doc/gcp-instructions.md b/doc/gcp-instructions.md
@@ -10,62 +10,74 @@ Example CIDR : 10.0.10.0/16
 
 ### Create subnets
 
-#### Infrastructure
-
-* Example subnet 1 name : infrastructure
-* Example subnet 1 CIDR : 10.0.10.0/24
-* Example subnet 1 region : northamerica-northeast1
-
-#### Confluent Platform
-
-* Example subnet 2 name : confluent-platform
-* Example subnet 2 CIDR : 10.0.20.0/24
-* Example subnet 2 region : northamerica-northeast1
+| Name | CIDR | Region |
+|---|---|---|
+| instrastructure | 10.0.10.0/24 | northamerica-northeast1 |
+| confluent-platform | 10.0.20.0/24 | northamerica-northeast1 |
 
 ### Create Firewall rules
 
 | Name | Targets | Filters | Protocols / ports | Network |
 | ------------- | ------------- | ------------- | ------------- | ------------- |
 | bosh-allow-ssh | allow-ssh | IP ranges: 0.0.0.0/0  | tcp:22 | cp-bosh |
 | bosh-unrestricted | confluent-platform | Tags: confluent-platform  | all | cp-bosh |
-|  bosh-allow-control-center | allow-control-center | IP ranges: 0.0.0.0/0 | tcp:9021 | cp-bosh |
-
-### Create a TCP Load Balancer for Confluent Server
-
-TODO
+| bosh-allow-control-center | allow-control-center | IP ranges: 0.0.0.0/0 | tcp:9021 | cp-bosh |
+| bosh-allow-ksql | allow-ksql | IP ranges: 0.0.0.0/0 | tcp:8088 | cp-bosh |
 
 ### Create unmanaged instance groups for Control Center
 
-Instance Group 1 Name : cp-control-center
-Instance Group 1 Zone : northamerica-northeast1-a
-Instance Group 1 Network : cp-bosh
-Instance Group 1 Subnet : confluent-platform
-
-
-Instance Group 2 Name : cp-control-center
-Instance Group 2 Zone : northamerica-northeast1-b
-Instance Group 2 Network : cp-bosh
-Instance Group 2 Subnet : confluent-platform
+| Number | Zone | Name | Network | Subnet |
+|---|---|---|---|---|
+| 1 | northamerica-northeast1-a | cp-control-center | cp-bosh | confluent-platform |
+| 2 | northamerica-northeast1-b | cp-control-center | cp-bosh | confluent-platform |
+| 3 | northamerica-northeast1-c | cp-control-center | cp-bosh | confluent-platform |
 
+### Create unmanaged instance groups for KSQL
 
-Instance Group 3 Name : cp-control-center
-Instance Group 3 Zone : northamerica-northeast1-c
-Instance Group 3 Network : cp-bosh
-Instance Group 3 Subnet : confluent-platform
+| Number | Zone | Name | Network | Subnet |
+|---|---|---|---|---|
+| 1 | northamerica-northeast1-a | cp-ksql | cp-bosh | confluent-platform |
+| 2 | northamerica-northeast1-b | cp-ksql | cp-bosh | confluent-platform |
+| 3 | northamerica-northeast1-c | cp-ksql | cp-bosh | confluent-platform |
 
 ### Create an Http Load Balancer for Control Center
 
 #### Backend services
 
 Instance Group : cp-control-center
+
 Port number : 9021
+
 Health check : HTTP on :9021/
+
 Backend Services : cp-control-center
 
 #### Frontend
 
 protocol :  http
+
 port : 80
+
+ip : Reserved ipv4
+
+### Create an Http Load Balancer for KSQL
+
+#### Backend services
+
+Instance Group : cp-ksql
+
+Port number : 8088
+
+Health check : HTTP on :8088/
+
+Backend Services : cp-ksql
+
+#### Frontend
+
+protocol :  https
+
+port : 443
+
 ip : Reserved ipv4
 
 ### Create a jumpbox to run Bosh CLI Commands

diff --git a/doc/state-of-security.md b/doc/state-of-security.md
@@ -1,91 +1,44 @@
 # State of security implementation
 
-- [ ] Broker
-  - [X] Brokers to brokers
-    - [X] Encryption
-    - [X] Authentication
-  - [ ] Metric reporter
-    - [X] Encryption
-    - [X] Authentication
-    - [ ] ACL
-    - [ ] RBAC
-
-- [ ] Connect
-  - [ ] Workers to Brokers
-    - [X] Encryption
-    - [X] Authentication
-    - [ ] ACL
-    - [ ] RBAC
-  - [ ] Rest API
-    - [ ] Encryption
-    - [ ] Authentication
-    - [ ] RBAC
-  - [ ] Interceptors
-    - [ ] Encryption
-    - [ ] Authentication
-    - [ ] ACL
-    - [ ] RBAC
-
-- [ ] KSQL
-  - [ ] KSQL nodes to Brokers
-    - [X] Encryption
-    - [X] Authentication
-    - [ ] ACL
-  - [ ] Rest API
-    - [ ] Encryption
-    - [ ] Authentication
-  - [ ] Schema Registry
-    - [ ] Encryption
-    - [ ] Authentication
-  - [ ] Interceptors
-    - [ ] Encryption
-    - [ ] Authentication
-    - [ ] ACL
-
-- [ ] Schema Registry
-  - [ ] Schema registry to Brokers
-    - [X] Encryption
-    - [X] Authentication
-    - [ ] ACL
-    - [ ] RBAC
-  - [ ] Rest API
-    - [ ] Encryption
-    - [ ] Authentication
-    - [ ] RBAC
-  - [ ] Interceptors
-    - [ ] SSL
-    - [ ] Authentication
-    - [ ] ACL
-    - [ ] RBAC
-
-- [ ] Control Center
-  - [ ] Rest API
-    - [ ] Encryption
-    - [ ] Authentication
-    - [ ] ACL
-    - [ ] RBAC
-  - [ ] Brokers
-    - [X] Encryption
-    - [X] Authentication
-    - [ ] ACL
-    - [ ] RBAC
-  - [ ] Connect
-    - [ ] Encryption
-    - [ ] Authentication
-    - [ ] ACL
-    - [ ] RBAC
-  - [ ] KSQL
-    - [ ] Encryption
-    - [ ] Authentication
-    - [ ] ACL
-    - [ ] RBAC
+- [X] Encryption
+  - [X] Kafka Broker
+    - [X] mTLS for broker intercommunication
+    - [X] mTLS between Metric Reporters and Kafka Cluster
+  - [X] Kafka Connect  
+    - [X] mTLS with Kafka cluster
+    - [X] Https for Connect REST endpoints
+  - [X] Schema Registry
+    - [X] mTLS with Kafka cluster
+    - [X] Https for REST endpoints
+  - [X] KSQL
+    - [X] mTLS with Kafka cluster
+    - [X] Https for REST endpoints
+  - [X] Control Center
+    - [X] mTLS with Kafka cluster
+    - [X] Https for REST endpoints
+- [ ] Authentication
+  - [X] Kafka Broker
+    - [X] SASL for broker intercommunication
+    - [X] SASL between Metric Reporters and Kafka Cluster (to test)
+  - [X] Kafka Connect
+    - [X] SASL with Kafka cluster
+    - [X] REST endpoints
   - [ ] Schema Registry
-    - [ ] Encryption
-    - [ ] Authentication
-    - [ ] ACL
-    - [ ] RBAC
-  - [ ] Zookeeper
-    - [ ] Authentication
-
-- [ ] Zookeeper
-  - [ ] Authentication
+    - [X] SASL with Kafka cluster
+    - [ ] REST endpoints
+  - [X] KSQL
+    - [X] SASL for with Kafka cluster
+    - [X] REST endpoints
+  - [X] Control Center
+    - [X] SASL with Kafka cluster
+    - [X] Basic Auth for REST endpoints
+- [ ] Kafka Topics ACL
+  - [ ] Kafka Connect
+  - [ ] Schema Regisry
+  - [ ] KSQL
+  - [ ] Control Center
+- [ ] RBAC
+  - [ ] Kafka Connect
+  - [ ] Schema Regisry
+  - [ ] KSQL
+  - [ ] Control Center
diff --git a/jobs/confluent-connect/spec b/jobs/confluent-connect/spec
@@ -3,16 +3,21 @@ name: confluent-connect
 
 templates:
   bin/ctl: bin/ctl
+  bin/download-connectors.sh: bin/download-connectors.sh
   bin/pre-start.erb: bin/pre-start
   config/bpm.yml: config/bpm.yml
   config/ca_certs.pem.erb: config/ca_certs.pem
   config/cert.pem.erb: config/cert.pem
+  config/connect-jaas.conf: config/connect-jaas.conf
+  config/connect-login.conf.erb: config/connect-login.conf
   config/connect.properties.erb: config/connect.properties
   config/key.pem.erb: config/key.pem
+  config/log4j.properties: config/log4j.properties
 
 packages:
 - openjdk-8
 - confluent-platform
+- minio-mc
 
 consumes:
 - name: confluent-server
@@ -29,8 +34,8 @@ provides:
 
 properties:
   listen_port:
-    description: The port to listen for client connections
-    default: 8083
+    description: "Https port for Confluent Connect REST endpoints"
+    default: 8443
   group_id:
     description: Unique identifier for the set of workers that form the Kafka Connect cluster
     default: connect-cluster
@@ -66,8 +71,34 @@ properties:
     description: "Keystore password"
     default: notasecret
 
-  jaas.username:
-    description: "Username used in JAAS configuration"
+  kafka.jaas.username:
+    description: "Username used for Kafka Broker"
 
-  jaas.password:
-    description: "Password used in JAAS configuration"
+  kafka.jaas.password:
+    description: "Password used for Kafka Broker"
+
+  basic.jaas.username:
+    description: "Username used for Basic Auth"
+
+  basic.jaas.password:
+    description: "Password used for Basic Auth"
+
+  schema_registry.basic.username:
+    description: Username for Basic Auth on Schema Registry
+
+  schema_registry.basic.password:
+    description: Password for Basic Auth on Schema Registry
+
+  connectors.s3.endpoint:
+    description: "S3 endpoint to lookup for connectors"
+
+  connectors.s3.access_key:
+    description: "S3 Access key to lookup for connectors"
+    default: ""
+
+  connectors.s3.secret_key:
+    description: "S3 Secret key to lookup for connectors"
+    default: ""
+
+  connectors.s3.bucket:
+    description: "Bucket to lookup for connectors"
diff --git a/jobs/confluent-connect/templates/bin/ctl b/jobs/confluent-connect/templates/bin/ctl
@@ -4,6 +4,14 @@ set -e
 
 source /var/vcap/packages/openjdk-8/bosh/runtime.env
 
+export KAFKA_OPTS="-Djava.security.auth.login.config=/var/vcap/jobs/confluent-connect/config/connect-jaas.conf"
+
+#export KAFKA_LOG4J_OPTS="-Dlog4j.configuration=file:/var/vcap/jobs/confluent-connect/config/log4j.properties"
+
+export SCHEMA_REGISTRY_OPTS="-Dssl.truststore.location=/var/vcap/jobs/confluent-connect/config/generated.truststore.jks -Dssl.truststore.password=<%= p("keystore_password") %> -Dssl.keystore.location=/var/vcap/jobs/confluent-connect/config/generated.keystore.jks -Dssl.keystore.password=<%= p("keystore_password") %> -Dssl.key.password=<%= p("keystore_password") %>"
+
+export LOG_DIR=/var/vcap/sys/log/confluent-connect
+
 case $1 in
 
   start)

diff --git a/jobs/confluent-connect/templates/bin/download-connectors.sh b/jobs/confluent-connect/templates/bin/download-connectors.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+function downloadConnectors() {
+  CONNECTORS_FOLDER=$1
+  S3_ENDPOINT=$2
+  S3_ACCESS_KEY=$3
+  S3_SECRET_KEY=$4
+  S3_BUCKET=$5
+
+  rm -rf $CONNECTORS_FOLDER
+
+  mkdir $CONNECTORS_FOLDER
+
+  /var/vcap/packages/minio-mc/mc config host add connectors $S3_ENDPOINT $S3_ACCESS_KEY $S3_SECRET_KEY
+
+  /var/vcap/packages/minio-mc/mc cp --recursive connectors/$S3_BUCKET/ $CONNECTORS_FOLDER
+
+  for i in $CONNECTORS_FOLDER/*.zip; do
+    newdir="${i:0:-4}" && mkdir "$newdir"
+    unzip "$i" -d  "$newdir"
+  done
+}