From 8d8ea25a2aa71ca29192eb0d8bff800d5bc8c9e6 Mon Sep 17 00:00:00 2001 From: David Sloan Date: Tue, 3 Sep 2024 09:15:21 +0100 Subject: [PATCH 01/12] Elastic SSL WTD --- .../connect/security/StoreInfo.scala | 97 +++++++++++++++++++ .../elastic6/config/ElasticConfig.scala | 1 + .../elastic6/config/ElasticSettings.scala | 3 + .../elastic7/config/ElasticConfig.scala | 1 + .../elastic7/config/ElasticSettings.scala | 5 +- 5 files changed, 106 insertions(+), 1 deletion(-) create mode 100644 kafka-connect-common/src/main/scala/io/lenses/streamreactor/connect/security/StoreInfo.scala diff --git a/kafka-connect-common/src/main/scala/io/lenses/streamreactor/connect/security/StoreInfo.scala b/kafka-connect-common/src/main/scala/io/lenses/streamreactor/connect/security/StoreInfo.scala new file mode 100644 index 0000000000..96541108ce --- /dev/null +++ b/kafka-connect-common/src/main/scala/io/lenses/streamreactor/connect/security/StoreInfo.scala @@ -0,0 +1,97 @@ +/* + * Copyright 2017-2024 Lenses.io Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.lenses.streamreactor.connect.security + +import io.lenses.streamreactor.common.config.base.traits.BaseConfig +import org.apache.kafka.common.config.SslConfigs + +import java.io.FileInputStream +import java.security.KeyStore +import javax.net.ssl.KeyManagerFactory +import javax.net.ssl.SSLContext +import javax.net.ssl.TrustManagerFactory + +case class StoreInfo( + storePath: String, + storeType: Option[String], + storePassword: Option[String] = None, + ) + +case class StoresInfo( + trustStore: Option[StoreInfo] = None, + keyStore: Option[StoreInfo] = None, + ) { + def toSslContext: Option[SSLContext] = { + val maybeTrustFactory: Option[TrustManagerFactory] = trustStore.map { + case StoreInfo(path, storeType, password) => + trustManagers(path, storeType, password) + } + val maybeKeyFactory: Option[KeyManagerFactory] = keyStore.map { + case StoreInfo(path, storeType, password) => + keyManagers(path, storeType, password) + } + + Option.when(maybeTrustFactory.nonEmpty || maybeKeyFactory.nonEmpty) { + val sslContext = SSLContext.getInstance("TLS") + sslContext.init( + maybeKeyFactory.map(_.getKeyManagers).orNull, + maybeTrustFactory.map(_.getTrustManagers).orNull, + null, + ) + sslContext + } + } + + private def trustManagers(path: String, storeType: Option[String], password: Option[String]) = { + val truststore: KeyStore = getJksStore(path, storeType, password) + + val trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm) + trustManagerFactory.init(truststore) + trustManagerFactory + } + + private def keyManagers(path: String, storeType: Option[String], password: Option[String]): KeyManagerFactory = { + val keyStore: KeyStore = getJksStore(path, storeType, password) + + val keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm) + keyManagerFactory.init(keyStore, password.getOrElse("").toCharArray) + keyManagerFactory + } + + private def getJksStore(path: String, storeType: Option[String], password: Option[String]) = { + val keyStore = KeyStore.getInstance(storeType.map(_.toUpperCase).getOrElse("JKS")) + val truststoreStream = new FileInputStream(path) + keyStore.load(truststoreStream, password.getOrElse("").toCharArray) + keyStore + } +} + +object StoresInfo { + def apply(config: BaseConfig): StoresInfo = { + val trustStore = for { + storePath <- Option(config.getString(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG)) + storeType = Option(config.getString(SslConfigs.SSL_TRUSTSTORE_TYPE_CONFIG)) + storePassword = Option(config.getPassword(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG)).map(_.value()) + } yield StoreInfo(storePath, storeType, storePassword) + val keyStore = for { + storePath <- Option(config.getString(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG)) + storeType = Option(config.getString(SslConfigs.SSL_KEYSTORE_TYPE_CONFIG)) + storePassword = Option(config.getPassword(SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG)).map(_.value()) + } yield StoreInfo(storePath, storeType, storePassword) + + StoresInfo(trustStore, keyStore) + } +} \ No newline at end of file diff --git a/kafka-connect-elastic6/src/main/scala/io/lenses/streamreactor/connect/elastic6/config/ElasticConfig.scala b/kafka-connect-elastic6/src/main/scala/io/lenses/streamreactor/connect/elastic6/config/ElasticConfig.scala index b44b10a5ee..24cd987258 100644 --- a/kafka-connect-elastic6/src/main/scala/io/lenses/streamreactor/connect/elastic6/config/ElasticConfig.scala +++ b/kafka-connect-elastic6/src/main/scala/io/lenses/streamreactor/connect/elastic6/config/ElasticConfig.scala @@ -191,6 +191,7 @@ object ElasticConfig { ConfigDef.Width.MEDIUM, ElasticConfigConstants.PROGRESS_COUNTER_ENABLED_DISPLAY, ) + .withClientSslSupport() } /** diff --git a/kafka-connect-elastic6/src/main/scala/io/lenses/streamreactor/connect/elastic6/config/ElasticSettings.scala b/kafka-connect-elastic6/src/main/scala/io/lenses/streamreactor/connect/elastic6/config/ElasticSettings.scala index 9d3b86713d..13b765baa7 100644 --- a/kafka-connect-elastic6/src/main/scala/io/lenses/streamreactor/connect/elastic6/config/ElasticSettings.scala +++ b/kafka-connect-elastic6/src/main/scala/io/lenses/streamreactor/connect/elastic6/config/ElasticSettings.scala @@ -17,6 +17,7 @@ package io.lenses.streamreactor.connect.elastic6.config import io.lenses.kcql.Kcql import io.lenses.streamreactor.common.errors.ErrorPolicy +import io.lenses.streamreactor.connect.security.StoresInfo /** * Created by andrew@datamountaineer.com on 13/05/16. @@ -31,6 +32,7 @@ case class ElasticSettings( pkJoinerSeparator: String = ElasticConfigConstants.PK_JOINER_SEPARATOR_DEFAULT, httpBasicAuthUsername: String = ElasticConfigConstants.CLIENT_HTTP_BASIC_AUTH_USERNAME_DEFAULT, httpBasicAuthPassword: String = ElasticConfigConstants.CLIENT_HTTP_BASIC_AUTH_USERNAME_DEFAULT, + storesInfo: StoresInfo = StoresInfo() ) object ElasticSettings { @@ -54,6 +56,7 @@ object ElasticSettings { pkJoinerSeparator, httpBasicAuthUsername, httpBasicAuthPassword, + StoresInfo(config) ) } } diff --git a/kafka-connect-elastic7/src/main/scala/io/lenses/streamreactor/connect/elastic7/config/ElasticConfig.scala b/kafka-connect-elastic7/src/main/scala/io/lenses/streamreactor/connect/elastic7/config/ElasticConfig.scala index 31237b4c6d..f1a542ca92 100644 --- a/kafka-connect-elastic7/src/main/scala/io/lenses/streamreactor/connect/elastic7/config/ElasticConfig.scala +++ b/kafka-connect-elastic7/src/main/scala/io/lenses/streamreactor/connect/elastic7/config/ElasticConfig.scala @@ -191,6 +191,7 @@ object ElasticConfig { ConfigDef.Width.MEDIUM, ElasticConfigConstants.PROGRESS_COUNTER_ENABLED_DISPLAY, ) + .withClientSslSupport() } /** diff --git a/kafka-connect-elastic7/src/main/scala/io/lenses/streamreactor/connect/elastic7/config/ElasticSettings.scala b/kafka-connect-elastic7/src/main/scala/io/lenses/streamreactor/connect/elastic7/config/ElasticSettings.scala index 83341edd3f..81d46a7fbd 100644 --- a/kafka-connect-elastic7/src/main/scala/io/lenses/streamreactor/connect/elastic7/config/ElasticSettings.scala +++ b/kafka-connect-elastic7/src/main/scala/io/lenses/streamreactor/connect/elastic7/config/ElasticSettings.scala @@ -17,6 +17,7 @@ package io.lenses.streamreactor.connect.elastic7.config import io.lenses.kcql.Kcql import io.lenses.streamreactor.common.errors.ErrorPolicy +import io.lenses.streamreactor.connect.security.StoresInfo /** * Created by andrew@datamountaineer.com on 13/05/16. @@ -31,7 +32,8 @@ case class ElasticSettings( pkJoinerSeparator: String = ElasticConfigConstants.PK_JOINER_SEPARATOR_DEFAULT, httpBasicAuthUsername: String = ElasticConfigConstants.CLIENT_HTTP_BASIC_AUTH_USERNAME_DEFAULT, httpBasicAuthPassword: String = ElasticConfigConstants.CLIENT_HTTP_BASIC_AUTH_USERNAME_DEFAULT, -) + storesInfo: StoresInfo = StoresInfo() + ) object ElasticSettings { @@ -54,6 +56,7 @@ object ElasticSettings { pkJoinerSeparator, httpBasicAuthUsername, httpBasicAuthPassword, + StoresInfo(config) ) } } From cacbf2de3508fb21923d710caeabf71636f7e1b0 Mon Sep 17 00:00:00 2001 From: David Sloan Date: Tue, 3 Sep 2024 14:26:02 +0100 Subject: [PATCH 02/12] Key/trust store setup for ES6/7 aargh --- .../common/security/StoreInfo.java | 17 +++ .../common/security/StoresInfo.java | 118 ++++++++++++++++ .../connect/security/StoreInfo.scala | 18 +-- .../connect/security/KeyStoreUtils.scala | 126 ++++++++++++++++++ .../connect/security/StoresInfoTest.scala | 120 +++++++++++++++++ .../connect/elastic6/KElasticClient.scala | 40 +++--- .../elastic6/config/ElasticSettings.scala | 33 ++--- .../connect/elastic7/KElasticClient.scala | 38 +++--- .../elastic7/config/ElasticSettings.scala | 35 ++--- project/Dependencies.scala | 2 +- 10 files changed, 463 insertions(+), 84 deletions(-) create mode 100644 java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/StoreInfo.java create mode 100644 java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/StoresInfo.java create mode 100644 kafka-connect-common/src/test/scala/io/lenses/streamreactor/connect/security/KeyStoreUtils.scala create mode 100644 kafka-connect-common/src/test/scala/io/lenses/streamreactor/connect/security/StoresInfoTest.scala diff --git a/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/StoreInfo.java b/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/StoreInfo.java new file mode 100644 index 0000000000..09d0c136a8 --- /dev/null +++ b/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/StoreInfo.java @@ -0,0 +1,17 @@ +package io.lenses.streamreactor.common.security; + +import cyclops.control.Option; +import lombok.AllArgsConstructor; +import lombok.Getter; + +@AllArgsConstructor +@Getter +class StoreInfo { + + private String storePath; + + private Option storeType; + + private Option storePassword; + +} diff --git a/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/StoresInfo.java b/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/StoresInfo.java new file mode 100644 index 0000000000..09c8c68789 --- /dev/null +++ b/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/StoresInfo.java @@ -0,0 +1,118 @@ +package io.lenses.streamreactor.common.security; + +import cyclops.control.Either; +import cyclops.control.Option; +import cyclops.control.Try; +import cyclops.data.Seq; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.val; + +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManagerFactory; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.security.GeneralSecurityException; +import java.security.KeyStore; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.security.cert.CertificateException; + +import static java.util.function.Function.identity; + +@AllArgsConstructor +@Getter +public class StoresInfo { + private Option maybeTrustStore; + private Option maybeKeyStore; + + private Try getJksStore(String path, Option storeType, Option password) { + return Try.withCatch( + ()-> KeyStore.getInstance(storeType.map(String::toUpperCase).orElse("JKS")), + GeneralSecurityException.class + ) + .forEach3( + (KeyStore keyStore) -> Try.withCatch( + () -> new FileInputStream(path), + FileNotFoundException.class + ), + (KeyStore keyStore, FileInputStream truststoreStream) -> Try.runWithCatch( + () -> keyStore.load(truststoreStream, (password.orElse("")).toCharArray()), + IOException.class, + NoSuchAlgorithmException.class, + CertificateException.class + ), + (KeyStore keyStore, FileInputStream truststoreStream, Void ignore) -> Try.success(keyStore)); + + } + + public Either> toSslContext() { + + + val maybeTrustFactory = maybeTrustStore.map( + trustStore -> + trustManagers( + trustStore.getStorePath(), + trustStore.getStoreType(), + trustStore.getStorePassword() + ) + ); + + val maybeKeyFactory = maybeKeyStore.flatMap( + keyStore -> + keyManagers( + keyStore.getStorePath(), + keyStore.getStoreType(), + keyStore.getStorePassword() + ).toOption() + ); + + + val newSeq = Seq.of(maybeTrustFactory, maybeKeyFactory).flatMap(identity()); + if (.stream().flatMap(identity())) + + Option.when(maybeTrustFactory.nonEmpty || maybeKeyFactory.nonEmpty) { + val sslContext = SSLContext.getInstance("TLS") + sslContext.init( + maybeKeyFactory.map(_.getKeyManagers).orNull, + maybeTrustFactory.map(_.getTrustManagers).orNull, + null, + ) + sslContext + } + } + + + + + private Try trustManagers(String path, Option storeType, Option password) { + return getJksStore(path, storeType, password) + .forEach2( + (keyStore) -> + Try.withCatch( + () -> { + val tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + return tmf; + }, + GeneralSecurityException.class + ), + (keyStore, trustManagerFactory) -> + Try.withCatch( () -> { + trustManagerFactory.init(keyStore, GeneralSecurityException.class); + return trustManagerFactory; + }) + ); + + } + + private Either keyManagers(String path, Option storeType, Option password) { + val keyStore = getJksStore(path, storeType, password); + + val keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm) + keyManagerFactory.init(keyStore, (password.orElse("")).toCharArray()); + return keyManagerFactory; + } + +} diff --git a/kafka-connect-common/src/main/scala/io/lenses/streamreactor/connect/security/StoreInfo.scala b/kafka-connect-common/src/main/scala/io/lenses/streamreactor/connect/security/StoreInfo.scala index 96541108ce..6302f2a1ec 100644 --- a/kafka-connect-common/src/main/scala/io/lenses/streamreactor/connect/security/StoreInfo.scala +++ b/kafka-connect-common/src/main/scala/io/lenses/streamreactor/connect/security/StoreInfo.scala @@ -24,16 +24,10 @@ import javax.net.ssl.KeyManagerFactory import javax.net.ssl.SSLContext import javax.net.ssl.TrustManagerFactory -case class StoreInfo( - storePath: String, - storeType: Option[String], - storePassword: Option[String] = None, - ) - case class StoresInfo( - trustStore: Option[StoreInfo] = None, - keyStore: Option[StoreInfo] = None, - ) { + trustStore: Option[StoreInfo] = None, + keyStore: Option[StoreInfo] = None, +) { def toSslContext: Option[SSLContext] = { val maybeTrustFactory: Option[TrustManagerFactory] = trustStore.map { case StoreInfo(path, storeType, password) => @@ -72,7 +66,7 @@ case class StoresInfo( } private def getJksStore(path: String, storeType: Option[String], password: Option[String]) = { - val keyStore = KeyStore.getInstance(storeType.map(_.toUpperCase).getOrElse("JKS")) + val keyStore = KeyStore.getInstance(storeType.map(_.toUpperCase).getOrElse("JKS")) val truststoreStream = new FileInputStream(path) keyStore.load(truststoreStream, password.getOrElse("").toCharArray) keyStore @@ -90,8 +84,8 @@ object StoresInfo { storePath <- Option(config.getString(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG)) storeType = Option(config.getString(SslConfigs.SSL_KEYSTORE_TYPE_CONFIG)) storePassword = Option(config.getPassword(SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG)).map(_.value()) - } yield StoreInfo(storePath, storeType, storePassword) + } yield new StoreInfo(storePath, storeType, storePassword) StoresInfo(trustStore, keyStore) } -} \ No newline at end of file +} diff --git a/kafka-connect-common/src/test/scala/io/lenses/streamreactor/connect/security/KeyStoreUtils.scala b/kafka-connect-common/src/test/scala/io/lenses/streamreactor/connect/security/KeyStoreUtils.scala new file mode 100644 index 0000000000..6b23cf034f --- /dev/null +++ b/kafka-connect-common/src/test/scala/io/lenses/streamreactor/connect/security/KeyStoreUtils.scala @@ -0,0 +1,126 @@ +/* + * Copyright 2017-2024 Lenses.io Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.lenses.streamreactor.connect.security + +import com.typesafe.scalalogging.LazyLogging +import org.bouncycastle.asn1.x500.X500Name +import org.bouncycastle.asn1.x509.SubjectPublicKeyInfo +import org.bouncycastle.cert.X509v3CertificateBuilder +import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter +import org.bouncycastle.jce.provider.BouncyCastleProvider +import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder + +import java.io.FileOutputStream +import java.math.BigInteger +import java.nio.file.Files +import java.nio.file.Path +import java.security.cert.X509Certificate +import java.security.interfaces.RSAPrivateKey +import java.security.KeyPairGenerator +import java.security.KeyStore +import java.security.Security +import java.util.Date + +object KeyStoreUtils extends LazyLogging { + + Security.addProvider(new BouncyCastleProvider()) + + def createKeystore( + commonName: String, + keyStorePassword: String, + trustStorePassword: String, + ): Path = { + + val tmpDir: Path = Files.createTempDirectory("security") + + val (certificate, privateKey) = KeyStoreUtils.generateSelfSignedCertificate(2048, 365, commonName) + val _ = KeyStoreUtils.createAndSaveKeystore(tmpDir, keyStorePassword, certificate, privateKey) + val _ = KeyStoreUtils.createAndSaveTruststore(tmpDir, trustStorePassword, certificate) + logger.info(s"container -> Creating keystore at $tmpDir") + tmpDir + } + + def generateSelfSignedCertificate( + keySize: Int, + certificateValidityDays: Int, + commonName: String, + ): (X509Certificate, RSAPrivateKey) = { + val keyPairGen = KeyPairGenerator.getInstance("RSA", "BC") + keyPairGen.initialize(keySize) + val keyPair = keyPairGen.generateKeyPair() + + val notBefore = new Date() + val notAfter = new Date(System.currentTimeMillis() + certificateValidityDays * 24L * 60 * 60 * 1000) + + val publicKeyInfo = SubjectPublicKeyInfo.getInstance(keyPair.getPublic.getEncoded) + + val certBuilder = new X509v3CertificateBuilder( + new X500Name(s"CN=$commonName"), + BigInteger.valueOf(System.currentTimeMillis()), + notBefore, + notAfter, + new X500Name(s"CN=$commonName"), + publicKeyInfo, + ) + + val contentSigner = + new JcaContentSignerBuilder("SHA256WithRSAEncryption").setProvider("BC").build(keyPair.getPrivate) + val certHolder = certBuilder.build(contentSigner) + val cert = new JcaX509CertificateConverter().setProvider("BC").getCertificate(certHolder) + + (cert, keyPair.getPrivate.asInstanceOf[RSAPrivateKey]) + } + + def createAndSaveKeystore( + tmpDir: Path, + password: String, + certificate: X509Certificate, + privateKey: RSAPrivateKey, + ): String = { + + val keyStore = KeyStore.getInstance("JKS") + keyStore.load(null, password.toCharArray) + + // Store the private key and certificate in the keystore + keyStore.setKeyEntry("alias", privateKey, password.toCharArray, Array(certificate)) + + val keyStorePath = tmpDir.resolve("keystore.jks").toString + // Save the keystore to a file + val keystoreOutputStream = new FileOutputStream(keyStorePath) + keyStore.store(keystoreOutputStream, password.toCharArray) + keystoreOutputStream.close() + + keyStorePath + } + + def createAndSaveTruststore(tmpDir: Path, password: String, certificate: X509Certificate): String = { + + val trustStore = KeyStore.getInstance("JKS") + trustStore.load(null, password.toCharArray) + + // Add the trusted certificate to the truststore + trustStore.setCertificateEntry("alias", certificate) + val trustStorePath = tmpDir.resolve("truststore.jks").toString + + // Save the truststore to a file + val truststoreOutputStream = new FileOutputStream(trustStorePath) + trustStore.store(truststoreOutputStream, password.toCharArray) + truststoreOutputStream.close() + + trustStorePath + } + +} diff --git a/kafka-connect-common/src/test/scala/io/lenses/streamreactor/connect/security/StoresInfoTest.scala b/kafka-connect-common/src/test/scala/io/lenses/streamreactor/connect/security/StoresInfoTest.scala new file mode 100644 index 0000000000..437bdc4616 --- /dev/null +++ b/kafka-connect-common/src/test/scala/io/lenses/streamreactor/connect/security/StoresInfoTest.scala @@ -0,0 +1,120 @@ +/* + * Copyright 2017-2024 Lenses.io Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.lenses.streamreactor.connect.security + +import io.lenses.streamreactor.common.config.base.traits.BaseConfig +import org.apache.kafka.common.config.SslConfigs +import org.mockito.MockitoSugar +import org.scalatest.BeforeAndAfterAll +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers + +import java.io.FileNotFoundException +import java.nio.file.Path + +class StoresInfoTest extends AnyFunSuite with Matchers with MockitoSugar with BeforeAndAfterAll { + + private val password = "changeIt" + private val keystoreDir: Path = KeyStoreUtils.createKeystore("TestCommonName", password, password) + private val keystoreFile = keystoreDir.toAbsolutePath.toString + "/keystore.jks" + private val truststoreFile = keystoreDir.toAbsolutePath.toString + "/truststore.jks" + + test("StoresInfo.toSslContext should return None when both trustStore and keyStore are None") { + val storesInfo = StoresInfo(None, None) + storesInfo.toSslContext should be(None) + } + + test("StoresInfo.toSslContext should return Some(SSLContext) when keyStore is defined") { + val storeInfo = StoreInfo(keystoreFile, Some("JKS"), Some(password)) + val storesInfo = StoresInfo(keyStore = Some(storeInfo)) + + val sslContext = storesInfo.toSslContext + + sslContext should not be empty + sslContext.get.getProtocol shouldEqual "TLS" + } + + test("StoresInfo.toSslContext should return Some(SSLContext) when trustStore is defined") { + val storeInfo = StoreInfo(keystoreFile, Some("JKS"), Some(password)) + val storesInfo = StoresInfo(trustStore = Some(storeInfo)) + + val sslContext = storesInfo.toSslContext + + sslContext should not be empty + sslContext.get.getProtocol shouldEqual "TLS" + } + + test("StoresInfo.toSslContext should return Some(SSLContext) when both keyStore and trustStore are defined") { + val keyStoreInfo = StoreInfo(keystoreFile, Some("JKS"), Some(password)) + val trustStoreInfo = StoreInfo(truststoreFile, Some("JKS"), Some(password)) + val storesInfo = StoresInfo(trustStore = Some(trustStoreInfo), keyStore = Some(keyStoreInfo)) + + val sslContext = storesInfo.toSslContext + + sslContext should not be empty + sslContext.get.getProtocol shouldEqual "TLS" + } + + test("StoresInfo.toSslContext should throw FileNotFoundException if the keyStore path is incorrect") { + val keyStoreInfo = StoreInfo("/invalid/path/to/keystore", Some("JKS"), Some(password)) + val storesInfo = StoresInfo(keyStore = Some(keyStoreInfo)) + + assertThrows[FileNotFoundException] { + storesInfo.toSslContext + } + } + + test("StoresInfo.toSslContext should throw FileNotFoundException if the trustStore path is incorrect") { + val trustStoreInfo = StoreInfo("/invalid/path/to/truststore", Some("JKS"), Some(password)) + val storesInfo = StoresInfo(trustStore = Some(trustStoreInfo)) + + assertThrows[FileNotFoundException] { + storesInfo.toSslContext + } + } + + test("StoresInfo.apply should create StoresInfo with correct values from BaseConfig") { + val mockConfig: BaseConfig = mock[BaseConfig] + when(mockConfig.getString(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG)).thenReturn("/path/to/truststore") + when(mockConfig.getString(SslConfigs.SSL_TRUSTSTORE_TYPE_CONFIG)).thenReturn("JKS") + when(mockConfig.getPassword(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG)).thenReturn(null) + + when(mockConfig.getString(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG)).thenReturn("/path/to/keystore") + when(mockConfig.getString(SslConfigs.SSL_KEYSTORE_TYPE_CONFIG)).thenReturn("JKS") + when(mockConfig.getPassword(SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG)).thenReturn(null) + + val storesInfo = StoresInfo(mockConfig) + + storesInfo.trustStore shouldEqual Some(StoreInfo("/path/to/truststore", Some("JKS"), None)) + storesInfo.keyStore shouldEqual Some(StoreInfo("/path/to/keystore", Some("JKS"), None)) + } + + test("StoresInfo.apply should create StoresInfo with None values if configs are missing") { + val mockConfig: BaseConfig = mock[BaseConfig] + when(mockConfig.getString(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG)).thenReturn(null) + when(mockConfig.getString(SslConfigs.SSL_TRUSTSTORE_TYPE_CONFIG)).thenReturn(null) + when(mockConfig.getPassword(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG)).thenReturn(null) + + when(mockConfig.getString(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG)).thenReturn(null) + when(mockConfig.getString(SslConfigs.SSL_KEYSTORE_TYPE_CONFIG)).thenReturn(null) + when(mockConfig.getPassword(SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG)).thenReturn(null) + + val storesInfo = StoresInfo(mockConfig) + + storesInfo.trustStore shouldEqual None + storesInfo.keyStore shouldEqual None + } +} diff --git a/kafka-connect-elastic6/src/main/scala/io/lenses/streamreactor/connect/elastic6/KElasticClient.scala b/kafka-connect-elastic6/src/main/scala/io/lenses/streamreactor/connect/elastic6/KElasticClient.scala index e20f5e487c..8546d28560 100644 --- a/kafka-connect-elastic6/src/main/scala/io/lenses/streamreactor/connect/elastic6/KElasticClient.scala +++ b/kafka-connect-elastic6/src/main/scala/io/lenses/streamreactor/connect/elastic6/KElasticClient.scala @@ -25,9 +25,9 @@ import io.lenses.streamreactor.connect.elastic6.config.ElasticSettings import io.lenses.streamreactor.connect.elastic6.indexname.CreateIndex.getIndexNameForAutoCreate import org.apache.http.auth.AuthScope import org.apache.http.auth.UsernamePasswordCredentials +import org.apache.http.client.config.RequestConfig.Builder import org.apache.http.impl.client.BasicCredentialsProvider import org.apache.http.impl.nio.client.HttpAsyncClientBuilder -import org.elasticsearch.client.RestClientBuilder.HttpClientConfigCallback import scala.concurrent.Future @@ -39,29 +39,29 @@ trait KElasticClient extends AutoCloseable { object KElasticClient extends StrictLogging { - def createHttpClient(settings: ElasticSettings, endpoints: Seq[ElasticNodeEndpoint]): KElasticClient = - if (settings.httpBasicAuthUsername.nonEmpty && settings.httpBasicAuthPassword.nonEmpty) { - lazy val provider = { - val provider = new BasicCredentialsProvider - val credentials = - new UsernamePasswordCredentials(settings.httpBasicAuthUsername, settings.httpBasicAuthPassword) + def createHttpClient(settings: ElasticSettings, endpoints: Seq[ElasticNodeEndpoint]): KElasticClient = { + val maybeProvider: Option[BasicCredentialsProvider] = { + for { + httpBasicAuthUsername <- Option.when(settings.httpBasicAuthUsername.nonEmpty)(settings.httpBasicAuthUsername) + httpBasicAuthPassword <- Option.when(settings.httpBasicAuthPassword.nonEmpty)(settings.httpBasicAuthPassword) + } yield { + val credentials = new UsernamePasswordCredentials(httpBasicAuthUsername, httpBasicAuthPassword) + val provider = new BasicCredentialsProvider provider.setCredentials(AuthScope.ANY, credentials) provider } - val callback = new HttpClientConfigCallback { - override def customizeHttpClient(httpClientBuilder: HttpAsyncClientBuilder): HttpAsyncClientBuilder = - httpClientBuilder.setDefaultCredentialsProvider(provider) - } - val client: ElasticClient = ElasticClient( - ElasticProperties(endpoints), - requestConfigCallback = NoOpRequestConfigCallback, - httpClientConfigCallback = callback, - ) - new HttpKElasticClient(client) - } else { - val client: ElasticClient = ElasticClient(ElasticProperties(endpoints)) - new HttpKElasticClient(client) } + val client: ElasticClient = ElasticClient( + ElasticProperties(endpoints), + (requestConfigBuilder: Builder) => requestConfigBuilder, + (httpClientBuilder: HttpAsyncClientBuilder) => { + maybeProvider.foreach(httpClientBuilder.setDefaultCredentialsProvider) + settings.storesInfo.toSslContext.foreach(httpClientBuilder.setSSLContext) + httpClientBuilder + }, + ) + new HttpKElasticClient(client) + } } class HttpKElasticClient(client: ElasticClient) extends KElasticClient { diff --git a/kafka-connect-elastic6/src/main/scala/io/lenses/streamreactor/connect/elastic6/config/ElasticSettings.scala b/kafka-connect-elastic6/src/main/scala/io/lenses/streamreactor/connect/elastic6/config/ElasticSettings.scala index 13b765baa7..05ef47fae1 100644 --- a/kafka-connect-elastic6/src/main/scala/io/lenses/streamreactor/connect/elastic6/config/ElasticSettings.scala +++ b/kafka-connect-elastic6/src/main/scala/io/lenses/streamreactor/connect/elastic6/config/ElasticSettings.scala @@ -26,13 +26,13 @@ import io.lenses.streamreactor.connect.security.StoresInfo case class ElasticSettings( kcqls: Seq[Kcql], errorPolicy: ErrorPolicy, - taskRetries: Int = ElasticConfigConstants.NBR_OF_RETIRES_DEFAULT, - writeTimeout: Int = ElasticConfigConstants.WRITE_TIMEOUT_DEFAULT, - batchSize: Int = ElasticConfigConstants.BATCH_SIZE_DEFAULT, - pkJoinerSeparator: String = ElasticConfigConstants.PK_JOINER_SEPARATOR_DEFAULT, - httpBasicAuthUsername: String = ElasticConfigConstants.CLIENT_HTTP_BASIC_AUTH_USERNAME_DEFAULT, - httpBasicAuthPassword: String = ElasticConfigConstants.CLIENT_HTTP_BASIC_AUTH_USERNAME_DEFAULT, - storesInfo: StoresInfo = StoresInfo() + taskRetries: Int = ElasticConfigConstants.NBR_OF_RETIRES_DEFAULT, + writeTimeout: Int = ElasticConfigConstants.WRITE_TIMEOUT_DEFAULT, + batchSize: Int = ElasticConfigConstants.BATCH_SIZE_DEFAULT, + pkJoinerSeparator: String = ElasticConfigConstants.PK_JOINER_SEPARATOR_DEFAULT, + httpBasicAuthUsername: String = ElasticConfigConstants.CLIENT_HTTP_BASIC_AUTH_USERNAME_DEFAULT, + httpBasicAuthPassword: String = ElasticConfigConstants.CLIENT_HTTP_BASIC_AUTH_USERNAME_DEFAULT, + storesInfo: StoresInfo = StoresInfo(), ) object ElasticSettings { @@ -48,15 +48,16 @@ object ElasticSettings { val batchSize = config.getInt(ElasticConfigConstants.BATCH_SIZE_CONFIG) - ElasticSettings(kcql, - errorPolicy, - retries, - writeTimeout, - batchSize, - pkJoinerSeparator, - httpBasicAuthUsername, - httpBasicAuthPassword, - StoresInfo(config) + ElasticSettings( + kcql, + errorPolicy, + retries, + writeTimeout, + batchSize, + pkJoinerSeparator, + httpBasicAuthUsername, + httpBasicAuthPassword, + StoresInfo(config), ) } } diff --git a/kafka-connect-elastic7/src/main/scala/io/lenses/streamreactor/connect/elastic7/KElasticClient.scala b/kafka-connect-elastic7/src/main/scala/io/lenses/streamreactor/connect/elastic7/KElasticClient.scala index 3733ab34cf..9ed8899cb0 100644 --- a/kafka-connect-elastic7/src/main/scala/io/lenses/streamreactor/connect/elastic7/KElasticClient.scala +++ b/kafka-connect-elastic7/src/main/scala/io/lenses/streamreactor/connect/elastic7/KElasticClient.scala @@ -43,28 +43,31 @@ trait KElasticClient extends AutoCloseable { object KElasticClient extends StrictLogging { - def createHttpClient(settings: ElasticSettings, endpoints: Seq[ElasticNodeEndpoint]): KElasticClient = - if (settings.httpBasicAuthUsername.nonEmpty && settings.httpBasicAuthPassword.nonEmpty) { - lazy val provider = { - val provider = new BasicCredentialsProvider - val credentials = - new UsernamePasswordCredentials(settings.httpBasicAuthUsername, settings.httpBasicAuthPassword) + def createHttpClient(settings: ElasticSettings, endpoints: Seq[ElasticNodeEndpoint]): KElasticClient = { + val maybeProvider: Option[BasicCredentialsProvider] = { + for { + httpBasicAuthUsername <- Option.when(settings.httpBasicAuthUsername.nonEmpty)(settings.httpBasicAuthUsername) + httpBasicAuthPassword <- Option.when(settings.httpBasicAuthPassword.nonEmpty)(settings.httpBasicAuthPassword) + } yield { + val credentials = new UsernamePasswordCredentials(httpBasicAuthUsername, httpBasicAuthPassword) + val provider = new BasicCredentialsProvider provider.setCredentials(AuthScope.ANY, credentials) provider } - - val javaClient = JavaClient( + } + val client: ElasticClient = ElasticClient( + JavaClient( ElasticProperties(endpoints), (requestConfigBuilder: Builder) => requestConfigBuilder, - (httpClientBuilder: HttpAsyncClientBuilder) => httpClientBuilder.setDefaultCredentialsProvider(provider), - ) - - val client: ElasticClient = ElasticClient(javaClient) - new HttpKElasticClient(client) - } else { - val client: ElasticClient = ElasticClient(JavaClient(ElasticProperties(endpoints))) - new HttpKElasticClient(client) - } + (httpClientBuilder: HttpAsyncClientBuilder) => { + maybeProvider.foreach(httpClientBuilder.setDefaultCredentialsProvider) + settings.storesInfo.toSslContext.foreach(httpClientBuilder.setSSLContext) + httpClientBuilder + }, + ), + ) + new HttpKElasticClient(client) + } } class HttpKElasticClient(client: ElasticClient) extends KElasticClient { @@ -80,7 +83,6 @@ class HttpKElasticClient(client: ElasticClient) extends KElasticClient { createIndex(indexName) } } - () } diff --git a/kafka-connect-elastic7/src/main/scala/io/lenses/streamreactor/connect/elastic7/config/ElasticSettings.scala b/kafka-connect-elastic7/src/main/scala/io/lenses/streamreactor/connect/elastic7/config/ElasticSettings.scala index 81d46a7fbd..ccc96957ec 100644 --- a/kafka-connect-elastic7/src/main/scala/io/lenses/streamreactor/connect/elastic7/config/ElasticSettings.scala +++ b/kafka-connect-elastic7/src/main/scala/io/lenses/streamreactor/connect/elastic7/config/ElasticSettings.scala @@ -26,14 +26,14 @@ import io.lenses.streamreactor.connect.security.StoresInfo case class ElasticSettings( kcqls: Seq[Kcql], errorPolicy: ErrorPolicy, - taskRetries: Int = ElasticConfigConstants.NBR_OF_RETIRES_DEFAULT, - writeTimeout: Int = ElasticConfigConstants.WRITE_TIMEOUT_DEFAULT, - batchSize: Int = ElasticConfigConstants.BATCH_SIZE_DEFAULT, - pkJoinerSeparator: String = ElasticConfigConstants.PK_JOINER_SEPARATOR_DEFAULT, - httpBasicAuthUsername: String = ElasticConfigConstants.CLIENT_HTTP_BASIC_AUTH_USERNAME_DEFAULT, - httpBasicAuthPassword: String = ElasticConfigConstants.CLIENT_HTTP_BASIC_AUTH_USERNAME_DEFAULT, - storesInfo: StoresInfo = StoresInfo() - ) + taskRetries: Int = ElasticConfigConstants.NBR_OF_RETIRES_DEFAULT, + writeTimeout: Int = ElasticConfigConstants.WRITE_TIMEOUT_DEFAULT, + batchSize: Int = ElasticConfigConstants.BATCH_SIZE_DEFAULT, + pkJoinerSeparator: String = ElasticConfigConstants.PK_JOINER_SEPARATOR_DEFAULT, + httpBasicAuthUsername: String = ElasticConfigConstants.CLIENT_HTTP_BASIC_AUTH_USERNAME_DEFAULT, + httpBasicAuthPassword: String = ElasticConfigConstants.CLIENT_HTTP_BASIC_AUTH_USERNAME_DEFAULT, + storesInfo: StoresInfo = StoresInfo(), +) object ElasticSettings { @@ -48,15 +48,16 @@ object ElasticSettings { val batchSize = config.getInt(ElasticConfigConstants.BATCH_SIZE_CONFIG) - ElasticSettings(kcql, - errorPolicy, - retries, - writeTimeout, - batchSize, - pkJoinerSeparator, - httpBasicAuthUsername, - httpBasicAuthPassword, - StoresInfo(config) + ElasticSettings( + kcql, + errorPolicy, + retries, + writeTimeout, + batchSize, + pkJoinerSeparator, + httpBasicAuthUsername, + httpBasicAuthPassword, + StoresInfo(config), ) } } diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 26042528e2..54756aab21 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -437,7 +437,7 @@ trait Dependencies { jerseyCommon, avro4s, kafkaClients, - ) ++ enumeratum ++ circe ++ http4s + ) ++ enumeratum ++ circe ++ http4s ++ bouncyCastle //Specific modules dependencies val sqlCommonDeps: Seq[ModuleID] = loggingDeps ++ Seq( From 781dd61e7777ad4aed1670cf5bfd917098b0265b Mon Sep 17 00:00:00 2001 From: David Sloan Date: Thu, 5 Sep 2024 15:17:37 +0100 Subject: [PATCH 03/12] ES6/7 SSL Support --- build.sbt | 3 +- java-connectors/build.gradle | 2 + .../kafka-connect-common/build.gradle | 6 + .../common/security/StoreInfo.java | 19 +- .../common/security/StoresInfo.java | 246 ++++++++++++------ .../common/security/KeyStoreUtils.java | 124 +++++++++ .../common/security/StoresInfoTest.java | 141 ++++++++++ .../common/utils/CyclopsToScalaOption.scala | 42 +++ .../connect/security/StoreInfo.scala | 91 ------- .../connect/security/KeyStoreUtils.scala | 126 --------- .../connect/security/StoresInfoTest.scala | 120 --------- .../connect/elastic6/KElasticClient.scala | 8 +- .../elastic6/config/ElasticSettings.scala | 7 +- .../connect/elastic7/KElasticClient.scala | 8 +- .../elastic7/config/ElasticSettings.scala | 7 +- project/Dependencies.scala | 8 +- 16 files changed, 521 insertions(+), 437 deletions(-) create mode 100644 java-connectors/kafka-connect-common/src/test/java/io/lenses/streamreactor/common/security/KeyStoreUtils.java create mode 100644 java-connectors/kafka-connect-common/src/test/java/io/lenses/streamreactor/common/security/StoresInfoTest.java create mode 100644 kafka-connect-common/src/main/scala/io/lenses/streamreactor/common/utils/CyclopsToScalaOption.scala delete mode 100644 kafka-connect-common/src/main/scala/io/lenses/streamreactor/connect/security/StoreInfo.scala delete mode 100644 kafka-connect-common/src/test/scala/io/lenses/streamreactor/connect/security/KeyStoreUtils.scala delete mode 100644 kafka-connect-common/src/test/scala/io/lenses/streamreactor/connect/security/StoresInfoTest.scala diff --git a/build.sbt b/build.sbt index b8264c77db..27a59589a8 100644 --- a/build.sbt +++ b/build.sbt @@ -1,4 +1,5 @@ import Dependencies.Versions +import Dependencies.`cyclopsPure` import Dependencies.`cyclops` import Dependencies.`lombok` import Dependencies.globalExcludeDeps @@ -71,7 +72,7 @@ lazy val `query-language` = (project in file("java-connectors/kafka-connect-quer Seq( name := "kafka-connect-query-language", description := "Kafka Connect compatible connectors to move data between Kafka and popular data stores", - libraryDependencies ++= Seq(cyclops, lombok), + libraryDependencies ++= Seq(cyclops, cyclopsPure, lombok), publish / skip := true, ), ) diff --git a/java-connectors/build.gradle b/java-connectors/build.gradle index fc2c6fdacf..f4e909470a 100644 --- a/java-connectors/build.gradle +++ b/java-connectors/build.gradle @@ -31,6 +31,7 @@ allprojects { apacheToConfluentVersionAxis = ["2.8.1": "6.2.2", "3.3.0": "7.3.1"] caffeineVersion = '3.1.8' cyclopsVersion = '10.4.1' + bouncyCastleVersion = "1.78.1" //Other Manifest Info mainClassName = '' @@ -64,6 +65,7 @@ allprojects { // functional java implementation group: 'com.oath.cyclops', name: 'cyclops', version: cyclopsVersion + implementation group: 'com.oath.cyclops', name: 'cyclops-pure', version: cyclopsVersion //tests testImplementation group: 'org.mockito', name: 'mockito-core', version: mockitoJupiterVersion diff --git a/java-connectors/kafka-connect-common/build.gradle b/java-connectors/kafka-connect-common/build.gradle index 800c506491..65456e61b6 100644 --- a/java-connectors/kafka-connect-common/build.gradle +++ b/java-connectors/kafka-connect-common/build.gradle @@ -12,6 +12,12 @@ project(":kafka-connect-common") { api group: 'org.apache.kafka', name: 'kafka-clients', version: kafkaVersion testImplementation(project(path: ':test-utils', configuration: 'testArtifacts')) + testImplementation group: 'org.bouncycastle', name:'bcprov-jdk18on', version: bouncyCastleVersion + testImplementation group: 'org.bouncycastle', name:'bcutil-jdk18on', version: bouncyCastleVersion + testImplementation group: 'org.bouncycastle', name:'bcpkix-jdk18on', version: bouncyCastleVersion + testImplementation group: 'org.bouncycastle', name:'bcpg-jdk18on', version: bouncyCastleVersion + testImplementation group: 'org.bouncycastle', name:'bctls-jdk18on', version: bouncyCastleVersion + //confluent - may be needed soon // implementation group: 'io.confluent', name: 'kafka-json-schema-serializer', version: apacheToConfluentVersionAxis.get(kafkaVersion) // implementation group: 'io.confluent', name: 'kafka-connect-avro-converter', version: apacheToConfluentVersionAxis.get(kafkaVersion) diff --git a/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/StoreInfo.java b/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/StoreInfo.java index 09d0c136a8..137e78da56 100644 --- a/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/StoreInfo.java +++ b/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/StoreInfo.java @@ -1,11 +1,26 @@ +/* + * Copyright 2017-2024 Lenses.io Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.lenses.streamreactor.common.security; import cyclops.control.Option; import lombok.AllArgsConstructor; -import lombok.Getter; +import lombok.Data; @AllArgsConstructor -@Getter +@Data class StoreInfo { private String storePath; diff --git a/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/StoresInfo.java b/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/StoresInfo.java index 09c8c68789..314aa2157f 100644 --- a/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/StoresInfo.java +++ b/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/StoresInfo.java @@ -1,118 +1,192 @@ +/* + * Copyright 2017-2024 Lenses.io Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.lenses.streamreactor.common.security; import cyclops.control.Either; import cyclops.control.Option; import cyclops.control.Try; -import cyclops.data.Seq; +import cyclops.instances.control.TryInstances; +import cyclops.typeclasses.Do; import lombok.AllArgsConstructor; -import lombok.Getter; +import lombok.Data; import lombok.val; +import org.apache.kafka.common.config.AbstractConfig; +import org.apache.kafka.common.config.SslConfigs; +import org.apache.kafka.common.config.types.Password; import javax.net.ssl.KeyManagerFactory; import javax.net.ssl.SSLContext; import javax.net.ssl.TrustManagerFactory; import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.security.GeneralSecurityException; import java.security.KeyStore; import java.security.KeyStoreException; import java.security.NoSuchAlgorithmException; -import java.security.cert.CertificateException; - -import static java.util.function.Function.identity; +import java.security.UnrecoverableKeyException; @AllArgsConstructor -@Getter +@Data public class StoresInfo { - private Option maybeTrustStore; - private Option maybeKeyStore; - - private Try getJksStore(String path, Option storeType, Option password) { - return Try.withCatch( - ()-> KeyStore.getInstance(storeType.map(String::toUpperCase).orElse("JKS")), - GeneralSecurityException.class - ) - .forEach3( - (KeyStore keyStore) -> Try.withCatch( - () -> new FileInputStream(path), - FileNotFoundException.class - ), - (KeyStore keyStore, FileInputStream truststoreStream) -> Try.runWithCatch( - () -> keyStore.load(truststoreStream, (password.orElse("")).toCharArray()), - IOException.class, - NoSuchAlgorithmException.class, - CertificateException.class - ), - (KeyStore keyStore, FileInputStream truststoreStream, Void ignore) -> Try.success(keyStore)); - - } - public Either> toSslContext() { - - - val maybeTrustFactory = maybeTrustStore.map( - trustStore -> - trustManagers( - trustStore.getStorePath(), - trustStore.getStoreType(), - trustStore.getStorePassword() - ) + private Option maybeTrustStore; + private Option maybeKeyStore; + + private Try getJksStore(String path, Option storeType, Option password) { + return Try.withCatch( + () -> { + val keyStore = getKeyStoreInstance(storeType); + val inputStream = new FileInputStream(path); + loadKeyStore(password, keyStore, inputStream); + return keyStore; + }, + Exception.class + ); + + } + + private static void loadKeyStore(Option password, KeyStore keyStore, FileInputStream truststoreStream) + throws Exception { + keyStore.load(truststoreStream, (password.orElse("")).toCharArray()); + } + + private static KeyStore getKeyStoreInstance(Option storeType) throws Exception { + return KeyStore.getInstance(storeType.map(String::toUpperCase).orElse("JKS")); + } + + public Either> toSslContext() { + + final Option> maybeTrustFactory = + maybeTrustStore.map( + trustStore -> trustManagers( + trustStore.getStorePath(), + trustStore.getStoreType(), + trustStore.getStorePassword() + ) ); - val maybeKeyFactory = maybeKeyStore.flatMap( - keyStore -> - keyManagers( - keyStore.getStorePath(), - keyStore.getStoreType(), - keyStore.getStorePassword() - ).toOption() + final Option> maybeKeyFactory = + maybeKeyStore.map( + keyStore -> keyManagers( + keyStore.getStorePath(), + keyStore.getStoreType(), + keyStore.getStorePassword() + ) ); + if (maybeTrustFactory.isPresent() || maybeKeyFactory.isPresent()) { - val newSeq = Seq.of(maybeTrustFactory, maybeKeyFactory).flatMap(identity()); - if (.stream().flatMap(identity())) - - Option.when(maybeTrustFactory.nonEmpty || maybeKeyFactory.nonEmpty) { - val sslContext = SSLContext.getInstance("TLS") - sslContext.init( - maybeKeyFactory.map(_.getKeyManagers).orNull, - maybeTrustFactory.map(_.getTrustManagers).orNull, - null, - ) - sslContext - } - } - + val trustFailure = maybeTrustFactory.filter(Try::isFailure).flatMap(Try::failureGet); + val keyFailure = maybeKeyFactory.filter(Try::isFailure).flatMap(Try::failureGet); + if (trustFailure.isPresent() || keyFailure.isPresent()) { + return Either.left(trustFailure.orElse(keyFailure.orElse(new IllegalStateException( + "Logic error retrieving trust factories")))); + } else { + val trustSuccess = maybeTrustFactory.filter(Try::isSuccess).flatMap(Try::get); + val keySuccess = maybeKeyFactory.filter(Try::isSuccess).flatMap(Try::get); - private Try trustManagers(String path, Option storeType, Option password) { - return getJksStore(path, storeType, password) - .forEach2( - (keyStore) -> - Try.withCatch( - () -> { - val tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); - return tmf; - }, - GeneralSecurityException.class - ), - (keyStore, trustManagerFactory) -> - Try.withCatch( () -> { - trustManagerFactory.init(keyStore, GeneralSecurityException.class); - return trustManagerFactory; - }) - ); - - } + return Try.withCatch(() -> { + val sslContext = SSLContext.getInstance("TLS"); + sslContext.init( + keySuccess.map(KeyManagerFactory::getKeyManagers).orElse(null), + trustSuccess.map(TrustManagerFactory::getTrustManagers).orElse(null), + null + ); + return sslContext; + } + ).toEither().bimap(l -> l, Option::some); - private Either keyManagers(String path, Option storeType, Option password) { - val keyStore = getJksStore(path, storeType, password); + } - val keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm) - keyManagerFactory.init(keyStore, (password.orElse("")).toCharArray()); - return keyManagerFactory; + } else { + return Either.right(Option.none()); } + } + + private Try trustManagers(String path, Option storeType, + Option password) { + return Try.narrowK( + Do.forEach( + TryInstances.monad() + ) + .__(getJksStore(path, storeType, password)) + .__((KeyStore keyStore) -> Try.withCatch( + () -> getTrustManagerFactoryFromKeyStore(keyStore), + Exception.class + ) + ) + .yield( + (KeyStore keyStore, TrustManagerFactory trustManagerFactory) -> trustManagerFactory + ) + .unwrap()); + + } + + private Try keyManagers(String path, Option storeType, + Option password) { + return Try.narrowK( + Do.forEach( + TryInstances.monad() + ) + .__(getJksStore(path, storeType, password)) + .__((KeyStore keyStore) -> Try.withCatch( + () -> getKeyManagerFactoryFromKeyStore(keyStore, password), + Exception.class + ) + ) + .yield( + (KeyStore keyStore, KeyManagerFactory trustManagerFactory) -> trustManagerFactory + ) + .unwrap()); + } + + private static TrustManagerFactory getTrustManagerFactoryFromKeyStore(KeyStore keyStore) + throws NoSuchAlgorithmException, KeyStoreException { + val trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + trustManagerFactory.init(keyStore); + return trustManagerFactory; + } + + private static KeyManagerFactory getKeyManagerFactoryFromKeyStore(KeyStore keyStore, Option password) + throws NoSuchAlgorithmException, KeyStoreException, UnrecoverableKeyException { + val keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); + keyManagerFactory.init(keyStore, (password.orElse("")).toCharArray()); + return keyManagerFactory; + } + + public static StoresInfo fromConfig(AbstractConfig config) { + val trustStore = + Option.fromNullable(config.getString(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG)) + .map(storePath -> { + val storeType = Option.fromNullable(config.getString(SslConfigs.SSL_TRUSTSTORE_TYPE_CONFIG)); + val storePassword = + Option.fromNullable(config.getPassword(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG)) + .map(Password::value); + return new StoreInfo(storePath, storeType, storePassword); + }); + val keyStore = + Option.fromNullable(config.getString(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG)) + .map(storePath -> { + val storeType = Option.fromNullable(config.getString(SslConfigs.SSL_KEYSTORE_TYPE_CONFIG)); + val storePassword = + Option.fromNullable(config.getPassword(SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG)).map(Password::value); + return new StoreInfo(storePath, storeType, storePassword); + }); + + return new StoresInfo(trustStore, keyStore); + } } diff --git a/java-connectors/kafka-connect-common/src/test/java/io/lenses/streamreactor/common/security/KeyStoreUtils.java b/java-connectors/kafka-connect-common/src/test/java/io/lenses/streamreactor/common/security/KeyStoreUtils.java new file mode 100644 index 0000000000..2d8a793569 --- /dev/null +++ b/java-connectors/kafka-connect-common/src/test/java/io/lenses/streamreactor/common/security/KeyStoreUtils.java @@ -0,0 +1,124 @@ +/* + * Copyright 2017-2024 Lenses.io Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.lenses.streamreactor.common.security; +/* + * Copyright 2017-2024 Lenses.io Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.bouncycastle.asn1.x500.X500Name; +import org.bouncycastle.asn1.x509.SubjectPublicKeyInfo; +import org.bouncycastle.cert.X509v3CertificateBuilder; +import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter; +import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder; + +import java.io.FileOutputStream; +import java.math.BigInteger; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.security.KeyStore; +import java.security.Security; +import java.security.cert.X509Certificate; +import java.security.interfaces.RSAPrivateKey; +import java.util.Date; + +public class KeyStoreUtils { + + static { + Security.addProvider(new BouncyCastleProvider()); + } + + public static Path createKeystore(String commonName, String keyStorePassword, String trustStorePassword) + throws Exception { + Path tmpDir = Files.createTempDirectory("security"); + + KeyPairGenerator keyPairGen = KeyPairGenerator.getInstance("RSA", "BC"); + keyPairGen.initialize(2048); + KeyPair keyPair = keyPairGen.generateKeyPair(); + + Date notBefore = new Date(); + Date notAfter = new Date(System.currentTimeMillis() + 365L * 24 * 60 * 60 * 1000); + + SubjectPublicKeyInfo publicKeyInfo = SubjectPublicKeyInfo.getInstance(keyPair.getPublic().getEncoded()); + + X509v3CertificateBuilder certBuilder = + new X509v3CertificateBuilder( + new X500Name("CN=" + commonName), + BigInteger.valueOf(System.currentTimeMillis()), + notBefore, + notAfter, + new X500Name("CN=" + commonName), + publicKeyInfo + ); + + JcaContentSignerBuilder contentSignerBuilder = + new JcaContentSignerBuilder("SHA256WithRSAEncryption").setProvider("BC"); + X509Certificate certificate = + new JcaX509CertificateConverter().setProvider("BC") + .getCertificate(certBuilder.build(contentSignerBuilder.build(keyPair.getPrivate()))); + + createAndSaveKeystore(tmpDir, keyStorePassword, certificate, (RSAPrivateKey) keyPair.getPrivate()); + createAndSaveTruststore(tmpDir, trustStorePassword, certificate); + + System.out.println("container -> Creating keystore at " + tmpDir); + return tmpDir; + } + + private static String createAndSaveKeystore(Path tmpDir, String password, X509Certificate certificate, + RSAPrivateKey privateKey) throws Exception { + KeyStore keyStore = KeyStore.getInstance("JKS"); + keyStore.load(null, password.toCharArray()); + + keyStore.setKeyEntry("alias", privateKey, password.toCharArray(), new java.security.cert.Certificate[]{ + certificate}); + + String keyStorePath = tmpDir.resolve("keystore.jks").toString(); + try (FileOutputStream keyStoreOutputStream = new FileOutputStream(keyStorePath)) { + keyStore.store(keyStoreOutputStream, password.toCharArray()); + } + + return keyStorePath; + } + + private static String createAndSaveTruststore(Path tmpDir, String password, X509Certificate certificate) + throws Exception { + KeyStore trustStore = KeyStore.getInstance("JKS"); + trustStore.load(null, password.toCharArray()); + + trustStore.setCertificateEntry("alias", certificate); + String trustStorePath = tmpDir.resolve("truststore.jks").toString(); + + try (FileOutputStream trustStoreOutputStream = new FileOutputStream(trustStorePath)) { + trustStore.store(trustStoreOutputStream, password.toCharArray()); + } + + return trustStorePath; + } +} diff --git a/java-connectors/kafka-connect-common/src/test/java/io/lenses/streamreactor/common/security/StoresInfoTest.java b/java-connectors/kafka-connect-common/src/test/java/io/lenses/streamreactor/common/security/StoresInfoTest.java new file mode 100644 index 0000000000..a58727d0c3 --- /dev/null +++ b/java-connectors/kafka-connect-common/src/test/java/io/lenses/streamreactor/common/security/StoresInfoTest.java @@ -0,0 +1,141 @@ +/* + * Copyright 2017-2024 Lenses.io Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.lenses.streamreactor.common.security; + +import io.lenses.streamreactor.common.config.base.BaseConfig; +import lombok.val; +import org.apache.kafka.common.config.SslConfigs; +import org.junit.jupiter.api.Test; + +import java.io.FileNotFoundException; +import java.nio.file.Path; + +import static cyclops.control.Either.right; +import static cyclops.control.Option.none; +import static cyclops.control.Option.some; +import static io.lenses.streamreactor.test.utils.EitherValues.getLeft; +import static io.lenses.streamreactor.test.utils.EitherValues.getRight; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class StoresInfoTest { + + private final String password = "changeIt"; + private final Path keystoreDir = KeyStoreUtils.createKeystore("TestCommonName", password, password); + private final String keystoreFile = keystoreDir.toAbsolutePath() + "/keystore.jks"; + private final String truststoreFile = keystoreDir.toAbsolutePath() + "/truststore.jks"; + + StoresInfoTest() throws Exception { + } + + @Test + void testToSslContextWithBothNone() { + val storesInfo = new StoresInfo(none(), none()); + assertEquals(right(none()), storesInfo.toSslContext()); + } + + @Test + void testToSslContextWithKeyStoreDefined() { + val storeInfo = new StoreInfo(keystoreFile, some("JKS"), some(password)); + val storesInfo = new StoresInfo(none(), some(storeInfo)); + + val sslContext = getRight(storesInfo.toSslContext()); + + assertTrue(sslContext.isPresent()); + assertTrue(sslContext.filter(e -> e.getProtocol().equals("TLS")).isPresent()); + } + + @Test + void testToSslContextWithTrustStoreDefined() { + val storeInfo = new StoreInfo(keystoreFile, some("JKS"), some(password)); + val storesInfo = new StoresInfo(some(storeInfo), none()); + + val sslContext = getRight(storesInfo.toSslContext()); + + assertTrue(sslContext.isPresent()); + assertTrue(sslContext.filter(e -> e.getProtocol().equals("TLS")).isPresent()); + } + + @Test + void testToSslContextWithBothStoresDefined() { + val keyStoreInfo = new StoreInfo(keystoreFile, some("JKS"), some(password)); + val trustStoreInfo = new StoreInfo(truststoreFile, some("JKS"), some(password)); + val storesInfo = new StoresInfo(some(trustStoreInfo), some(keyStoreInfo)); + + val sslContext = getRight(storesInfo.toSslContext()); + + assertTrue(sslContext.isPresent()); + assertTrue(sslContext.filter(e -> e.getProtocol().equals("TLS")).isPresent()); + } + + @Test + void testToSslContextThrowsFileNotFoundExceptionForInvalidKeyStorePath() { + val keyStoreInfo = new StoreInfo("/invalid/path/to/keystore", some("JKS"), some(password)); + val storesInfo = new StoresInfo(none(), some(keyStoreInfo)); + + assertEquals(FileNotFoundException.class, getLeft(storesInfo.toSslContext()).getClass()); + } + + @Test + void testToSslContextThrowsFileNotFoundExceptionForInvalidTrustStorePath() { + val trustStoreInfo = new StoreInfo("/invalid/path/to/truststore", some("JKS"), some(password)); + val storesInfo = new StoresInfo(some(trustStoreInfo), none()); + + assertEquals(FileNotFoundException.class, getLeft(storesInfo.toSslContext()).getClass()); + + } + + @Test + void testStoresInfoCreationFromBaseConfig() { + BaseConfig mockConfig = mock(BaseConfig.class); + when(mockConfig.getString(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG)).thenReturn("/path/to/truststore"); + when(mockConfig.getString(SslConfigs.SSL_TRUSTSTORE_TYPE_CONFIG)).thenReturn("JKS"); + when(mockConfig.getPassword(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG)).thenReturn(null); + + when(mockConfig.getString(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG)).thenReturn("/path/to/keystore"); + when(mockConfig.getString(SslConfigs.SSL_KEYSTORE_TYPE_CONFIG)).thenReturn("JKS"); + when(mockConfig.getPassword(SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG)).thenReturn(null); + + val storesInfo = StoresInfo.fromConfig(mockConfig); + + assertEquals( + some(new StoreInfo("/path/to/truststore", some("JKS"), none())), + storesInfo.getMaybeTrustStore() + ); + assertEquals( + some(new StoreInfo("/path/to/keystore", some("JKS"), none())), + storesInfo.getMaybeKeyStore() + ); + } + + @Test + void testStoresInfoCreationWithNoneValuesWithMissingConfigs() { + BaseConfig mockConfig = mock(BaseConfig.class); + when(mockConfig.getString(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG)).thenReturn(null); + when(mockConfig.getString(SslConfigs.SSL_TRUSTSTORE_TYPE_CONFIG)).thenReturn(null); + when(mockConfig.getPassword(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG)).thenReturn(null); + + when(mockConfig.getString(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG)).thenReturn(null); + when(mockConfig.getString(SslConfigs.SSL_KEYSTORE_TYPE_CONFIG)).thenReturn(null); + when(mockConfig.getPassword(SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG)).thenReturn(null); + + val storesInfo = StoresInfo.fromConfig(mockConfig); + assertEquals(none(), storesInfo.getMaybeKeyStore()); + assertEquals(none(), storesInfo.getMaybeTrustStore()); + } +} diff --git a/kafka-connect-common/src/main/scala/io/lenses/streamreactor/common/utils/CyclopsToScalaOption.scala b/kafka-connect-common/src/main/scala/io/lenses/streamreactor/common/utils/CyclopsToScalaOption.scala new file mode 100644 index 0000000000..550d1a3a64 --- /dev/null +++ b/kafka-connect-common/src/main/scala/io/lenses/streamreactor/common/utils/CyclopsToScalaOption.scala @@ -0,0 +1,42 @@ +/* + * Copyright 2017-2024 Lenses.io Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.lenses.streamreactor.common.utils + +import cyclops.control.{ Option => CyclopsOption } + +import scala.jdk.OptionConverters.RichOptional +import scala.{ Option => ScalaOption } + +/** + * Utility object for converting Cyclops Option to Scala Option. + * + * This object provides a method to convert an instance of Cyclops Option to a Scala Option. + */ +object CyclopsToScalaOption { + + /** + * Converts a Cyclops Option to a Scala Option. + * + * This method converts an instance of Cyclops Option to a Scala Option. + * + * @tparam M the type of the value contained in the Option + * @param cyclopsOption the Cyclops Option to convert + * @return the converted Scala Option + */ + def convertToScalaOption[M](cyclopsOption: CyclopsOption[M]): ScalaOption[M] = + cyclopsOption.toOptional.toScala + +} diff --git a/kafka-connect-common/src/main/scala/io/lenses/streamreactor/connect/security/StoreInfo.scala b/kafka-connect-common/src/main/scala/io/lenses/streamreactor/connect/security/StoreInfo.scala deleted file mode 100644 index 6302f2a1ec..0000000000 --- a/kafka-connect-common/src/main/scala/io/lenses/streamreactor/connect/security/StoreInfo.scala +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Copyright 2017-2024 Lenses.io Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.lenses.streamreactor.connect.security - -import io.lenses.streamreactor.common.config.base.traits.BaseConfig -import org.apache.kafka.common.config.SslConfigs - -import java.io.FileInputStream -import java.security.KeyStore -import javax.net.ssl.KeyManagerFactory -import javax.net.ssl.SSLContext -import javax.net.ssl.TrustManagerFactory - -case class StoresInfo( - trustStore: Option[StoreInfo] = None, - keyStore: Option[StoreInfo] = None, -) { - def toSslContext: Option[SSLContext] = { - val maybeTrustFactory: Option[TrustManagerFactory] = trustStore.map { - case StoreInfo(path, storeType, password) => - trustManagers(path, storeType, password) - } - val maybeKeyFactory: Option[KeyManagerFactory] = keyStore.map { - case StoreInfo(path, storeType, password) => - keyManagers(path, storeType, password) - } - - Option.when(maybeTrustFactory.nonEmpty || maybeKeyFactory.nonEmpty) { - val sslContext = SSLContext.getInstance("TLS") - sslContext.init( - maybeKeyFactory.map(_.getKeyManagers).orNull, - maybeTrustFactory.map(_.getTrustManagers).orNull, - null, - ) - sslContext - } - } - - private def trustManagers(path: String, storeType: Option[String], password: Option[String]) = { - val truststore: KeyStore = getJksStore(path, storeType, password) - - val trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm) - trustManagerFactory.init(truststore) - trustManagerFactory - } - - private def keyManagers(path: String, storeType: Option[String], password: Option[String]): KeyManagerFactory = { - val keyStore: KeyStore = getJksStore(path, storeType, password) - - val keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm) - keyManagerFactory.init(keyStore, password.getOrElse("").toCharArray) - keyManagerFactory - } - - private def getJksStore(path: String, storeType: Option[String], password: Option[String]) = { - val keyStore = KeyStore.getInstance(storeType.map(_.toUpperCase).getOrElse("JKS")) - val truststoreStream = new FileInputStream(path) - keyStore.load(truststoreStream, password.getOrElse("").toCharArray) - keyStore - } -} - -object StoresInfo { - def apply(config: BaseConfig): StoresInfo = { - val trustStore = for { - storePath <- Option(config.getString(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG)) - storeType = Option(config.getString(SslConfigs.SSL_TRUSTSTORE_TYPE_CONFIG)) - storePassword = Option(config.getPassword(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG)).map(_.value()) - } yield StoreInfo(storePath, storeType, storePassword) - val keyStore = for { - storePath <- Option(config.getString(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG)) - storeType = Option(config.getString(SslConfigs.SSL_KEYSTORE_TYPE_CONFIG)) - storePassword = Option(config.getPassword(SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG)).map(_.value()) - } yield new StoreInfo(storePath, storeType, storePassword) - - StoresInfo(trustStore, keyStore) - } -} diff --git a/kafka-connect-common/src/test/scala/io/lenses/streamreactor/connect/security/KeyStoreUtils.scala b/kafka-connect-common/src/test/scala/io/lenses/streamreactor/connect/security/KeyStoreUtils.scala deleted file mode 100644 index 6b23cf034f..0000000000 --- a/kafka-connect-common/src/test/scala/io/lenses/streamreactor/connect/security/KeyStoreUtils.scala +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright 2017-2024 Lenses.io Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.lenses.streamreactor.connect.security - -import com.typesafe.scalalogging.LazyLogging -import org.bouncycastle.asn1.x500.X500Name -import org.bouncycastle.asn1.x509.SubjectPublicKeyInfo -import org.bouncycastle.cert.X509v3CertificateBuilder -import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter -import org.bouncycastle.jce.provider.BouncyCastleProvider -import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder - -import java.io.FileOutputStream -import java.math.BigInteger -import java.nio.file.Files -import java.nio.file.Path -import java.security.cert.X509Certificate -import java.security.interfaces.RSAPrivateKey -import java.security.KeyPairGenerator -import java.security.KeyStore -import java.security.Security -import java.util.Date - -object KeyStoreUtils extends LazyLogging { - - Security.addProvider(new BouncyCastleProvider()) - - def createKeystore( - commonName: String, - keyStorePassword: String, - trustStorePassword: String, - ): Path = { - - val tmpDir: Path = Files.createTempDirectory("security") - - val (certificate, privateKey) = KeyStoreUtils.generateSelfSignedCertificate(2048, 365, commonName) - val _ = KeyStoreUtils.createAndSaveKeystore(tmpDir, keyStorePassword, certificate, privateKey) - val _ = KeyStoreUtils.createAndSaveTruststore(tmpDir, trustStorePassword, certificate) - logger.info(s"container -> Creating keystore at $tmpDir") - tmpDir - } - - def generateSelfSignedCertificate( - keySize: Int, - certificateValidityDays: Int, - commonName: String, - ): (X509Certificate, RSAPrivateKey) = { - val keyPairGen = KeyPairGenerator.getInstance("RSA", "BC") - keyPairGen.initialize(keySize) - val keyPair = keyPairGen.generateKeyPair() - - val notBefore = new Date() - val notAfter = new Date(System.currentTimeMillis() + certificateValidityDays * 24L * 60 * 60 * 1000) - - val publicKeyInfo = SubjectPublicKeyInfo.getInstance(keyPair.getPublic.getEncoded) - - val certBuilder = new X509v3CertificateBuilder( - new X500Name(s"CN=$commonName"), - BigInteger.valueOf(System.currentTimeMillis()), - notBefore, - notAfter, - new X500Name(s"CN=$commonName"), - publicKeyInfo, - ) - - val contentSigner = - new JcaContentSignerBuilder("SHA256WithRSAEncryption").setProvider("BC").build(keyPair.getPrivate) - val certHolder = certBuilder.build(contentSigner) - val cert = new JcaX509CertificateConverter().setProvider("BC").getCertificate(certHolder) - - (cert, keyPair.getPrivate.asInstanceOf[RSAPrivateKey]) - } - - def createAndSaveKeystore( - tmpDir: Path, - password: String, - certificate: X509Certificate, - privateKey: RSAPrivateKey, - ): String = { - - val keyStore = KeyStore.getInstance("JKS") - keyStore.load(null, password.toCharArray) - - // Store the private key and certificate in the keystore - keyStore.setKeyEntry("alias", privateKey, password.toCharArray, Array(certificate)) - - val keyStorePath = tmpDir.resolve("keystore.jks").toString - // Save the keystore to a file - val keystoreOutputStream = new FileOutputStream(keyStorePath) - keyStore.store(keystoreOutputStream, password.toCharArray) - keystoreOutputStream.close() - - keyStorePath - } - - def createAndSaveTruststore(tmpDir: Path, password: String, certificate: X509Certificate): String = { - - val trustStore = KeyStore.getInstance("JKS") - trustStore.load(null, password.toCharArray) - - // Add the trusted certificate to the truststore - trustStore.setCertificateEntry("alias", certificate) - val trustStorePath = tmpDir.resolve("truststore.jks").toString - - // Save the truststore to a file - val truststoreOutputStream = new FileOutputStream(trustStorePath) - trustStore.store(truststoreOutputStream, password.toCharArray) - truststoreOutputStream.close() - - trustStorePath - } - -} diff --git a/kafka-connect-common/src/test/scala/io/lenses/streamreactor/connect/security/StoresInfoTest.scala b/kafka-connect-common/src/test/scala/io/lenses/streamreactor/connect/security/StoresInfoTest.scala deleted file mode 100644 index 437bdc4616..0000000000 --- a/kafka-connect-common/src/test/scala/io/lenses/streamreactor/connect/security/StoresInfoTest.scala +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Copyright 2017-2024 Lenses.io Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.lenses.streamreactor.connect.security - -import io.lenses.streamreactor.common.config.base.traits.BaseConfig -import org.apache.kafka.common.config.SslConfigs -import org.mockito.MockitoSugar -import org.scalatest.BeforeAndAfterAll -import org.scalatest.funsuite.AnyFunSuite -import org.scalatest.matchers.should.Matchers - -import java.io.FileNotFoundException -import java.nio.file.Path - -class StoresInfoTest extends AnyFunSuite with Matchers with MockitoSugar with BeforeAndAfterAll { - - private val password = "changeIt" - private val keystoreDir: Path = KeyStoreUtils.createKeystore("TestCommonName", password, password) - private val keystoreFile = keystoreDir.toAbsolutePath.toString + "/keystore.jks" - private val truststoreFile = keystoreDir.toAbsolutePath.toString + "/truststore.jks" - - test("StoresInfo.toSslContext should return None when both trustStore and keyStore are None") { - val storesInfo = StoresInfo(None, None) - storesInfo.toSslContext should be(None) - } - - test("StoresInfo.toSslContext should return Some(SSLContext) when keyStore is defined") { - val storeInfo = StoreInfo(keystoreFile, Some("JKS"), Some(password)) - val storesInfo = StoresInfo(keyStore = Some(storeInfo)) - - val sslContext = storesInfo.toSslContext - - sslContext should not be empty - sslContext.get.getProtocol shouldEqual "TLS" - } - - test("StoresInfo.toSslContext should return Some(SSLContext) when trustStore is defined") { - val storeInfo = StoreInfo(keystoreFile, Some("JKS"), Some(password)) - val storesInfo = StoresInfo(trustStore = Some(storeInfo)) - - val sslContext = storesInfo.toSslContext - - sslContext should not be empty - sslContext.get.getProtocol shouldEqual "TLS" - } - - test("StoresInfo.toSslContext should return Some(SSLContext) when both keyStore and trustStore are defined") { - val keyStoreInfo = StoreInfo(keystoreFile, Some("JKS"), Some(password)) - val trustStoreInfo = StoreInfo(truststoreFile, Some("JKS"), Some(password)) - val storesInfo = StoresInfo(trustStore = Some(trustStoreInfo), keyStore = Some(keyStoreInfo)) - - val sslContext = storesInfo.toSslContext - - sslContext should not be empty - sslContext.get.getProtocol shouldEqual "TLS" - } - - test("StoresInfo.toSslContext should throw FileNotFoundException if the keyStore path is incorrect") { - val keyStoreInfo = StoreInfo("/invalid/path/to/keystore", Some("JKS"), Some(password)) - val storesInfo = StoresInfo(keyStore = Some(keyStoreInfo)) - - assertThrows[FileNotFoundException] { - storesInfo.toSslContext - } - } - - test("StoresInfo.toSslContext should throw FileNotFoundException if the trustStore path is incorrect") { - val trustStoreInfo = StoreInfo("/invalid/path/to/truststore", Some("JKS"), Some(password)) - val storesInfo = StoresInfo(trustStore = Some(trustStoreInfo)) - - assertThrows[FileNotFoundException] { - storesInfo.toSslContext - } - } - - test("StoresInfo.apply should create StoresInfo with correct values from BaseConfig") { - val mockConfig: BaseConfig = mock[BaseConfig] - when(mockConfig.getString(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG)).thenReturn("/path/to/truststore") - when(mockConfig.getString(SslConfigs.SSL_TRUSTSTORE_TYPE_CONFIG)).thenReturn("JKS") - when(mockConfig.getPassword(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG)).thenReturn(null) - - when(mockConfig.getString(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG)).thenReturn("/path/to/keystore") - when(mockConfig.getString(SslConfigs.SSL_KEYSTORE_TYPE_CONFIG)).thenReturn("JKS") - when(mockConfig.getPassword(SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG)).thenReturn(null) - - val storesInfo = StoresInfo(mockConfig) - - storesInfo.trustStore shouldEqual Some(StoreInfo("/path/to/truststore", Some("JKS"), None)) - storesInfo.keyStore shouldEqual Some(StoreInfo("/path/to/keystore", Some("JKS"), None)) - } - - test("StoresInfo.apply should create StoresInfo with None values if configs are missing") { - val mockConfig: BaseConfig = mock[BaseConfig] - when(mockConfig.getString(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG)).thenReturn(null) - when(mockConfig.getString(SslConfigs.SSL_TRUSTSTORE_TYPE_CONFIG)).thenReturn(null) - when(mockConfig.getPassword(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG)).thenReturn(null) - - when(mockConfig.getString(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG)).thenReturn(null) - when(mockConfig.getString(SslConfigs.SSL_KEYSTORE_TYPE_CONFIG)).thenReturn(null) - when(mockConfig.getPassword(SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG)).thenReturn(null) - - val storesInfo = StoresInfo(mockConfig) - - storesInfo.trustStore shouldEqual None - storesInfo.keyStore shouldEqual None - } -} diff --git a/kafka-connect-elastic6/src/main/scala/io/lenses/streamreactor/connect/elastic6/KElasticClient.scala b/kafka-connect-elastic6/src/main/scala/io/lenses/streamreactor/connect/elastic6/KElasticClient.scala index 8546d28560..72efa789d7 100644 --- a/kafka-connect-elastic6/src/main/scala/io/lenses/streamreactor/connect/elastic6/KElasticClient.scala +++ b/kafka-connect-elastic6/src/main/scala/io/lenses/streamreactor/connect/elastic6/KElasticClient.scala @@ -21,6 +21,8 @@ import com.sksamuel.elastic4s.http._ import com.sksamuel.elastic4s.http.bulk.BulkResponse import com.typesafe.scalalogging.StrictLogging import io.lenses.kcql.Kcql +import io.lenses.streamreactor.common.utils.CyclopsToScalaEither.convertToScalaEither +import io.lenses.streamreactor.common.utils.CyclopsToScalaOption.convertToScalaOption import io.lenses.streamreactor.connect.elastic6.config.ElasticSettings import io.lenses.streamreactor.connect.elastic6.indexname.CreateIndex.getIndexNameForAutoCreate import org.apache.http.auth.AuthScope @@ -56,7 +58,11 @@ object KElasticClient extends StrictLogging { (requestConfigBuilder: Builder) => requestConfigBuilder, (httpClientBuilder: HttpAsyncClientBuilder) => { maybeProvider.foreach(httpClientBuilder.setDefaultCredentialsProvider) - settings.storesInfo.toSslContext.foreach(httpClientBuilder.setSSLContext) + convertToScalaEither(settings.storesInfo.toSslContext) + .map(convertToScalaOption) + .leftMap(throw _) + .merge + .map(httpClientBuilder.setSSLContext) httpClientBuilder }, ) diff --git a/kafka-connect-elastic6/src/main/scala/io/lenses/streamreactor/connect/elastic6/config/ElasticSettings.scala b/kafka-connect-elastic6/src/main/scala/io/lenses/streamreactor/connect/elastic6/config/ElasticSettings.scala index 05ef47fae1..d626a444b0 100644 --- a/kafka-connect-elastic6/src/main/scala/io/lenses/streamreactor/connect/elastic6/config/ElasticSettings.scala +++ b/kafka-connect-elastic6/src/main/scala/io/lenses/streamreactor/connect/elastic6/config/ElasticSettings.scala @@ -15,9 +15,10 @@ */ package io.lenses.streamreactor.connect.elastic6.config +import cyclops.control.Option.none import io.lenses.kcql.Kcql import io.lenses.streamreactor.common.errors.ErrorPolicy -import io.lenses.streamreactor.connect.security.StoresInfo +import io.lenses.streamreactor.common.security.StoresInfo /** * Created by andrew@datamountaineer.com on 13/05/16. @@ -32,7 +33,7 @@ case class ElasticSettings( pkJoinerSeparator: String = ElasticConfigConstants.PK_JOINER_SEPARATOR_DEFAULT, httpBasicAuthUsername: String = ElasticConfigConstants.CLIENT_HTTP_BASIC_AUTH_USERNAME_DEFAULT, httpBasicAuthPassword: String = ElasticConfigConstants.CLIENT_HTTP_BASIC_AUTH_USERNAME_DEFAULT, - storesInfo: StoresInfo = StoresInfo(), + storesInfo: StoresInfo = new StoresInfo(none(), none()), ) object ElasticSettings { @@ -57,7 +58,7 @@ object ElasticSettings { pkJoinerSeparator, httpBasicAuthUsername, httpBasicAuthPassword, - StoresInfo(config), + StoresInfo.fromConfig(config), ) } } diff --git a/kafka-connect-elastic7/src/main/scala/io/lenses/streamreactor/connect/elastic7/KElasticClient.scala b/kafka-connect-elastic7/src/main/scala/io/lenses/streamreactor/connect/elastic7/KElasticClient.scala index 9ed8899cb0..75af0ae34f 100644 --- a/kafka-connect-elastic7/src/main/scala/io/lenses/streamreactor/connect/elastic7/KElasticClient.scala +++ b/kafka-connect-elastic7/src/main/scala/io/lenses/streamreactor/connect/elastic7/KElasticClient.scala @@ -25,6 +25,8 @@ import com.sksamuel.elastic4s.requests.bulk.BulkRequest import com.sksamuel.elastic4s.requests.bulk.BulkResponse import com.typesafe.scalalogging.StrictLogging import io.lenses.kcql.Kcql +import io.lenses.streamreactor.common.utils.CyclopsToScalaEither.convertToScalaEither +import io.lenses.streamreactor.common.utils.CyclopsToScalaOption.convertToScalaOption import io.lenses.streamreactor.connect.elastic7.config.ElasticSettings import io.lenses.streamreactor.connect.elastic7.indexname.CreateIndex.getIndexNameForAutoCreate import org.apache.http.auth.AuthScope @@ -61,7 +63,11 @@ object KElasticClient extends StrictLogging { (requestConfigBuilder: Builder) => requestConfigBuilder, (httpClientBuilder: HttpAsyncClientBuilder) => { maybeProvider.foreach(httpClientBuilder.setDefaultCredentialsProvider) - settings.storesInfo.toSslContext.foreach(httpClientBuilder.setSSLContext) + convertToScalaEither(settings.storesInfo.toSslContext) + .map(convertToScalaOption) + .leftMap(throw _) + .merge + .map(httpClientBuilder.setSSLContext) httpClientBuilder }, ), diff --git a/kafka-connect-elastic7/src/main/scala/io/lenses/streamreactor/connect/elastic7/config/ElasticSettings.scala b/kafka-connect-elastic7/src/main/scala/io/lenses/streamreactor/connect/elastic7/config/ElasticSettings.scala index ccc96957ec..33eeece10b 100644 --- a/kafka-connect-elastic7/src/main/scala/io/lenses/streamreactor/connect/elastic7/config/ElasticSettings.scala +++ b/kafka-connect-elastic7/src/main/scala/io/lenses/streamreactor/connect/elastic7/config/ElasticSettings.scala @@ -15,9 +15,10 @@ */ package io.lenses.streamreactor.connect.elastic7.config +import cyclops.control.Option.none import io.lenses.kcql.Kcql import io.lenses.streamreactor.common.errors.ErrorPolicy -import io.lenses.streamreactor.connect.security.StoresInfo +import io.lenses.streamreactor.common.security.StoresInfo /** * Created by andrew@datamountaineer.com on 13/05/16. @@ -32,7 +33,7 @@ case class ElasticSettings( pkJoinerSeparator: String = ElasticConfigConstants.PK_JOINER_SEPARATOR_DEFAULT, httpBasicAuthUsername: String = ElasticConfigConstants.CLIENT_HTTP_BASIC_AUTH_USERNAME_DEFAULT, httpBasicAuthPassword: String = ElasticConfigConstants.CLIENT_HTTP_BASIC_AUTH_USERNAME_DEFAULT, - storesInfo: StoresInfo = StoresInfo(), + storesInfo: StoresInfo = new StoresInfo(none(), none()), ) object ElasticSettings { @@ -57,7 +58,7 @@ object ElasticSettings { pkJoinerSeparator, httpBasicAuthUsername, httpBasicAuthPassword, - StoresInfo(config), + StoresInfo.fromConfig(config), ) } } diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 54756aab21..2dd45a0b18 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -192,7 +192,8 @@ object Dependencies { val `junitJupiterParams` = "org.junit.jupiter" % "junit-jupiter-params" % junitJupiterVersion val `assertjCore` = "org.assertj" % "assertj-core" % assertjCoreVersion - val `cyclops` = "com.oath.cyclops" % "cyclops" % cyclopsVersion + val `cyclops` = "com.oath.cyclops" % "cyclops" % cyclopsVersion + val `cyclopsPure` = "com.oath.cyclops" % "cyclops-pure" % cyclopsVersion val catsEffectScalatest = "org.typelevel" %% "cats-effect-testing-scalatest" % `cats-effect-testing` @@ -468,8 +469,9 @@ trait Dependencies { confluentJsonSchemaSerializer, ) ++ enumeratum ++ circe - val javaCommonDeps: Seq[ModuleID] = Seq(lombok, kafkaConnectJson, kafkaClients, cyclops) - val javaCommonTestDeps: Seq[ModuleID] = Seq(junitJupiter, junitJupiterParams, assertjCore, `mockitoJava`, logback) + val javaCommonDeps: Seq[ModuleID] = Seq(lombok, kafkaConnectJson, kafkaClients, cyclops, `cyclopsPure`) + val javaCommonTestDeps: Seq[ModuleID] = + Seq(junitJupiter, junitJupiterParams, assertjCore, `mockitoJava`, logback) ++ bouncyCastle //Specific modules dependencies From ebccac18a4e42f34151cb7fff30c55caa7f8386b Mon Sep 17 00:00:00 2001 From: David Sloan <33483659+davidsloan@users.noreply.github.com> Date: Wed, 11 Sep 2024 09:02:42 +0100 Subject: [PATCH 04/12] Update java-connectors/kafka-connect-common/src/test/java/io/lenses/streamreactor/common/security/KeyStoreUtils.java Co-authored-by: Mati Urban <157909548+GoMati-MU@users.noreply.github.com> --- .../lenses/streamreactor/common/security/KeyStoreUtils.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/java-connectors/kafka-connect-common/src/test/java/io/lenses/streamreactor/common/security/KeyStoreUtils.java b/java-connectors/kafka-connect-common/src/test/java/io/lenses/streamreactor/common/security/KeyStoreUtils.java index 2d8a793569..92c0a52c93 100644 --- a/java-connectors/kafka-connect-common/src/test/java/io/lenses/streamreactor/common/security/KeyStoreUtils.java +++ b/java-connectors/kafka-connect-common/src/test/java/io/lenses/streamreactor/common/security/KeyStoreUtils.java @@ -63,8 +63,8 @@ public static Path createKeystore(String commonName, String keyStorePassword, St keyPairGen.initialize(2048); KeyPair keyPair = keyPairGen.generateKeyPair(); - Date notBefore = new Date(); - Date notAfter = new Date(System.currentTimeMillis() + 365L * 24 * 60 * 60 * 1000); + Date notBefore = new Date(Instant.now().toEpochMilli()); + Date notAfter = new Date(Instant.now().plus(365, ChronoUnit.DAYS).toEpochMilli()); SubjectPublicKeyInfo publicKeyInfo = SubjectPublicKeyInfo.getInstance(keyPair.getPublic().getEncoded()); From 590e9847588317e33f1f2fde49e1dee375a6a752 Mon Sep 17 00:00:00 2001 From: David Sloan Date: Wed, 11 Sep 2024 09:04:33 +0100 Subject: [PATCH 05/12] Changes following review --- .../exception/SecuritySetupException.java | 23 ++ .../common/security/StoreInfo.java | 2 +- .../common/security/StoreType.java | 44 ++++ .../common/security/StoresInfo.java | 198 +++++++++++------- .../common/security/StoresInfoTest.java | 48 ++--- .../test/utils/OptionValues.java | 29 +++ .../connect/elastic6/KElasticClient.scala | 9 +- .../elastic6/config/ElasticSettings.scala | 3 +- .../connect/elastic7/KElasticClient.scala | 9 +- .../elastic7/config/ElasticSettings.scala | 3 +- 10 files changed, 246 insertions(+), 122 deletions(-) create mode 100644 java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/exception/SecuritySetupException.java create mode 100644 java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/StoreType.java create mode 100644 java-connectors/test-utils/src/test/java/io/lenses/streamreactor/test/utils/OptionValues.java diff --git a/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/exception/SecuritySetupException.java b/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/exception/SecuritySetupException.java new file mode 100644 index 0000000000..3e690a5292 --- /dev/null +++ b/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/exception/SecuritySetupException.java @@ -0,0 +1,23 @@ +/* + * Copyright 2017-2024 Lenses.io Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.lenses.streamreactor.common.exception; + +public class SecuritySetupException extends StreamReactorException { + + public SecuritySetupException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/StoreInfo.java b/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/StoreInfo.java index 137e78da56..ffca232fdc 100644 --- a/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/StoreInfo.java +++ b/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/StoreInfo.java @@ -25,7 +25,7 @@ class StoreInfo { private String storePath; - private Option storeType; + private StoreType storeType; private Option storePassword; diff --git a/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/StoreType.java b/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/StoreType.java new file mode 100644 index 0000000000..3ad2d48f3a --- /dev/null +++ b/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/StoreType.java @@ -0,0 +1,44 @@ +/* + * Copyright 2017-2024 Lenses.io Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.lenses.streamreactor.common.security; + +import cyclops.control.Either; +import cyclops.control.Try; +import io.lenses.streamreactor.common.exception.SecuritySetupException; +import lombok.Getter; + +@Getter +public enum StoreType { + + JKS("JKS"), + PKCS12("PKCS12"); + + private final String type; + + StoreType(String type) { + this.type = type; + } + + public static Either valueOfCaseInsensitive(String storeType) { + return Try + .withCatch(() -> StoreType.valueOf(storeType.toUpperCase())) + .toEither() + .mapLeft(ex -> new SecuritySetupException(String.format("Unable to retrieve Store type %s", storeType), ex)); + } + + public static final StoreType DEFAULT_STORE_TYPE = StoreType.JKS; + +} diff --git a/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/StoresInfo.java b/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/StoresInfo.java index 314aa2157f..250780122b 100644 --- a/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/StoresInfo.java +++ b/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/StoresInfo.java @@ -20,6 +20,7 @@ import cyclops.control.Try; import cyclops.instances.control.TryInstances; import cyclops.typeclasses.Do; +import io.lenses.streamreactor.common.exception.SecuritySetupException; import lombok.AllArgsConstructor; import lombok.Data; import lombok.val; @@ -35,39 +36,37 @@ import java.security.KeyStoreException; import java.security.NoSuchAlgorithmException; import java.security.UnrecoverableKeyException; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static io.lenses.streamreactor.common.security.StoreType.DEFAULT_STORE_TYPE; @AllArgsConstructor @Data public class StoresInfo { + private static final String PROTOCOL_TLS = "TLS"; + private Option maybeTrustStore; private Option maybeKeyStore; - private Try getJksStore(String path, Option storeType, Option password) { + private Try getJksStore(String path, StoreType storeType, Option password) { return Try.withCatch( () -> { - val keyStore = getKeyStoreInstance(storeType); + val keyStore = KeyStore.getInstance(storeType.toString()); val inputStream = new FileInputStream(path); - loadKeyStore(password, keyStore, inputStream); + keyStore.load(inputStream, (password.orElse("")).toCharArray()); return keyStore; }, Exception.class - ); + ).mapFailure(ex -> new SecuritySetupException("unable to retrieve keystore", ex)); } - private static void loadKeyStore(Option password, KeyStore keyStore, FileInputStream truststoreStream) - throws Exception { - keyStore.load(truststoreStream, (password.orElse("")).toCharArray()); - } + public Either> toSslContext() { - private static KeyStore getKeyStoreInstance(Option storeType) throws Exception { - return KeyStore.getInstance(storeType.map(String::toUpperCase).orElse("JKS")); - } - - public Either> toSslContext() { - - final Option> maybeTrustFactory = + final Option> maybeTrustFactory = maybeTrustStore.map( trustStore -> trustManagers( trustStore.getStorePath(), @@ -76,7 +75,7 @@ public Either> toSslContext() { ) ); - final Option> maybeKeyFactory = + final Option> maybeKeyFactory = maybeKeyStore.map( keyStore -> keyManagers( keyStore.getStorePath(), @@ -85,49 +84,56 @@ public Either> toSslContext() { ) ); - if (maybeTrustFactory.isPresent() || maybeKeyFactory.isPresent()) { - - val trustFailure = maybeTrustFactory.filter(Try::isFailure).flatMap(Try::failureGet); - val keyFailure = maybeKeyFactory.filter(Try::isFailure).flatMap(Try::failureGet); - - if (trustFailure.isPresent() || keyFailure.isPresent()) { - return Either.left(trustFailure.orElse(keyFailure.orElse(new IllegalStateException( - "Logic error retrieving trust factories")))); - } else { + val failures = + Stream.of( + maybeTrustFactory.filter(Try::isFailure).flatMap(Try::failureGet).stream(), + maybeKeyFactory.filter(Try::isFailure).flatMap(Try::failureGet).stream() + ) + .flatMap(Function.identity()) + .collect(Collectors.toUnmodifiableList()); + + val maybeFailure = + Option.fromOptional( + failures + .stream() + .findFirst() + ); - val trustSuccess = maybeTrustFactory.filter(Try::isSuccess).flatMap(Try::get); - val keySuccess = maybeKeyFactory.filter(Try::isSuccess).flatMap(Try::get); + return maybeFailure + .toEither(getAndInitSslContext(maybeKeyFactory, maybeTrustFactory)) + .swap() + .fold(Either::left, either -> either.fold(Either::left, Either::right)); - return Try.withCatch(() -> { - val sslContext = SSLContext.getInstance("TLS"); - sslContext.init( - keySuccess.map(KeyManagerFactory::getKeyManagers).orElse(null), - trustSuccess.map(TrustManagerFactory::getTrustManagers).orElse(null), - null - ); - return sslContext; - } - ).toEither().bimap(l -> l, Option::some); + } + private static Either> getAndInitSslContext( + Option> maybeKeyFactory, + Option> maybeTrustFactory + ) { + return Try.withCatch(() -> { + // If either factory is present, initialize SSLContext + if (maybeKeyFactory.isPresent() || maybeTrustFactory.isPresent()) { + val sslContext = SSLContext.getInstance(PROTOCOL_TLS); + sslContext.init( + maybeKeyFactory.flatMap(Try::toOption).map(KeyManagerFactory::getKeyManagers).orElse(null), + maybeTrustFactory.flatMap(Try::toOption).map(TrustManagerFactory::getTrustManagers).orElse(null), + null + ); + return Option.of(sslContext); } - - } else { - return Either.right(Option.none()); - } + return Option.none(); + }).mapFailure(ex -> new SecuritySetupException("unable to retrieve keystore", ex)) + .toEither(); } - private Try trustManagers(String path, Option storeType, + private Try trustManagers(String path, StoreType storeType, Option password) { return Try.narrowK( Do.forEach( - TryInstances.monad() + TryInstances.monad() ) .__(getJksStore(path, storeType, password)) - .__((KeyStore keyStore) -> Try.withCatch( - () -> getTrustManagerFactoryFromKeyStore(keyStore), - Exception.class - ) - ) + .__(StoresInfo::getTrustManagerFactoryFromKeyStore) .yield( (KeyStore keyStore, TrustManagerFactory trustManagerFactory) -> trustManagerFactory ) @@ -135,58 +141,88 @@ private Try trustManagers(String path, Option keyManagers(String path, Option storeType, + private Try keyManagers(String path, StoreType storeType, Option password) { return Try.narrowK( Do.forEach( - TryInstances.monad() + TryInstances.monad() ) .__(getJksStore(path, storeType, password)) - .__((KeyStore keyStore) -> Try.withCatch( - () -> getKeyManagerFactoryFromKeyStore(keyStore, password), - Exception.class - ) - ) + .__(s -> StoresInfo.getKeyManagerFactoryFromKeyStore(s, password)) .yield( (KeyStore keyStore, KeyManagerFactory trustManagerFactory) -> trustManagerFactory ) .unwrap()); } - private static TrustManagerFactory getTrustManagerFactoryFromKeyStore(KeyStore keyStore) - throws NoSuchAlgorithmException, KeyStoreException { - val trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); - trustManagerFactory.init(keyStore); - return trustManagerFactory; + private static Try getTrustManagerFactoryFromKeyStore( + KeyStore keyStore) { + return Try.withCatch(() -> { + val trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + trustManagerFactory.init(keyStore); + return trustManagerFactory; + }, NoSuchAlgorithmException.class, KeyStoreException.class) + .mapFailure(ex -> new SecuritySetupException("Unable to get trust manager factory from keystore", ex)); } - private static KeyManagerFactory getKeyManagerFactoryFromKeyStore(KeyStore keyStore, Option password) - throws NoSuchAlgorithmException, KeyStoreException, UnrecoverableKeyException { - val keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); - keyManagerFactory.init(keyStore, (password.orElse("")).toCharArray()); - return keyManagerFactory; + private static Try getKeyManagerFactoryFromKeyStore(KeyStore keyStore, + Option password) { + return Try.withCatch(() -> { + val keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); + keyManagerFactory.init(keyStore, (password.orElse("")).toCharArray()); + return keyManagerFactory; + }, NoSuchAlgorithmException.class, KeyStoreException.class, UnrecoverableKeyException.class) + .mapFailure(ex -> new SecuritySetupException("Unable to get trust manager factory from truststore", ex)); } - public static StoresInfo fromConfig(AbstractConfig config) { + public static Either fromConfig(AbstractConfig config) { val trustStore = - Option.fromNullable(config.getString(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG)) - .map(storePath -> { - val storeType = Option.fromNullable(config.getString(SslConfigs.SSL_TRUSTSTORE_TYPE_CONFIG)); - val storePassword = - Option.fromNullable(config.getPassword(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG)) - .map(Password::value); - return new StoreInfo(storePath, storeType, storePassword); - }); + configToTrustStoreInfo( + config, + SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, + SslConfigs.SSL_TRUSTSTORE_TYPE_CONFIG, + SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG + ); + val keyStore = - Option.fromNullable(config.getString(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG)) - .map(storePath -> { - val storeType = Option.fromNullable(config.getString(SslConfigs.SSL_KEYSTORE_TYPE_CONFIG)); + configToTrustStoreInfo( + config, + SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG, + SslConfigs.SSL_KEYSTORE_TYPE_CONFIG, + SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG + ); + + val failures = + Stream.of(trustStore, keyStore) + .flatMap(option -> option.stream().flatMap(either -> either.swap().stream())) + .collect(Collectors.toUnmodifiableList()); + + return failures.isEmpty() + ? Either.right(new StoresInfo( + trustStore.flatMap(Either::toOption), + keyStore.flatMap(Either::toOption) + )) + : Either.left(failures.iterator().next()); + } + + private static Option> configToTrustStoreInfo(AbstractConfig config, + String sslTruststoreLocationConfig, String sslTruststoreTypeConfig, String sslTruststorePasswordConfig) { + return Option.fromNullable(config.getString(sslTruststoreLocationConfig)) + .map(storePath -> fromConfigOption(config, sslTruststoreTypeConfig) + .map(sT -> { val storePassword = - Option.fromNullable(config.getPassword(SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG)).map(Password::value); - return new StoreInfo(storePath, storeType, storePassword); - }); + Option.fromNullable(config.getPassword(sslTruststorePasswordConfig)) + .map(Password::value); + return new StoreInfo(storePath, sT, storePassword); + } + )); + } - return new StoresInfo(trustStore, keyStore); + private static Either fromConfigOption(AbstractConfig config, String configKey) { + return Option + .fromNullable(config.getString(configKey)) + .map(StoreType::valueOfCaseInsensitive) + .orElse(Either.right(DEFAULT_STORE_TYPE)); } } diff --git a/java-connectors/kafka-connect-common/src/test/java/io/lenses/streamreactor/common/security/StoresInfoTest.java b/java-connectors/kafka-connect-common/src/test/java/io/lenses/streamreactor/common/security/StoresInfoTest.java index a58727d0c3..39af7823ff 100644 --- a/java-connectors/kafka-connect-common/src/test/java/io/lenses/streamreactor/common/security/StoresInfoTest.java +++ b/java-connectors/kafka-connect-common/src/test/java/io/lenses/streamreactor/common/security/StoresInfoTest.java @@ -26,10 +26,11 @@ import static cyclops.control.Either.right; import static cyclops.control.Option.none; import static cyclops.control.Option.some; +import static io.lenses.streamreactor.test.utils.EitherValues.assertRight; import static io.lenses.streamreactor.test.utils.EitherValues.getLeft; import static io.lenses.streamreactor.test.utils.EitherValues.getRight; +import static io.lenses.streamreactor.test.utils.OptionValues.getValue; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -51,52 +52,49 @@ void testToSslContextWithBothNone() { @Test void testToSslContextWithKeyStoreDefined() { - val storeInfo = new StoreInfo(keystoreFile, some("JKS"), some(password)); + val storeInfo = new StoreInfo(keystoreFile, StoreType.JKS, some(password)); val storesInfo = new StoresInfo(none(), some(storeInfo)); val sslContext = getRight(storesInfo.toSslContext()); - assertTrue(sslContext.isPresent()); - assertTrue(sslContext.filter(e -> e.getProtocol().equals("TLS")).isPresent()); + assertEquals("TLS", getValue(sslContext).getProtocol()); } @Test void testToSslContextWithTrustStoreDefined() { - val storeInfo = new StoreInfo(keystoreFile, some("JKS"), some(password)); + val storeInfo = new StoreInfo(keystoreFile, StoreType.JKS, some(password)); val storesInfo = new StoresInfo(some(storeInfo), none()); val sslContext = getRight(storesInfo.toSslContext()); - assertTrue(sslContext.isPresent()); - assertTrue(sslContext.filter(e -> e.getProtocol().equals("TLS")).isPresent()); + assertEquals("TLS", getValue(sslContext).getProtocol()); } @Test void testToSslContextWithBothStoresDefined() { - val keyStoreInfo = new StoreInfo(keystoreFile, some("JKS"), some(password)); - val trustStoreInfo = new StoreInfo(truststoreFile, some("JKS"), some(password)); + val keyStoreInfo = new StoreInfo(keystoreFile, StoreType.JKS, some(password)); + val trustStoreInfo = new StoreInfo(truststoreFile, StoreType.JKS, some(password)); val storesInfo = new StoresInfo(some(trustStoreInfo), some(keyStoreInfo)); val sslContext = getRight(storesInfo.toSslContext()); - assertTrue(sslContext.isPresent()); - assertTrue(sslContext.filter(e -> e.getProtocol().equals("TLS")).isPresent()); + assertEquals("TLS", getValue(sslContext).getProtocol()); } @Test void testToSslContextThrowsFileNotFoundExceptionForInvalidKeyStorePath() { - val keyStoreInfo = new StoreInfo("/invalid/path/to/keystore", some("JKS"), some(password)); + val keyStoreInfo = new StoreInfo("/invalid/path/to/keystore", StoreType.JKS, some(password)); val storesInfo = new StoresInfo(none(), some(keyStoreInfo)); - assertEquals(FileNotFoundException.class, getLeft(storesInfo.toSslContext()).getClass()); + assertEquals(FileNotFoundException.class, getLeft(storesInfo.toSslContext()).getCause().getClass()); } @Test void testToSslContextThrowsFileNotFoundExceptionForInvalidTrustStorePath() { - val trustStoreInfo = new StoreInfo("/invalid/path/to/truststore", some("JKS"), some(password)); + val trustStoreInfo = new StoreInfo("/invalid/path/to/truststore", StoreType.JKS, some(password)); val storesInfo = new StoresInfo(some(trustStoreInfo), none()); - assertEquals(FileNotFoundException.class, getLeft(storesInfo.toSslContext()).getClass()); + assertEquals(FileNotFoundException.class, getLeft(storesInfo.toSslContext()).getCause().getClass()); } @@ -113,13 +111,11 @@ void testStoresInfoCreationFromBaseConfig() { val storesInfo = StoresInfo.fromConfig(mockConfig); - assertEquals( - some(new StoreInfo("/path/to/truststore", some("JKS"), none())), - storesInfo.getMaybeTrustStore() - ); - assertEquals( - some(new StoreInfo("/path/to/keystore", some("JKS"), none())), - storesInfo.getMaybeKeyStore() + assertRight(storesInfo).isEqualTo( + new StoresInfo( + some(new StoreInfo("/path/to/truststore", StoreType.JKS, none())), + some(new StoreInfo("/path/to/keystore", StoreType.JKS, none())) + ) ); } @@ -135,7 +131,11 @@ void testStoresInfoCreationWithNoneValuesWithMissingConfigs() { when(mockConfig.getPassword(SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG)).thenReturn(null); val storesInfo = StoresInfo.fromConfig(mockConfig); - assertEquals(none(), storesInfo.getMaybeKeyStore()); - assertEquals(none(), storesInfo.getMaybeTrustStore()); + assertRight(storesInfo).isEqualTo( + new StoresInfo( + none(), + none() + ) + ); } } diff --git a/java-connectors/test-utils/src/test/java/io/lenses/streamreactor/test/utils/OptionValues.java b/java-connectors/test-utils/src/test/java/io/lenses/streamreactor/test/utils/OptionValues.java new file mode 100644 index 0000000000..0d43868dc8 --- /dev/null +++ b/java-connectors/test-utils/src/test/java/io/lenses/streamreactor/test/utils/OptionValues.java @@ -0,0 +1,29 @@ +/* + * Copyright 2017-2024 Lenses.io Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.lenses.streamreactor.test.utils; + +import cyclops.control.Option; +import lombok.val; + +public class OptionValues { + + public static X getValue(Option opt) { + val ex = new AssertionError("Expected Some, got None"); + return opt.orElseGet(() -> { + throw ex; + }); + } +} diff --git a/kafka-connect-elastic6/src/main/scala/io/lenses/streamreactor/connect/elastic6/KElasticClient.scala b/kafka-connect-elastic6/src/main/scala/io/lenses/streamreactor/connect/elastic6/KElasticClient.scala index 72efa789d7..dd2d416d0d 100644 --- a/kafka-connect-elastic6/src/main/scala/io/lenses/streamreactor/connect/elastic6/KElasticClient.scala +++ b/kafka-connect-elastic6/src/main/scala/io/lenses/streamreactor/connect/elastic6/KElasticClient.scala @@ -21,8 +21,7 @@ import com.sksamuel.elastic4s.http._ import com.sksamuel.elastic4s.http.bulk.BulkResponse import com.typesafe.scalalogging.StrictLogging import io.lenses.kcql.Kcql -import io.lenses.streamreactor.common.utils.CyclopsToScalaEither.convertToScalaEither -import io.lenses.streamreactor.common.utils.CyclopsToScalaOption.convertToScalaOption +import io.lenses.streamreactor.common.util.EitherUtils.unpackOrThrow import io.lenses.streamreactor.connect.elastic6.config.ElasticSettings import io.lenses.streamreactor.connect.elastic6.indexname.CreateIndex.getIndexNameForAutoCreate import org.apache.http.auth.AuthScope @@ -58,11 +57,7 @@ object KElasticClient extends StrictLogging { (requestConfigBuilder: Builder) => requestConfigBuilder, (httpClientBuilder: HttpAsyncClientBuilder) => { maybeProvider.foreach(httpClientBuilder.setDefaultCredentialsProvider) - convertToScalaEither(settings.storesInfo.toSslContext) - .map(convertToScalaOption) - .leftMap(throw _) - .merge - .map(httpClientBuilder.setSSLContext) + unpackOrThrow(settings.storesInfo.toSslContext).map(httpClientBuilder.setSSLContext(_)) httpClientBuilder }, ) diff --git a/kafka-connect-elastic6/src/main/scala/io/lenses/streamreactor/connect/elastic6/config/ElasticSettings.scala b/kafka-connect-elastic6/src/main/scala/io/lenses/streamreactor/connect/elastic6/config/ElasticSettings.scala index d626a444b0..f25b7946d0 100644 --- a/kafka-connect-elastic6/src/main/scala/io/lenses/streamreactor/connect/elastic6/config/ElasticSettings.scala +++ b/kafka-connect-elastic6/src/main/scala/io/lenses/streamreactor/connect/elastic6/config/ElasticSettings.scala @@ -19,6 +19,7 @@ import cyclops.control.Option.none import io.lenses.kcql.Kcql import io.lenses.streamreactor.common.errors.ErrorPolicy import io.lenses.streamreactor.common.security.StoresInfo +import io.lenses.streamreactor.common.util.EitherUtils.unpackOrThrow /** * Created by andrew@datamountaineer.com on 13/05/16. @@ -58,7 +59,7 @@ object ElasticSettings { pkJoinerSeparator, httpBasicAuthUsername, httpBasicAuthPassword, - StoresInfo.fromConfig(config), + unpackOrThrow(StoresInfo.fromConfig(config)), ) } } diff --git a/kafka-connect-elastic7/src/main/scala/io/lenses/streamreactor/connect/elastic7/KElasticClient.scala b/kafka-connect-elastic7/src/main/scala/io/lenses/streamreactor/connect/elastic7/KElasticClient.scala index 75af0ae34f..209ff9217c 100644 --- a/kafka-connect-elastic7/src/main/scala/io/lenses/streamreactor/connect/elastic7/KElasticClient.scala +++ b/kafka-connect-elastic7/src/main/scala/io/lenses/streamreactor/connect/elastic7/KElasticClient.scala @@ -25,8 +25,7 @@ import com.sksamuel.elastic4s.requests.bulk.BulkRequest import com.sksamuel.elastic4s.requests.bulk.BulkResponse import com.typesafe.scalalogging.StrictLogging import io.lenses.kcql.Kcql -import io.lenses.streamreactor.common.utils.CyclopsToScalaEither.convertToScalaEither -import io.lenses.streamreactor.common.utils.CyclopsToScalaOption.convertToScalaOption +import io.lenses.streamreactor.common.util.EitherUtils.unpackOrThrow import io.lenses.streamreactor.connect.elastic7.config.ElasticSettings import io.lenses.streamreactor.connect.elastic7.indexname.CreateIndex.getIndexNameForAutoCreate import org.apache.http.auth.AuthScope @@ -63,11 +62,7 @@ object KElasticClient extends StrictLogging { (requestConfigBuilder: Builder) => requestConfigBuilder, (httpClientBuilder: HttpAsyncClientBuilder) => { maybeProvider.foreach(httpClientBuilder.setDefaultCredentialsProvider) - convertToScalaEither(settings.storesInfo.toSslContext) - .map(convertToScalaOption) - .leftMap(throw _) - .merge - .map(httpClientBuilder.setSSLContext) + unpackOrThrow(settings.storesInfo.toSslContext).map(httpClientBuilder.setSSLContext(_)) httpClientBuilder }, ), diff --git a/kafka-connect-elastic7/src/main/scala/io/lenses/streamreactor/connect/elastic7/config/ElasticSettings.scala b/kafka-connect-elastic7/src/main/scala/io/lenses/streamreactor/connect/elastic7/config/ElasticSettings.scala index 33eeece10b..9de0c3c9e7 100644 --- a/kafka-connect-elastic7/src/main/scala/io/lenses/streamreactor/connect/elastic7/config/ElasticSettings.scala +++ b/kafka-connect-elastic7/src/main/scala/io/lenses/streamreactor/connect/elastic7/config/ElasticSettings.scala @@ -19,6 +19,7 @@ import cyclops.control.Option.none import io.lenses.kcql.Kcql import io.lenses.streamreactor.common.errors.ErrorPolicy import io.lenses.streamreactor.common.security.StoresInfo +import io.lenses.streamreactor.common.util.EitherUtils.unpackOrThrow /** * Created by andrew@datamountaineer.com on 13/05/16. @@ -58,7 +59,7 @@ object ElasticSettings { pkJoinerSeparator, httpBasicAuthUsername, httpBasicAuthPassword, - StoresInfo.fromConfig(config), + unpackOrThrow(StoresInfo.fromConfig(config)), ) } } From 1315ba1284712d97df977c8899e290abd87d4352 Mon Sep 17 00:00:00 2001 From: David Sloan Date: Wed, 11 Sep 2024 09:05:21 +0100 Subject: [PATCH 06/12] Fixing suggestion --- .../io/lenses/streamreactor/common/security/KeyStoreUtils.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/java-connectors/kafka-connect-common/src/test/java/io/lenses/streamreactor/common/security/KeyStoreUtils.java b/java-connectors/kafka-connect-common/src/test/java/io/lenses/streamreactor/common/security/KeyStoreUtils.java index 92c0a52c93..71e97e9f7d 100644 --- a/java-connectors/kafka-connect-common/src/test/java/io/lenses/streamreactor/common/security/KeyStoreUtils.java +++ b/java-connectors/kafka-connect-common/src/test/java/io/lenses/streamreactor/common/security/KeyStoreUtils.java @@ -47,6 +47,8 @@ import java.security.Security; import java.security.cert.X509Certificate; import java.security.interfaces.RSAPrivateKey; +import java.time.Instant; +import java.time.temporal.ChronoUnit; import java.util.Date; public class KeyStoreUtils { From 985c4035e4767a836f4b8e629483921d007219f0 Mon Sep 17 00:00:00 2001 From: David Sloan Date: Wed, 11 Sep 2024 09:19:19 +0100 Subject: [PATCH 07/12] Add logger --- .../lenses/streamreactor/common/security/KeyStoreUtils.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/java-connectors/kafka-connect-common/src/test/java/io/lenses/streamreactor/common/security/KeyStoreUtils.java b/java-connectors/kafka-connect-common/src/test/java/io/lenses/streamreactor/common/security/KeyStoreUtils.java index 71e97e9f7d..3e0c7b90ab 100644 --- a/java-connectors/kafka-connect-common/src/test/java/io/lenses/streamreactor/common/security/KeyStoreUtils.java +++ b/java-connectors/kafka-connect-common/src/test/java/io/lenses/streamreactor/common/security/KeyStoreUtils.java @@ -30,6 +30,7 @@ * limitations under the License. */ +import lombok.extern.slf4j.Slf4j; import org.bouncycastle.asn1.x500.X500Name; import org.bouncycastle.asn1.x509.SubjectPublicKeyInfo; import org.bouncycastle.cert.X509v3CertificateBuilder; @@ -51,6 +52,7 @@ import java.time.temporal.ChronoUnit; import java.util.Date; +@Slf4j public class KeyStoreUtils { static { @@ -89,7 +91,7 @@ public static Path createKeystore(String commonName, String keyStorePassword, St createAndSaveKeystore(tmpDir, keyStorePassword, certificate, (RSAPrivateKey) keyPair.getPrivate()); createAndSaveTruststore(tmpDir, trustStorePassword, certificate); - System.out.println("container -> Creating keystore at " + tmpDir); + log.info("container -> Creating keystore at " + tmpDir); return tmpDir; } From 26cbcc00089d894ba009ef3bc48d7dc805e56d09 Mon Sep 17 00:00:00 2001 From: David Sloan Date: Wed, 11 Sep 2024 09:28:40 +0100 Subject: [PATCH 08/12] Constants --- .../common/security/KeyStoreUtils.java | 42 ++++++++++++------- 1 file changed, 27 insertions(+), 15 deletions(-) diff --git a/java-connectors/kafka-connect-common/src/test/java/io/lenses/streamreactor/common/security/KeyStoreUtils.java b/java-connectors/kafka-connect-common/src/test/java/io/lenses/streamreactor/common/security/KeyStoreUtils.java index 3e0c7b90ab..6a2c80cd0a 100644 --- a/java-connectors/kafka-connect-common/src/test/java/io/lenses/streamreactor/common/security/KeyStoreUtils.java +++ b/java-connectors/kafka-connect-common/src/test/java/io/lenses/streamreactor/common/security/KeyStoreUtils.java @@ -48,44 +48,56 @@ import java.security.Security; import java.security.cert.X509Certificate; import java.security.interfaces.RSAPrivateKey; +import java.time.Duration; import java.time.Instant; -import java.time.temporal.ChronoUnit; import java.util.Date; @Slf4j public class KeyStoreUtils { + private static final String COMMON_NAME = "CN"; + private static final String DIRECTORY_SECURITY = "security"; + private static final String ALGORITHM_RSA = "RSA"; + private static final String PROVIDER_BC = "BC"; + private static final int KEY_SIZE = 2048; + private static final int DAYS_IN_YEAR = 365; + private static final String SIGNER_SIGNATURE_ALGORITHM = "SHA256WithRSAEncryption"; + private static final String KEYSTORE_TYPE = "JKS"; + private static final String KEYSTORE_FILE = "keystore.jks"; + private static final String TRUSTSTORE_FILE = "truststore.jks"; + private static final String TEST_ALIAS = "alias"; + static { Security.addProvider(new BouncyCastleProvider()); } public static Path createKeystore(String commonName, String keyStorePassword, String trustStorePassword) throws Exception { - Path tmpDir = Files.createTempDirectory("security"); + Path tmpDir = Files.createTempDirectory(DIRECTORY_SECURITY); - KeyPairGenerator keyPairGen = KeyPairGenerator.getInstance("RSA", "BC"); - keyPairGen.initialize(2048); + KeyPairGenerator keyPairGen = KeyPairGenerator.getInstance(ALGORITHM_RSA, PROVIDER_BC); + keyPairGen.initialize(KEY_SIZE); KeyPair keyPair = keyPairGen.generateKeyPair(); Date notBefore = new Date(Instant.now().toEpochMilli()); - Date notAfter = new Date(Instant.now().plus(365, ChronoUnit.DAYS).toEpochMilli()); + Date notAfter = new Date(Instant.now().plus(Duration.ofDays(DAYS_IN_YEAR)).toEpochMilli()); SubjectPublicKeyInfo publicKeyInfo = SubjectPublicKeyInfo.getInstance(keyPair.getPublic().getEncoded()); X509v3CertificateBuilder certBuilder = new X509v3CertificateBuilder( - new X500Name("CN=" + commonName), + new X500Name(COMMON_NAME + "=" + commonName), BigInteger.valueOf(System.currentTimeMillis()), notBefore, notAfter, - new X500Name("CN=" + commonName), + new X500Name(COMMON_NAME + "=" + commonName), publicKeyInfo ); JcaContentSignerBuilder contentSignerBuilder = - new JcaContentSignerBuilder("SHA256WithRSAEncryption").setProvider("BC"); + new JcaContentSignerBuilder(SIGNER_SIGNATURE_ALGORITHM).setProvider(PROVIDER_BC); X509Certificate certificate = - new JcaX509CertificateConverter().setProvider("BC") + new JcaX509CertificateConverter().setProvider(PROVIDER_BC) .getCertificate(certBuilder.build(contentSignerBuilder.build(keyPair.getPrivate()))); createAndSaveKeystore(tmpDir, keyStorePassword, certificate, (RSAPrivateKey) keyPair.getPrivate()); @@ -97,13 +109,13 @@ public static Path createKeystore(String commonName, String keyStorePassword, St private static String createAndSaveKeystore(Path tmpDir, String password, X509Certificate certificate, RSAPrivateKey privateKey) throws Exception { - KeyStore keyStore = KeyStore.getInstance("JKS"); + KeyStore keyStore = KeyStore.getInstance(KEYSTORE_TYPE); keyStore.load(null, password.toCharArray()); - keyStore.setKeyEntry("alias", privateKey, password.toCharArray(), new java.security.cert.Certificate[]{ + keyStore.setKeyEntry(TEST_ALIAS, privateKey, password.toCharArray(), new java.security.cert.Certificate[]{ certificate}); - String keyStorePath = tmpDir.resolve("keystore.jks").toString(); + String keyStorePath = tmpDir.resolve(KEYSTORE_FILE).toString(); try (FileOutputStream keyStoreOutputStream = new FileOutputStream(keyStorePath)) { keyStore.store(keyStoreOutputStream, password.toCharArray()); } @@ -113,11 +125,11 @@ private static String createAndSaveKeystore(Path tmpDir, String password, X509Ce private static String createAndSaveTruststore(Path tmpDir, String password, X509Certificate certificate) throws Exception { - KeyStore trustStore = KeyStore.getInstance("JKS"); + KeyStore trustStore = KeyStore.getInstance(KEYSTORE_TYPE); trustStore.load(null, password.toCharArray()); - trustStore.setCertificateEntry("alias", certificate); - String trustStorePath = tmpDir.resolve("truststore.jks").toString(); + trustStore.setCertificateEntry(TEST_ALIAS, certificate); + String trustStorePath = tmpDir.resolve(TRUSTSTORE_FILE).toString(); try (FileOutputStream trustStoreOutputStream = new FileOutputStream(trustStorePath)) { trustStore.store(trustStoreOutputStream, password.toCharArray()); From a094353dcfc743a3fca2f29ff8fd4937a523a1fe Mon Sep 17 00:00:00 2001 From: David Sloan Date: Wed, 11 Sep 2024 11:16:54 +0100 Subject: [PATCH 09/12] TrustStore can have an optional password. KeyStore must have a password. Making this clear via type hierarchy. --- .../exception/SecuritySetupException.java | 4 ++ .../common/security/KeyStoreInfo.java | 31 ++++++++++ .../common/security/StoreInfo.java | 14 +---- .../common/security/StoresInfo.java | 58 +++++++++---------- .../common/security/TrustStoreInfo.java | 32 ++++++++++ .../common/security/KeyStoreUtils.java | 31 ++++++++++ .../common/security/StoresInfoTest.java | 37 +++++++++--- 7 files changed, 158 insertions(+), 49 deletions(-) create mode 100644 java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/KeyStoreInfo.java create mode 100644 java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/TrustStoreInfo.java diff --git a/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/exception/SecuritySetupException.java b/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/exception/SecuritySetupException.java index 3e690a5292..f7e7e994af 100644 --- a/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/exception/SecuritySetupException.java +++ b/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/exception/SecuritySetupException.java @@ -17,6 +17,10 @@ public class SecuritySetupException extends StreamReactorException { + public SecuritySetupException(String message) { + super(message); + } + public SecuritySetupException(String message, Throwable cause) { super(message, cause); } diff --git a/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/KeyStoreInfo.java b/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/KeyStoreInfo.java new file mode 100644 index 0000000000..d3f583678d --- /dev/null +++ b/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/KeyStoreInfo.java @@ -0,0 +1,31 @@ +/* + * Copyright 2017-2024 Lenses.io Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.lenses.streamreactor.common.security; + +import lombok.AllArgsConstructor; +import lombok.Data; + +@AllArgsConstructor +@Data +public class KeyStoreInfo { + + private String storePath; + + private StoreType storeType; + + private String storePassword; + +} diff --git a/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/StoreInfo.java b/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/StoreInfo.java index ffca232fdc..d259fccc04 100644 --- a/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/StoreInfo.java +++ b/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/StoreInfo.java @@ -15,18 +15,10 @@ */ package io.lenses.streamreactor.common.security; -import cyclops.control.Option; -import lombok.AllArgsConstructor; -import lombok.Data; +interface StoreInfo { -@AllArgsConstructor -@Data -class StoreInfo { + String storePath(); - private String storePath; - - private StoreType storeType; - - private Option storePassword; + StoreType storeType(); } diff --git a/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/StoresInfo.java b/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/StoresInfo.java index 250780122b..23c486a6a2 100644 --- a/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/StoresInfo.java +++ b/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/StoresInfo.java @@ -48,15 +48,15 @@ public class StoresInfo { private static final String PROTOCOL_TLS = "TLS"; - private Option maybeTrustStore; - private Option maybeKeyStore; + private Option maybeTrustStore; + private Option maybeKeyStore; private Try getJksStore(String path, StoreType storeType, Option password) { return Try.withCatch( () -> { val keyStore = KeyStore.getInstance(storeType.toString()); val inputStream = new FileInputStream(path); - keyStore.load(inputStream, (password.orElse("")).toCharArray()); + keyStore.load(inputStream, password.map(String::toCharArray).orElse(null)); return keyStore; }, Exception.class @@ -142,12 +142,12 @@ private Try trustManagers(String pa } private Try keyManagers(String path, StoreType storeType, - Option password) { + String password) { return Try.narrowK( Do.forEach( TryInstances.monad() ) - .__(getJksStore(path, storeType, password)) + .__(getJksStore(path, storeType, Option.of(password))) .__(s -> StoresInfo.getKeyManagerFactoryFromKeyStore(s, password)) .yield( (KeyStore keyStore, KeyManagerFactory trustManagerFactory) -> trustManagerFactory @@ -166,31 +166,18 @@ private static Try getTrustManagerF } private static Try getKeyManagerFactoryFromKeyStore(KeyStore keyStore, - Option password) { + String password) { return Try.withCatch(() -> { val keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); - keyManagerFactory.init(keyStore, (password.orElse("")).toCharArray()); + keyManagerFactory.init(keyStore, password.toCharArray()); return keyManagerFactory; }, NoSuchAlgorithmException.class, KeyStoreException.class, UnrecoverableKeyException.class) .mapFailure(ex -> new SecuritySetupException("Unable to get trust manager factory from truststore", ex)); } public static Either fromConfig(AbstractConfig config) { - val trustStore = - configToTrustStoreInfo( - config, - SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, - SslConfigs.SSL_TRUSTSTORE_TYPE_CONFIG, - SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG - ); - - val keyStore = - configToTrustStoreInfo( - config, - SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG, - SslConfigs.SSL_KEYSTORE_TYPE_CONFIG, - SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG - ); + val trustStore = configToTrustStoreInfo(config); + val keyStore = configToKeyStoreInfo(config); val failures = Stream.of(trustStore, keyStore) @@ -205,19 +192,32 @@ public static Either fromConfig(AbstractConf : Either.left(failures.iterator().next()); } - private static Option> configToTrustStoreInfo(AbstractConfig config, - String sslTruststoreLocationConfig, String sslTruststoreTypeConfig, String sslTruststorePasswordConfig) { - return Option.fromNullable(config.getString(sslTruststoreLocationConfig)) - .map(storePath -> fromConfigOption(config, sslTruststoreTypeConfig) - .map(sT -> { + private static Option> configToTrustStoreInfo(AbstractConfig config) { + return Option.ofNullable(config.getString(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG)) + .map(storePath -> fromConfigOption(config, SslConfigs.SSL_TRUSTSTORE_TYPE_CONFIG) + .map(storeType -> { val storePassword = - Option.fromNullable(config.getPassword(sslTruststorePasswordConfig)) + Option.ofNullable(config.getPassword(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG)) .map(Password::value); - return new StoreInfo(storePath, sT, storePassword); + return new TrustStoreInfo(storePath, storeType, storePassword); } )); } + private static Option> configToKeyStoreInfo( + AbstractConfig config + ) { + return Option.ofNullable(config.getString(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG)) + .flatMap(storePath -> fromConfigOption(config, SslConfigs.SSL_KEYSTORE_TYPE_CONFIG) + .map(storeType -> Option.ofNullable(config.getPassword(SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG)) + .map(Password::value) + .toEither(new SecuritySetupException("Password is required for key store")) + .map(pw -> new KeyStoreInfo(storePath, storeType, pw)) + ) + .toOption() + ); + } + private static Either fromConfigOption(AbstractConfig config, String configKey) { return Option .fromNullable(config.getString(configKey)) diff --git a/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/TrustStoreInfo.java b/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/TrustStoreInfo.java new file mode 100644 index 0000000000..0379013256 --- /dev/null +++ b/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/TrustStoreInfo.java @@ -0,0 +1,32 @@ +/* + * Copyright 2017-2024 Lenses.io Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.lenses.streamreactor.common.security; + +import cyclops.control.Option; +import lombok.AllArgsConstructor; +import lombok.Data; + +@AllArgsConstructor +@Data +public class TrustStoreInfo { + + private String storePath; + + private StoreType storeType; + + private Option storePassword; + +} diff --git a/java-connectors/kafka-connect-common/src/test/java/io/lenses/streamreactor/common/security/KeyStoreUtils.java b/java-connectors/kafka-connect-common/src/test/java/io/lenses/streamreactor/common/security/KeyStoreUtils.java index 6a2c80cd0a..9adb3eee40 100644 --- a/java-connectors/kafka-connect-common/src/test/java/io/lenses/streamreactor/common/security/KeyStoreUtils.java +++ b/java-connectors/kafka-connect-common/src/test/java/io/lenses/streamreactor/common/security/KeyStoreUtils.java @@ -52,6 +52,9 @@ import java.time.Instant; import java.util.Date; +/** + * Utility class for creating and managing keystore and truststore files using RSA keys and X.509 certificates. + */ @Slf4j public class KeyStoreUtils { @@ -71,6 +74,15 @@ public class KeyStoreUtils { Security.addProvider(new BouncyCastleProvider()); } + /** + * Creates a new keystore and truststore with an RSA key pair and X.509 certificate. + * + * @param commonName the common name for the certificate + * @param keyStorePassword the password for the keystore + * @param trustStorePassword the password for the truststore + * @return Path to the temporary directory where keystore and truststore are created + * @throws Exception if an error occurs during the creation process + */ public static Path createKeystore(String commonName, String keyStorePassword, String trustStorePassword) throws Exception { Path tmpDir = Files.createTempDirectory(DIRECTORY_SECURITY); @@ -107,6 +119,16 @@ public static Path createKeystore(String commonName, String keyStorePassword, St return tmpDir; } + /** + * Creates and saves the keystore with the given password and certificate. + * + * @param tmpDir the temporary directory where the keystore will be saved + * @param password the password for the keystore + * @param certificate the certificate to store in the keystore + * @param privateKey the private key to store in the keystore + * @return the path to the created keystore file + * @throws Exception if an error occurs during the keystore creation process + */ private static String createAndSaveKeystore(Path tmpDir, String password, X509Certificate certificate, RSAPrivateKey privateKey) throws Exception { KeyStore keyStore = KeyStore.getInstance(KEYSTORE_TYPE); @@ -123,6 +145,15 @@ private static String createAndSaveKeystore(Path tmpDir, String password, X509Ce return keyStorePath; } + /** + * Creates and saves the truststore with the given password and certificate. + * + * @param tmpDir the temporary directory where the truststore will be saved + * @param password the password for the truststore + * @param certificate the certificate to store in the truststore + * @return the path to the created truststore file + * @throws Exception if an error occurs during the truststore creation process + */ private static String createAndSaveTruststore(Path tmpDir, String password, X509Certificate certificate) throws Exception { KeyStore trustStore = KeyStore.getInstance(KEYSTORE_TYPE); diff --git a/java-connectors/kafka-connect-common/src/test/java/io/lenses/streamreactor/common/security/StoresInfoTest.java b/java-connectors/kafka-connect-common/src/test/java/io/lenses/streamreactor/common/security/StoresInfoTest.java index 39af7823ff..ab812a8892 100644 --- a/java-connectors/kafka-connect-common/src/test/java/io/lenses/streamreactor/common/security/StoresInfoTest.java +++ b/java-connectors/kafka-connect-common/src/test/java/io/lenses/streamreactor/common/security/StoresInfoTest.java @@ -18,6 +18,7 @@ import io.lenses.streamreactor.common.config.base.BaseConfig; import lombok.val; import org.apache.kafka.common.config.SslConfigs; +import org.apache.kafka.common.config.types.Password; import org.junit.jupiter.api.Test; import java.io.FileNotFoundException; @@ -26,6 +27,7 @@ import static cyclops.control.Either.right; import static cyclops.control.Option.none; import static cyclops.control.Option.some; +import static io.lenses.streamreactor.test.utils.EitherValues.assertLeft; import static io.lenses.streamreactor.test.utils.EitherValues.assertRight; import static io.lenses.streamreactor.test.utils.EitherValues.getLeft; import static io.lenses.streamreactor.test.utils.EitherValues.getRight; @@ -52,7 +54,7 @@ void testToSslContextWithBothNone() { @Test void testToSslContextWithKeyStoreDefined() { - val storeInfo = new StoreInfo(keystoreFile, StoreType.JKS, some(password)); + val storeInfo = new KeyStoreInfo(keystoreFile, StoreType.JKS, password); val storesInfo = new StoresInfo(none(), some(storeInfo)); val sslContext = getRight(storesInfo.toSslContext()); @@ -62,7 +64,7 @@ void testToSslContextWithKeyStoreDefined() { @Test void testToSslContextWithTrustStoreDefined() { - val storeInfo = new StoreInfo(keystoreFile, StoreType.JKS, some(password)); + val storeInfo = new TrustStoreInfo(keystoreFile, StoreType.JKS, some(password)); val storesInfo = new StoresInfo(some(storeInfo), none()); val sslContext = getRight(storesInfo.toSslContext()); @@ -72,8 +74,8 @@ void testToSslContextWithTrustStoreDefined() { @Test void testToSslContextWithBothStoresDefined() { - val keyStoreInfo = new StoreInfo(keystoreFile, StoreType.JKS, some(password)); - val trustStoreInfo = new StoreInfo(truststoreFile, StoreType.JKS, some(password)); + val keyStoreInfo = new KeyStoreInfo(keystoreFile, StoreType.JKS, password); + val trustStoreInfo = new TrustStoreInfo(truststoreFile, StoreType.JKS, some(password)); val storesInfo = new StoresInfo(some(trustStoreInfo), some(keyStoreInfo)); val sslContext = getRight(storesInfo.toSslContext()); @@ -83,7 +85,7 @@ void testToSslContextWithBothStoresDefined() { @Test void testToSslContextThrowsFileNotFoundExceptionForInvalidKeyStorePath() { - val keyStoreInfo = new StoreInfo("/invalid/path/to/keystore", StoreType.JKS, some(password)); + val keyStoreInfo = new KeyStoreInfo("/invalid/path/to/keystore", StoreType.JKS, password); val storesInfo = new StoresInfo(none(), some(keyStoreInfo)); assertEquals(FileNotFoundException.class, getLeft(storesInfo.toSslContext()).getCause().getClass()); @@ -91,7 +93,7 @@ void testToSslContextThrowsFileNotFoundExceptionForInvalidKeyStorePath() { @Test void testToSslContextThrowsFileNotFoundExceptionForInvalidTrustStorePath() { - val trustStoreInfo = new StoreInfo("/invalid/path/to/truststore", StoreType.JKS, some(password)); + val trustStoreInfo = new TrustStoreInfo("/invalid/path/to/truststore", StoreType.JKS, some(password)); val storesInfo = new StoresInfo(some(trustStoreInfo), none()); assertEquals(FileNotFoundException.class, getLeft(storesInfo.toSslContext()).getCause().getClass()); @@ -107,18 +109,35 @@ void testStoresInfoCreationFromBaseConfig() { when(mockConfig.getString(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG)).thenReturn("/path/to/keystore"); when(mockConfig.getString(SslConfigs.SSL_KEYSTORE_TYPE_CONFIG)).thenReturn("JKS"); - when(mockConfig.getPassword(SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG)).thenReturn(null); + when(mockConfig.getPassword(SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG)).thenReturn(new Password(password)); val storesInfo = StoresInfo.fromConfig(mockConfig); assertRight(storesInfo).isEqualTo( new StoresInfo( - some(new StoreInfo("/path/to/truststore", StoreType.JKS, none())), - some(new StoreInfo("/path/to/keystore", StoreType.JKS, none())) + some(new TrustStoreInfo("/path/to/truststore", StoreType.JKS, none())), + some(new KeyStoreInfo("/path/to/keystore", StoreType.JKS, password)) ) ); } + @Test + void testStoresInfoCreationFromBaseConfigFailsWithoutTrustStorePassword() { + BaseConfig mockConfig = mock(BaseConfig.class); + when(mockConfig.getString(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG)).thenReturn("/path/to/truststore"); + when(mockConfig.getString(SslConfigs.SSL_TRUSTSTORE_TYPE_CONFIG)).thenReturn("JKS"); + when(mockConfig.getPassword(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG)).thenReturn(null); + + when(mockConfig.getString(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG)).thenReturn("/path/to/keystore"); + when(mockConfig.getString(SslConfigs.SSL_KEYSTORE_TYPE_CONFIG)).thenReturn("JKS"); + when(mockConfig.getPassword(SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG)).thenReturn(null); + + val storesInfo = StoresInfo.fromConfig(mockConfig); + + assertLeft(storesInfo).withFailMessage(() -> " Password is required for key store"); + + } + @Test void testStoresInfoCreationWithNoneValuesWithMissingConfigs() { BaseConfig mockConfig = mock(BaseConfig.class); From 8e9d220427e92dabaaf6d70074e4c328951055bc Mon Sep 17 00:00:00 2001 From: David Sloan Date: Wed, 11 Sep 2024 11:40:18 +0100 Subject: [PATCH 10/12] Add config for key/trust manager algorithm --- .../common/security/KeyStoreInfo.java | 3 ++ .../common/security/StoreInfo.java | 3 ++ .../common/security/StoresInfo.java | 32 ++++++++++++------- .../common/security/TrustStoreInfo.java | 2 ++ .../common/security/StoresInfoTest.java | 28 +++++++++------- 5 files changed, 45 insertions(+), 23 deletions(-) diff --git a/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/KeyStoreInfo.java b/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/KeyStoreInfo.java index d3f583678d..490fbd714f 100644 --- a/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/KeyStoreInfo.java +++ b/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/KeyStoreInfo.java @@ -15,6 +15,7 @@ */ package io.lenses.streamreactor.common.security; +import cyclops.control.Option; import lombok.AllArgsConstructor; import lombok.Data; @@ -28,4 +29,6 @@ public class KeyStoreInfo { private String storePassword; + private Option managerAlgorithm; + } diff --git a/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/StoreInfo.java b/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/StoreInfo.java index d259fccc04..c6adbfa279 100644 --- a/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/StoreInfo.java +++ b/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/StoreInfo.java @@ -15,10 +15,13 @@ */ package io.lenses.streamreactor.common.security; +import cyclops.control.Option; + interface StoreInfo { String storePath(); StoreType storeType(); + Option managerAlgorithm(); } diff --git a/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/StoresInfo.java b/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/StoresInfo.java index 23c486a6a2..05bed66fe1 100644 --- a/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/StoresInfo.java +++ b/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/StoresInfo.java @@ -71,7 +71,8 @@ public Either> toSslContext() { trustStore -> trustManagers( trustStore.getStorePath(), trustStore.getStoreType(), - trustStore.getStorePassword() + trustStore.getStorePassword(), + trustStore.getManagerAlgorithm() ) ); @@ -80,7 +81,8 @@ public Either> toSslContext() { keyStore -> keyManagers( keyStore.getStorePath(), keyStore.getStoreType(), - keyStore.getStorePassword() + keyStore.getStorePassword(), + keyStore.getManagerAlgorithm() ) ); @@ -127,13 +129,13 @@ private static Either> getAndInitSslC } private Try trustManagers(String path, StoreType storeType, - Option password) { + Option password, Option algorithm) { return Try.narrowK( Do.forEach( TryInstances.monad() ) .__(getJksStore(path, storeType, password)) - .__(StoresInfo::getTrustManagerFactoryFromKeyStore) + .__((KeyStore keyStore) -> StoresInfo.getTrustManagerFactoryFromKeyStore(keyStore, algorithm)) .yield( (KeyStore keyStore, TrustManagerFactory trustManagerFactory) -> trustManagerFactory ) @@ -142,13 +144,13 @@ private Try trustManagers(String pa } private Try keyManagers(String path, StoreType storeType, - String password) { + String password, Option algorithm) { return Try.narrowK( Do.forEach( TryInstances.monad() ) .__(getJksStore(path, storeType, Option.of(password))) - .__(s -> StoresInfo.getKeyManagerFactoryFromKeyStore(s, password)) + .__((KeyStore keyStore) -> StoresInfo.getKeyManagerFactoryFromKeyStore(keyStore, password, algorithm)) .yield( (KeyStore keyStore, KeyManagerFactory trustManagerFactory) -> trustManagerFactory ) @@ -156,9 +158,10 @@ private Try keyManagers(String path, } private static Try getTrustManagerFactoryFromKeyStore( - KeyStore keyStore) { + KeyStore keyStore, Option algorithm) { return Try.withCatch(() -> { - val trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + val trustManagerFactory = + TrustManagerFactory.getInstance(algorithm.orElse(TrustManagerFactory.getDefaultAlgorithm())); trustManagerFactory.init(keyStore); return trustManagerFactory; }, NoSuchAlgorithmException.class, KeyStoreException.class) @@ -166,9 +169,9 @@ private static Try getTrustManagerF } private static Try getKeyManagerFactoryFromKeyStore(KeyStore keyStore, - String password) { + String password, Option algorithm) { return Try.withCatch(() -> { - val keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); + val keyManagerFactory = KeyManagerFactory.getInstance(algorithm.orElse(KeyManagerFactory.getDefaultAlgorithm())); keyManagerFactory.init(keyStore, password.toCharArray()); return keyManagerFactory; }, NoSuchAlgorithmException.class, KeyStoreException.class, UnrecoverableKeyException.class) @@ -199,7 +202,8 @@ private static Option> configToTr val storePassword = Option.ofNullable(config.getPassword(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG)) .map(Password::value); - return new TrustStoreInfo(storePath, storeType, storePassword); + val managerAlgorithm = Option.ofNullable(config.getString(SslConfigs.SSL_TRUSTMANAGER_ALGORITHM_CONFIG)); + return new TrustStoreInfo(storePath, storeType, storePassword, managerAlgorithm); } )); } @@ -212,7 +216,11 @@ private static Option> configToKeyS .map(storeType -> Option.ofNullable(config.getPassword(SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG)) .map(Password::value) .toEither(new SecuritySetupException("Password is required for key store")) - .map(pw -> new KeyStoreInfo(storePath, storeType, pw)) + .map(pw -> { + val managerAlgorithm = + Option.ofNullable(config.getString(SslConfigs.SSL_TRUSTMANAGER_ALGORITHM_CONFIG)); + return new KeyStoreInfo(storePath, storeType, pw, managerAlgorithm); + }) ) .toOption() ); diff --git a/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/TrustStoreInfo.java b/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/TrustStoreInfo.java index 0379013256..3838431c05 100644 --- a/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/TrustStoreInfo.java +++ b/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/TrustStoreInfo.java @@ -29,4 +29,6 @@ public class TrustStoreInfo { private Option storePassword; + private Option managerAlgorithm; + } diff --git a/java-connectors/kafka-connect-common/src/test/java/io/lenses/streamreactor/common/security/StoresInfoTest.java b/java-connectors/kafka-connect-common/src/test/java/io/lenses/streamreactor/common/security/StoresInfoTest.java index ab812a8892..65e30c0887 100644 --- a/java-connectors/kafka-connect-common/src/test/java/io/lenses/streamreactor/common/security/StoresInfoTest.java +++ b/java-connectors/kafka-connect-common/src/test/java/io/lenses/streamreactor/common/security/StoresInfoTest.java @@ -38,8 +38,10 @@ class StoresInfoTest { - private final String password = "changeIt"; - private final Path keystoreDir = KeyStoreUtils.createKeystore("TestCommonName", password, password); + private static final String KEY_OR_TRUST_MANAGER_ALGORITHM = "PKIX"; + private static final String STORE_PASSWORD = "changeIt"; + + private final Path keystoreDir = KeyStoreUtils.createKeystore("TestCommonName", STORE_PASSWORD, STORE_PASSWORD); private final String keystoreFile = keystoreDir.toAbsolutePath() + "/keystore.jks"; private final String truststoreFile = keystoreDir.toAbsolutePath() + "/truststore.jks"; @@ -54,7 +56,7 @@ void testToSslContextWithBothNone() { @Test void testToSslContextWithKeyStoreDefined() { - val storeInfo = new KeyStoreInfo(keystoreFile, StoreType.JKS, password); + val storeInfo = new KeyStoreInfo(keystoreFile, StoreType.JKS, STORE_PASSWORD, none()); val storesInfo = new StoresInfo(none(), some(storeInfo)); val sslContext = getRight(storesInfo.toSslContext()); @@ -64,7 +66,7 @@ void testToSslContextWithKeyStoreDefined() { @Test void testToSslContextWithTrustStoreDefined() { - val storeInfo = new TrustStoreInfo(keystoreFile, StoreType.JKS, some(password)); + val storeInfo = new TrustStoreInfo(keystoreFile, StoreType.JKS, some(STORE_PASSWORD), none()); val storesInfo = new StoresInfo(some(storeInfo), none()); val sslContext = getRight(storesInfo.toSslContext()); @@ -74,8 +76,8 @@ void testToSslContextWithTrustStoreDefined() { @Test void testToSslContextWithBothStoresDefined() { - val keyStoreInfo = new KeyStoreInfo(keystoreFile, StoreType.JKS, password); - val trustStoreInfo = new TrustStoreInfo(truststoreFile, StoreType.JKS, some(password)); + val keyStoreInfo = new KeyStoreInfo(keystoreFile, StoreType.JKS, STORE_PASSWORD, none()); + val trustStoreInfo = new TrustStoreInfo(truststoreFile, StoreType.JKS, some(STORE_PASSWORD), none()); val storesInfo = new StoresInfo(some(trustStoreInfo), some(keyStoreInfo)); val sslContext = getRight(storesInfo.toSslContext()); @@ -85,7 +87,7 @@ void testToSslContextWithBothStoresDefined() { @Test void testToSslContextThrowsFileNotFoundExceptionForInvalidKeyStorePath() { - val keyStoreInfo = new KeyStoreInfo("/invalid/path/to/keystore", StoreType.JKS, password); + val keyStoreInfo = new KeyStoreInfo("/invalid/path/to/keystore", StoreType.JKS, STORE_PASSWORD, none()); val storesInfo = new StoresInfo(none(), some(keyStoreInfo)); assertEquals(FileNotFoundException.class, getLeft(storesInfo.toSslContext()).getCause().getClass()); @@ -93,7 +95,7 @@ void testToSslContextThrowsFileNotFoundExceptionForInvalidKeyStorePath() { @Test void testToSslContextThrowsFileNotFoundExceptionForInvalidTrustStorePath() { - val trustStoreInfo = new TrustStoreInfo("/invalid/path/to/truststore", StoreType.JKS, some(password)); + val trustStoreInfo = new TrustStoreInfo("/invalid/path/to/truststore", StoreType.JKS, some(STORE_PASSWORD), none()); val storesInfo = new StoresInfo(some(trustStoreInfo), none()); assertEquals(FileNotFoundException.class, getLeft(storesInfo.toSslContext()).getCause().getClass()); @@ -106,17 +108,21 @@ void testStoresInfoCreationFromBaseConfig() { when(mockConfig.getString(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG)).thenReturn("/path/to/truststore"); when(mockConfig.getString(SslConfigs.SSL_TRUSTSTORE_TYPE_CONFIG)).thenReturn("JKS"); when(mockConfig.getPassword(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG)).thenReturn(null); + when(mockConfig.getString(SslConfigs.SSL_TRUSTMANAGER_ALGORITHM_CONFIG)).thenReturn(KEY_OR_TRUST_MANAGER_ALGORITHM); when(mockConfig.getString(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG)).thenReturn("/path/to/keystore"); when(mockConfig.getString(SslConfigs.SSL_KEYSTORE_TYPE_CONFIG)).thenReturn("JKS"); - when(mockConfig.getPassword(SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG)).thenReturn(new Password(password)); + when(mockConfig.getPassword(SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG)).thenReturn(new Password(STORE_PASSWORD)); + when(mockConfig.getString(SslConfigs.SSL_KEYMANAGER_ALGORITHM_CONFIG)).thenReturn(KEY_OR_TRUST_MANAGER_ALGORITHM); val storesInfo = StoresInfo.fromConfig(mockConfig); assertRight(storesInfo).isEqualTo( new StoresInfo( - some(new TrustStoreInfo("/path/to/truststore", StoreType.JKS, none())), - some(new KeyStoreInfo("/path/to/keystore", StoreType.JKS, password)) + some(new TrustStoreInfo("/path/to/truststore", StoreType.JKS, none(), some( + KEY_OR_TRUST_MANAGER_ALGORITHM))), + some(new KeyStoreInfo("/path/to/keystore", StoreType.JKS, STORE_PASSWORD, some( + KEY_OR_TRUST_MANAGER_ALGORITHM))) ) ); } From 6458131eff7bfd7389c3cfb5e659cd9c45fc2810 Mon Sep 17 00:00:00 2001 From: David Sloan Date: Wed, 11 Sep 2024 12:24:30 +0100 Subject: [PATCH 11/12] SSL Protocol property --- .../common/security/StoresInfo.java | 16 +++++++++++--- .../common/security/StoresInfoTest.java | 21 +++++++++++-------- 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/StoresInfo.java b/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/StoresInfo.java index 05bed66fe1..d03f6061bc 100644 --- a/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/StoresInfo.java +++ b/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/StoresInfo.java @@ -48,6 +48,7 @@ public class StoresInfo { private static final String PROTOCOL_TLS = "TLS"; + private Option sslProtocol; private Option maybeTrustStore; private Option maybeKeyStore; @@ -102,7 +103,7 @@ public Either> toSslContext() { ); return maybeFailure - .toEither(getAndInitSslContext(maybeKeyFactory, maybeTrustFactory)) + .toEither(getAndInitSslContext(maybeKeyFactory, maybeTrustFactory, sslProtocol)) .swap() .fold(Either::left, either -> either.fold(Either::left, Either::right)); @@ -110,12 +111,13 @@ public Either> toSslContext() { private static Either> getAndInitSslContext( Option> maybeKeyFactory, - Option> maybeTrustFactory + Option> maybeTrustFactory, + Option sslProtocol ) { return Try.withCatch(() -> { // If either factory is present, initialize SSLContext if (maybeKeyFactory.isPresent() || maybeTrustFactory.isPresent()) { - val sslContext = SSLContext.getInstance(PROTOCOL_TLS); + val sslContext = SSLContext.getInstance(sslProtocol.orElse(PROTOCOL_TLS)); sslContext.init( maybeKeyFactory.flatMap(Try::toOption).map(KeyManagerFactory::getKeyManagers).orElse(null), maybeTrustFactory.flatMap(Try::toOption).map(TrustManagerFactory::getTrustManagers).orElse(null), @@ -179,6 +181,7 @@ private static Try getKeyManagerFacto } public static Either fromConfig(AbstractConfig config) { + val sslProtocol = configToSslProtocol(config); val trustStore = configToTrustStoreInfo(config); val keyStore = configToKeyStoreInfo(config); @@ -189,6 +192,7 @@ public static Either fromConfig(AbstractConf return failures.isEmpty() ? Either.right(new StoresInfo( + sslProtocol, trustStore.flatMap(Either::toOption), keyStore.flatMap(Either::toOption) )) @@ -226,6 +230,12 @@ private static Option> configToKeyS ); } + private static Option configToSslProtocol( + AbstractConfig config + ) { + return Option.ofNullable(config.getString(SslConfigs.SSL_PROTOCOL_CONFIG)); + } + private static Either fromConfigOption(AbstractConfig config, String configKey) { return Option .fromNullable(config.getString(configKey)) diff --git a/java-connectors/kafka-connect-common/src/test/java/io/lenses/streamreactor/common/security/StoresInfoTest.java b/java-connectors/kafka-connect-common/src/test/java/io/lenses/streamreactor/common/security/StoresInfoTest.java index 65e30c0887..1220eb758f 100644 --- a/java-connectors/kafka-connect-common/src/test/java/io/lenses/streamreactor/common/security/StoresInfoTest.java +++ b/java-connectors/kafka-connect-common/src/test/java/io/lenses/streamreactor/common/security/StoresInfoTest.java @@ -40,6 +40,7 @@ class StoresInfoTest { private static final String KEY_OR_TRUST_MANAGER_ALGORITHM = "PKIX"; private static final String STORE_PASSWORD = "changeIt"; + private static final String SSL_PROTOCOL_TLS = "TLS"; private final Path keystoreDir = KeyStoreUtils.createKeystore("TestCommonName", STORE_PASSWORD, STORE_PASSWORD); private final String keystoreFile = keystoreDir.toAbsolutePath() + "/keystore.jks"; @@ -50,45 +51,45 @@ class StoresInfoTest { @Test void testToSslContextWithBothNone() { - val storesInfo = new StoresInfo(none(), none()); + val storesInfo = new StoresInfo(none(), none(), none()); assertEquals(right(none()), storesInfo.toSslContext()); } @Test void testToSslContextWithKeyStoreDefined() { val storeInfo = new KeyStoreInfo(keystoreFile, StoreType.JKS, STORE_PASSWORD, none()); - val storesInfo = new StoresInfo(none(), some(storeInfo)); + val storesInfo = new StoresInfo(some(SSL_PROTOCOL_TLS), none(), some(storeInfo)); val sslContext = getRight(storesInfo.toSslContext()); - assertEquals("TLS", getValue(sslContext).getProtocol()); + assertEquals(SSL_PROTOCOL_TLS, getValue(sslContext).getProtocol()); } @Test void testToSslContextWithTrustStoreDefined() { val storeInfo = new TrustStoreInfo(keystoreFile, StoreType.JKS, some(STORE_PASSWORD), none()); - val storesInfo = new StoresInfo(some(storeInfo), none()); + val storesInfo = new StoresInfo(none(), some(storeInfo), none()); val sslContext = getRight(storesInfo.toSslContext()); - assertEquals("TLS", getValue(sslContext).getProtocol()); + assertEquals(SSL_PROTOCOL_TLS, getValue(sslContext).getProtocol()); } @Test void testToSslContextWithBothStoresDefined() { val keyStoreInfo = new KeyStoreInfo(keystoreFile, StoreType.JKS, STORE_PASSWORD, none()); val trustStoreInfo = new TrustStoreInfo(truststoreFile, StoreType.JKS, some(STORE_PASSWORD), none()); - val storesInfo = new StoresInfo(some(trustStoreInfo), some(keyStoreInfo)); + val storesInfo = new StoresInfo(some(SSL_PROTOCOL_TLS), some(trustStoreInfo), some(keyStoreInfo)); val sslContext = getRight(storesInfo.toSslContext()); - assertEquals("TLS", getValue(sslContext).getProtocol()); + assertEquals(SSL_PROTOCOL_TLS, getValue(sslContext).getProtocol()); } @Test void testToSslContextThrowsFileNotFoundExceptionForInvalidKeyStorePath() { val keyStoreInfo = new KeyStoreInfo("/invalid/path/to/keystore", StoreType.JKS, STORE_PASSWORD, none()); - val storesInfo = new StoresInfo(none(), some(keyStoreInfo)); + val storesInfo = new StoresInfo(none(), none(), some(keyStoreInfo)); assertEquals(FileNotFoundException.class, getLeft(storesInfo.toSslContext()).getCause().getClass()); } @@ -96,7 +97,7 @@ void testToSslContextThrowsFileNotFoundExceptionForInvalidKeyStorePath() { @Test void testToSslContextThrowsFileNotFoundExceptionForInvalidTrustStorePath() { val trustStoreInfo = new TrustStoreInfo("/invalid/path/to/truststore", StoreType.JKS, some(STORE_PASSWORD), none()); - val storesInfo = new StoresInfo(some(trustStoreInfo), none()); + val storesInfo = new StoresInfo(some(SSL_PROTOCOL_TLS), some(trustStoreInfo), none()); assertEquals(FileNotFoundException.class, getLeft(storesInfo.toSslContext()).getCause().getClass()); @@ -119,6 +120,7 @@ void testStoresInfoCreationFromBaseConfig() { assertRight(storesInfo).isEqualTo( new StoresInfo( + none(), some(new TrustStoreInfo("/path/to/truststore", StoreType.JKS, none(), some( KEY_OR_TRUST_MANAGER_ALGORITHM))), some(new KeyStoreInfo("/path/to/keystore", StoreType.JKS, STORE_PASSWORD, some( @@ -158,6 +160,7 @@ void testStoresInfoCreationWithNoneValuesWithMissingConfigs() { val storesInfo = StoresInfo.fromConfig(mockConfig); assertRight(storesInfo).isEqualTo( new StoresInfo( + none(), none(), none() ) From 70afcaf4f3f2db1df84d10f2a52ad1b4db428c3b Mon Sep 17 00:00:00 2001 From: David Sloan Date: Wed, 11 Sep 2024 13:09:28 +0100 Subject: [PATCH 12/12] Change String to Path --- .../common/security/KeyStoreInfo.java | 6 ++++-- .../common/security/StoreInfo.java | 8 +++++--- .../common/security/StoresInfo.java | 11 +++++++---- .../common/security/TrustStoreInfo.java | 6 ++++-- .../common/security/StoresInfoTest.java | 19 ++++++++++++------- .../elastic6/config/ElasticSettings.scala | 2 +- .../elastic7/config/ElasticSettings.scala | 2 +- 7 files changed, 34 insertions(+), 20 deletions(-) diff --git a/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/KeyStoreInfo.java b/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/KeyStoreInfo.java index 490fbd714f..9bf4ceed93 100644 --- a/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/KeyStoreInfo.java +++ b/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/KeyStoreInfo.java @@ -19,11 +19,13 @@ import lombok.AllArgsConstructor; import lombok.Data; +import java.nio.file.Path; + @AllArgsConstructor @Data -public class KeyStoreInfo { +public class KeyStoreInfo implements StoreInfo { - private String storePath; + private Path storePath; private StoreType storeType; diff --git a/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/StoreInfo.java b/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/StoreInfo.java index c6adbfa279..0f72defa98 100644 --- a/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/StoreInfo.java +++ b/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/StoreInfo.java @@ -17,11 +17,13 @@ import cyclops.control.Option; +import java.nio.file.Path; + interface StoreInfo { - String storePath(); + Path getStorePath(); - StoreType storeType(); + StoreType getStoreType(); - Option managerAlgorithm(); + Option getManagerAlgorithm(); } diff --git a/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/StoresInfo.java b/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/StoresInfo.java index d03f6061bc..9159ca7d02 100644 --- a/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/StoresInfo.java +++ b/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/StoresInfo.java @@ -32,6 +32,7 @@ import javax.net.ssl.SSLContext; import javax.net.ssl.TrustManagerFactory; import java.io.FileInputStream; +import java.nio.file.Path; import java.security.KeyStore; import java.security.KeyStoreException; import java.security.NoSuchAlgorithmException; @@ -52,11 +53,11 @@ public class StoresInfo { private Option maybeTrustStore; private Option maybeKeyStore; - private Try getJksStore(String path, StoreType storeType, Option password) { + private Try getJksStore(Path path, StoreType storeType, Option password) { return Try.withCatch( () -> { val keyStore = KeyStore.getInstance(storeType.toString()); - val inputStream = new FileInputStream(path); + val inputStream = new FileInputStream(path.toFile()); keyStore.load(inputStream, password.map(String::toCharArray).orElse(null)); return keyStore; }, @@ -130,7 +131,7 @@ private static Either> getAndInitSslC .toEither(); } - private Try trustManagers(String path, StoreType storeType, + private Try trustManagers(Path path, StoreType storeType, Option password, Option algorithm) { return Try.narrowK( Do.forEach( @@ -145,7 +146,7 @@ private Try trustManagers(String pa } - private Try keyManagers(String path, StoreType storeType, + private Try keyManagers(Path path, StoreType storeType, String password, Option algorithm) { return Try.narrowK( Do.forEach( @@ -201,6 +202,7 @@ public static Either fromConfig(AbstractConf private static Option> configToTrustStoreInfo(AbstractConfig config) { return Option.ofNullable(config.getString(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG)) + .map(Path::of) .map(storePath -> fromConfigOption(config, SslConfigs.SSL_TRUSTSTORE_TYPE_CONFIG) .map(storeType -> { val storePassword = @@ -216,6 +218,7 @@ private static Option> configToKeyS AbstractConfig config ) { return Option.ofNullable(config.getString(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG)) + .map(Path::of) .flatMap(storePath -> fromConfigOption(config, SslConfigs.SSL_KEYSTORE_TYPE_CONFIG) .map(storeType -> Option.ofNullable(config.getPassword(SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG)) .map(Password::value) diff --git a/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/TrustStoreInfo.java b/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/TrustStoreInfo.java index 3838431c05..6809907fd1 100644 --- a/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/TrustStoreInfo.java +++ b/java-connectors/kafka-connect-common/src/main/java/io/lenses/streamreactor/common/security/TrustStoreInfo.java @@ -19,11 +19,13 @@ import lombok.AllArgsConstructor; import lombok.Data; +import java.nio.file.Path; + @AllArgsConstructor @Data -public class TrustStoreInfo { +public class TrustStoreInfo implements StoreInfo { - private String storePath; + private Path storePath; private StoreType storeType; diff --git a/java-connectors/kafka-connect-common/src/test/java/io/lenses/streamreactor/common/security/StoresInfoTest.java b/java-connectors/kafka-connect-common/src/test/java/io/lenses/streamreactor/common/security/StoresInfoTest.java index 1220eb758f..987e429eb4 100644 --- a/java-connectors/kafka-connect-common/src/test/java/io/lenses/streamreactor/common/security/StoresInfoTest.java +++ b/java-connectors/kafka-connect-common/src/test/java/io/lenses/streamreactor/common/security/StoresInfoTest.java @@ -16,6 +16,7 @@ package io.lenses.streamreactor.common.security; import io.lenses.streamreactor.common.config.base.BaseConfig; +import io.lenses.streamreactor.common.exception.SecuritySetupException; import lombok.val; import org.apache.kafka.common.config.SslConfigs; import org.apache.kafka.common.config.types.Password; @@ -32,6 +33,7 @@ import static io.lenses.streamreactor.test.utils.EitherValues.getLeft; import static io.lenses.streamreactor.test.utils.EitherValues.getRight; import static io.lenses.streamreactor.test.utils.OptionValues.getValue; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -43,8 +45,8 @@ class StoresInfoTest { private static final String SSL_PROTOCOL_TLS = "TLS"; private final Path keystoreDir = KeyStoreUtils.createKeystore("TestCommonName", STORE_PASSWORD, STORE_PASSWORD); - private final String keystoreFile = keystoreDir.toAbsolutePath() + "/keystore.jks"; - private final String truststoreFile = keystoreDir.toAbsolutePath() + "/truststore.jks"; + private final Path keystoreFile = keystoreDir.resolve("keystore.jks"); + private final Path truststoreFile = keystoreDir.resolve("truststore.jks"); StoresInfoTest() throws Exception { } @@ -88,7 +90,7 @@ void testToSslContextWithBothStoresDefined() { @Test void testToSslContextThrowsFileNotFoundExceptionForInvalidKeyStorePath() { - val keyStoreInfo = new KeyStoreInfo("/invalid/path/to/keystore", StoreType.JKS, STORE_PASSWORD, none()); + val keyStoreInfo = new KeyStoreInfo(Path.of("/invalid/path/to/keystore"), StoreType.JKS, STORE_PASSWORD, none()); val storesInfo = new StoresInfo(none(), none(), some(keyStoreInfo)); assertEquals(FileNotFoundException.class, getLeft(storesInfo.toSslContext()).getCause().getClass()); @@ -96,7 +98,8 @@ void testToSslContextThrowsFileNotFoundExceptionForInvalidKeyStorePath() { @Test void testToSslContextThrowsFileNotFoundExceptionForInvalidTrustStorePath() { - val trustStoreInfo = new TrustStoreInfo("/invalid/path/to/truststore", StoreType.JKS, some(STORE_PASSWORD), none()); + val trustStoreInfo = + new TrustStoreInfo(Path.of("/invalid/path/to/truststore"), StoreType.JKS, some(STORE_PASSWORD), none()); val storesInfo = new StoresInfo(some(SSL_PROTOCOL_TLS), some(trustStoreInfo), none()); assertEquals(FileNotFoundException.class, getLeft(storesInfo.toSslContext()).getCause().getClass()); @@ -121,9 +124,9 @@ void testStoresInfoCreationFromBaseConfig() { assertRight(storesInfo).isEqualTo( new StoresInfo( none(), - some(new TrustStoreInfo("/path/to/truststore", StoreType.JKS, none(), some( + some(new TrustStoreInfo(Path.of("/path/to/truststore"), StoreType.JKS, none(), some( KEY_OR_TRUST_MANAGER_ALGORITHM))), - some(new KeyStoreInfo("/path/to/keystore", StoreType.JKS, STORE_PASSWORD, some( + some(new KeyStoreInfo(Path.of("/path/to/keystore"), StoreType.JKS, STORE_PASSWORD, some( KEY_OR_TRUST_MANAGER_ALGORITHM))) ) ); @@ -142,7 +145,9 @@ void testStoresInfoCreationFromBaseConfigFailsWithoutTrustStorePassword() { val storesInfo = StoresInfo.fromConfig(mockConfig); - assertLeft(storesInfo).withFailMessage(() -> " Password is required for key store"); + assertLeft(storesInfo) + .isInstanceOf(SecuritySetupException.class) + .satisfies(ex -> assertThat(ex.getMessage()).contains("Password is required for key store")); } diff --git a/kafka-connect-elastic6/src/main/scala/io/lenses/streamreactor/connect/elastic6/config/ElasticSettings.scala b/kafka-connect-elastic6/src/main/scala/io/lenses/streamreactor/connect/elastic6/config/ElasticSettings.scala index f25b7946d0..15f194afc9 100644 --- a/kafka-connect-elastic6/src/main/scala/io/lenses/streamreactor/connect/elastic6/config/ElasticSettings.scala +++ b/kafka-connect-elastic6/src/main/scala/io/lenses/streamreactor/connect/elastic6/config/ElasticSettings.scala @@ -34,7 +34,7 @@ case class ElasticSettings( pkJoinerSeparator: String = ElasticConfigConstants.PK_JOINER_SEPARATOR_DEFAULT, httpBasicAuthUsername: String = ElasticConfigConstants.CLIENT_HTTP_BASIC_AUTH_USERNAME_DEFAULT, httpBasicAuthPassword: String = ElasticConfigConstants.CLIENT_HTTP_BASIC_AUTH_USERNAME_DEFAULT, - storesInfo: StoresInfo = new StoresInfo(none(), none()), + storesInfo: StoresInfo = new StoresInfo(none(), none(), none()), ) object ElasticSettings { diff --git a/kafka-connect-elastic7/src/main/scala/io/lenses/streamreactor/connect/elastic7/config/ElasticSettings.scala b/kafka-connect-elastic7/src/main/scala/io/lenses/streamreactor/connect/elastic7/config/ElasticSettings.scala index 9de0c3c9e7..e1cd3e5066 100644 --- a/kafka-connect-elastic7/src/main/scala/io/lenses/streamreactor/connect/elastic7/config/ElasticSettings.scala +++ b/kafka-connect-elastic7/src/main/scala/io/lenses/streamreactor/connect/elastic7/config/ElasticSettings.scala @@ -34,7 +34,7 @@ case class ElasticSettings( pkJoinerSeparator: String = ElasticConfigConstants.PK_JOINER_SEPARATOR_DEFAULT, httpBasicAuthUsername: String = ElasticConfigConstants.CLIENT_HTTP_BASIC_AUTH_USERNAME_DEFAULT, httpBasicAuthPassword: String = ElasticConfigConstants.CLIENT_HTTP_BASIC_AUTH_USERNAME_DEFAULT, - storesInfo: StoresInfo = new StoresInfo(none(), none()), + storesInfo: StoresInfo = new StoresInfo(none(), none(), none()), ) object ElasticSettings {