From 582569453766eb829e9710fa69f17a18a84e3551 Mon Sep 17 00:00:00 2001 From: David Sloan Date: Mon, 2 Oct 2023 13:00:18 +0100 Subject: [PATCH] Further refactoring --- build.sbt | 6 +- .../formats/AvroFormatWriterStreamTest.scala | 43 +++--- .../ParquetFormatWriterStreamTest.scala | 94 ++++++------ .../aws/s3/sink/S3AvroWriterManagerTest.scala | 67 +++++---- .../aws/s3/sink/S3JsonWriterManagerTest.scala | 52 +++---- .../s3/sink/S3ParquetWriterManagerTest.scala | 70 +++++---- .../s3/sink/S3SinkTaskAvroEnvelopeTest.scala | 4 +- .../s3/sink/S3SinkTaskJsonEnvelopeTest.scala | 2 +- .../sink/S3SinkTaskParquetEnvelopeTest.scala | 4 +- .../connect/aws/s3/sink/S3SinkTaskTest.scala | 22 +-- .../aws/s3/sink/S3WriterManagerTest.scala | 23 +-- .../connect/aws/s3/source/BucketSetup.scala | 6 +- .../s3/source/S3SourceAvroEnvelopeTest.scala | 6 +- .../s3/source/S3SourceJsonEnvelopeTest.scala | 6 +- .../source/S3SourceParquetEnvelopeTest.scala | 10 +- .../source/S3SourceTaskBucketRootTest.scala | 6 +- .../aws/s3/source/S3SourceTaskTest.scala | 22 +-- .../s3/source/S3SourceTaskXmlReaderTest.scala | 6 +- .../aws/s3/storage/ListDirectoryTest.scala | 8 +- .../aws/s3/utils/ITSampleSchemaAndData.scala | 2 +- .../aws/s3/utils/RemoteFileHelper.scala | 6 +- .../aws/s3/utils/S3ProxyContainerTest.scala | 2 +- .../aws/s3/config/CommonConfigDef.scala | 4 +- .../aws/s3/config/S3ConfigSettings.scala | 14 -- .../kcql/DeprecationConfigDefProcessor.scala | 2 +- .../model/location/S3LocationValidator.scala | 4 +- .../connect/aws/s3/sink/S3SinkConnector.scala | 2 +- .../connect/aws/s3/sink/S3SinkTask.scala | 25 +-- .../aws/s3/sink/S3WriterManagerCreator.scala | 140 +++++++++++++++++ .../aws/s3/sink/config/S3SinkConfig.scala | 36 ++--- .../aws/s3/sink/config/S3SinkConfigDef.scala | 84 +++++------ .../s3/sink/transformers/Transformers.scala | 14 +- .../aws/s3/source/S3SourceConnector.scala | 2 +- .../connect/aws/s3/source/S3SourceTask.scala | 7 +- .../s3/source/WrappedSourceException.scala | 18 --- .../aws/s3/source/config/S3SourceConfig.scala | 23 +-- .../s3/source/config/S3SourceConfigDef.scala | 115 ++++++-------- .../SourcePartitionSearcherSettings.scala | 60 +++++++- .../distribution/PartitionSearcher.scala | 9 +- .../source/state/ReaderManagerBuilder.scala | 14 +- .../aws/s3/source/state/S3SourceBuilder.scala | 13 +- .../aws/s3/storage/AwsS3DirectoryLister.scala | 6 +- .../s3/storage/AwsS3StorageInterface.scala | 29 +++- .../LowerCaseKeyConfigDefProcessorTest.scala | 2 +- .../aws/s3/model/CloudLocationTest.scala | 4 +- .../HierarchicalPartitionExtractorTest.scala | 2 +- .../aws/s3/model/PartitionExtractorTest.scala | 2 +- .../s3/sink/config/PartitionDisplayTest.scala | 14 +- .../config/S3SinkConfigDefBuilderTest.scala | 14 +- .../aws/s3/sink/config/S3SinkConfigTest.scala | 6 +- .../s3/sink/config/TestConfigDefBuilder.scala | 30 ---- .../SequenceTransformerTest.scala | 21 +-- .../transformers/TopicsTransformersTest.scala | 21 +-- .../aws/s3/source/S3SourceConnectorTest.scala | 2 +- .../config/S3SourceConfigTest.scala | 18 +-- .../source/config/S3SourceConfigTests.scala | 7 +- .../kcqlprops/S3SourcePropsSchemaTest.scala | 10 +- .../s3/source/reader/ReaderManagerTest.scala | 10 +- .../s3/source/reader/ResultReaderTest.scala | 3 +- ...t.scala => S3PartitionDiscoveryTest.scala} | 94 ++---------- .../state/ReaderManagerBuilderTest.scala | 12 +- .../s3/storage/AwsS3DirectoryListerTest.scala | 8 +- .../config/CompressionCodecSettings.scala | 8 +- .../{ => common}/config/ConnectorTaskId.scala | 5 +- .../config/DataStorageSettings.scala | 2 +- .../config/DeleteModeSettings.scala | 2 +- .../cloud/{ => common}/config/Format.scala | 2 +- .../{ => common}/config/FormatOptions.scala | 2 +- .../{ => common}/config/FormatSelection.scala | 50 +++--- .../{ => common}/config/TaskDistributor.scala | 2 +- .../config/kcqlprops/S3PropsKeyEnum.scala | 4 +- .../processors/ConfigDefProcessor.scala | 2 +- .../LowerCaseKeyConfigDefProcessor.scala | 2 +- .../kcql/KcqlMapToStringConverter.scala | 2 +- .../config/processors/kcql/KcqlProp.scala | 2 +- .../kcql/StringToKcqlMapConverter.scala | 2 +- .../kcql/YamlToKcqlMapConverter.scala | 2 +- .../formats/FormatWriterException.scala | 2 +- .../formats/bytes/BytesOutputRow.scala | 2 +- .../formats/reader/AvroStreamReader.scala | 2 +- .../reader/BytesStreamFileReader.scala | 4 +- .../formats/reader/CsvStreamReader.scala | 4 +- .../reader/CustomTextStreamReader.scala | 2 +- .../formats/reader/ParquetFormatReader.scala | 6 +- .../reader/ParquetReaderIteratorAdaptor.scala | 4 +- .../formats/reader/ParquetStreamReader.scala | 6 +- .../formats/reader/S3StreamReader.scala | 4 +- .../formats/reader/TextStreamReader.scala | 6 +- .../{ => common}/formats/reader/Using.scala | 2 +- .../converters/BytesOutputRowConverter.scala | 12 +- .../converters/SchemaAndValueConverter.scala | 10 +- .../SchemaAndValueEnvelopeConverter.scala | 10 +- .../SchemalessEnvelopeConverter.scala | 10 +- .../reader/converters/TextConverter.scala | 12 +- .../reader/parquet/ParquetInputFile.scala | 2 +- .../parquet/ParquetSeekableInputStream.scala | 2 +- .../parquet/ParquetStreamingInputFile.scala | 2 +- .../formats/writer/AvroFormatWriter.scala | 17 ++- .../formats/writer/BytesFormatWriter.scala | 6 +- .../formats/writer/CsvFormatWriter.scala | 12 +- .../formats/writer/JsonFormatWriter.scala | 6 +- .../formats/writer/LineSeparatorUtil.scala | 2 +- .../formats/writer/MessageDetail.scala | 6 +- .../formats/writer/ParquetFormatWriter.scala | 20 ++- .../formats/writer/S3FormatWriter.scala | 22 +-- .../formats/writer/SinkData.scala | 2 +- .../formats/writer/Suppress.scala | 4 +- .../formats/writer/TextFormatWriter.scala | 8 +- .../writer/parquet/ParquetOutputFile.scala | 4 +- .../model/CompressionCodecName.scala | 2 +- .../model/TopicPartitionOffset.scala | 2 +- .../model/location/CloudLocation.scala | 2 +- .../location/CloudLocationValidator.scala | 2 +- .../model/location/FileUtils.scala | 2 +- .../model/location/Location.scala | 2 +- .../cloud/{ => common}/sink/Exceptions.scala | 4 +- .../sink/commit/CommitContext.scala | 4 +- .../sink/commit/CommitPolicy.scala | 8 +- .../sink/commit/CommitPolicyConditions.scala | 2 +- .../sink/config/LocalStagingArea.scala | 15 +- .../sink/config/PartitionDisplay.scala | 8 +- .../sink/config/PartitionField.scala | 2 +- .../sink/config/PartitionFieldSplitter.scala | 2 +- .../sink/config/PartitionNamePath.scala | 2 +- .../sink/config/PartitionSelection.scala | 10 +- .../sink/config/PartitionSpecifier.scala | 2 +- .../sink/config/S3FlushSettings.scala | 12 +- .../config/kcqlprops/S3SinkPropsSchema.scala | 10 +- .../sink/config/padding/PaddingService.scala | 14 +- .../sink/config/padding/PaddingStrategy.scala | 2 +- .../padding/PaddingStrategySettings.scala | 25 ++- .../sink/config/padding/PaddingType.scala | 2 +- .../conversion/HeaderToStringConverter.scala | 4 +- .../sink/conversion/ToAvroDataConverter.scala | 16 +- .../sink/conversion/ToJsonDataConverter.scala | 8 +- .../conversion/ValueToSinkDataConverter.scala | 30 ++-- .../sink/extractors/ArrayExtractor.scala | 4 +- .../sink/extractors/ArrayIndexUtil.scala | 2 +- .../extractors/ComplexTypeExtractor.scala | 4 +- .../sink/extractors/ExtractorErrorType.scala | 2 +- .../sink/extractors/MapExtractor.scala | 4 +- .../sink/extractors/PrimitiveExtractor.scala | 2 +- .../sink/extractors/SinkDataExtractor.scala | 16 +- .../sink/extractors/StructExtractor.scala | 4 +- .../extractors/WrappedArrayExtractor.scala | 4 +- .../WrappedComplexTypeExtractor.scala | 4 +- .../sink/extractors/WrappedMapExtractor.scala | 4 +- .../WrappedPrimitiveExtractor.scala | 2 +- .../common/sink/naming}/IndexFilenames.scala | 6 +- .../cloud/common}/sink/naming/KeyNamer.scala | 17 ++- .../common}/sink/naming/S3FileNamer.scala | 6 +- .../common}/sink/naming/S3KeyNamer.scala | 41 +++-- .../common}/sink/seek/IndexManager.scala | 25 +-- .../AddConnectSchemaTransformer.scala | 15 +- .../EnvelopeWithSchemaTransformer.scala | 14 +- .../SchemalessEnvelopeTransformer.scala | 15 +- .../sink/transformers/Transformer.scala | 4 +- .../common}/sink/writer/CommitState.scala | 6 +- .../cloud/common}/sink/writer/S3Writer.scala | 34 ++--- .../common/sink/writer}/S3WriterManager.scala | 142 +++--------------- .../common}/sink/writer/WriteState.scala | 6 +- .../source/ContextConstants.scala | 2 +- .../common}/source/SourceContextReader.scala | 12 +- .../{ => common}/source/SourceWatermark.scala | 14 +- .../common}/source/config/OrderingType.scala | 8 +- .../source/config/PartitionExtractor.scala | 4 +- .../config/PartitionSearcherOptions.scala | 2 +- .../source/config/ReadTextMode.scala | 14 +- .../config/distribution/PartitionHasher.scala | 2 +- .../config/kcqlprops/ReadTextMode.scala | 2 +- .../kcqlprops/S3SourcePropsSchema.scala | 8 +- .../PartitionSearcherResponse.scala | 6 +- .../common}/source/files/BatchLister.scala | 10 +- .../files/DateOrderingBatchLister.scala | 13 +- .../files/DefaultOrderingBatchLister.scala | 10 +- .../source/files/S3SourceFileQueue.scala | 16 +- .../source/reader/PartitionDiscovery.scala | 10 +- .../common}/source/reader/ReaderManager.scala | 8 +- .../source/reader/ReaderManagerState.scala | 4 +- .../common}/source/reader/ResultReader.scala | 23 ++- .../source/state/S3SourceTaskState.scala | 4 +- .../storage/PartitionSearchResults.scala | 4 +- .../common}/storage/ResultProcessors.scala | 23 +-- .../common}/storage/StorageInterface.scala | 7 +- .../cloud/common}/storage/UploadError.scala | 2 +- .../stream/BuildLocalOutputStream.scala | 8 +- .../{ => common}/stream/S3OutputStream.scala | 5 +- .../{ => common}/utils/IteratorOps.scala | 0 .../cloud/{ => common}/utils/MapUtils.scala | 0 .../cloud/{ => common}/utils/PollLoop.scala | 0 .../{ => common}/utils/TimestampUtils.scala | 0 .../common}/config/ConnectorTaskIdTest.scala | 68 ++++----- .../config/DataStorageSettingsTests.scala | 5 +- .../common}/config/FormatSelectionTest.scala | 10 +- .../common}/config/TaskDistributorTest.scala | 11 +- .../common}/formats/AvroFormatReader.scala | 4 +- .../formats/AvroFormatStreamReaderTest.scala | 24 +-- .../formats/AvroFormatWriterTest.scala | 25 ++- .../BytesFormatStreamFileReaderTest.scala | 6 +- .../formats/BytesFormatWriterTest.scala | 24 +-- .../formats/CsvFormatStreamReaderTest.scala | 7 +- .../common}/formats/CsvFormatWriterTest.scala | 22 +-- .../formats/JsonFormatWriterTest.scala | 20 +-- .../ParquetFormatStreamReaderTest.scala | 4 +- .../formats/TextFormatStreamReaderTest.scala | 20 +-- .../formats/TextFormatWriterTest.scala | 17 +-- .../ParquetSeekableInputStreamTest.scala | 4 +- .../ParquetStreamingInputFileTest.scala | 6 +- .../DelegateIteratorS3StreamReaderTest.scala | 8 +- .../BytesOutputRowConverterTest.scala | 17 ++- .../SchemaAndValueConverterTest.scala | 13 +- .../SchemaAndValueEnvelopeConverterTest.scala | 13 +- .../SchemalessEnvelopeConverterTest.scala | 13 +- .../reader/converters/TextConverterTest.scala | 14 +- .../common}/model/PartitionFieldTest.scala | 8 +- .../model/location/FileUtilsTest.scala | 3 +- .../common}/sink/PaddingStrategyTest.scala | 6 +- .../sink/commit/CommitPolicyTest.scala | 18 +-- .../cloud/common}/sink/commit/CountTest.scala | 8 +- .../common}/sink/commit/FileSizeTest.scala | 8 +- .../common}/sink/commit/IntervalTest.scala | 8 +- .../sink/config/LocalStagingAreaTest.scala | 26 ++-- .../config/PartitionFieldSplitterTest.scala | 4 +- .../sink/config/TestConfigDefBuilder.scala | 57 +++++++ .../config/padding/PaddingServiceTest.scala | 20 +-- .../padding/PaddingStrategySettingsTest.scala | 20 +-- ...JsonDataConverterTestFormatSelection.scala | 5 +- .../ValueToSinkDataConverterTest.scala | 6 +- .../sink/extractors/MapExtractorTest.scala | 4 +- .../extractors/SinkDataExtractorTest.scala | 18 +-- .../common}/sink/naming/S3FileNamerTest.scala | 6 +- .../common}/sink/naming/S3KeyNamerTest.scala | 34 ++--- .../common}/sink/seek/IndexManagerTest.scala | 41 +++-- .../AddConnectSchemaTransformerTest.scala | 36 ++--- .../transformers/DataSchemaBuilderTest.scala | 26 ++-- .../EnvelopeWithSchemaTransformerTests.scala | 39 +++-- .../SchemalessEnvelopeTransformerTests.scala | 41 +++-- .../source/SourceContextReaderTest.scala | 10 +- .../common}/source/SourceWatermarkTest.scala | 11 +- .../ReadTextModeTestFormatSelection.scala | 14 +- .../distribution/PartitionHasherTest.scala | 4 +- .../files/DateOrderingBatchListerTest.scala | 17 +-- .../DefaultOrderingBatchListerTest.scala | 8 +- .../source/files/S3SourceFileQueueTest.scala | 20 +-- .../reader/PartitionDiscoveryTest.scala | 111 ++++++++++++++ .../stream/BuildLocalOutputStreamTest.scala | 10 +- .../stream/S3ByteArrayOutputStream.scala | 5 +- .../cloud/common}/utils/IteratorOpsTest.scala | 2 +- .../cloud/common}/utils/MapUtilsTest.scala | 2 +- .../cloud/common}/utils/PollLoopTests.scala | 2 +- .../cloud/common}/utils/SampleData.scala | 12 +- .../common}/utils/TimestampUtilsTest.scala | 2 +- project/Dependencies.scala | 15 +- 253 files changed, 1822 insertions(+), 1688 deletions(-) create mode 100644 kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/S3WriterManagerCreator.scala delete mode 100644 kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/WrappedSourceException.scala delete mode 100644 kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/config/TestConfigDefBuilder.scala rename kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/{ => source}/config/S3SourceConfigTest.scala (75%) rename kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/source/reader/{PartitionDiscoveryTest.scala => S3PartitionDiscoveryTest.scala} (75%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/config/CompressionCodecSettings.scala (84%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/config/ConnectorTaskId.scala (94%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/config/DataStorageSettings.scala (98%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/config/DeleteModeSettings.scala (96%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/config/Format.scala (94%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/config/FormatOptions.scala (93%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/config/FormatSelection.scala (80%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/config/TaskDistributor.scala (94%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/config/kcqlprops/S3PropsKeyEnum.scala (93%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/config/processors/ConfigDefProcessor.scala (91%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/config/processors/LowerCaseKeyConfigDefProcessor.scala (93%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/config/processors/kcql/KcqlMapToStringConverter.scala (98%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/config/processors/kcql/KcqlProp.scala (97%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/config/processors/kcql/StringToKcqlMapConverter.scala (95%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/config/processors/kcql/YamlToKcqlMapConverter.scala (94%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/formats/FormatWriterException.scala (92%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/formats/bytes/BytesOutputRow.scala (92%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/formats/reader/AvroStreamReader.scala (95%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/formats/reader/BytesStreamFileReader.scala (88%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/formats/reader/CsvStreamReader.scala (87%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/formats/reader/CustomTextStreamReader.scala (93%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/formats/reader/ParquetFormatReader.scala (88%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/formats/reader/ParquetReaderIteratorAdaptor.scala (89%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/formats/reader/ParquetStreamReader.scala (89%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/formats/reader/S3StreamReader.scala (92%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/formats/reader/TextStreamReader.scala (87%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/formats/reader/Using.scala (91%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/formats/reader/converters/BytesOutputRowConverter.scala (73%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/formats/reader/converters/SchemaAndValueConverter.scala (77%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/formats/reader/converters/SchemaAndValueEnvelopeConverter.scala (91%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/formats/reader/converters/SchemalessEnvelopeConverter.scala (94%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/formats/reader/converters/TextConverter.scala (77%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/formats/reader/parquet/ParquetInputFile.scala (95%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/formats/reader/parquet/ParquetSeekableInputStream.scala (97%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/formats/reader/parquet/ParquetStreamingInputFile.scala (93%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/formats/writer/AvroFormatWriter.scala (79%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/formats/writer/BytesFormatWriter.scala (91%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/formats/writer/CsvFormatWriter.scala (82%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/formats/writer/JsonFormatWriter.scala (92%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/formats/writer/LineSeparatorUtil.scala (91%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/formats/writer/MessageDetail.scala (80%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/formats/writer/ParquetFormatWriter.scala (74%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/formats/writer/S3FormatWriter.scala (73%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/formats/writer/SinkData.scala (97%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/formats/writer/Suppress.scala (86%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/formats/writer/TextFormatWriter.scala (86%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/formats/writer/parquet/ParquetOutputFile.scala (90%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/model/CompressionCodecName.scala (96%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/model/TopicPartitionOffset.scala (96%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/model/location/CloudLocation.scala (97%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/model/location/CloudLocationValidator.scala (91%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/model/location/FileUtils.scala (94%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/model/location/Location.scala (91%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/sink/Exceptions.scala (95%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/sink/commit/CommitContext.scala (89%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/sink/commit/CommitPolicy.scala (87%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/sink/commit/CommitPolicyConditions.scala (97%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/sink/config/LocalStagingArea.scala (79%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/sink/config/PartitionDisplay.scala (83%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/sink/config/PartitionField.scala (98%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/sink/config/PartitionFieldSplitter.scala (96%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/sink/config/PartitionNamePath.scala (95%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/sink/config/PartitionSelection.scala (80%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/sink/config/PartitionSpecifier.scala (95%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/sink/config/S3FlushSettings.scala (83%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/sink/config/kcqlprops/S3SinkPropsSchema.scala (80%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/sink/config/padding/PaddingService.scala (86%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/sink/config/padding/PaddingStrategy.scala (94%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/sink/config/padding/PaddingStrategySettings.scala (73%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/sink/config/padding/PaddingType.scala (94%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/sink/conversion/HeaderToStringConverter.scala (86%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/sink/conversion/ToAvroDataConverter.scala (80%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/sink/conversion/ToJsonDataConverter.scala (73%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/sink/conversion/ValueToSinkDataConverter.scala (69%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/sink/extractors/ArrayExtractor.scala (93%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/sink/extractors/ArrayIndexUtil.scala (94%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/sink/extractors/ComplexTypeExtractor.scala (90%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/sink/extractors/ExtractorErrorType.scala (96%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/sink/extractors/MapExtractor.scala (92%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/sink/extractors/PrimitiveExtractor.scala (96%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/sink/extractors/SinkDataExtractor.scala (73%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/sink/extractors/StructExtractor.scala (95%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/sink/extractors/WrappedArrayExtractor.scala (89%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/sink/extractors/WrappedComplexTypeExtractor.scala (90%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/sink/extractors/WrappedMapExtractor.scala (91%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/sink/extractors/WrappedPrimitiveExtractor.scala (96%) rename {kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/seek => kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/naming}/IndexFilenames.scala (87%) rename {kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common}/sink/naming/KeyNamer.scala (67%) rename {kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common}/sink/naming/S3FileNamer.scala (86%) rename {kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common}/sink/naming/S3KeyNamer.scala (77%) rename {kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common}/sink/seek/IndexManager.scala (89%) rename {kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common}/sink/transformers/AddConnectSchemaTransformer.scala (93%) rename {kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common}/sink/transformers/EnvelopeWithSchemaTransformer.scala (93%) rename {kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common}/sink/transformers/SchemalessEnvelopeTransformer.scala (86%) rename {kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common}/sink/transformers/Transformer.scala (85%) rename {kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common}/sink/writer/CommitState.scala (90%) rename {kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common}/sink/writer/S3Writer.scala (86%) rename {kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink => kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/writer}/S3WriterManager.scala (63%) rename {kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common}/sink/writer/WriteState.scala (91%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/source/ContextConstants.scala (92%) rename {kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common}/source/SourceContextReader.scala (78%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/source/SourceWatermark.scala (75%) rename {kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common}/source/config/OrderingType.scala (77%) rename {kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common}/source/config/PartitionExtractor.scala (93%) rename {kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common}/source/config/PartitionSearcherOptions.scala (94%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/source/config/ReadTextMode.scala (85%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/source/config/distribution/PartitionHasher.scala (90%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/source/config/kcqlprops/ReadTextMode.scala (92%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/source/config/kcqlprops/S3SourcePropsSchema.scala (79%) rename {kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common}/source/distribution/PartitionSearcherResponse.scala (76%) rename {kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common}/source/files/BatchLister.scala (70%) rename {kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common}/source/files/DateOrderingBatchLister.scala (80%) rename {kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common}/source/files/DefaultOrderingBatchLister.scala (75%) rename {kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common}/source/files/S3SourceFileQueue.scala (86%) rename {kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common}/source/reader/PartitionDiscovery.scala (87%) rename {kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common}/source/reader/ReaderManager.scala (94%) rename {kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common}/source/reader/ReaderManagerState.scala (81%) rename {kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common}/source/reader/ResultReader.scala (84%) rename {kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common}/source/state/S3SourceTaskState.scala (88%) rename {kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common}/storage/PartitionSearchResults.scala (86%) rename {kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common}/storage/ResultProcessors.scala (72%) rename {kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common}/storage/StorageInterface.scala (87%) rename {kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common}/storage/UploadError.scala (97%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/stream/BuildLocalOutputStream.scala (87%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/stream/S3OutputStream.scala (85%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/utils/IteratorOps.scala (100%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/utils/MapUtils.scala (100%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/utils/PollLoop.scala (100%) rename kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/{ => common}/utils/TimestampUtils.scala (100%) rename {kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common}/config/ConnectorTaskIdTest.scala (71%) rename {kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common}/config/DataStorageSettingsTests.scala (96%) rename {kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common}/config/FormatSelectionTest.scala (78%) rename {kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common}/config/TaskDistributorTest.scala (69%) rename {kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common}/formats/AvroFormatReader.scala (93%) rename {kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common}/formats/AvroFormatStreamReaderTest.scala (74%) rename {kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common}/formats/AvroFormatWriterTest.scala (90%) rename {kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common}/formats/BytesFormatStreamFileReaderTest.scala (88%) rename {kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common}/formats/BytesFormatWriterTest.scala (86%) rename {kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common}/formats/CsvFormatStreamReaderTest.scala (91%) rename {kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common}/formats/CsvFormatWriterTest.scala (89%) rename {kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common}/formats/JsonFormatWriterTest.scala (89%) rename {kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common}/formats/ParquetFormatStreamReaderTest.scala (92%) rename {kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common}/formats/TextFormatStreamReaderTest.scala (74%) rename {kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common}/formats/TextFormatWriterTest.scala (82%) rename {kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common}/formats/parquet/ParquetSeekableInputStreamTest.scala (91%) rename {kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common}/formats/parquet/ParquetStreamingInputFileTest.scala (85%) rename {kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common}/formats/reader/DelegateIteratorS3StreamReaderTest.scala (92%) rename {kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common}/formats/reader/converters/BytesOutputRowConverterTest.scala (77%) rename {kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common}/formats/reader/converters/SchemaAndValueConverterTest.scala (80%) rename {kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common}/formats/reader/converters/SchemaAndValueEnvelopeConverterTest.scala (95%) rename {kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common}/formats/reader/converters/SchemalessEnvelopeConverterTest.scala (95%) rename {kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common}/formats/reader/converters/TextConverterTest.scala (78%) rename {kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common}/model/PartitionFieldTest.scala (87%) rename {kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common}/model/location/FileUtilsTest.scala (89%) rename {kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common}/sink/PaddingStrategyTest.scala (80%) rename {kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common}/sink/commit/CommitPolicyTest.scala (83%) rename {kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common}/sink/commit/CountTest.scala (87%) rename {kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common}/sink/commit/FileSizeTest.scala (87%) rename {kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common}/sink/commit/IntervalTest.scala (93%) rename {kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common}/sink/config/LocalStagingAreaTest.scala (70%) rename {kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common}/sink/config/PartitionFieldSplitterTest.scala (92%) create mode 100644 kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/config/TestConfigDefBuilder.scala rename {kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common}/sink/config/padding/PaddingServiceTest.scala (81%) rename {kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common}/sink/config/padding/PaddingStrategySettingsTest.scala (68%) rename {kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common}/sink/conversion/ToJsonDataConverterTestFormatSelection.scala (89%) rename {kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common}/sink/conversion/ValueToSinkDataConverterTest.scala (87%) rename {kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/cloud => kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common}/sink/extractors/MapExtractorTest.scala (90%) rename {kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common}/sink/extractors/SinkDataExtractorTest.scala (94%) rename {kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common}/sink/naming/S3FileNamerTest.scala (86%) rename {kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common}/sink/naming/S3KeyNamerTest.scala (72%) rename {kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common}/sink/seek/IndexManagerTest.scala (89%) rename {kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common}/sink/transformers/AddConnectSchemaTransformerTest.scala (89%) rename {kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common}/sink/transformers/DataSchemaBuilderTest.scala (87%) rename {kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common}/sink/transformers/EnvelopeWithSchemaTransformerTests.scala (88%) rename {kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common}/sink/transformers/SchemalessEnvelopeTransformerTests.scala (90%) rename {kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common}/source/SourceContextReaderTest.scala (88%) rename {kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common}/source/SourceWatermarkTest.scala (82%) rename {kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common}/source/config/ReadTextModeTestFormatSelection.scala (87%) rename {kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common}/source/distribution/PartitionHasherTest.scala (93%) rename {kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common}/source/files/DateOrderingBatchListerTest.scala (87%) rename {kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common}/source/files/DefaultOrderingBatchListerTest.scala (88%) rename {kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common}/source/files/S3SourceFileQueueTest.scala (89%) create mode 100644 kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/source/reader/PartitionDiscoveryTest.scala rename {kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common}/stream/BuildLocalOutputStreamTest.scala (84%) rename {kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common}/stream/S3ByteArrayOutputStream.scala (86%) rename {kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common}/utils/IteratorOpsTest.scala (97%) rename {kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common}/utils/MapUtilsTest.scala (96%) rename {kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common}/utils/PollLoopTests.scala (98%) rename {kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common}/utils/SampleData.scala (90%) rename {kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3 => kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common}/utils/TimestampUtilsTest.scala (95%) diff --git a/build.sbt b/build.sbt index 8788c64a97..9f8b643e98 100644 --- a/build.sbt +++ b/build.sbt @@ -81,7 +81,7 @@ lazy val `cloud-common` = (project in file("kafka-connect-cloud-common")) Seq( name := "kafka-connect-cloud-common", description := "Kafka Connect compatible connectors to move data between Kafka and popular data stores", - libraryDependencies ++= baseDeps ++ kafkaConnectS3Deps, + libraryDependencies ++= baseDeps ++ kafkaConnectCloudCommonDeps, publish / skip := true, packExcludeJars := Seq( "scala-.*\\.jar", @@ -91,13 +91,11 @@ lazy val `cloud-common` = (project in file("kafka-connect-cloud-common")) ) .configureAssembly() .configureTests(baseTestDeps) - //.configureIntegrationTests(kafkaConnectS3TestDeps) - //.configureFunctionalTests(kafkaConnectS3FuncTestDeps) .enablePlugins(PackPlugin) lazy val `aws-s3` = (project in file("kafka-connect-aws-s3")) .dependsOn(common) - .dependsOn(`cloud-common`) + .dependsOn(`cloud-common` % "compile->compile;test->test") .dependsOn(`test-common` % "fun->compile") .settings( settings ++ diff --git a/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/formats/AvroFormatWriterStreamTest.scala b/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/formats/AvroFormatWriterStreamTest.scala index 5dde88c18d..b7c3b29ebe 100644 --- a/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/formats/AvroFormatWriterStreamTest.scala +++ b/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/formats/AvroFormatWriterStreamTest.scala @@ -19,19 +19,20 @@ package io.lenses.streamreactor.connect.aws.s3.formats import io.lenses.streamreactor.connect.aws.s3.utils.ITSampleSchemaAndData.firstUsers import io.lenses.streamreactor.connect.aws.s3.utils.ITSampleSchemaAndData.users import io.lenses.streamreactor.connect.aws.s3.utils.S3ProxyContainerTest -import io.lenses.streamreactor.connect.aws.s3.utils.SampleData.checkRecord -import io.lenses.streamreactor.connect.aws.s3.utils.SampleData.topic -import io.lenses.streamreactor.connect.cloud.model.CompressionCodecName.UNCOMPRESSED -import io.lenses.streamreactor.connect.cloud.model.location.FileUtils.toBufferedOutputStream -import io.lenses.streamreactor.connect.cloud.config.AvroFormatSelection -import io.lenses.streamreactor.connect.cloud.formats.writer.AvroFormatWriter -import io.lenses.streamreactor.connect.cloud.formats.writer.MessageDetail -import io.lenses.streamreactor.connect.cloud.formats.writer.NullSinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.StructSinkData -import io.lenses.streamreactor.connect.cloud.model.CompressionCodec -import io.lenses.streamreactor.connect.cloud.model.Offset -import io.lenses.streamreactor.connect.cloud.model.Topic -import io.lenses.streamreactor.connect.cloud.stream.BuildLocalOutputStream +import io.lenses.streamreactor.connect.cloud.common.utils.SampleData.checkRecord +import io.lenses.streamreactor.connect.cloud.common.utils.SampleData.topic +import io.lenses.streamreactor.connect.cloud.common.config.AvroFormatSelection +import io.lenses.streamreactor.connect.cloud.common.formats.AvroFormatReader +import io.lenses.streamreactor.connect.cloud.common.formats.writer +import io.lenses.streamreactor.connect.cloud.common.formats.writer.AvroFormatWriter +import io.lenses.streamreactor.connect.cloud.common.formats.writer.NullSinkData +import io.lenses.streamreactor.connect.cloud.common.formats.writer.StructSinkData +import io.lenses.streamreactor.connect.cloud.common.model.CompressionCodecName.UNCOMPRESSED +import io.lenses.streamreactor.connect.cloud.common.model.CompressionCodec +import io.lenses.streamreactor.connect.cloud.common.model.Offset +import io.lenses.streamreactor.connect.cloud.common.model.Topic +import io.lenses.streamreactor.connect.cloud.common.model.location.FileUtils.toBufferedOutputStream +import io.lenses.streamreactor.connect.cloud.common.stream.BuildLocalOutputStream import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -47,13 +48,13 @@ class AvroFormatWriterStreamTest extends AnyFlatSpec with Matchers with S3ProxyC val blobStream = new BuildLocalOutputStream(toBufferedOutputStream(localFile), Topic("testTopic").withPartition(1)) val avroFormatWriter = new AvroFormatWriter(blobStream) - avroFormatWriter.write(MessageDetail(NullSinkData(None), - StructSinkData(users.head), - Map.empty, - None, - topic, - 1, - Offset(1), + avroFormatWriter.write(writer.MessageDetail(NullSinkData(None), + StructSinkData(users.head), + Map.empty, + None, + topic, + 1, + Offset(1), )) avroFormatWriter.complete() should be(Right(())) val bytes = localFileAsBytes(localFile) @@ -109,7 +110,7 @@ class AvroFormatWriterStreamTest extends AnyFlatSpec with Matchers with S3ProxyC val avroFormatWriter = new AvroFormatWriter(blobStream) firstUsers.foreach(u => avroFormatWriter.write( - MessageDetail(NullSinkData(None), StructSinkData(u), Map.empty, None, topic, 1, Offset(2)), + writer.MessageDetail(NullSinkData(None), StructSinkData(u), Map.empty, None, topic, 1, Offset(2)), ) should be( Right(()), ), diff --git a/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/formats/ParquetFormatWriterStreamTest.scala b/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/formats/ParquetFormatWriterStreamTest.scala index b6ad5b2e6c..1585a6b126 100644 --- a/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/formats/ParquetFormatWriterStreamTest.scala +++ b/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/formats/ParquetFormatWriterStreamTest.scala @@ -16,28 +16,24 @@ package io.lenses.streamreactor.connect.aws.s3.formats -import io.lenses.streamreactor.connect.cloud.model.CompressionCodecName.BROTLI -import io.lenses.streamreactor.connect.cloud.model.CompressionCodecName.LZ4 -import io.lenses.streamreactor.connect.cloud.model.CompressionCodecName.LZO -import io.lenses.streamreactor.connect.cloud.model.CompressionCodecName.UNCOMPRESSED import io.lenses.streamreactor.connect.aws.s3.utils.ITSampleSchemaAndData.firstUsers import io.lenses.streamreactor.connect.aws.s3.utils.ITSampleSchemaAndData.users import io.lenses.streamreactor.connect.aws.s3.utils.S3ProxyContainerTest -import io.lenses.streamreactor.connect.aws.s3.utils.SampleData.checkRecord -import io.lenses.streamreactor.connect.aws.s3.utils.SampleData.topic -import io.lenses.streamreactor.connect.cloud.model.location.FileUtils.toBufferedOutputStream -import io.lenses.streamreactor.connect.cloud.config.ParquetFormatSelection -import io.lenses.streamreactor.connect.cloud.formats.reader.ParquetFormatReader -import io.lenses.streamreactor.connect.cloud.formats.writer.ArraySinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.MapSinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.MessageDetail -import io.lenses.streamreactor.connect.cloud.formats.writer.NullSinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.ParquetFormatWriter -import io.lenses.streamreactor.connect.cloud.formats.writer.StructSinkData -import io.lenses.streamreactor.connect.cloud.model.CompressionCodec -import io.lenses.streamreactor.connect.cloud.model.Offset -import io.lenses.streamreactor.connect.cloud.model.Topic -import io.lenses.streamreactor.connect.cloud.stream.BuildLocalOutputStream +import io.lenses.streamreactor.connect.cloud.common.utils.SampleData.checkRecord +import io.lenses.streamreactor.connect.cloud.common.utils.SampleData.topic +import io.lenses.streamreactor.connect.cloud.common.config.ParquetFormatSelection +import io.lenses.streamreactor.connect.cloud.common.formats.reader.ParquetFormatReader +import io.lenses.streamreactor.connect.cloud.common.formats.writer +import io.lenses.streamreactor.connect.cloud.common.formats.writer._ +import io.lenses.streamreactor.connect.cloud.common.model.CompressionCodec +import io.lenses.streamreactor.connect.cloud.common.model.Offset +import io.lenses.streamreactor.connect.cloud.common.model.Topic +import io.lenses.streamreactor.connect.cloud.common.model.CompressionCodecName.BROTLI +import io.lenses.streamreactor.connect.cloud.common.model.CompressionCodecName.LZ4 +import io.lenses.streamreactor.connect.cloud.common.model.CompressionCodecName.LZO +import io.lenses.streamreactor.connect.cloud.common.model.CompressionCodecName.UNCOMPRESSED +import io.lenses.streamreactor.connect.cloud.common.model.location.FileUtils.toBufferedOutputStream +import io.lenses.streamreactor.connect.cloud.common.stream.BuildLocalOutputStream import org.apache.kafka.connect.data.Schema import org.apache.kafka.connect.data.SchemaBuilder import org.scalatest.EitherValues @@ -60,31 +56,31 @@ class ParquetFormatWriterStreamTest extends AnyFlatSpec with Matchers with S3Pro val blobStream = new BuildLocalOutputStream(toBufferedOutputStream(localFile), Topic("testTopic").withPartition(1)) val parquetFormatWriter = new ParquetFormatWriter(blobStream) - parquetFormatWriter.write(MessageDetail(NullSinkData(None), - StructSinkData(users.head), - Map.empty, - None, - topic, - 1, - Offset(1), + parquetFormatWriter.write(writer.MessageDetail(NullSinkData(None), + StructSinkData(users.head), + Map.empty, + None, + topic, + 1, + Offset(1), )) parquetFormatWriter.getPointer should be(21) - parquetFormatWriter.write(MessageDetail(NullSinkData(None), - StructSinkData(users(1)), - Map.empty, - None, - topic, - 1, - Offset(2), + parquetFormatWriter.write(writer.MessageDetail(NullSinkData(None), + StructSinkData(users(1)), + Map.empty, + None, + topic, + 1, + Offset(2), )) parquetFormatWriter.getPointer should be(44) - parquetFormatWriter.write(MessageDetail(NullSinkData(None), - StructSinkData(users(2)), - Map.empty, - None, - topic, - 1, - Offset(3), + parquetFormatWriter.write(writer.MessageDetail(NullSinkData(None), + StructSinkData(users(2)), + Map.empty, + None, + topic, + 1, + Offset(3), )) parquetFormatWriter.getPointer should be(59) parquetFormatWriter.complete() should be(Right(())) @@ -111,7 +107,7 @@ class ParquetFormatWriterStreamTest extends AnyFlatSpec with Matchers with S3Pro val blobStream = new BuildLocalOutputStream(toBufferedOutputStream(localFile), Topic("testTopic").withPartition(1)) val parquetFormatWriter = new ParquetFormatWriter(blobStream) parquetFormatWriter.write( - MessageDetail( + writer.MessageDetail( NullSinkData(None), ArraySinkData( Seq( @@ -136,7 +132,7 @@ class ParquetFormatWriterStreamTest extends AnyFlatSpec with Matchers with S3Pro val blobStream = new BuildLocalOutputStream(toBufferedOutputStream(localFile), Topic("testTopic").withPartition(1)) val parquetFormatWriter = new ParquetFormatWriter(blobStream) parquetFormatWriter.write( - MessageDetail( + writer.MessageDetail( NullSinkData(None), MapSinkData( Map( @@ -189,13 +185,13 @@ class ParquetFormatWriterStreamTest extends AnyFlatSpec with Matchers with S3Pro val parquetFormatWriter = new ParquetFormatWriter(blobStream) firstUsers.foreach(u => - parquetFormatWriter.write(MessageDetail(NullSinkData(None), - StructSinkData(u), - Map.empty, - None, - topic, - 1, - Offset(1), + parquetFormatWriter.write(writer.MessageDetail(NullSinkData(None), + StructSinkData(u), + Map.empty, + None, + topic, + 1, + Offset(1), )) should be( Right(()), ), diff --git a/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/sink/S3AvroWriterManagerTest.scala b/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/sink/S3AvroWriterManagerTest.scala index 82f751979b..0b5c8e8125 100644 --- a/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/sink/S3AvroWriterManagerTest.scala +++ b/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/sink/S3AvroWriterManagerTest.scala @@ -18,36 +18,37 @@ package io.lenses.streamreactor.connect.aws.s3.sink import cats.implicits.catsSyntaxOptionId import io.lenses.streamreactor.connect.aws.s3.config._ -import io.lenses.streamreactor.connect.aws.s3.formats.AvroFormatReader -import io.lenses.streamreactor.connect.cloud.model.CompressionCodecName.UNCOMPRESSED import io.lenses.streamreactor.connect.aws.s3.model.location.S3LocationValidator -import io.lenses.streamreactor.connect.cloud.sink.config.PartitionSelection.defaultPartitionSelection import io.lenses.streamreactor.connect.aws.s3.sink.config.OffsetSeekerOptions -import io.lenses.streamreactor.connect.cloud.sink.config.PartitionDisplay.Values import io.lenses.streamreactor.connect.aws.s3.sink.config.S3SinkConfig import io.lenses.streamreactor.connect.aws.s3.sink.config.SinkBucketOptions -import io.lenses.streamreactor.connect.aws.s3.sink.naming.OffsetS3FileNamer -import io.lenses.streamreactor.connect.aws.s3.sink.naming.S3KeyNamer import io.lenses.streamreactor.connect.aws.s3.utils.ITSampleSchemaAndData.firstUsers import io.lenses.streamreactor.connect.aws.s3.utils.S3ProxyContainerTest -import io.lenses.streamreactor.connect.cloud.config.AvroFormatSelection -import io.lenses.streamreactor.connect.cloud.config.ConnectorTaskId -import io.lenses.streamreactor.connect.cloud.config.DataStorageSettings -import io.lenses.streamreactor.connect.cloud.formats.writer.MessageDetail -import io.lenses.streamreactor.connect.cloud.formats.writer.NullSinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.SinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.StructSinkData -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocation -import io.lenses.streamreactor.connect.cloud.model.Offset -import io.lenses.streamreactor.connect.cloud.model.Topic -import io.lenses.streamreactor.connect.cloud.model.TopicPartitionOffset -import io.lenses.streamreactor.connect.cloud.sink.commit.CommitPolicy -import io.lenses.streamreactor.connect.cloud.sink.commit.Count -import io.lenses.streamreactor.connect.cloud.sink.config.LocalStagingArea -import io.lenses.streamreactor.connect.cloud.sink.config.padding.LeftPadPaddingStrategy -import io.lenses.streamreactor.connect.cloud.sink.config.padding.NoOpPaddingStrategy -import io.lenses.streamreactor.connect.cloud.sink.config.padding.PaddingService -import io.lenses.streamreactor.connect.cloud.sink.config.padding.PaddingStrategy +import io.lenses.streamreactor.connect.cloud.common.config.AvroFormatSelection +import io.lenses.streamreactor.connect.cloud.common.config.ConnectorTaskId +import io.lenses.streamreactor.connect.cloud.common.config.DataStorageSettings +import io.lenses.streamreactor.connect.cloud.common.formats.AvroFormatReader +import io.lenses.streamreactor.connect.cloud.common.formats.writer +import io.lenses.streamreactor.connect.cloud.common.formats.writer.MessageDetail +import io.lenses.streamreactor.connect.cloud.common.formats.writer.NullSinkData +import io.lenses.streamreactor.connect.cloud.common.formats.writer.SinkData +import io.lenses.streamreactor.connect.cloud.common.formats.writer.StructSinkData +import io.lenses.streamreactor.connect.cloud.common.model.CompressionCodecName.UNCOMPRESSED +import io.lenses.streamreactor.connect.cloud.common.model.Offset +import io.lenses.streamreactor.connect.cloud.common.model.Topic +import io.lenses.streamreactor.connect.cloud.common.model.TopicPartitionOffset +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocation +import io.lenses.streamreactor.connect.cloud.common.sink.commit.CommitPolicy +import io.lenses.streamreactor.connect.cloud.common.sink.commit.Count +import io.lenses.streamreactor.connect.cloud.common.sink.config.LocalStagingArea +import io.lenses.streamreactor.connect.cloud.common.sink.config.PartitionDisplay.Values +import io.lenses.streamreactor.connect.cloud.common.sink.config.PartitionSelection.defaultPartitionSelection +import io.lenses.streamreactor.connect.cloud.common.sink.config.padding.LeftPadPaddingStrategy +import io.lenses.streamreactor.connect.cloud.common.sink.config.padding.NoOpPaddingStrategy +import io.lenses.streamreactor.connect.cloud.common.sink.config.padding.PaddingService +import io.lenses.streamreactor.connect.cloud.common.sink.config.padding.PaddingStrategy +import io.lenses.streamreactor.connect.cloud.common.sink.naming.OffsetS3FileNamer +import io.lenses.streamreactor.connect.cloud.common.sink.naming.S3KeyNamer import org.apache.avro.generic.GenericRecord import org.apache.kafka.connect.data.Schema import org.apache.kafka.connect.data.SchemaBuilder @@ -104,7 +105,7 @@ class S3AvroWriterManagerTest extends AnyFlatSpec with Matchers with S3ProxyCont ) "avro sink" should "write 2 records to avro format in s3" in { - val sink = S3WriterManager.from(avroConfig) + val sink = S3WriterManagerCreator.from(avroConfig) firstUsers.zipWithIndex.foreach { case (struct: Struct, index: Int) => val writeRes = sink.write( @@ -148,18 +149,18 @@ class S3AvroWriterManagerTest extends AnyFlatSpec with Matchers with S3ProxyCont new Struct(secondSchema).put("name", "coco").put("designation", null).put("salary", 395.44), ) - val sink = S3WriterManager.from(avroConfig) + val sink = S3WriterManagerCreator.from(avroConfig) firstUsers.concat(usersWithNewSchema).zipWithIndex.foreach { case (user, index) => sink.write( TopicPartitionOffset(Topic(TopicName), 1, Offset((index + 1).toLong)), - MessageDetail(NullSinkData(None), - StructSinkData(user), - Map.empty[String, SinkData], - None, - Topic(TopicName), - 1, - Offset((index + 1).toLong), + writer.MessageDetail(NullSinkData(None), + StructSinkData(user), + Map.empty[String, SinkData], + None, + Topic(TopicName), + 1, + Offset((index + 1).toLong), ), ) } diff --git a/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/sink/S3JsonWriterManagerTest.scala b/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/sink/S3JsonWriterManagerTest.scala index 69c0843bef..3b37513a26 100644 --- a/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/sink/S3JsonWriterManagerTest.scala +++ b/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/sink/S3JsonWriterManagerTest.scala @@ -22,33 +22,33 @@ import io.lenses.streamreactor.connect.aws.s3.model.location.S3LocationValidator import io.lenses.streamreactor.connect.aws.s3.sink.config.OffsetSeekerOptions import io.lenses.streamreactor.connect.aws.s3.sink.config.S3SinkConfig import io.lenses.streamreactor.connect.aws.s3.sink.config.SinkBucketOptions -import io.lenses.streamreactor.connect.aws.s3.sink.naming.OffsetS3FileNamer -import io.lenses.streamreactor.connect.aws.s3.sink.naming.S3KeyNamer import io.lenses.streamreactor.connect.aws.s3.utils.ITSampleSchemaAndData.firstUsers import io.lenses.streamreactor.connect.aws.s3.utils.ITSampleSchemaAndData.users import io.lenses.streamreactor.connect.aws.s3.utils.S3ProxyContainerTest -import io.lenses.streamreactor.connect.cloud.config.AvroFormatSelection -import io.lenses.streamreactor.connect.cloud.config.ConnectorTaskId -import io.lenses.streamreactor.connect.cloud.config.DataStorageSettings -import io.lenses.streamreactor.connect.cloud.config.JsonFormatSelection -import io.lenses.streamreactor.connect.cloud.formats.writer.MessageDetail -import io.lenses.streamreactor.connect.cloud.formats.writer.NullSinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.SinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.StructSinkData -import io.lenses.streamreactor.connect.cloud.model.CompressionCodecName.UNCOMPRESSED -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocation -import io.lenses.streamreactor.connect.cloud.model.Offset -import io.lenses.streamreactor.connect.cloud.model.Topic -import io.lenses.streamreactor.connect.cloud.model.TopicPartitionOffset -import io.lenses.streamreactor.connect.cloud.sink.commit.CommitPolicy -import io.lenses.streamreactor.connect.cloud.sink.commit.Count -import io.lenses.streamreactor.connect.cloud.sink.config.LocalStagingArea -import io.lenses.streamreactor.connect.cloud.sink.config.PartitionDisplay.Values -import io.lenses.streamreactor.connect.cloud.sink.config.PartitionSelection.defaultPartitionSelection -import io.lenses.streamreactor.connect.cloud.sink.config.padding.LeftPadPaddingStrategy -import io.lenses.streamreactor.connect.cloud.sink.config.padding.NoOpPaddingStrategy -import io.lenses.streamreactor.connect.cloud.sink.config.padding.PaddingService -import io.lenses.streamreactor.connect.cloud.sink.config.padding.PaddingStrategy +import io.lenses.streamreactor.connect.cloud.common.config.AvroFormatSelection +import io.lenses.streamreactor.connect.cloud.common.config.ConnectorTaskId +import io.lenses.streamreactor.connect.cloud.common.config.DataStorageSettings +import io.lenses.streamreactor.connect.cloud.common.config.JsonFormatSelection +import io.lenses.streamreactor.connect.cloud.common.formats.writer.MessageDetail +import io.lenses.streamreactor.connect.cloud.common.formats.writer.NullSinkData +import io.lenses.streamreactor.connect.cloud.common.formats.writer.SinkData +import io.lenses.streamreactor.connect.cloud.common.formats.writer.StructSinkData +import io.lenses.streamreactor.connect.cloud.common.model.CompressionCodecName.UNCOMPRESSED +import io.lenses.streamreactor.connect.cloud.common.model.Offset +import io.lenses.streamreactor.connect.cloud.common.model.Topic +import io.lenses.streamreactor.connect.cloud.common.model.TopicPartitionOffset +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocation +import io.lenses.streamreactor.connect.cloud.common.sink.commit.CommitPolicy +import io.lenses.streamreactor.connect.cloud.common.sink.commit.Count +import io.lenses.streamreactor.connect.cloud.common.sink.config.LocalStagingArea +import io.lenses.streamreactor.connect.cloud.common.sink.config.PartitionDisplay.Values +import io.lenses.streamreactor.connect.cloud.common.sink.config.PartitionSelection.defaultPartitionSelection +import io.lenses.streamreactor.connect.cloud.common.sink.config.padding.LeftPadPaddingStrategy +import io.lenses.streamreactor.connect.cloud.common.sink.config.padding.NoOpPaddingStrategy +import io.lenses.streamreactor.connect.cloud.common.sink.config.padding.PaddingService +import io.lenses.streamreactor.connect.cloud.common.sink.config.padding.PaddingStrategy +import io.lenses.streamreactor.connect.cloud.common.sink.naming.OffsetS3FileNamer +import io.lenses.streamreactor.connect.cloud.common.sink.naming.S3KeyNamer import org.apache.kafka.connect.data.Struct import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -102,7 +102,7 @@ class S3JsonWriterManagerTest extends AnyFlatSpec with Matchers with S3ProxyCont batchDelete = true, ) - val sink = S3WriterManager.from(config) + val sink = S3WriterManagerCreator.from(config) val topic = Topic(TopicName) val offset = Offset(1) sink.write( @@ -156,7 +156,7 @@ class S3JsonWriterManagerTest extends AnyFlatSpec with Matchers with S3ProxyCont batchDelete = true, ) - val sink = S3WriterManager.from(config) + val sink = S3WriterManagerCreator.from(config) firstUsers.zipWithIndex.foreach { case (struct: Struct, index: Int) => val topic = Topic(TopicName) diff --git a/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/sink/S3ParquetWriterManagerTest.scala b/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/sink/S3ParquetWriterManagerTest.scala index bb5587ffe0..3eee78fdb6 100644 --- a/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/sink/S3ParquetWriterManagerTest.scala +++ b/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/sink/S3ParquetWriterManagerTest.scala @@ -22,32 +22,32 @@ import io.lenses.streamreactor.connect.aws.s3.model.location.S3LocationValidator import io.lenses.streamreactor.connect.aws.s3.sink.config.OffsetSeekerOptions import io.lenses.streamreactor.connect.aws.s3.sink.config.S3SinkConfig import io.lenses.streamreactor.connect.aws.s3.sink.config.SinkBucketOptions -import io.lenses.streamreactor.connect.aws.s3.sink.naming.OffsetS3FileNamer -import io.lenses.streamreactor.connect.aws.s3.sink.naming.S3KeyNamer import io.lenses.streamreactor.connect.aws.s3.utils.ITSampleSchemaAndData.firstUsers import io.lenses.streamreactor.connect.aws.s3.utils.S3ProxyContainerTest -import io.lenses.streamreactor.connect.cloud.config.ConnectorTaskId -import io.lenses.streamreactor.connect.cloud.config.DataStorageSettings -import io.lenses.streamreactor.connect.cloud.config.ParquetFormatSelection -import io.lenses.streamreactor.connect.cloud.formats.reader.ParquetFormatReader -import io.lenses.streamreactor.connect.cloud.formats.writer.MessageDetail -import io.lenses.streamreactor.connect.cloud.formats.writer.NullSinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.SinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.StructSinkData -import io.lenses.streamreactor.connect.cloud.model.CompressionCodecName.UNCOMPRESSED -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocation -import io.lenses.streamreactor.connect.cloud.model.Offset -import io.lenses.streamreactor.connect.cloud.model.Topic -import io.lenses.streamreactor.connect.cloud.model.TopicPartitionOffset -import io.lenses.streamreactor.connect.cloud.sink.commit.CommitPolicy -import io.lenses.streamreactor.connect.cloud.sink.commit.Count -import io.lenses.streamreactor.connect.cloud.sink.config.LocalStagingArea -import io.lenses.streamreactor.connect.cloud.sink.config.PartitionDisplay.Values -import io.lenses.streamreactor.connect.cloud.sink.config.PartitionSelection.defaultPartitionSelection -import io.lenses.streamreactor.connect.cloud.sink.config.padding.LeftPadPaddingStrategy -import io.lenses.streamreactor.connect.cloud.sink.config.padding.NoOpPaddingStrategy -import io.lenses.streamreactor.connect.cloud.sink.config.padding.PaddingService -import io.lenses.streamreactor.connect.cloud.sink.config.padding.PaddingStrategy +import io.lenses.streamreactor.connect.cloud.common.config.ConnectorTaskId +import io.lenses.streamreactor.connect.cloud.common.config.DataStorageSettings +import io.lenses.streamreactor.connect.cloud.common.config.ParquetFormatSelection +import io.lenses.streamreactor.connect.cloud.common.formats.reader.ParquetFormatReader +import io.lenses.streamreactor.connect.cloud.common.formats.writer +import io.lenses.streamreactor.connect.cloud.common.formats.writer.NullSinkData +import io.lenses.streamreactor.connect.cloud.common.formats.writer.SinkData +import io.lenses.streamreactor.connect.cloud.common.formats.writer.StructSinkData +import io.lenses.streamreactor.connect.cloud.common.model.CompressionCodecName.UNCOMPRESSED +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocation +import io.lenses.streamreactor.connect.cloud.common.model.Offset +import io.lenses.streamreactor.connect.cloud.common.model.Topic +import io.lenses.streamreactor.connect.cloud.common.model.TopicPartitionOffset +import io.lenses.streamreactor.connect.cloud.common.sink.commit.CommitPolicy +import io.lenses.streamreactor.connect.cloud.common.sink.commit.Count +import io.lenses.streamreactor.connect.cloud.common.sink.config.LocalStagingArea +import io.lenses.streamreactor.connect.cloud.common.sink.config.PartitionDisplay.Values +import io.lenses.streamreactor.connect.cloud.common.sink.config.PartitionSelection.defaultPartitionSelection +import io.lenses.streamreactor.connect.cloud.common.sink.config.padding.LeftPadPaddingStrategy +import io.lenses.streamreactor.connect.cloud.common.sink.config.padding.NoOpPaddingStrategy +import io.lenses.streamreactor.connect.cloud.common.sink.config.padding.PaddingService +import io.lenses.streamreactor.connect.cloud.common.sink.config.padding.PaddingStrategy +import io.lenses.streamreactor.connect.cloud.common.sink.naming.OffsetS3FileNamer +import io.lenses.streamreactor.connect.cloud.common.sink.naming.S3KeyNamer import org.apache.avro.generic.GenericRecord import org.apache.kafka.connect.data.Schema import org.apache.kafka.connect.data.SchemaBuilder @@ -105,14 +105,21 @@ class S3ParquetWriterManagerTest extends AnyFlatSpec with Matchers with S3ProxyC "parquet sink" should "write 2 records to parquet format in s3" in { - val sink = S3WriterManager.from(parquetConfig) + val sink = S3WriterManagerCreator.from(parquetConfig) firstUsers.zipWithIndex.foreach { case (struct: Struct, index: Int) => val topic = Topic(TopicName) val offset = Offset(index.toLong + 1) sink.write( TopicPartitionOffset(topic, 1, offset), - MessageDetail(NullSinkData(None), StructSinkData(struct), Map.empty[String, SinkData], None, topic, 1, offset), + writer.MessageDetail(NullSinkData(None), + StructSinkData(struct), + Map.empty[String, SinkData], + None, + topic, + 1, + offset, + ), ) } @@ -143,14 +150,21 @@ class S3ParquetWriterManagerTest extends AnyFlatSpec with Matchers with S3ProxyC new Struct(secondSchema).put("name", "coco").put("designation", null).put("salary", 395.44), ) - val sink = S3WriterManager.from(parquetConfig) + val sink = S3WriterManagerCreator.from(parquetConfig) firstUsers.concat(usersWithNewSchema).zipWithIndex.foreach { case (user, index) => val topic = Topic(TopicName) val offset = Offset(index.toLong + 1) sink.write( TopicPartitionOffset(topic, 1, offset), - MessageDetail(NullSinkData(None), StructSinkData(user), Map.empty[String, SinkData], None, topic, 1, offset), + writer.MessageDetail(NullSinkData(None), + StructSinkData(user), + Map.empty[String, SinkData], + None, + topic, + 1, + offset, + ), ) } sink.close() diff --git a/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/sink/S3SinkTaskAvroEnvelopeTest.scala b/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/sink/S3SinkTaskAvroEnvelopeTest.scala index ba83d2f3c7..6c8814ee3c 100644 --- a/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/sink/S3SinkTaskAvroEnvelopeTest.scala +++ b/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/sink/S3SinkTaskAvroEnvelopeTest.scala @@ -20,10 +20,10 @@ import cats.implicits._ import com.typesafe.scalalogging.LazyLogging import io.lenses.streamreactor.connect.aws.s3.config.AuthMode import io.lenses.streamreactor.connect.aws.s3.config.S3ConfigSettings._ -import io.lenses.streamreactor.connect.aws.s3.formats.AvroFormatReader import io.lenses.streamreactor.connect.aws.s3.utils.ITSampleSchemaAndData._ import io.lenses.streamreactor.connect.aws.s3.utils.S3ProxyContainerTest -import io.lenses.streamreactor.connect.cloud.config.TaskIndexKey +import io.lenses.streamreactor.connect.cloud.common.config.TaskIndexKey +import io.lenses.streamreactor.connect.cloud.common.formats.AvroFormatReader import org.apache.avro.generic.GenericRecord import org.apache.kafka.common.TopicPartition import org.apache.kafka.common.record.TimestampType diff --git a/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/sink/S3SinkTaskJsonEnvelopeTest.scala b/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/sink/S3SinkTaskJsonEnvelopeTest.scala index 212bfdf477..8d55925ff1 100644 --- a/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/sink/S3SinkTaskJsonEnvelopeTest.scala +++ b/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/sink/S3SinkTaskJsonEnvelopeTest.scala @@ -24,7 +24,7 @@ import io.lenses.streamreactor.connect.aws.s3.config.AuthMode import io.lenses.streamreactor.connect.aws.s3.config.S3ConfigSettings._ import io.lenses.streamreactor.connect.aws.s3.utils.ITSampleSchemaAndData._ import io.lenses.streamreactor.connect.aws.s3.utils.S3ProxyContainerTest -import io.lenses.streamreactor.connect.cloud.config.TaskIndexKey +import io.lenses.streamreactor.connect.cloud.common.config.TaskIndexKey import org.apache.kafka.common.TopicPartition import org.apache.kafka.common.record.TimestampType import org.apache.kafka.connect.data.Schema diff --git a/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/sink/S3SinkTaskParquetEnvelopeTest.scala b/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/sink/S3SinkTaskParquetEnvelopeTest.scala index c857ba1c1a..8f6fc9b6a1 100644 --- a/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/sink/S3SinkTaskParquetEnvelopeTest.scala +++ b/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/sink/S3SinkTaskParquetEnvelopeTest.scala @@ -22,8 +22,8 @@ import io.lenses.streamreactor.connect.aws.s3.config.AuthMode import io.lenses.streamreactor.connect.aws.s3.config.S3ConfigSettings._ import io.lenses.streamreactor.connect.aws.s3.utils.ITSampleSchemaAndData._ import io.lenses.streamreactor.connect.aws.s3.utils.S3ProxyContainerTest -import io.lenses.streamreactor.connect.cloud.config.TaskIndexKey -import io.lenses.streamreactor.connect.cloud.formats.reader.ParquetFormatReader +import io.lenses.streamreactor.connect.cloud.common.config.TaskIndexKey +import io.lenses.streamreactor.connect.cloud.common.formats.reader.ParquetFormatReader import org.apache.avro.generic.GenericRecord import org.apache.kafka.common.TopicPartition import org.apache.kafka.common.record.TimestampType diff --git a/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/sink/S3SinkTaskTest.scala b/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/sink/S3SinkTaskTest.scala index 7a3e5fccc3..1800a82217 100644 --- a/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/sink/S3SinkTaskTest.scala +++ b/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/sink/S3SinkTaskTest.scala @@ -24,16 +24,16 @@ import com.opencsv.CSVReader import com.typesafe.scalalogging.LazyLogging import io.lenses.streamreactor.connect.aws.s3.config.AuthMode import io.lenses.streamreactor.connect.aws.s3.config.S3ConfigSettings._ -import io.lenses.streamreactor.connect.aws.s3.formats.AvroFormatReader import io.lenses.streamreactor.connect.aws.s3.utils.ITSampleSchemaAndData.checkArray import io.lenses.streamreactor.connect.aws.s3.utils.ITSampleSchemaAndData.firstUsers import io.lenses.streamreactor.connect.aws.s3.utils.ITSampleSchemaAndData.schema import io.lenses.streamreactor.connect.aws.s3.utils.ITSampleSchemaAndData.users import io.lenses.streamreactor.connect.aws.s3.utils.S3ProxyContainerTest -import io.lenses.streamreactor.connect.aws.s3.utils.SampleData.checkRecord -import io.lenses.streamreactor.connect.cloud.config.TaskIndexKey -import io.lenses.streamreactor.connect.cloud.formats.reader.ParquetFormatReader -import io.lenses.streamreactor.connect.cloud.formats.writer.BytesFormatWriter +import io.lenses.streamreactor.connect.cloud.common.utils.SampleData.checkRecord +import io.lenses.streamreactor.connect.cloud.common.config.TaskIndexKey +import io.lenses.streamreactor.connect.cloud.common.formats.AvroFormatReader +import io.lenses.streamreactor.connect.cloud.common.formats.reader.ParquetFormatReader +import io.lenses.streamreactor.connect.cloud.common.formats.writer.BytesFormatWriter import org.apache.avro.generic.GenericData import org.apache.avro.util.Utf8 import org.apache.commons.io.FileUtils @@ -53,7 +53,6 @@ import org.mockito.MockitoSugar import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers -import java.io.File import java.io.StringReader import java.nio.file.Files import java.time.LocalDate @@ -2307,16 +2306,5 @@ class S3SinkTaskTest new CSVReader(file1Reader) } - private def getResourcesDirectory(): Try[String] = { - val url = classOf[S3SinkTaskTest].getResource("/profiles/") - Try { - val uri = url.toURI - logger.info("Profile uri: {}", uri) - val profilePath = new File(uri).getAbsolutePath - logger.info("Profile path: {}", profilePath) - profilePath - } - } - override def connectorPrefix: String = CONNECTOR_PREFIX } diff --git a/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/sink/S3WriterManagerTest.scala b/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/sink/S3WriterManagerTest.scala index f06d0649e4..583898f27c 100644 --- a/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/sink/S3WriterManagerTest.scala +++ b/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/sink/S3WriterManagerTest.scala @@ -2,18 +2,19 @@ package io.lenses.streamreactor.connect.aws.s3.sink import cats.implicits.catsSyntaxEitherId import io.lenses.streamreactor.connect.aws.s3.model.location.S3LocationValidator -import io.lenses.streamreactor.connect.aws.s3.sink.naming.S3KeyNamer -import io.lenses.streamreactor.connect.aws.s3.sink.seek.IndexManager import io.lenses.streamreactor.connect.aws.s3.utils.S3ProxyContainerTest -import io.lenses.streamreactor.connect.cloud.config.ConnectorTaskId -import io.lenses.streamreactor.connect.cloud.formats.writer.S3FormatWriter -import io.lenses.streamreactor.connect.cloud.model.Topic -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocation -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocationValidator -import io.lenses.streamreactor.connect.cloud.sink.commit.CommitPolicy -import io.lenses.streamreactor.connect.cloud.sink.commit.Count -import io.lenses.streamreactor.connect.cloud.sink.commit.FileSize -import io.lenses.streamreactor.connect.cloud.sink.commit.Interval +import io.lenses.streamreactor.connect.cloud.common.config.ConnectorTaskId +import io.lenses.streamreactor.connect.cloud.common.formats.writer.S3FormatWriter +import io.lenses.streamreactor.connect.cloud.common.model.Topic +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocation +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocationValidator +import io.lenses.streamreactor.connect.cloud.common.sink.commit.CommitPolicy +import io.lenses.streamreactor.connect.cloud.common.sink.commit.Count +import io.lenses.streamreactor.connect.cloud.common.sink.commit.FileSize +import io.lenses.streamreactor.connect.cloud.common.sink.commit.Interval +import io.lenses.streamreactor.connect.cloud.common.sink.naming.S3KeyNamer +import io.lenses.streamreactor.connect.cloud.common.sink.seek.IndexManager +import io.lenses.streamreactor.connect.cloud.common.sink.writer.S3WriterManager import org.apache.kafka.clients.consumer.OffsetAndMetadata import org.mockito.MockitoSugar import org.scalatest.flatspec.AnyFlatSpec diff --git a/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/source/BucketSetup.scala b/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/source/BucketSetup.scala index a88a2c1d7c..809561b9e1 100644 --- a/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/source/BucketSetup.scala +++ b/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/source/BucketSetup.scala @@ -16,10 +16,10 @@ package io.lenses.streamreactor.connect.aws.s3.source -import io.lenses.streamreactor.connect.aws.s3.storage.StorageInterface import io.lenses.streamreactor.connect.aws.s3.utils.S3ProxyContainerTest -import io.lenses.streamreactor.connect.cloud.config.Format -import io.lenses.streamreactor.connect.cloud.config.FormatOptions +import io.lenses.streamreactor.connect.cloud.common.config.Format +import io.lenses.streamreactor.connect.cloud.common.config.FormatOptions +import io.lenses.streamreactor.connect.cloud.common.storage.StorageInterface import org.scalatest.matchers.should.Matchers import java.io.File diff --git a/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/source/S3SourceAvroEnvelopeTest.scala b/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/source/S3SourceAvroEnvelopeTest.scala index 7cfa9840b1..1e191fb98b 100644 --- a/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/source/S3SourceAvroEnvelopeTest.scala +++ b/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/source/S3SourceAvroEnvelopeTest.scala @@ -3,8 +3,9 @@ package io.lenses.streamreactor.connect.aws.s3.source import cats.implicits._ import io.lenses.streamreactor.connect.aws.s3.config.AuthMode import io.lenses.streamreactor.connect.aws.s3.config.S3ConfigSettings._ +import io.lenses.streamreactor.connect.aws.s3.source.config.SourcePartitionSearcherSettingsKeys import io.lenses.streamreactor.connect.aws.s3.utils.S3ProxyContainerTest -import io.lenses.streamreactor.connect.cloud.config.TaskIndexKey +import io.lenses.streamreactor.connect.cloud.common.config.TaskIndexKey import org.apache.avro.SchemaBuilder import org.apache.avro.file.CodecFactory import org.apache.avro.file.DataFileWriter @@ -26,7 +27,8 @@ class S3SourceAvroEnvelopeTest with AnyFlatSpecLike with Matchers with EitherValues - with TaskIndexKey { + with TaskIndexKey + with SourcePartitionSearcherSettingsKeys { def DefaultProps: Map[String, String] = Map( AWS_ACCESS_KEY -> Identity, diff --git a/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/source/S3SourceJsonEnvelopeTest.scala b/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/source/S3SourceJsonEnvelopeTest.scala index e90b19db74..a23b0f7c09 100644 --- a/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/source/S3SourceJsonEnvelopeTest.scala +++ b/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/source/S3SourceJsonEnvelopeTest.scala @@ -3,8 +3,9 @@ package io.lenses.streamreactor.connect.aws.s3.source import cats.implicits._ import io.lenses.streamreactor.connect.aws.s3.config.AuthMode import io.lenses.streamreactor.connect.aws.s3.config.S3ConfigSettings._ +import io.lenses.streamreactor.connect.aws.s3.source.config.SourcePartitionSearcherSettingsKeys import io.lenses.streamreactor.connect.aws.s3.utils.S3ProxyContainerTest -import io.lenses.streamreactor.connect.cloud.config.TaskIndexKey +import io.lenses.streamreactor.connect.cloud.common.config.TaskIndexKey import org.apache.kafka.connect.source.SourceRecord import org.scalatest.EitherValues import org.scalatest.concurrent.Eventually.eventually @@ -22,7 +23,8 @@ class S3SourceJsonEnvelopeTest with AnyFlatSpecLike with Matchers with EitherValues - with TaskIndexKey { + with TaskIndexKey + with SourcePartitionSearcherSettingsKeys { def DefaultProps: Map[String, String] = Map( AWS_ACCESS_KEY -> Identity, diff --git a/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/source/S3SourceParquetEnvelopeTest.scala b/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/source/S3SourceParquetEnvelopeTest.scala index db74074216..43acfd1d3f 100644 --- a/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/source/S3SourceParquetEnvelopeTest.scala +++ b/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/source/S3SourceParquetEnvelopeTest.scala @@ -3,10 +3,11 @@ package io.lenses.streamreactor.connect.aws.s3.source import cats.implicits._ import io.lenses.streamreactor.connect.aws.s3.config.AuthMode import io.lenses.streamreactor.connect.aws.s3.config.S3ConfigSettings._ -import io.lenses.streamreactor.connect.aws.s3.stream.S3ByteArrayOutputStream +import io.lenses.streamreactor.connect.aws.s3.source.config.SourcePartitionSearcherSettingsKeys import io.lenses.streamreactor.connect.aws.s3.utils.S3ProxyContainerTest -import io.lenses.streamreactor.connect.cloud.config.TaskIndexKey -import io.lenses.streamreactor.connect.cloud.formats.writer.parquet.ParquetOutputFile +import io.lenses.streamreactor.connect.cloud.common.config.TaskIndexKey +import io.lenses.streamreactor.connect.cloud.common.formats.writer.parquet.ParquetOutputFile +import io.lenses.streamreactor.connect.cloud.common.stream.S3ByteArrayOutputStream import org.apache.avro.SchemaBuilder import org.apache.kafka.connect.source.SourceRecord import org.apache.parquet.avro.AvroParquetWriter @@ -28,7 +29,8 @@ class S3SourceParquetEnvelopeTest with AnyFlatSpecLike with Matchers with EitherValues - with TaskIndexKey { + with TaskIndexKey + with SourcePartitionSearcherSettingsKeys { def DefaultProps: Map[String, String] = Map( AWS_ACCESS_KEY -> Identity, diff --git a/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/source/S3SourceTaskBucketRootTest.scala b/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/source/S3SourceTaskBucketRootTest.scala index d105113b9e..41b4e54882 100644 --- a/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/source/S3SourceTaskBucketRootTest.scala +++ b/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/source/S3SourceTaskBucketRootTest.scala @@ -4,8 +4,9 @@ import cats.implicits._ import io.lenses.streamreactor.connect.aws.s3.config.AuthMode import io.lenses.streamreactor.connect.aws.s3.config.S3ConfigSettings._ import io.lenses.streamreactor.connect.aws.s3.source.S3SourceTaskTest.formats +import io.lenses.streamreactor.connect.aws.s3.source.config.SourcePartitionSearcherSettingsKeys import io.lenses.streamreactor.connect.aws.s3.utils.S3ProxyContainerTest -import io.lenses.streamreactor.connect.cloud.config.TaskIndexKey +import io.lenses.streamreactor.connect.cloud.common.config.TaskIndexKey import org.scalatest.EitherValues import org.scalatest.concurrent.Eventually.eventually import org.scalatest.flatspec.AnyFlatSpecLike @@ -21,7 +22,8 @@ class S3SourceTaskBucketRootTest with AnyFlatSpecLike with Matchers with EitherValues - with TaskIndexKey { + with TaskIndexKey + with SourcePartitionSearcherSettingsKeys { def DefaultProps: Map[String, String] = Map( AWS_ACCESS_KEY -> Identity, diff --git a/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/source/S3SourceTaskTest.scala b/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/source/S3SourceTaskTest.scala index 8e5ccd6df9..27702415ec 100644 --- a/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/source/S3SourceTaskTest.scala +++ b/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/source/S3SourceTaskTest.scala @@ -3,21 +3,22 @@ package io.lenses.streamreactor.connect.aws.s3.source import cats.effect.unsafe.implicits.global import cats.implicits._ import com.typesafe.scalalogging.LazyLogging -import io.lenses.streamreactor.connect.cloud.config.Format.Bytes import io.lenses.streamreactor.connect.aws.s3.config.AuthMode import io.lenses.streamreactor.connect.aws.s3.config.S3ConfigSettings._ import io.lenses.streamreactor.connect.aws.s3.model.location.S3LocationValidator import io.lenses.streamreactor.connect.aws.s3.source.S3SourceTaskTest.formats +import io.lenses.streamreactor.connect.aws.s3.source.config.SourcePartitionSearcherSettingsKeys import io.lenses.streamreactor.connect.aws.s3.storage.AwsS3DirectoryLister -import io.lenses.streamreactor.connect.aws.s3.storage.DirectoryFindCompletionConfig -import io.lenses.streamreactor.connect.aws.s3.storage.DirectoryFindResults import io.lenses.streamreactor.connect.aws.s3.utils.S3ProxyContainerTest -import io.lenses.streamreactor.connect.cloud.config.ConnectorTaskId -import io.lenses.streamreactor.connect.cloud.config.Format -import io.lenses.streamreactor.connect.cloud.config.FormatOptions -import io.lenses.streamreactor.connect.cloud.config.TaskIndexKey -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocation -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocationValidator +import io.lenses.streamreactor.connect.cloud.common.config.Format.Bytes +import io.lenses.streamreactor.connect.cloud.common.config.ConnectorTaskId +import io.lenses.streamreactor.connect.cloud.common.config.Format +import io.lenses.streamreactor.connect.cloud.common.config.FormatOptions +import io.lenses.streamreactor.connect.cloud.common.config.TaskIndexKey +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocation +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocationValidator +import io.lenses.streamreactor.connect.cloud.common.storage.DirectoryFindCompletionConfig +import io.lenses.streamreactor.connect.cloud.common.storage.DirectoryFindResults import org.apache.kafka.connect.source.SourceTaskContext import org.apache.kafka.connect.storage.OffsetStorageReader import org.scalatest.BeforeAndAfter @@ -51,7 +52,8 @@ class S3SourceTaskTest with LazyLogging with BeforeAndAfter with Eventually - with TaskIndexKey { + with TaskIndexKey + with SourcePartitionSearcherSettingsKeys { override implicit def patienceConfig: PatienceConfig = PatienceConfig(timeout = Span(10, Seconds), interval = Span(500, Milliseconds)) diff --git a/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/source/S3SourceTaskXmlReaderTest.scala b/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/source/S3SourceTaskXmlReaderTest.scala index 8be948106f..ed51ebdf6a 100644 --- a/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/source/S3SourceTaskXmlReaderTest.scala +++ b/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/source/S3SourceTaskXmlReaderTest.scala @@ -3,8 +3,9 @@ package io.lenses.streamreactor.connect.aws.s3.source import cats.implicits._ import io.lenses.streamreactor.connect.aws.s3.config.AuthMode import io.lenses.streamreactor.connect.aws.s3.config.S3ConfigSettings._ +import io.lenses.streamreactor.connect.aws.s3.source.config.SourcePartitionSearcherSettingsKeys import io.lenses.streamreactor.connect.aws.s3.utils.S3ProxyContainerTest -import io.lenses.streamreactor.connect.cloud.config.TaskIndexKey +import io.lenses.streamreactor.connect.cloud.common.config.TaskIndexKey import org.apache.kafka.connect.source.SourceRecord import org.scalatest.EitherValues import org.scalatest.concurrent.Eventually.eventually @@ -22,7 +23,8 @@ class S3SourceTaskXmlReaderTest with AnyFlatSpecLike with Matchers with EitherValues - with TaskIndexKey { + with TaskIndexKey + with SourcePartitionSearcherSettingsKeys { def DefaultProps: Map[String, String] = Map( AWS_ACCESS_KEY -> Identity, diff --git a/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/storage/ListDirectoryTest.scala b/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/storage/ListDirectoryTest.scala index 300ed291cc..f159ae1e31 100644 --- a/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/storage/ListDirectoryTest.scala +++ b/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/storage/ListDirectoryTest.scala @@ -5,9 +5,11 @@ import cats.implicits.catsSyntaxOptionId import com.typesafe.scalalogging.LazyLogging import io.lenses.streamreactor.connect.aws.s3.model.location.S3LocationValidator import io.lenses.streamreactor.connect.aws.s3.utils.S3ProxyContainerTest -import io.lenses.streamreactor.connect.cloud.config.ConnectorTaskId -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocation -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocationValidator +import io.lenses.streamreactor.connect.cloud.common.config.ConnectorTaskId +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocation +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocationValidator +import io.lenses.streamreactor.connect.cloud.common.storage.DirectoryFindCompletionConfig +import io.lenses.streamreactor.connect.cloud.common.storage.DirectoryFindResults import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers import software.amazon.awssdk.core.sync.RequestBody diff --git a/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/utils/ITSampleSchemaAndData.scala b/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/utils/ITSampleSchemaAndData.scala index af511856ea..a6cfc87957 100644 --- a/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/utils/ITSampleSchemaAndData.scala +++ b/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/utils/ITSampleSchemaAndData.scala @@ -1,6 +1,6 @@ package io.lenses.streamreactor.connect.aws.s3.utils -import io.lenses.streamreactor.connect.cloud.model.Topic +import io.lenses.streamreactor.connect.cloud.common.model.Topic import org.apache.avro.generic.GenericData import org.apache.avro.generic.GenericRecord import org.apache.avro.util.Utf8 diff --git a/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/utils/RemoteFileHelper.scala b/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/utils/RemoteFileHelper.scala index 13a54fdda3..8dda4c71c7 100644 --- a/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/utils/RemoteFileHelper.scala +++ b/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/utils/RemoteFileHelper.scala @@ -18,9 +18,9 @@ package io.lenses.streamreactor.connect.aws.s3.utils import cats.implicits.catsSyntaxOptionId import com.google.common.io.ByteStreams -import io.lenses.streamreactor.connect.aws.s3.storage.ResultProcessors.processAsKey -import io.lenses.streamreactor.connect.aws.s3.storage.StorageInterface -import io.lenses.streamreactor.connect.cloud.config.ObjectMetadata +import io.lenses.streamreactor.connect.cloud.common.storage.ResultProcessors.processAsKey +import io.lenses.streamreactor.connect.cloud.common.config.ObjectMetadata +import io.lenses.streamreactor.connect.cloud.common.storage.StorageInterface import java.io.File import java.io.InputStream diff --git a/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/utils/S3ProxyContainerTest.scala b/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/utils/S3ProxyContainerTest.scala index 0fcfcef4b4..d42e1db82e 100644 --- a/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/utils/S3ProxyContainerTest.scala +++ b/kafka-connect-aws-s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/utils/S3ProxyContainerTest.scala @@ -7,7 +7,7 @@ import io.lenses.streamreactor.connect.aws.s3.auth.AwsS3ClientCreator import io.lenses.streamreactor.connect.aws.s3.config.AuthMode import io.lenses.streamreactor.connect.aws.s3.config.S3Config import io.lenses.streamreactor.connect.aws.s3.storage.AwsS3StorageInterface -import io.lenses.streamreactor.connect.cloud.config.ConnectorTaskId +import io.lenses.streamreactor.connect.cloud.common.config.ConnectorTaskId import org.scalatest.BeforeAndAfter import org.scalatest.flatspec.AnyFlatSpec import org.testcontainers.containers.wait.strategy.Wait diff --git a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/config/CommonConfigDef.scala b/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/config/CommonConfigDef.scala index 042c13cde9..9029e168cd 100644 --- a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/config/CommonConfigDef.scala +++ b/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/config/CommonConfigDef.scala @@ -17,8 +17,8 @@ package io.lenses.streamreactor.connect.aws.s3.config import io.lenses.streamreactor.connect.aws.s3.config.S3ConfigSettings._ import io.lenses.streamreactor.connect.aws.s3.source.config.S3SourceConfigDef -import io.lenses.streamreactor.connect.cloud.config.CompressionCodecConfigKeys -import io.lenses.streamreactor.connect.cloud.config.DeleteModeConfigKeys +import io.lenses.streamreactor.connect.cloud.common.config.CompressionCodecConfigKeys +import io.lenses.streamreactor.connect.cloud.common.config.DeleteModeConfigKeys import org.apache.kafka.common.config.ConfigDef import org.apache.kafka.common.config.ConfigDef.Importance import org.apache.kafka.common.config.ConfigDef.Type diff --git a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/config/S3ConfigSettings.scala b/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/config/S3ConfigSettings.scala index e1181a0d82..4450ac59f8 100644 --- a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/config/S3ConfigSettings.scala +++ b/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/config/S3ConfigSettings.scala @@ -88,20 +88,6 @@ object S3ConfigSettings { val POOL_MAX_CONNECTIONS_DOC = "Max connections in pool. -1: Use default according to underlying client." val POOL_MAX_CONNECTIONS_DEFAULT: Int = -1 - val SOURCE_PARTITION_SEARCH_RECURSE_LEVELS: String = s"$CONNECTOR_PREFIX.partition.search.recurse.levels" - val SOURCE_PARTITION_SEARCH_RECURSE_LEVELS_DOC: String = - "When searching for new partitions on the S3 filesystem, how many levels deep to recurse." - val SOURCE_PARTITION_SEARCH_RECURSE_LEVELS_DEFAULT: Int = 0 - - val SOURCE_PARTITION_SEARCH_INTERVAL_MILLIS: String = s"$CONNECTOR_PREFIX.partition.search.interval" - val SOURCE_PARTITION_SEARCH_INTERVAL_MILLIS_DOC: String = - "The interval in milliseconds between searching for new partitions. Defaults to 5 minutes." - val SOURCE_PARTITION_SEARCH_INTERVAL_MILLIS_DEFAULT: Long = 300000L - - val SOURCE_PARTITION_SEARCH_MODE: String = s"$CONNECTOR_PREFIX.partition.search.continuous" - val SOURCE_PARTITION_SEARCH_MODE_DOC: String = - "If set to true, it will be continuously search for new partitions. Otherwise it is a one-off operation. Defaults to true." - val SOURCE_ORDERING_TYPE: String = s"$CONNECTOR_PREFIX.ordering.type" val SOURCE_ORDERING_TYPE_DOC: String = "AlphaNumeric (the default)" val SOURCE_ORDERING_TYPE_DEFAULT: String = "AlphaNumeric" diff --git a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/config/processors/kcql/DeprecationConfigDefProcessor.scala b/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/config/processors/kcql/DeprecationConfigDefProcessor.scala index 285c2b76a6..b0a2d6764c 100644 --- a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/config/processors/kcql/DeprecationConfigDefProcessor.scala +++ b/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/config/processors/kcql/DeprecationConfigDefProcessor.scala @@ -26,7 +26,7 @@ import io.lenses.streamreactor.connect.aws.s3.config.processors.kcql.Deprecation import io.lenses.streamreactor.connect.aws.s3.config.processors.kcql.DeprecationConfigDefProcessor.DEP_AWS_SECRET_KEY import io.lenses.streamreactor.connect.aws.s3.config.processors.kcql.DeprecationConfigDefProcessor.DEP_CUSTOM_ENDPOINT import io.lenses.streamreactor.connect.aws.s3.config.processors.kcql.DeprecationConfigDefProcessor.DEP_ENABLE_VIRTUAL_HOST_BUCKETS -import io.lenses.streamreactor.connect.cloud.config.processors.ConfigDefProcessor +import io.lenses.streamreactor.connect.cloud.common.config.processors.ConfigDefProcessor import scala.collection.MapView import scala.collection.immutable.ListMap diff --git a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/model/location/S3LocationValidator.scala b/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/model/location/S3LocationValidator.scala index f6a685431d..ea633ddf3d 100644 --- a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/model/location/S3LocationValidator.scala +++ b/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/model/location/S3LocationValidator.scala @@ -16,8 +16,8 @@ package io.lenses.streamreactor.connect.aws.s3.model.location import cats.data.Validated -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocation -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocationValidator +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocation +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocationValidator import software.amazon.awssdk.services.s3.internal.BucketUtils import scala.util.Try diff --git a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/S3SinkConnector.scala b/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/S3SinkConnector.scala index 77379a8c9f..83b9d911b5 100644 --- a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/S3SinkConnector.scala +++ b/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/S3SinkConnector.scala @@ -19,7 +19,7 @@ import com.datamountaineer.streamreactor.common.utils.JarManifest import com.typesafe.scalalogging.LazyLogging import io.lenses.streamreactor.connect.aws.s3.config.S3ConfigSettings import io.lenses.streamreactor.connect.aws.s3.sink.config.S3SinkConfigDef -import io.lenses.streamreactor.connect.cloud.config.TaskDistributor +import io.lenses.streamreactor.connect.cloud.common.config.TaskDistributor import org.apache.kafka.common.config.ConfigDef import org.apache.kafka.connect.connector.Task import org.apache.kafka.connect.sink.SinkConnector diff --git a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/S3SinkTask.scala b/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/S3SinkTask.scala index c22c568478..383f1a6838 100644 --- a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/S3SinkTask.scala +++ b/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/S3SinkTask.scala @@ -26,18 +26,19 @@ import io.lenses.streamreactor.connect.aws.s3.config.S3ConfigSettings import io.lenses.streamreactor.connect.aws.s3.model.location.S3LocationValidator import io.lenses.streamreactor.connect.aws.s3.sink.config.S3SinkConfig import io.lenses.streamreactor.connect.aws.s3.storage.AwsS3StorageInterface +import io.lenses.streamreactor.connect.cloud.common.config.ConnectorTaskId +import io.lenses.streamreactor.connect.cloud.common.config.ConnectorTaskIdCreator +import io.lenses.streamreactor.connect.cloud.common.formats.writer.MessageDetail +import io.lenses.streamreactor.connect.cloud.common.formats.writer.NullSinkData +import io.lenses.streamreactor.connect.cloud.common.model.Offset +import io.lenses.streamreactor.connect.cloud.common.model.Topic +import io.lenses.streamreactor.connect.cloud.common.model.TopicPartition +import io.lenses.streamreactor.connect.cloud.common.sink.SinkError +import io.lenses.streamreactor.connect.cloud.common.sink.conversion.HeaderToStringConverter +import io.lenses.streamreactor.connect.cloud.common.sink.conversion.ValueToSinkDataConverter +import io.lenses.streamreactor.connect.cloud.common.sink.writer.S3WriterManager import io.lenses.streamreactor.connect.cloud.common.utils.MapUtils import io.lenses.streamreactor.connect.cloud.common.utils.TimestampUtils -import io.lenses.streamreactor.connect.cloud.config.ConnectorTaskId -import io.lenses.streamreactor.connect.cloud.config.ConnectorTaskIdCreator -import io.lenses.streamreactor.connect.cloud.formats.writer -import io.lenses.streamreactor.connect.cloud.formats.writer.NullSinkData -import io.lenses.streamreactor.connect.cloud.model.Offset -import io.lenses.streamreactor.connect.cloud.model.Topic -import io.lenses.streamreactor.connect.cloud.model.TopicPartition -import io.lenses.streamreactor.connect.cloud.sink.SinkError -import io.lenses.streamreactor.connect.cloud.sink.conversion.HeaderToStringConverter -import io.lenses.streamreactor.connect.cloud.sink.conversion.ValueToSinkDataConverter import org.apache.kafka.clients.consumer.OffsetAndMetadata import org.apache.kafka.common.{ TopicPartition => KafkaTopicPartition } import org.apache.kafka.connect.sink.SinkRecord @@ -78,7 +79,7 @@ class S3SinkTask extends SinkTask with ErrorHandler { s3Client <- AwsS3ClientCreator.make(config.s3Config) storageInterface = new AwsS3StorageInterface(connectorTaskId, s3Client, config.batchDelete) _ <- Try(setErrorRetryInterval(config.s3Config)).toEither - writerManager <- Try(S3WriterManager.from(config)(connectorTaskId, storageInterface)).toEither + writerManager <- Try(S3WriterManagerCreator.from(config)(connectorTaskId, storageInterface)).toEither _ <- Try(initialize( config.s3Config.connectorRetryConfig.numberOfRetries, config.s3Config.errorPolicy, @@ -157,7 +158,7 @@ class S3SinkTask extends SinkTask with ErrorHandler { case Some(k) => ValueToSinkDataConverter(k, Option(record.keySchema())) case None => NullSinkData(Option(record.keySchema())) } - val msgDetails = writer.MessageDetail( + val msgDetails = MessageDetail( key = key, value = ValueToSinkDataConverter(record.value(), Option(record.valueSchema())), headers = HeaderToStringConverter(record), diff --git a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/S3WriterManagerCreator.scala b/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/S3WriterManagerCreator.scala new file mode 100644 index 0000000000..49e08eff32 --- /dev/null +++ b/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/S3WriterManagerCreator.scala @@ -0,0 +1,140 @@ +/* + * Copyright 2017-2023 Lenses.io Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.lenses.streamreactor.connect.aws.s3.sink + +import cats.implicits.catsSyntaxEitherId +import com.typesafe.scalalogging.LazyLogging +import io.lenses.streamreactor.connect.aws.s3.sink.config.S3SinkConfig +import io.lenses.streamreactor.connect.aws.s3.sink.config.SinkBucketOptions +import io.lenses.streamreactor.connect.aws.s3.sink.transformers.TopicsTransformers +import io.lenses.streamreactor.connect.cloud.common.config.ConnectorTaskId +import io.lenses.streamreactor.connect.cloud.common.formats +import io.lenses.streamreactor.connect.cloud.common.formats.writer.S3FormatWriter +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocation +import io.lenses.streamreactor.connect.cloud.common.model.Offset +import io.lenses.streamreactor.connect.cloud.common.model.Topic +import io.lenses.streamreactor.connect.cloud.common.model.TopicPartition +import io.lenses.streamreactor.connect.cloud.common.sink.commit.CommitPolicy +import io.lenses.streamreactor.connect.cloud.common.sink.config.PartitionField +import io.lenses.streamreactor.connect.cloud.common.sink.naming.KeyNamer +import io.lenses.streamreactor.connect.cloud.common.sink.seek.IndexManager +import io.lenses.streamreactor.connect.cloud.common.sink.FatalS3SinkError +import io.lenses.streamreactor.connect.cloud.common.sink.SinkError +import io.lenses.streamreactor.connect.cloud.common.sink.writer.S3WriterManager +import io.lenses.streamreactor.connect.cloud.common.storage.StorageInterface + +import java.io.File +import scala.collection.immutable + +object S3WriterManagerCreator extends LazyLogging { + + def from( + config: S3SinkConfig, + )( + implicit + connectorTaskId: ConnectorTaskId, + storageInterface: StorageInterface, + ): S3WriterManager = { + + val bucketAndPrefixFn: TopicPartition => Either[SinkError, CloudLocation] = topicPartition => { + bucketOptsForTopic(config, topicPartition.topic) match { + case Some(sBO) => sBO.bucketAndPrefix.asRight + case None => fatalErrorTopicNotConfigured(topicPartition).asLeft + } + } + + val commitPolicyFn: TopicPartition => Either[SinkError, CommitPolicy] = topicPartition => + bucketOptsForTopic(config, topicPartition.topic) match { + case Some(bucketOptions) => bucketOptions.commitPolicy.asRight + case None => fatalErrorTopicNotConfigured(topicPartition).asLeft + } + + val keyNamerFn: TopicPartition => Either[SinkError, KeyNamer] = topicPartition => + bucketOptsForTopic(config, topicPartition.topic) match { + case Some(bucketOptions) => bucketOptions.keyNamer.asRight + case None => fatalErrorTopicNotConfigured(topicPartition).asLeft + } + + val stagingFilenameFn: (TopicPartition, immutable.Map[PartitionField, String]) => Either[SinkError, File] = + (topicPartition, partitionValues) => + bucketOptsForTopic(config, topicPartition.topic) match { + case Some(bucketOptions) => + for { + keyNamer <- keyNamerFn(topicPartition) + stagingFilename <- keyNamer.stagingFile(bucketOptions.localStagingArea.dir, + bucketOptions.bucketAndPrefix, + topicPartition, + partitionValues, + ) + } yield stagingFilename + case None => fatalErrorTopicNotConfigured(topicPartition).asLeft + } + + val finalFilenameFn: ( + TopicPartition, + immutable.Map[PartitionField, String], + Offset, + ) => Either[SinkError, CloudLocation] = (topicPartition, partitionValues, offset) => + bucketOptsForTopic(config, topicPartition.topic) match { + case Some(bucketOptions) => + for { + keyNamer <- keyNamerFn(topicPartition) + stagingFilename <- keyNamer.finalFilename(bucketOptions.bucketAndPrefix, + topicPartition.withOffset(offset), + partitionValues, + ) + } yield stagingFilename + case None => fatalErrorTopicNotConfigured(topicPartition).asLeft + } + + val formatWriterFn: (TopicPartition, File) => Either[SinkError, S3FormatWriter] = + (topicPartition: TopicPartition, stagingFilename) => + bucketOptsForTopic(config, topicPartition.topic) match { + case Some(bucketOptions) => + for { + formatWriter <- formats.writer.S3FormatWriter( + bucketOptions.formatSelection, + stagingFilename, + topicPartition, + )(config.compressionCodec) + } yield formatWriter + case None => FatalS3SinkError("Can't find format choice in config", topicPartition).asLeft + } + + val indexManager = new IndexManager(config.offsetSeekerOptions.maxIndexFiles) + + val transformers = TopicsTransformers.from(config.bucketOptions) + new S3WriterManager( + commitPolicyFn, + bucketAndPrefixFn, + keyNamerFn, + stagingFilenameFn, + finalFilenameFn, + formatWriterFn, + indexManager, + transformers.transform, + ) + } + + private def bucketOptsForTopic(config: S3SinkConfig, topic: Topic): Option[SinkBucketOptions] = + config.bucketOptions.find(bo => bo.sourceTopic.isEmpty || bo.sourceTopic.contains(topic.value)) + + private def fatalErrorTopicNotConfigured(topicPartition: TopicPartition): SinkError = + FatalS3SinkError( + s"Can't find the KCQL for source topic [${topicPartition.topic}]. The topics defined via [topics] or [topics.regex] need to have an equivalent KCQL statement: INSERT INTO {S3_BUCKET} SELECT * FROM {TOPIC}.", + topicPartition, + ) +} diff --git a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/config/S3SinkConfig.scala b/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/config/S3SinkConfig.scala index 0abb875575..fca6266295 100644 --- a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/config/S3SinkConfig.scala +++ b/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/config/S3SinkConfig.scala @@ -19,24 +19,24 @@ import com.datamountaineer.kcql.Kcql import com.typesafe.scalalogging.LazyLogging import io.lenses.streamreactor.connect.aws.s3.config.S3Config import io.lenses.streamreactor.connect.aws.s3.config.S3ConfigSettings.SEEK_MAX_INDEX_FILES -import io.lenses.streamreactor.connect.aws.s3.sink.naming.KeyNamer -import io.lenses.streamreactor.connect.aws.s3.sink.naming.OffsetS3FileNamer -import io.lenses.streamreactor.connect.aws.s3.sink.naming.S3KeyNamer -import io.lenses.streamreactor.connect.aws.s3.sink.naming.TopicPartitionOffsetS3FileNamer -import io.lenses.streamreactor.connect.cloud.config.BytesFormatSelection -import io.lenses.streamreactor.connect.cloud.config.ConnectorTaskId -import io.lenses.streamreactor.connect.cloud.config.DataStorageSettings -import io.lenses.streamreactor.connect.cloud.config.FormatSelection -import io.lenses.streamreactor.connect.cloud.model.CompressionCodec -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocation -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocationValidator -import io.lenses.streamreactor.connect.cloud.sink.commit.CommitPolicy -import io.lenses.streamreactor.connect.cloud.sink.commit.Count -import io.lenses.streamreactor.connect.cloud.sink.config.kcqlprops.S3SinkProps -import io.lenses.streamreactor.connect.cloud.sink.config.kcqlprops.S3SinkPropsSchema -import io.lenses.streamreactor.connect.cloud.sink.config.padding.PaddingService -import io.lenses.streamreactor.connect.cloud.sink.config.LocalStagingArea -import io.lenses.streamreactor.connect.cloud.sink.config.PartitionSelection +import io.lenses.streamreactor.connect.cloud.common.config.BytesFormatSelection +import io.lenses.streamreactor.connect.cloud.common.config.ConnectorTaskId +import io.lenses.streamreactor.connect.cloud.common.config.DataStorageSettings +import io.lenses.streamreactor.connect.cloud.common.config.FormatSelection +import io.lenses.streamreactor.connect.cloud.common.model.CompressionCodec +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocation +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocationValidator +import io.lenses.streamreactor.connect.cloud.common.sink.commit.CommitPolicy +import io.lenses.streamreactor.connect.cloud.common.sink.commit.Count +import io.lenses.streamreactor.connect.cloud.common.sink.config.kcqlprops.S3SinkProps +import io.lenses.streamreactor.connect.cloud.common.sink.config.kcqlprops.S3SinkPropsSchema +import io.lenses.streamreactor.connect.cloud.common.sink.config.LocalStagingArea +import io.lenses.streamreactor.connect.cloud.common.sink.config.PartitionSelection +import io.lenses.streamreactor.connect.cloud.common.sink.config.padding.PaddingService +import io.lenses.streamreactor.connect.cloud.common.sink.naming.KeyNamer +import io.lenses.streamreactor.connect.cloud.common.sink.naming.OffsetS3FileNamer +import io.lenses.streamreactor.connect.cloud.common.sink.naming.S3KeyNamer +import io.lenses.streamreactor.connect.cloud.common.sink.naming.TopicPartitionOffsetS3FileNamer import java.util import scala.jdk.CollectionConverters._ diff --git a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/config/S3SinkConfigDef.scala b/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/config/S3SinkConfigDef.scala index d9cc529c13..41a1ba56e8 100644 --- a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/config/S3SinkConfigDef.scala +++ b/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/config/S3SinkConfigDef.scala @@ -27,16 +27,16 @@ import java.util import scala.jdk.CollectionConverters._ import S3ConfigSettings._ import io.lenses.streamreactor.connect.aws.s3.config.processors.kcql.DeprecationConfigDefProcessor -import io.lenses.streamreactor.connect.cloud.config.CompressionCodecSettings -import io.lenses.streamreactor.connect.cloud.config.DeleteModeSettings -import io.lenses.streamreactor.connect.cloud.config.processors.ConfigDefProcessor -import io.lenses.streamreactor.connect.cloud.config.processors.LowerCaseKeyConfigDefProcessor -import io.lenses.streamreactor.connect.cloud.sink.config.LocalStagingAreaConfigKeys -import io.lenses.streamreactor.connect.cloud.sink.config.LocalStagingAreaSettings -import io.lenses.streamreactor.connect.cloud.sink.config.S3FlushConfigKeys -import io.lenses.streamreactor.connect.cloud.sink.config.S3FlushSettings -import io.lenses.streamreactor.connect.cloud.sink.config.padding.PaddingStrategyConfigKeys -import io.lenses.streamreactor.connect.cloud.sink.config.padding.PaddingStrategySettings +import io.lenses.streamreactor.connect.cloud.common.config.CompressionCodecSettings +import io.lenses.streamreactor.connect.cloud.common.config.DeleteModeSettings +import io.lenses.streamreactor.connect.cloud.common.config.processors.ConfigDefProcessor +import io.lenses.streamreactor.connect.cloud.common.config.processors.LowerCaseKeyConfigDefProcessor +import io.lenses.streamreactor.connect.cloud.common.sink.config.LocalStagingAreaConfigKeys +import io.lenses.streamreactor.connect.cloud.common.sink.config.LocalStagingAreaSettings +import io.lenses.streamreactor.connect.cloud.common.sink.config.S3FlushConfigKeys +import io.lenses.streamreactor.connect.cloud.common.sink.config.S3FlushSettings +import io.lenses.streamreactor.connect.cloud.common.sink.config.padding.PaddingStrategyConfigKeys +import io.lenses.streamreactor.connect.cloud.common.sink.config.padding.PaddingStrategySettings object S3SinkConfigDef extends CommonConfigDef @@ -46,46 +46,30 @@ object S3SinkConfigDef override def connectorPrefix: String = CONNECTOR_PREFIX - override val config: ConfigDef = super.config - .define( - DISABLE_FLUSH_COUNT, - Type.BOOLEAN, - false, - Importance.LOW, - "Disable flush on reaching count", - ) - .define( - LOCAL_TMP_DIRECTORY, - Type.STRING, - "", - Importance.LOW, - s"Local tmp directory for preparing the files", - ) - .define( - SEEK_MAX_INDEX_FILES, - Type.INT, - SEEK_MAX_INDEX_FILES_DEFAULT, - Importance.LOW, - SEEK_MAX_INDEX_FILES_DOC, - "Sink Seek", - 2, - ConfigDef.Width.LONG, - SEEK_MAX_INDEX_FILES, - ) - .define( - PADDING_STRATEGY, - Type.STRING, - PADDING_STRATEGY_DEFAULT, - Importance.LOW, - PADDING_STRATEGY_DOC, - ) - .define( - PADDING_LENGTH, - Type.INT, - PADDING_LENGTH_DEFAULT, - Importance.LOW, - PADDING_LENGTH_DOC, - ) + override val config: ConfigDef = { + val configDef = super.config + .define( + DISABLE_FLUSH_COUNT, + Type.BOOLEAN, + false, + Importance.LOW, + "Disable flush on reaching count", + ) + .define( + SEEK_MAX_INDEX_FILES, + Type.INT, + SEEK_MAX_INDEX_FILES_DEFAULT, + Importance.LOW, + SEEK_MAX_INDEX_FILES_DOC, + "Sink Seek", + 2, + ConfigDef.Width.LONG, + SEEK_MAX_INDEX_FILES, + ) + addLocalStagingAreaToConfigDef(configDef) + addPaddingToConfigDef(configDef) + } + } class S3SinkConfigDef() extends ConfigDef with LazyLogging { diff --git a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/transformers/Transformers.scala b/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/transformers/Transformers.scala index e5d1976854..887e4b3778 100644 --- a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/transformers/Transformers.scala +++ b/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/transformers/Transformers.scala @@ -17,11 +17,15 @@ package io.lenses.streamreactor.connect.aws.s3.sink.transformers import cats.implicits.catsSyntaxEitherId import io.lenses.streamreactor.connect.aws.s3.sink.config.SinkBucketOptions -import io.lenses.streamreactor.connect.cloud.config.AvroFormatSelection -import io.lenses.streamreactor.connect.cloud.config.JsonFormatSelection -import io.lenses.streamreactor.connect.cloud.config.ParquetFormatSelection -import io.lenses.streamreactor.connect.cloud.formats.writer.MessageDetail -import io.lenses.streamreactor.connect.cloud.model.Topic +import io.lenses.streamreactor.connect.cloud.common.config.AvroFormatSelection +import io.lenses.streamreactor.connect.cloud.common.config.JsonFormatSelection +import io.lenses.streamreactor.connect.cloud.common.config.ParquetFormatSelection +import io.lenses.streamreactor.connect.cloud.common.formats.writer.MessageDetail +import io.lenses.streamreactor.connect.cloud.common.model.Topic +import io.lenses.streamreactor.connect.cloud.common.sink.transformers.AddConnectSchemaTransformer +import io.lenses.streamreactor.connect.cloud.common.sink.transformers.EnvelopeWithSchemaTransformer +import io.lenses.streamreactor.connect.cloud.common.sink.transformers.SchemalessEnvelopeTransformer +import io.lenses.streamreactor.connect.cloud.common.sink.transformers.Transformer /** * Applies a sequence of transformations to a message. diff --git a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/S3SourceConnector.scala b/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/S3SourceConnector.scala index 26a236cf85..0670bf0af2 100644 --- a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/S3SourceConnector.scala +++ b/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/S3SourceConnector.scala @@ -19,7 +19,7 @@ import com.datamountaineer.streamreactor.common.utils.JarManifest import com.typesafe.scalalogging.LazyLogging import io.lenses.streamreactor.connect.aws.s3.config.S3ConfigSettings.CONNECTOR_PREFIX import io.lenses.streamreactor.connect.aws.s3.source.config.S3SourceConfigDef -import io.lenses.streamreactor.connect.cloud.config.TaskDistributor +import io.lenses.streamreactor.connect.cloud.common.config.TaskDistributor import org.apache.kafka.common.config.ConfigDef import org.apache.kafka.connect.connector.Task import org.apache.kafka.connect.source.ExactlyOnceSupport diff --git a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/S3SourceTask.scala b/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/S3SourceTask.scala index bc57140264..3bcccee93f 100644 --- a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/S3SourceTask.scala +++ b/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/S3SourceTask.scala @@ -25,10 +25,11 @@ import com.datamountaineer.streamreactor.common.utils.JarManifest import com.typesafe.scalalogging.LazyLogging import io.lenses.streamreactor.connect.aws.s3.model.location.S3LocationValidator import io.lenses.streamreactor.connect.aws.s3.source.state.S3SourceState -import io.lenses.streamreactor.connect.aws.s3.source.state.S3SourceTaskState +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocation +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocationValidator +import io.lenses.streamreactor.connect.cloud.common.source.SourceContextReader +import io.lenses.streamreactor.connect.cloud.common.source.state.S3SourceTaskState import io.lenses.streamreactor.connect.cloud.common.utils.MapUtils -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocation -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocationValidator import org.apache.kafka.connect.source.SourceRecord import org.apache.kafka.connect.source.SourceTask diff --git a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/WrappedSourceException.scala b/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/WrappedSourceException.scala deleted file mode 100644 index 9fca1fe8f6..0000000000 --- a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/WrappedSourceException.scala +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Copyright 2017-2023 Lenses.io Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.lenses.streamreactor.connect.aws.s3.source - -class WrappedSourceException(ex: String) extends Throwable(ex) {} diff --git a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/config/S3SourceConfig.scala b/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/config/S3SourceConfig.scala index 1831ec1bcb..03de729a17 100644 --- a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/config/S3SourceConfig.scala +++ b/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/config/S3SourceConfig.scala @@ -24,16 +24,19 @@ import io.lenses.streamreactor.connect.aws.s3.config.S3ConfigSettings.SOURCE_ORD import io.lenses.streamreactor.connect.aws.s3.config.S3ConfigSettings.SOURCE_PARTITION_EXTRACTOR_REGEX import io.lenses.streamreactor.connect.aws.s3.config.S3ConfigSettings.SOURCE_PARTITION_EXTRACTOR_TYPE import io.lenses.streamreactor.connect.aws.s3.model.location.S3LocationValidator -import io.lenses.streamreactor.connect.aws.s3.storage.FileListError -import io.lenses.streamreactor.connect.aws.s3.storage.FileMetadata -import io.lenses.streamreactor.connect.aws.s3.storage.ListResponse -import io.lenses.streamreactor.connect.aws.s3.storage.StorageInterface -import io.lenses.streamreactor.connect.cloud.config.DataStorageSettings -import io.lenses.streamreactor.connect.cloud.config.FormatSelection -import io.lenses.streamreactor.connect.cloud.model.CompressionCodec -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocation -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocationValidator -import io.lenses.streamreactor.connect.cloud.source.config.kcqlprops.S3SourcePropsSchema +import io.lenses.streamreactor.connect.cloud.common.config.DataStorageSettings +import io.lenses.streamreactor.connect.cloud.common.config.FormatSelection +import io.lenses.streamreactor.connect.cloud.common.model.CompressionCodec +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocation +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocationValidator +import io.lenses.streamreactor.connect.cloud.common.source.config.OrderingType +import io.lenses.streamreactor.connect.cloud.common.source.config.PartitionExtractor +import io.lenses.streamreactor.connect.cloud.common.source.config.PartitionSearcherOptions +import io.lenses.streamreactor.connect.cloud.common.source.config.kcqlprops.S3SourcePropsSchema +import io.lenses.streamreactor.connect.cloud.common.storage.FileListError +import io.lenses.streamreactor.connect.cloud.common.storage.FileMetadata +import io.lenses.streamreactor.connect.cloud.common.storage.ListResponse +import io.lenses.streamreactor.connect.cloud.common.storage.StorageInterface import java.util import scala.jdk.CollectionConverters.MapHasAsScala diff --git a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/config/S3SourceConfigDef.scala b/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/config/S3SourceConfigDef.scala index 32f8c6fc82..3426f8b35a 100644 --- a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/config/S3SourceConfigDef.scala +++ b/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/config/S3SourceConfigDef.scala @@ -21,10 +21,10 @@ import com.typesafe.scalalogging.LazyLogging import io.lenses.streamreactor.connect.aws.s3.config.S3ConfigSettings._ import io.lenses.streamreactor.connect.aws.s3.config._ import io.lenses.streamreactor.connect.aws.s3.config.processors.kcql.DeprecationConfigDefProcessor -import io.lenses.streamreactor.connect.cloud.config.CompressionCodecSettings -import io.lenses.streamreactor.connect.cloud.config.DeleteModeSettings -import io.lenses.streamreactor.connect.cloud.config.processors.ConfigDefProcessor -import io.lenses.streamreactor.connect.cloud.config.processors.LowerCaseKeyConfigDefProcessor +import io.lenses.streamreactor.connect.cloud.common.config.CompressionCodecSettings +import io.lenses.streamreactor.connect.cloud.common.config.DeleteModeSettings +import io.lenses.streamreactor.connect.cloud.common.config.processors.ConfigDefProcessor +import io.lenses.streamreactor.connect.cloud.common.config.processors.LowerCaseKeyConfigDefProcessor import org.apache.kafka.common.config.ConfigDef import org.apache.kafka.common.config.ConfigDef.Importance import org.apache.kafka.common.config.ConfigDef.Type @@ -33,77 +33,48 @@ import java.util import scala.collection.immutable.ListMap import scala.jdk.CollectionConverters._ -object S3SourceConfigDef extends CommonConfigDef { +object S3SourceConfigDef extends CommonConfigDef with SourcePartitionSearcherSettingsKeys { override def connectorPrefix: String = CONNECTOR_PREFIX - override val config: ConfigDef = super.config - .define( - SOURCE_PARTITION_EXTRACTOR_TYPE, - Type.STRING, - null, - Importance.LOW, - SOURCE_PARTITION_EXTRACTOR_TYPE_DOC, - "Source", - 1, - ConfigDef.Width.MEDIUM, - SOURCE_PARTITION_EXTRACTOR_TYPE, - ) - .define( - SOURCE_PARTITION_EXTRACTOR_REGEX, - Type.STRING, - null, - Importance.LOW, - SOURCE_PARTITION_EXTRACTOR_REGEX_DOC, - "Source", - 2, - ConfigDef.Width.MEDIUM, - SOURCE_PARTITION_EXTRACTOR_REGEX, - ) - .define( - SOURCE_PARTITION_SEARCH_RECURSE_LEVELS, - Type.INT, - SOURCE_PARTITION_SEARCH_RECURSE_LEVELS_DEFAULT, - Importance.LOW, - SOURCE_PARTITION_SEARCH_RECURSE_LEVELS_DOC, - "Source", - 3, - ConfigDef.Width.MEDIUM, - SOURCE_PARTITION_SEARCH_RECURSE_LEVELS, - ) - .define( - SOURCE_PARTITION_SEARCH_MODE, - Type.BOOLEAN, - true, - Importance.LOW, - SOURCE_PARTITION_SEARCH_MODE_DOC, - "Source", - 4, - ConfigDef.Width.MEDIUM, - SOURCE_PARTITION_SEARCH_MODE, - ) - .define( - SOURCE_PARTITION_SEARCH_INTERVAL_MILLIS, - Type.LONG, - SOURCE_PARTITION_SEARCH_INTERVAL_MILLIS_DEFAULT, - Importance.LOW, - SOURCE_PARTITION_SEARCH_INTERVAL_MILLIS_DOC, - "Source", - 5, - ConfigDef.Width.MEDIUM, - SOURCE_PARTITION_SEARCH_INTERVAL_MILLIS, - ) - .define( - SOURCE_ORDERING_TYPE, - Type.STRING, - SOURCE_ORDERING_TYPE_DEFAULT, - Importance.LOW, - SOURCE_ORDERING_TYPE_DOC, - "Source", - 6, - ConfigDef.Width.MEDIUM, - SOURCE_ORDERING_TYPE, - ) + override val config: ConfigDef = { + val settings = super.config + .define( + SOURCE_PARTITION_EXTRACTOR_TYPE, + Type.STRING, + null, + Importance.LOW, + SOURCE_PARTITION_EXTRACTOR_TYPE_DOC, + "Source", + 1, + ConfigDef.Width.MEDIUM, + SOURCE_PARTITION_EXTRACTOR_TYPE, + ) + .define( + SOURCE_PARTITION_EXTRACTOR_REGEX, + Type.STRING, + null, + Importance.LOW, + SOURCE_PARTITION_EXTRACTOR_REGEX_DOC, + "Source", + 2, + ConfigDef.Width.MEDIUM, + SOURCE_PARTITION_EXTRACTOR_REGEX, + ) + .define( + SOURCE_ORDERING_TYPE, + Type.STRING, + SOURCE_ORDERING_TYPE_DEFAULT, + Importance.LOW, + SOURCE_ORDERING_TYPE_DOC, + "Source", + 6, + ConfigDef.Width.MEDIUM, + SOURCE_ORDERING_TYPE, + ) + + addSourcePartitionSearcherSettings(settings) + } } class S3SourceConfigDef() extends ConfigDef with LazyLogging { diff --git a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/config/SourcePartitionSearcherSettings.scala b/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/config/SourcePartitionSearcherSettings.scala index 704bece165..249935c9ae 100644 --- a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/config/SourcePartitionSearcherSettings.scala +++ b/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/config/SourcePartitionSearcherSettings.scala @@ -16,13 +16,69 @@ package io.lenses.streamreactor.connect.aws.s3.source.config import com.datamountaineer.streamreactor.common.config.base.traits.BaseSettings +import com.datamountaineer.streamreactor.common.config.base.traits.WithConnectorPrefix import io.lenses.streamreactor.connect.aws.s3.config.S3Config import io.lenses.streamreactor.connect.aws.s3.config.S3ConfigSettings._ -import io.lenses.streamreactor.connect.aws.s3.source.config.PartitionSearcherOptions.ExcludeIndexes +import io.lenses.streamreactor.connect.cloud.common.source.config.PartitionSearcherOptions +import io.lenses.streamreactor.connect.cloud.common.source.config.PartitionSearcherOptions.ExcludeIndexes +import org.apache.kafka.common.config.ConfigDef +import org.apache.kafka.common.config.ConfigDef.Importance +import org.apache.kafka.common.config.ConfigDef.Type import scala.concurrent.duration.DurationLong -trait SourcePartitionSearcherSettings extends BaseSettings { +trait SourcePartitionSearcherSettingsKeys extends WithConnectorPrefix { + + val SOURCE_PARTITION_SEARCH_RECURSE_LEVELS: String = s"$CONNECTOR_PREFIX.partition.search.recurse.levels" + val SOURCE_PARTITION_SEARCH_RECURSE_LEVELS_DOC: String = + "When searching for new partitions on the S3 filesystem, how many levels deep to recurse." + val SOURCE_PARTITION_SEARCH_RECURSE_LEVELS_DEFAULT: Int = 0 + + val SOURCE_PARTITION_SEARCH_INTERVAL_MILLIS: String = s"$CONNECTOR_PREFIX.partition.search.interval" + val SOURCE_PARTITION_SEARCH_INTERVAL_MILLIS_DOC: String = + "The interval in milliseconds between searching for new partitions. Defaults to 5 minutes." + val SOURCE_PARTITION_SEARCH_INTERVAL_MILLIS_DEFAULT: Long = 300000L + + val SOURCE_PARTITION_SEARCH_MODE: String = s"$CONNECTOR_PREFIX.partition.search.continuous" + val SOURCE_PARTITION_SEARCH_MODE_DOC: String = + "If set to true, it will be continuously search for new partitions. Otherwise it is a one-off operation. Defaults to true." + + def addSourcePartitionSearcherSettings(configDef: ConfigDef): ConfigDef = + configDef.define( + SOURCE_PARTITION_SEARCH_RECURSE_LEVELS, + Type.INT, + SOURCE_PARTITION_SEARCH_RECURSE_LEVELS_DEFAULT, + Importance.LOW, + SOURCE_PARTITION_SEARCH_RECURSE_LEVELS_DOC, + "Source", + 3, + ConfigDef.Width.MEDIUM, + SOURCE_PARTITION_SEARCH_RECURSE_LEVELS, + ) + .define( + SOURCE_PARTITION_SEARCH_MODE, + Type.BOOLEAN, + true, + Importance.LOW, + SOURCE_PARTITION_SEARCH_MODE_DOC, + "Source", + 4, + ConfigDef.Width.MEDIUM, + SOURCE_PARTITION_SEARCH_MODE, + ) + .define( + SOURCE_PARTITION_SEARCH_INTERVAL_MILLIS, + Type.LONG, + SOURCE_PARTITION_SEARCH_INTERVAL_MILLIS_DEFAULT, + Importance.LOW, + SOURCE_PARTITION_SEARCH_INTERVAL_MILLIS_DOC, + "Source", + 5, + ConfigDef.Width.MEDIUM, + SOURCE_PARTITION_SEARCH_INTERVAL_MILLIS, + ) +} +trait SourcePartitionSearcherSettings extends BaseSettings with SourcePartitionSearcherSettingsKeys { def getPartitionSearcherOptions(props: Map[String, _]): PartitionSearcherOptions = PartitionSearcherOptions( diff --git a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/distribution/PartitionSearcher.scala b/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/distribution/PartitionSearcher.scala index 138050021f..b7671fda49 100644 --- a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/distribution/PartitionSearcher.scala +++ b/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/distribution/PartitionSearcher.scala @@ -18,11 +18,12 @@ package io.lenses.streamreactor.connect.aws.s3.source.distribution import cats.effect.IO import cats.implicits._ import com.typesafe.scalalogging.LazyLogging -import io.lenses.streamreactor.connect.aws.s3.source.config.PartitionSearcherOptions import io.lenses.streamreactor.connect.aws.s3.storage.AwsS3DirectoryLister -import io.lenses.streamreactor.connect.aws.s3.storage.DirectoryFindCompletionConfig -import io.lenses.streamreactor.connect.cloud.config.ConnectorTaskId -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocation +import io.lenses.streamreactor.connect.cloud.common.config.ConnectorTaskId +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocation +import io.lenses.streamreactor.connect.cloud.common.source.config.PartitionSearcherOptions +import io.lenses.streamreactor.connect.cloud.common.source.distribution.PartitionSearcherResponse +import io.lenses.streamreactor.connect.cloud.common.storage.DirectoryFindCompletionConfig import software.amazon.awssdk.services.s3.model.ListObjectsV2Request import software.amazon.awssdk.services.s3.model.ListObjectsV2Response diff --git a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/state/ReaderManagerBuilder.scala b/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/state/ReaderManagerBuilder.scala index ee0a3ef7ad..8571eeb8e6 100644 --- a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/state/ReaderManagerBuilder.scala +++ b/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/state/ReaderManagerBuilder.scala @@ -18,13 +18,13 @@ package io.lenses.streamreactor.connect.aws.s3.source.state import cats.effect.IO import cats.effect.Ref import io.lenses.streamreactor.connect.aws.s3.source.config.SourceBucketOptions -import io.lenses.streamreactor.connect.aws.s3.source.files.S3SourceFileQueue -import io.lenses.streamreactor.connect.aws.s3.source.reader.ReaderManager -import io.lenses.streamreactor.connect.aws.s3.source.reader.ResultReader -import io.lenses.streamreactor.connect.aws.s3.storage.StorageInterface -import io.lenses.streamreactor.connect.cloud.config.ConnectorTaskId -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocation -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocationValidator +import io.lenses.streamreactor.connect.cloud.common.config.ConnectorTaskId +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocation +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocationValidator +import io.lenses.streamreactor.connect.cloud.common.source.files.S3SourceFileQueue +import io.lenses.streamreactor.connect.cloud.common.source.reader.ReaderManager +import io.lenses.streamreactor.connect.cloud.common.source.reader.ResultReader +import io.lenses.streamreactor.connect.cloud.common.storage.StorageInterface import org.apache.kafka.connect.errors.ConnectException /** diff --git a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/state/S3SourceBuilder.scala b/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/state/S3SourceBuilder.scala index 3f166e1aaf..82ffe3d9aa 100644 --- a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/state/S3SourceBuilder.scala +++ b/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/state/S3SourceBuilder.scala @@ -22,13 +22,14 @@ import io.lenses.streamreactor.connect.aws.s3.auth.AwsS3ClientCreator import io.lenses.streamreactor.connect.aws.s3.config.S3ConfigSettings.CONNECTOR_PREFIX import io.lenses.streamreactor.connect.aws.s3.source.config.S3SourceConfig import io.lenses.streamreactor.connect.aws.s3.source.distribution.PartitionSearcher -import io.lenses.streamreactor.connect.aws.s3.source.reader.PartitionDiscovery -import io.lenses.streamreactor.connect.aws.s3.source.reader.ReaderManager -import io.lenses.streamreactor.connect.aws.s3.source.reader.ReaderManagerState import io.lenses.streamreactor.connect.aws.s3.storage.AwsS3StorageInterface -import io.lenses.streamreactor.connect.cloud.config.ConnectorTaskIdCreator -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocation -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocationValidator +import io.lenses.streamreactor.connect.cloud.common.config.ConnectorTaskIdCreator +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocation +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocationValidator +import io.lenses.streamreactor.connect.cloud.common.source.reader.PartitionDiscovery +import io.lenses.streamreactor.connect.cloud.common.source.reader.ReaderManager +import io.lenses.streamreactor.connect.cloud.common.source.reader.ReaderManagerState +import io.lenses.streamreactor.connect.cloud.common.source.state.S3SourceTaskState import java.util import scala.jdk.CollectionConverters.IteratorHasAsScala diff --git a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/storage/AwsS3DirectoryLister.scala b/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/storage/AwsS3DirectoryLister.scala index caeef8f84e..f8589e392d 100644 --- a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/storage/AwsS3DirectoryLister.scala +++ b/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/storage/AwsS3DirectoryLister.scala @@ -18,8 +18,10 @@ package io.lenses.streamreactor.connect.aws.s3.storage import cats.effect.IO import cats.implicits._ import com.typesafe.scalalogging.LazyLogging -import io.lenses.streamreactor.connect.cloud.config.ConnectorTaskId -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocation +import io.lenses.streamreactor.connect.cloud.common.config.ConnectorTaskId +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocation +import io.lenses.streamreactor.connect.cloud.common.storage.DirectoryFindCompletionConfig +import io.lenses.streamreactor.connect.cloud.common.storage.DirectoryFindResults import software.amazon.awssdk.services.s3.model._ import scala.jdk.CollectionConverters.IteratorHasAsScala diff --git a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/storage/AwsS3StorageInterface.scala b/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/storage/AwsS3StorageInterface.scala index 11698db580..4bb1ed5651 100644 --- a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/storage/AwsS3StorageInterface.scala +++ b/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/storage/AwsS3StorageInterface.scala @@ -16,9 +16,21 @@ package io.lenses.streamreactor.connect.aws.s3.storage import cats.implicits._ import com.typesafe.scalalogging.LazyLogging -import io.lenses.streamreactor.connect.aws.s3.storage.ResultProcessors.processAsKey -import io.lenses.streamreactor.connect.cloud.config.ConnectorTaskId -import io.lenses.streamreactor.connect.cloud.config.ObjectMetadata +import io.lenses.streamreactor.connect.cloud.common.storage.ResultProcessors.processAsKey +import io.lenses.streamreactor.connect.cloud.common.config.ConnectorTaskId +import io.lenses.streamreactor.connect.cloud.common.config.ObjectMetadata +import io.lenses.streamreactor.connect.cloud.common.storage.EmptyContentsStringError +import io.lenses.streamreactor.connect.cloud.common.storage.FileCreateError +import io.lenses.streamreactor.connect.cloud.common.storage.FileDeleteError +import io.lenses.streamreactor.connect.cloud.common.storage.FileListError +import io.lenses.streamreactor.connect.cloud.common.storage.FileLoadError +import io.lenses.streamreactor.connect.cloud.common.storage.FileMetadata +import io.lenses.streamreactor.connect.cloud.common.storage.ListResponse +import io.lenses.streamreactor.connect.cloud.common.storage.NonExistingFileError +import io.lenses.streamreactor.connect.cloud.common.storage.StorageInterface +import io.lenses.streamreactor.connect.cloud.common.storage.UploadError +import io.lenses.streamreactor.connect.cloud.common.storage.UploadFailedError +import io.lenses.streamreactor.connect.cloud.common.storage.ZeroByteFileError import org.apache.commons.io.IOUtils import software.amazon.awssdk.core.ResponseInputStream import software.amazon.awssdk.core.sync.RequestBody @@ -61,6 +73,7 @@ class AwsS3StorageInterface(val connectorTaskId: ConnectorTaskId, val s3Client: listObjectsV2Response .contents() .asScala + .map(o => FileMetadata(o.key(), o.lastModified())) .toSeq, ) @@ -71,7 +84,7 @@ class AwsS3StorageInterface(val connectorTaskId: ConnectorTaskId, val s3Client: override def listRecursive[T]( bucket: String, prefix: Option[String], - processFn: (String, Option[String], Seq[S3Object]) => Option[ListResponse[T]], + processFn: (String, Option[String], Seq[FileMetadata]) => Option[ListResponse[T]], ): Either[FileListError, Option[ListResponse[T]]] = { logger.debug(s"[{}] List path {}:{}", connectorTaskId.show, bucket, prefix) Try { @@ -83,7 +96,13 @@ class AwsS3StorageInterface(val connectorTaskId: ConnectorTaskId, val s3Client: val pagReq = s3Client.listObjectsV2Paginator(options) - processFn(bucket, prefix, pagReq.iterator().asScala.flatMap(_.contents().asScala.toSeq).toSeq) + processFn( + bucket, + prefix, + pagReq.iterator().asScala.flatMap(_.contents().asScala.toSeq.map(o => + new FileMetadata(o.key(), o.lastModified()), + )).toSeq, + ) }.toEither.leftMap { ex: Throwable => FileListError(ex, bucket, prefix) } diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/config/processors/LowerCaseKeyConfigDefProcessorTest.scala b/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/config/processors/LowerCaseKeyConfigDefProcessorTest.scala index 28036b1437..723f3f27e6 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/config/processors/LowerCaseKeyConfigDefProcessorTest.scala +++ b/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/config/processors/LowerCaseKeyConfigDefProcessorTest.scala @@ -16,7 +16,7 @@ package io.lenses.streamreactor.connect.aws.s3.config.processors import io.lenses.streamreactor.connect.aws.s3.config.S3ConfigSettings._ -import io.lenses.streamreactor.connect.cloud.config.processors.LowerCaseKeyConfigDefProcessor +import io.lenses.streamreactor.connect.cloud.common.config.processors.LowerCaseKeyConfigDefProcessor import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/model/CloudLocationTest.scala b/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/model/CloudLocationTest.scala index 12ec1fd649..5bc451d604 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/model/CloudLocationTest.scala +++ b/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/model/CloudLocationTest.scala @@ -17,8 +17,8 @@ package io.lenses.streamreactor.connect.aws.s3.model import cats.implicits.catsSyntaxOptionId import io.lenses.streamreactor.connect.aws.s3.model.location.S3LocationValidator -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocation -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocationValidator +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocation +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocationValidator import org.scalatest.EitherValues import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/model/HierarchicalPartitionExtractorTest.scala b/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/model/HierarchicalPartitionExtractorTest.scala index 025ff21ccd..b7a9f4d4ee 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/model/HierarchicalPartitionExtractorTest.scala +++ b/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/model/HierarchicalPartitionExtractorTest.scala @@ -17,7 +17,7 @@ package io.lenses.streamreactor.connect.aws.s3.model import cats.implicits.catsSyntaxOptionId import com.typesafe.scalalogging.LazyLogging -import io.lenses.streamreactor.connect.aws.s3.source.config.HierarchicalPartitionExtractor +import io.lenses.streamreactor.connect.cloud.common.source.config.HierarchicalPartitionExtractor import org.scalatest.flatspec.AnyFlatSpecLike import org.scalatest.matchers.should.Matchers diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/model/PartitionExtractorTest.scala b/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/model/PartitionExtractorTest.scala index e9b2158217..31263b3432 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/model/PartitionExtractorTest.scala +++ b/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/model/PartitionExtractorTest.scala @@ -15,7 +15,7 @@ */ package io.lenses.streamreactor.connect.aws.s3.model -import io.lenses.streamreactor.connect.aws.s3.source.config.HierarchicalPartitionExtractor +import io.lenses.streamreactor.connect.cloud.common.source.config.HierarchicalPartitionExtractor import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/config/PartitionDisplayTest.scala b/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/config/PartitionDisplayTest.scala index 088b852e20..267c3a4c88 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/config/PartitionDisplayTest.scala +++ b/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/config/PartitionDisplayTest.scala @@ -16,13 +16,13 @@ package io.lenses.streamreactor.connect.aws.s3.sink.config import com.datamountaineer.kcql.Kcql -import io.lenses.streamreactor.connect.cloud.config.kcqlprops.S3PropsKeyEnum.PartitionIncludeKeys -import io.lenses.streamreactor.connect.cloud.config.kcqlprops.S3PropsKeyEntry -import io.lenses.streamreactor.connect.cloud.config.kcqlprops.S3PropsKeyEnum -import io.lenses.streamreactor.connect.cloud.sink.config.PartitionDisplay -import io.lenses.streamreactor.connect.cloud.sink.config.PartitionDisplay.KeysAndValues -import io.lenses.streamreactor.connect.cloud.sink.config.PartitionDisplay.Values -import io.lenses.streamreactor.connect.cloud.sink.config.kcqlprops.S3SinkPropsSchema +import io.lenses.streamreactor.connect.cloud.common.config.kcqlprops.S3PropsKeyEnum.PartitionIncludeKeys +import io.lenses.streamreactor.connect.cloud.common.config.kcqlprops.S3PropsKeyEntry +import io.lenses.streamreactor.connect.cloud.common.config.kcqlprops.S3PropsKeyEnum +import io.lenses.streamreactor.connect.cloud.common.sink.config.kcqlprops.S3SinkPropsSchema +import io.lenses.streamreactor.connect.cloud.common.sink.config.PartitionDisplay +import io.lenses.streamreactor.connect.cloud.common.sink.config.PartitionDisplay.KeysAndValues +import io.lenses.streamreactor.connect.cloud.common.sink.config.PartitionDisplay.Values import io.lenses.streamreactor.connect.config.kcqlprops.KcqlProperties import org.mockito.MockitoSugar import org.scalatest.BeforeAndAfter diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/config/S3SinkConfigDefBuilderTest.scala b/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/config/S3SinkConfigDefBuilderTest.scala index 1116a7dcc1..19bd92a21a 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/config/S3SinkConfigDefBuilderTest.scala +++ b/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/config/S3SinkConfigDefBuilderTest.scala @@ -16,13 +16,13 @@ package io.lenses.streamreactor.connect.aws.s3.sink.config import io.lenses.streamreactor.connect.aws.s3.model.location.S3LocationValidator -import io.lenses.streamreactor.connect.cloud.config.ConnectorTaskId -import io.lenses.streamreactor.connect.cloud.config.DataStorageSettings -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocationValidator -import io.lenses.streamreactor.connect.cloud.sink.commit.Count -import io.lenses.streamreactor.connect.cloud.sink.commit.FileSize -import io.lenses.streamreactor.connect.cloud.sink.commit.Interval -import io.lenses.streamreactor.connect.cloud.sink.config.S3FlushSettings +import io.lenses.streamreactor.connect.cloud.common.config.ConnectorTaskId +import io.lenses.streamreactor.connect.cloud.common.config.DataStorageSettings +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocationValidator +import io.lenses.streamreactor.connect.cloud.common.sink.commit.Count +import io.lenses.streamreactor.connect.cloud.common.sink.commit.FileSize +import io.lenses.streamreactor.connect.cloud.common.sink.commit.Interval +import io.lenses.streamreactor.connect.cloud.common.sink.config.S3FlushSettings import org.mockito.MockitoSugar import org.scalatest.EitherValues import org.scalatest.flatspec.AnyFlatSpec diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/config/S3SinkConfigTest.scala b/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/config/S3SinkConfigTest.scala index 3a9ceddff1..efe2790553 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/config/S3SinkConfigTest.scala +++ b/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/config/S3SinkConfigTest.scala @@ -16,9 +16,9 @@ package io.lenses.streamreactor.connect.aws.s3.sink.config import io.lenses.streamreactor.connect.aws.s3.model.location.S3LocationValidator -import io.lenses.streamreactor.connect.cloud.config.ConnectorTaskId -import io.lenses.streamreactor.connect.cloud.config.DataStorageSettings -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocationValidator +import io.lenses.streamreactor.connect.cloud.common.config.ConnectorTaskId +import io.lenses.streamreactor.connect.cloud.common.config.DataStorageSettings +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocationValidator import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/config/TestConfigDefBuilder.scala b/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/config/TestConfigDefBuilder.scala deleted file mode 100644 index 1a36d07a9f..0000000000 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/config/TestConfigDefBuilder.scala +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright 2017-2023 Lenses.io Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.lenses.streamreactor.connect.aws.s3.sink.config - -import scala.jdk.CollectionConverters.MapHasAsJava - -object TestConfigDefBuilder { - - def apply(pairs: (String, String)*): S3SinkConfigDefBuilder = { - val map: Map[String, String] = pairs.toMap - val newMap = map + { - "connect.s3.kcql" -> "dummy value" - } - S3SinkConfigDefBuilder(newMap.asJava) - } - -} diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/transformers/SequenceTransformerTest.scala b/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/transformers/SequenceTransformerTest.scala index bf76e269ca..c5a4b3ebf7 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/transformers/SequenceTransformerTest.scala +++ b/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/transformers/SequenceTransformerTest.scala @@ -16,13 +16,14 @@ package io.lenses.streamreactor.connect.aws.s3.sink.transformers import cats.implicits.catsSyntaxEitherId -import io.lenses.streamreactor.connect.aws.s3.utils.SampleData -import io.lenses.streamreactor.connect.cloud.formats.writer.ByteArraySinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.MessageDetail -import io.lenses.streamreactor.connect.cloud.formats.writer.StringSinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.StructSinkData -import io.lenses.streamreactor.connect.cloud.model.Offset -import io.lenses.streamreactor.connect.cloud.model.Topic +import io.lenses.streamreactor.connect.cloud.common.formats.writer +import io.lenses.streamreactor.connect.cloud.common.formats.writer.ByteArraySinkData +import io.lenses.streamreactor.connect.cloud.common.formats.writer.MessageDetail +import io.lenses.streamreactor.connect.cloud.common.formats.writer.StringSinkData +import io.lenses.streamreactor.connect.cloud.common.formats.writer.StructSinkData +import io.lenses.streamreactor.connect.cloud.common.model.Offset +import io.lenses.streamreactor.connect.cloud.common.model.Topic +import io.lenses.streamreactor.connect.cloud.common.utils.SampleData import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers @@ -30,7 +31,7 @@ import java.time.Instant class SequenceTransformerTest extends AnyFunSuite with Matchers { test("empty sequence returns the same message") { - val messageDetail = MessageDetail( + val messageDetail = writer.MessageDetail( StringSinkData("key"), StructSinkData(SampleData.Users.head), Map("header1" -> StringSinkData("value1"), "header2" -> ByteArraySinkData("value2".getBytes())), @@ -46,7 +47,7 @@ class SequenceTransformerTest extends AnyFunSuite with Matchers { } } test("apply all the transformers in the sequence") { - val messageDetail = MessageDetail( + val messageDetail = writer.MessageDetail( StringSinkData("key"), StructSinkData(SampleData.Users.head), Map("header1" -> StringSinkData("value1"), "header2" -> ByteArraySinkData("value2".getBytes())), @@ -66,7 +67,7 @@ class SequenceTransformerTest extends AnyFunSuite with Matchers { } } test("return the error from the first transformer") { - val messageDetail = MessageDetail( + val messageDetail = writer.MessageDetail( StringSinkData("key"), StructSinkData(SampleData.Users.head), Map("header1" -> StringSinkData("value1"), "header2" -> ByteArraySinkData("value2".getBytes())), diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/transformers/TopicsTransformersTest.scala b/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/transformers/TopicsTransformersTest.scala index ad0546ad27..a9496cccd8 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/transformers/TopicsTransformersTest.scala +++ b/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/transformers/TopicsTransformersTest.scala @@ -16,13 +16,14 @@ package io.lenses.streamreactor.connect.aws.s3.sink.transformers import cats.implicits.catsSyntaxEitherId -import io.lenses.streamreactor.connect.aws.s3.utils.SampleData -import io.lenses.streamreactor.connect.cloud.formats.writer.ByteArraySinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.MessageDetail -import io.lenses.streamreactor.connect.cloud.formats.writer.StringSinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.StructSinkData -import io.lenses.streamreactor.connect.cloud.model.Offset -import io.lenses.streamreactor.connect.cloud.model.Topic +import io.lenses.streamreactor.connect.cloud.common.formats.writer +import io.lenses.streamreactor.connect.cloud.common.formats.writer.ByteArraySinkData +import io.lenses.streamreactor.connect.cloud.common.formats.writer.MessageDetail +import io.lenses.streamreactor.connect.cloud.common.formats.writer.StringSinkData +import io.lenses.streamreactor.connect.cloud.common.formats.writer.StructSinkData +import io.lenses.streamreactor.connect.cloud.common.model.Offset +import io.lenses.streamreactor.connect.cloud.common.model.Topic +import io.lenses.streamreactor.connect.cloud.common.utils.SampleData import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers @@ -30,7 +31,7 @@ import java.time.Instant class TopicsTransformersTest extends AnyFunSuite with Matchers { test("unmatched topic returns the same message detail") { - val messageDetail = MessageDetail( + val messageDetail = writer.MessageDetail( StringSinkData("key"), StructSinkData(SampleData.Users.head), Map("header1" -> StringSinkData("value1"), "header2" -> ByteArraySinkData("value2".getBytes())), @@ -45,7 +46,7 @@ class TopicsTransformersTest extends AnyFunSuite with Matchers { ) eq messageDetail shouldBe true } test("apply the transformation for a given topic") { - val messageDetail = MessageDetail( + val messageDetail = writer.MessageDetail( StringSinkData("key"), StructSinkData(SampleData.Users.head), Map("header1" -> StringSinkData("value1"), "header2" -> ByteArraySinkData("value2".getBytes())), @@ -70,7 +71,7 @@ class TopicsTransformersTest extends AnyFunSuite with Matchers { ) shouldBe messageDetail.copy(key = actualKey) } test("return the error from the transformer") { - val messageDetail = MessageDetail( + val messageDetail = writer.MessageDetail( StringSinkData("key"), StructSinkData(SampleData.Users.head), Map("header1" -> StringSinkData("value1"), "header2" -> ByteArraySinkData("value2".getBytes())), diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/source/S3SourceConnectorTest.scala b/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/source/S3SourceConnectorTest.scala index 31f51e4699..181dd5ab40 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/source/S3SourceConnectorTest.scala +++ b/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/source/S3SourceConnectorTest.scala @@ -16,7 +16,7 @@ package io.lenses.streamreactor.connect.aws.s3.source import io.lenses.streamreactor.connect.aws.s3.config.S3ConfigSettings.CONNECTOR_PREFIX -import io.lenses.streamreactor.connect.cloud.config.TaskIndexKey +import io.lenses.streamreactor.connect.cloud.common.config.TaskIndexKey import org.scalatest.OptionValues import org.scalatest.flatspec.AnyFlatSpecLike import org.scalatest.matchers.should.Matchers diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/config/S3SourceConfigTest.scala b/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/source/config/S3SourceConfigTest.scala similarity index 75% rename from kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/config/S3SourceConfigTest.scala rename to kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/source/config/S3SourceConfigTest.scala index a2e83fcd17..7709f215f5 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/config/S3SourceConfigTest.scala +++ b/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/source/config/S3SourceConfigTest.scala @@ -13,25 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.config +package io.lenses.streamreactor.connect.aws.s3.source.config -import io.lenses.streamreactor.connect.aws.s3.config.S3ConfigSettings.AUTH_MODE -import io.lenses.streamreactor.connect.aws.s3.config.S3ConfigSettings.AWS_ACCESS_KEY -import io.lenses.streamreactor.connect.aws.s3.config.S3ConfigSettings.AWS_REGION -import io.lenses.streamreactor.connect.aws.s3.config.S3ConfigSettings.AWS_SECRET_KEY -import io.lenses.streamreactor.connect.aws.s3.config.S3ConfigSettings.CONNECTOR_PREFIX -import io.lenses.streamreactor.connect.aws.s3.config.S3ConfigSettings.CUSTOM_ENDPOINT -import io.lenses.streamreactor.connect.aws.s3.config.S3ConfigSettings.ENABLE_VIRTUAL_HOST_BUCKETS -import io.lenses.streamreactor.connect.aws.s3.config.S3ConfigSettings.SOURCE_PARTITION_SEARCH_INTERVAL_MILLIS -import io.lenses.streamreactor.connect.aws.s3.source.config.S3SourceConfig -import io.lenses.streamreactor.connect.aws.s3.source.config.S3SourceConfigDefBuilder -import io.lenses.streamreactor.connect.cloud.config.TaskIndexKey +import io.lenses.streamreactor.connect.aws.s3.config.AuthMode +import io.lenses.streamreactor.connect.aws.s3.config.S3ConfigSettings._ +import io.lenses.streamreactor.connect.cloud.common.config.TaskIndexKey import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers import scala.jdk.CollectionConverters.MapHasAsJava -class S3SourceConfigTest extends AnyFunSuite with Matchers with TaskIndexKey { +class S3SourceConfigTest extends AnyFunSuite with Matchers with TaskIndexKey with SourcePartitionSearcherSettingsKeys { private val Identity: String = "identity" private val Credential: String = "credential" private val BucketName: String = "mybucket" diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/source/config/S3SourceConfigTests.scala b/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/source/config/S3SourceConfigTests.scala index 66829e5503..3c9348cb3c 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/source/config/S3SourceConfigTests.scala +++ b/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/source/config/S3SourceConfigTests.scala @@ -16,15 +16,16 @@ package io.lenses.streamreactor.connect.aws.s3.source.config import io.lenses.streamreactor.connect.aws.s3.config.S3ConfigSettings._ -import io.lenses.streamreactor.connect.aws.s3.source.config.PartitionSearcherOptions.ExcludeIndexes -import io.lenses.streamreactor.connect.cloud.config.TaskIndexKey +import io.lenses.streamreactor.connect.cloud.common.config.TaskIndexKey +import io.lenses.streamreactor.connect.cloud.common.source.config.PartitionSearcherOptions +import io.lenses.streamreactor.connect.cloud.common.source.config.PartitionSearcherOptions.ExcludeIndexes import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers import scala.concurrent.duration._ import scala.jdk.CollectionConverters._ -class S3SourceConfigTests extends AnyFunSuite with Matchers with TaskIndexKey { +class S3SourceConfigTests extends AnyFunSuite with Matchers with TaskIndexKey with SourcePartitionSearcherSettingsKeys { test("default recursive levels is 0") { S3SourceConfig.fromProps( Map( diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/source/config/kcqlprops/S3SourcePropsSchemaTest.scala b/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/source/config/kcqlprops/S3SourcePropsSchemaTest.scala index bdc4e0d3b2..a18f25fc00 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/source/config/kcqlprops/S3SourcePropsSchemaTest.scala +++ b/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/source/config/kcqlprops/S3SourcePropsSchemaTest.scala @@ -15,11 +15,11 @@ */ package io.lenses.streamreactor.connect.aws.s3.source.config.kcqlprops -import io.lenses.streamreactor.connect.cloud.config.kcqlprops.S3PropsKeyEnum -import io.lenses.streamreactor.connect.cloud.source.config.kcqlprops.ReadTextModeEnum.Regex -import io.lenses.streamreactor.connect.cloud.source.config.kcqlprops.ReadTextModeEntry -import io.lenses.streamreactor.connect.cloud.source.config.kcqlprops.ReadTextModeEnum -import io.lenses.streamreactor.connect.cloud.source.config.kcqlprops.S3SourcePropsSchema +import io.lenses.streamreactor.connect.cloud.common.config.kcqlprops.S3PropsKeyEnum +import io.lenses.streamreactor.connect.cloud.common.source.config.kcqlprops.ReadTextModeEnum.Regex +import io.lenses.streamreactor.connect.cloud.common.source.config.kcqlprops.ReadTextModeEntry +import io.lenses.streamreactor.connect.cloud.common.source.config.kcqlprops.ReadTextModeEnum +import io.lenses.streamreactor.connect.cloud.common.source.config.kcqlprops.S3SourcePropsSchema import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/source/reader/ReaderManagerTest.scala b/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/source/reader/ReaderManagerTest.scala index 0a7482a638..54a8129e1a 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/source/reader/ReaderManagerTest.scala +++ b/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/source/reader/ReaderManagerTest.scala @@ -22,10 +22,12 @@ import cats.implicits.catsSyntaxEitherId import cats.implicits.catsSyntaxOptionId import com.typesafe.scalalogging.LazyLogging import io.lenses.streamreactor.connect.aws.s3.model.location.S3LocationValidator -import io.lenses.streamreactor.connect.aws.s3.source.files.SourceFileQueue -import io.lenses.streamreactor.connect.cloud.config.ConnectorTaskId -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocation -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocationValidator +import io.lenses.streamreactor.connect.cloud.common.config.ConnectorTaskId +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocation +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocationValidator +import io.lenses.streamreactor.connect.cloud.common.source.files.SourceFileQueue +import io.lenses.streamreactor.connect.cloud.common.source.reader.ReaderManager +import io.lenses.streamreactor.connect.cloud.common.source.reader.ResultReader import org.apache.kafka.connect.data.Schema import org.apache.kafka.connect.source.SourceRecord import org.mockito.MockitoSugar diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/source/reader/ResultReaderTest.scala b/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/source/reader/ResultReaderTest.scala index 960857f827..f0fffcb557 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/source/reader/ResultReaderTest.scala +++ b/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/source/reader/ResultReaderTest.scala @@ -15,7 +15,8 @@ */ package io.lenses.streamreactor.connect.aws.s3.source.reader -import io.lenses.streamreactor.connect.cloud.formats.reader.S3StreamReader +import io.lenses.streamreactor.connect.cloud.common.formats.reader.S3StreamReader +import io.lenses.streamreactor.connect.cloud.common.source.reader.ResultReader import org.apache.kafka.connect.data.Schema import org.apache.kafka.connect.source.SourceRecord import org.mockito.MockitoSugar diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/source/reader/PartitionDiscoveryTest.scala b/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/source/reader/S3PartitionDiscoveryTest.scala similarity index 75% rename from kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/source/reader/PartitionDiscoveryTest.scala rename to kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/source/reader/S3PartitionDiscoveryTest.scala index e30e946e72..8769010756 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/source/reader/PartitionDiscoveryTest.scala +++ b/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/source/reader/S3PartitionDiscoveryTest.scala @@ -14,22 +14,27 @@ * limitations under the License. */ package io.lenses.streamreactor.connect.aws.s3.source.reader -import io.lenses.streamreactor.connect.aws.s3.source.config.PartitionSearcherOptions.ExcludeIndexes + import cats.effect.IO import cats.effect.kernel.Ref import cats.effect.unsafe.implicits.global import cats.implicits.catsSyntaxOptionId import io.lenses.streamreactor.connect.aws.s3.model.location.S3LocationValidator -import io.lenses.streamreactor.connect.aws.s3.source.config.PartitionSearcherOptions import io.lenses.streamreactor.connect.aws.s3.source.distribution.PartitionSearcher -import io.lenses.streamreactor.connect.aws.s3.source.distribution.PartitionSearcherResponse -import io.lenses.streamreactor.connect.aws.s3.source.files.SourceFileQueue -import io.lenses.streamreactor.connect.aws.s3.storage.DirectoryFindResults import io.lenses.streamreactor.connect.aws.s3.storage.MockS3Client import io.lenses.streamreactor.connect.aws.s3.storage.S3Page -import io.lenses.streamreactor.connect.cloud.config.ConnectorTaskId -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocation -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocationValidator +import io.lenses.streamreactor.connect.cloud.common.config.ConnectorTaskId +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocation +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocationValidator +import io.lenses.streamreactor.connect.cloud.common.source.config.PartitionSearcherOptions +import io.lenses.streamreactor.connect.cloud.common.source.config.PartitionSearcherOptions.ExcludeIndexes +import io.lenses.streamreactor.connect.cloud.common.source.distribution.PartitionSearcherResponse +import io.lenses.streamreactor.connect.cloud.common.source.files.SourceFileQueue +import io.lenses.streamreactor.connect.cloud.common.source.reader.PartitionDiscovery +import io.lenses.streamreactor.connect.cloud.common.source.reader.ReaderManager +import io.lenses.streamreactor.connect.cloud.common.source.reader.ReaderManagerState +import io.lenses.streamreactor.connect.cloud.common.source.reader.ResultReader +import io.lenses.streamreactor.connect.cloud.common.storage.DirectoryFindResults import org.mockito.MockitoSugar import org.scalatest.flatspec.AnyFlatSpecLike import org.scalatest.matchers.should.Matchers @@ -37,80 +42,9 @@ import org.scalatest.matchers.should.Matchers import scala.concurrent.duration.DurationInt import scala.jdk.CollectionConverters.IteratorHasAsScala -class PartitionDiscoveryTest extends AnyFlatSpecLike with Matchers with MockitoSugar { +class S3PartitionDiscoveryTest extends AnyFlatSpecLike with Matchers with MockitoSugar { private implicit val cloudLocationValidator: CloudLocationValidator = S3LocationValidator private val connectorTaskId: ConnectorTaskId = ConnectorTaskId("sinkName", 1, 1) - "PartitionDiscovery" should "handle failure on PartitionSearcher and resume" in { - val fileQueueProcessor: SourceFileQueue = mock[SourceFileQueue] - val limit = 10 - val options = PartitionSearcherOptions(1, continuous = true, 100.millis, ExcludeIndexes) - - trait Count { - def getCount: IO[Int] - - def find( - lastFound: Seq[PartitionSearcherResponse], - ): IO[Seq[PartitionSearcherResponse]] - } - val searcherMock = new Count { - private val count = Ref[IO].of(0).unsafeRunSync() - def getCount: IO[Int] = count.get - - def find( - lastFound: Seq[PartitionSearcherResponse], - ): IO[Seq[PartitionSearcherResponse]] = - for { - c <- count.getAndUpdate(_ + 1) - _ <- if (c == 0) IO.raiseError(new RuntimeException("error")) else IO.unit - } yield { - List( - PartitionSearcherResponse(CloudLocation("bucket", None), - Set("prefix1/", "prefix2/"), - DirectoryFindResults(Set("prefix1/", "prefix2/")), - None, - ), - ) - } - } - - val io = for { - cancelledRef <- Ref[IO].of(false) - readerRef <- Ref[IO].of(Option.empty[ResultReader]) - state <- Ref[IO].of(ReaderManagerState(Seq.empty, Seq.empty)) - fiber <- PartitionDiscovery.run( - connectorTaskId, - options, - searcherMock.find, - (_, _) => - IO(new ReaderManager(limit, - fileQueueProcessor, - _ => Left(new RuntimeException()), - connectorTaskId, - readerRef, - )), - state, - cancelledRef, - ).start - _ <- IO.sleep(1000.millis) - _ <- cancelledRef.set(true) - _ <- fiber.join - readerMgrState <- state.get - callsMade <- searcherMock.getCount - } yield readerMgrState -> callsMade - - val (state, callsMade) = io.unsafeRunSync() - assert( - state.partitionResponses == List( - PartitionSearcherResponse( - CloudLocation("bucket", None), - Set("prefix1/", "prefix2/"), - DirectoryFindResults(Set("prefix1/", "prefix2/")), - None, - ), - ), - ) - callsMade >= 1 - } "PartitionDiscovery" should "discover all partitions" in { val fileQueueProcessor: SourceFileQueue = mock[SourceFileQueue] val limit = 10 diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/source/state/ReaderManagerBuilderTest.scala b/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/source/state/ReaderManagerBuilderTest.scala index 2c024b3cf7..dcaea2f95a 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/source/state/ReaderManagerBuilderTest.scala +++ b/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/source/state/ReaderManagerBuilderTest.scala @@ -17,13 +17,13 @@ package io.lenses.streamreactor.connect.aws.s3.source.state import cats.effect.testing.scalatest.AsyncIOSpec import io.lenses.streamreactor.connect.aws.s3.model.location.S3LocationValidator -import io.lenses.streamreactor.connect.aws.s3.source.config.OrderingType import io.lenses.streamreactor.connect.aws.s3.source.config.SourceBucketOptions -import io.lenses.streamreactor.connect.aws.s3.storage.StorageInterface -import io.lenses.streamreactor.connect.cloud.config.AvroFormatSelection -import io.lenses.streamreactor.connect.cloud.config.ConnectorTaskId -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocation -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocationValidator +import io.lenses.streamreactor.connect.cloud.common.config.AvroFormatSelection +import io.lenses.streamreactor.connect.cloud.common.config.ConnectorTaskId +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocation +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocationValidator +import io.lenses.streamreactor.connect.cloud.common.source.config.OrderingType +import io.lenses.streamreactor.connect.cloud.common.storage.StorageInterface import org.mockito.MockitoSugar.mock import org.scalatest.flatspec.AsyncFlatSpec import org.scalatest.matchers.should.Matchers diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/storage/AwsS3DirectoryListerTest.scala b/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/storage/AwsS3DirectoryListerTest.scala index f89e1c792d..010918568c 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/storage/AwsS3DirectoryListerTest.scala +++ b/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/storage/AwsS3DirectoryListerTest.scala @@ -18,9 +18,11 @@ package io.lenses.streamreactor.connect.aws.s3.storage import cats.effect.unsafe.implicits.global import cats.implicits._ import io.lenses.streamreactor.connect.aws.s3.model.location.S3LocationValidator -import io.lenses.streamreactor.connect.cloud.config.ConnectorTaskId -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocation -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocationValidator +import io.lenses.streamreactor.connect.cloud.common.config.ConnectorTaskId +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocation +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocationValidator +import io.lenses.streamreactor.connect.cloud.common.storage.DirectoryFindCompletionConfig +import io.lenses.streamreactor.connect.cloud.common.storage.DirectoryFindResults import org.scalatest.flatspec.AnyFlatSpecLike import org.scalatest.matchers.should.Matchers import software.amazon.awssdk.services.s3.S3Client diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/config/CompressionCodecSettings.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/config/CompressionCodecSettings.scala similarity index 84% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/config/CompressionCodecSettings.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/config/CompressionCodecSettings.scala index d090ba87e3..b03dd5379b 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/config/CompressionCodecSettings.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/config/CompressionCodecSettings.scala @@ -13,13 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.config +package io.lenses.streamreactor.connect.cloud.common.config import com.datamountaineer.streamreactor.common.config.base.traits.BaseSettings import com.datamountaineer.streamreactor.common.config.base.traits.WithConnectorPrefix -import io.lenses.streamreactor.connect.cloud.model.CompressionCodecName.UNCOMPRESSED -import io.lenses.streamreactor.connect.cloud.model.CompressionCodec -import io.lenses.streamreactor.connect.cloud.model.CompressionCodecName +import io.lenses.streamreactor.connect.cloud.common.model.CompressionCodec +import io.lenses.streamreactor.connect.cloud.common.model.CompressionCodecName +import io.lenses.streamreactor.connect.cloud.common.model.CompressionCodecName.UNCOMPRESSED trait CompressionCodecConfigKeys extends WithConnectorPrefix { diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/config/ConnectorTaskId.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/config/ConnectorTaskId.scala similarity index 94% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/config/ConnectorTaskId.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/config/ConnectorTaskId.scala index 83fb1f0615..c8fdb6d124 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/config/ConnectorTaskId.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/config/ConnectorTaskId.scala @@ -13,11 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.config +package io.lenses.streamreactor.connect.cloud.common.config + import cats.Show import cats.implicits.toBifunctorOps import com.datamountaineer.streamreactor.common.config.base.traits.WithConnectorPrefix -import io.lenses.streamreactor.connect.cloud.source.config.distribution.PartitionHasher +import io.lenses.streamreactor.connect.cloud.common.source.config.distribution.PartitionHasher import java.util diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/config/DataStorageSettings.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/config/DataStorageSettings.scala similarity index 98% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/config/DataStorageSettings.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/config/DataStorageSettings.scala index a675a05764..be37b5ebf3 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/config/DataStorageSettings.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/config/DataStorageSettings.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.config +package io.lenses.streamreactor.connect.cloud.common.config import cats.data.ValidatedNel import cats.implicits.catsSyntaxEitherId diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/config/DeleteModeSettings.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/config/DeleteModeSettings.scala similarity index 96% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/config/DeleteModeSettings.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/config/DeleteModeSettings.scala index 8d1c61993f..bac0d75e3b 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/config/DeleteModeSettings.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/config/DeleteModeSettings.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.config +package io.lenses.streamreactor.connect.cloud.common.config import com.datamountaineer.streamreactor.common.config.base.traits.BaseSettings import com.datamountaineer.streamreactor.common.config.base.traits.WithConnectorPrefix diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/config/Format.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/config/Format.scala similarity index 94% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/config/Format.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/config/Format.scala index 942c7310d2..ce3fd2b626 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/config/Format.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/config/Format.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.config +package io.lenses.streamreactor.connect.cloud.common.config import enumeratum.Enum import enumeratum.EnumEntry diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/config/FormatOptions.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/config/FormatOptions.scala similarity index 93% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/config/FormatOptions.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/config/FormatOptions.scala index 2a3e98b16c..72be8c320c 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/config/FormatOptions.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/config/FormatOptions.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.config +package io.lenses.streamreactor.connect.cloud.common.config import enumeratum.Enum import enumeratum.EnumEntry diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/config/FormatSelection.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/config/FormatSelection.scala similarity index 80% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/config/FormatSelection.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/config/FormatSelection.scala index 9a81457bde..338abf3b25 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/config/FormatSelection.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/config/FormatSelection.scala @@ -13,30 +13,32 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.config +package io.lenses.streamreactor.connect.cloud.common.config import cats.implicits.catsSyntaxEitherId import com.datamountaineer.kcql.Kcql -import io.lenses.streamreactor.connect.cloud.config.FormatOptions.WithHeaders -import io.lenses.streamreactor.connect.cloud.config.kcqlprops.S3PropsKeyEntry -import io.lenses.streamreactor.connect.cloud.config.kcqlprops.S3PropsKeyEnum -import io.lenses.streamreactor.connect.cloud.formats.reader.converters.BytesOutputRowConverter -import io.lenses.streamreactor.connect.cloud.formats.reader.converters.SchemaAndValueConverter -import io.lenses.streamreactor.connect.cloud.formats.reader.converters.SchemaAndValueEnvelopeConverter -import io.lenses.streamreactor.connect.cloud.formats.reader.converters.SchemalessEnvelopeConverter -import io.lenses.streamreactor.connect.cloud.formats.reader.converters.TextConverter -import io.lenses.streamreactor.connect.cloud.formats.reader.AvroStreamReader -import io.lenses.streamreactor.connect.cloud.formats.reader.BytesStreamFileReader -import io.lenses.streamreactor.connect.cloud.formats.reader.CsvStreamReader -import io.lenses.streamreactor.connect.cloud.formats.reader.DelegateIteratorS3StreamReader -import io.lenses.streamreactor.connect.cloud.formats.reader.ParquetStreamReader -import io.lenses.streamreactor.connect.cloud.formats.reader.S3StreamReader -import io.lenses.streamreactor.connect.cloud.formats.reader.TextStreamReader -import io.lenses.streamreactor.connect.cloud.model.CompressionCodecName._ -import io.lenses.streamreactor.connect.cloud.model.CompressionCodecName -import io.lenses.streamreactor.connect.cloud.model.Topic -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocation -import io.lenses.streamreactor.connect.cloud.source.config.ReadTextMode +import io.lenses.streamreactor.connect.cloud.common.config.FormatOptions.WithHeaders +import io.lenses.streamreactor.connect.cloud.common.config.kcqlprops.S3PropsKeyEntry +import io.lenses.streamreactor.connect.cloud.common.config.kcqlprops.S3PropsKeyEnum +import io.lenses.streamreactor.connect.cloud.common.formats.reader.converters.BytesOutputRowConverter +import io.lenses.streamreactor.connect.cloud.common.formats.reader.converters.SchemaAndValueConverter +import io.lenses.streamreactor.connect.cloud.common.formats.reader.converters.SchemaAndValueEnvelopeConverter +import io.lenses.streamreactor.connect.cloud.common.formats.reader.converters.SchemalessEnvelopeConverter +import io.lenses.streamreactor.connect.cloud.common.formats.reader._ +import io.lenses.streamreactor.connect.cloud.common.model.CompressionCodecName +import io.lenses.streamreactor.connect.cloud.common.model.Topic +import io.lenses.streamreactor.connect.cloud.common.model.CompressionCodecName.BROTLI +import io.lenses.streamreactor.connect.cloud.common.model.CompressionCodecName.BZIP2 +import io.lenses.streamreactor.connect.cloud.common.model.CompressionCodecName.DEFLATE +import io.lenses.streamreactor.connect.cloud.common.model.CompressionCodecName.GZIP +import io.lenses.streamreactor.connect.cloud.common.model.CompressionCodecName.LZ4 +import io.lenses.streamreactor.connect.cloud.common.model.CompressionCodecName.LZO +import io.lenses.streamreactor.connect.cloud.common.model.CompressionCodecName.SNAPPY +import io.lenses.streamreactor.connect.cloud.common.model.CompressionCodecName.UNCOMPRESSED +import io.lenses.streamreactor.connect.cloud.common.model.CompressionCodecName.XZ +import io.lenses.streamreactor.connect.cloud.common.model.CompressionCodecName.ZSTD +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocation +import io.lenses.streamreactor.connect.cloud.common.source.config.ReadTextMode import io.lenses.streamreactor.connect.config.kcqlprops.KcqlPropsSchema import java.io.InputStream @@ -122,7 +124,7 @@ case object JsonFormatSelection extends FormatSelection { ) } else { - TextConverter(input) + converters.TextConverter(input) } new DelegateIteratorS3StreamReader[String](inner, converter, input.bucketAndPath) @@ -217,7 +219,7 @@ case class TextFormatSelection(readTextMode: Option[ReadTextMode]) extends Forma readTextMode, input.stream, ) - val converter = TextConverter(input) + val converter = converters.TextConverter(input) new DelegateIteratorS3StreamReader( inner, converter, @@ -234,7 +236,7 @@ case class CsvFormatSelection(formatOptions: Set[FormatOptions]) extends FormatS input: ReaderBuilderContext, ): S3StreamReader = { val inner = new CsvStreamReader(input.stream, hasHeaders = formatOptions.contains(WithHeaders)) - val converter = TextConverter(input) + val converter = converters.TextConverter(input) new DelegateIteratorS3StreamReader( inner, converter, diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/config/TaskDistributor.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/config/TaskDistributor.scala similarity index 94% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/config/TaskDistributor.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/config/TaskDistributor.scala index 8a687e28f9..8d3ede7e60 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/config/TaskDistributor.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/config/TaskDistributor.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.config +package io.lenses.streamreactor.connect.cloud.common.config import java.util import scala.jdk.CollectionConverters.SeqHasAsJava diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/config/kcqlprops/S3PropsKeyEnum.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/config/kcqlprops/S3PropsKeyEnum.scala similarity index 93% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/config/kcqlprops/S3PropsKeyEnum.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/config/kcqlprops/S3PropsKeyEnum.scala index fb92946cc8..c63671744a 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/config/kcqlprops/S3PropsKeyEnum.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/config/kcqlprops/S3PropsKeyEnum.scala @@ -13,11 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.config.kcqlprops +package io.lenses.streamreactor.connect.cloud.common.config.kcqlprops import enumeratum.Enum import enumeratum.EnumEntry -import io.lenses.streamreactor.connect.cloud.config.DataStorageSettings +import io.lenses.streamreactor.connect.cloud.common.config.DataStorageSettings sealed abstract class S3PropsKeyEntry(override val entryName: String) extends EnumEntry diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/config/processors/ConfigDefProcessor.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/config/processors/ConfigDefProcessor.scala similarity index 91% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/config/processors/ConfigDefProcessor.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/config/processors/ConfigDefProcessor.scala index 98b3653eb7..6055d25d6d 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/config/processors/ConfigDefProcessor.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/config/processors/ConfigDefProcessor.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.config.processors +package io.lenses.streamreactor.connect.cloud.common.config.processors /** * ConfigDefProcessor provides an interface to process the configs on entry diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/config/processors/LowerCaseKeyConfigDefProcessor.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/config/processors/LowerCaseKeyConfigDefProcessor.scala similarity index 93% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/config/processors/LowerCaseKeyConfigDefProcessor.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/config/processors/LowerCaseKeyConfigDefProcessor.scala index 836a8197ec..5ad6b85a2e 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/config/processors/LowerCaseKeyConfigDefProcessor.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/config/processors/LowerCaseKeyConfigDefProcessor.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.config.processors +package io.lenses.streamreactor.connect.cloud.common.config.processors import cats.implicits.catsSyntaxEitherId diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/config/processors/kcql/KcqlMapToStringConverter.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/config/processors/kcql/KcqlMapToStringConverter.scala similarity index 98% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/config/processors/kcql/KcqlMapToStringConverter.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/config/processors/kcql/KcqlMapToStringConverter.scala index 4b9912f0cc..cf796e9970 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/config/processors/kcql/KcqlMapToStringConverter.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/config/processors/kcql/KcqlMapToStringConverter.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.config.processors.kcql +package io.lenses.streamreactor.connect.cloud.common.config.processors.kcql import cats.implicits.catsSyntaxEitherId diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/config/processors/kcql/KcqlProp.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/config/processors/kcql/KcqlProp.scala similarity index 97% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/config/processors/kcql/KcqlProp.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/config/processors/kcql/KcqlProp.scala index 29de4e2948..513f7da7ee 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/config/processors/kcql/KcqlProp.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/config/processors/kcql/KcqlProp.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.config.processors.kcql +package io.lenses.streamreactor.connect.cloud.common.config.processors.kcql import com.datamountaineer.kcql.Kcql import enumeratum._ diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/config/processors/kcql/StringToKcqlMapConverter.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/config/processors/kcql/StringToKcqlMapConverter.scala similarity index 95% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/config/processors/kcql/StringToKcqlMapConverter.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/config/processors/kcql/StringToKcqlMapConverter.scala index 03b88cb66e..5d1b5c97e0 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/config/processors/kcql/StringToKcqlMapConverter.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/config/processors/kcql/StringToKcqlMapConverter.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.config.processors.kcql +package io.lenses.streamreactor.connect.cloud.common.config.processors.kcql import cats.implicits.catsSyntaxEitherId import com.datamountaineer.kcql.Kcql diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/config/processors/kcql/YamlToKcqlMapConverter.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/config/processors/kcql/YamlToKcqlMapConverter.scala similarity index 94% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/config/processors/kcql/YamlToKcqlMapConverter.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/config/processors/kcql/YamlToKcqlMapConverter.scala index e7a915d628..b1ddfcbaa0 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/config/processors/kcql/YamlToKcqlMapConverter.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/config/processors/kcql/YamlToKcqlMapConverter.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.config.processors.kcql +package io.lenses.streamreactor.connect.cloud.common.config.processors.kcql import cats.implicits.catsSyntaxEitherId diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/FormatWriterException.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/FormatWriterException.scala similarity index 92% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/FormatWriterException.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/FormatWriterException.scala index d99b315e1b..182f33e52d 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/FormatWriterException.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/FormatWriterException.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.formats +package io.lenses.streamreactor.connect.cloud.common.formats case class FormatWriterException(message: String, exception: Option[Throwable] = None) extends Exception(message, exception.orNull) diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/bytes/BytesOutputRow.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/bytes/BytesOutputRow.scala similarity index 92% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/bytes/BytesOutputRow.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/bytes/BytesOutputRow.scala index a98cc90287..ac556532fb 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/bytes/BytesOutputRow.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/bytes/BytesOutputRow.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.formats.bytes +package io.lenses.streamreactor.connect.cloud.common.formats.bytes /** * Now that we only read/write the value for a BYTES record, this is little more than a wrapper for a byte array. diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/reader/AvroStreamReader.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/AvroStreamReader.scala similarity index 95% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/reader/AvroStreamReader.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/AvroStreamReader.scala index 32f66903ad..7dd2d705c2 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/reader/AvroStreamReader.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/AvroStreamReader.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.formats.reader +package io.lenses.streamreactor.connect.cloud.common.formats.reader import io.confluent.connect.avro.AvroData import org.apache.avro.file.DataFileStream diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/reader/BytesStreamFileReader.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/BytesStreamFileReader.scala similarity index 88% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/reader/BytesStreamFileReader.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/BytesStreamFileReader.scala index 6d53007d4e..5dd487bbf2 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/reader/BytesStreamFileReader.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/BytesStreamFileReader.scala @@ -13,10 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.formats.reader +package io.lenses.streamreactor.connect.cloud.common.formats.reader import com.google.common.io.ByteStreams -import io.lenses.streamreactor.connect.cloud.formats.bytes.BytesOutputRow +import io.lenses.streamreactor.connect.cloud.common.formats.bytes.BytesOutputRow import java.io.InputStream import scala.util.Try diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/reader/CsvStreamReader.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/CsvStreamReader.scala similarity index 87% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/reader/CsvStreamReader.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/CsvStreamReader.scala index f01d38f38a..649b5c6394 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/reader/CsvStreamReader.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/CsvStreamReader.scala @@ -13,9 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.formats.reader +package io.lenses.streamreactor.connect.cloud.common.formats.reader -import io.lenses.streamreactor.connect.cloud.formats.FormatWriterException +import io.lenses.streamreactor.connect.cloud.common.formats.FormatWriterException import java.io.InputStream class CsvStreamReader(input: InputStream, hasHeaders: Boolean) extends TextStreamReader(input) { diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/reader/CustomTextStreamReader.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/CustomTextStreamReader.scala similarity index 93% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/reader/CustomTextStreamReader.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/CustomTextStreamReader.scala index fae974890d..e901af0de3 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/reader/CustomTextStreamReader.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/CustomTextStreamReader.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.formats.reader +package io.lenses.streamreactor.connect.cloud.common.formats.reader import io.lenses.streamreactor.connect.io.text.OptionIteratorAdaptor diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/reader/ParquetFormatReader.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/ParquetFormatReader.scala similarity index 88% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/reader/ParquetFormatReader.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/ParquetFormatReader.scala index 2ae08a9c71..80d17bddff 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/reader/ParquetFormatReader.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/ParquetFormatReader.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.formats.reader +package io.lenses.streamreactor.connect.cloud.common.formats.reader /* * Copyright 2020 Lenses.io @@ -31,8 +31,8 @@ package io.lenses.streamreactor.connect.cloud.formats.reader * limitations under the License. */ -import io.lenses.streamreactor.connect.cloud.formats.reader.parquet.ParquetInputFile -import io.lenses.streamreactor.connect.cloud.formats.reader.parquet.SeekableByteArrayInputStream +import io.lenses.streamreactor.connect.cloud.common.formats.reader.parquet.ParquetInputFile +import io.lenses.streamreactor.connect.cloud.common.formats.reader.parquet.SeekableByteArrayInputStream import org.apache.avro.generic.GenericRecord import org.apache.parquet.avro.AvroParquetReader import org.apache.parquet.hadoop.ParquetReader diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/reader/ParquetReaderIteratorAdaptor.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/ParquetReaderIteratorAdaptor.scala similarity index 89% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/reader/ParquetReaderIteratorAdaptor.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/ParquetReaderIteratorAdaptor.scala index 69756350c4..1d66220ebb 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/reader/ParquetReaderIteratorAdaptor.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/ParquetReaderIteratorAdaptor.scala @@ -13,9 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.formats.reader +package io.lenses.streamreactor.connect.cloud.common.formats.reader -import io.lenses.streamreactor.connect.cloud.formats.FormatWriterException +import io.lenses.streamreactor.connect.cloud.common.formats.FormatWriterException import org.apache.avro.generic.GenericRecord import org.apache.parquet.hadoop.ParquetReader diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/reader/ParquetStreamReader.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/ParquetStreamReader.scala similarity index 89% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/reader/ParquetStreamReader.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/ParquetStreamReader.scala index f8df71c7c2..15aaeb9eff 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/reader/ParquetStreamReader.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/ParquetStreamReader.scala @@ -13,11 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.formats.reader +package io.lenses.streamreactor.connect.cloud.common.formats.reader import io.confluent.connect.avro.AvroData -import io.lenses.streamreactor.connect.cloud.formats.reader.parquet.ParquetSeekableInputStream -import io.lenses.streamreactor.connect.cloud.formats.reader.parquet.ParquetStreamingInputFile +import io.lenses.streamreactor.connect.cloud.common.formats.reader.parquet.ParquetSeekableInputStream +import io.lenses.streamreactor.connect.cloud.common.formats.reader.parquet.ParquetStreamingInputFile import org.apache.avro.generic.GenericRecord import org.apache.hadoop.conf.Configuration import org.apache.kafka.connect.data.SchemaAndValue diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/reader/S3StreamReader.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/S3StreamReader.scala similarity index 92% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/reader/S3StreamReader.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/S3StreamReader.scala index e99128f582..a590d3bc42 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/reader/S3StreamReader.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/S3StreamReader.scala @@ -13,9 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.formats.reader +package io.lenses.streamreactor.connect.cloud.common.formats.reader -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocation +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocation import org.apache.kafka.connect.source.SourceRecord trait S3StreamReader extends AutoCloseable with Iterator[SourceRecord] { diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/reader/TextStreamReader.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/TextStreamReader.scala similarity index 87% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/reader/TextStreamReader.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/TextStreamReader.scala index 8adb4d38be..cba5cfec74 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/reader/TextStreamReader.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/TextStreamReader.scala @@ -13,10 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.formats.reader +package io.lenses.streamreactor.connect.cloud.common.formats.reader -import io.lenses.streamreactor.connect.cloud.formats.FormatWriterException -import io.lenses.streamreactor.connect.cloud.source.config.ReadTextMode +import io.lenses.streamreactor.connect.cloud.common.formats.FormatWriterException +import io.lenses.streamreactor.connect.cloud.common.source.config.ReadTextMode import java.io.InputStream import scala.io.Source diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/reader/Using.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/Using.scala similarity index 91% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/reader/Using.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/Using.scala index a10fa5eb8b..b0863e898b 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/reader/Using.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/Using.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.formats.reader +package io.lenses.streamreactor.connect.cloud.common.formats.reader trait Using { diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/reader/converters/BytesOutputRowConverter.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/converters/BytesOutputRowConverter.scala similarity index 73% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/reader/converters/BytesOutputRowConverter.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/converters/BytesOutputRowConverter.scala index 4fca6f2fa4..db373de869 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/reader/converters/BytesOutputRowConverter.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/converters/BytesOutputRowConverter.scala @@ -13,13 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.formats.reader.converters +package io.lenses.streamreactor.connect.cloud.common.formats.reader.converters -import io.lenses.streamreactor.connect.cloud.formats.bytes.BytesOutputRow -import io.lenses.streamreactor.connect.cloud.formats.reader.Converter -import io.lenses.streamreactor.connect.cloud.model.Topic -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocation -import io.lenses.streamreactor.connect.cloud.source.SourceWatermark +import io.lenses.streamreactor.connect.cloud.common.formats.bytes.BytesOutputRow +import io.lenses.streamreactor.connect.cloud.common.formats.reader.Converter +import io.lenses.streamreactor.connect.cloud.common.model.Topic +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocation +import io.lenses.streamreactor.connect.cloud.common.source.SourceWatermark import org.apache.kafka.connect.data.Schema import org.apache.kafka.connect.source.SourceRecord diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/reader/converters/SchemaAndValueConverter.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/converters/SchemaAndValueConverter.scala similarity index 77% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/reader/converters/SchemaAndValueConverter.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/converters/SchemaAndValueConverter.scala index d2b1dad9d2..199cd4a97b 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/reader/converters/SchemaAndValueConverter.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/converters/SchemaAndValueConverter.scala @@ -13,12 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.formats.reader.converters +package io.lenses.streamreactor.connect.cloud.common.formats.reader.converters -import io.lenses.streamreactor.connect.cloud.formats.reader.Converter -import io.lenses.streamreactor.connect.cloud.model.Topic -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocation -import io.lenses.streamreactor.connect.cloud.source.SourceWatermark +import io.lenses.streamreactor.connect.cloud.common.formats.reader.Converter +import io.lenses.streamreactor.connect.cloud.common.model.Topic +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocation +import io.lenses.streamreactor.connect.cloud.common.source.SourceWatermark import org.apache.kafka.connect.data.SchemaAndValue import org.apache.kafka.connect.source.SourceRecord diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/reader/converters/SchemaAndValueEnvelopeConverter.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/converters/SchemaAndValueEnvelopeConverter.scala similarity index 91% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/reader/converters/SchemaAndValueEnvelopeConverter.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/converters/SchemaAndValueEnvelopeConverter.scala index b4820188f0..f6d6c7da77 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/reader/converters/SchemaAndValueEnvelopeConverter.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/converters/SchemaAndValueEnvelopeConverter.scala @@ -13,12 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.formats.reader.converters +package io.lenses.streamreactor.connect.cloud.common.formats.reader.converters -import io.lenses.streamreactor.connect.cloud.formats.reader.Converter -import io.lenses.streamreactor.connect.cloud.model.Topic -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocation -import io.lenses.streamreactor.connect.cloud.source.SourceWatermark +import io.lenses.streamreactor.connect.cloud.common.formats.reader.Converter +import io.lenses.streamreactor.connect.cloud.common.model.Topic +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocation +import io.lenses.streamreactor.connect.cloud.common.source.SourceWatermark import org.apache.kafka.connect.data.Schema import org.apache.kafka.connect.data.SchemaAndValue import org.apache.kafka.connect.header.ConnectHeaders diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/reader/converters/SchemalessEnvelopeConverter.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/converters/SchemalessEnvelopeConverter.scala similarity index 94% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/reader/converters/SchemalessEnvelopeConverter.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/converters/SchemalessEnvelopeConverter.scala index d90111a2fb..2106bc76c8 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/reader/converters/SchemalessEnvelopeConverter.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/converters/SchemalessEnvelopeConverter.scala @@ -13,14 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.formats.reader.converters +package io.lenses.streamreactor.connect.cloud.common.formats.reader.converters import io.circe.Json import io.circe.parser._ -import io.lenses.streamreactor.connect.cloud.source.SourceWatermark -import io.lenses.streamreactor.connect.cloud.formats.reader.Converter -import io.lenses.streamreactor.connect.cloud.model.Topic -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocation +import io.lenses.streamreactor.connect.cloud.common.formats.reader.Converter +import io.lenses.streamreactor.connect.cloud.common.model.Topic +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocation +import io.lenses.streamreactor.connect.cloud.common.source.SourceWatermark import org.apache.kafka.connect.data.Schema import org.apache.kafka.connect.source.SourceRecord diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/reader/converters/TextConverter.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/converters/TextConverter.scala similarity index 77% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/reader/converters/TextConverter.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/converters/TextConverter.scala index 5c972e6c15..dc8cd14abb 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/reader/converters/TextConverter.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/converters/TextConverter.scala @@ -13,13 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.formats.reader.converters +package io.lenses.streamreactor.connect.cloud.common.formats.reader.converters -import io.lenses.streamreactor.connect.cloud.source.SourceWatermark -import io.lenses.streamreactor.connect.cloud.config.ReaderBuilderContext -import io.lenses.streamreactor.connect.cloud.formats.reader.Converter -import io.lenses.streamreactor.connect.cloud.model.Topic -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocation +import io.lenses.streamreactor.connect.cloud.common.config.ReaderBuilderContext +import io.lenses.streamreactor.connect.cloud.common.formats.reader.Converter +import io.lenses.streamreactor.connect.cloud.common.model.Topic +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocation +import io.lenses.streamreactor.connect.cloud.common.source.SourceWatermark import org.apache.kafka.connect.data.Schema import org.apache.kafka.connect.source.SourceRecord diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/reader/parquet/ParquetInputFile.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/parquet/ParquetInputFile.scala similarity index 95% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/reader/parquet/ParquetInputFile.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/parquet/ParquetInputFile.scala index 8019b647fd..1673023af3 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/reader/parquet/ParquetInputFile.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/parquet/ParquetInputFile.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.formats.reader.parquet +package io.lenses.streamreactor.connect.cloud.common.formats.reader.parquet import org.apache.parquet.io.DelegatingSeekableInputStream import org.apache.parquet.io.InputFile diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/reader/parquet/ParquetSeekableInputStream.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/parquet/ParquetSeekableInputStream.scala similarity index 97% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/reader/parquet/ParquetSeekableInputStream.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/parquet/ParquetSeekableInputStream.scala index 75a0ad803b..ee6fc44c3e 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/reader/parquet/ParquetSeekableInputStream.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/parquet/ParquetSeekableInputStream.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.formats.reader.parquet +package io.lenses.streamreactor.connect.cloud.common.formats.reader.parquet import com.typesafe.scalalogging.LazyLogging import org.apache.parquet.io.DelegatingSeekableInputStream diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/reader/parquet/ParquetStreamingInputFile.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/parquet/ParquetStreamingInputFile.scala similarity index 93% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/reader/parquet/ParquetStreamingInputFile.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/parquet/ParquetStreamingInputFile.scala index ac5cd5696b..a53c5ee23d 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/reader/parquet/ParquetStreamingInputFile.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/parquet/ParquetStreamingInputFile.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.formats.reader.parquet +package io.lenses.streamreactor.connect.cloud.common.formats.reader.parquet import com.typesafe.scalalogging.LazyLogging import org.apache.parquet.io.InputFile diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/writer/AvroFormatWriter.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/writer/AvroFormatWriter.scala similarity index 79% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/writer/AvroFormatWriter.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/writer/AvroFormatWriter.scala index e312d3be8d..397d1f005a 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/writer/AvroFormatWriter.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/writer/AvroFormatWriter.scala @@ -13,15 +13,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.formats.writer +package io.lenses.streamreactor.connect.cloud.common.formats.writer import cats.implicits._ import com.typesafe.scalalogging.LazyLogging -import io.lenses.streamreactor.connect.cloud.model.CompressionCodec -import io.lenses.streamreactor.connect.cloud.model.CompressionCodecName._ -import io.lenses.streamreactor.connect.cloud.sink.SinkError -import io.lenses.streamreactor.connect.cloud.sink.conversion.ToAvroDataConverter -import io.lenses.streamreactor.connect.cloud.stream.S3OutputStream +import io.lenses.streamreactor.connect.cloud.common.model.CompressionCodec +import io.lenses.streamreactor.connect.cloud.common.model.CompressionCodecName.BZIP2 +import io.lenses.streamreactor.connect.cloud.common.model.CompressionCodecName.DEFLATE +import io.lenses.streamreactor.connect.cloud.common.model.CompressionCodecName.SNAPPY +import io.lenses.streamreactor.connect.cloud.common.model.CompressionCodecName.UNCOMPRESSED +import io.lenses.streamreactor.connect.cloud.common.model.CompressionCodecName.XZ +import io.lenses.streamreactor.connect.cloud.common.model.CompressionCodecName.ZSTD +import io.lenses.streamreactor.connect.cloud.common.sink.conversion.ToAvroDataConverter +import io.lenses.streamreactor.connect.cloud.common.stream.S3OutputStream +import io.lenses.streamreactor.connect.cloud.common.sink.SinkError import org.apache.avro.Schema import org.apache.avro.file.CodecFactory import org.apache.avro.file.DataFileWriter diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/writer/BytesFormatWriter.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/writer/BytesFormatWriter.scala similarity index 91% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/writer/BytesFormatWriter.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/writer/BytesFormatWriter.scala index 893252f532..241449cf80 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/writer/BytesFormatWriter.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/writer/BytesFormatWriter.scala @@ -13,12 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.formats.writer +package io.lenses.streamreactor.connect.cloud.common.formats.writer import cats.implicits.catsSyntaxEitherId import com.typesafe.scalalogging.LazyLogging -import io.lenses.streamreactor.connect.cloud.sink.SinkError -import io.lenses.streamreactor.connect.cloud.stream.S3OutputStream +import io.lenses.streamreactor.connect.cloud.common.sink.SinkError +import io.lenses.streamreactor.connect.cloud.common.stream.S3OutputStream import scala.util.Try diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/writer/CsvFormatWriter.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/writer/CsvFormatWriter.scala similarity index 82% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/writer/CsvFormatWriter.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/writer/CsvFormatWriter.scala index 14d0e5a10b..097aa20ad8 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/writer/CsvFormatWriter.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/writer/CsvFormatWriter.scala @@ -13,15 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.formats.writer +package io.lenses.streamreactor.connect.cloud.common.formats.writer import com.opencsv.CSVWriter import com.typesafe.scalalogging.LazyLogging -import io.lenses.streamreactor.connect.cloud.sink.config.PartitionNamePath -import io.lenses.streamreactor.connect.cloud.sink.extractors.ExtractorErrorAdaptor.adaptErrorResponse -import io.lenses.streamreactor.connect.cloud.sink.extractors.SinkDataExtractor -import io.lenses.streamreactor.connect.cloud.sink.SinkError -import io.lenses.streamreactor.connect.cloud.stream.S3OutputStream +import io.lenses.streamreactor.connect.cloud.common.sink.SinkError +import io.lenses.streamreactor.connect.cloud.common.sink.config.PartitionNamePath +import io.lenses.streamreactor.connect.cloud.common.sink.extractors.ExtractorErrorAdaptor.adaptErrorResponse +import io.lenses.streamreactor.connect.cloud.common.sink.extractors.SinkDataExtractor +import io.lenses.streamreactor.connect.cloud.common.stream.S3OutputStream import org.apache.kafka.connect.data.Schema import java.io.OutputStreamWriter diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/writer/JsonFormatWriter.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/writer/JsonFormatWriter.scala similarity index 92% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/writer/JsonFormatWriter.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/writer/JsonFormatWriter.scala index 8901667c61..1ef5b3bf93 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/writer/JsonFormatWriter.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/writer/JsonFormatWriter.scala @@ -13,12 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.formats.writer +package io.lenses.streamreactor.connect.cloud.common.formats.writer import JsonFormatWriter._ import LineSeparatorUtil.LineSeparatorBytes -import io.lenses.streamreactor.connect.cloud.sink.SinkError -import io.lenses.streamreactor.connect.cloud.stream.S3OutputStream +import io.lenses.streamreactor.connect.cloud.common.sink.SinkError +import io.lenses.streamreactor.connect.cloud.common.stream.S3OutputStream import org.apache.kafka.connect.json.JsonConverter import scala.jdk.CollectionConverters.MapHasAsJava diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/writer/LineSeparatorUtil.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/writer/LineSeparatorUtil.scala similarity index 91% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/writer/LineSeparatorUtil.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/writer/LineSeparatorUtil.scala index 7059b21f34..b6c2f4f777 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/writer/LineSeparatorUtil.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/writer/LineSeparatorUtil.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.formats.writer +package io.lenses.streamreactor.connect.cloud.common.formats.writer import java.nio.charset.StandardCharsets diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/writer/MessageDetail.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/writer/MessageDetail.scala similarity index 80% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/writer/MessageDetail.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/writer/MessageDetail.scala index 0379a3bc96..447e84c7a7 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/writer/MessageDetail.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/writer/MessageDetail.scala @@ -13,10 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.formats.writer +package io.lenses.streamreactor.connect.cloud.common.formats.writer -import io.lenses.streamreactor.connect.cloud.model.Offset -import io.lenses.streamreactor.connect.cloud.model.Topic +import io.lenses.streamreactor.connect.cloud.common.model.Offset +import io.lenses.streamreactor.connect.cloud.common.model.Topic import java.time.Instant diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/writer/ParquetFormatWriter.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/writer/ParquetFormatWriter.scala similarity index 74% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/writer/ParquetFormatWriter.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/writer/ParquetFormatWriter.scala index 9064cb5a4a..e6ec663537 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/writer/ParquetFormatWriter.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/writer/ParquetFormatWriter.scala @@ -13,15 +13,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.formats.writer +package io.lenses.streamreactor.connect.cloud.common.formats.writer import com.typesafe.scalalogging.LazyLogging -import io.lenses.streamreactor.connect.cloud.model.CompressionCodecName._ -import io.lenses.streamreactor.connect.cloud.formats.writer.parquet.ParquetOutputFile -import io.lenses.streamreactor.connect.cloud.model.CompressionCodec -import io.lenses.streamreactor.connect.cloud.sink.SinkError -import io.lenses.streamreactor.connect.cloud.sink.conversion.ToAvroDataConverter -import io.lenses.streamreactor.connect.cloud.stream.S3OutputStream +import io.lenses.streamreactor.connect.cloud.common.formats.writer.parquet.ParquetOutputFile +import io.lenses.streamreactor.connect.cloud.common.model.CompressionCodec +import io.lenses.streamreactor.connect.cloud.common.model.CompressionCodecName.BROTLI +import io.lenses.streamreactor.connect.cloud.common.model.CompressionCodecName.GZIP +import io.lenses.streamreactor.connect.cloud.common.model.CompressionCodecName.LZ4 +import io.lenses.streamreactor.connect.cloud.common.model.CompressionCodecName.LZO +import io.lenses.streamreactor.connect.cloud.common.model.CompressionCodecName.SNAPPY +import io.lenses.streamreactor.connect.cloud.common.model.CompressionCodecName.UNCOMPRESSED +import io.lenses.streamreactor.connect.cloud.common.model.CompressionCodecName.ZSTD +import io.lenses.streamreactor.connect.cloud.common.sink.SinkError +import io.lenses.streamreactor.connect.cloud.common.sink.conversion.ToAvroDataConverter +import io.lenses.streamreactor.connect.cloud.common.stream.S3OutputStream import org.apache.avro.Schema import org.apache.kafka.connect.data.{ Schema => ConnectSchema } import org.apache.parquet.avro.AvroParquetWriter diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/writer/S3FormatWriter.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/writer/S3FormatWriter.scala similarity index 73% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/writer/S3FormatWriter.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/writer/S3FormatWriter.scala index 4fcce75d24..06559cb8b4 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/writer/S3FormatWriter.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/writer/S3FormatWriter.scala @@ -13,19 +13,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.formats.writer +package io.lenses.streamreactor.connect.cloud.common.formats.writer import cats.implicits._ -import io.lenses.streamreactor.connect.cloud.config.FormatOptions.WithHeaders -import io.lenses.streamreactor.connect.cloud.config._ -import io.lenses.streamreactor.connect.cloud.formats.FormatWriterException -import io.lenses.streamreactor.connect.cloud.model.location.FileUtils.toBufferedOutputStream -import io.lenses.streamreactor.connect.cloud.model.CompressionCodec -import io.lenses.streamreactor.connect.cloud.model.TopicPartition -import io.lenses.streamreactor.connect.cloud.sink.NonFatalS3SinkError -import io.lenses.streamreactor.connect.cloud.sink.SinkError -import io.lenses.streamreactor.connect.cloud.stream.BuildLocalOutputStream -import io.lenses.streamreactor.connect.cloud.stream.S3OutputStream +import io.lenses.streamreactor.connect.cloud.common.config.FormatOptions.WithHeaders +import io.lenses.streamreactor.connect.cloud.common.config._ +import io.lenses.streamreactor.connect.cloud.common.formats.FormatWriterException +import io.lenses.streamreactor.connect.cloud.common.model.location.FileUtils.toBufferedOutputStream +import io.lenses.streamreactor.connect.cloud.common.model.CompressionCodec +import io.lenses.streamreactor.connect.cloud.common.model.TopicPartition +import io.lenses.streamreactor.connect.cloud.common.sink.NonFatalS3SinkError +import io.lenses.streamreactor.connect.cloud.common.sink.SinkError +import io.lenses.streamreactor.connect.cloud.common.stream.BuildLocalOutputStream +import io.lenses.streamreactor.connect.cloud.common.stream.S3OutputStream import java.io.File import scala.util.Try diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/writer/SinkData.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/writer/SinkData.scala similarity index 97% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/writer/SinkData.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/writer/SinkData.scala index ffa31df030..82034ba959 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/writer/SinkData.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/writer/SinkData.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.formats.writer +package io.lenses.streamreactor.connect.cloud.common.formats.writer import org.apache.kafka.connect.data.Schema import org.apache.kafka.connect.data.Struct diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/writer/Suppress.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/writer/Suppress.scala similarity index 86% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/writer/Suppress.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/writer/Suppress.scala index 18ffaf5c82..3103e466e2 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/writer/Suppress.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/writer/Suppress.scala @@ -13,10 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.formats.writer +package io.lenses.streamreactor.connect.cloud.common.formats.writer import cats.implicits._ -import io.lenses.streamreactor.connect.cloud.sink.FatalS3SinkError +import io.lenses.streamreactor.connect.cloud.common.sink.FatalS3SinkError import scala.util.Failure import scala.util.Success diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/writer/TextFormatWriter.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/writer/TextFormatWriter.scala similarity index 86% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/writer/TextFormatWriter.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/writer/TextFormatWriter.scala index c5519c16d0..8d766cfa94 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/writer/TextFormatWriter.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/writer/TextFormatWriter.scala @@ -13,12 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.formats.writer +package io.lenses.streamreactor.connect.cloud.common.formats.writer import LineSeparatorUtil.LineSeparatorBytes -import io.lenses.streamreactor.connect.cloud.formats.FormatWriterException -import io.lenses.streamreactor.connect.cloud.sink.SinkError -import io.lenses.streamreactor.connect.cloud.stream.S3OutputStream +import io.lenses.streamreactor.connect.cloud.common.formats.FormatWriterException +import io.lenses.streamreactor.connect.cloud.common.sink.SinkError +import io.lenses.streamreactor.connect.cloud.common.stream.S3OutputStream import scala.util.Failure import scala.util.Success diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/writer/parquet/ParquetOutputFile.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/writer/parquet/ParquetOutputFile.scala similarity index 90% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/writer/parquet/ParquetOutputFile.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/writer/parquet/ParquetOutputFile.scala index f6d0e0e61e..90e5afdb88 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/formats/writer/parquet/ParquetOutputFile.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/formats/writer/parquet/ParquetOutputFile.scala @@ -13,9 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.formats.writer.parquet +package io.lenses.streamreactor.connect.cloud.common.formats.writer.parquet -import io.lenses.streamreactor.connect.cloud.stream.S3OutputStream +import io.lenses.streamreactor.connect.cloud.common.stream.S3OutputStream import org.apache.parquet.io.OutputFile import org.apache.parquet.io.PositionOutputStream diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/model/CompressionCodecName.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/model/CompressionCodecName.scala similarity index 96% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/model/CompressionCodecName.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/model/CompressionCodecName.scala index ca8a99182a..15e6f90214 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/model/CompressionCodecName.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/model/CompressionCodecName.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.model +package io.lenses.streamreactor.connect.cloud.common.model import cats.implicits.catsSyntaxOptionId import enumeratum.Enum diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/model/TopicPartitionOffset.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/model/TopicPartitionOffset.scala similarity index 96% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/model/TopicPartitionOffset.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/model/TopicPartitionOffset.scala index eecaf8dd8e..3269bd8b88 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/model/TopicPartitionOffset.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/model/TopicPartitionOffset.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.model +package io.lenses.streamreactor.connect.cloud.common.model import org.apache.kafka.common.{ TopicPartition => KafkaTopicPartition } diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/model/location/CloudLocation.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/model/location/CloudLocation.scala similarity index 97% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/model/location/CloudLocation.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/model/location/CloudLocation.scala index 431c850ba7..2819659316 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/model/location/CloudLocation.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/model/location/CloudLocation.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.model.location +package io.lenses.streamreactor.connect.cloud.common.model.location import cats.Show import cats.data.Validated diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/model/location/CloudLocationValidator.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/model/location/CloudLocationValidator.scala similarity index 91% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/model/location/CloudLocationValidator.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/model/location/CloudLocationValidator.scala index b5ab74e180..cf8186e636 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/model/location/CloudLocationValidator.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/model/location/CloudLocationValidator.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.model.location +package io.lenses.streamreactor.connect.cloud.common.model.location import cats.data.Validated diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/model/location/FileUtils.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/model/location/FileUtils.scala similarity index 94% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/model/location/FileUtils.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/model/location/FileUtils.scala index 1a5897e5ce..8b6c63e7ed 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/model/location/FileUtils.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/model/location/FileUtils.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.model.location +package io.lenses.streamreactor.connect.cloud.common.model.location import com.typesafe.scalalogging.LazyLogging diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/model/location/Location.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/model/location/Location.scala similarity index 91% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/model/location/Location.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/model/location/Location.scala index 6d391cc99f..de910f4ad1 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/model/location/Location.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/model/location/Location.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.model.location +package io.lenses.streamreactor.connect.cloud.common.model.location trait Location {} diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/Exceptions.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/Exceptions.scala similarity index 95% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/Exceptions.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/Exceptions.scala index e0f5c1730a..73fcc5e633 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/Exceptions.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/Exceptions.scala @@ -13,9 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.sink +package io.lenses.streamreactor.connect.cloud.common.sink -import io.lenses.streamreactor.connect.cloud.model.TopicPartition +import io.lenses.streamreactor.connect.cloud.common.model.TopicPartition trait SinkError { def exception(): Throwable diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/commit/CommitContext.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/commit/CommitContext.scala similarity index 89% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/commit/CommitContext.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/commit/CommitContext.scala index 51e1d13c77..c36c0f88e6 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/commit/CommitContext.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/commit/CommitContext.scala @@ -13,9 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.sink.commit +package io.lenses.streamreactor.connect.cloud.common.sink.commit -import io.lenses.streamreactor.connect.cloud.model.TopicPartitionOffset +import io.lenses.streamreactor.connect.cloud.common.model.TopicPartitionOffset /** * @param tpo the [[TopicPartitionOffset]] of the last record written diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/commit/CommitPolicy.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/commit/CommitPolicy.scala similarity index 87% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/commit/CommitPolicy.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/commit/CommitPolicy.scala index cf4e2a067a..85016386ca 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/commit/CommitPolicy.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/commit/CommitPolicy.scala @@ -13,13 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.sink.commit +package io.lenses.streamreactor.connect.cloud.common.sink.commit import com.typesafe.scalalogging.LazyLogging import com.typesafe.scalalogging.Logger -import io.lenses.streamreactor.connect.cloud.sink.config.S3FlushSettings.defaultFlushCount -import io.lenses.streamreactor.connect.cloud.sink.config.S3FlushSettings.defaultFlushInterval -import io.lenses.streamreactor.connect.cloud.sink.config.S3FlushSettings.defaultFlushSize +import io.lenses.streamreactor.connect.cloud.common.sink.config.S3FlushSettings.defaultFlushCount +import io.lenses.streamreactor.connect.cloud.common.sink.config.S3FlushSettings.defaultFlushInterval +import io.lenses.streamreactor.connect.cloud.common.sink.config.S3FlushSettings.defaultFlushSize import scala.util.Try diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/commit/CommitPolicyConditions.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/commit/CommitPolicyConditions.scala similarity index 97% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/commit/CommitPolicyConditions.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/commit/CommitPolicyConditions.scala index 4e129e0f72..bf3ab574fa 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/commit/CommitPolicyConditions.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/commit/CommitPolicyConditions.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.sink.commit +package io.lenses.streamreactor.connect.cloud.common.sink.commit import com.typesafe.scalalogging.LazyLogging diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/config/LocalStagingArea.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/config/LocalStagingArea.scala similarity index 79% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/config/LocalStagingArea.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/config/LocalStagingArea.scala index 29787ad0bc..3c1233b9ac 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/config/LocalStagingArea.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/config/LocalStagingArea.scala @@ -13,12 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.sink.config +package io.lenses.streamreactor.connect.cloud.common.sink.config import cats.syntax.all._ import com.datamountaineer.streamreactor.common.config.base.traits.BaseSettings import com.datamountaineer.streamreactor.common.config.base.traits.WithConnectorPrefix -import io.lenses.streamreactor.connect.cloud.config.ConnectorTaskId +import io.lenses.streamreactor.connect.cloud.common.config.ConnectorTaskId +import org.apache.kafka.common.config.ConfigDef +import org.apache.kafka.common.config.ConfigDef.Importance +import org.apache.kafka.common.config.ConfigDef.Type import java.io.File import java.nio.file.Files @@ -28,6 +31,14 @@ import scala.util.Try trait LocalStagingAreaConfigKeys extends WithConnectorPrefix { val LOCAL_TMP_DIRECTORY: String = s"$connectorPrefix.local.tmp.directory" + def addLocalStagingAreaToConfigDef(configDef: ConfigDef): ConfigDef = + configDef.define( + LOCAL_TMP_DIRECTORY, + Type.STRING, + "", + Importance.LOW, + s"Local tmp directory for preparing the files", + ) } trait LocalStagingAreaSettings extends BaseSettings with LocalStagingAreaConfigKeys { diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/config/PartitionDisplay.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/config/PartitionDisplay.scala similarity index 83% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/config/PartitionDisplay.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/config/PartitionDisplay.scala index d0a4b64281..d83a4cac3c 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/config/PartitionDisplay.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/config/PartitionDisplay.scala @@ -13,14 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.sink.config +package io.lenses.streamreactor.connect.cloud.common.sink.config import com.datamountaineer.kcql.Kcql import enumeratum.Enum import enumeratum.EnumEntry -import io.lenses.streamreactor.connect.cloud.config.kcqlprops.S3PropsKeyEnum.PartitionIncludeKeys -import io.lenses.streamreactor.connect.cloud.config.kcqlprops.S3PropsKeyEntry -import io.lenses.streamreactor.connect.cloud.config.kcqlprops.S3PropsKeyEnum +import io.lenses.streamreactor.connect.cloud.common.config.kcqlprops.S3PropsKeyEnum.PartitionIncludeKeys +import io.lenses.streamreactor.connect.cloud.common.config.kcqlprops.S3PropsKeyEntry +import io.lenses.streamreactor.connect.cloud.common.config.kcqlprops.S3PropsKeyEnum import io.lenses.streamreactor.connect.config.kcqlprops.KcqlProperties import scala.collection.immutable diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/config/PartitionField.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/config/PartitionField.scala similarity index 98% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/config/PartitionField.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/config/PartitionField.scala index b42dd9d7f2..58fa469a24 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/config/PartitionField.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/config/PartitionField.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.sink.config +package io.lenses.streamreactor.connect.cloud.common.sink.config import com.datamountaineer.kcql.Kcql diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/config/PartitionFieldSplitter.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/config/PartitionFieldSplitter.scala similarity index 96% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/config/PartitionFieldSplitter.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/config/PartitionFieldSplitter.scala index 916a4315c3..24a64a27e5 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/config/PartitionFieldSplitter.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/config/PartitionFieldSplitter.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.sink.config +package io.lenses.streamreactor.connect.cloud.common.sink.config /** * A utility for splitting a string into parts while respecting backticks. diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/config/PartitionNamePath.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/config/PartitionNamePath.scala similarity index 95% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/config/PartitionNamePath.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/config/PartitionNamePath.scala index bef5844003..6886ef8fb8 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/config/PartitionNamePath.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/config/PartitionNamePath.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.sink.config +package io.lenses.streamreactor.connect.cloud.common.sink.config case class PartitionNamePath(path: String*) { override def toString: String = path.mkString(".") diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/config/PartitionSelection.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/config/PartitionSelection.scala similarity index 80% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/config/PartitionSelection.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/config/PartitionSelection.scala index 12f31a5d9e..4fcb67e15e 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/config/PartitionSelection.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/config/PartitionSelection.scala @@ -13,13 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.sink.config +package io.lenses.streamreactor.connect.cloud.common.sink.config import com.datamountaineer.kcql.Kcql -import io.lenses.streamreactor.connect.cloud.config.kcqlprops.S3PropsKeyEntry -import io.lenses.streamreactor.connect.cloud.config.kcqlprops.S3PropsKeyEnum -import io.lenses.streamreactor.connect.cloud.sink.config.PartitionDisplay.KeysAndValues -import io.lenses.streamreactor.connect.cloud.sink.config.PartitionDisplay.Values +import io.lenses.streamreactor.connect.cloud.common.config.kcqlprops.S3PropsKeyEntry +import io.lenses.streamreactor.connect.cloud.common.config.kcqlprops.S3PropsKeyEnum +import PartitionDisplay.KeysAndValues +import PartitionDisplay.Values import io.lenses.streamreactor.connect.config.kcqlprops.KcqlProperties case class PartitionSelection( diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/config/PartitionSpecifier.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/config/PartitionSpecifier.scala similarity index 95% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/config/PartitionSpecifier.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/config/PartitionSpecifier.scala index 8911fb7732..fe71296d39 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/config/PartitionSpecifier.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/config/PartitionSpecifier.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.sink.config +package io.lenses.streamreactor.connect.cloud.common.sink.config import enumeratum.Enum import enumeratum.EnumEntry diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/config/S3FlushSettings.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/config/S3FlushSettings.scala similarity index 83% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/config/S3FlushSettings.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/config/S3FlushSettings.scala index 78b82f2fd6..e842a9b891 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/config/S3FlushSettings.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/config/S3FlushSettings.scala @@ -13,16 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.sink.config +package io.lenses.streamreactor.connect.cloud.common.sink.config import com.datamountaineer.kcql.Kcql import com.datamountaineer.streamreactor.common.config.base.traits.BaseSettings import com.datamountaineer.streamreactor.common.config.base.traits.WithConnectorPrefix -import io.lenses.streamreactor.connect.cloud.sink.commit.CommitPolicy -import io.lenses.streamreactor.connect.cloud.sink.commit.CommitPolicyCondition -import io.lenses.streamreactor.connect.cloud.sink.commit.Count -import io.lenses.streamreactor.connect.cloud.sink.commit.FileSize -import io.lenses.streamreactor.connect.cloud.sink.commit.Interval +import io.lenses.streamreactor.connect.cloud.common.sink.commit.CommitPolicy +import io.lenses.streamreactor.connect.cloud.common.sink.commit.CommitPolicyCondition +import io.lenses.streamreactor.connect.cloud.common.sink.commit.Count +import io.lenses.streamreactor.connect.cloud.common.sink.commit.FileSize +import io.lenses.streamreactor.connect.cloud.common.sink.commit.Interval import scala.concurrent.duration.DurationInt import scala.concurrent.duration.DurationLong diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/config/kcqlprops/S3SinkPropsSchema.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/config/kcqlprops/S3SinkPropsSchema.scala similarity index 80% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/config/kcqlprops/S3SinkPropsSchema.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/config/kcqlprops/S3SinkPropsSchema.scala index cb93a99bf7..35c29126b8 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/config/kcqlprops/S3SinkPropsSchema.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/config/kcqlprops/S3SinkPropsSchema.scala @@ -13,13 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.sink.config.kcqlprops +package io.lenses.streamreactor.connect.cloud.common.sink.config.kcqlprops import com.datamountaineer.kcql.Kcql -import io.lenses.streamreactor.connect.cloud.config.kcqlprops.S3PropsKeyEntry -import io.lenses.streamreactor.connect.cloud.config.kcqlprops.S3PropsKeyEnum -import io.lenses.streamreactor.connect.cloud.config.kcqlprops.S3PropsKeyEnum._ -import io.lenses.streamreactor.connect.cloud.sink.config.padding.PaddingType +import io.lenses.streamreactor.connect.cloud.common.config.kcqlprops.S3PropsKeyEntry +import io.lenses.streamreactor.connect.cloud.common.config.kcqlprops.S3PropsKeyEnum +import io.lenses.streamreactor.connect.cloud.common.sink.config.padding.PaddingType +import S3PropsKeyEnum._ import io.lenses.streamreactor.connect.config.kcqlprops._ import scala.jdk.CollectionConverters.MapHasAsScala diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/config/padding/PaddingService.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/config/padding/PaddingService.scala similarity index 86% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/config/padding/PaddingService.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/config/padding/PaddingService.scala index 6e6105b3e2..e4ae2518ab 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/config/padding/PaddingService.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/config/padding/PaddingService.scala @@ -13,15 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.sink.config.padding +package io.lenses.streamreactor.connect.cloud.common.sink.config.padding import cats.implicits.catsSyntaxEitherId -import io.lenses.streamreactor.connect.cloud.config.kcqlprops.S3PropsKeyEntry -import io.lenses.streamreactor.connect.cloud.config.kcqlprops.S3PropsKeyEnum -import io.lenses.streamreactor.connect.cloud.config.kcqlprops.S3PropsKeyEnum.PaddingCharacter -import io.lenses.streamreactor.connect.cloud.config.kcqlprops.S3PropsKeyEnum.PaddingLength -import io.lenses.streamreactor.connect.cloud.config.kcqlprops.S3PropsKeyEnum.PaddingSelection -import io.lenses.streamreactor.connect.cloud.sink.config.padding.PaddingType.LeftPad +import io.lenses.streamreactor.connect.cloud.common.config.kcqlprops.S3PropsKeyEnum.PaddingCharacter +import io.lenses.streamreactor.connect.cloud.common.config.kcqlprops.S3PropsKeyEnum.PaddingLength +import io.lenses.streamreactor.connect.cloud.common.config.kcqlprops.S3PropsKeyEnum.PaddingSelection +import io.lenses.streamreactor.connect.cloud.common.config.kcqlprops.S3PropsKeyEntry +import io.lenses.streamreactor.connect.cloud.common.config.kcqlprops.S3PropsKeyEnum +import io.lenses.streamreactor.connect.cloud.common.sink.config.padding.PaddingType.LeftPad import io.lenses.streamreactor.connect.config.kcqlprops.KcqlProperties import io.lenses.streamreactor.connect.config.kcqlprops.KcqlProperties.stringToInt import io.lenses.streamreactor.connect.config.kcqlprops.KcqlProperties.stringToString diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/config/padding/PaddingStrategy.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/config/padding/PaddingStrategy.scala similarity index 94% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/config/padding/PaddingStrategy.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/config/padding/PaddingStrategy.scala index 12f800a187..ee7b1be145 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/config/padding/PaddingStrategy.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/config/padding/PaddingStrategy.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.sink.config.padding +package io.lenses.streamreactor.connect.cloud.common.sink.config.padding trait PaddingStrategy { def padString(padMe: String): String diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/config/padding/PaddingStrategySettings.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/config/padding/PaddingStrategySettings.scala similarity index 73% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/config/padding/PaddingStrategySettings.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/config/padding/PaddingStrategySettings.scala index b7c2d6cc1d..1bda3b20e4 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/config/padding/PaddingStrategySettings.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/config/padding/PaddingStrategySettings.scala @@ -13,12 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.sink.config.padding +package io.lenses.streamreactor.connect.cloud.common.sink.config.padding import com.datamountaineer.streamreactor.common.config.base.traits.BaseSettings import com.datamountaineer.streamreactor.common.config.base.traits.WithConnectorPrefix -import io.lenses.streamreactor.connect.cloud.sink.config.padding.PaddingService.DefaultPadChar -import io.lenses.streamreactor.connect.cloud.sink.config.padding.PaddingService.DefaultPadLength +import PaddingService.DefaultPadChar +import PaddingService.DefaultPadLength +import org.apache.kafka.common.config.ConfigDef +import org.apache.kafka.common.config.ConfigDef.Importance +import org.apache.kafka.common.config.ConfigDef.Type /** * Retrieves Padding settings from KCQL @@ -34,6 +37,22 @@ trait PaddingStrategyConfigKeys extends WithConnectorPrefix { val PADDING_LENGTH_DOC = s"Length to pad the string up to if $PADDING_STRATEGY is set." val PADDING_LENGTH_DEFAULT: Int = -1 + def addPaddingToConfigDef(configDef: ConfigDef) = + configDef + .define( + PADDING_STRATEGY, + Type.STRING, + PADDING_STRATEGY_DEFAULT, + Importance.LOW, + PADDING_STRATEGY_DOC, + ) + .define( + PADDING_LENGTH, + Type.INT, + PADDING_LENGTH_DEFAULT, + Importance.LOW, + PADDING_LENGTH_DOC, + ) } trait PaddingStrategySettings extends BaseSettings with PaddingStrategyConfigKeys { diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/config/padding/PaddingType.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/config/padding/PaddingType.scala similarity index 94% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/config/padding/PaddingType.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/config/padding/PaddingType.scala index 2d455e34a0..ee2aca1ef3 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/config/padding/PaddingType.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/config/padding/PaddingType.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.sink.config.padding +package io.lenses.streamreactor.connect.cloud.common.sink.config.padding import enumeratum._ diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/conversion/HeaderToStringConverter.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/conversion/HeaderToStringConverter.scala similarity index 86% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/conversion/HeaderToStringConverter.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/conversion/HeaderToStringConverter.scala index 892c837620..a07eb31aa7 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/conversion/HeaderToStringConverter.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/conversion/HeaderToStringConverter.scala @@ -13,9 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.sink.conversion +package io.lenses.streamreactor.connect.cloud.common.sink.conversion -import io.lenses.streamreactor.connect.cloud.formats.writer.SinkData +import io.lenses.streamreactor.connect.cloud.common.formats.writer.SinkData import org.apache.kafka.connect.sink.SinkRecord import scala.jdk.CollectionConverters.IterableHasAsScala diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/conversion/ToAvroDataConverter.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/conversion/ToAvroDataConverter.scala similarity index 80% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/conversion/ToAvroDataConverter.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/conversion/ToAvroDataConverter.scala index ff9547252c..ab5bde3ec7 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/conversion/ToAvroDataConverter.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/conversion/ToAvroDataConverter.scala @@ -13,16 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.sink.conversion +package io.lenses.streamreactor.connect.cloud.common.sink.conversion import io.confluent.connect.avro.AvroData -import io.lenses.streamreactor.connect.cloud.formats.writer.ArraySinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.ByteArraySinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.MapSinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.NullSinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.PrimitiveSinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.SinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.StructSinkData +import io.lenses.streamreactor.connect.cloud.common.formats.writer.ArraySinkData +import io.lenses.streamreactor.connect.cloud.common.formats.writer.ByteArraySinkData +import io.lenses.streamreactor.connect.cloud.common.formats.writer.MapSinkData +import io.lenses.streamreactor.connect.cloud.common.formats.writer.NullSinkData +import io.lenses.streamreactor.connect.cloud.common.formats.writer.PrimitiveSinkData +import io.lenses.streamreactor.connect.cloud.common.formats.writer.SinkData +import io.lenses.streamreactor.connect.cloud.common.formats.writer.StructSinkData import org.apache.avro.Schema import org.apache.kafka.connect.data.Struct import org.apache.kafka.connect.data.{ Schema => ConnectSchema } diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/conversion/ToJsonDataConverter.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/conversion/ToJsonDataConverter.scala similarity index 73% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/conversion/ToJsonDataConverter.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/conversion/ToJsonDataConverter.scala index 0c07778367..94c452e54c 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/conversion/ToJsonDataConverter.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/conversion/ToJsonDataConverter.scala @@ -13,11 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.sink.conversion +package io.lenses.streamreactor.connect.cloud.common.sink.conversion -import io.lenses.streamreactor.connect.cloud.formats.writer.ByteArraySinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.PrimitiveSinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.SinkData +import io.lenses.streamreactor.connect.cloud.common.formats.writer.ByteArraySinkData +import io.lenses.streamreactor.connect.cloud.common.formats.writer.PrimitiveSinkData +import io.lenses.streamreactor.connect.cloud.common.formats.writer.SinkData import java.nio.ByteBuffer diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/conversion/ValueToSinkDataConverter.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/conversion/ValueToSinkDataConverter.scala similarity index 69% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/conversion/ValueToSinkDataConverter.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/conversion/ValueToSinkDataConverter.scala index 17d4c97f10..9ed8d34f19 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/conversion/ValueToSinkDataConverter.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/conversion/ValueToSinkDataConverter.scala @@ -13,22 +13,22 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.sink.conversion +package io.lenses.streamreactor.connect.cloud.common.sink.conversion -import io.lenses.streamreactor.connect.cloud.formats.writer.ArraySinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.BooleanSinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.ByteArraySinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.ByteSinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.DoubleSinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.FloatSinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.IntSinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.LongSinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.MapSinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.NullSinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.ShortSinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.SinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.StringSinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.StructSinkData +import io.lenses.streamreactor.connect.cloud.common.formats.writer.ArraySinkData +import io.lenses.streamreactor.connect.cloud.common.formats.writer.BooleanSinkData +import io.lenses.streamreactor.connect.cloud.common.formats.writer.ByteArraySinkData +import io.lenses.streamreactor.connect.cloud.common.formats.writer.ByteSinkData +import io.lenses.streamreactor.connect.cloud.common.formats.writer.DoubleSinkData +import io.lenses.streamreactor.connect.cloud.common.formats.writer.FloatSinkData +import io.lenses.streamreactor.connect.cloud.common.formats.writer.IntSinkData +import io.lenses.streamreactor.connect.cloud.common.formats.writer.LongSinkData +import io.lenses.streamreactor.connect.cloud.common.formats.writer.MapSinkData +import io.lenses.streamreactor.connect.cloud.common.formats.writer.NullSinkData +import io.lenses.streamreactor.connect.cloud.common.formats.writer.ShortSinkData +import io.lenses.streamreactor.connect.cloud.common.formats.writer.SinkData +import io.lenses.streamreactor.connect.cloud.common.formats.writer.StringSinkData +import io.lenses.streamreactor.connect.cloud.common.formats.writer.StructSinkData import org.apache.kafka.connect.data.Schema import org.apache.kafka.connect.data.Struct import org.apache.kafka.connect.errors.ConnectException diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/extractors/ArrayExtractor.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/extractors/ArrayExtractor.scala similarity index 93% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/extractors/ArrayExtractor.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/extractors/ArrayExtractor.scala index 7bbd14460b..461d6c5523 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/extractors/ArrayExtractor.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/extractors/ArrayExtractor.scala @@ -13,12 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.sink.extractors +package io.lenses.streamreactor.connect.cloud.common.sink.extractors import cats.implicits._ import com.typesafe.scalalogging.LazyLogging import ArrayIndexUtil.getArrayIndex -import io.lenses.streamreactor.connect.cloud.sink.config.PartitionNamePath +import io.lenses.streamreactor.connect.cloud.common.sink.config.PartitionNamePath import org.apache.kafka.connect.data.Schema import java.util diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/extractors/ArrayIndexUtil.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/extractors/ArrayIndexUtil.scala similarity index 94% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/extractors/ArrayIndexUtil.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/extractors/ArrayIndexUtil.scala index ccd13d8761..fae4c6d0a9 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/extractors/ArrayIndexUtil.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/extractors/ArrayIndexUtil.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.sink.extractors +package io.lenses.streamreactor.connect.cloud.common.sink.extractors import cats.implicits._ import com.typesafe.scalalogging.LazyLogging diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/extractors/ComplexTypeExtractor.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/extractors/ComplexTypeExtractor.scala similarity index 90% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/extractors/ComplexTypeExtractor.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/extractors/ComplexTypeExtractor.scala index fb8907b46b..2193663300 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/extractors/ComplexTypeExtractor.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/extractors/ComplexTypeExtractor.scala @@ -13,11 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.sink.extractors +package io.lenses.streamreactor.connect.cloud.common.sink.extractors import cats.implicits._ import com.typesafe.scalalogging.LazyLogging -import io.lenses.streamreactor.connect.cloud.sink.config.PartitionNamePath +import io.lenses.streamreactor.connect.cloud.common.sink.config.PartitionNamePath import org.apache.kafka.connect.data.Schema import org.apache.kafka.connect.data.Struct diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/extractors/ExtractorErrorType.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/extractors/ExtractorErrorType.scala similarity index 96% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/extractors/ExtractorErrorType.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/extractors/ExtractorErrorType.scala index 764c7900c9..1358770fcd 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/extractors/ExtractorErrorType.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/extractors/ExtractorErrorType.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.sink.extractors +package io.lenses.streamreactor.connect.cloud.common.sink.extractors import enumeratum._ import ExtractorErrorType.FieldNameNotSpecified diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/extractors/MapExtractor.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/extractors/MapExtractor.scala similarity index 92% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/extractors/MapExtractor.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/extractors/MapExtractor.scala index f7a7899699..98e1b60c6d 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/extractors/MapExtractor.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/extractors/MapExtractor.scala @@ -13,11 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.sink.extractors +package io.lenses.streamreactor.connect.cloud.common.sink.extractors import cats.implicits._ import com.typesafe.scalalogging.LazyLogging -import io.lenses.streamreactor.connect.cloud.sink.config.PartitionNamePath +import io.lenses.streamreactor.connect.cloud.common.sink.config.PartitionNamePath import org.apache.kafka.connect.data.Schema import java.util diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/extractors/PrimitiveExtractor.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/extractors/PrimitiveExtractor.scala similarity index 96% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/extractors/PrimitiveExtractor.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/extractors/PrimitiveExtractor.scala index 5c5777ffa9..79198d982b 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/extractors/PrimitiveExtractor.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/extractors/PrimitiveExtractor.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.sink.extractors +package io.lenses.streamreactor.connect.cloud.common.sink.extractors import cats.implicits._ import com.typesafe.scalalogging.LazyLogging diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/extractors/SinkDataExtractor.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/extractors/SinkDataExtractor.scala similarity index 73% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/extractors/SinkDataExtractor.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/extractors/SinkDataExtractor.scala index d7d1c3427f..ede47b710b 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/extractors/SinkDataExtractor.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/extractors/SinkDataExtractor.scala @@ -13,17 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.sink.extractors +package io.lenses.streamreactor.connect.cloud.common.sink.extractors import cats.implicits._ import com.typesafe.scalalogging.LazyLogging -import io.lenses.streamreactor.connect.cloud.formats.writer.ArraySinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.ByteArraySinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.MapSinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.PrimitiveSinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.SinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.StructSinkData -import io.lenses.streamreactor.connect.cloud.sink.config.PartitionNamePath +import io.lenses.streamreactor.connect.cloud.common.formats.writer.ArraySinkData +import io.lenses.streamreactor.connect.cloud.common.formats.writer.ByteArraySinkData +import io.lenses.streamreactor.connect.cloud.common.formats.writer.MapSinkData +import io.lenses.streamreactor.connect.cloud.common.formats.writer.PrimitiveSinkData +import io.lenses.streamreactor.connect.cloud.common.formats.writer.SinkData +import io.lenses.streamreactor.connect.cloud.common.formats.writer.StructSinkData +import io.lenses.streamreactor.connect.cloud.common.sink.config.PartitionNamePath /** * Extracts values from a SinkData wrapper type diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/extractors/StructExtractor.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/extractors/StructExtractor.scala similarity index 95% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/extractors/StructExtractor.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/extractors/StructExtractor.scala index 1f55976620..08c177c20d 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/extractors/StructExtractor.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/extractors/StructExtractor.scala @@ -13,13 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.sink.extractors +package io.lenses.streamreactor.connect.cloud.common.sink.extractors import cats.implicits._ import com.typesafe.scalalogging.LazyLogging import ExtractorErrorType.UnexpectedType import PrimitiveExtractor.anyToEither -import io.lenses.streamreactor.connect.cloud.sink.config.PartitionNamePath +import io.lenses.streamreactor.connect.cloud.common.sink.config.PartitionNamePath import org.apache.kafka.connect.data.Schema.Type._ import org.apache.kafka.connect.data.Struct diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/extractors/WrappedArrayExtractor.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/extractors/WrappedArrayExtractor.scala similarity index 89% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/extractors/WrappedArrayExtractor.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/extractors/WrappedArrayExtractor.scala index a12bc08dc8..72fb667d39 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/extractors/WrappedArrayExtractor.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/extractors/WrappedArrayExtractor.scala @@ -13,11 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.sink.extractors +package io.lenses.streamreactor.connect.cloud.common.sink.extractors import cats.implicits._ import ArrayIndexUtil.getArrayIndex -import io.lenses.streamreactor.connect.cloud.sink.config.PartitionNamePath +import io.lenses.streamreactor.connect.cloud.common.sink.config.PartitionNamePath object WrappedArrayExtractor { diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/extractors/WrappedComplexTypeExtractor.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/extractors/WrappedComplexTypeExtractor.scala similarity index 90% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/extractors/WrappedComplexTypeExtractor.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/extractors/WrappedComplexTypeExtractor.scala index 59aa477e9e..97a9e414eb 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/extractors/WrappedComplexTypeExtractor.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/extractors/WrappedComplexTypeExtractor.scala @@ -13,12 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.sink.extractors +package io.lenses.streamreactor.connect.cloud.common.sink.extractors import cats.implicits._ import com.typesafe.scalalogging.LazyLogging import WrappedMapExtractor.extractPathFromMap -import io.lenses.streamreactor.connect.cloud.sink.config.PartitionNamePath +import io.lenses.streamreactor.connect.cloud.common.sink.config.PartitionNamePath import org.apache.kafka.connect.data.Struct object WrappedComplexTypeExtractor extends LazyLogging { diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/extractors/WrappedMapExtractor.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/extractors/WrappedMapExtractor.scala similarity index 91% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/extractors/WrappedMapExtractor.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/extractors/WrappedMapExtractor.scala index 15abdd1d01..58440204d7 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/extractors/WrappedMapExtractor.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/extractors/WrappedMapExtractor.scala @@ -13,10 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.sink.extractors +package io.lenses.streamreactor.connect.cloud.common.sink.extractors import cats.implicits._ -import io.lenses.streamreactor.connect.cloud.sink.config.PartitionNamePath +import io.lenses.streamreactor.connect.cloud.common.sink.config.PartitionNamePath /** * Extracts values from a Map. diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/extractors/WrappedPrimitiveExtractor.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/extractors/WrappedPrimitiveExtractor.scala similarity index 96% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/extractors/WrappedPrimitiveExtractor.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/extractors/WrappedPrimitiveExtractor.scala index efd73285eb..2e5291b0c7 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/sink/extractors/WrappedPrimitiveExtractor.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/extractors/WrappedPrimitiveExtractor.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.sink.extractors +package io.lenses.streamreactor.connect.cloud.common.sink.extractors import cats.implicits._ import com.typesafe.scalalogging.LazyLogging diff --git a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/seek/IndexFilenames.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/naming/IndexFilenames.scala similarity index 87% rename from kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/seek/IndexFilenames.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/naming/IndexFilenames.scala index fd4f75936a..8606f8c1d6 100644 --- a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/seek/IndexFilenames.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/naming/IndexFilenames.scala @@ -13,10 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.sink.seek +package io.lenses.streamreactor.connect.cloud.common.sink.naming -import io.lenses.streamreactor.connect.cloud.config.ConnectorTaskId -import io.lenses.streamreactor.connect.cloud.model.Offset +import io.lenses.streamreactor.connect.cloud.common.config.ConnectorTaskId +import io.lenses.streamreactor.connect.cloud.common.model.Offset import scala.util.Try diff --git a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/naming/KeyNamer.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/naming/KeyNamer.scala similarity index 67% rename from kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/naming/KeyNamer.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/naming/KeyNamer.scala index 2ec4345f66..1967bd6c06 100644 --- a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/naming/KeyNamer.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/naming/KeyNamer.scala @@ -13,14 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.sink.naming -import io.lenses.streamreactor.connect.cloud.formats.writer.MessageDetail -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocation -import io.lenses.streamreactor.connect.cloud.model.TopicPartition -import io.lenses.streamreactor.connect.cloud.model.TopicPartitionOffset -import io.lenses.streamreactor.connect.cloud.sink.config.PartitionField -import io.lenses.streamreactor.connect.cloud.sink.FatalS3SinkError -import io.lenses.streamreactor.connect.cloud.sink.SinkError +package io.lenses.streamreactor.connect.cloud.common.sink.naming + +import io.lenses.streamreactor.connect.cloud.common.formats.writer.MessageDetail +import io.lenses.streamreactor.connect.cloud.common.model.TopicPartition +import io.lenses.streamreactor.connect.cloud.common.model.TopicPartitionOffset +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocation +import io.lenses.streamreactor.connect.cloud.common.sink.FatalS3SinkError +import io.lenses.streamreactor.connect.cloud.common.sink.SinkError +import io.lenses.streamreactor.connect.cloud.common.sink.config.PartitionField import java.io.File diff --git a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/naming/S3FileNamer.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/naming/S3FileNamer.scala similarity index 86% rename from kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/naming/S3FileNamer.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/naming/S3FileNamer.scala index e48228da31..f089d6e913 100644 --- a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/naming/S3FileNamer.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/naming/S3FileNamer.scala @@ -13,10 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.sink.naming +package io.lenses.streamreactor.connect.cloud.common.sink.naming -import io.lenses.streamreactor.connect.cloud.model.TopicPartitionOffset -import io.lenses.streamreactor.connect.cloud.sink.config.padding.PaddingStrategy +import io.lenses.streamreactor.connect.cloud.common.model.TopicPartitionOffset +import io.lenses.streamreactor.connect.cloud.common.sink.config.padding.PaddingStrategy trait S3FileNamer { def fileName( diff --git a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/naming/S3KeyNamer.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/naming/S3KeyNamer.scala similarity index 77% rename from kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/naming/S3KeyNamer.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/naming/S3KeyNamer.scala index b96c328ad4..7ad2898e83 100644 --- a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/naming/S3KeyNamer.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/naming/S3KeyNamer.scala @@ -13,25 +13,34 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.sink.naming +package io.lenses.streamreactor.connect.cloud.common.sink.naming import cats.implicits.catsSyntaxEitherId import cats.implicits.toTraverseOps -import io.lenses.streamreactor.connect.cloud.config.FormatSelection -import io.lenses.streamreactor.connect.cloud.formats.writer.MessageDetail -import io.lenses.streamreactor.connect.cloud.formats.writer.NullSinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.SinkData -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocation -import io.lenses.streamreactor.connect.cloud.model.location.FileUtils.createFileAndParents -import io.lenses.streamreactor.connect.cloud.model.TopicPartition -import io.lenses.streamreactor.connect.cloud.model.TopicPartitionOffset -import io.lenses.streamreactor.connect.cloud.sink.config.PartitionDisplay.KeysAndValues -import io.lenses.streamreactor.connect.cloud.sink.config.padding.PaddingService -import io.lenses.streamreactor.connect.cloud.sink.config._ -import io.lenses.streamreactor.connect.cloud.sink.extractors.ExtractorErrorAdaptor.adaptErrorResponse -import io.lenses.streamreactor.connect.cloud.sink.extractors.SinkDataExtractor -import io.lenses.streamreactor.connect.cloud.sink.FatalS3SinkError -import io.lenses.streamreactor.connect.cloud.sink.SinkError +import io.lenses.streamreactor.connect.cloud.common.config.FormatSelection +import io.lenses.streamreactor.connect.cloud.common.formats.writer.MessageDetail +import io.lenses.streamreactor.connect.cloud.common.formats.writer.NullSinkData +import io.lenses.streamreactor.connect.cloud.common.formats.writer.SinkData +import io.lenses.streamreactor.connect.cloud.common.model.TopicPartition +import io.lenses.streamreactor.connect.cloud.common.model.TopicPartitionOffset +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocation +import io.lenses.streamreactor.connect.cloud.common.model.location.FileUtils.createFileAndParents +import io.lenses.streamreactor.connect.cloud.common.sink.config.padding.PaddingService +import io.lenses.streamreactor.connect.cloud.common.sink.FatalS3SinkError +import io.lenses.streamreactor.connect.cloud.common.sink.SinkError +import io.lenses.streamreactor.connect.cloud.common.sink.config.PartitionDisplay.KeysAndValues +import io.lenses.streamreactor.connect.cloud.common.sink.config.DatePartitionField +import io.lenses.streamreactor.connect.cloud.common.sink.config.HeaderPartitionField +import io.lenses.streamreactor.connect.cloud.common.sink.config.KeyPartitionField +import io.lenses.streamreactor.connect.cloud.common.sink.config.PartitionField +import io.lenses.streamreactor.connect.cloud.common.sink.config.PartitionNamePath +import io.lenses.streamreactor.connect.cloud.common.sink.config.PartitionPartitionField +import io.lenses.streamreactor.connect.cloud.common.sink.config.PartitionSelection +import io.lenses.streamreactor.connect.cloud.common.sink.config.TopicPartitionField +import io.lenses.streamreactor.connect.cloud.common.sink.config.ValuePartitionField +import io.lenses.streamreactor.connect.cloud.common.sink.config.WholeKeyPartitionField +import io.lenses.streamreactor.connect.cloud.common.sink.extractors.ExtractorErrorAdaptor.adaptErrorResponse +import io.lenses.streamreactor.connect.cloud.common.sink.extractors.SinkDataExtractor import java.io.File import java.util.UUID diff --git a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/seek/IndexManager.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/seek/IndexManager.scala similarity index 89% rename from kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/seek/IndexManager.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/seek/IndexManager.scala index ff45e68014..597d6d8aed 100644 --- a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/seek/IndexManager.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/seek/IndexManager.scala @@ -13,21 +13,22 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.sink.seek +package io.lenses.streamreactor.connect.cloud.common.sink.seek import cats.implicits._ import com.typesafe.scalalogging.LazyLogging -import io.lenses.streamreactor.connect.aws.s3.storage.ResultProcessors.processAsKey -import io.lenses.streamreactor.connect.aws.s3.storage.FileDeleteError -import io.lenses.streamreactor.connect.aws.s3.storage.FileLoadError -import io.lenses.streamreactor.connect.aws.s3.storage.ListResponse -import io.lenses.streamreactor.connect.aws.s3.storage.StorageInterface -import io.lenses.streamreactor.connect.cloud.config.ConnectorTaskId -import io.lenses.streamreactor.connect.cloud.model.TopicPartition -import io.lenses.streamreactor.connect.cloud.model.TopicPartitionOffset -import io.lenses.streamreactor.connect.cloud.sink.FatalS3SinkError -import io.lenses.streamreactor.connect.cloud.sink.NonFatalS3SinkError -import io.lenses.streamreactor.connect.cloud.sink.SinkError +import io.lenses.streamreactor.connect.cloud.common.storage.ResultProcessors.processAsKey +import io.lenses.streamreactor.connect.cloud.common.config.ConnectorTaskId +import io.lenses.streamreactor.connect.cloud.common.model.TopicPartition +import io.lenses.streamreactor.connect.cloud.common.model.TopicPartitionOffset +import io.lenses.streamreactor.connect.cloud.common.sink.FatalS3SinkError +import io.lenses.streamreactor.connect.cloud.common.sink.NonFatalS3SinkError +import io.lenses.streamreactor.connect.cloud.common.sink.SinkError +import io.lenses.streamreactor.connect.cloud.common.sink.naming.IndexFilenames +import io.lenses.streamreactor.connect.cloud.common.storage.FileDeleteError +import io.lenses.streamreactor.connect.cloud.common.storage.FileLoadError +import io.lenses.streamreactor.connect.cloud.common.storage.ListResponse +import io.lenses.streamreactor.connect.cloud.common.storage.StorageInterface class IndexManager( maxIndexes: Int, diff --git a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/transformers/AddConnectSchemaTransformer.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/transformers/AddConnectSchemaTransformer.scala similarity index 93% rename from kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/transformers/AddConnectSchemaTransformer.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/transformers/AddConnectSchemaTransformer.scala index 4356d6743e..d10678601e 100644 --- a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/transformers/AddConnectSchemaTransformer.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/transformers/AddConnectSchemaTransformer.scala @@ -13,16 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.sink.transformers +package io.lenses.streamreactor.connect.cloud.common.sink.transformers + import cats.implicits.catsSyntaxEitherId import cats.implicits.catsSyntaxOptionId import cats.implicits.toTraverseOps -import io.lenses.streamreactor.connect.cloud.config.DataStorageSettings -import io.lenses.streamreactor.connect.cloud.formats.writer.ArraySinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.MessageDetail -import io.lenses.streamreactor.connect.cloud.formats.writer.SinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.StructSinkData -import io.lenses.streamreactor.connect.cloud.model.Topic +import io.lenses.streamreactor.connect.cloud.common.config.DataStorageSettings +import io.lenses.streamreactor.connect.cloud.common.formats.writer.ArraySinkData +import io.lenses.streamreactor.connect.cloud.common.formats.writer.MessageDetail +import io.lenses.streamreactor.connect.cloud.common.formats.writer.SinkData +import io.lenses.streamreactor.connect.cloud.common.formats.writer.StructSinkData +import io.lenses.streamreactor.connect.cloud.common.model.Topic import org.apache.kafka.connect.data.Schema import org.apache.kafka.connect.data.SchemaAndValue import org.apache.kafka.connect.data.SchemaBuilder diff --git a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/transformers/EnvelopeWithSchemaTransformer.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/transformers/EnvelopeWithSchemaTransformer.scala similarity index 93% rename from kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/transformers/EnvelopeWithSchemaTransformer.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/transformers/EnvelopeWithSchemaTransformer.scala index a7c967bee2..319434277a 100644 --- a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/transformers/EnvelopeWithSchemaTransformer.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/transformers/EnvelopeWithSchemaTransformer.scala @@ -13,15 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.sink.transformers +package io.lenses.streamreactor.connect.cloud.common.sink.transformers import cats.implicits.catsSyntaxEitherId -import io.lenses.streamreactor.connect.aws.s3.sink.transformers.EnvelopeWithSchemaTransformer.envelope -import io.lenses.streamreactor.connect.cloud.config.DataStorageSettings -import io.lenses.streamreactor.connect.cloud.formats.writer.MessageDetail -import io.lenses.streamreactor.connect.cloud.formats.writer.SinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.StructSinkData -import io.lenses.streamreactor.connect.cloud.model.Topic +import io.lenses.streamreactor.connect.cloud.common.config.DataStorageSettings +import io.lenses.streamreactor.connect.cloud.common.formats.writer.MessageDetail +import io.lenses.streamreactor.connect.cloud.common.formats.writer.SinkData +import io.lenses.streamreactor.connect.cloud.common.formats.writer.StructSinkData +import io.lenses.streamreactor.connect.cloud.common.model.Topic +import io.lenses.streamreactor.connect.cloud.common.sink.transformers.EnvelopeWithSchemaTransformer.envelope import org.apache.kafka.connect.data._ import scala.jdk.CollectionConverters.CollectionHasAsScala diff --git a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/transformers/SchemalessEnvelopeTransformer.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/transformers/SchemalessEnvelopeTransformer.scala similarity index 86% rename from kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/transformers/SchemalessEnvelopeTransformer.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/transformers/SchemalessEnvelopeTransformer.scala index b214999940..280ab70c5b 100644 --- a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/transformers/SchemalessEnvelopeTransformer.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/transformers/SchemalessEnvelopeTransformer.scala @@ -13,19 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.sink.transformers +package io.lenses.streamreactor.connect.cloud.common.sink.transformers import cats.implicits.catsSyntaxEitherId -import io.lenses.streamreactor.connect.cloud.config.DataStorageSettings -import io.lenses.streamreactor.connect.cloud.formats.writer.ArraySinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.ByteArraySinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.MapSinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.MessageDetail -import io.lenses.streamreactor.connect.cloud.formats.writer.NullSinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.PrimitiveSinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.SinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.StructSinkData -import io.lenses.streamreactor.connect.cloud.model.Topic +import io.lenses.streamreactor.connect.cloud.common.config.DataStorageSettings +import io.lenses.streamreactor.connect.cloud.common.formats.writer._ +import io.lenses.streamreactor.connect.cloud.common.model.Topic import org.apache.kafka.connect.data._ import scala.jdk.CollectionConverters.ListHasAsScala diff --git a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/transformers/Transformer.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/transformers/Transformer.scala similarity index 85% rename from kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/transformers/Transformer.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/transformers/Transformer.scala index 7395de8ae6..e3267e8437 100644 --- a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/transformers/Transformer.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/transformers/Transformer.scala @@ -13,10 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.sink.transformers +package io.lenses.streamreactor.connect.cloud.common.sink.transformers import cats.implicits.catsSyntaxEitherId -import io.lenses.streamreactor.connect.cloud.formats.writer.MessageDetail +import io.lenses.streamreactor.connect.cloud.common.formats.writer.MessageDetail /** * A generic interface for transforming messages. diff --git a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/writer/CommitState.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/writer/CommitState.scala similarity index 90% rename from kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/writer/CommitState.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/writer/CommitState.scala index 02be2c7578..7357090e65 100644 --- a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/writer/CommitState.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/writer/CommitState.scala @@ -13,10 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.sink.writer +package io.lenses.streamreactor.connect.cloud.common.sink.writer -import io.lenses.streamreactor.connect.cloud.model.Offset -import io.lenses.streamreactor.connect.cloud.model.TopicPartition +import io.lenses.streamreactor.connect.cloud.common.model.Offset +import io.lenses.streamreactor.connect.cloud.common.model.TopicPartition import org.apache.kafka.connect.data.Schema case class CommitState( diff --git a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/writer/S3Writer.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/writer/S3Writer.scala similarity index 86% rename from kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/writer/S3Writer.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/writer/S3Writer.scala index 952b84ac62..2d0a8a68c9 100644 --- a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/writer/S3Writer.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/writer/S3Writer.scala @@ -13,26 +13,26 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.sink.writer +package io.lenses.streamreactor.connect.cloud.common.sink.writer import cats.implicits._ import com.typesafe.scalalogging.LazyLogging -import io.lenses.streamreactor.connect.aws.s3.sink.seek.IndexManager -import io.lenses.streamreactor.connect.aws.s3.storage.NonExistingFileError -import io.lenses.streamreactor.connect.aws.s3.storage.StorageInterface -import io.lenses.streamreactor.connect.aws.s3.storage.UploadFailedError -import io.lenses.streamreactor.connect.aws.s3.storage.ZeroByteFileError -import io.lenses.streamreactor.connect.cloud.config.ConnectorTaskId -import io.lenses.streamreactor.connect.cloud.formats.writer.MessageDetail -import io.lenses.streamreactor.connect.cloud.formats.writer.S3FormatWriter -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocation -import io.lenses.streamreactor.connect.cloud.model.Offset -import io.lenses.streamreactor.connect.cloud.model.TopicPartition -import io.lenses.streamreactor.connect.cloud.sink.commit.CommitContext -import io.lenses.streamreactor.connect.cloud.sink.commit.CommitPolicy -import io.lenses.streamreactor.connect.cloud.sink.FatalS3SinkError -import io.lenses.streamreactor.connect.cloud.sink.NonFatalS3SinkError -import io.lenses.streamreactor.connect.cloud.sink.SinkError +import io.lenses.streamreactor.connect.cloud.common.config.ConnectorTaskId +import io.lenses.streamreactor.connect.cloud.common.formats.writer.MessageDetail +import io.lenses.streamreactor.connect.cloud.common.formats.writer.S3FormatWriter +import io.lenses.streamreactor.connect.cloud.common.model.Offset +import io.lenses.streamreactor.connect.cloud.common.model.TopicPartition +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocation +import io.lenses.streamreactor.connect.cloud.common.sink.FatalS3SinkError +import io.lenses.streamreactor.connect.cloud.common.sink.NonFatalS3SinkError +import io.lenses.streamreactor.connect.cloud.common.sink.SinkError +import io.lenses.streamreactor.connect.cloud.common.sink.commit.CommitContext +import io.lenses.streamreactor.connect.cloud.common.sink.commit.CommitPolicy +import io.lenses.streamreactor.connect.cloud.common.sink.seek.IndexManager +import io.lenses.streamreactor.connect.cloud.common.storage.NonExistingFileError +import io.lenses.streamreactor.connect.cloud.common.storage.StorageInterface +import io.lenses.streamreactor.connect.cloud.common.storage.UploadFailedError +import io.lenses.streamreactor.connect.cloud.common.storage.ZeroByteFileError import org.apache.kafka.connect.data.Schema import java.io.File diff --git a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/S3WriterManager.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/writer/S3WriterManager.scala similarity index 63% rename from kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/S3WriterManager.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/writer/S3WriterManager.scala index 5ffcab3339..2bd70a3567 100644 --- a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/S3WriterManager.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/writer/S3WriterManager.scala @@ -13,32 +13,26 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.sink +package io.lenses.streamreactor.connect.cloud.common.sink.writer import cats.implicits._ -import com.typesafe.scalalogging.LazyLogging import com.typesafe.scalalogging.StrictLogging -import io.lenses.streamreactor.connect.cloud.model.Offset.orderingByOffsetValue -import io.lenses.streamreactor.connect.aws.s3.sink.config.S3SinkConfig -import io.lenses.streamreactor.connect.aws.s3.sink.config.SinkBucketOptions -import io.lenses.streamreactor.connect.aws.s3.sink.naming.KeyNamer -import io.lenses.streamreactor.connect.aws.s3.sink.seek._ -import io.lenses.streamreactor.connect.aws.s3.sink.transformers.TopicsTransformers -import io.lenses.streamreactor.connect.aws.s3.sink.writer.S3Writer -import io.lenses.streamreactor.connect.aws.s3.storage.StorageInterface -import io.lenses.streamreactor.connect.cloud.config.ConnectorTaskId -import io.lenses.streamreactor.connect.cloud.formats.writer.MessageDetail -import io.lenses.streamreactor.connect.cloud.formats.writer.S3FormatWriter -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocation -import io.lenses.streamreactor.connect.cloud.model.Offset -import io.lenses.streamreactor.connect.cloud.model.Topic -import io.lenses.streamreactor.connect.cloud.model.TopicPartition -import io.lenses.streamreactor.connect.cloud.model.TopicPartitionOffset -import io.lenses.streamreactor.connect.cloud.sink.commit.CommitPolicy -import io.lenses.streamreactor.connect.cloud.sink.config.PartitionField -import io.lenses.streamreactor.connect.cloud.sink.BatchS3SinkError -import io.lenses.streamreactor.connect.cloud.sink.FatalS3SinkError -import io.lenses.streamreactor.connect.cloud.sink.SinkError +import io.lenses.streamreactor.connect.cloud.common.config.ConnectorTaskId +import io.lenses.streamreactor.connect.cloud.common.formats.writer.MessageDetail +import io.lenses.streamreactor.connect.cloud.common.formats.writer.S3FormatWriter +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocation +import io.lenses.streamreactor.connect.cloud.common.model.Offset +import io.lenses.streamreactor.connect.cloud.common.model.TopicPartition +import io.lenses.streamreactor.connect.cloud.common.model.TopicPartitionOffset +import io.lenses.streamreactor.connect.cloud.common.sink +import io.lenses.streamreactor.connect.cloud.common.sink.commit.CommitPolicy +import io.lenses.streamreactor.connect.cloud.common.sink.config.PartitionField +import io.lenses.streamreactor.connect.cloud.common.sink.naming.KeyNamer +import io.lenses.streamreactor.connect.cloud.common.sink.seek.IndexManager +import io.lenses.streamreactor.connect.cloud.common.sink.BatchS3SinkError +import io.lenses.streamreactor.connect.cloud.common.sink.FatalS3SinkError +import io.lenses.streamreactor.connect.cloud.common.sink.SinkError +import io.lenses.streamreactor.connect.cloud.common.storage.StorageInterface import org.apache.kafka.clients.consumer.OffsetAndMetadata import org.apache.kafka.connect.data.Schema @@ -124,7 +118,7 @@ class S3WriterManager( }.toSet if (writerCommitErrors.nonEmpty) { - BatchS3SinkError(writerCommitErrors).asLeft + sink.BatchS3SinkError(writerCommitErrors).asLeft } else { ().asRight } @@ -299,103 +293,3 @@ class S3WriterManager( .foreach(writers.remove) } - -object S3WriterManager extends LazyLogging { - - def from( - config: S3SinkConfig, - )( - implicit - connectorTaskId: ConnectorTaskId, - storageInterface: StorageInterface, - ): S3WriterManager = { - - val bucketAndPrefixFn: TopicPartition => Either[SinkError, CloudLocation] = topicPartition => { - bucketOptsForTopic(config, topicPartition.topic) match { - case Some(sBO) => sBO.bucketAndPrefix.asRight - case None => fatalErrorTopicNotConfigured(topicPartition).asLeft - } - } - - val commitPolicyFn: TopicPartition => Either[SinkError, CommitPolicy] = topicPartition => - bucketOptsForTopic(config, topicPartition.topic) match { - case Some(bucketOptions) => bucketOptions.commitPolicy.asRight - case None => fatalErrorTopicNotConfigured(topicPartition).asLeft - } - - val keyNamerFn: TopicPartition => Either[SinkError, KeyNamer] = topicPartition => - bucketOptsForTopic(config, topicPartition.topic) match { - case Some(bucketOptions) => bucketOptions.keyNamer.asRight - case None => fatalErrorTopicNotConfigured(topicPartition).asLeft - } - - val stagingFilenameFn: (TopicPartition, immutable.Map[PartitionField, String]) => Either[SinkError, File] = - (topicPartition, partitionValues) => - bucketOptsForTopic(config, topicPartition.topic) match { - case Some(bucketOptions) => - for { - keyNamer <- keyNamerFn(topicPartition) - stagingFilename <- keyNamer.stagingFile(bucketOptions.localStagingArea.dir, - bucketOptions.bucketAndPrefix, - topicPartition, - partitionValues, - ) - } yield stagingFilename - case None => fatalErrorTopicNotConfigured(topicPartition).asLeft - } - - val finalFilenameFn: ( - TopicPartition, - immutable.Map[PartitionField, String], - Offset, - ) => Either[SinkError, CloudLocation] = (topicPartition, partitionValues, offset) => - bucketOptsForTopic(config, topicPartition.topic) match { - case Some(bucketOptions) => - for { - keyNamer <- keyNamerFn(topicPartition) - stagingFilename <- keyNamer.finalFilename(bucketOptions.bucketAndPrefix, - topicPartition.withOffset(offset), - partitionValues, - ) - } yield stagingFilename - case None => fatalErrorTopicNotConfigured(topicPartition).asLeft - } - - val formatWriterFn: (TopicPartition, File) => Either[SinkError, S3FormatWriter] = - (topicPartition: TopicPartition, stagingFilename) => - bucketOptsForTopic(config, topicPartition.topic) match { - case Some(bucketOptions) => - for { - formatWriter <- S3FormatWriter( - bucketOptions.formatSelection, - stagingFilename, - topicPartition, - )(config.compressionCodec) - } yield formatWriter - case None => FatalS3SinkError("Can't find format choice in config", topicPartition).asLeft - } - - val indexManager = new IndexManager(config.offsetSeekerOptions.maxIndexFiles) - - val transformers = TopicsTransformers.from(config.bucketOptions) - new S3WriterManager( - commitPolicyFn, - bucketAndPrefixFn, - keyNamerFn, - stagingFilenameFn, - finalFilenameFn, - formatWriterFn, - indexManager, - transformers.transform, - ) - } - - private def bucketOptsForTopic(config: S3SinkConfig, topic: Topic): Option[SinkBucketOptions] = - config.bucketOptions.find(bo => bo.sourceTopic.isEmpty || bo.sourceTopic.contains(topic.value)) - - private def fatalErrorTopicNotConfigured(topicPartition: TopicPartition): SinkError = - FatalS3SinkError( - s"Can't find the KCQL for source topic [${topicPartition.topic}]. The topics defined via [topics] or [topics.regex] need to have an equivalent KCQL statement: INSERT INTO {S3_BUCKET} SELECT * FROM {TOPIC}.", - topicPartition, - ) -} diff --git a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/writer/WriteState.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/writer/WriteState.scala similarity index 91% rename from kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/writer/WriteState.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/writer/WriteState.scala index fc80e07853..7bd7f7e779 100644 --- a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/sink/writer/WriteState.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/sink/writer/WriteState.scala @@ -13,11 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.sink.writer +package io.lenses.streamreactor.connect.cloud.common.sink.writer import com.typesafe.scalalogging.LazyLogging -import io.lenses.streamreactor.connect.cloud.formats.writer.S3FormatWriter -import io.lenses.streamreactor.connect.cloud.model.Offset +import io.lenses.streamreactor.connect.cloud.common.formats.writer.S3FormatWriter +import io.lenses.streamreactor.connect.cloud.common.model.Offset import org.apache.kafka.connect.data.Schema import java.io.File diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/source/ContextConstants.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/source/ContextConstants.scala similarity index 92% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/source/ContextConstants.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/source/ContextConstants.scala index 727c7adbaf..28cebe29c2 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/source/ContextConstants.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/source/ContextConstants.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.source +package io.lenses.streamreactor.connect.cloud.common.source object ContextConstants { diff --git a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/SourceContextReader.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/source/SourceContextReader.scala similarity index 78% rename from kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/SourceContextReader.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/source/SourceContextReader.scala index bebe0a1d4c..777b96c93e 100644 --- a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/SourceContextReader.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/source/SourceContextReader.scala @@ -13,14 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.source +package io.lenses.streamreactor.connect.cloud.common.source import cats.implicits.catsSyntaxOptionId -import io.lenses.streamreactor.connect.cloud.source.ContextConstants.LineKey -import io.lenses.streamreactor.connect.cloud.source.ContextConstants.PathKey -import io.lenses.streamreactor.connect.cloud.source.ContextConstants.TimeStampKey -import io.lenses.streamreactor.connect.cloud.source.SourceWatermark.partition -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocation +import ContextConstants.LineKey +import ContextConstants.PathKey +import ContextConstants.TimeStampKey +import SourceWatermark.partition +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocation import org.apache.kafka.connect.source.SourceTaskContext import java.time.Instant diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/source/SourceWatermark.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/source/SourceWatermark.scala similarity index 75% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/source/SourceWatermark.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/source/SourceWatermark.scala index 944bf3c7bc..d58c9139f9 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/source/SourceWatermark.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/source/SourceWatermark.scala @@ -13,14 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.source +package io.lenses.streamreactor.connect.cloud.common.source -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocation -import io.lenses.streamreactor.connect.cloud.source.ContextConstants.ContainerKey -import io.lenses.streamreactor.connect.cloud.source.ContextConstants.LineKey -import io.lenses.streamreactor.connect.cloud.source.ContextConstants.PathKey -import io.lenses.streamreactor.connect.cloud.source.ContextConstants.PrefixKey -import io.lenses.streamreactor.connect.cloud.source.ContextConstants.TimeStampKey +import ContextConstants.ContainerKey +import ContextConstants.LineKey +import ContextConstants.PathKey +import ContextConstants.PrefixKey +import ContextConstants.TimeStampKey +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocation import java.time.Instant import scala.jdk.CollectionConverters.MapHasAsJava diff --git a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/config/OrderingType.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/source/config/OrderingType.scala similarity index 77% rename from kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/config/OrderingType.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/source/config/OrderingType.scala index 329990a860..4ea3edbef2 100644 --- a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/config/OrderingType.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/source/config/OrderingType.scala @@ -13,13 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.source.config +package io.lenses.streamreactor.connect.cloud.common.source.config import enumeratum.Enum import enumeratum.EnumEntry -import io.lenses.streamreactor.connect.aws.s3.source.files.BatchLister -import io.lenses.streamreactor.connect.aws.s3.source.files.DateOrderingBatchLister -import io.lenses.streamreactor.connect.aws.s3.source.files.DefaultOrderingBatchLister +import io.lenses.streamreactor.connect.cloud.common.source.files.BatchLister +import io.lenses.streamreactor.connect.cloud.common.source.files.DateOrderingBatchLister +import io.lenses.streamreactor.connect.cloud.common.source.files.DefaultOrderingBatchLister sealed trait OrderingType extends EnumEntry { diff --git a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/config/PartitionExtractor.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/source/config/PartitionExtractor.scala similarity index 93% rename from kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/config/PartitionExtractor.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/source/config/PartitionExtractor.scala index 1f57163f42..fc07505ffd 100644 --- a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/config/PartitionExtractor.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/source/config/PartitionExtractor.scala @@ -13,10 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.source.config +package io.lenses.streamreactor.connect.cloud.common.source.config import com.typesafe.scalalogging.LazyLogging -import io.lenses.streamreactor.connect.cloud.config.Format +import io.lenses.streamreactor.connect.cloud.common.config.Format /** * For a source, can extract the original partition so that the message can be returned to the original partition. diff --git a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/config/PartitionSearcherOptions.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/source/config/PartitionSearcherOptions.scala similarity index 94% rename from kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/config/PartitionSearcherOptions.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/source/config/PartitionSearcherOptions.scala index 8a09ea6bf7..3ece509927 100644 --- a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/config/PartitionSearcherOptions.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/source/config/PartitionSearcherOptions.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.source.config +package io.lenses.streamreactor.connect.cloud.common.source.config import scala.concurrent.duration.FiniteDuration diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/source/config/ReadTextMode.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/source/config/ReadTextMode.scala similarity index 85% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/source/config/ReadTextMode.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/source/config/ReadTextMode.scala index 937c19c8e1..df28d58124 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/source/config/ReadTextMode.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/source/config/ReadTextMode.scala @@ -13,14 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.source.config +package io.lenses.streamreactor.connect.cloud.common.source.config -import io.lenses.streamreactor.connect.cloud.config.kcqlprops.S3PropsKeyEntry -import io.lenses.streamreactor.connect.cloud.config.kcqlprops.S3PropsKeyEnum -import io.lenses.streamreactor.connect.cloud.formats.reader.CustomTextStreamReader -import io.lenses.streamreactor.connect.cloud.formats.reader.S3DataIterator -import io.lenses.streamreactor.connect.cloud.source.config.kcqlprops.ReadTextModeEntry -import io.lenses.streamreactor.connect.cloud.source.config.kcqlprops.ReadTextModeEnum +import io.lenses.streamreactor.connect.cloud.common.config.kcqlprops.S3PropsKeyEntry +import io.lenses.streamreactor.connect.cloud.common.config.kcqlprops.S3PropsKeyEnum +import io.lenses.streamreactor.connect.cloud.common.formats.reader.CustomTextStreamReader +import io.lenses.streamreactor.connect.cloud.common.formats.reader.S3DataIterator +import io.lenses.streamreactor.connect.cloud.common.source.config.kcqlprops.ReadTextModeEntry +import io.lenses.streamreactor.connect.cloud.common.source.config.kcqlprops.ReadTextModeEnum import io.lenses.streamreactor.connect.config.kcqlprops.KcqlProperties import io.lenses.streamreactor.connect.io.text.LineStartLineEndReader import io.lenses.streamreactor.connect.io.text.PrefixSuffixReader diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/source/config/distribution/PartitionHasher.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/source/config/distribution/PartitionHasher.scala similarity index 90% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/source/config/distribution/PartitionHasher.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/source/config/distribution/PartitionHasher.scala index 56e0803565..d5988847eb 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/source/config/distribution/PartitionHasher.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/source/config/distribution/PartitionHasher.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.source.config.distribution +package io.lenses.streamreactor.connect.cloud.common.source.config.distribution object PartitionHasher { def hash(maxTasks: Int, fileName: String): Int = Math.abs(fileName.hashCode) % maxTasks diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/source/config/kcqlprops/ReadTextMode.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/source/config/kcqlprops/ReadTextMode.scala similarity index 92% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/source/config/kcqlprops/ReadTextMode.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/source/config/kcqlprops/ReadTextMode.scala index ded810e564..af93bd4b55 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/source/config/kcqlprops/ReadTextMode.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/source/config/kcqlprops/ReadTextMode.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.source.config.kcqlprops +package io.lenses.streamreactor.connect.cloud.common.source.config.kcqlprops import enumeratum._ diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/source/config/kcqlprops/S3SourcePropsSchema.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/source/config/kcqlprops/S3SourcePropsSchema.scala similarity index 79% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/source/config/kcqlprops/S3SourcePropsSchema.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/source/config/kcqlprops/S3SourcePropsSchema.scala index 1eff209c61..913a5a59ee 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/source/config/kcqlprops/S3SourcePropsSchema.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/source/config/kcqlprops/S3SourcePropsSchema.scala @@ -13,11 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.source.config.kcqlprops +package io.lenses.streamreactor.connect.cloud.common.source.config.kcqlprops -import io.lenses.streamreactor.connect.cloud.config.kcqlprops.S3PropsKeyEntry -import io.lenses.streamreactor.connect.cloud.config.kcqlprops.S3PropsKeyEnum -import io.lenses.streamreactor.connect.cloud.config.kcqlprops.S3PropsKeyEnum._ +import io.lenses.streamreactor.connect.cloud.common.config.kcqlprops.S3PropsKeyEntry +import io.lenses.streamreactor.connect.cloud.common.config.kcqlprops.S3PropsKeyEnum +import S3PropsKeyEnum._ import io.lenses.streamreactor.connect.config.kcqlprops._ object S3SourcePropsSchema { diff --git a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/distribution/PartitionSearcherResponse.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/source/distribution/PartitionSearcherResponse.scala similarity index 76% rename from kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/distribution/PartitionSearcherResponse.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/source/distribution/PartitionSearcherResponse.scala index c36045ee3c..ecff59363f 100644 --- a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/distribution/PartitionSearcherResponse.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/source/distribution/PartitionSearcherResponse.scala @@ -13,10 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.source.distribution +package io.lenses.streamreactor.connect.cloud.common.source.distribution -import io.lenses.streamreactor.connect.aws.s3.storage.DirectoryFindResults -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocation +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocation +import io.lenses.streamreactor.connect.cloud.common.storage.DirectoryFindResults case class PartitionSearcherResponse( root: CloudLocation, diff --git a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/files/BatchLister.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/source/files/BatchLister.scala similarity index 70% rename from kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/files/BatchLister.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/source/files/BatchLister.scala index 859cdc919a..2a6199623b 100644 --- a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/files/BatchLister.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/source/files/BatchLister.scala @@ -13,12 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.source.files +package io.lenses.streamreactor.connect.cloud.common.source.files -import io.lenses.streamreactor.connect.aws.s3.storage.FileListError -import io.lenses.streamreactor.connect.aws.s3.storage.FileMetadata -import io.lenses.streamreactor.connect.aws.s3.storage.ListResponse -import io.lenses.streamreactor.connect.aws.s3.storage.StorageInterface +import io.lenses.streamreactor.connect.cloud.common.storage.FileListError +import io.lenses.streamreactor.connect.cloud.common.storage.FileMetadata +import io.lenses.streamreactor.connect.cloud.common.storage.ListResponse +import io.lenses.streamreactor.connect.cloud.common.storage.StorageInterface trait BatchLister { diff --git a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/files/DateOrderingBatchLister.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/source/files/DateOrderingBatchLister.scala similarity index 80% rename from kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/files/DateOrderingBatchLister.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/source/files/DateOrderingBatchLister.scala index f945a0ceaa..1ac02eb768 100644 --- a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/files/DateOrderingBatchLister.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/source/files/DateOrderingBatchLister.scala @@ -13,13 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.source.files +package io.lenses.streamreactor.connect.cloud.common.source.files + import cats.implicits.catsSyntaxOptionId -import io.lenses.streamreactor.connect.aws.s3.storage.ResultProcessors.processObjectsAsFileMeta -import io.lenses.streamreactor.connect.aws.s3.storage.FileListError -import io.lenses.streamreactor.connect.aws.s3.storage.FileMetadata -import io.lenses.streamreactor.connect.aws.s3.storage.ListResponse -import io.lenses.streamreactor.connect.aws.s3.storage.StorageInterface +import io.lenses.streamreactor.connect.cloud.common.storage.ResultProcessors.processObjectsAsFileMeta +import io.lenses.streamreactor.connect.cloud.common.storage.FileListError +import io.lenses.streamreactor.connect.cloud.common.storage.FileMetadata +import io.lenses.streamreactor.connect.cloud.common.storage.ListResponse +import io.lenses.streamreactor.connect.cloud.common.storage.StorageInterface /** * If an order other than UTF-8 binary order is desired, then this can only be achieved by downloading all the results from the bucket and ordering them in memory. It is recommended to be able to use the default ordering wherever possible, but if you need to sort by file modification date then use this. diff --git a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/files/DefaultOrderingBatchLister.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/source/files/DefaultOrderingBatchLister.scala similarity index 75% rename from kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/files/DefaultOrderingBatchLister.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/source/files/DefaultOrderingBatchLister.scala index 823094c742..d94caf2fc6 100644 --- a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/files/DefaultOrderingBatchLister.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/source/files/DefaultOrderingBatchLister.scala @@ -13,12 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.source.files +package io.lenses.streamreactor.connect.cloud.common.source.files -import io.lenses.streamreactor.connect.aws.s3.storage.FileListError -import io.lenses.streamreactor.connect.aws.s3.storage.FileMetadata -import io.lenses.streamreactor.connect.aws.s3.storage.ListResponse -import io.lenses.streamreactor.connect.aws.s3.storage.StorageInterface +import io.lenses.streamreactor.connect.cloud.common.storage.FileListError +import io.lenses.streamreactor.connect.cloud.common.storage.FileMetadata +import io.lenses.streamreactor.connect.cloud.common.storage.ListResponse +import io.lenses.streamreactor.connect.cloud.common.storage.StorageInterface /** * As AWS's List results are always returned in UTF-8 binary order, the default mode of listing simply defers to Amazon to decide the order. diff --git a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/files/S3SourceFileQueue.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/source/files/S3SourceFileQueue.scala similarity index 86% rename from kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/files/S3SourceFileQueue.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/source/files/S3SourceFileQueue.scala index 091b52fb6b..ab85faeba1 100644 --- a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/files/S3SourceFileQueue.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/source/files/S3SourceFileQueue.scala @@ -13,19 +13,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.source.files +package io.lenses.streamreactor.connect.cloud.common.source.files import cats.implicits.catsSyntaxEitherId import cats.implicits.catsSyntaxOptionId import cats.implicits.toShow import com.typesafe.scalalogging.LazyLogging -import io.lenses.streamreactor.connect.aws.s3.storage.FileListError -import io.lenses.streamreactor.connect.aws.s3.storage.FileLoadError -import io.lenses.streamreactor.connect.aws.s3.storage.FileMetadata -import io.lenses.streamreactor.connect.aws.s3.storage.ListResponse -import io.lenses.streamreactor.connect.cloud.config.ConnectorTaskId -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocation -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocationValidator +import io.lenses.streamreactor.connect.cloud.common.config.ConnectorTaskId +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocation +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocationValidator +import io.lenses.streamreactor.connect.cloud.common.storage.FileListError +import io.lenses.streamreactor.connect.cloud.common.storage.FileLoadError +import io.lenses.streamreactor.connect.cloud.common.storage.FileMetadata +import io.lenses.streamreactor.connect.cloud.common.storage.ListResponse import java.time.Instant diff --git a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/reader/PartitionDiscovery.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/source/reader/PartitionDiscovery.scala similarity index 87% rename from kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/reader/PartitionDiscovery.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/source/reader/PartitionDiscovery.scala index 919eab11e2..54b1c237ca 100644 --- a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/reader/PartitionDiscovery.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/source/reader/PartitionDiscovery.scala @@ -13,18 +13,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.source.reader +package io.lenses.streamreactor.connect.cloud.common.source.reader import cats.effect.IO import cats.effect.kernel.Ref import cats.implicits.toShow import cats.implicits.toTraverseOps import com.typesafe.scalalogging.LazyLogging -import io.lenses.streamreactor.connect.aws.s3.source.config.PartitionSearcherOptions -import io.lenses.streamreactor.connect.aws.s3.source.distribution.PartitionSearcherResponse +import io.lenses.streamreactor.connect.cloud.common.config.ConnectorTaskId +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocation +import io.lenses.streamreactor.connect.cloud.common.source.config.PartitionSearcherOptions +import io.lenses.streamreactor.connect.cloud.common.source.distribution.PartitionSearcherResponse import io.lenses.streamreactor.connect.cloud.common.utils.PollLoop -import io.lenses.streamreactor.connect.cloud.config.ConnectorTaskId -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocation object PartitionDiscovery extends LazyLogging { diff --git a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/reader/ReaderManager.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/source/reader/ReaderManager.scala similarity index 94% rename from kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/reader/ReaderManager.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/source/reader/ReaderManager.scala index 5b2b9f1fdb..5fbfeceae0 100644 --- a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/reader/ReaderManager.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/source/reader/ReaderManager.scala @@ -13,16 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.source.reader +package io.lenses.streamreactor.connect.cloud.common.source.reader import cats.effect.IO import cats.effect.Ref import cats.implicits.toBifunctorOps import cats.implicits.toShow import com.typesafe.scalalogging.LazyLogging -import io.lenses.streamreactor.connect.aws.s3.source.files.SourceFileQueue -import io.lenses.streamreactor.connect.cloud.config.ConnectorTaskId -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocation +import io.lenses.streamreactor.connect.cloud.common.config.ConnectorTaskId +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocation +import io.lenses.streamreactor.connect.cloud.common.source.files.SourceFileQueue import org.apache.kafka.connect.source.SourceRecord import scala.util.Try diff --git a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/reader/ReaderManagerState.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/source/reader/ReaderManagerState.scala similarity index 81% rename from kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/reader/ReaderManagerState.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/source/reader/ReaderManagerState.scala index cf4366326d..b65de46c3e 100644 --- a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/reader/ReaderManagerState.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/source/reader/ReaderManagerState.scala @@ -13,9 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.source.reader +package io.lenses.streamreactor.connect.cloud.common.source.reader -import io.lenses.streamreactor.connect.aws.s3.source.distribution.PartitionSearcherResponse +import io.lenses.streamreactor.connect.cloud.common.source.distribution.PartitionSearcherResponse case class ReaderManagerState( partitionResponses: Seq[PartitionSearcherResponse], diff --git a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/reader/ResultReader.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/source/reader/ResultReader.scala similarity index 84% rename from kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/reader/ResultReader.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/source/reader/ResultReader.scala index d6365c6328..d79449f351 100644 --- a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/reader/ResultReader.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/source/reader/ResultReader.scala @@ -13,22 +13,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.source.reader +package io.lenses.streamreactor.connect.cloud.common.source.reader import cats.implicits.toBifunctorOps import cats.implicits.toShow import com.typesafe.scalalogging.LazyLogging -import io.lenses.streamreactor.connect.aws.s3.storage.StorageInterface +import io.lenses.streamreactor.connect.cloud.common.config.ConnectorTaskId +import io.lenses.streamreactor.connect.cloud.common.config.FormatSelection +import io.lenses.streamreactor.connect.cloud.common.config.ReaderBuilderContext +import io.lenses.streamreactor.connect.cloud.common.formats.reader.S3StreamReader +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocation +import io.lenses.streamreactor.connect.cloud.common.source.SourceWatermark +import io.lenses.streamreactor.connect.cloud.common.storage.StorageInterface import io.lenses.streamreactor.connect.cloud.common.utils.IteratorOps -import io.lenses.streamreactor.connect.cloud.config -import io.lenses.streamreactor.connect.cloud.config.ConnectorTaskId -import io.lenses.streamreactor.connect.cloud.config.FormatSelection -import io.lenses.streamreactor.connect.cloud.formats.reader.S3StreamReader -import io.lenses.streamreactor.connect.cloud.model.Topic -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocation -import io.lenses.streamreactor.connect.cloud.source.SourceWatermark import org.apache.kafka.connect.source.SourceRecord - +import io.lenses.streamreactor.connect.cloud.common.model.Topic import scala.annotation.tailrec import scala.util.Try @@ -66,7 +65,7 @@ class ResultReader( object ResultReader extends LazyLogging { - def create( + def create[I]( format: FormatSelection, targetTopic: String, partitionFn: String => Option[Int], @@ -88,7 +87,7 @@ object ResultReader extends LazyLogging { partition = partitionFn(path).map(Int.box).orNull reader = format.toStreamReader( - config.ReaderBuilderContext( + ReaderBuilderContext( inputStream, pathWithLine, metadata, diff --git a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/state/S3SourceTaskState.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/source/state/S3SourceTaskState.scala similarity index 88% rename from kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/state/S3SourceTaskState.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/source/state/S3SourceTaskState.scala index 84e95ad678..f43ffcd8c2 100644 --- a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/source/state/S3SourceTaskState.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/source/state/S3SourceTaskState.scala @@ -13,11 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.source.state +package io.lenses.streamreactor.connect.cloud.common.source.state import cats.effect.IO import cats.implicits._ -import io.lenses.streamreactor.connect.aws.s3.source.reader.ReaderManager +import io.lenses.streamreactor.connect.cloud.common.source.reader.ReaderManager import org.apache.kafka.connect.source.SourceRecord class S3SourceTaskState( diff --git a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/storage/PartitionSearchResults.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/storage/PartitionSearchResults.scala similarity index 86% rename from kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/storage/PartitionSearchResults.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/storage/PartitionSearchResults.scala index 9ac91069a4..d330b237d5 100644 --- a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/storage/PartitionSearchResults.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/storage/PartitionSearchResults.scala @@ -13,9 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.storage +package io.lenses.streamreactor.connect.cloud.common.storage -import io.lenses.streamreactor.connect.aws.s3.source.config.PartitionSearcherOptions +import io.lenses.streamreactor.connect.cloud.common.source.config.PartitionSearcherOptions object DirectoryFindCompletionConfig { def fromSearchOptions( diff --git a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/storage/ResultProcessors.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/storage/ResultProcessors.scala similarity index 72% rename from kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/storage/ResultProcessors.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/storage/ResultProcessors.scala index 317e8b0880..110ee31bb8 100644 --- a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/storage/ResultProcessors.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/storage/ResultProcessors.scala @@ -13,46 +13,39 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.storage - -import software.amazon.awssdk.services.s3.model.S3Object +package io.lenses.streamreactor.connect.cloud.common.storage object ResultProcessors { def processObjectsAsFileMeta( bucket: String, prefix: Option[String], - objectSeq: Seq[S3Object], - ): Option[ListResponse[FileMetadata]] = { - val list = objectSeq - .map(s3O => FileMetadata(s3O.key(), s3O.lastModified())) - - list + objectSeq: Seq[FileMetadata], + ): Option[ListResponse[FileMetadata]] = + objectSeq .lastOption .map { last => ListResponse[FileMetadata]( bucket, prefix, - list, + objectSeq, last, ) } - } - def processAsKey( bucket: String, prefix: Option[String], - objectSeq: Seq[S3Object], + objectSeq: Seq[FileMetadata], ): Option[ListResponse[String]] = objectSeq.lastOption.map { last => ListResponse( bucket, prefix, - objectSeq.map(_.key()), - FileMetadata(last.key(), last.lastModified()), + objectSeq.map(_.file), + last, ) } } diff --git a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/storage/StorageInterface.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/storage/StorageInterface.scala similarity index 87% rename from kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/storage/StorageInterface.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/storage/StorageInterface.scala index 60d53648c7..aded73b559 100644 --- a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/storage/StorageInterface.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/storage/StorageInterface.scala @@ -13,10 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.storage +package io.lenses.streamreactor.connect.cloud.common.storage -import io.lenses.streamreactor.connect.cloud.config.ObjectMetadata -import software.amazon.awssdk.services.s3.model.S3Object +import io.lenses.streamreactor.connect.cloud.common.config.ObjectMetadata import java.io.File import java.io.InputStream @@ -51,7 +50,7 @@ trait StorageInterface { def listRecursive[T]( bucket: String, prefix: Option[String], - processFn: (String, Option[String], Seq[S3Object]) => Option[ListResponse[T]], + processFn: (String, Option[String], Seq[FileMetadata]) => Option[ListResponse[T]], ): Either[FileListError, Option[ListResponse[T]]] def getBlob(bucket: String, path: String): Either[FileLoadError, InputStream] diff --git a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/storage/UploadError.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/storage/UploadError.scala similarity index 97% rename from kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/storage/UploadError.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/storage/UploadError.scala index 8a544c945e..467cc4d0d6 100644 --- a/kafka-connect-aws-s3/src/main/scala/io/lenses/streamreactor/connect/aws/s3/storage/UploadError.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/storage/UploadError.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.storage +package io.lenses.streamreactor.connect.cloud.common.storage import java.io.File diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/stream/BuildLocalOutputStream.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/stream/BuildLocalOutputStream.scala similarity index 87% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/stream/BuildLocalOutputStream.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/stream/BuildLocalOutputStream.scala index ec23194ea1..b848588f72 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/stream/BuildLocalOutputStream.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/stream/BuildLocalOutputStream.scala @@ -13,13 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.stream +package io.lenses.streamreactor.connect.cloud.common.stream import cats.implicits._ import com.typesafe.scalalogging.LazyLogging -import io.lenses.streamreactor.connect.cloud.model.TopicPartition -import io.lenses.streamreactor.connect.cloud.sink.FatalS3SinkError -import io.lenses.streamreactor.connect.cloud.sink.SinkError +import io.lenses.streamreactor.connect.cloud.common.model.TopicPartition +import io.lenses.streamreactor.connect.cloud.common.sink.FatalS3SinkError +import io.lenses.streamreactor.connect.cloud.common.sink.SinkError import java.io.BufferedOutputStream import scala.util.Try diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/stream/S3OutputStream.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/stream/S3OutputStream.scala similarity index 85% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/stream/S3OutputStream.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/stream/S3OutputStream.scala index 4c6173b96f..fbc36c2fee 100644 --- a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/stream/S3OutputStream.scala +++ b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/stream/S3OutputStream.scala @@ -13,10 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.stream - -import io.lenses.streamreactor.connect.cloud.sink.SinkError +package io.lenses.streamreactor.connect.cloud.common.stream +import io.lenses.streamreactor.connect.cloud.common.sink.SinkError import java.io.OutputStream trait S3OutputStream extends OutputStream { diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/utils/IteratorOps.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/utils/IteratorOps.scala similarity index 100% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/utils/IteratorOps.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/utils/IteratorOps.scala diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/utils/MapUtils.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/utils/MapUtils.scala similarity index 100% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/utils/MapUtils.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/utils/MapUtils.scala diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/utils/PollLoop.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/utils/PollLoop.scala similarity index 100% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/utils/PollLoop.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/utils/PollLoop.scala diff --git a/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/utils/TimestampUtils.scala b/kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/utils/TimestampUtils.scala similarity index 100% rename from kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/utils/TimestampUtils.scala rename to kafka-connect-cloud-common/src/main/scala/io/lenses/streamreactor/connect/cloud/common/utils/TimestampUtils.scala diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/config/ConnectorTaskIdTest.scala b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/config/ConnectorTaskIdTest.scala similarity index 71% rename from kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/config/ConnectorTaskIdTest.scala rename to kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/config/ConnectorTaskIdTest.scala index 9743eef08c..45b37714e4 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/config/ConnectorTaskIdTest.scala +++ b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/config/ConnectorTaskIdTest.scala @@ -13,14 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.config +package io.lenses.streamreactor.connect.cloud.common.config import cats.implicits.catsSyntaxEitherId -import io.lenses.streamreactor.connect.aws.s3.config.S3ConfigSettings.CONNECTOR_PREFIX -import io.lenses.streamreactor.connect.cloud.config.ConnectorTaskId -import io.lenses.streamreactor.connect.cloud.config.ConnectorTaskIdCreator -import io.lenses.streamreactor.connect.cloud.config.TaskIndexKey -import io.lenses.streamreactor.connect.cloud.source.config.distribution.PartitionHasher +import io.lenses.streamreactor.connect.cloud.common.source.config.distribution.PartitionHasher import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec @@ -30,14 +26,14 @@ class ConnectorTaskIdTest extends AnyWordSpec with Matchers with TaskIndexKey { "ConnectorTaskId" should { "create the instance" in { val from = Map("a" -> "1", "b" -> "2", TASK_INDEX -> "0:2", "name" -> connectorName) - new ConnectorTaskIdCreator(CONNECTOR_PREFIX).fromProps(from.asJava) shouldBe ConnectorTaskId(connectorName, - 2, - 0, + new ConnectorTaskIdCreator(connectorPrefix).fromProps(from.asJava) shouldBe ConnectorTaskId(connectorName, + 2, + 0, ).asRight[String] } "fail if max tasks is not valid integer" in { val from = Map("a" -> "1", "b" -> "2", TASK_INDEX -> "0:2a", "name" -> connectorName) - val actual = new ConnectorTaskIdCreator(CONNECTOR_PREFIX).fromProps(from.asJava) + val actual = new ConnectorTaskIdCreator(connectorPrefix).fromProps(from.asJava) actual match { case Left(e) => e.getMessage shouldBe s"Invalid $TASK_INDEX. Expecting an integer but found:2a" case Right(_) => fail("Should have failed") @@ -45,14 +41,14 @@ class ConnectorTaskIdTest extends AnyWordSpec with Matchers with TaskIndexKey { } "fail if task number is not a valid integer" in { val from = Map("a" -> "1", "b" -> "2", TASK_INDEX -> "0a:2", "name" -> connectorName) - new ConnectorTaskIdCreator(CONNECTOR_PREFIX).fromProps(from.asJava) match { + new ConnectorTaskIdCreator(connectorPrefix).fromProps(from.asJava) match { case Left(value) => value.getMessage shouldBe s"Invalid $TASK_INDEX. Expecting an integer but found:0a" case Right(_) => fail("Should have failed") } } "fail if task number < 0" in { val from = Map("a" -> "1", "b" -> "2", TASK_INDEX -> "-1:2", "name" -> connectorName) - new ConnectorTaskIdCreator(CONNECTOR_PREFIX).fromProps(from.asJava) match { + new ConnectorTaskIdCreator(connectorPrefix).fromProps(from.asJava) match { case Left(value) => value.getMessage shouldBe s"Invalid $TASK_INDEX. Expecting a positive integer but found:-1" case Right(value) => fail(s"Should have failed but got $value") } @@ -60,14 +56,14 @@ class ConnectorTaskIdTest extends AnyWordSpec with Matchers with TaskIndexKey { } "fail if max tasks is zero" in { val from = Map("a" -> "1", "b" -> "2", TASK_INDEX -> "0:0", "name" -> connectorName) - new ConnectorTaskIdCreator(CONNECTOR_PREFIX).fromProps(from.asJava) match { + new ConnectorTaskIdCreator(connectorPrefix).fromProps(from.asJava) match { case Left(value) => value.getMessage shouldBe s"Invalid $TASK_INDEX. Expecting a positive integer but found:0" case Right(value) => fail(s"Should have failed but got $value") } } "fail if max tasks is negative" in { val from = Map("a" -> "1", "b" -> "2", TASK_INDEX -> "0:-1", "name" -> connectorName) - new ConnectorTaskIdCreator(CONNECTOR_PREFIX).fromProps(from.asJava) match { + new ConnectorTaskIdCreator(connectorPrefix).fromProps(from.asJava) match { case Left(value) => value.getMessage shouldBe s"Invalid $TASK_INDEX. Expecting a positive integer but found:-1" case Right(value) => fail(s"Should have failed but got $value") } @@ -76,7 +72,7 @@ class ConnectorTaskIdTest extends AnyWordSpec with Matchers with TaskIndexKey { "own the partitions when max task is 1" in { val from = Map("a" -> "1", "b" -> "2", TASK_INDEX -> "0:1", "name" -> connectorName) val actual = - new ConnectorTaskIdCreator(CONNECTOR_PREFIX).fromProps(from.asJava).getOrElse(fail("Should be valid")) + new ConnectorTaskIdCreator(connectorPrefix).fromProps(from.asJava).getOrElse(fail("Should be valid")) Seq("/myTopic/", "/anotherTopic/", "/thirdTopic/") .flatMap { value => @@ -89,15 +85,15 @@ class ConnectorTaskIdTest extends AnyWordSpec with Matchers with TaskIndexKey { } "distribute the directory between two tasks" in { - val one = new ConnectorTaskIdCreator(CONNECTOR_PREFIX).fromProps(Map("a" -> "1", - "b" -> "2", - TASK_INDEX -> "0:2", - "name" -> connectorName, + val one = new ConnectorTaskIdCreator(connectorPrefix).fromProps(Map("a" -> "1", + "b" -> "2", + TASK_INDEX -> "0:2", + "name" -> connectorName, ).asJava).getOrElse(fail("Should be valid")) - val two = new ConnectorTaskIdCreator(CONNECTOR_PREFIX).fromProps(Map("a" -> "1", - "b" -> "2", - TASK_INDEX -> "1:2", - "name" -> connectorName, + val two = new ConnectorTaskIdCreator(connectorPrefix).fromProps(Map("a" -> "1", + "b" -> "2", + TASK_INDEX -> "1:2", + "name" -> connectorName, ).asJava).getOrElse(fail("Should be valid")) PartitionHasher.hash(2, "1") shouldBe 1 @@ -111,20 +107,20 @@ class ConnectorTaskIdTest extends AnyWordSpec with Matchers with TaskIndexKey { "distribute the directories between three tasks" in { - val one = new ConnectorTaskIdCreator(CONNECTOR_PREFIX).fromProps(Map("a" -> "1", - "b" -> "2", - TASK_INDEX -> "0:3", - "name" -> connectorName, + val one = new ConnectorTaskIdCreator(connectorPrefix).fromProps(Map("a" -> "1", + "b" -> "2", + TASK_INDEX -> "0:3", + "name" -> connectorName, ).asJava).getOrElse(fail("Should be valid")) - val two = new ConnectorTaskIdCreator(CONNECTOR_PREFIX).fromProps(Map("a" -> "1", - "b" -> "2", - TASK_INDEX -> "1:3", - "name" -> connectorName, + val two = new ConnectorTaskIdCreator(connectorPrefix).fromProps(Map("a" -> "1", + "b" -> "2", + TASK_INDEX -> "1:3", + "name" -> connectorName, ).asJava).getOrElse(fail("Should be valid")) - val three = new ConnectorTaskIdCreator(CONNECTOR_PREFIX).fromProps(Map("a" -> "1", - "b" -> "2", - TASK_INDEX -> "2:3", - "name" -> connectorName, + val three = new ConnectorTaskIdCreator(connectorPrefix).fromProps(Map("a" -> "1", + "b" -> "2", + TASK_INDEX -> "2:3", + "name" -> connectorName, ).asJava).getOrElse(fail("Should be valid")) PartitionHasher.hash(3, "1") shouldBe 1 @@ -144,5 +140,5 @@ class ConnectorTaskIdTest extends AnyWordSpec with Matchers with TaskIndexKey { } } - override def connectorPrefix: String = CONNECTOR_PREFIX + override def connectorPrefix: String = "connect.testing" } diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/config/DataStorageSettingsTests.scala b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/config/DataStorageSettingsTests.scala similarity index 96% rename from kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/config/DataStorageSettingsTests.scala rename to kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/config/DataStorageSettingsTests.scala index d8c6b77909..48bccbb570 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/config/DataStorageSettingsTests.scala +++ b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/config/DataStorageSettingsTests.scala @@ -13,10 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.config +package io.lenses.streamreactor.connect.cloud.common.config -import io.lenses.streamreactor.connect.cloud.config.DataStorageSettings -import io.lenses.streamreactor.connect.cloud.config.DataStorageSettings.AllEnvelopeFields +import DataStorageSettings.AllEnvelopeFields import org.apache.kafka.common.config.ConfigException import org.scalatest.Assertion import org.scalatest.funsuite.AnyFunSuite diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/config/FormatSelectionTest.scala b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/config/FormatSelectionTest.scala similarity index 78% rename from kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/config/FormatSelectionTest.scala rename to kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/config/FormatSelectionTest.scala index 213c8947b2..2f372d030a 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/config/FormatSelectionTest.scala +++ b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/config/FormatSelectionTest.scala @@ -13,13 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.config +package io.lenses.streamreactor.connect.cloud.common.config -import io.lenses.streamreactor.connect.cloud.config.FormatOptions.WithHeaders -import io.lenses.streamreactor.connect.cloud.model.CompressionCodecName.UNCOMPRESSED -import io.lenses.streamreactor.connect.cloud.config.CsvFormatSelection -import io.lenses.streamreactor.connect.cloud.config.FormatSelection -import io.lenses.streamreactor.connect.cloud.model.CompressionCodec +import FormatOptions.WithHeaders +import io.lenses.streamreactor.connect.cloud.common.model.CompressionCodec +import io.lenses.streamreactor.connect.cloud.common.model.CompressionCodecName.UNCOMPRESSED import org.scalatest.EitherValues import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/config/TaskDistributorTest.scala b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/config/TaskDistributorTest.scala similarity index 69% rename from kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/config/TaskDistributorTest.scala rename to kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/config/TaskDistributorTest.scala index d242bc2ef7..03157ef856 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/config/TaskDistributorTest.scala +++ b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/config/TaskDistributorTest.scala @@ -13,11 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.config +package io.lenses.streamreactor.connect.cloud.common.config -import io.lenses.streamreactor.connect.aws.s3.config.S3ConfigSettings.CONNECTOR_PREFIX -import io.lenses.streamreactor.connect.cloud.config.TaskDistributor -import io.lenses.streamreactor.connect.cloud.config.TaskIndexKey import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec @@ -26,7 +23,7 @@ class TaskDistributorTest extends AnyWordSpec with Matchers with TaskIndexKey { "TaskDistributor" should { "distribute tasks" in { val tasks = - new TaskDistributor(CONNECTOR_PREFIX).distributeTasks(Map.empty[String, String].asJava, 5).asScala.map( + new TaskDistributor(connectorPrefix).distributeTasks(Map.empty[String, String].asJava, 5).asScala.map( _.asScala.toMap, ).toList tasks shouldBe List( @@ -39,12 +36,12 @@ class TaskDistributorTest extends AnyWordSpec with Matchers with TaskIndexKey { } "handle 0 max tasks" in { val tasks = - new TaskDistributor(CONNECTOR_PREFIX).distributeTasks(Map.empty[String, String].asJava, 0).asScala.map( + new TaskDistributor(connectorPrefix).distributeTasks(Map.empty[String, String].asJava, 0).asScala.map( _.asScala.toMap, ).toList tasks shouldBe List() } } - override def connectorPrefix: String = CONNECTOR_PREFIX + override def connectorPrefix: String = "connect.testing" } diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/formats/AvroFormatReader.scala b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/formats/AvroFormatReader.scala similarity index 93% rename from kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/formats/AvroFormatReader.scala rename to kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/formats/AvroFormatReader.scala index c2f0c7c0b1..cba96a07e4 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/formats/AvroFormatReader.scala +++ b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/formats/AvroFormatReader.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.formats +package io.lenses.streamreactor.connect.cloud.common.formats /* * Copyright 2020 Lenses.io @@ -31,7 +31,7 @@ package io.lenses.streamreactor.connect.aws.s3.formats * limitations under the License. */ -import io.lenses.streamreactor.connect.cloud.formats.reader.Using +import io.lenses.streamreactor.connect.cloud.common.formats.reader.Using import org.apache.avro.file.DataFileReader import org.apache.avro.file.SeekableByteArrayInput import org.apache.avro.generic.GenericDatumReader diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/formats/AvroFormatStreamReaderTest.scala b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/formats/AvroFormatStreamReaderTest.scala similarity index 74% rename from kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/formats/AvroFormatStreamReaderTest.scala rename to kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/formats/AvroFormatStreamReaderTest.scala index c19fa7ba27..56fde86858 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/formats/AvroFormatStreamReaderTest.scala +++ b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/formats/AvroFormatStreamReaderTest.scala @@ -13,19 +13,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.formats +package io.lenses.streamreactor.connect.cloud.common.formats -import io.lenses.streamreactor.connect.aws.s3.stream.S3ByteArrayOutputStream -import io.lenses.streamreactor.connect.aws.s3.utils.SampleData -import io.lenses.streamreactor.connect.aws.s3.utils.SampleData.topic -import io.lenses.streamreactor.connect.cloud.formats.reader.AvroStreamReader -import io.lenses.streamreactor.connect.cloud.formats.writer.AvroFormatWriter -import io.lenses.streamreactor.connect.cloud.formats.writer.MessageDetail -import io.lenses.streamreactor.connect.cloud.formats.writer.NullSinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.StructSinkData -import io.lenses.streamreactor.connect.cloud.model.CompressionCodec -import io.lenses.streamreactor.connect.cloud.model.Offset -import io.lenses.streamreactor.connect.cloud.model.CompressionCodecName.UNCOMPRESSED +import io.lenses.streamreactor.connect.cloud.common.formats.reader.AvroStreamReader +import io.lenses.streamreactor.connect.cloud.common.formats.writer.AvroFormatWriter +import io.lenses.streamreactor.connect.cloud.common.formats.writer.MessageDetail +import io.lenses.streamreactor.connect.cloud.common.formats.writer.NullSinkData +import io.lenses.streamreactor.connect.cloud.common.formats.writer.StructSinkData +import io.lenses.streamreactor.connect.cloud.common.model.CompressionCodec +import io.lenses.streamreactor.connect.cloud.common.model.Offset +import io.lenses.streamreactor.connect.cloud.common.model.CompressionCodecName.UNCOMPRESSED +import io.lenses.streamreactor.connect.cloud.common.stream.S3ByteArrayOutputStream +import io.lenses.streamreactor.connect.cloud.common.utils.SampleData +import io.lenses.streamreactor.connect.cloud.common.utils.SampleData.topic import org.apache.kafka.connect.data.Struct import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/formats/AvroFormatWriterTest.scala b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/formats/AvroFormatWriterTest.scala similarity index 90% rename from kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/formats/AvroFormatWriterTest.scala rename to kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/formats/AvroFormatWriterTest.scala index 0fe819b2ba..4deabfb99d 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/formats/AvroFormatWriterTest.scala +++ b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/formats/AvroFormatWriterTest.scala @@ -13,22 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.formats - -import io.lenses.streamreactor.connect.aws.s3.stream.S3ByteArrayOutputStream -import io.lenses.streamreactor.connect.aws.s3.utils.SampleData -import io.lenses.streamreactor.connect.aws.s3.utils.SampleData._ -import io.lenses.streamreactor.connect.cloud.formats.writer.ArraySinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.AvroFormatWriter -import io.lenses.streamreactor.connect.cloud.formats.writer.ByteArraySinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.IntSinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.MapSinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.MessageDetail -import io.lenses.streamreactor.connect.cloud.formats.writer.NullSinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.StructSinkData -import io.lenses.streamreactor.connect.cloud.model.CompressionCodec -import io.lenses.streamreactor.connect.cloud.model.Offset -import io.lenses.streamreactor.connect.cloud.model.CompressionCodecName.UNCOMPRESSED +package io.lenses.streamreactor.connect.cloud.common.formats + +import io.lenses.streamreactor.connect.cloud.common.stream.S3ByteArrayOutputStream +import io.lenses.streamreactor.connect.cloud.common.utils.SampleData +import io.lenses.streamreactor.connect.cloud.common.utils.SampleData._ +import io.lenses.streamreactor.connect.cloud.common.formats.writer._ +import io.lenses.streamreactor.connect.cloud.common.model.CompressionCodec +import io.lenses.streamreactor.connect.cloud.common.model.Offset +import io.lenses.streamreactor.connect.cloud.common.model.CompressionCodecName.UNCOMPRESSED import org.apache.avro.generic.GenericData import org.apache.avro.util.Utf8 import org.apache.kafka.connect.data.Schema diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/formats/BytesFormatStreamFileReaderTest.scala b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/formats/BytesFormatStreamFileReaderTest.scala similarity index 88% rename from kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/formats/BytesFormatStreamFileReaderTest.scala rename to kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/formats/BytesFormatStreamFileReaderTest.scala index 02185b478a..47e3daedbd 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/formats/BytesFormatStreamFileReaderTest.scala +++ b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/formats/BytesFormatStreamFileReaderTest.scala @@ -13,10 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.formats +package io.lenses.streamreactor.connect.cloud.common.formats -import io.lenses.streamreactor.connect.cloud.formats.bytes.BytesOutputRow -import io.lenses.streamreactor.connect.cloud.formats.reader.BytesStreamFileReader +import io.lenses.streamreactor.connect.cloud.common.formats.bytes.BytesOutputRow +import io.lenses.streamreactor.connect.cloud.common.formats.reader.BytesStreamFileReader import org.mockito.MockitoSugar import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/formats/BytesFormatWriterTest.scala b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/formats/BytesFormatWriterTest.scala similarity index 86% rename from kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/formats/BytesFormatWriterTest.scala rename to kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/formats/BytesFormatWriterTest.scala index ebca4f21e7..826966c079 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/formats/BytesFormatWriterTest.scala +++ b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/formats/BytesFormatWriterTest.scala @@ -13,18 +13,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.formats - -import io.lenses.streamreactor.connect.aws.s3.stream.S3ByteArrayOutputStream -import io.lenses.streamreactor.connect.aws.s3.utils.SampleData -import io.lenses.streamreactor.connect.aws.s3.utils.SampleData.topic -import io.lenses.streamreactor.connect.cloud.formats.writer.ByteArraySinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.BytesFormatWriter -import io.lenses.streamreactor.connect.cloud.formats.writer.MessageDetail -import io.lenses.streamreactor.connect.cloud.formats.writer.NullSinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.StructSinkData -import io.lenses.streamreactor.connect.cloud.model.Offset -import io.lenses.streamreactor.connect.cloud.model.Topic +package io.lenses.streamreactor.connect.cloud.common.formats + +import io.lenses.streamreactor.connect.cloud.common.stream.S3ByteArrayOutputStream +import io.lenses.streamreactor.connect.cloud.common.utils.SampleData +import io.lenses.streamreactor.connect.cloud.common.utils.SampleData.topic +import io.lenses.streamreactor.connect.cloud.common.formats.writer.ByteArraySinkData +import io.lenses.streamreactor.connect.cloud.common.formats.writer.BytesFormatWriter +import io.lenses.streamreactor.connect.cloud.common.formats.writer.MessageDetail +import io.lenses.streamreactor.connect.cloud.common.formats.writer.NullSinkData +import io.lenses.streamreactor.connect.cloud.common.formats.writer.StructSinkData +import io.lenses.streamreactor.connect.cloud.common.model.Offset +import io.lenses.streamreactor.connect.cloud.common.model.Topic import org.apache.commons.io.IOUtils import org.scalatest.EitherValues import org.scalatest.flatspec.AnyFlatSpec diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/formats/CsvFormatStreamReaderTest.scala b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/formats/CsvFormatStreamReaderTest.scala similarity index 91% rename from kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/formats/CsvFormatStreamReaderTest.scala rename to kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/formats/CsvFormatStreamReaderTest.scala index 0690c0a6dc..463dac1eae 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/formats/CsvFormatStreamReaderTest.scala +++ b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/formats/CsvFormatStreamReaderTest.scala @@ -13,11 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.formats +package io.lenses.streamreactor.connect.cloud.common.formats -import io.lenses.streamreactor.connect.aws.s3.utils.SampleData -import io.lenses.streamreactor.connect.cloud.formats.FormatWriterException -import io.lenses.streamreactor.connect.cloud.formats.reader.CsvStreamReader +import io.lenses.streamreactor.connect.cloud.common.formats.reader.CsvStreamReader +import io.lenses.streamreactor.connect.cloud.common.utils.SampleData import org.mockito.MockitoSugar import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/formats/CsvFormatWriterTest.scala b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/formats/CsvFormatWriterTest.scala similarity index 89% rename from kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/formats/CsvFormatWriterTest.scala rename to kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/formats/CsvFormatWriterTest.scala index eeeca9a6b1..25e8947fd0 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/formats/CsvFormatWriterTest.scala +++ b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/formats/CsvFormatWriterTest.scala @@ -13,19 +13,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.formats +package io.lenses.streamreactor.connect.cloud.common.formats import com.opencsv.CSVReader -import io.lenses.streamreactor.connect.aws.s3.stream.S3ByteArrayOutputStream -import io.lenses.streamreactor.connect.aws.s3.utils.SampleData -import io.lenses.streamreactor.connect.aws.s3.utils.SampleData.topic -import io.lenses.streamreactor.connect.cloud.formats.writer.CsvFormatWriter -import io.lenses.streamreactor.connect.cloud.formats.writer.MessageDetail -import io.lenses.streamreactor.connect.cloud.formats.writer.NullSinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.StructSinkData -import io.lenses.streamreactor.connect.cloud.model.Offset -import io.lenses.streamreactor.connect.cloud.sink.extractors.ExtractorError -import io.lenses.streamreactor.connect.cloud.sink.extractors.ExtractorErrorType.UnexpectedType +import io.lenses.streamreactor.connect.cloud.common.formats.writer.CsvFormatWriter +import io.lenses.streamreactor.connect.cloud.common.formats.writer.MessageDetail +import io.lenses.streamreactor.connect.cloud.common.formats.writer.NullSinkData +import io.lenses.streamreactor.connect.cloud.common.formats.writer.StructSinkData +import io.lenses.streamreactor.connect.cloud.common.model.Offset +import io.lenses.streamreactor.connect.cloud.common.sink.extractors.ExtractorError +import io.lenses.streamreactor.connect.cloud.common.sink.extractors.ExtractorErrorType.UnexpectedType +import io.lenses.streamreactor.connect.cloud.common.stream.S3ByteArrayOutputStream +import io.lenses.streamreactor.connect.cloud.common.utils.SampleData +import io.lenses.streamreactor.connect.cloud.common.utils.SampleData.topic import org.apache.kafka.connect.data.Schema.STRING_SCHEMA import org.apache.kafka.connect.data.Schema import org.apache.kafka.connect.data.SchemaBuilder diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/formats/JsonFormatWriterTest.scala b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/formats/JsonFormatWriterTest.scala similarity index 89% rename from kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/formats/JsonFormatWriterTest.scala rename to kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/formats/JsonFormatWriterTest.scala index f2555207d0..44e35ac837 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/formats/JsonFormatWriterTest.scala +++ b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/formats/JsonFormatWriterTest.scala @@ -13,22 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.formats +package io.lenses.streamreactor.connect.cloud.common.formats import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.databind.node.NullNode -import io.lenses.streamreactor.connect.aws.s3.stream.S3ByteArrayOutputStream -import io.lenses.streamreactor.connect.aws.s3.utils.SampleData -import io.lenses.streamreactor.connect.aws.s3.utils.SampleData.recordsAsJson -import io.lenses.streamreactor.connect.aws.s3.utils.SampleData.topic -import io.lenses.streamreactor.connect.cloud.formats.writer.ArraySinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.JsonFormatWriter -import io.lenses.streamreactor.connect.cloud.formats.writer.MapSinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.MessageDetail -import io.lenses.streamreactor.connect.cloud.formats.writer.NullSinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.StringSinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.StructSinkData -import io.lenses.streamreactor.connect.cloud.model.Offset +import io.lenses.streamreactor.connect.cloud.common.formats.writer._ +import io.lenses.streamreactor.connect.cloud.common.model.Offset +import io.lenses.streamreactor.connect.cloud.common.stream.S3ByteArrayOutputStream +import io.lenses.streamreactor.connect.cloud.common.utils.SampleData +import io.lenses.streamreactor.connect.cloud.common.utils.SampleData.recordsAsJson +import io.lenses.streamreactor.connect.cloud.common.utils.SampleData.topic import org.apache.kafka.connect.data.SchemaBuilder import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/formats/ParquetFormatStreamReaderTest.scala b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/formats/ParquetFormatStreamReaderTest.scala similarity index 92% rename from kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/formats/ParquetFormatStreamReaderTest.scala rename to kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/formats/ParquetFormatStreamReaderTest.scala index 30af202e1e..4648f897c1 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/formats/ParquetFormatStreamReaderTest.scala +++ b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/formats/ParquetFormatStreamReaderTest.scala @@ -13,9 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.formats +package io.lenses.streamreactor.connect.cloud.common.formats -import io.lenses.streamreactor.connect.cloud.formats.reader.ParquetStreamReader +import io.lenses.streamreactor.connect.cloud.common.formats.reader.ParquetStreamReader import org.apache.kafka.connect.data.Struct import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/formats/TextFormatStreamReaderTest.scala b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/formats/TextFormatStreamReaderTest.scala similarity index 74% rename from kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/formats/TextFormatStreamReaderTest.scala rename to kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/formats/TextFormatStreamReaderTest.scala index 6014a15cf7..159929e01c 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/formats/TextFormatStreamReaderTest.scala +++ b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/formats/TextFormatStreamReaderTest.scala @@ -13,17 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.formats +package io.lenses.streamreactor.connect.cloud.common.formats -import io.lenses.streamreactor.connect.aws.s3.stream.S3ByteArrayOutputStream -import io.lenses.streamreactor.connect.aws.s3.utils.SampleData -import io.lenses.streamreactor.connect.aws.s3.utils.SampleData.topic -import io.lenses.streamreactor.connect.cloud.formats.reader.TextStreamReader -import io.lenses.streamreactor.connect.cloud.formats.writer.JsonFormatWriter -import io.lenses.streamreactor.connect.cloud.formats.writer.MessageDetail -import io.lenses.streamreactor.connect.cloud.formats.writer.NullSinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.StructSinkData -import io.lenses.streamreactor.connect.cloud.model.Offset +import io.lenses.streamreactor.connect.cloud.common.formats.reader.TextStreamReader +import io.lenses.streamreactor.connect.cloud.common.formats.writer.JsonFormatWriter +import io.lenses.streamreactor.connect.cloud.common.formats.writer.MessageDetail +import io.lenses.streamreactor.connect.cloud.common.formats.writer.NullSinkData +import io.lenses.streamreactor.connect.cloud.common.formats.writer.StructSinkData +import io.lenses.streamreactor.connect.cloud.common.model.Offset +import io.lenses.streamreactor.connect.cloud.common.stream.S3ByteArrayOutputStream +import io.lenses.streamreactor.connect.cloud.common.utils.SampleData +import io.lenses.streamreactor.connect.cloud.common.utils.SampleData.topic import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/formats/TextFormatWriterTest.scala b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/formats/TextFormatWriterTest.scala similarity index 82% rename from kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/formats/TextFormatWriterTest.scala rename to kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/formats/TextFormatWriterTest.scala index cafc9013ee..77120698d9 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/formats/TextFormatWriterTest.scala +++ b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/formats/TextFormatWriterTest.scala @@ -13,18 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.formats +package io.lenses.streamreactor.connect.cloud.common.formats -import io.lenses.streamreactor.connect.aws.s3.stream.S3ByteArrayOutputStream -import io.lenses.streamreactor.connect.aws.s3.utils.SampleData -import io.lenses.streamreactor.connect.aws.s3.utils.SampleData.topic -import io.lenses.streamreactor.connect.cloud.formats.FormatWriterException -import io.lenses.streamreactor.connect.cloud.formats.writer.MessageDetail -import io.lenses.streamreactor.connect.cloud.formats.writer.NullSinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.StringSinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.StructSinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.TextFormatWriter -import io.lenses.streamreactor.connect.cloud.model.Offset +import io.lenses.streamreactor.connect.cloud.common.formats.writer._ +import io.lenses.streamreactor.connect.cloud.common.model.Offset +import io.lenses.streamreactor.connect.cloud.common.stream.S3ByteArrayOutputStream +import io.lenses.streamreactor.connect.cloud.common.utils.SampleData +import io.lenses.streamreactor.connect.cloud.common.utils.SampleData.topic import org.scalatest.EitherValues import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/formats/parquet/ParquetSeekableInputStreamTest.scala b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/formats/parquet/ParquetSeekableInputStreamTest.scala similarity index 91% rename from kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/formats/parquet/ParquetSeekableInputStreamTest.scala rename to kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/formats/parquet/ParquetSeekableInputStreamTest.scala index 356d35f746..73692d781e 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/formats/parquet/ParquetSeekableInputStreamTest.scala +++ b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/formats/parquet/ParquetSeekableInputStreamTest.scala @@ -13,9 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.formats.parquet +package io.lenses.streamreactor.connect.cloud.common.formats.parquet -import io.lenses.streamreactor.connect.cloud.formats.reader.parquet.ParquetSeekableInputStream +import io.lenses.streamreactor.connect.cloud.common.formats.reader.parquet.ParquetSeekableInputStream import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/formats/parquet/ParquetStreamingInputFileTest.scala b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/formats/parquet/ParquetStreamingInputFileTest.scala similarity index 85% rename from kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/formats/parquet/ParquetStreamingInputFileTest.scala rename to kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/formats/parquet/ParquetStreamingInputFileTest.scala index b39022ddf0..aa9c41efc2 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/formats/parquet/ParquetStreamingInputFileTest.scala +++ b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/formats/parquet/ParquetStreamingInputFileTest.scala @@ -13,10 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.formats.parquet +package io.lenses.streamreactor.connect.cloud.common.formats.parquet -import io.lenses.streamreactor.connect.cloud.formats.reader.parquet.ParquetSeekableInputStream -import io.lenses.streamreactor.connect.cloud.formats.reader.parquet.ParquetStreamingInputFile +import io.lenses.streamreactor.connect.cloud.common.formats.reader.parquet.ParquetSeekableInputStream +import io.lenses.streamreactor.connect.cloud.common.formats.reader.parquet.ParquetStreamingInputFile import org.mockito.MockitoSugar import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/formats/reader/DelegateIteratorS3StreamReaderTest.scala b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/DelegateIteratorS3StreamReaderTest.scala similarity index 92% rename from kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/formats/reader/DelegateIteratorS3StreamReaderTest.scala rename to kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/DelegateIteratorS3StreamReaderTest.scala index 5c666f4e4a..bca82fd164 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/formats/reader/DelegateIteratorS3StreamReaderTest.scala +++ b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/DelegateIteratorS3StreamReaderTest.scala @@ -13,11 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.formats.reader +package io.lenses.streamreactor.connect.cloud.common.formats.reader -import io.lenses.streamreactor.connect.cloud.formats.reader.Converter -import io.lenses.streamreactor.connect.cloud.formats.reader.DelegateIteratorS3StreamReader -import io.lenses.streamreactor.connect.cloud.formats.reader.S3DataIterator +import io.lenses.streamreactor.connect.cloud.common.formats.reader.Converter +import io.lenses.streamreactor.connect.cloud.common.formats.reader.DelegateIteratorS3StreamReader +import io.lenses.streamreactor.connect.cloud.common.formats.reader.S3DataIterator import org.apache.kafka.connect.data.Schema import org.apache.kafka.connect.source.SourceRecord import org.scalatest.funsuite.AnyFunSuite diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/formats/reader/converters/BytesOutputRowConverterTest.scala b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/converters/BytesOutputRowConverterTest.scala similarity index 77% rename from kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/formats/reader/converters/BytesOutputRowConverterTest.scala rename to kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/converters/BytesOutputRowConverterTest.scala index 78c4eec85d..144c77fcf6 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/formats/reader/converters/BytesOutputRowConverterTest.scala +++ b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/converters/BytesOutputRowConverterTest.scala @@ -13,15 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.formats.reader.converters +package io.lenses.streamreactor.connect.cloud.common.formats.reader.converters import cats.implicits.catsSyntaxOptionId -import io.lenses.streamreactor.connect.aws.s3.model.location.S3LocationValidator -import io.lenses.streamreactor.connect.cloud.formats.bytes.BytesOutputRow -import io.lenses.streamreactor.connect.cloud.formats.reader.converters.BytesOutputRowConverter -import io.lenses.streamreactor.connect.cloud.model.Topic -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocation -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocationValidator +import io.lenses.streamreactor.connect.cloud.common.formats.bytes.BytesOutputRow +import io.lenses.streamreactor.connect.cloud.common.model.Topic +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocation +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocationValidator +import io.lenses.streamreactor.connect.cloud.common.utils.SampleData import org.apache.kafka.connect.data.Schema import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers @@ -30,7 +29,9 @@ import java.time.Instant import scala.jdk.CollectionConverters.MapHasAsJava class BytesOutputRowConverterTest extends AnyFunSuite with Matchers { - implicit val cloudLocationValidator: CloudLocationValidator = S3LocationValidator + + implicit val cloudLocationValidator: CloudLocationValidator = SampleData.cloudLocationValidator + test("convert to SourceRecord and populate the key and value") { val location = CloudLocation("bucket", "myprefix".some, "a/b/c.txt".some) val lastModified = Instant.ofEpochMilli(10001) diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/formats/reader/converters/SchemaAndValueConverterTest.scala b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/converters/SchemaAndValueConverterTest.scala similarity index 80% rename from kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/formats/reader/converters/SchemaAndValueConverterTest.scala rename to kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/converters/SchemaAndValueConverterTest.scala index 3f8b86c842..6c7b4e0a01 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/formats/reader/converters/SchemaAndValueConverterTest.scala +++ b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/converters/SchemaAndValueConverterTest.scala @@ -13,14 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.formats.reader.converters +package io.lenses.streamreactor.connect.cloud.common.formats.reader.converters import cats.implicits.catsSyntaxOptionId -import io.lenses.streamreactor.connect.aws.s3.model.location.S3LocationValidator -import io.lenses.streamreactor.connect.cloud.formats.reader.converters.SchemaAndValueConverter -import io.lenses.streamreactor.connect.cloud.model.Topic -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocation -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocationValidator +import io.lenses.streamreactor.connect.cloud.common.model.Topic +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocation +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocationValidator +import io.lenses.streamreactor.connect.cloud.common.utils.SampleData import org.apache.kafka.connect.data.Schema import org.apache.kafka.connect.data.SchemaAndValue import org.scalatest.funsuite.AnyFunSuite @@ -30,7 +29,7 @@ import java.time.Instant import scala.jdk.CollectionConverters.MapHasAsJava class SchemaAndValueConverterTest extends AnyFunSuite with Matchers { - implicit val cloudLocationValidator: CloudLocationValidator = S3LocationValidator + implicit val cloudLocationValidator: CloudLocationValidator = SampleData.cloudLocationValidator test("converts a schema and value and populates the source record") { val value = new SchemaAndValue( diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/formats/reader/converters/SchemaAndValueEnvelopeConverterTest.scala b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/converters/SchemaAndValueEnvelopeConverterTest.scala similarity index 95% rename from kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/formats/reader/converters/SchemaAndValueEnvelopeConverterTest.scala rename to kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/converters/SchemaAndValueEnvelopeConverterTest.scala index d56b175f80..768341e8f8 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/formats/reader/converters/SchemaAndValueEnvelopeConverterTest.scala +++ b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/converters/SchemaAndValueEnvelopeConverterTest.scala @@ -13,14 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.formats.reader.converters +package io.lenses.streamreactor.connect.cloud.common.formats.reader.converters import cats.implicits.catsSyntaxOptionId -import io.lenses.streamreactor.connect.aws.s3.model.location.S3LocationValidator -import io.lenses.streamreactor.connect.cloud.formats.reader.converters.SchemaAndValueEnvelopeConverter -import io.lenses.streamreactor.connect.cloud.model.Topic -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocation -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocationValidator +import io.lenses.streamreactor.connect.cloud.common.model.Topic +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocation +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocationValidator +import io.lenses.streamreactor.connect.cloud.common.utils.SampleData import org.apache.kafka.connect.data.Schema import org.apache.kafka.connect.data.SchemaAndValue import org.apache.kafka.connect.data.SchemaBuilder @@ -36,7 +35,7 @@ import scala.jdk.CollectionConverters.MapHasAsJava import scala.jdk.CollectionConverters.MapHasAsScala class SchemaAndValueEnvelopeConverterTest extends AnyFunSuite with Matchers { - implicit val cloudLocationValidator: CloudLocationValidator = S3LocationValidator + implicit val cloudLocationValidator: CloudLocationValidator = SampleData.cloudLocationValidator private val HeadersSchema = SchemaBuilder.struct() .field("header1", Schema.STRING_SCHEMA) diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/formats/reader/converters/SchemalessEnvelopeConverterTest.scala b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/converters/SchemalessEnvelopeConverterTest.scala similarity index 95% rename from kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/formats/reader/converters/SchemalessEnvelopeConverterTest.scala rename to kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/converters/SchemalessEnvelopeConverterTest.scala index fcd94b3370..100945f42d 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/formats/reader/converters/SchemalessEnvelopeConverterTest.scala +++ b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/converters/SchemalessEnvelopeConverterTest.scala @@ -13,15 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.formats.reader.converters +package io.lenses.streamreactor.connect.cloud.common.formats.reader.converters import cats.implicits.catsSyntaxOptionId import io.circe.Json -import io.lenses.streamreactor.connect.aws.s3.model.location.S3LocationValidator -import io.lenses.streamreactor.connect.cloud.formats.reader.converters.SchemalessEnvelopeConverter -import io.lenses.streamreactor.connect.cloud.model.Topic -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocation -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocationValidator +import io.lenses.streamreactor.connect.cloud.common.model.Topic +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocation +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocationValidator +import io.lenses.streamreactor.connect.cloud.common.utils.SampleData import org.apache.kafka.connect.data.Schema import org.apache.kafka.connect.source.SourceRecord import org.scalatest.Assertion @@ -35,7 +34,7 @@ import scala.jdk.CollectionConverters.MapHasAsJava import scala.jdk.CollectionConverters.MapHasAsScala class SchemalessEnvelopeConverterTest extends AnyFunSuite with Matchers { - implicit val cloudLocationValidator: CloudLocationValidator = S3LocationValidator + implicit val cloudLocationValidator: CloudLocationValidator = SampleData.cloudLocationValidator private val Header1Value = "header1" private val Header2Value = "header2" diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/formats/reader/converters/TextConverterTest.scala b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/converters/TextConverterTest.scala similarity index 78% rename from kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/formats/reader/converters/TextConverterTest.scala rename to kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/converters/TextConverterTest.scala index 544348c64a..f8b968ea89 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/formats/reader/converters/TextConverterTest.scala +++ b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/formats/reader/converters/TextConverterTest.scala @@ -13,15 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.formats.reader.converters +package io.lenses.streamreactor.connect.cloud.common.formats.reader.converters import cats.implicits.catsSyntaxOptionId -import io.lenses.streamreactor.connect.aws.s3.model.location.S3LocationValidator -import io.lenses.streamreactor.connect.cloud.formats.reader.converters.TextConverter -import io.lenses.streamreactor.connect.cloud.model.Topic -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocation -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocationValidator +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocation +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocationValidator +import io.lenses.streamreactor.connect.cloud.common.utils.SampleData import org.apache.kafka.connect.data.Schema +import io.lenses.streamreactor.connect.cloud.common.model.Topic + import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers @@ -29,7 +29,7 @@ import java.time.Instant import scala.jdk.CollectionConverters.MapHasAsJava class TextConverterTest extends AnyFunSuite with Matchers { - implicit val cloudLocationValidator: CloudLocationValidator = S3LocationValidator + implicit val cloudLocationValidator: CloudLocationValidator = SampleData.cloudLocationValidator test("convert to SourceRecord setting the value and value schema") { val location = CloudLocation("bucket", "prefix".some, "a/b/c.txt".some) val lastModified = Instant.ofEpochMilli(1000) diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/model/PartitionFieldTest.scala b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/model/PartitionFieldTest.scala similarity index 87% rename from kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/model/PartitionFieldTest.scala rename to kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/model/PartitionFieldTest.scala index 9c848c25fd..2f55cfeb8c 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/model/PartitionFieldTest.scala +++ b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/model/PartitionFieldTest.scala @@ -13,14 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.model +package io.lenses.streamreactor.connect.cloud.common.model import com.datamountaineer.kcql.Kcql -import io.lenses.streamreactor.connect.cloud.sink.config.KeyPartitionField -import io.lenses.streamreactor.connect.cloud.sink.config.PartitionField -import io.lenses.streamreactor.connect.cloud.sink.config.PartitionNamePath -import io.lenses.streamreactor.connect.cloud.sink.config.ValuePartitionField -import io.lenses.streamreactor.connect.cloud.sink.config.WholeKeyPartitionField +import io.lenses.streamreactor.connect.cloud.common.sink.config._ import org.mockito.MockitoSugar import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/model/location/FileUtilsTest.scala b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/model/location/FileUtilsTest.scala similarity index 89% rename from kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/model/location/FileUtilsTest.scala rename to kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/model/location/FileUtilsTest.scala index 0f208e1b3a..aaa5adda37 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/model/location/FileUtilsTest.scala +++ b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/model/location/FileUtilsTest.scala @@ -13,9 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.model.location +package io.lenses.streamreactor.connect.cloud.common.model.location -import io.lenses.streamreactor.connect.cloud.model.location.FileUtils import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/PaddingStrategyTest.scala b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/PaddingStrategyTest.scala similarity index 80% rename from kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/PaddingStrategyTest.scala rename to kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/PaddingStrategyTest.scala index 1908e65359..d3b9c7d6e1 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/PaddingStrategyTest.scala +++ b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/PaddingStrategyTest.scala @@ -13,10 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.sink +package io.lenses.streamreactor.connect.cloud.common.sink -import io.lenses.streamreactor.connect.cloud.sink.config.padding.LeftPadPaddingStrategy -import io.lenses.streamreactor.connect.cloud.sink.config.padding.RightPadPaddingStrategy +import io.lenses.streamreactor.connect.cloud.common.sink.config.padding.LeftPadPaddingStrategy +import io.lenses.streamreactor.connect.cloud.common.sink.config.padding.RightPadPaddingStrategy import org.scalatest.flatspec.AnyFlatSpecLike import org.scalatest.matchers.should.Matchers diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/commit/CommitPolicyTest.scala b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/commit/CommitPolicyTest.scala similarity index 83% rename from kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/commit/CommitPolicyTest.scala rename to kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/commit/CommitPolicyTest.scala index 511f45cee9..d39b291809 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/commit/CommitPolicyTest.scala +++ b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/commit/CommitPolicyTest.scala @@ -13,17 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.sink.commit +package io.lenses.streamreactor.connect.cloud.common.sink.commit import com.typesafe.scalalogging.Logger -import io.lenses.streamreactor.connect.cloud.model.Offset -import io.lenses.streamreactor.connect.cloud.model.Topic -import io.lenses.streamreactor.connect.cloud.model.TopicPartitionOffset -import io.lenses.streamreactor.connect.cloud.sink.commit.CommitContext -import io.lenses.streamreactor.connect.cloud.sink.commit.CommitPolicy -import io.lenses.streamreactor.connect.cloud.sink.commit.Count -import io.lenses.streamreactor.connect.cloud.sink.commit.FileSize -import io.lenses.streamreactor.connect.cloud.sink.commit.Interval +import io.lenses.streamreactor.connect.cloud.common.model.Offset +import io.lenses.streamreactor.connect.cloud.common.model.Topic +import io.lenses.streamreactor.connect.cloud.common.model.TopicPartitionOffset +import io.lenses.streamreactor.connect.cloud.common.sink.commit.CommitContext +import io.lenses.streamreactor.connect.cloud.common.sink.commit.CommitPolicy +import io.lenses.streamreactor.connect.cloud.common.sink.commit.Count +import io.lenses.streamreactor.connect.cloud.common.sink.commit.FileSize +import io.lenses.streamreactor.connect.cloud.common.sink.commit.Interval import org.mockito.MockitoSugar import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/commit/CountTest.scala b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/commit/CountTest.scala similarity index 87% rename from kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/commit/CountTest.scala rename to kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/commit/CountTest.scala index 71af14a401..3f811eb144 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/commit/CountTest.scala +++ b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/commit/CountTest.scala @@ -13,13 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.sink.commit +package io.lenses.streamreactor.connect.cloud.common.sink.commit import cats.implicits.catsSyntaxOptionId import cats.implicits.none -import io.lenses.streamreactor.connect.cloud.sink.commit.CommitContext -import io.lenses.streamreactor.connect.cloud.sink.commit.ConditionCommitResult -import io.lenses.streamreactor.connect.cloud.sink.commit.Count +import io.lenses.streamreactor.connect.cloud.common.sink.commit.CommitContext +import io.lenses.streamreactor.connect.cloud.common.sink.commit.ConditionCommitResult +import io.lenses.streamreactor.connect.cloud.common.sink.commit.Count import org.mockito.MockitoSugar import org.scalatest.EitherValues import org.scalatest.flatspec.AnyFlatSpec diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/commit/FileSizeTest.scala b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/commit/FileSizeTest.scala similarity index 87% rename from kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/commit/FileSizeTest.scala rename to kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/commit/FileSizeTest.scala index a5781a9be2..449a090dde 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/commit/FileSizeTest.scala +++ b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/commit/FileSizeTest.scala @@ -13,13 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.sink.commit +package io.lenses.streamreactor.connect.cloud.common.sink.commit import cats.implicits.catsSyntaxOptionId import cats.implicits.none -import io.lenses.streamreactor.connect.cloud.sink.commit.CommitContext -import io.lenses.streamreactor.connect.cloud.sink.commit.ConditionCommitResult -import io.lenses.streamreactor.connect.cloud.sink.commit.FileSize +import io.lenses.streamreactor.connect.cloud.common.sink.commit.CommitContext +import io.lenses.streamreactor.connect.cloud.common.sink.commit.ConditionCommitResult +import io.lenses.streamreactor.connect.cloud.common.sink.commit.FileSize import org.mockito.MockitoSugar import org.scalatest.EitherValues import org.scalatest.flatspec.AnyFlatSpec diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/commit/IntervalTest.scala b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/commit/IntervalTest.scala similarity index 93% rename from kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/commit/IntervalTest.scala rename to kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/commit/IntervalTest.scala index b67b0e1be4..fcf6519bd3 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/commit/IntervalTest.scala +++ b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/commit/IntervalTest.scala @@ -13,13 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.sink.commit +package io.lenses.streamreactor.connect.cloud.common.sink.commit import cats.implicits.catsSyntaxOptionId import cats.implicits.none -import io.lenses.streamreactor.connect.cloud.sink.commit.CommitContext -import io.lenses.streamreactor.connect.cloud.sink.commit.ConditionCommitResult -import io.lenses.streamreactor.connect.cloud.sink.commit.Interval +import io.lenses.streamreactor.connect.cloud.common.sink.commit.CommitContext +import io.lenses.streamreactor.connect.cloud.common.sink.commit.ConditionCommitResult +import io.lenses.streamreactor.connect.cloud.common.sink.commit.Interval import org.mockito.MockitoSugar import org.scalatest.BeforeAndAfter import org.scalatest.EitherValues diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/config/LocalStagingAreaTest.scala b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/config/LocalStagingAreaTest.scala similarity index 70% rename from kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/config/LocalStagingAreaTest.scala rename to kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/config/LocalStagingAreaTest.scala index fed4ca313a..51862e509a 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/config/LocalStagingAreaTest.scala +++ b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/config/LocalStagingAreaTest.scala @@ -13,14 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.sink.config - -import io.lenses.streamreactor.connect.aws.s3.config.S3ConfigSettings.CONNECTOR_PREFIX -import io.lenses.streamreactor.connect.aws.s3.model.location.S3LocationValidator -import io.lenses.streamreactor.connect.cloud.config.ConnectorTaskId -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocationValidator -import io.lenses.streamreactor.connect.cloud.sink.config.LocalStagingArea -import io.lenses.streamreactor.connect.cloud.sink.config.LocalStagingAreaConfigKeys +package io.lenses.streamreactor.connect.cloud.common.sink.config + +import io.lenses.streamreactor.connect.cloud.common.config.ConnectorTaskId +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocationValidator +import io.lenses.streamreactor.connect.cloud.common.sink.config.LocalStagingArea +import io.lenses.streamreactor.connect.cloud.common.sink.config.LocalStagingAreaConfigKeys +import io.lenses.streamreactor.connect.cloud.common.utils.SampleData import org.scalatest.EitherValues import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -29,28 +28,29 @@ import java.io.File import java.nio.file.Files class LocalStagingAreaTest extends AnyFlatSpec with Matchers with EitherValues with LocalStagingAreaConfigKeys { - implicit val cloudLocationValidator: CloudLocationValidator = S3LocationValidator + implicit val cloudLocationValidator: CloudLocationValidator = SampleData.cloudLocationValidator private val tmpDir = Files.createTempDirectory("S3OutputStreamOptionsTest") + behavior of "LocalStagingArea" it should "create BuildLocalOutputStreamOptions when temp directory has been supplied" in { implicit val connectorTaskId: ConnectorTaskId = ConnectorTaskId("unusedSinkName", 1, 1) - TestConfigDefBuilder(LOCAL_TMP_DIRECTORY -> s"$tmpDir/my/path").getLocalStagingArea() should + TestConfig(LOCAL_TMP_DIRECTORY -> s"$tmpDir/my/path").getLocalStagingArea() should be(Right(LocalStagingArea(new File(s"$tmpDir/my/path")))) } it should "create BuildLocalOutputStreamOptions when temp directory and sink name has been supplied" in { implicit val connectorTaskId: ConnectorTaskId = ConnectorTaskId("unusedSinkName", 1, 1) // should ignore the sinkName - TestConfigDefBuilder(LOCAL_TMP_DIRECTORY -> s"$tmpDir/my/path").getLocalStagingArea() should + TestConfig(LOCAL_TMP_DIRECTORY -> s"$tmpDir/my/path").getLocalStagingArea() should be(Right(LocalStagingArea(new File(s"$tmpDir/my/path")))) } it should "create BuildLocalOutputStreamOptions when only sink name has been supplied" in { implicit val connectorTaskId: ConnectorTaskId = ConnectorTaskId("superSleekSinkName", 1, 1) val tempDir = System.getProperty("java.io.tmpdir") - val result = TestConfigDefBuilder().getLocalStagingArea() + val result = TestConfig().getLocalStagingArea() result.isRight should be(true) result.value match { case LocalStagingArea(file) => file.toString should startWith(s"$tempDir/superSleekSinkName".replace("//", "/")) @@ -58,5 +58,5 @@ class LocalStagingAreaTest extends AnyFlatSpec with Matchers with EitherValues w } } - override def connectorPrefix: String = CONNECTOR_PREFIX + override def connectorPrefix: String = "connect.testing" } diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/config/PartitionFieldSplitterTest.scala b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/config/PartitionFieldSplitterTest.scala similarity index 92% rename from kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/config/PartitionFieldSplitterTest.scala rename to kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/config/PartitionFieldSplitterTest.scala index f340ce9499..f403718fce 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/config/PartitionFieldSplitterTest.scala +++ b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/config/PartitionFieldSplitterTest.scala @@ -13,9 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.sink.config +package io.lenses.streamreactor.connect.cloud.common.sink.config -import io.lenses.streamreactor.connect.cloud.sink.config.PartitionFieldSplitter +import io.lenses.streamreactor.connect.cloud.common.sink.config.PartitionFieldSplitter import org.scalatest.flatspec.AnyFlatSpecLike import org.scalatest.matchers.should.Matchers diff --git a/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/config/TestConfigDefBuilder.scala b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/config/TestConfigDefBuilder.scala new file mode 100644 index 0000000000..4bc8da6ef7 --- /dev/null +++ b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/config/TestConfigDefBuilder.scala @@ -0,0 +1,57 @@ +/* + * Copyright 2017-2023 Lenses.io Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.lenses.streamreactor.connect.cloud.common.sink.config + +import com.datamountaineer.streamreactor.common.config.base.traits.BaseConfig +import io.lenses.streamreactor.connect.cloud.common.sink.config.padding.PaddingStrategyConfigKeys +import io.lenses.streamreactor.connect.cloud.common.sink.config.padding.PaddingStrategySettings +import io.lenses.streamreactor.connect.cloud.common.sink.config.LocalStagingAreaConfigKeys +import io.lenses.streamreactor.connect.cloud.common.sink.config.LocalStagingAreaSettings +import org.apache.kafka.common.config.ConfigDef + +import java.util +import scala.jdk.CollectionConverters.MapHasAsJava +import scala.jdk.CollectionConverters.MapHasAsScala + +case class TestConfigDefBuilder(configDef: ConfigDef, props: util.Map[String, String]) + extends BaseConfig("connect.testing", configDef, props) + with PaddingStrategySettings + with LocalStagingAreaSettings { + + def getParsedValues: Map[String, _] = values().asScala.toMap + +} + +object TestConfig { + + def apply(pairs: (String, String)*): TestConfigDefBuilder = { + + def defineProps(configDef: ConfigDef) = { + val keys = new PaddingStrategyConfigKeys with LocalStagingAreaConfigKeys { + override def connectorPrefix: String = "connect.testing" + } + keys.addPaddingToConfigDef(configDef) + keys.addLocalStagingAreaToConfigDef(configDef) + } + + val map: Map[String, String] = pairs.toMap + val newMap = map + { + "connect.s3.kcql" -> "dummy value" + } + TestConfigDefBuilder(defineProps(new ConfigDef()), newMap.asJava) + } + +} diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/config/padding/PaddingServiceTest.scala b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/config/padding/PaddingServiceTest.scala similarity index 81% rename from kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/config/padding/PaddingServiceTest.scala rename to kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/config/padding/PaddingServiceTest.scala index f0b9519a0d..2b01223bc2 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/config/padding/PaddingServiceTest.scala +++ b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/config/padding/PaddingServiceTest.scala @@ -13,15 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.sink.config.padding +package io.lenses.streamreactor.connect.cloud.common.sink.config.padding import cats.implicits.catsSyntaxOptionId import cats.implicits.none -import io.lenses.streamreactor.connect.aws.s3.sink.config.S3SinkConfigDefBuilder -import io.lenses.streamreactor.connect.cloud.sink.config.kcqlprops.S3SinkPropsSchema -import io.lenses.streamreactor.connect.cloud.sink.config.padding.PaddingService -import io.lenses.streamreactor.connect.cloud.sink.config.padding.PaddingType -import io.lenses.streamreactor.connect.cloud.sink.config.padding.RightPadPaddingStrategy +import io.lenses.streamreactor.connect.cloud.common.sink.config.kcqlprops.S3SinkPropsSchema +import io.lenses.streamreactor.connect.cloud.common.sink.config.padding.PaddingService +import io.lenses.streamreactor.connect.cloud.common.sink.config.padding.PaddingStrategySettings +import io.lenses.streamreactor.connect.cloud.common.sink.config.padding.PaddingType +import io.lenses.streamreactor.connect.cloud.common.sink.config.padding.RightPadPaddingStrategy import org.mockito.MockitoSugar._ import org.scalatest.EitherValues import org.scalatest.funsuite.AnyFunSuite @@ -52,7 +52,7 @@ class PaddingServiceTest extends AnyFunSuite with Matchers with EitherValues { } test("PaddingService should return an error when both KCQL properties and ConfigDef padding configurations are set") { - val rightPadConfigDef: S3SinkConfigDefBuilder = mockConfigDefPadding(RightPadPaddingStrategy(10, '-').some) + val rightPadConfigDef: PaddingStrategySettings = mockConfigDefPadding(RightPadPaddingStrategy(10, '-').some) val kcqlProps = S3SinkPropsSchema.schema.readProps( "padding.length.partition" -> "5", "padding.char" -> "#", @@ -77,7 +77,7 @@ class PaddingServiceTest extends AnyFunSuite with Matchers with EitherValues { } test("fromConfig should create a PaddingService from S3SinkConfigDefBuilder operating only on 'offset'") { - val rightPadConfigDef: S3SinkConfigDefBuilder = mockConfigDefPadding(RightPadPaddingStrategy(10, '-').some) + val rightPadConfigDef: PaddingStrategySettings = mockConfigDefPadding(RightPadPaddingStrategy(10, '-').some) val paddingService = PaddingService.fromConfig(rightPadConfigDef, emptyProps).getOrElse(fail("No padding service found")) @@ -86,7 +86,7 @@ class PaddingServiceTest extends AnyFunSuite with Matchers with EitherValues { } test("fromConfig should return a PaddingService with default values when no config is provided") { - val emptyConfigDef: S3SinkConfigDefBuilder = mockConfigDefPadding(none) + val emptyConfigDef: PaddingStrategySettings = mockConfigDefPadding(none) val paddingService = PaddingService.fromConfig(emptyConfigDef, emptyProps).getOrElse(fail("No padding service found")) @@ -95,7 +95,7 @@ class PaddingServiceTest extends AnyFunSuite with Matchers with EitherValues { } private def mockConfigDefPadding(maybePaddingStrategy: Option[RightPadPaddingStrategy]) = { - val configDef = mock[S3SinkConfigDefBuilder] + val configDef = mock[PaddingStrategySettings] when(configDef.getPaddingStrategy).thenReturn(maybePaddingStrategy) configDef } diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/config/padding/PaddingStrategySettingsTest.scala b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/config/padding/PaddingStrategySettingsTest.scala similarity index 68% rename from kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/config/padding/PaddingStrategySettingsTest.scala rename to kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/config/padding/PaddingStrategySettingsTest.scala index 41a2ee3c18..9117e18238 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/config/padding/PaddingStrategySettingsTest.scala +++ b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/config/padding/PaddingStrategySettingsTest.scala @@ -13,39 +13,39 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.sink.config.padding +package io.lenses.streamreactor.connect.cloud.common.sink.config.padding + import cats.implicits.catsSyntaxOptionId -import io.lenses.streamreactor.connect.aws.s3.config.S3ConfigSettings.CONNECTOR_PREFIX -import io.lenses.streamreactor.connect.aws.s3.sink.config.TestConfigDefBuilder -import io.lenses.streamreactor.connect.cloud.sink.config.padding.LeftPadPaddingStrategy -import io.lenses.streamreactor.connect.cloud.sink.config.padding.PaddingStrategyConfigKeys +import io.lenses.streamreactor.connect.cloud.common.sink.config.TestConfig +import io.lenses.streamreactor.connect.cloud.common.sink.config.padding.LeftPadPaddingStrategy +import io.lenses.streamreactor.connect.cloud.common.sink.config.padding.PaddingStrategyConfigKeys import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers class PaddingStrategySettingsTest extends AnyFunSuite with Matchers with PaddingStrategyConfigKeys { test("getPaddingStrategy should return None when padding strategy is not provided") { - val settings = TestConfigDefBuilder() + val settings = TestConfig() settings.getPaddingStrategy should equal(None) } test("getPaddingStrategy should return None when padding length is less than 0") { - val settings = TestConfigDefBuilder(PADDING_STRATEGY -> "-1") + val settings = TestConfig(PADDING_STRATEGY -> "-1") settings.getPaddingStrategy should equal(None) } test("getPaddingStrategy should return None when padding length is much less than 0") { - val settings = TestConfigDefBuilder(PADDING_LENGTH -> "-199") + val settings = TestConfig(PADDING_LENGTH -> "-199") settings.getPaddingStrategy should equal(None) } test("getPaddingStrategy should return the appropriate PaddingStrategy") { - val settings = TestConfigDefBuilder( + val settings = TestConfig( PADDING_LENGTH -> "12", PADDING_STRATEGY -> "LeftPad", ) settings.getPaddingStrategy should equal(LeftPadPaddingStrategy(12, '0').some) } - override def connectorPrefix: String = CONNECTOR_PREFIX + override def connectorPrefix: String = "connect.testing" } diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/conversion/ToJsonDataConverterTestFormatSelection.scala b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/conversion/ToJsonDataConverterTestFormatSelection.scala similarity index 89% rename from kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/conversion/ToJsonDataConverterTestFormatSelection.scala rename to kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/conversion/ToJsonDataConverterTestFormatSelection.scala index 0e997fd74c..197fe59c6b 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/conversion/ToJsonDataConverterTestFormatSelection.scala +++ b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/conversion/ToJsonDataConverterTestFormatSelection.scala @@ -13,10 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.sink.conversion +package io.lenses.streamreactor.connect.cloud.common.sink.conversion -import io.lenses.streamreactor.connect.cloud.formats.writer.MapSinkData -import io.lenses.streamreactor.connect.cloud.sink.conversion.ToJsonDataConverter +import io.lenses.streamreactor.connect.cloud.common.formats.writer.MapSinkData import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/conversion/ValueToSinkDataConverterTest.scala b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/conversion/ValueToSinkDataConverterTest.scala similarity index 87% rename from kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/conversion/ValueToSinkDataConverterTest.scala rename to kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/conversion/ValueToSinkDataConverterTest.scala index f0e3f2ae92..0cc31fcdc4 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/conversion/ValueToSinkDataConverterTest.scala +++ b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/conversion/ValueToSinkDataConverterTest.scala @@ -13,11 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.sink.conversion +package io.lenses.streamreactor.connect.cloud.common.sink.conversion -import io.lenses.streamreactor.connect.cloud.formats.writer.MapSinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.ShortSinkData -import io.lenses.streamreactor.connect.cloud.sink.conversion.ValueToSinkDataConverter +import io.lenses.streamreactor.connect.cloud.common.formats.writer._ import org.apache.kafka.connect.data.Schema import org.apache.kafka.connect.data.SchemaBuilder import org.apache.kafka.connect.data.Struct diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/cloud/sink/extractors/MapExtractorTest.scala b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/extractors/MapExtractorTest.scala similarity index 90% rename from kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/cloud/sink/extractors/MapExtractorTest.scala rename to kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/extractors/MapExtractorTest.scala index 1da52406ed..ac2604aadd 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/cloud/sink/extractors/MapExtractorTest.scala +++ b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/extractors/MapExtractorTest.scala @@ -13,9 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.cloud.sink.extractors +package io.lenses.streamreactor.connect.cloud.common.sink.extractors -import io.lenses.streamreactor.connect.cloud.sink.config.PartitionNamePath +import io.lenses.streamreactor.connect.cloud.common.sink.config.PartitionNamePath import org.apache.kafka.connect.data.SchemaBuilder import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/extractors/SinkDataExtractorTest.scala b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/extractors/SinkDataExtractorTest.scala similarity index 94% rename from kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/extractors/SinkDataExtractorTest.scala rename to kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/extractors/SinkDataExtractorTest.scala index 15366a7c19..1a86d62d0e 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/extractors/SinkDataExtractorTest.scala +++ b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/extractors/SinkDataExtractorTest.scala @@ -13,16 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.sink.extractors +package io.lenses.streamreactor.connect.cloud.common.sink.extractors -import io.lenses.streamreactor.connect.cloud.formats.writer.ArraySinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.MapSinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.StructSinkData -import io.lenses.streamreactor.connect.cloud.sink.config.PartitionNamePath -import io.lenses.streamreactor.connect.cloud.sink.extractors.ExtractorErrorType.MissingValue -import io.lenses.streamreactor.connect.cloud.sink.extractors.ExtractorErrorType.UnexpectedType -import io.lenses.streamreactor.connect.cloud.sink.extractors.ExtractorError -import io.lenses.streamreactor.connect.cloud.sink.extractors.SinkDataExtractor +import io.lenses.streamreactor.connect.cloud.common.formats.writer.ArraySinkData +import io.lenses.streamreactor.connect.cloud.common.formats.writer.MapSinkData +import io.lenses.streamreactor.connect.cloud.common.formats.writer.StructSinkData +import io.lenses.streamreactor.connect.cloud.common.sink.config.PartitionNamePath +import io.lenses.streamreactor.connect.cloud.common.sink.extractors.ExtractorErrorType.MissingValue +import io.lenses.streamreactor.connect.cloud.common.sink.extractors.ExtractorErrorType.UnexpectedType +import io.lenses.streamreactor.connect.cloud.common.sink.extractors.ExtractorError +import io.lenses.streamreactor.connect.cloud.common.sink.extractors.SinkDataExtractor import org.apache.kafka.connect.data.Schema import org.apache.kafka.connect.data.SchemaBuilder import org.apache.kafka.connect.data.Struct diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/naming/S3FileNamerTest.scala b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/naming/S3FileNamerTest.scala similarity index 86% rename from kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/naming/S3FileNamerTest.scala rename to kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/naming/S3FileNamerTest.scala index 04c001a641..a6e2d4ba75 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/naming/S3FileNamerTest.scala +++ b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/naming/S3FileNamerTest.scala @@ -13,10 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.sink.naming +package io.lenses.streamreactor.connect.cloud.common.sink.naming -import io.lenses.streamreactor.connect.cloud.model.Topic -import io.lenses.streamreactor.connect.cloud.sink.config.padding.PaddingType.LeftPad +import io.lenses.streamreactor.connect.cloud.common.model.Topic +import io.lenses.streamreactor.connect.cloud.common.sink.config.padding.PaddingType.LeftPad import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers class S3FileNamerTest extends AnyFunSuite with Matchers { diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/naming/S3KeyNamerTest.scala b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/naming/S3KeyNamerTest.scala similarity index 72% rename from kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/naming/S3KeyNamerTest.scala rename to kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/naming/S3KeyNamerTest.scala index c4f9e0db1e..42394c5184 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/naming/S3KeyNamerTest.scala +++ b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/naming/S3KeyNamerTest.scala @@ -13,24 +13,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.sink.naming +package io.lenses.streamreactor.connect.cloud.common.sink.naming import cats.implicits.none -import io.lenses.streamreactor.connect.aws.s3.model.location.S3LocationValidator -import io.lenses.streamreactor.connect.cloud.sink.config.PartitionDisplay.Values -import io.lenses.streamreactor.connect.cloud.sink.config.PartitionSelection.defaultPartitionSelection -import io.lenses.streamreactor.connect.cloud.config.FormatSelection -import io.lenses.streamreactor.connect.cloud.config.JsonFormatSelection -import io.lenses.streamreactor.connect.cloud.model.Topic -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocation -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocationValidator -import io.lenses.streamreactor.connect.cloud.sink.config.padding.LeftPadPaddingStrategy -import io.lenses.streamreactor.connect.cloud.sink.config.padding.PaddingService -import io.lenses.streamreactor.connect.cloud.sink.config.padding.PaddingStrategy -import io.lenses.streamreactor.connect.cloud.sink.config.PartitionField -import io.lenses.streamreactor.connect.cloud.sink.config.PartitionPartitionField -import io.lenses.streamreactor.connect.cloud.sink.config.PartitionSelection -import io.lenses.streamreactor.connect.cloud.sink.config.TopicPartitionField +import io.lenses.streamreactor.connect.cloud.common.config.FormatSelection +import io.lenses.streamreactor.connect.cloud.common.config.JsonFormatSelection +import io.lenses.streamreactor.connect.cloud.common.model.Topic +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocation +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocationValidator +import io.lenses.streamreactor.connect.cloud.common.sink.config.PartitionDisplay.Values +import io.lenses.streamreactor.connect.cloud.common.sink.config.PartitionSelection.defaultPartitionSelection +import io.lenses.streamreactor.connect.cloud.common.sink.config.padding.LeftPadPaddingStrategy +import io.lenses.streamreactor.connect.cloud.common.sink.config.padding.PaddingService +import io.lenses.streamreactor.connect.cloud.common.sink.config.padding.PaddingStrategy +import io.lenses.streamreactor.connect.cloud.common.sink.config.PartitionField +import io.lenses.streamreactor.connect.cloud.common.sink.config.PartitionPartitionField +import io.lenses.streamreactor.connect.cloud.common.sink.config.PartitionSelection +import io.lenses.streamreactor.connect.cloud.common.sink.config.TopicPartitionField +import io.lenses.streamreactor.connect.cloud.common.utils.SampleData import org.mockito.ArgumentMatchers.anyString import org.mockito.MockitoSugar import org.scalatest.EitherValues @@ -42,7 +42,7 @@ import java.nio.file.Files import java.util.UUID class S3KeyNamerTest extends AnyFunSuite with Matchers with OptionValues with EitherValues with MockitoSugar { - private implicit val cloudLocationValidator: CloudLocationValidator = S3LocationValidator + implicit val cloudLocationValidator: CloudLocationValidator = SampleData.cloudLocationValidator private val formatSelection: FormatSelection = JsonFormatSelection private val paddingStrategy: PaddingStrategy = LeftPadPaddingStrategy(3, '0') diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/seek/IndexManagerTest.scala b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/seek/IndexManagerTest.scala similarity index 89% rename from kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/seek/IndexManagerTest.scala rename to kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/seek/IndexManagerTest.scala index 2ab2e2f345..a1ee9df1f4 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/seek/IndexManagerTest.scala +++ b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/seek/IndexManagerTest.scala @@ -13,26 +13,25 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.sink.seek +package io.lenses.streamreactor.connect.cloud.common.sink.seek import cats.implicits.catsSyntaxEitherId import cats.implicits.catsSyntaxOptionId -import io.lenses.streamreactor.connect.aws.s3.storage._ -import io.lenses.streamreactor.connect.cloud.config.ConnectorTaskId -import io.lenses.streamreactor.connect.cloud.model.Offset -import io.lenses.streamreactor.connect.cloud.model.Topic -import io.lenses.streamreactor.connect.cloud.sink.FatalS3SinkError -import io.lenses.streamreactor.connect.cloud.sink.NonFatalS3SinkError +import io.lenses.streamreactor.connect.cloud.common.config.ConnectorTaskId +import io.lenses.streamreactor.connect.cloud.common.model.Offset +import io.lenses.streamreactor.connect.cloud.common.model.Topic +import io.lenses.streamreactor.connect.cloud.common.sink.FatalS3SinkError +import io.lenses.streamreactor.connect.cloud.common.sink.NonFatalS3SinkError +import io.lenses.streamreactor.connect.cloud.common.storage._ import org.mockito.ArgumentMatchers.any import org.mockito.ArgumentMatchers.anyString import org.mockito.ArgumentMatchers.{ eq => eqTo } import org.mockito.MockitoSugar +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers._ import org.scalatest.BeforeAndAfter import org.scalatest.EitherValues import org.scalatest.OptionValues -import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers._ -import software.amazon.awssdk.services.s3.model.S3Object import java.time.Instant @@ -96,7 +95,7 @@ class IndexManagerTest extends AnyFlatSpec with MockitoSugar with EitherValues w storageInterface.listRecursive[String]( eqTo(bucketName), eqTo(topicPartitionRootPath.some), - any[(String, Option[String], Seq[S3Object]) => Option[ListResponse[String]]], + any[(String, Option[String], Seq[FileMetadata]) => Option[ListResponse[String]]], ), ).thenReturn( ListResponse[String]( @@ -114,7 +113,7 @@ class IndexManagerTest extends AnyFlatSpec with MockitoSugar with EitherValues w cleanInOrder.verify(storageInterface).listRecursive[String]( eqTo(bucketName), eqTo(topicPartitionRootPath.some), - any[(String, Option[String], Seq[S3Object]) => Option[ListResponse[String]]], + any[(String, Option[String], Seq[FileMetadata]) => Option[ListResponse[String]]], ) cleanInOrder.verify(storageInterface).deleteFiles( bucketName, @@ -136,7 +135,7 @@ class IndexManagerTest extends AnyFlatSpec with MockitoSugar with EitherValues w cleanInOrder.verify(storageInterface).listRecursive[String]( eqTo(bucketName), eqTo(topicPartitionRootPath.some), - any[(String, Option[String], Seq[S3Object]) => Option[ListResponse[String]]], + any[(String, Option[String], Seq[FileMetadata]) => Option[ListResponse[String]]], ) cleanInOrder.verifyNoMoreInteractions() } @@ -147,7 +146,7 @@ class IndexManagerTest extends AnyFlatSpec with MockitoSugar with EitherValues w storageInterface.listRecursive[String]( any[String], any[Option[String]], - any[(String, Option[String], Seq[S3Object]) => Option[ListResponse[String]]], + any[(String, Option[String], Seq[FileMetadata]) => Option[ListResponse[String]]], ), ).thenReturn( ListResponse[String]( @@ -165,7 +164,7 @@ class IndexManagerTest extends AnyFlatSpec with MockitoSugar with EitherValues w storageInterface.listRecursive[String]( any[String], any[Option[String]], - any[(String, Option[String], Seq[S3Object]) => Option[ListResponse[String]]], + any[(String, Option[String], Seq[FileMetadata]) => Option[ListResponse[String]]], ), ).thenReturn( ListResponse( @@ -184,7 +183,7 @@ class IndexManagerTest extends AnyFlatSpec with MockitoSugar with EitherValues w cleanInOrder.verify(storageInterface).listRecursive[String]( eqTo(bucketName), eqTo(topicPartitionRootPath.some), - any[(String, Option[String], Seq[S3Object]) => Option[ListResponse[String]]], + any[(String, Option[String], Seq[FileMetadata]) => Option[ListResponse[String]]], ) cleanInOrder.verifyNoMoreInteractions() } @@ -199,7 +198,7 @@ class IndexManagerTest extends AnyFlatSpec with MockitoSugar with EitherValues w storageInterface.listRecursive[String]( eqTo(bucketName), eqTo(topicPartitionRootPath.some), - any[(String, Option[String], Seq[S3Object]) => Option[ListResponse[String]]], + any[(String, Option[String], Seq[FileMetadata]) => Option[ListResponse[String]]], ), ).thenReturn( ListResponse( @@ -220,7 +219,7 @@ class IndexManagerTest extends AnyFlatSpec with MockitoSugar with EitherValues w cleanInOrder.verify(storageInterface).listRecursive[String]( eqTo(bucketName), eqTo(topicPartitionRootPath.some), - any[(String, Option[String], Seq[S3Object]) => Option[ListResponse[String]]], + any[(String, Option[String], Seq[FileMetadata]) => Option[ListResponse[String]]], ) cleanInOrder.verify(storageInterface).deleteFiles( bucketName, @@ -238,7 +237,7 @@ class IndexManagerTest extends AnyFlatSpec with MockitoSugar with EitherValues w storageInterface.listRecursive[String]( any[String], any[Option[String]], - any[(String, Option[String], Seq[S3Object]) => Option[ListResponse[String]]], + any[(String, Option[String], Seq[FileMetadata]) => Option[ListResponse[String]]], ), ).thenReturn( ListResponse( @@ -256,7 +255,7 @@ class IndexManagerTest extends AnyFlatSpec with MockitoSugar with EitherValues w seekInOrder.verify(storageInterface).listRecursive[String]( eqTo(bucketName), eqTo(topicPartitionRootPath.some), - any[(String, Option[String], Seq[S3Object]) => Option[ListResponse[String]]], + any[(String, Option[String], Seq[FileMetadata]) => Option[ListResponse[String]]], ) seekInOrder.verify(storageInterface).getBlobAsString( bucketName, @@ -297,7 +296,7 @@ class IndexManagerTest extends AnyFlatSpec with MockitoSugar with EitherValues w seekInOrder.verify(storageInterface).listRecursive[String]( eqTo(bucketName), eqTo(topicPartitionRootPath.some), - any[(String, Option[String], Seq[S3Object]) => Option[ListResponse[String]]], + any[(String, Option[String], Seq[FileMetadata]) => Option[ListResponse[String]]], ) seekInOrder.verify(storageInterface).deleteFiles(anyString(), any[Seq[String]]) } diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/transformers/AddConnectSchemaTransformerTest.scala b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/transformers/AddConnectSchemaTransformerTest.scala similarity index 89% rename from kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/transformers/AddConnectSchemaTransformerTest.scala rename to kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/transformers/AddConnectSchemaTransformerTest.scala index 601ec34c1a..7ae28232d1 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/transformers/AddConnectSchemaTransformerTest.scala +++ b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/transformers/AddConnectSchemaTransformerTest.scala @@ -13,17 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.sink.transformers +package io.lenses.streamreactor.connect.cloud.common.sink.transformers -import io.lenses.streamreactor.connect.aws.s3.utils.SampleData -import io.lenses.streamreactor.connect.cloud.config.DataStorageSettings -import io.lenses.streamreactor.connect.cloud.formats.writer.ByteArraySinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.MapSinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.MessageDetail -import io.lenses.streamreactor.connect.cloud.formats.writer.StringSinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.StructSinkData -import io.lenses.streamreactor.connect.cloud.model.Offset -import io.lenses.streamreactor.connect.cloud.model.Topic +import io.lenses.streamreactor.connect.cloud.common.config.DataStorageSettings +import io.lenses.streamreactor.connect.cloud.common.formats.writer +import io.lenses.streamreactor.connect.cloud.common.formats.writer.ByteArraySinkData +import io.lenses.streamreactor.connect.cloud.common.formats.writer.MapSinkData +import io.lenses.streamreactor.connect.cloud.common.formats.writer.StringSinkData +import io.lenses.streamreactor.connect.cloud.common.formats.writer.StructSinkData +import io.lenses.streamreactor.connect.cloud.common.model.Offset +import io.lenses.streamreactor.connect.cloud.common.model.Topic +import io.lenses.streamreactor.connect.cloud.common.utils.SampleData import org.apache.kafka.connect.data.Schema import org.apache.kafka.connect.data.Struct import org.scalatest.Assertion @@ -36,7 +36,7 @@ import scala.jdk.CollectionConverters.MapHasAsJava class AddConnectSchemaTransformerTest extends AnyFunSuite with Matchers { test("return an error if the topic is not matching the message") { val transformer = new AddConnectSchemaTransformer(Topic("different"), DataStorageSettings.enabled) - val expected = MessageDetail( + val expected = writer.MessageDetail( StringSinkData("key"), StructSinkData(SampleData.Users.head), Map("header1" -> StringSinkData("value1"), "header2" -> ByteArraySinkData("value2".getBytes())), @@ -53,7 +53,7 @@ class AddConnectSchemaTransformerTest extends AnyFunSuite with Matchers { } test("ignores the key transformation if the settings does not store the key") { val transformer = new AddConnectSchemaTransformer(Topic("topic1"), DataStorageSettings.enabled.copy(key = false)) - val expected = MessageDetail( + val expected = writer.MessageDetail( ByteArraySinkData(Array[Byte](1, 2, 3)), StructSinkData(SampleData.Users.head), Map("header1" -> StringSinkData("value1"), "header2" -> ByteArraySinkData("value2".getBytes())), @@ -66,7 +66,7 @@ class AddConnectSchemaTransformerTest extends AnyFunSuite with Matchers { } test("ignores the value transformation if the settings does not store the value") { val transformer = new AddConnectSchemaTransformer(Topic("topic1"), DataStorageSettings.enabled.copy(value = false)) - val expected = MessageDetail( + val expected = writer.MessageDetail( StringSinkData("key"), ByteArraySinkData(Array[Byte](1, 2, 3)), Map("header1" -> StringSinkData("value1"), "header2" -> ByteArraySinkData("value2".getBytes())), @@ -79,7 +79,7 @@ class AddConnectSchemaTransformerTest extends AnyFunSuite with Matchers { } test("ignores the key and value if the envelope is disabled") { val transformer = new AddConnectSchemaTransformer(Topic("topic1"), DataStorageSettings.disabled) - val expected = MessageDetail( + val expected = writer.MessageDetail( ByteArraySinkData(Array[Byte](1, 2, 3)), ByteArraySinkData(Array[Byte](1, 2, 3)), Map("header1" -> StringSinkData("value1"), "header2" -> ByteArraySinkData("value2".getBytes())), @@ -103,7 +103,7 @@ class AddConnectSchemaTransformerTest extends AnyFunSuite with Matchers { "key8" -> 6.toShort, "key9" -> 7.toByte, ).asJava - val expected = MessageDetail( + val expected = writer.MessageDetail( MapSinkData( map, None, @@ -141,7 +141,7 @@ class AddConnectSchemaTransformerTest extends AnyFunSuite with Matchers { "key8" -> 6.toShort, "key9" -> 7.toByte, ).asJava - val expected = MessageDetail( + val expected = writer.MessageDetail( StringSinkData("key"), MapSinkData( map, @@ -191,7 +191,7 @@ class AddConnectSchemaTransformerTest extends AnyFunSuite with Matchers { ).asJava, ).asJava - val expected = MessageDetail( + val expected = writer.MessageDetail( StringSinkData("key"), MapSinkData( map, @@ -230,7 +230,7 @@ class AddConnectSchemaTransformerTest extends AnyFunSuite with Matchers { "key9" -> 7.toByte, ).asJava - val expected = MessageDetail( + val expected = writer.MessageDetail( StringSinkData("key"), MapSinkData( map, diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/transformers/DataSchemaBuilderTest.scala b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/transformers/DataSchemaBuilderTest.scala similarity index 87% rename from kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/transformers/DataSchemaBuilderTest.scala rename to kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/transformers/DataSchemaBuilderTest.scala index a7592ec66b..09e8eb0004 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/transformers/DataSchemaBuilderTest.scala +++ b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/transformers/DataSchemaBuilderTest.scala @@ -13,17 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.sink.transformers +package io.lenses.streamreactor.connect.cloud.common.sink.transformers -import io.lenses.streamreactor.connect.aws.s3.utils.SampleData -import io.lenses.streamreactor.connect.cloud.config.DataStorageSettings -import io.lenses.streamreactor.connect.cloud.formats.writer.BooleanSinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.IntSinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.MessageDetail -import io.lenses.streamreactor.connect.cloud.formats.writer.StringSinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.StructSinkData -import io.lenses.streamreactor.connect.cloud.model.Offset -import io.lenses.streamreactor.connect.cloud.model.Topic +import io.lenses.streamreactor.connect.cloud.common.config.DataStorageSettings +import io.lenses.streamreactor.connect.cloud.common.formats.writer +import io.lenses.streamreactor.connect.cloud.common.formats.writer.BooleanSinkData +import io.lenses.streamreactor.connect.cloud.common.formats.writer.IntSinkData +import io.lenses.streamreactor.connect.cloud.common.formats.writer.StringSinkData +import io.lenses.streamreactor.connect.cloud.common.formats.writer.StructSinkData +import io.lenses.streamreactor.connect.cloud.common.model.Offset +import io.lenses.streamreactor.connect.cloud.common.model.Topic +import io.lenses.streamreactor.connect.cloud.common.utils.SampleData import org.apache.kafka.connect.data.Schema import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers @@ -36,7 +36,7 @@ class DataSchemaBuilderTest extends AnyFunSuite with Matchers { DataStorageSettings(envelope = true, value = true, key = true, metadata = true, headers = true) val ts = Instant.now() - val message = MessageDetail( + val message = writer.MessageDetail( StringSinkData("key", Some(Schema.STRING_SCHEMA)), StructSinkData(SampleData.Users.head), Map( @@ -73,7 +73,7 @@ class DataSchemaBuilderTest extends AnyFunSuite with Matchers { val storageSettings = DataStorageSettings(envelope = true, value = true, key = true, metadata = true, headers = false) val ts = Instant.now() - val message = MessageDetail( + val message = writer.MessageDetail( StringSinkData("key", Some(Schema.STRING_SCHEMA)), StructSinkData(SampleData.Users.head), Map( @@ -107,7 +107,7 @@ class DataSchemaBuilderTest extends AnyFunSuite with Matchers { val storageSettings = DataStorageSettings(key = false, metadata = true, headers = false, envelope = true, value = true) val ts = Instant.now() - val message = MessageDetail( + val message = writer.MessageDetail( StringSinkData("key", Some(Schema.STRING_SCHEMA)), StructSinkData(SampleData.Users.head), Map( diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/transformers/EnvelopeWithSchemaTransformerTests.scala b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/transformers/EnvelopeWithSchemaTransformerTests.scala similarity index 88% rename from kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/transformers/EnvelopeWithSchemaTransformerTests.scala rename to kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/transformers/EnvelopeWithSchemaTransformerTests.scala index 5c8207c261..3f16b297b4 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/transformers/EnvelopeWithSchemaTransformerTests.scala +++ b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/transformers/EnvelopeWithSchemaTransformerTests.scala @@ -13,19 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.sink.transformers - -import io.lenses.streamreactor.connect.aws.s3.utils.SampleData -import io.lenses.streamreactor.connect.cloud.config.DataStorageSettings -import io.lenses.streamreactor.connect.cloud.formats.writer.ArraySinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.ByteArraySinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.MapSinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.MessageDetail -import io.lenses.streamreactor.connect.cloud.formats.writer.SinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.StringSinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.StructSinkData -import io.lenses.streamreactor.connect.cloud.model.Offset -import io.lenses.streamreactor.connect.cloud.model.Topic +package io.lenses.streamreactor.connect.cloud.common.sink.transformers + +import io.lenses.streamreactor.connect.cloud.common.config.DataStorageSettings +import io.lenses.streamreactor.connect.cloud.common.formats.writer +import io.lenses.streamreactor.connect.cloud.common.formats.writer._ +import io.lenses.streamreactor.connect.cloud.common.model.Offset +import io.lenses.streamreactor.connect.cloud.common.model.Topic +import io.lenses.streamreactor.connect.cloud.common.utils.SampleData import org.apache.kafka.connect.data.Schema import org.apache.kafka.connect.data.SchemaBuilder import org.apache.kafka.connect.data.Struct @@ -40,7 +35,7 @@ import scala.jdk.CollectionConverters.SeqHasAsJava class EnvelopeWithSchemaTransformerTests extends AnyFunSuite with Matchers { test("returns an error when the topic does not match") { val transformer = EnvelopeWithSchemaTransformer(Topic("different"), DataStorageSettings.enabled) - val expected = MessageDetail( + val expected = writer.MessageDetail( StringSinkData("key"), StructSinkData(SampleData.Users.head), Map("header1" -> StringSinkData("value1"), "header2" -> ByteArraySinkData("value2".getBytes())), @@ -58,7 +53,7 @@ class EnvelopeWithSchemaTransformerTests extends AnyFunSuite with Matchers { test("returns the message when the envelope settings is disabled") { val transformer = EnvelopeWithSchemaTransformer(Topic("topic1"), DataStorageSettings.disabled) - val expected = MessageDetail( + val expected = writer.MessageDetail( StringSinkData("key"), StructSinkData(SampleData.Users.head), Map("header1" -> StringSinkData("value1"), "header2" -> ByteArraySinkData("value2".getBytes())), @@ -71,7 +66,7 @@ class EnvelopeWithSchemaTransformerTests extends AnyFunSuite with Matchers { } test("full envelope") { - val expected = MessageDetail( + val expected = writer.MessageDetail( StringSinkData("key", Some(Schema.STRING_SCHEMA)), StructSinkData(SampleData.Users.head), Map( @@ -87,7 +82,7 @@ class EnvelopeWithSchemaTransformerTests extends AnyFunSuite with Matchers { } test("envelope without metadata") { val settings = DataStorageSettings.enabled.copy(metadata = false) - val expected = MessageDetail( + val expected = writer.MessageDetail( StringSinkData("key", Some(Schema.OPTIONAL_STRING_SCHEMA)), StructSinkData(SampleData.Users.head), Map( @@ -104,7 +99,7 @@ class EnvelopeWithSchemaTransformerTests extends AnyFunSuite with Matchers { test("envelope without headers") { val settings = DataStorageSettings.enabled.copy(headers = false) - val expected = MessageDetail( + val expected = writer.MessageDetail( StringSinkData("key", Some(Schema.OPTIONAL_STRING_SCHEMA)), StructSinkData(SampleData.Users.head), Map( @@ -121,7 +116,7 @@ class EnvelopeWithSchemaTransformerTests extends AnyFunSuite with Matchers { test("envelope without key") { val settings = DataStorageSettings.enabled.copy(key = false) - val expected = MessageDetail( + val expected = writer.MessageDetail( StringSinkData("key", Some(Schema.OPTIONAL_STRING_SCHEMA)), StructSinkData(SampleData.Users.head), Map( @@ -138,7 +133,7 @@ class EnvelopeWithSchemaTransformerTests extends AnyFunSuite with Matchers { test("envelope without value") { val settings = DataStorageSettings.enabled.copy(value = false) - val expected = MessageDetail( + val expected = writer.MessageDetail( StringSinkData("key", Some(Schema.OPTIONAL_STRING_SCHEMA)), StructSinkData(SampleData.Users.head), Map( @@ -155,7 +150,7 @@ class EnvelopeWithSchemaTransformerTests extends AnyFunSuite with Matchers { test("envelope value is an array") { val settings = DataStorageSettings.enabled - val expected = MessageDetail( + val expected = writer.MessageDetail( StringSinkData("key", Some(Schema.OPTIONAL_STRING_SCHEMA)), ArraySinkData(SampleData.Users.asJava, Some(SchemaBuilder.array(SampleData.UsersSchema).build())), Map( @@ -172,7 +167,7 @@ class EnvelopeWithSchemaTransformerTests extends AnyFunSuite with Matchers { test("envelope value is a map") { val settings = DataStorageSettings.enabled - val expected = MessageDetail( + val expected = writer.MessageDetail( StringSinkData("key", Some(Schema.OPTIONAL_STRING_SCHEMA)), MapSinkData( Map( diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/transformers/SchemalessEnvelopeTransformerTests.scala b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/transformers/SchemalessEnvelopeTransformerTests.scala similarity index 90% rename from kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/transformers/SchemalessEnvelopeTransformerTests.scala rename to kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/transformers/SchemalessEnvelopeTransformerTests.scala index 14422da096..b53fac682a 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/sink/transformers/SchemalessEnvelopeTransformerTests.scala +++ b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/sink/transformers/SchemalessEnvelopeTransformerTests.scala @@ -13,19 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.sink.transformers +package io.lenses.streamreactor.connect.cloud.common.sink.transformers -import io.lenses.streamreactor.connect.aws.s3.utils.SampleData -import io.lenses.streamreactor.connect.cloud.config.DataStorageSettings -import io.lenses.streamreactor.connect.cloud.formats.writer.ArraySinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.ByteArraySinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.MapSinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.MessageDetail -import io.lenses.streamreactor.connect.cloud.formats.writer.SinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.StringSinkData -import io.lenses.streamreactor.connect.cloud.formats.writer.StructSinkData -import io.lenses.streamreactor.connect.cloud.model.Offset -import io.lenses.streamreactor.connect.cloud.model.Topic +import io.lenses.streamreactor.connect.cloud.common.config.DataStorageSettings +import io.lenses.streamreactor.connect.cloud.common.formats.writer +import io.lenses.streamreactor.connect.cloud.common.formats.writer._ +import io.lenses.streamreactor.connect.cloud.common.model.Offset +import io.lenses.streamreactor.connect.cloud.common.model.Topic +import io.lenses.streamreactor.connect.cloud.common.utils.SampleData import org.apache.kafka.connect.data.Schema import org.apache.kafka.connect.data.SchemaBuilder import org.apache.kafka.connect.data.Struct @@ -42,7 +37,7 @@ import scala.jdk.CollectionConverters.SeqHasAsJava class SchemalessEnvelopeTransformerTests extends AnyFunSuite with Matchers { test("returns an error when the topic does not match") { val transformer = SchemalessEnvelopeTransformer(Topic("different"), DataStorageSettings.enabled) - val expected = MessageDetail( + val expected = writer.MessageDetail( StringSinkData("key"), StructSinkData(SampleData.Users.head), Map("header1" -> StringSinkData("value1"), "header2" -> ByteArraySinkData("value2".getBytes())), @@ -60,7 +55,7 @@ class SchemalessEnvelopeTransformerTests extends AnyFunSuite with Matchers { test("returns the message when the envelope settings is disabled") { val transformer = SchemalessEnvelopeTransformer(Topic("topic1"), DataStorageSettings.disabled) - val expected = MessageDetail( + val expected = writer.MessageDetail( StringSinkData("key"), StructSinkData(SampleData.Users.head), Map("header1" -> StringSinkData("value1"), "header2" -> ByteArraySinkData("value2".getBytes())), @@ -73,7 +68,7 @@ class SchemalessEnvelopeTransformerTests extends AnyFunSuite with Matchers { } test("full envelope") { - val expected = MessageDetail( + val expected = writer.MessageDetail( StringSinkData("key", Some(Schema.STRING_SCHEMA)), StructSinkData(SampleData.Users.head), Map( @@ -89,7 +84,7 @@ class SchemalessEnvelopeTransformerTests extends AnyFunSuite with Matchers { } test("envelope without metadata") { val settings = DataStorageSettings.enabled.copy(metadata = false) - val expected = MessageDetail( + val expected = writer.MessageDetail( StringSinkData("key", Some(Schema.OPTIONAL_STRING_SCHEMA)), StructSinkData(SampleData.Users.head), Map( @@ -106,7 +101,7 @@ class SchemalessEnvelopeTransformerTests extends AnyFunSuite with Matchers { test("envelope without headers") { val settings = DataStorageSettings.enabled.copy(headers = false) - val expected = MessageDetail( + val expected = writer.MessageDetail( StringSinkData("key", Some(Schema.OPTIONAL_STRING_SCHEMA)), StructSinkData(SampleData.Users.head), Map( @@ -123,7 +118,7 @@ class SchemalessEnvelopeTransformerTests extends AnyFunSuite with Matchers { test("envelope without key") { val settings = DataStorageSettings.enabled.copy(key = false) - val expected = MessageDetail( + val expected = writer.MessageDetail( StringSinkData("key", Some(Schema.OPTIONAL_STRING_SCHEMA)), StructSinkData(SampleData.Users.head), Map( @@ -140,7 +135,7 @@ class SchemalessEnvelopeTransformerTests extends AnyFunSuite with Matchers { test("envelope without value") { val settings = DataStorageSettings.enabled.copy(value = false) - val expected = MessageDetail( + val expected = writer.MessageDetail( StringSinkData("key", Some(Schema.OPTIONAL_STRING_SCHEMA)), StructSinkData(SampleData.Users.head), Map( @@ -157,7 +152,7 @@ class SchemalessEnvelopeTransformerTests extends AnyFunSuite with Matchers { test("envelope value is an array") { val settings = DataStorageSettings.enabled - val expected = MessageDetail( + val expected = writer.MessageDetail( StringSinkData("key", Some(Schema.OPTIONAL_STRING_SCHEMA)), ArraySinkData(SampleData.Users.asJava, Some(SchemaBuilder.array(SampleData.UsersSchema).build())), Map( @@ -174,7 +169,7 @@ class SchemalessEnvelopeTransformerTests extends AnyFunSuite with Matchers { test("envelope value is a map") { val settings = DataStorageSettings.enabled - val expected = MessageDetail( + val expected = writer.MessageDetail( StringSinkData("key", Some(Schema.OPTIONAL_STRING_SCHEMA)), MapSinkData( Map( @@ -282,7 +277,7 @@ class SchemalessEnvelopeTransformerTests extends AnyFunSuite with Matchers { test("key is bytes array adds a keyIsArray field in the envelope") { val settings = DataStorageSettings.enabled - val expected = MessageDetail( + val expected = writer.MessageDetail( ByteArraySinkData("key".getBytes(), Some(Schema.OPTIONAL_BYTES_SCHEMA)), StructSinkData(SampleData.Users.head), Map( @@ -304,7 +299,7 @@ class SchemalessEnvelopeTransformerTests extends AnyFunSuite with Matchers { } test("value is bytes array adds isValueArray field in the envelope") { val settings = DataStorageSettings.enabled - val expected = MessageDetail( + val expected = writer.MessageDetail( StringSinkData("key", Some(Schema.OPTIONAL_STRING_SCHEMA)), ByteArraySinkData("value".getBytes(), Some(Schema.OPTIONAL_BYTES_SCHEMA)), Map( diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/source/SourceContextReaderTest.scala b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/source/SourceContextReaderTest.scala similarity index 88% rename from kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/source/SourceContextReaderTest.scala rename to kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/source/SourceContextReaderTest.scala index 3dd72a02ef..4c487dcd4e 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/source/SourceContextReaderTest.scala +++ b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/source/SourceContextReaderTest.scala @@ -13,12 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.source +package io.lenses.streamreactor.connect.cloud.common.source import cats.implicits.catsSyntaxOptionId -import io.lenses.streamreactor.connect.aws.s3.model.location.S3LocationValidator -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocation -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocationValidator +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocation +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocationValidator +import io.lenses.streamreactor.connect.cloud.common.utils.SampleData import org.apache.kafka.connect.source.SourceTaskContext import org.mockito.Answers import org.mockito.MockitoSugar @@ -35,7 +35,7 @@ class SourceContextReaderTest extends AnyFlatSpec with Matchers with MockitoSuga private val bucketName = "bucket" private val prefixName = "prefixName" - private implicit val cloudLocationValidator: CloudLocationValidator = S3LocationValidator + private implicit val cloudLocationValidator: CloudLocationValidator = SampleData.cloudLocationValidator private val rootLocation = CloudLocation(bucketName, prefixName.some) private val filePath = "prefixName/file.json" diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/source/SourceWatermarkTest.scala b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/source/SourceWatermarkTest.scala similarity index 82% rename from kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/source/SourceWatermarkTest.scala rename to kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/source/SourceWatermarkTest.scala index 038b09f18f..286d79a72e 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/source/SourceWatermarkTest.scala +++ b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/source/SourceWatermarkTest.scala @@ -13,13 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.source +package io.lenses.streamreactor.connect.cloud.common.source import cats.implicits.catsSyntaxOptionId -import io.lenses.streamreactor.connect.aws.s3.model.location.S3LocationValidator -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocation -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocationValidator -import io.lenses.streamreactor.connect.cloud.source.SourceWatermark +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocation +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocationValidator +import io.lenses.streamreactor.connect.cloud.common.utils.SampleData import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -27,7 +26,7 @@ import java.time.Instant import scala.jdk.CollectionConverters.MapHasAsScala class SourceWatermarkTest extends AnyFlatSpec with Matchers { - private implicit val cloudLocationValidator: CloudLocationValidator = S3LocationValidator + private implicit val cloudLocationValidator: CloudLocationValidator = SampleData.cloudLocationValidator "fromSourcePartition" should "convert S3Location to Map" in { SourceWatermark.partition(CloudLocation("test-bucket", "test-prefix".some)).asScala.toMap shouldBe Map( diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/source/config/ReadTextModeTestFormatSelection.scala b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/source/config/ReadTextModeTestFormatSelection.scala similarity index 87% rename from kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/source/config/ReadTextModeTestFormatSelection.scala rename to kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/source/config/ReadTextModeTestFormatSelection.scala index e5da42c0c1..39468e781c 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/source/config/ReadTextModeTestFormatSelection.scala +++ b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/source/config/ReadTextModeTestFormatSelection.scala @@ -13,16 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.source.config +package io.lenses.streamreactor.connect.cloud.common.source.config -import io.lenses.streamreactor.connect.cloud.config.kcqlprops.S3PropsKeyEntry -import io.lenses.streamreactor.connect.cloud.config.kcqlprops.S3PropsKeyEnum -import io.lenses.streamreactor.connect.cloud.source.config.kcqlprops.ReadTextModeEnum -import io.lenses.streamreactor.connect.cloud.source.config.kcqlprops.S3SourcePropsSchema -import io.lenses.streamreactor.connect.cloud.source.config.ReadTextMode -import io.lenses.streamreactor.connect.cloud.source.config.RegexReadTextMode -import io.lenses.streamreactor.connect.cloud.source.config.StartEndLineReadTextMode -import io.lenses.streamreactor.connect.cloud.source.config.StartEndTagReadTextMode +import io.lenses.streamreactor.connect.cloud.common.config.kcqlprops.S3PropsKeyEntry +import io.lenses.streamreactor.connect.cloud.common.config.kcqlprops.S3PropsKeyEnum +import io.lenses.streamreactor.connect.cloud.common.source.config.kcqlprops.ReadTextModeEnum +import io.lenses.streamreactor.connect.cloud.common.source.config.kcqlprops.S3SourcePropsSchema import io.lenses.streamreactor.connect.config.kcqlprops.KcqlProperties import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/source/distribution/PartitionHasherTest.scala b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/source/distribution/PartitionHasherTest.scala similarity index 93% rename from kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/source/distribution/PartitionHasherTest.scala rename to kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/source/distribution/PartitionHasherTest.scala index 270c85692a..c2f26b9250 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/source/distribution/PartitionHasherTest.scala +++ b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/source/distribution/PartitionHasherTest.scala @@ -13,9 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.source.distribution +package io.lenses.streamreactor.connect.cloud.common.source.distribution -import io.lenses.streamreactor.connect.cloud.source.config.distribution.PartitionHasher +import io.lenses.streamreactor.connect.cloud.common.source.config.distribution.PartitionHasher import org.scalatest.Assertion import org.scalatest.flatspec.AnyFlatSpecLike import org.scalatest.matchers.should.Matchers diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/source/files/DateOrderingBatchListerTest.scala b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/source/files/DateOrderingBatchListerTest.scala similarity index 87% rename from kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/source/files/DateOrderingBatchListerTest.scala rename to kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/source/files/DateOrderingBatchListerTest.scala index 196a3bab6c..e0b7103d0b 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/source/files/DateOrderingBatchListerTest.scala +++ b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/source/files/DateOrderingBatchListerTest.scala @@ -13,13 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.source.files +package io.lenses.streamreactor.connect.cloud.common.source.files import cats.implicits._ -import io.lenses.streamreactor.connect.aws.s3.storage.FileListError -import io.lenses.streamreactor.connect.aws.s3.storage.FileMetadata -import io.lenses.streamreactor.connect.aws.s3.storage.ListResponse -import io.lenses.streamreactor.connect.aws.s3.storage.StorageInterface +import io.lenses.streamreactor.connect.cloud.common.storage.FileListError +import io.lenses.streamreactor.connect.cloud.common.storage.FileMetadata +import io.lenses.streamreactor.connect.cloud.common.storage.ListResponse +import io.lenses.streamreactor.connect.cloud.common.storage.StorageInterface import org.mockito.ArgumentMatchersSugar._ import org.mockito.MockitoSugar import org.scalatest.BeforeAndAfter @@ -27,7 +27,6 @@ import org.scalatest.EitherValues import org.scalatest.OptionValues import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers -import software.amazon.awssdk.services.s3.model.S3Object import java.time.Instant import java.time.temporal.ChronoUnit @@ -79,7 +78,7 @@ class DateOrderingBatchListerTest storageInterface.listRecursive( eqTo(bucket), eqTo(prefix.some), - any[(String, Option[String], Seq[S3Object]) => Option[ListResponse[FileMetadata]]], + any[(String, Option[String], Seq[FileMetadata]) => Option[ListResponse[FileMetadata]]], ), ).thenReturn(none.asRight) @@ -93,7 +92,7 @@ class DateOrderingBatchListerTest storageInterface.listRecursive( eqTo(bucket), eqTo(prefix.some), - any[(String, Option[String], Seq[S3Object]) => Option[ListResponse[FileMetadata]]], + any[(String, Option[String], Seq[FileMetadata]) => Option[ListResponse[FileMetadata]]], ), ).thenReturn( exception.asLeft, @@ -167,7 +166,7 @@ class DateOrderingBatchListerTest storageInterface.listRecursive( eqTo(bucket), eqTo(prefix.some), - any[(String, Option[String], Seq[S3Object]) => Option[ListResponse[FileMetadata]]], + any[(String, Option[String], Seq[FileMetadata]) => Option[ListResponse[FileMetadata]]], ), ).thenReturn(storageInterfaceResponse.some.asRight) } diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/source/files/DefaultOrderingBatchListerTest.scala b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/source/files/DefaultOrderingBatchListerTest.scala similarity index 88% rename from kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/source/files/DefaultOrderingBatchListerTest.scala rename to kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/source/files/DefaultOrderingBatchListerTest.scala index 2101435853..1dc44d5f07 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/source/files/DefaultOrderingBatchListerTest.scala +++ b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/source/files/DefaultOrderingBatchListerTest.scala @@ -13,14 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.source.files +package io.lenses.streamreactor.connect.cloud.common.source.files import cats.implicits.catsSyntaxEitherId import cats.implicits.catsSyntaxOptionId import cats.implicits.none -import io.lenses.streamreactor.connect.aws.s3.storage.FileListError -import io.lenses.streamreactor.connect.aws.s3.storage.ListResponse -import io.lenses.streamreactor.connect.aws.s3.storage.StorageInterface +import io.lenses.streamreactor.connect.cloud.common.storage.FileListError +import io.lenses.streamreactor.connect.cloud.common.storage.ListResponse +import io.lenses.streamreactor.connect.cloud.common.storage.StorageInterface import org.mockito.MockitoSugar import org.scalatest.BeforeAndAfter import org.scalatest.EitherValues diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/source/files/S3SourceFileQueueTest.scala b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/source/files/S3SourceFileQueueTest.scala similarity index 89% rename from kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/source/files/S3SourceFileQueueTest.scala rename to kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/source/files/S3SourceFileQueueTest.scala index 9ae7d88f52..18b363eaa9 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/source/files/S3SourceFileQueueTest.scala +++ b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/source/files/S3SourceFileQueueTest.scala @@ -13,19 +13,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.source.files +package io.lenses.streamreactor.connect.cloud.common.source.files import cats.implicits.catsSyntaxEitherId import cats.implicits.catsSyntaxOptionId import cats.implicits.none -import io.lenses.streamreactor.connect.aws.s3.model.location.S3LocationValidator -import io.lenses.streamreactor.connect.aws.s3.storage.FileListError -import io.lenses.streamreactor.connect.aws.s3.storage.FileLoadError -import io.lenses.streamreactor.connect.aws.s3.storage.FileMetadata -import io.lenses.streamreactor.connect.aws.s3.storage.ListResponse -import io.lenses.streamreactor.connect.cloud.config.ConnectorTaskId -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocation -import io.lenses.streamreactor.connect.cloud.model.location.CloudLocationValidator +import io.lenses.streamreactor.connect.cloud.common.config.ConnectorTaskId +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocation +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocationValidator +import io.lenses.streamreactor.connect.cloud.common.storage.FileListError +import io.lenses.streamreactor.connect.cloud.common.storage.FileLoadError +import io.lenses.streamreactor.connect.cloud.common.storage.FileMetadata +import io.lenses.streamreactor.connect.cloud.common.storage.ListResponse +import io.lenses.streamreactor.connect.cloud.common.utils.SampleData import org.mockito.ArgumentMatchers._ import org.mockito.MockitoSugar import org.scalatest.BeforeAndAfter @@ -35,7 +35,7 @@ import org.scalatest.matchers.should.Matchers import java.time.Instant class S3SourceFileQueueTest extends AnyFlatSpec with Matchers with MockitoSugar with BeforeAndAfter { - private implicit val cloudLocationValidator: CloudLocationValidator = S3LocationValidator + private implicit val cloudLocationValidator: CloudLocationValidator = SampleData.cloudLocationValidator private val taskId = ConnectorTaskId("topic", 1, 0) private val bucket = "bucket" diff --git a/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/source/reader/PartitionDiscoveryTest.scala b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/source/reader/PartitionDiscoveryTest.scala new file mode 100644 index 0000000000..983849b64f --- /dev/null +++ b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/source/reader/PartitionDiscoveryTest.scala @@ -0,0 +1,111 @@ +/* + * Copyright 2017-2023 Lenses.io Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.lenses.streamreactor.connect.cloud.common.source.reader + +import cats.effect.IO +import cats.effect.kernel.Ref +import cats.effect.unsafe.implicits.global +import io.lenses.streamreactor.connect.cloud.common.config.ConnectorTaskId +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocation +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocationValidator +import io.lenses.streamreactor.connect.cloud.common.source.config.PartitionSearcherOptions +import io.lenses.streamreactor.connect.cloud.common.source.config.PartitionSearcherOptions.ExcludeIndexes +import io.lenses.streamreactor.connect.cloud.common.source.distribution.PartitionSearcherResponse +import io.lenses.streamreactor.connect.cloud.common.source.files.SourceFileQueue +import io.lenses.streamreactor.connect.cloud.common.storage.DirectoryFindResults +import io.lenses.streamreactor.connect.cloud.common.utils.SampleData +import org.mockito.MockitoSugar +import org.scalatest.flatspec.AnyFlatSpecLike +import org.scalatest.matchers.should.Matchers + +import scala.concurrent.duration.DurationInt + +class PartitionDiscoveryTest extends AnyFlatSpecLike with Matchers with MockitoSugar { + private implicit val cloudLocationValidator: CloudLocationValidator = SampleData.cloudLocationValidator + private val connectorTaskId: ConnectorTaskId = ConnectorTaskId("sinkName", 1, 1) + "PartitionDiscovery" should "handle failure on PartitionSearcher and resume" in { + val fileQueueProcessor: SourceFileQueue = mock[SourceFileQueue] + val limit = 10 + val options = PartitionSearcherOptions(1, continuous = true, 100.millis, ExcludeIndexes) + + trait Count { + def getCount: IO[Int] + + def find( + lastFound: Seq[PartitionSearcherResponse], + ): IO[Seq[PartitionSearcherResponse]] + } + val searcherMock = new Count { + private val count = Ref[IO].of(0).unsafeRunSync() + def getCount: IO[Int] = count.get + + def find( + lastFound: Seq[PartitionSearcherResponse], + ): IO[Seq[PartitionSearcherResponse]] = + for { + c <- count.getAndUpdate(_ + 1) + _ <- if (c == 0) IO.raiseError(new RuntimeException("error")) else IO.unit + } yield { + List( + PartitionSearcherResponse(CloudLocation("bucket", None), + Set("prefix1/", "prefix2/"), + DirectoryFindResults(Set("prefix1/", "prefix2/")), + None, + ), + ) + } + } + + val io = for { + cancelledRef <- Ref[IO].of(false) + readerRef <- Ref[IO].of(Option.empty[ResultReader]) + state <- Ref[IO].of(ReaderManagerState(Seq.empty, Seq.empty)) + fiber <- PartitionDiscovery.run( + connectorTaskId, + options, + searcherMock.find, + (_, _) => + IO(new ReaderManager(limit, + fileQueueProcessor, + _ => Left(new RuntimeException()), + connectorTaskId, + readerRef, + )), + state, + cancelledRef, + ).start + _ <- IO.sleep(1000.millis) + _ <- cancelledRef.set(true) + _ <- fiber.join + readerMgrState <- state.get + callsMade <- searcherMock.getCount + } yield readerMgrState -> callsMade + + val (state, callsMade) = io.unsafeRunSync() + assert( + state.partitionResponses == List( + PartitionSearcherResponse( + CloudLocation("bucket", None), + Set("prefix1/", "prefix2/"), + DirectoryFindResults(Set("prefix1/", "prefix2/")), + None, + ), + ), + ) + callsMade >= 1 + } + +} diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/stream/BuildLocalOutputStreamTest.scala b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/stream/BuildLocalOutputStreamTest.scala similarity index 84% rename from kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/stream/BuildLocalOutputStreamTest.scala rename to kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/stream/BuildLocalOutputStreamTest.scala index 27838c7143..83441f21a0 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/stream/BuildLocalOutputStreamTest.scala +++ b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/stream/BuildLocalOutputStreamTest.scala @@ -13,12 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.stream +package io.lenses.streamreactor.connect.cloud.common.stream -import io.lenses.streamreactor.connect.cloud.formats.reader.Using -import io.lenses.streamreactor.connect.cloud.model.location.FileUtils.toBufferedOutputStream -import io.lenses.streamreactor.connect.cloud.model.Topic -import io.lenses.streamreactor.connect.cloud.stream.BuildLocalOutputStream +import io.lenses.streamreactor.connect.cloud.common.formats.reader.Using +import io.lenses.streamreactor.connect.cloud.common.model.Topic +import io.lenses.streamreactor.connect.cloud.common.model.location.FileUtils.toBufferedOutputStream +import io.lenses.streamreactor.connect.cloud.common.stream.BuildLocalOutputStream import org.scalatest.BeforeAndAfter import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/stream/S3ByteArrayOutputStream.scala b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/stream/S3ByteArrayOutputStream.scala similarity index 86% rename from kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/stream/S3ByteArrayOutputStream.scala rename to kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/stream/S3ByteArrayOutputStream.scala index 627d37ff82..50925d3c87 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/stream/S3ByteArrayOutputStream.scala +++ b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/stream/S3ByteArrayOutputStream.scala @@ -13,11 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.stream +package io.lenses.streamreactor.connect.cloud.common.stream import cats.implicits.catsSyntaxEitherId -import io.lenses.streamreactor.connect.cloud.sink.SinkError -import io.lenses.streamreactor.connect.cloud.stream.S3OutputStream +import io.lenses.streamreactor.connect.cloud.common.sink.SinkError import java.io.ByteArrayOutputStream diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/utils/IteratorOpsTest.scala b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/utils/IteratorOpsTest.scala similarity index 97% rename from kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/utils/IteratorOpsTest.scala rename to kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/utils/IteratorOpsTest.scala index e53a04c799..8db8e8822f 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/utils/IteratorOpsTest.scala +++ b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/utils/IteratorOpsTest.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.utils +package io.lenses.streamreactor.connect.cloud.common.utils import io.lenses.streamreactor.connect.cloud.common.utils.IteratorOps import org.scalatest.flatspec.AnyFlatSpecLike diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/utils/MapUtilsTest.scala b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/utils/MapUtilsTest.scala similarity index 96% rename from kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/utils/MapUtilsTest.scala rename to kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/utils/MapUtilsTest.scala index 7711d0c3e5..fde0d04b7e 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/utils/MapUtilsTest.scala +++ b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/utils/MapUtilsTest.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.utils +package io.lenses.streamreactor.connect.cloud.common.utils import io.lenses.streamreactor.connect.cloud.common.utils.MapUtils import org.scalatest.funsuite.AnyFunSuite diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/utils/PollLoopTests.scala b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/utils/PollLoopTests.scala similarity index 98% rename from kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/utils/PollLoopTests.scala rename to kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/utils/PollLoopTests.scala index 86c653aba7..d1af124807 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/utils/PollLoopTests.scala +++ b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/utils/PollLoopTests.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.utils +package io.lenses.streamreactor.connect.cloud.common.utils import cats.effect.IO import cats.effect.kernel.Ref diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/utils/SampleData.scala b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/utils/SampleData.scala similarity index 90% rename from kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/utils/SampleData.scala rename to kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/utils/SampleData.scala index fca0f6fb31..1ceca098af 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/utils/SampleData.scala +++ b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/utils/SampleData.scala @@ -13,9 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.utils +package io.lenses.streamreactor.connect.cloud.common.utils -import io.lenses.streamreactor.connect.cloud.model.Topic +import cats.data.Validated +import io.lenses.streamreactor.connect.cloud.common.model.Topic +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocation +import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocationValidator import org.apache.avro.generic.GenericData import org.apache.avro.generic.GenericRecord import org.apache.avro.util.Utf8 @@ -26,10 +29,15 @@ import org.scalacheck.Gen import org.scalatest.Assertion import org.scalatest.matchers.should.Matchers import org.scalacheck.Gen.Choose.chooseDouble + import java.util import scala.jdk.CollectionConverters.MapHasAsScala object SampleData extends Matchers { + + implicit val cloudLocationValidator: CloudLocationValidator = (s3Location: CloudLocation, allowSlash: Boolean) => + Validated.fromEither(Right(s3Location)) + val topic: Topic = Topic("niceTopic") val UsersSchema: Schema = SchemaBuilder.struct() diff --git a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/utils/TimestampUtilsTest.scala b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/utils/TimestampUtilsTest.scala similarity index 95% rename from kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/utils/TimestampUtilsTest.scala rename to kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/utils/TimestampUtilsTest.scala index e5cbbeb135..f7664bfc33 100644 --- a/kafka-connect-aws-s3/src/test/scala/io/lenses/streamreactor/connect/aws/s3/utils/TimestampUtilsTest.scala +++ b/kafka-connect-cloud-common/src/test/scala/io/lenses/streamreactor/connect/cloud/common/utils/TimestampUtilsTest.scala @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.lenses.streamreactor.connect.aws.s3.utils +package io.lenses.streamreactor.connect.cloud.common.utils import io.lenses.streamreactor.connect.cloud.common.utils.TimestampUtils import org.scalatest.funsuite.AnyFunSuite diff --git a/project/Dependencies.scala b/project/Dependencies.scala index ecb2f95828..aa8eafc1e0 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -80,7 +80,7 @@ object Dependencies { val jerseyCommonVersion = "3.1.1" val calciteVersion = "1.34.0" - val awsSdkVersion = "2.20.54" + val awsSdkVersion = "2.20.68" val guavaVersion = "31.0.1-jre" val guiceVersion = "5.1.0" val javaxBindVersion = "2.3.1" @@ -512,6 +512,19 @@ trait Dependencies { //Specific modules dependencies + val kafkaConnectCloudCommonDeps: Seq[ModuleID] = Seq( + parquetAvro, + parquetHadoop, + hadoopCommon, + hadoopMapReduce, + hadoopMapReduceClient, + hadoopMapReduceClientCore, + //javaxBind, + openCsv, + //guice, + //guiceAssistedInject, + ) + val kafkaConnectS3Deps: Seq[ModuleID] = Seq( s3Sdk, stsSdk,