Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

build: upgrade kafka client #620

Merged
merged 7 commits into from
Mar 19, 2024
Merged

build: upgrade kafka client #620

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
c827ee0
refactor: remove dead code
venikkin Mar 15, 2024
2e2deff
refactor: remove unused junit rules
venikkin Mar 15, 2024
5f742de
build: update kafka client
venikkin Mar 15, 2024
1842284
build: downgrade kafka client version to 3.4.1
venikkin Mar 18, 2024
dbfc75e
build: remove duplicate java version property
venikkin Mar 18, 2024
bca814c
build: update actions
venikkin Mar 18, 2024
6551a92
build: add distribution
venikkin Mar 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 0 additions & 40 deletions common/src/main/kotlin/streams/extensions/CommonExtensions.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,17 +1,10 @@
package streams.extensions

import org.apache.kafka.clients.consumer.ConsumerRecord
import org.apache.kafka.clients.consumer.OffsetAndMetadata
import org.apache.kafka.common.TopicPartition
import org.neo4j.driver.types.Node
import org.neo4j.driver.types.Relationship
import streams.service.StreamsSinkEntity
import streams.utils.JSONUtils
import java.nio.ByteBuffer
import java.util.*
import javax.lang.model.SourceVersion

fun Map<String,String>.getInt(name:String, defaultValue: Int) = this.get(name)?.toInt() ?: defaultValue
fun Map<*, *>.asProperties() = this.let {
val properties = Properties()
properties.putAll(it)
Expand All @@ -34,10 +27,6 @@ fun Relationship.asStreamsMap(): Map<String, Any?> {
return relMap
}

fun String.toPointCase(): String {
return this.split("(?<=[a-z])(?=[A-Z])".toRegex()).joinToString(separator = ".").toLowerCase()
}

fun String.quote(): String = if (SourceVersion.isIdentifier(this)) this else "`$this`"

fun Map<String, Any?>.flatten(map: Map<String, Any?> = this, prefix: String = ""): Map<String, Any?> {
Expand All @@ -51,33 +40,4 @@ fun Map<String, Any?>.flatten(map: Map<String, Any?> = this, prefix: String = ""
listOf(newKey to value)
}
}.toMap()
}

fun ConsumerRecord<*, *>.topicPartition() = TopicPartition(this.topic(), this.partition())
fun ConsumerRecord<*, *>.offsetAndMetadata(metadata: String = "") = OffsetAndMetadata(this.offset() + 1, metadata)

private fun convertAvroData(rawValue: Any?): Any? = when (rawValue) {
is Collection<*> -> rawValue.map(::convertAvroData)
is Array<*> -> if (rawValue.javaClass.componentType.isPrimitive) rawValue else rawValue.map(::convertAvroData)
is Map<*, *> -> rawValue
.mapKeys { it.key.toString() }
.mapValues { convertAvroData(it.value) }
is ByteBuffer -> rawValue.array()
is CharSequence -> rawValue.toString()
else -> rawValue
}



private fun convertData(data: Any?, stringWhenFailure: Boolean = false): Any? {
return when (data) {
null -> null
is ByteArray -> JSONUtils.readValue<Any>(data, stringWhenFailure)
else -> if (stringWhenFailure) data.toString() else throw RuntimeException("Unsupported type ${data::class.java.name}")
}
}
fun ConsumerRecord<*, *>.toStreamsSinkEntity(): StreamsSinkEntity {
val key = convertData(this.key(), true)
val value = convertData(this.value())
return StreamsSinkEntity(key, value)
}
17 changes: 2 additions & 15 deletions common/src/main/kotlin/streams/service/errors/ErrorService.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,7 @@
package streams.service.errors

import org.apache.kafka.clients.consumer.ConsumerRecord
import org.apache.kafka.common.record.RecordBatch
import java.util.Properties


data class ErrorData(val originalTopic: String,
val timestamp: Long,
val key: ByteArray?,
Expand All @@ -17,21 +14,11 @@ data class ErrorData(val originalTopic: String,

constructor(originalTopic: String, timestamp: Long?, key: Any?, value: Any?,
partition: Int, offset: Long, executingClass: Class<*>?, databaseName: String?, exception: Exception?) :
this(originalTopic, timestamp ?: RecordBatch.NO_TIMESTAMP, toByteArray(key), toByteArray(value), partition.toString(),offset.toString(), executingClass, databaseName, exception)
this(originalTopic, timestamp ?: NO_TIMESTAMP, toByteArray(key), toByteArray(value), partition.toString(),offset.toString(), executingClass, databaseName, exception)

companion object {

fun from(consumerRecord: ConsumerRecord<out Any, out Any>, exception: Exception?, executingClass: Class<*>?, databaseName: String?): ErrorData {
return ErrorData(offset = consumerRecord.offset().toString(),
originalTopic = consumerRecord.topic(),
partition = consumerRecord.partition().toString(),
timestamp = consumerRecord.timestamp(),
exception = exception,
executingClass = executingClass,
key = toByteArray(consumerRecord.key()),
value = toByteArray(consumerRecord.value()),
databaseName = databaseName)
}
private const val NO_TIMESTAMP: Long = -1L

fun toByteArray(v:Any?) = try {
when (v) {
Expand Down
69 changes: 0 additions & 69 deletions common/src/main/kotlin/streams/utils/KafkaValidationUtils.kt
View file
Open in desktop

This file was deleted.

3 changes: 1 addition & 2 deletions common/src/test/kotlin/streams/service/sink/errors/KafkaErrorServiceTest.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ import org.apache.kafka.clients.producer.ProducerRecord
import org.apache.kafka.clients.producer.internals.FutureRecordMetadata
import org.apache.kafka.common.record.RecordBatch
import org.apache.kafka.common.utils.SystemTime
import org.apache.kafka.common.utils.Time
import org.junit.Test
import org.mockito.ArgumentMatchers
import org.mockito.Mockito
Expand All @@ -25,7 +24,7 @@ class KafkaErrorServiceTest {
val counter = AtomicInteger(0)
Mockito.`when`(producer.send(ArgumentMatchers.any<ProducerRecord<ByteArray, ByteArray>>())).then {
counter.incrementAndGet()
FutureRecordMetadata(null, 0, RecordBatch.NO_TIMESTAMP, 0L, 0, 0, SystemTime())
FutureRecordMetadata(null, 0, RecordBatch.NO_TIMESTAMP, 0, 0, SystemTime())
}
val dlqService = KafkaErrorService(producer, ErrorService.ErrorConfig(fail=false,dlqTopic = "dlqTopic"), { s, e -> })
dlqService.report(listOf(dlqData()))
Expand Down
3 changes: 2 additions & 1 deletion pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,10 +49,11 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<java.version>11</java.version>
<java.version>11</java.version>
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

duplicate?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for spotting it. Fixed.

<kotlin.version>1.9.20</kotlin.version>
<kotlin.coroutines.version>1.7.3</kotlin.coroutines.version>
<neo4j.version>4.4.27</neo4j.version>
<kafka.version>2.6.3</kafka.version>
<kafka.version>3.7.0</kafka.version>
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same as in #619.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

3.4 is the lowest version containing the fix. As we discussed used it.

<jackson.version>2.15.2</jackson.version>
<kotlin.compiler.incremental>true</kotlin.compiler.incremental>
<neo4j.java.driver.version>4.4.12</neo4j.java.driver.version>
Expand Down
Loading
Loading