diff --git a/common/pom.xml b/common/pom.xml index ce28b3f8..6c605547 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -34,6 +34,12 @@ kafka-clients provided + + + org.neo4j + neo4j-configuration + provided + diff --git a/common/src/main/kotlin/streams/events/StreamsEvent.kt b/common/src/main/kotlin/streams/events/StreamsEvent.kt index 7b0e7f8e..6fd2deda 100644 --- a/common/src/main/kotlin/streams/events/StreamsEvent.kt +++ b/common/src/main/kotlin/streams/events/StreamsEvent.kt @@ -1,7 +1,5 @@ package streams.events -import org.neo4j.graphdb.schema.ConstraintType - enum class OperationType { created, updated, deleted } data class Meta(val timestamp: Long, diff --git a/common/src/main/kotlin/streams/utils/JSONUtils.kt b/common/src/main/kotlin/streams/utils/JSONUtils.kt index d6c1c41c..283314b5 100644 --- a/common/src/main/kotlin/streams/utils/JSONUtils.kt +++ b/common/src/main/kotlin/streams/utils/JSONUtils.kt @@ -2,16 +2,38 @@ package streams.utils import com.fasterxml.jackson.core.JsonGenerator import com.fasterxml.jackson.core.JsonParseException +import com.fasterxml.jackson.core.JsonParser import com.fasterxml.jackson.core.JsonProcessingException -import com.fasterxml.jackson.databind.* +import com.fasterxml.jackson.databind.DeserializationContext +import com.fasterxml.jackson.databind.DeserializationFeature +import com.fasterxml.jackson.databind.JsonDeserializer +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.JsonSerializer +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.SerializationFeature +import com.fasterxml.jackson.databind.SerializerProvider import com.fasterxml.jackson.databind.module.SimpleModule import com.fasterxml.jackson.module.kotlin.convertValue import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import com.fasterxml.jackson.module.kotlin.readValue import org.neo4j.driver.internal.value.PointValue +import org.neo4j.function.ThrowingBiConsumer import org.neo4j.graphdb.spatial.Point +import org.neo4j.values.AnyValue import org.neo4j.values.storable.CoordinateReferenceSystem -import streams.events.* +import org.neo4j.values.storable.Values +import org.neo4j.values.virtual.MapValue +import org.neo4j.values.virtual.MapValueBuilder +import streams.events.EntityType +import streams.events.Meta +import streams.events.NodePayload +import streams.events.Payload +import streams.events.RecordChange +import streams.events.RelationshipPayload +import streams.events.Schema +import streams.events.StreamsTransactionEvent +import streams.events.StreamsTransactionNodeEvent +import streams.events.StreamsTransactionRelationshipEvent import streams.extensions.asStreamsMap import java.io.IOException import java.time.temporal.TemporalAccessor @@ -33,6 +55,13 @@ fun Point.toStreamsPoint(): StreamsPoint { } } +fun Map.toMapValue(): MapValue { + val map = this + val builder = MapValueBuilder() + map.forEach { (t, u) -> builder.add(t, Values.of(u)) } + return builder.build() +} + fun PointValue.toStreamsPoint(): StreamsPoint { val point = this.asPoint() return when (val crsType = point.srid()) { @@ -121,6 +150,100 @@ class DriverRelationshipSerializer : JsonSerializer() { + override fun createEvent(meta: Meta, payload: RelationshipPayload, schema: Schema): StreamsTransactionRelationshipEvent { + return StreamsTransactionRelationshipEvent(meta, payload, schema) + } + + override fun convertPayload(payloadMap: JsonNode): RelationshipPayload { + return JSONUtils.convertValue(payloadMap) + } + + override fun fillPayload(payload: RelationshipPayload, + beforeProps: Map?, + afterProps: Map?): RelationshipPayload { + return payload.copy( + before = payload.before?.copy(properties = beforeProps), + after = payload.after?.copy(properties = afterProps) + ) + } + + override fun deserialize(parser: JsonParser, context: DeserializationContext): StreamsTransactionRelationshipEvent { + val deserialized = super.deserialize(parser, context) + if (deserialized.payload.type == EntityType.node) { + throw IllegalArgumentException("Relationship event expected, but node type found") + } + return deserialized + } + +} + +class StreamsTransactionNodeEventDeserializer : StreamsTransactionEventDeserializer() { + override fun createEvent(meta: Meta, payload: NodePayload, schema: Schema): StreamsTransactionNodeEvent { + return StreamsTransactionNodeEvent(meta, payload, schema) + } + + override fun convertPayload(payloadMap: JsonNode): NodePayload { + return JSONUtils.convertValue(payloadMap) + } + + override fun fillPayload(payload: NodePayload, + beforeProps: Map?, + afterProps: Map?): NodePayload { + return payload.copy( + before = payload.before?.copy(properties = beforeProps), + after = payload.after?.copy(properties = afterProps) + ) + } + + override fun deserialize(parser: JsonParser, context: DeserializationContext): StreamsTransactionNodeEvent { + val deserialized = super.deserialize(parser, context) + if (deserialized.payload.type == EntityType.relationship) { + throw IllegalArgumentException("Node event expected, but relationship type found") + } + return deserialized + } + +} + +abstract class StreamsTransactionEventDeserializer : JsonDeserializer() { + + abstract fun createEvent(meta: Meta, payload: PAYLOAD, schema: Schema): EVENT + abstract fun convertPayload(payloadMap: JsonNode): PAYLOAD + abstract fun fillPayload(payload: PAYLOAD, + beforeProps: Map?, + afterProps: Map?): PAYLOAD + + @Throws(IOException::class, JsonProcessingException::class) + override fun deserialize(parser: JsonParser, context: DeserializationContext): EVENT { + val root: JsonNode = parser.codec.readTree(parser) + val meta = JSONUtils.convertValue(root["meta"]) + val schema = JSONUtils.convertValue(root["schema"]) + val points = schema.properties.filterValues { it == "PointValue" }.keys + var payload = convertPayload(root["payload"]) + if (points.isNotEmpty()) { + val beforeProps = convertPoints(payload.before, points) + val afterProps = convertPoints(payload.after, points) + payload = fillPayload(payload, beforeProps, afterProps) + } + return createEvent(meta, payload, schema) + } + + private fun convertPoints( + recordChange: RecordChange?, + points: Set + ) = recordChange + ?.properties + ?.mapValues { + if (points.contains(it.key)) { + org.neo4j.values.storable.PointValue.fromMap((it.value as Map).toMapValue()) + } else { + it.value + } + } + +} + object JSONUtils { private val OBJECT_MAPPER: ObjectMapper = jacksonObjectMapper() @@ -133,6 +256,8 @@ object JSONUtils { StreamsUtils.ignoreExceptions({ module.addSerializer(org.neo4j.driver.types.Point::class.java, DriverPointSerializer()) }, NoClassDefFoundError::class.java) // in case is loaded from StreamsUtils.ignoreExceptions({ module.addSerializer(org.neo4j.driver.types.Node::class.java, DriverNodeSerializer()) }, NoClassDefFoundError::class.java) // in case is loaded from StreamsUtils.ignoreExceptions({ module.addSerializer(org.neo4j.driver.types.Relationship::class.java, DriverRelationshipSerializer()) }, NoClassDefFoundError::class.java) // in case is loaded from + StreamsUtils.ignoreExceptions({ module.addDeserializer(StreamsTransactionRelationshipEvent::class.java, StreamsTransactionRelationshipEventDeserializer()) }, NoClassDefFoundError::class.java) // in case is loaded from + StreamsUtils.ignoreExceptions({ module.addDeserializer(StreamsTransactionNodeEvent::class.java, StreamsTransactionNodeEventDeserializer()) }, NoClassDefFoundError::class.java) // in case is loaded from module.addSerializer(TemporalAccessor::class.java, TemporalAccessorSerializer()) OBJECT_MAPPER.registerModule(module) OBJECT_MAPPER.disable(SerializationFeature.FAIL_ON_EMPTY_BEANS) diff --git a/common/src/main/kotlin/streams/utils/ProcedureUtils.kt b/common/src/main/kotlin/streams/utils/ProcedureUtils.kt index 33477f90..63b96be0 100644 --- a/common/src/main/kotlin/streams/utils/ProcedureUtils.kt +++ b/common/src/main/kotlin/streams/utils/ProcedureUtils.kt @@ -1,7 +1,5 @@ package streams.utils -import org.neo4j.configuration.Config -import org.neo4j.configuration.GraphDatabaseSettings import org.neo4j.dbms.api.DatabaseManagementService import org.neo4j.exceptions.UnsatisfiedDependencyException import org.neo4j.kernel.impl.factory.DbmsInfo diff --git a/consumer/src/test/kotlin/integrations/kafka/KafkaEventSinkCDCTSE.kt b/consumer/src/test/kotlin/integrations/kafka/KafkaEventSinkCDCTSE.kt index c69697b5..e6465e7b 100644 --- a/consumer/src/test/kotlin/integrations/kafka/KafkaEventSinkCDCTSE.kt +++ b/consumer/src/test/kotlin/integrations/kafka/KafkaEventSinkCDCTSE.kt @@ -175,6 +175,156 @@ class KafkaEventSinkCDCTSE: KafkaEventSinkBaseTSE() { }, Matchers.equalTo(true), 30, TimeUnit.SECONDS) } + @Test + fun shouldWriteDataFromSinkWithCDCSchemaTopicAndPointValue() = runBlocking { + val topic = UUID.randomUUID().toString() + db.setConfig("streams.sink.topic.cdc.schema", topic) + db.start() + + val constraints = listOf(Constraint(label = "User", type = StreamsConstraintType.UNIQUE, properties = setOf("name", "surname"))) + val nodeSchema = Schema( + properties = mapOf( + "name" to "String", + "surname" to "String", + "comp@ny" to "String", + "bornIn2d" to "PointValue", + "bornIn3d" to "PointValue", + "livesIn2d" to "PointValue", + "livesIn3d" to "PointValue", + "worksIn2d" to "PointValue", + "worksIn3d" to "PointValue" + ), + constraints = constraints) + val relSchema = Schema(properties = mapOf( + "since" to "Long", + "where" to "PointValue" + ), constraints = constraints) + val cdcDataStart = StreamsTransactionEvent( + meta = Meta(timestamp = System.currentTimeMillis(), + username = "user", + txId = 1, + txEventId = 0, + txEventsCount = 3, + operation = OperationType.created + ), + payload = NodePayload( + id = "0", + before = null, + after = NodeChange( + properties = mapOf( + "name" to "Andrea", + "surname" to "Santurbano", + "comp@ny" to "LARUS-BA", + "bornIn3d" to mapOf( + "crs" to "wgs-84-3d", + "latitude" to 12.78, + "longitude" to 56.7, + "height" to 100.0, + ), + "bornIn2d" to mapOf( + "crs" to "wgs-84", + "latitude" to 12.78, + "longitude" to 56.7 + ), + "livesIn3d" to mapOf( + "crs" to "wgs-84-3d", + "latitude" to 12.79, + "longitude" to 56.71, + "height" to 100.0, + ), + "livesIn2d" to mapOf( + "crs" to "wgs-84", + "latitude" to 12.79, + "longitude" to 56.71 + ), + "worksIn2d" to mapOf( + "crs" to "cartesian", + "x" to 1.2, + "y" to 10.1 + ), + "worksIn3d" to mapOf( + "crs" to "cartesian-3d", + "x" to 1.2, + "y" to 10.1, + "z" to 7.1 + ) + ), + labels = listOf("User") + ) + ), + schema = nodeSchema + ) + val cdcDataEnd = StreamsTransactionEvent( + meta = Meta(timestamp = System.currentTimeMillis(), + username = "user", + txId = 1, + txEventId = 1, + txEventsCount = 3, + operation = OperationType.created + ), + payload = NodePayload(id = "1", + before = null, + after = NodeChange(properties = mapOf("name" to "Michael", "surname" to "Hunger", "comp@ny" to "Neo4j"), labels = listOf("User")) + ), + schema = nodeSchema + ) + val cdcDataRelationship = StreamsTransactionEvent( + meta = Meta(timestamp = System.currentTimeMillis(), + username = "user", + txId = 1, + txEventId = 2, + txEventsCount = 3, + operation = OperationType.created + ), + payload = RelationshipPayload( + id = "2", + start = RelationshipNodeChange(id = "0", labels = listOf("User"), ids = mapOf("name" to "Andrea", "surname" to "Santurbano")), + end = RelationshipNodeChange(id = "1", labels = listOf("User"), ids = mapOf("name" to "Michael", "surname" to "Hunger")), + after = RelationshipChange(properties = mapOf( + "since" to 2014, + "where" to mapOf( + "crs" to "wgs-84-3d", + "latitude" to 12.78, + "longitude" to 56.7, + "height" to 80.0, + ) + )), + before = null, + label = "MEET" + ), + schema = relSchema + ) + var producerRecord = ProducerRecord(topic, UUID.randomUUID().toString(), JSONUtils.writeValueAsBytes(cdcDataStart)) + kafkaProducer.send(producerRecord).get() + producerRecord = ProducerRecord(topic, UUID.randomUUID().toString(), JSONUtils.writeValueAsBytes(cdcDataEnd)) + kafkaProducer.send(producerRecord).get() + producerRecord = ProducerRecord(topic, UUID.randomUUID().toString(), JSONUtils.writeValueAsBytes(cdcDataRelationship)) + kafkaProducer.send(producerRecord).get() + + Assert.assertEventually(ThrowingSupplier { + val query = """ + |MATCH (s:User{ + | name:'Andrea', + | surname:'Santurbano', + | `comp@ny`:'LARUS-BA', + | bornIn3d: point({x: 56.7, y: 12.78, z: 100.0, crs: 'wgs-84-3d'}), + | bornIn2d: point({x: 56.7, y: 12.78, crs: 'wgs-84'}), + | livesIn3d: point({longitude: 56.71, latitude: 12.79, height: 100}), + | livesIn2d: point({longitude: 56.71, latitude: 12.79}), + | worksIn2d: point({x: 1.2, y: 10.1, crs: 'cartesian'}), + | worksIn3d: point({x: 1.2, y: 10.1, z: 7.1, crs: 'cartesian-3d'}) + |}) + |MATCH (t:User{name:'Michael', surname:'Hunger', `comp@ny`:'Neo4j'}) + |MATCH p = (s)-[r:MEET{since: 2014, where: point({x: 56.7, y: 12.78, z: 80.0, crs: 'wgs-84-3d'})}]->(t) + |RETURN count(p) AS count + |""".trimMargin() + db.execute(query) { + val result = it.columnAs("count") + result.hasNext() && result.next() == 1L && !result.hasNext() + } + }, Matchers.equalTo(true), 30, TimeUnit.SECONDS) + } + @Test fun writeDataFromSinkWithCDCSchemaTopicMultipleConstraintsAndLabels() = runBlocking { val topic = UUID.randomUUID().toString() diff --git a/kafka-connect-neo4j/README.md b/kafka-connect-neo4j/README.md deleted file mode 100644 index 154945b3..00000000 --- a/kafka-connect-neo4j/README.md +++ /dev/null @@ -1,15 +0,0 @@ -# Introduction - -Welcome to your Kafka Connect Neo4j Connector! - -# Build it locally - -Build the project by running the following command: - - $ mvn clean install - -Inside the directory `/kafka-connect-neo4j/target/component/packages` you'll find a file named `neo4j-kafka-connect-neo4j-.zip` - -# Run with docker - -Please refer to this file [readme.adoc](doc/readme.adoc) diff --git a/kafka-connect-neo4j/assets/neo4j-logo.png b/kafka-connect-neo4j/assets/neo4j-logo.png deleted file mode 100644 index 9b568bce..00000000 Binary files a/kafka-connect-neo4j/assets/neo4j-logo.png and /dev/null differ diff --git a/kafka-connect-neo4j/config/sink-quickstart.properties b/kafka-connect-neo4j/config/sink-quickstart.properties deleted file mode 100644 index 07a5db9c..00000000 --- a/kafka-connect-neo4j/config/sink-quickstart.properties +++ /dev/null @@ -1,14 +0,0 @@ -# A simple configuration properties, is the same as contrib.sink.avro.neo4j.json -name=Neo4jSinkConnector -topics=my-topic -connector.class=streams.kafka.connect.sink.Neo4jSinkConnector -errors.retry.timeout=-1 -errors.retry.delay.max.ms=1000 -errors.tolerance=all -errors.log.enable=true -errors.log.include.messages=true -neo4j.server.uri=bolt://neo4j:7687 -neo4j.authentication.basic.username=neo4j -neo4j.authentication.basic.password=connect -neo4j.encryption.enabled=false -neo4j.topic.cypher.my-topic=MERGE (p:Person{name: event.name, surname: event.surname}) MERGE (f:Family{name: event.surname}) MERGE (p)-[:BELONGS_TO]->(f) \ No newline at end of file diff --git a/kafka-connect-neo4j/doc/contrib.sink.avro.neo4j.json b/kafka-connect-neo4j/doc/contrib.sink.avro.neo4j.json deleted file mode 100644 index 9a9ffc89..00000000 --- a/kafka-connect-neo4j/doc/contrib.sink.avro.neo4j.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "name": "Neo4jSinkConnector", - "config": { - "topics": "my-topic", - "connector.class": "streams.kafka.connect.sink.Neo4jSinkConnector", - "errors.retry.timeout": "-1", - "errors.retry.delay.max.ms": "1000", - "errors.tolerance": "all", - "errors.log.enable": true, - "errors.log.include.messages": true, - "neo4j.server.uri": "bolt://neo4j:7687", - "neo4j.authentication.basic.username": "neo4j", - "neo4j.authentication.basic.password": "connect", - "neo4j.encryption.enabled": false, - "neo4j.topic.cypher.my-topic": "MERGE (p:Person{name: event.name, surname: event.surname, from: 'AVRO'}) MERGE (f:Family{name: event.surname}) MERGE (p)-[:BELONGS_TO]->(f)" - } -} diff --git a/kafka-connect-neo4j/doc/contrib.sink.string-json.neo4j.json b/kafka-connect-neo4j/doc/contrib.sink.string-json.neo4j.json deleted file mode 100644 index d23e8c9d..00000000 --- a/kafka-connect-neo4j/doc/contrib.sink.string-json.neo4j.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "name": "Neo4jSinkConnectorJSON", - "config": { - "key.converter": "org.apache.kafka.connect.storage.StringConverter", - "value.converter": "org.apache.kafka.connect.json.JsonConverter", - "value.converter.schemas.enable": false, - "topics": "my-topic", - "connector.class": "streams.kafka.connect.sink.Neo4jSinkConnector", - "errors.retry.timeout": "-1", - "errors.retry.delay.max.ms": "1000", - "errors.tolerance": "all", - "errors.log.enable": true, - "errors.log.include.messages": true, - "neo4j.server.uri": "bolt://neo4j:7687", - "neo4j.authentication.basic.username": "neo4j", - "neo4j.authentication.basic.password": "connect", - "neo4j.encryption.enabled": false, - "neo4j.topic.cypher.my-topic": "MERGE (p:Person{name: event.name, surname: event.surname, from: 'JSON'}) MERGE (f:Family{name: event.surname}) MERGE (p)-[:BELONGS_TO]->(f)" - } -} diff --git a/kafka-connect-neo4j/doc/contrib.source.avro.neo4j.json b/kafka-connect-neo4j/doc/contrib.source.avro.neo4j.json deleted file mode 100644 index 280ae06a..00000000 --- a/kafka-connect-neo4j/doc/contrib.source.avro.neo4j.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "name": "Neo4jSourceConnector", - "config": { - "topic": "my-topic", - "connector.class": "streams.kafka.connect.source.Neo4jSourceConnector", - "key.converter": "io.confluent.connect.avro.AvroConverter", - "value.converter": "io.confluent.connect.avro.AvroConverter", - "neo4j.server.uri": "bolt://neo4j:7687", - "neo4j.authentication.basic.username": "neo4j", - "neo4j.authentication.basic.password": "connect", - "neo4j.encryption.enabled": false, - "neo4j.streaming.poll.interval.msecs": 5000, - "neo4j.streaming.property": "timestamp", - "neo4j.streaming.from": "LAST_COMMITTED", - "neo4j.enforce.schema": true, - "neo4j.source.query": "MATCH (ts:TestSource) WHERE ts.timestamp > $lastCheck RETURN ts.name AS name, ts.timestamp AS timestamp" - } -} \ No newline at end of file diff --git a/kafka-connect-neo4j/doc/contrib.source.string-json.neo4j.json b/kafka-connect-neo4j/doc/contrib.source.string-json.neo4j.json deleted file mode 100644 index c9d8346c..00000000 --- a/kafka-connect-neo4j/doc/contrib.source.string-json.neo4j.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "name": "Neo4jSourceConnector", - "config": { - "topic": "my-topic", - "connector.class": "streams.kafka.connect.source.Neo4jSourceConnector", - "key.converter": "org.apache.kafka.connect.json.JsonConverter", - "value.converter": "org.apache.kafka.connect.json.JsonConverter", - "neo4j.server.uri": "bolt://neo4j:7687", - "neo4j.authentication.basic.username": "neo4j", - "neo4j.authentication.basic.password": "connect", - "neo4j.encryption.enabled": false, - "neo4j.streaming.poll.interval.msecs": 5000, - "neo4j.streaming.property": "timestamp", - "neo4j.streaming.from": "LAST_COMMITTED", - "neo4j.source.query": "MATCH (ts:TestSource) WHERE ts.timestamp > $lastCheck RETURN ts.name AS name, ts.timestamp AS timestamp" - } -} \ No newline at end of file diff --git a/kafka-connect-neo4j/doc/contrib.source.string.neo4j.json b/kafka-connect-neo4j/doc/contrib.source.string.neo4j.json deleted file mode 100644 index def80fd0..00000000 --- a/kafka-connect-neo4j/doc/contrib.source.string.neo4j.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "name": "Neo4jSourceConnectorString", - "config": { - "topic": "my-topic", - "connector.class": "streams.kafka.connect.source.Neo4jSourceConnector", - "key.converter": "org.apache.kafka.connect.storage.StringConverter", - "value.converter": "org.apache.kafka.connect.storage.StringConverter", - "neo4j.server.uri": "bolt://neo4j:7687", - "neo4j.authentication.basic.username": "neo4j", - "neo4j.authentication.basic.password": "connect", - "neo4j.encryption.enabled": false, - "neo4j.streaming.poll.interval.msecs": 5000, - "neo4j.streaming.property": "timestamp", - "neo4j.streaming.from": "LAST_COMMITTED", - "neo4j.source.query": "MATCH (ts:TestSource) WHERE ts.timestamp > $lastCheck RETURN ts.name AS name, ts.timestamp AS timestamp" - } -} \ No newline at end of file diff --git a/kafka-connect-neo4j/doc/docker-compose.yml b/kafka-connect-neo4j/doc/docker-compose.yml deleted file mode 100644 index 5e45aed2..00000000 --- a/kafka-connect-neo4j/doc/docker-compose.yml +++ /dev/null @@ -1,122 +0,0 @@ ---- -version: '2' -services: - neo4j: - image: neo4j:4.3-enterprise - hostname: neo4j - container_name: neo4j - ports: - - "7474:7474" - - "7687:7687" - environment: - NEO4J_kafka_bootstrap_servers: broker:9093 - NEO4J_AUTH: neo4j/connect - NEO4J_dbms_memory_heap_max__size: 8G - NEO4J_ACCEPT_LICENSE_AGREEMENT: yes - - zookeeper: - image: confluentinc/cp-zookeeper - hostname: zookeeper - container_name: zookeeper - ports: - - "2181:2181" - environment: - ZOOKEEPER_CLIENT_PORT: 2181 - ZOOKEEPER_TICK_TIME: 2000 - - broker: - image: confluentinc/cp-enterprise-kafka - hostname: broker - container_name: broker - depends_on: - - zookeeper - ports: - - "9092:9092" - expose: - - "9093" - environment: - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker:9093,OUTSIDE://localhost:9092 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,OUTSIDE:PLAINTEXT - KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9093,OUTSIDE://0.0.0.0:9092 - CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: broker:9093 - - # workaround if we change to a custom name the schema_registry fails to start - KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT - - KAFKA_BROKER_ID: 1 - KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181' - KAFKA_METRIC_REPORTERS: io.confluent.metrics.reporter.ConfluentMetricsReporter - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 - CONFLUENT_METRICS_REPORTER_ZOOKEEPER_CONNECT: zookeeper:2181 - CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1 - CONFLUENT_METRICS_ENABLE: 'true' - CONFLUENT_SUPPORT_CUSTOMER_ID: 'anonymous' - - schema_registry: - image: confluentinc/cp-schema-registry - hostname: schema_registry - container_name: schema_registry - depends_on: - - zookeeper - - broker - ports: - - "8081:8081" - environment: - SCHEMA_REGISTRY_HOST_NAME: schema_registry - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181' - - connect: - image: confluentinc/cp-kafka-connect - hostname: connect - container_name: connect - depends_on: - - zookeeper - - broker - - schema_registry - ports: - - "8083:8083" - volumes: - - ./plugins:/tmp/connect-plugins - environment: - CONNECT_BOOTSTRAP_SERVERS: 'broker:9093' - CONNECT_REST_ADVERTISED_HOST_NAME: connect - CONNECT_REST_PORT: 8083 - CONNECT_GROUP_ID: compose-connect-group - CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs - CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1 - CONNECT_OFFSET_FLUSH_INTERVAL_MS: 10000 - CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets - CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1 - CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status - CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1 - CONNECT_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter - CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema_registry:8081' - CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter - CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema_registry:8081' - CONNECT_INTERNAL_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter - CONNECT_INTERNAL_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter - CONNECT_ZOOKEEPER_CONNECT: 'zookeeper:2181' - CONNECT_PLUGIN_PATH: /usr/share/java,/tmp/connect-plugins - CONNECT_LOG4J_LOGGERS: org.apache.zookeeper=DEBUG,org.I0Itec.zkclient=DEBUG,org.reflections=ERROR - - control-center: - image: confluentinc/cp-enterprise-control-center - hostname: control-center - container_name: control-center - depends_on: - - zookeeper - - broker - - schema_registry - - connect - ports: - - "9021:9021" - environment: - CONTROL_CENTER_BOOTSTRAP_SERVERS: 'broker:9093' - CONTROL_CENTER_ZOOKEEPER_CONNECT: 'zookeeper:2181' - CONTROL_CENTER_CONNECT_CLUSTER: 'connect:8083' - CONTROL_CENTER_REPLICATION_FACTOR: 1 - CONTROL_CENTER_INTERNAL_TOPICS_PARTITIONS: 1 - CONTROL_CENTER_MONITORING_INTERCEPTOR_TOPIC_PARTITIONS: 1 - CONFLUENT_METRICS_TOPIC_REPLICATION: 1 - PORT: 9021 \ No newline at end of file diff --git a/kafka-connect-neo4j/doc/readme.adoc b/kafka-connect-neo4j/doc/readme.adoc deleted file mode 100644 index 6db97828..00000000 --- a/kafka-connect-neo4j/doc/readme.adoc +++ /dev/null @@ -1,201 +0,0 @@ -= Build it locally - -Build the project by running the following command: - - mvn clean install - -Inside the directory `/kafka-connect-neo4j/target/component/packages` you'll find a file named `neo4j-kafka-connect-neo4j-.zip` - -== Sink - -=== Configuring the stack - -Create a directory `plugins` at the same level of the compose file and unzip the file `neo4j-kafka-connect-neo4j-.zip` inside it, then start the compose file - - docker-compose up -d - -Create the Sink instance: - -We'll define the Sink configuration as follows: - -[source,json] ----- -include::contrib.sink.avro.neo4j.json[] ----- - -In particular this line: - ----- -"neo4j.topic.cypher.my-topic": "MERGE (p:Person{name: event.name, surname: event.surname}) MERGE (f:Family{name: event.surname}) MERGE (p)-[:BELONGS_TO]->(f)" ----- - -defines that all the data that comes from the topic `neo4j` will be unpacked by the Sink into Neo4j with the following Cypher query: - -[source,cypher] ----- -MERGE (p:Person{name: event.name, surname: event.surname}) -MERGE (f:Family{name: event.surname}) -MERGE (p)-[:BELONGS_TO]->(f) ----- - - -Under the hood the Sink inject the event object in this way - -[source,cypher] ----- -UNWIND {batch} AS event -MERGE (p:Person{name: event.name, surname: event.surname}) -MERGE (f:Family{name: event.surname}) -MERGE (p)-[:BELONGS_TO]->(f) ----- - -Where `{batch}` is a list of event objects. - -You can change the query or remove the property and add your own, but you must follow the following convention: - -[source,javascript] ----- -"neo4j.topic.cypher.": "" ----- - -Let's load the configuration into the Confluent Platform with this REST call: - -[source,shell] ----- -curl -X POST http://localhost:8083/connectors \ - -H 'Content-Type:application/json' \ - -H 'Accept:application/json' \ - -d @contrib.sink.avro.neo4j.json ----- - -The file `contrib.sink.string-json.neo4j.json` contains a configuration that manage a simple JSON producer example - -Please check that everything is fine by going into: - -http://localhost:9021/management/connect - -and click to the **Sink** tab. You must find a table just like this: - -[cols="4*",options="header"] -|=== -|Status -|Active Tasks -|Name -|Topics - -|Running -|1 -|Neo4jSinkConnector -|my-topic -|=== - -=== Use the data generator - -You can download and use the https://github.com/conker84/neo4j-streams-sink-tester/releases/download/1/neo4j-streams-sink-tester-1.0.jar[neo4j-streams-sink-tester-1.0.jar] in order to generate a sample dataset. - -This package sends records to the Neo4j Kafka Sink by using the following in two data formats: - -JSON example: - -[source,json] ----- -{"name": "Name", "surname": "Surname"} ----- - -AVRO, with the schema: - -[source,json] ----- -{ - "type":"record", - "name":"User", - "fields":[{"name":"name","type":"string"}, {"name":"surname","type":"string"}] -} ----- - -Please type: - ----- -java -jar neo4j-streams-sink-tester-1.0.jar -h ----- - -to print the option list with default values. - -In order to choose the data format please use the `-f` flag: `-f AVRO` or `-f JSON` (the default value). -So: - ----- -java -jar neo4j-streams-sink-tester-1.0.jar -f AVRO ----- - -Will send data in AVRO format. - -For a complete overview of the **Neo4j Steams Sink Tester** please refer to https://github.com/conker84/neo4j-streams-sink-tester[this repo] - -== Source - -=== Configuring the stack - -Create a directory `plugins` at the same level of the compose file and unzip the file `neo4j-kafka-connect-neo4j-.zip` inside it, then start the compose file - - docker-compose up -d - -=== Create the Source instance: - -In this chapter we'll discuss about how the Source instance works - -You can create a new Source instance with this REST call: - -[source,shell] ----- -curl -X POST http://localhost:8083/connectors \ - -H 'Content-Type:application/json' \ - -H 'Accept:application/json' \ - -d @contrib.source.avro.neo4j.json ----- - -Let's look at the `contrib.source.avro.neo4j.json` file: - -[source,json] ----- -include::contrib.source.avro.neo4j.json[] ----- - -This will create a Kafka Connect Source instance that will send `AVRO` message over the topic named `my-topic`. Every message in the -topic will have the following structure: - -[source,json] ----- -{"name": , "timestamp": } ----- - -**Nb.** Please check the <> for a detailed guide about the supported configuration -parameters - -=== How the Source module pushes the data to the defined Kafka topic - -We use the query provided in the `neo4j.source.query` field by polling the database every value is into the -`neo4j.streaming.poll.interval.msecs` field. - -So given the JSON configuration we have that we'll perform: - -[source,cypher] ----- -MATCH (ts:TestSource) WHERE ts.timestamp > $lastCheck RETURN ts.name AS name, ts.timestamp AS timestamp ----- - -every 5000 milliseconds by publishing events like: - -[source,json] ----- -{"name":{"string":"John Doe"},"timestamp":{"long":1624551349362}} ----- - -In this case we use `neo4j.enforce.schema=true` and this means that we will attach a schema for each record, in case -you want to stream pure simple JSON strings just use the relative serializer with `neo4j.enforce.schema=false` with the -following output: - -[source,json] ----- -{"name": "John Doe", "timestamp": 1624549598834} ----- \ No newline at end of file diff --git a/kafka-connect-neo4j/docker/contrib.sink.avro.neo4j.json b/kafka-connect-neo4j/docker/contrib.sink.avro.neo4j.json deleted file mode 100644 index fa4d4e96..00000000 --- a/kafka-connect-neo4j/docker/contrib.sink.avro.neo4j.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "name": "Neo4jSinkConnector", - "config": { - "topics": "my-topic", - "connector.class": "streams.kafka.connect.sink.Neo4jSinkConnector", - "errors.retry.timeout": "-1", - "errors.retry.delay.max.ms": "1000", - "errors.tolerance": "all", - "errors.log.enable": true, - "errors.deadletterqueue.topic.name": "test-error-topic", - "errors.log.include.messages": true, - "neo4j.server.uri": "bolt://neo4j:7687", - "neo4j.authentication.basic.username": "neo4j", - "neo4j.authentication.basic.password": "connect", - "neo4j.encryption.enabled": false, - "neo4j.topic.cypher.my-topic": "MERGE (p:Person{name: event.name, surname: event.surname}) MERGE (f:Family{name: event.surname}) MERGE (p)-[:BELONGS_TO]->(f)" - } -} \ No newline at end of file diff --git a/kafka-connect-neo4j/docker/contrib.sink.string-json.neo4j.json b/kafka-connect-neo4j/docker/contrib.sink.string-json.neo4j.json deleted file mode 100644 index cda74429..00000000 --- a/kafka-connect-neo4j/docker/contrib.sink.string-json.neo4j.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "name": "Neo4jSinkConnector", - "config": { - "topics": "my-topic", - "connector.class": "streams.kafka.connect.sink.Neo4jSinkConnector", - "key.converter": "org.apache.kafka.connect.json.JsonConverter", - "key.converter.schemas.enable": false, - "value.converter": "org.apache.kafka.connect.json.JsonConverter", - "value.converter.schemas.enable": false, - "errors.retry.timeout": "-1", - "errors.retry.delay.max.ms": "1000", - "errors.tolerance": "all", - "errors.log.enable": true, - "errors.log.include.messages": true, - "neo4j.server.uri": "bolt://neo4j:7687", - "neo4j.authentication.basic.username": "neo4j", - "neo4j.authentication.basic.password": "connect", - "neo4j.encryption.enabled": false, - "neo4j.topic.cypher.my-topic": "MERGE (p:Person{name: event.name, surname: event.surname}) MERGE (f:Family{name: event.surname}) MERGE (p)-[:BELONGS_TO]->(f)" - } -} \ No newline at end of file diff --git a/kafka-connect-neo4j/docker/contrib.source.avro.neo4j.json b/kafka-connect-neo4j/docker/contrib.source.avro.neo4j.json deleted file mode 100644 index 58015c81..00000000 --- a/kafka-connect-neo4j/docker/contrib.source.avro.neo4j.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "name": "Neo4jSourceConnectorAVRO", - "config": { - "topic": "my-topic", - "connector.class": "streams.kafka.connect.source.Neo4jSourceConnector", - "key.converter": "io.confluent.connect.avro.AvroConverter", - "value.converter": "io.confluent.connect.avro.AvroConverter", - "key.converter.schema.registry.url": "http://schema_registry:8081", - "value.converter.schema.registry.url": "http://schema_registry:8081", - "neo4j.server.uri": "bolt://neo4j:7687", - "neo4j.authentication.basic.username": "neo4j", - "neo4j.authentication.basic.password": "connect", - "neo4j.encryption.enabled": false, - "neo4j.streaming.poll.interval.msecs": 5000, - "neo4j.streaming.property": "timestamp", - "neo4j.streaming.from": "LAST_COMMITTED", - "neo4j.enforce.schema": true, - "neo4j.source.query": "MATCH (ts:TestSource) WHERE ts.timestamp > $lastCheck RETURN ts.name AS name, ts.timestamp AS timestamp" - } -} \ No newline at end of file diff --git a/kafka-connect-neo4j/docker/contrib.source.string-json.neo4j.json b/kafka-connect-neo4j/docker/contrib.source.string-json.neo4j.json deleted file mode 100644 index 4fed8eaf..00000000 --- a/kafka-connect-neo4j/docker/contrib.source.string-json.neo4j.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "name": "Neo4jSourceConnectorJSON", - "config": { - "topic": "my-topic", - "connector.class": "streams.kafka.connect.source.Neo4jSourceConnector", - "key.converter": "org.apache.kafka.connect.json.JsonConverter", - "value.converter": "org.apache.kafka.connect.json.JsonConverter", - "neo4j.server.uri": "bolt://neo4j:7687", - "neo4j.authentication.basic.username": "neo4j", - "neo4j.authentication.basic.password": "connect", - "neo4j.encryption.enabled": false, - "neo4j.streaming.poll.interval.msecs": 5000, - "neo4j.streaming.property": "timestamp", - "neo4j.streaming.from": "LAST_COMMITTED", - "neo4j.source.query": "MATCH (ts:TestSource) WHERE ts.timestamp > $lastCheck RETURN ts.name AS name, ts.timestamp AS timestamp" - } -} \ No newline at end of file diff --git a/kafka-connect-neo4j/docker/contrib.source.string.neo4j.json b/kafka-connect-neo4j/docker/contrib.source.string.neo4j.json deleted file mode 100644 index def80fd0..00000000 --- a/kafka-connect-neo4j/docker/contrib.source.string.neo4j.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "name": "Neo4jSourceConnectorString", - "config": { - "topic": "my-topic", - "connector.class": "streams.kafka.connect.source.Neo4jSourceConnector", - "key.converter": "org.apache.kafka.connect.storage.StringConverter", - "value.converter": "org.apache.kafka.connect.storage.StringConverter", - "neo4j.server.uri": "bolt://neo4j:7687", - "neo4j.authentication.basic.username": "neo4j", - "neo4j.authentication.basic.password": "connect", - "neo4j.encryption.enabled": false, - "neo4j.streaming.poll.interval.msecs": 5000, - "neo4j.streaming.property": "timestamp", - "neo4j.streaming.from": "LAST_COMMITTED", - "neo4j.source.query": "MATCH (ts:TestSource) WHERE ts.timestamp > $lastCheck RETURN ts.name AS name, ts.timestamp AS timestamp" - } -} \ No newline at end of file diff --git a/kafka-connect-neo4j/docker/docker-compose.yml b/kafka-connect-neo4j/docker/docker-compose.yml deleted file mode 100644 index d2dae351..00000000 --- a/kafka-connect-neo4j/docker/docker-compose.yml +++ /dev/null @@ -1,129 +0,0 @@ ---- -version: '2' - -services: - neo4j: - image: neo4j:4.3-enterprise - hostname: neo4j - container_name: neo4j - ports: - - "7474:7474" - - "7687:7687" - environment: - NEO4J_kafka_bootstrap_servers: broker:9093 - NEO4J_AUTH: neo4j/connect - NEO4J_dbms_memory_heap_max__size: 8G - NEO4J_ACCEPT_LICENSE_AGREEMENT: 'yes' - - zookeeper: - image: confluentinc/cp-zookeeper - hostname: zookeeper - container_name: zookeeper - ports: - - "2181:2181" - environment: - ZOOKEEPER_CLIENT_PORT: 2181 - ZOOKEEPER_TICK_TIME: 2000 - - broker: - image: confluentinc/cp-enterprise-kafka - hostname: broker - container_name: broker - depends_on: - - zookeeper - ports: - - "9092:9092" - expose: - - "9093" - environment: - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker:9093,OUTSIDE://localhost:9092 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,OUTSIDE:PLAINTEXT - KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9093,OUTSIDE://0.0.0.0:9092 - CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: broker:9093 - - # workaround if we change to a custom name the schema_registry fails to start - KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT - - KAFKA_BROKER_ID: 1 - KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181' - KAFKA_METRIC_REPORTERS: io.confluent.metrics.reporter.ConfluentMetricsReporter - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 - CONFLUENT_METRICS_REPORTER_ZOOKEEPER_CONNECT: zookeeper:2181 - CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1 - CONFLUENT_METRICS_ENABLE: 'true' - CONFLUENT_SUPPORT_CUSTOMER_ID: 'anonymous' - - schema_registry: - image: confluentinc/cp-schema-registry - hostname: schema_registry - container_name: schema_registry - depends_on: - - zookeeper - - broker - ports: - - "8081:8081" - environment: - SCHEMA_REGISTRY_HOST_NAME: schema_registry - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181' - - connect: - image: confluentinc/cp-kafka-connect - hostname: connect - container_name: connect - depends_on: - - zookeeper - - broker - - schema_registry - ports: - - "8083:8083" - volumes: - - ./plugins:/tmp/connect-plugins - environment: - CONNECT_BOOTSTRAP_SERVERS: 'broker:9093' - CONNECT_REST_ADVERTISED_HOST_NAME: connect - CONNECT_REST_PORT: 8083 - CONNECT_GROUP_ID: compose-connect-group - CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs - CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1 - CONNECT_OFFSET_FLUSH_INTERVAL_MS: 10000 - CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets - CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1 - CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status - CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1 - CONNECT_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter - CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema_registry:8081' - CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter - CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema_registry:8081' - CONNECT_INTERNAL_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter - CONNECT_INTERNAL_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter - CONNECT_ZOOKEEPER_CONNECT: 'zookeeper:2181' - CONNECT_PLUGIN_PATH: /usr/share/java,/usr/share/confluent-hub-components,/tmp/connect-plugins - CONNECT_LOG4J_LOGGERS: org.apache.zookeeper=DEBUG,org.I0Itec.zkclient=DEBUG,org.reflections=ERROR - command: -# - bash -# - -c -# - | -# confluent-hub install --no-prompt neo4j/kafka-connect-neo4j:latest - /etc/confluent/docker/run - - control-center: - image: confluentinc/cp-enterprise-control-center - hostname: control-center - container_name: control-center - depends_on: - - zookeeper - - broker - - schema_registry - - connect - ports: - - "9021:9021" - environment: - CONTROL_CENTER_BOOTSTRAP_SERVERS: 'broker:9093' - CONTROL_CENTER_ZOOKEEPER_CONNECT: 'zookeeper:2181' - CONTROL_CENTER_CONNECT_CLUSTER: 'connect:8083' - CONTROL_CENTER_REPLICATION_FACTOR: 1 - CONTROL_CENTER_INTERNAL_TOPICS_PARTITIONS: 1 - CONTROL_CENTER_MONITORING_INTERCEPTOR_TOPIC_PARTITIONS: 1 - CONFLUENT_METRICS_TOPIC_REPLICATION: 1 - PORT: 9021 diff --git a/kafka-connect-neo4j/docker/readme.adoc b/kafka-connect-neo4j/docker/readme.adoc deleted file mode 100644 index 4804aac9..00000000 --- a/kafka-connect-neo4j/docker/readme.adoc +++ /dev/null @@ -1,121 +0,0 @@ - -==== Configuration parameters -:environment: neo4j -:id: neo4j - -You can set the following configuration values via Confluent Connect UI, or via REST endpoint - -[cols="3*",subs="attributes",options="header"] -|=== -|Field|Type|Description - -|{environment}.server.uri|String|The Bolt URI (default bolt://localhost:7687) -|{environment}.authentication.type|enum[NONE, BASIC, KERBEROS]| The authentication type (default BASIC) -|{environment}.batch.size|Int|The max number of events processed by the Cypher query (default 1000) -|{environment}.batch.timeout.msecs|Long|The execution timeout for the cypher query (default 30000) -|{environment}.authentication.basic.username|String| The authentication username -|{environment}.authentication.basic.password|String| The authentication password -|{environment}.authentication.basic.realm|String| The authentication realm -|{environment}.authentication.kerberos.ticket|String| The Kerberos ticket -|{environment}.encryption.enabled|Boolean| If the encryption is enabled (default false) -|{environment}.encryption.trust.strategy|enum[TRUST_ALL_CERTIFICATES, TRUST_CUSTOM_CA_SIGNED_CERTIFICATES, TRUST_SYSTEM_CA_SIGNED_CERTIFICATES]| The Neo4j trust strategy (default TRUST_ALL_CERTIFICATES) -|{environment}.encryption.ca.certificate.path|String| The path of the certificate -|{environment}.connection.max.lifetime.msecs|Long| The max Neo4j connection lifetime (default 1 hour) -|{environment}.connection.acquisition.timeout.msecs|Long| The max Neo4j acquisition timeout (default 1 hour) -|{environment}.connection.liveness.check.timeout.msecs|Long| The max Neo4j liveness check timeout (default 1 hour) -|{environment}.connection.max.pool.size|Int| The max pool size (default 100) -|{environment}.load.balance.strategy|enum[ROUND_ROBIN, LEAST_CONNECTED]| The Neo4j load balance strategy (default LEAST_CONNECTED) -|{environment}.batch.parallelize|Boolean|(default true) While concurrent batch processing improves throughput, it might cause out-of-order handling of events. Set to `false` if you need application of messages with strict ordering, e.g. for change-data-capture (CDC) events. -|=== - -==== Configuring the stack - -Start the compose file - -[source,bash] ----- -docker-compose up -d ----- - -You can access your Neo4j instance under: http://localhost:7474, log in with `neo4j` as username and `connect` as password (see the docker-compose file to change it). - -===== Plugin installation - -You can choose your preferred way in order to install the plugin: - -* *Build it locally* -+ --- -Build the project by running the following command: - -[source,bash] ----- -mvn clean install ----- - -Create a directory `plugins` at the same level of the compose file and unzip the file `neo4j-kafka-connect-neo4j-.zip` inside it. --- - -* *Download the zip from the Confluent Hub* - -+ --- -Please go to the Confluent Hub page of the plugin: - -https://www.confluent.io/connector/kafka-connect-neo4j-sink/ - -And click to the **Download Connector** button. - -Create a directory `plugins` at the same level of the compose file and unzip the file `neo4j-kafka-connect-neo4j-.zip` inside it. --- - -* *Download and install the plugin via Confluent Hub client* -+ --- -If you are using the provided compose file you can easily install the plugin by using the Confluent Hub. - -Once the compose file is up and running you can install the plugin by executing the following command: - -[source,bash] ----- -docker exec -it connect confluent-hub install neo4j/kafka-connect-neo4j: ----- - -When the installation will ask: - -[source,bash] ----- -The component can be installed in any of the following Confluent Platform installations: ----- - -Please prefer the solution `(where this tool is installed)` and then go ahead with the default options. - -At the end of the process the plugin is automatically installed. --- - -==== Multi Database Support - -Neo4j 4.0 Enterprise has https://neo4j.com/docs/operations-manual/4.0/manage-databases/[multi-tenancy support], -in order to support this feature you can define into the json (or via the Confluent UI) -a param named `neo4j.database` which is the targeted database name. - -*N.b.* If no value is specified the connector will use the Neo4j's default db. - -==== Create the Sink Instance - -To create the Sink instance and configure your preferred ingestion strategy, you can follow instructions described -into <> and <> -sections. - -==== Create the Source Instance - -To create the Source instance and configure your preferred ingestion strategy, you can follow instructions described -into <> section. - -===== Use the Kafka Connect Datagen - -In order to generate a sample dataset you can use Kafka Connect Datagen as explained in <> section. - -[NOTE] -Before start using the data generator please create indexes in Neo4j (in order to speed-up the import process) - diff --git a/kafka-connect-neo4j/pom.xml b/kafka-connect-neo4j/pom.xml deleted file mode 100644 index 33e8662b..00000000 --- a/kafka-connect-neo4j/pom.xml +++ /dev/null @@ -1,160 +0,0 @@ - - 4.0.0 - - org.neo4j - kafka-connect-neo4j - 2.0.2 - jar - - Kafka Connect Neo4j - A Kafka Connect Neo4j Connector for kafka-connect-neo4j - - - org.neo4j - neo4j-streams-parent - 4.1.2 - - - - 5.0.0 - 0.11.1 - 3.1.0 - 0.3.141 - 32.1.1-jre - - - - - confluent - http://packages.confluent.io/maven/ - - - - - - org.apache.kafka - connect-api - ${kafka.version} - provided - - - com.github.jcustenborder.kafka.connect - connect-utils - ${kafka.connect.utils.version} - - - - com.google.guava - guava - ${google.guava.version} - provided - - - - org.neo4j - neo4j-streams-common - ${project.parent.version} - - - - org.neo4j - neo4j-streams-test-support - ${project.parent.version} - test - - - - org.neo4j.driver - neo4j-java-driver - - - - - - - maven-resources-plugin - - ${project.build.outputDirectory} - - - src/main/resources - true - - - - - - io.confluent - kafka-connect-maven-plugin - ${confluent.connect.plugin.version} - - - - kafka-connect - - - - - ${project.basedir} - doc/ - - README* - LICENSE* - NOTICE* - licenses/ - docker/ - - - - - sink - source - - neo4j - organization - Neo4j, Inc. - https://neo4j.com/ - Neo4j Connector - https://neo4j-contrib.github.io/neo4j-streams/#_kafka_connect - It's a basic Apache Kafka Connect Neo4j Connector which allows moving data from Kafka topics into Neo4j via Cypher templated queries and vice versa. - assets/neo4j-logo.png - Neo4j Labs]]> - https://github.com/neo4j-contrib/neo4j-streams/tree/master/kafka-connect-neo4j - ${project.issueManagement.url} - true - - neo4j - nosql - json - graph - nodes - relationships - cypher - - - - - - - - - - - oss-kafka-connect - - - com.google.guava - guava - ${google.guava.version} - compile - - - - false - - - - - diff --git a/kafka-connect-neo4j/src/main/assembly/package.xml b/kafka-connect-neo4j/src/main/assembly/package.xml deleted file mode 100644 index 5305539e..00000000 --- a/kafka-connect-neo4j/src/main/assembly/package.xml +++ /dev/null @@ -1,40 +0,0 @@ - - - package - - dir - - false - - - ${project.basedir} - share/doc/${project.name}/ - - README* - LICENSE* - NOTICE* - licenses/ - - - - ${project.basedir}/config - etc/${project.name} - - * - - - - - - share/kotlin/${project.name} - true - true - - org.apache.kafka:connect-api - - - - diff --git a/kafka-connect-neo4j/src/main/kotlin/streams/kafka/connect/common/Neo4jConnectorConfig.kt b/kafka-connect-neo4j/src/main/kotlin/streams/kafka/connect/common/Neo4jConnectorConfig.kt deleted file mode 100644 index 6a759ec7..00000000 --- a/kafka-connect-neo4j/src/main/kotlin/streams/kafka/connect/common/Neo4jConnectorConfig.kt +++ /dev/null @@ -1,351 +0,0 @@ -package streams.kafka.connect.common - -import com.github.jcustenborder.kafka.connect.utils.config.ConfigKeyBuilder -import com.github.jcustenborder.kafka.connect.utils.config.ConfigUtils -import com.github.jcustenborder.kafka.connect.utils.config.ValidEnum -import com.github.jcustenborder.kafka.connect.utils.config.recommenders.Recommenders -import com.github.jcustenborder.kafka.connect.utils.config.validators.Validators -import com.github.jcustenborder.kafka.connect.utils.config.validators.filesystem.ValidFile -import org.apache.kafka.common.config.AbstractConfig -import org.apache.kafka.common.config.ConfigDef -import org.apache.kafka.common.config.ConfigException -import org.neo4j.driver.* -import org.neo4j.driver.internal.async.pool.PoolSettings -import org.neo4j.driver.net.ServerAddress -import streams.kafka.connect.sink.AuthenticationType -import streams.kafka.connect.utils.PropertiesUtil -import java.io.File -import java.net.URI -import java.time.Duration -import java.util.concurrent.TimeUnit - -object ConfigGroup { - const val ENCRYPTION = "Encryption" - const val CONNECTION = "Connection" - const val AUTHENTICATION = "Authentication" - const val TOPIC_CYPHER_MAPPING = "Topic Cypher Mapping" - const val ERROR_REPORTING = "Error Reporting" - const val BATCH = "Batch Management" - const val RETRY = "Retry Strategy" - const val DEPRECATED = "Deprecated Properties (please check the documentation)" -} - -enum class ConnectorType { SINK, SOURCE } - -open class Neo4jConnectorConfig(configDef: ConfigDef, - originals: Map<*, *>, - private val type: ConnectorType): AbstractConfig(configDef, originals) { - val encryptionEnabled: Boolean - val encryptionTrustStrategy: Config.TrustStrategy.Strategy - var encryptionCACertificateFile: File? = null - - val authenticationType: AuthenticationType - val authenticationUsername: String - val authenticationPassword: String - val authenticationRealm: String - val authenticationKerberosTicket: String - - val serverUri: List - val connectionMaxConnectionLifetime: Long - val connectionLivenessCheckTimeout: Long - val connectionPoolMaxSize: Int - val connectionAcquisitionTimeout: Long - - val retryBackoff: Long - val retryMaxAttempts: Int - - val batchTimeout: Long - val batchSize: Int - - val database: String - - init { - database = getString(DATABASE) - encryptionEnabled = getBoolean(ENCRYPTION_ENABLED) - encryptionTrustStrategy = ConfigUtils - .getEnum(Config.TrustStrategy.Strategy::class.java, this, ENCRYPTION_TRUST_STRATEGY) - val encryptionCACertificatePATH = getString(ENCRYPTION_CA_CERTIFICATE_PATH) ?: "" - if (encryptionCACertificatePATH != "") { - encryptionCACertificateFile = File(encryptionCACertificatePATH) - } - - authenticationType = ConfigUtils - .getEnum(AuthenticationType::class.java, this, AUTHENTICATION_TYPE) - authenticationRealm = getString(AUTHENTICATION_BASIC_REALM) - authenticationUsername = getString(AUTHENTICATION_BASIC_USERNAME) - authenticationPassword = getPassword(AUTHENTICATION_BASIC_PASSWORD).value() - authenticationKerberosTicket = getPassword(AUTHENTICATION_KERBEROS_TICKET).value() - - serverUri = getString(SERVER_URI).split(",").map { URI(it) } - connectionLivenessCheckTimeout = getLong(CONNECTION_LIVENESS_CHECK_TIMEOUT_MSECS) - connectionMaxConnectionLifetime = getLong(CONNECTION_MAX_CONNECTION_LIFETIME_MSECS) - connectionPoolMaxSize = getInt(CONNECTION_POOL_MAX_SIZE) - connectionAcquisitionTimeout = getLong(CONNECTION_MAX_CONNECTION_ACQUISITION_TIMEOUT_MSECS) - - retryBackoff = getLong(RETRY_BACKOFF_MSECS) - retryMaxAttempts = getInt(RETRY_MAX_ATTEMPTS) - - batchTimeout = getLong(BATCH_TIMEOUT_MSECS) - batchSize = getInt(BATCH_SIZE) - } - - fun hasSecuredURI() = serverUri.any { it.scheme.endsWith("+s", true) || it.scheme.endsWith("+ssc", true) } - - fun createDriver(): Driver { - val configBuilder = Config.builder() - configBuilder.withUserAgent("neo4j-kafka-connect-$type/${PropertiesUtil.getVersion()}") - - if (!this.hasSecuredURI()) { - if (this.encryptionEnabled) { - configBuilder.withEncryption() - val trustStrategy: Config.TrustStrategy = when (this.encryptionTrustStrategy) { - Config.TrustStrategy.Strategy.TRUST_ALL_CERTIFICATES -> Config.TrustStrategy.trustAllCertificates() - Config.TrustStrategy.Strategy.TRUST_SYSTEM_CA_SIGNED_CERTIFICATES -> Config.TrustStrategy.trustSystemCertificates() - Config.TrustStrategy.Strategy.TRUST_CUSTOM_CA_SIGNED_CERTIFICATES -> Config.TrustStrategy.trustCustomCertificateSignedBy(this.encryptionCACertificateFile) - else -> { - throw ConfigException(ENCRYPTION_TRUST_STRATEGY, this.encryptionTrustStrategy.toString(), "Encryption Trust Strategy is not supported.") - } - } - configBuilder.withTrustStrategy(trustStrategy) - } else { - configBuilder.withoutEncryption() - } - } - - val authToken = when (this.authenticationType) { - AuthenticationType.NONE -> AuthTokens.none() - AuthenticationType.BASIC -> { - if (this.authenticationRealm != "") { - AuthTokens.basic(this.authenticationUsername, this.authenticationPassword, this.authenticationRealm) - } else { - AuthTokens.basic(this.authenticationUsername, this.authenticationPassword) - } - } - AuthenticationType.KERBEROS -> AuthTokens.kerberos(this.authenticationKerberosTicket) - } - configBuilder.withMaxConnectionPoolSize(this.connectionPoolMaxSize) - configBuilder.withMaxConnectionLifetime(this.connectionMaxConnectionLifetime, TimeUnit.MILLISECONDS) - configBuilder.withConnectionAcquisitionTimeout(this.connectionAcquisitionTimeout, TimeUnit.MILLISECONDS) - configBuilder.withMaxTransactionRetryTime(this.retryBackoff, TimeUnit.MILLISECONDS) - configBuilder.withConnectionLivenessCheckTimeout(this.connectionLivenessCheckTimeout, TimeUnit.MINUTES) - configBuilder.withResolver { address -> this.serverUri.map { ServerAddress.of(it.host, it.port) }.toSet() } - val neo4jConfig = configBuilder.build() - - return GraphDatabase.driver(this.serverUri.firstOrNull(), authToken, neo4jConfig) - } - - fun createSessionConfig(bookmarks: List = emptyList()): SessionConfig { - val sessionConfigBuilder = SessionConfig.builder() - if (this.database.isNotBlank()) { - sessionConfigBuilder.withDatabase(this.database) - } - val accessMode = if (type == ConnectorType.SOURCE) { - AccessMode.READ - } else { - AccessMode.WRITE - } - sessionConfigBuilder.withDefaultAccessMode(accessMode) - sessionConfigBuilder.withBookmarks(bookmarks) - return sessionConfigBuilder.build() - } - - fun createTransactionConfig(): TransactionConfig { - val batchTimeout = this.batchTimeout - return if (batchTimeout > 0) { - TransactionConfig.builder() - .withTimeout(Duration.ofMillis(batchTimeout)) - .build() - } else { - TransactionConfig.empty() - } - } - - companion object { - const val SERVER_URI = "neo4j.server.uri" - const val DATABASE = "neo4j.database" - - const val AUTHENTICATION_TYPE = "neo4j.authentication.type" - const val AUTHENTICATION_BASIC_USERNAME = "neo4j.authentication.basic.username" - const val AUTHENTICATION_BASIC_PASSWORD = "neo4j.authentication.basic.password" - const val AUTHENTICATION_BASIC_REALM = "neo4j.authentication.basic.realm" - const val AUTHENTICATION_KERBEROS_TICKET = "neo4j.authentication.kerberos.ticket" - - const val ENCRYPTION_ENABLED = "neo4j.encryption.enabled" - const val ENCRYPTION_TRUST_STRATEGY = "neo4j.encryption.trust.strategy" - const val ENCRYPTION_CA_CERTIFICATE_PATH = "neo4j.encryption.ca.certificate.path" - - const val CONNECTION_MAX_CONNECTION_LIFETIME_MSECS = "neo4j.connection.max.lifetime.msecs" - const val CONNECTION_MAX_CONNECTION_ACQUISITION_TIMEOUT_MSECS = "neo4j.connection.acquisition.timeout.msecs" - const val CONNECTION_LIVENESS_CHECK_TIMEOUT_MSECS = "neo4j.connection.liveness.check.timeout.msecs" - const val CONNECTION_POOL_MAX_SIZE = "neo4j.connection.max.pool.size" - - const val BATCH_SIZE = "neo4j.batch.size" - const val BATCH_TIMEOUT_MSECS = "neo4j.batch.timeout.msecs" - - const val RETRY_BACKOFF_MSECS = "neo4j.retry.backoff.msecs" - const val RETRY_MAX_ATTEMPTS = "neo4j.retry.max.attemps" - - const val CONNECTION_POOL_MAX_SIZE_DEFAULT = 100 - val BATCH_TIMEOUT_DEFAULT = TimeUnit.SECONDS.toMillis(0L) - const val BATCH_SIZE_DEFAULT = 1000 - val RETRY_BACKOFF_DEFAULT = TimeUnit.SECONDS.toMillis(30L) - const val RETRY_MAX_ATTEMPTS_DEFAULT = 5 - - // Default values optimizations for Aura please look at: https://aura.support.neo4j.com/hc/en-us/articles/1500002493281-Neo4j-Java-driver-settings-for-Aura - val CONNECTION_MAX_CONNECTION_LIFETIME_MSECS_DEFAULT = Duration.ofMinutes(8).toMillis() - val CONNECTION_LIVENESS_CHECK_TIMEOUT_MSECS_DEFAULT = Duration.ofMinutes(2).toMillis() - - - fun isValidQuery(session: Session, query: String) = try { - session.run("EXPLAIN $query") - true - } catch (e: Exception) { - false - } - - fun config(): ConfigDef = ConfigDef() - .define(ConfigKeyBuilder - .of(AUTHENTICATION_TYPE, ConfigDef.Type.STRING) - .documentation(PropertiesUtil.getProperty(AUTHENTICATION_TYPE)) - .importance(ConfigDef.Importance.HIGH) - .defaultValue(AuthenticationType.BASIC.toString()) - .group(ConfigGroup.AUTHENTICATION) - .validator(ValidEnum.of(AuthenticationType::class.java)) - .build()) - .define(ConfigKeyBuilder - .of(AUTHENTICATION_BASIC_USERNAME, ConfigDef.Type.STRING) - .documentation(PropertiesUtil.getProperty(AUTHENTICATION_BASIC_USERNAME)) - .importance(ConfigDef.Importance.HIGH) - .defaultValue("") - .group(ConfigGroup.AUTHENTICATION) - .recommender(Recommenders.visibleIf(AUTHENTICATION_TYPE, AuthenticationType.BASIC.toString())) - .build()) - .define(ConfigKeyBuilder - .of(AUTHENTICATION_BASIC_PASSWORD, ConfigDef.Type.PASSWORD) - .documentation(PropertiesUtil.getProperty(AUTHENTICATION_BASIC_PASSWORD)) - .importance(ConfigDef.Importance.HIGH) - .defaultValue("") - .group(ConfigGroup.AUTHENTICATION) - .recommender(Recommenders.visibleIf(AUTHENTICATION_TYPE, AuthenticationType.BASIC.toString())) - .build()) - .define(ConfigKeyBuilder - .of(AUTHENTICATION_BASIC_REALM, ConfigDef.Type.STRING) - .documentation(PropertiesUtil.getProperty(AUTHENTICATION_BASIC_REALM)) - .importance(ConfigDef.Importance.HIGH) - .defaultValue("") - .group(ConfigGroup.AUTHENTICATION) - .recommender(Recommenders.visibleIf(AUTHENTICATION_TYPE, AuthenticationType.BASIC.toString())) - .build()) - .define(ConfigKeyBuilder - .of(AUTHENTICATION_KERBEROS_TICKET, ConfigDef.Type.PASSWORD) - .documentation(PropertiesUtil.getProperty(AUTHENTICATION_KERBEROS_TICKET)) - .importance(ConfigDef.Importance.HIGH) - .defaultValue("") - .group(ConfigGroup.AUTHENTICATION) - .recommender(Recommenders.visibleIf(AUTHENTICATION_TYPE, AuthenticationType.KERBEROS.toString())) - .build()) - .define(ConfigKeyBuilder - .of(SERVER_URI, ConfigDef.Type.STRING) - .documentation(PropertiesUtil.getProperty(SERVER_URI)) - .importance(ConfigDef.Importance.HIGH) - .defaultValue("bolt://localhost:7687") - .group(ConfigGroup.CONNECTION) - .validator(Validators.validURI("bolt", "bolt+routing", "bolt+s", "bolt+ssc","neo4j", "neo4j+s", "neo4j+ssc")) - .build()) - .define(ConfigKeyBuilder - .of(CONNECTION_POOL_MAX_SIZE, ConfigDef.Type.INT) - .documentation(PropertiesUtil.getProperty(CONNECTION_POOL_MAX_SIZE)) - .importance(ConfigDef.Importance.LOW) - .defaultValue(CONNECTION_POOL_MAX_SIZE_DEFAULT) - .group(ConfigGroup.CONNECTION) - .validator(ConfigDef.Range.atLeast(1)) - .build()) - .define(ConfigKeyBuilder - .of(CONNECTION_MAX_CONNECTION_LIFETIME_MSECS, ConfigDef.Type.LONG) - .documentation(PropertiesUtil.getProperty(CONNECTION_MAX_CONNECTION_LIFETIME_MSECS)) - .importance(ConfigDef.Importance.LOW) - .defaultValue(CONNECTION_MAX_CONNECTION_LIFETIME_MSECS_DEFAULT) - .group(ConfigGroup.CONNECTION) - .validator(ConfigDef.Range.atLeast(1)) - .build()) - .define(ConfigKeyBuilder - .of(CONNECTION_LIVENESS_CHECK_TIMEOUT_MSECS, ConfigDef.Type.LONG) - .documentation(PropertiesUtil.getProperty(CONNECTION_LIVENESS_CHECK_TIMEOUT_MSECS)) - .importance(ConfigDef.Importance.LOW) - .defaultValue(CONNECTION_LIVENESS_CHECK_TIMEOUT_MSECS_DEFAULT) - .group(ConfigGroup.CONNECTION) - .validator(ConfigDef.Range.atLeast(1)) - .build()) - .define(ConfigKeyBuilder - .of(CONNECTION_MAX_CONNECTION_ACQUISITION_TIMEOUT_MSECS, ConfigDef.Type.LONG) - .documentation(PropertiesUtil.getProperty(CONNECTION_MAX_CONNECTION_ACQUISITION_TIMEOUT_MSECS)) - .importance(ConfigDef.Importance.LOW) - .defaultValue(PoolSettings.DEFAULT_CONNECTION_ACQUISITION_TIMEOUT) - .group(ConfigGroup.CONNECTION) - .validator(ConfigDef.Range.atLeast(1)) - .build()) - .define(ConfigKeyBuilder - .of(ENCRYPTION_ENABLED, ConfigDef.Type.BOOLEAN) - .documentation(PropertiesUtil.getProperty(ENCRYPTION_ENABLED)) - .importance(ConfigDef.Importance.HIGH) - .defaultValue(false) - .group(ConfigGroup.ENCRYPTION).build()) - .define(ConfigKeyBuilder - .of(ENCRYPTION_TRUST_STRATEGY, ConfigDef.Type.STRING) - .documentation(PropertiesUtil.getProperty(ENCRYPTION_TRUST_STRATEGY)) - .importance(ConfigDef.Importance.MEDIUM) - .defaultValue(Config.TrustStrategy.Strategy.TRUST_ALL_CERTIFICATES.toString()) - .group(ConfigGroup.ENCRYPTION) - .validator(ValidEnum.of(Config.TrustStrategy.Strategy::class.java)) - .recommender(Recommenders.visibleIf(ENCRYPTION_ENABLED, true)) - .build()) - .define(ConfigKeyBuilder - .of(ENCRYPTION_CA_CERTIFICATE_PATH, ConfigDef.Type.STRING) - .documentation(PropertiesUtil.getProperty(ENCRYPTION_CA_CERTIFICATE_PATH)) - .importance(ConfigDef.Importance.MEDIUM) - .defaultValue("") - .group(ConfigGroup.ENCRYPTION) - .validator(Validators.blankOr(ValidFile.of())) // TODO check - .recommender(Recommenders.visibleIf( - ENCRYPTION_TRUST_STRATEGY, - Config.TrustStrategy.Strategy.TRUST_CUSTOM_CA_SIGNED_CERTIFICATES.toString())) - .build()) - .define(ConfigKeyBuilder - .of(BATCH_SIZE, ConfigDef.Type.INT) - .documentation(PropertiesUtil.getProperty(BATCH_SIZE)) - .importance(ConfigDef.Importance.LOW) - .defaultValue(BATCH_SIZE_DEFAULT) - .group(ConfigGroup.BATCH) - .validator(ConfigDef.Range.atLeast(1)) - .build()) - .define(ConfigKeyBuilder - .of(BATCH_TIMEOUT_MSECS, ConfigDef.Type.LONG) - .documentation(PropertiesUtil.getProperty(BATCH_TIMEOUT_MSECS)) - .importance(ConfigDef.Importance.LOW) - .defaultValue(BATCH_TIMEOUT_DEFAULT) - .group(ConfigGroup.BATCH) - .validator(ConfigDef.Range.atLeast(0)).build()) - .define(ConfigKeyBuilder - .of(RETRY_BACKOFF_MSECS, ConfigDef.Type.LONG) - .documentation(PropertiesUtil.getProperty(RETRY_BACKOFF_MSECS)) - .importance(ConfigDef.Importance.MEDIUM) - .defaultValue(RETRY_BACKOFF_DEFAULT) - .group(ConfigGroup.RETRY) - .validator(ConfigDef.Range.atLeast(1)) - .build()) - .define(ConfigKeyBuilder - .of(RETRY_MAX_ATTEMPTS, ConfigDef.Type.INT) - .documentation(PropertiesUtil.getProperty(RETRY_MAX_ATTEMPTS)) - .importance(ConfigDef.Importance.MEDIUM) - .defaultValue(RETRY_MAX_ATTEMPTS_DEFAULT) - .group(ConfigGroup.RETRY) - .validator(ConfigDef.Range.atLeast(1)).build()) - .define(ConfigKeyBuilder - .of(DATABASE, ConfigDef.Type.STRING) - .documentation(PropertiesUtil.getProperty(DATABASE)) - .importance(ConfigDef.Importance.HIGH) - .group(ConfigGroup.CONNECTION) - .defaultValue("") - .build()) - } -} \ No newline at end of file diff --git a/kafka-connect-neo4j/src/main/kotlin/streams/kafka/connect/sink/EventBuilder.kt b/kafka-connect-neo4j/src/main/kotlin/streams/kafka/connect/sink/EventBuilder.kt deleted file mode 100644 index ce3cedaf..00000000 --- a/kafka-connect-neo4j/src/main/kotlin/streams/kafka/connect/sink/EventBuilder.kt +++ /dev/null @@ -1,33 +0,0 @@ -package streams.kafka.connect.sink - -import org.apache.kafka.connect.sink.SinkRecord -import org.slf4j.Logger -import org.slf4j.LoggerFactory -import streams.kafka.connect.utils.toStreamsSinkEntity -import streams.service.StreamsSinkEntity - -class EventBuilder { - private var batchSize: Int? = null - private lateinit var sinkRecords: Collection - - fun withBatchSize(batchSize: Int): EventBuilder { - this.batchSize = batchSize - return this - } - - fun withSinkRecords(sinkRecords: Collection): EventBuilder { - this.sinkRecords = sinkRecords - return this - } - - fun build(): Map>> { // > - val batchSize = this.batchSize!! - return this.sinkRecords - .groupBy { it.topic() } - .mapValues { entry -> - val value = entry.value.map { it.toStreamsSinkEntity() } - if (batchSize > value.size) listOf(value) else value.chunked(batchSize) - } - } - -} \ No newline at end of file diff --git a/kafka-connect-neo4j/src/main/kotlin/streams/kafka/connect/sink/Neo4jSinkConnector.kt b/kafka-connect-neo4j/src/main/kotlin/streams/kafka/connect/sink/Neo4jSinkConnector.kt deleted file mode 100644 index cfbd042e..00000000 --- a/kafka-connect-neo4j/src/main/kotlin/streams/kafka/connect/sink/Neo4jSinkConnector.kt +++ /dev/null @@ -1,40 +0,0 @@ -package streams.kafka.connect.sink - -import com.github.jcustenborder.kafka.connect.utils.config.* -import org.apache.kafka.common.config.ConfigDef -import org.apache.kafka.connect.connector.Task -import org.apache.kafka.connect.sink.SinkConnector -import org.slf4j.LoggerFactory -import streams.kafka.connect.utils.PropertiesUtil - -@Title("Neo4j Sink Connector") -@Description("The Neo4j Sink connector reads data from Kafka and and writes the data to Neo4j using a Cypher Template") -@DocumentationTip("If you need to control the size of transaction that is submitted to Neo4j you try adjusting the ``consumer.max.poll.records`` setting in the worker.properties for Kafka Connect.") -@DocumentationNote("For each topic you can provide a Cypher Template by using the following syntax ``neo4j.topic.cypher.=``") -class Neo4jSinkConnector: SinkConnector() { - private lateinit var settings: Map - private lateinit var config: Neo4jSinkConnectorConfig - override fun taskConfigs(maxTasks: Int): MutableList> { - return TaskConfigs.multiple(settings, maxTasks) - } - - override fun start(props: MutableMap?) { - settings = props!! - config = Neo4jSinkConnectorConfig(settings) - } - - override fun stop() {} - - override fun version(): String { - return PropertiesUtil.getVersion() - } - - override fun taskClass(): Class { - return Neo4jSinkTask::class.java - } - - override fun config(): ConfigDef { - return Neo4jSinkConnectorConfig.config() - } - -} \ No newline at end of file diff --git a/kafka-connect-neo4j/src/main/kotlin/streams/kafka/connect/sink/Neo4jSinkConnectorConfig.kt b/kafka-connect-neo4j/src/main/kotlin/streams/kafka/connect/sink/Neo4jSinkConnectorConfig.kt deleted file mode 100644 index f7b4da7b..00000000 --- a/kafka-connect-neo4j/src/main/kotlin/streams/kafka/connect/sink/Neo4jSinkConnectorConfig.kt +++ /dev/null @@ -1,102 +0,0 @@ -package streams.kafka.connect.sink - -import com.github.jcustenborder.kafka.connect.utils.config.ConfigKeyBuilder -import org.apache.kafka.common.config.ConfigDef -import org.apache.kafka.common.config.ConfigException -import org.apache.kafka.connect.sink.SinkTask -import streams.kafka.connect.common.ConfigGroup -import streams.kafka.connect.common.ConnectorType -import streams.kafka.connect.common.Neo4jConnectorConfig -import streams.kafka.connect.utils.PropertiesUtil -import streams.service.TopicType -import streams.service.TopicUtils -import streams.service.Topics -import streams.service.sink.strategy.SourceIdIngestionStrategyConfig - -enum class AuthenticationType { - NONE, BASIC, KERBEROS -} - -class Neo4jSinkConnectorConfig(originals: Map<*, *>): Neo4jConnectorConfig(config(), originals, ConnectorType.SINK) { - - val parallelBatches: Boolean - - val topics: Topics - - val strategyMap: Map - - val kafkaBrokerProperties: Map - - init { - val sourceIdStrategyConfig = SourceIdIngestionStrategyConfig(getString(TOPIC_CDC_SOURCE_ID_LABEL_NAME), getString(TOPIC_CDC_SOURCE_ID_ID_NAME)) - topics = Topics.from(originals as Map, "streams.sink." to "neo4j.") - strategyMap = TopicUtils.toStrategyMap(topics, sourceIdStrategyConfig) - - parallelBatches = getBoolean(BATCH_PARALLELIZE) - val kafkaPrefix = "kafka." - kafkaBrokerProperties = originals - .filterKeys { it.startsWith(kafkaPrefix) } - .mapKeys { it.key.substring(kafkaPrefix.length) } - validateAllTopics(originals) - } - - private fun validateAllTopics(originals: Map<*, *>) { - TopicUtils.validate(this.topics) - val topics = if (originals.containsKey(SinkTask.TOPICS_CONFIG)) { - originals[SinkTask.TOPICS_CONFIG].toString() - .split(",") - .map { it.trim() } - .sorted() - } else { // TODO manage regexp - emptyList() - } - val allTopics = this.topics - .allTopics() - .sorted() - if (topics != allTopics) { - throw ConfigException("There is a mismatch between topics defined into the property `${SinkTask.TOPICS_CONFIG}` ($topics) and configured topics ($allTopics)") - } - } - - companion object { - - const val BATCH_PARALLELIZE = "neo4j.batch.parallelize" - - const val TOPIC_CYPHER_PREFIX = "neo4j.topic.cypher." - const val TOPIC_CDC_SOURCE_ID = "neo4j.topic.cdc.sourceId" - const val TOPIC_CDC_SOURCE_ID_LABEL_NAME = "neo4j.topic.cdc.sourceId.labelName" - const val TOPIC_CDC_SOURCE_ID_ID_NAME = "neo4j.topic.cdc.sourceId.idName" - const val TOPIC_PATTERN_NODE_PREFIX = "neo4j.topic.pattern.node." - const val TOPIC_PATTERN_RELATIONSHIP_PREFIX = "neo4j.topic.pattern.relationship." - const val TOPIC_CDC_SCHEMA = "neo4j.topic.cdc.schema" - const val TOPIC_CUD = "neo4j.topic.cud" - - private val sourceIdIngestionStrategyConfig = SourceIdIngestionStrategyConfig() - - fun config(): ConfigDef = Neo4jConnectorConfig.config() - .define(ConfigKeyBuilder.of(TOPIC_CDC_SOURCE_ID, ConfigDef.Type.STRING) - .documentation(PropertiesUtil.getProperty(TOPIC_CDC_SOURCE_ID)).importance(ConfigDef.Importance.HIGH) - .defaultValue("").group(ConfigGroup.TOPIC_CYPHER_MAPPING) - .build()) - .define(ConfigKeyBuilder.of(TOPIC_CDC_SOURCE_ID_LABEL_NAME, ConfigDef.Type.STRING) - .documentation(PropertiesUtil.getProperty(TOPIC_CDC_SOURCE_ID_LABEL_NAME)).importance(ConfigDef.Importance.HIGH) - .defaultValue(sourceIdIngestionStrategyConfig.labelName).group(ConfigGroup.TOPIC_CYPHER_MAPPING) - .build()) - .define(ConfigKeyBuilder.of(TOPIC_CDC_SOURCE_ID_ID_NAME, ConfigDef.Type.STRING) - .documentation(PropertiesUtil.getProperty(TOPIC_CDC_SOURCE_ID_ID_NAME)).importance(ConfigDef.Importance.HIGH) - .defaultValue(sourceIdIngestionStrategyConfig.idName).group(ConfigGroup.TOPIC_CYPHER_MAPPING) - .build()) - .define(ConfigKeyBuilder.of(TOPIC_CDC_SCHEMA, ConfigDef.Type.STRING) - .documentation(PropertiesUtil.getProperty(TOPIC_CDC_SCHEMA)).importance(ConfigDef.Importance.HIGH) - .defaultValue("").group(ConfigGroup.TOPIC_CYPHER_MAPPING) - .build()) - .define(ConfigKeyBuilder.of(BATCH_PARALLELIZE, ConfigDef.Type.BOOLEAN) - .documentation(PropertiesUtil.getProperty(BATCH_PARALLELIZE)).importance(ConfigDef.Importance.MEDIUM) - .defaultValue(true).group(ConfigGroup.BATCH) - .build()) - .define(ConfigKeyBuilder.of(TOPIC_CUD, ConfigDef.Type.STRING) - .documentation(PropertiesUtil.getProperty(TOPIC_CUD)).importance(ConfigDef.Importance.HIGH) - .defaultValue("").group(ConfigGroup.TOPIC_CYPHER_MAPPING) - .build()) - } -} \ No newline at end of file diff --git a/kafka-connect-neo4j/src/main/kotlin/streams/kafka/connect/sink/Neo4jSinkService.kt b/kafka-connect-neo4j/src/main/kotlin/streams/kafka/connect/sink/Neo4jSinkService.kt deleted file mode 100644 index 33fda096..00000000 --- a/kafka-connect-neo4j/src/main/kotlin/streams/kafka/connect/sink/Neo4jSinkService.kt +++ /dev/null @@ -1,103 +0,0 @@ -package streams.kafka.connect.sink - -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.ExperimentalCoroutinesApi -import kotlinx.coroutines.ObsoleteCoroutinesApi -import kotlinx.coroutines.async -import kotlinx.coroutines.awaitAll -import kotlinx.coroutines.runBlocking -import org.apache.kafka.connect.errors.ConnectException -import org.neo4j.driver.Bookmark -import org.neo4j.driver.Driver -import org.neo4j.driver.TransactionConfig -import org.neo4j.driver.exceptions.ClientException -import org.neo4j.driver.exceptions.TransientException -import org.slf4j.Logger -import org.slf4j.LoggerFactory -import streams.extensions.errors -import streams.service.StreamsSinkEntity -import streams.service.StreamsSinkService -import streams.utils.StreamsUtils -import streams.utils.retryForException -import kotlin.streams.toList - - -class Neo4jSinkService(private val config: Neo4jSinkConnectorConfig): - StreamsSinkService(Neo4jStrategyStorage(config)) { - - private val log: Logger = LoggerFactory.getLogger(Neo4jSinkService::class.java) - - private val driver: Driver = config.createDriver() - private val transactionConfig: TransactionConfig = config.createTransactionConfig() - - private val bookmarks = mutableListOf() - - fun close() { - StreamsUtils.closeSafetely(driver) { - log.info("Error while closing Driver instance:", it) - } - } - - override fun write(query: String, events: Collection) { - val data = mapOf("events" to events) - driver.session(config.createSessionConfig(bookmarks)).use { session -> - try { - runBlocking { - retryForException(exceptions = arrayOf(ClientException::class.java, TransientException::class.java), - retries = config.retryMaxAttempts, delayTime = 0) { // we use the delayTime = 0, because we delegate the retryBackoff to the Neo4j Java Driver - - session.writeTransaction({ - val result = it.run(query, data) - if (log.isDebugEnabled) { - val summary = result.consume() - log.debug("Successfully executed query: `$query`. Summary: $summary") - } - }, transactionConfig) - } - } - } catch (e: Exception) { - bookmarks += session.lastBookmark() - if (log.isDebugEnabled) { - val subList = events.stream() - .limit(5.coerceAtMost(events.size).toLong()) - .toList() - log.debug("Exception `${e.message}` while executing query: `$query`, with data: `$subList` total-records ${events.size}") - } - throw e - } - } - } - - fun writeData(data: Map>>) { - val errors = if (config.parallelBatches) writeDataAsync(data) else writeDataSync(data); - if (errors.isNotEmpty()) { - throw ConnectException(errors.map { it.message }.toSet() - .joinToString("\n", "Errors executing ${data.values.map { it.size }.sum()} jobs:\n")) - } - } - - @ExperimentalCoroutinesApi - @ObsoleteCoroutinesApi - private fun writeDataAsync(data: Map>>) = runBlocking { - val jobs = data - .flatMap { (topic, records) -> - records.map { async (Dispatchers.IO) { writeForTopic(topic, it) } } - } - - // timeout starts in writeTransaction() - jobs.awaitAll() - jobs.mapNotNull { it.errors() } - } - - private fun writeDataSync(data: Map>>) = - data.flatMap { (topic, records) -> - records.mapNotNull { - try { - writeForTopic(topic, it) - null - } catch (e: Exception) { - e - } - } - } -} \ No newline at end of file diff --git a/kafka-connect-neo4j/src/main/kotlin/streams/kafka/connect/sink/Neo4jSinkTask.kt b/kafka-connect-neo4j/src/main/kotlin/streams/kafka/connect/sink/Neo4jSinkTask.kt deleted file mode 100644 index a3c34b73..00000000 --- a/kafka-connect-neo4j/src/main/kotlin/streams/kafka/connect/sink/Neo4jSinkTask.kt +++ /dev/null @@ -1,55 +0,0 @@ -package streams.kafka.connect.sink - -import com.github.jcustenborder.kafka.connect.utils.VersionUtil -import org.apache.kafka.connect.sink.SinkRecord -import org.apache.kafka.connect.sink.SinkTask -import org.slf4j.Logger -import org.slf4j.LoggerFactory -import streams.extensions.asProperties -import streams.service.errors.ErrorData -import streams.service.errors.ErrorService -import streams.service.errors.KafkaErrorService -import streams.utils.StreamsUtils - - -class Neo4jSinkTask : SinkTask() { - private val log: Logger = LoggerFactory.getLogger(Neo4jSinkTask::class.java) - private lateinit var config: Neo4jSinkConnectorConfig - private lateinit var neo4jSinkService: Neo4jSinkService - private lateinit var errorService: ErrorService - - override fun version(): String { - return VersionUtil.version(this.javaClass as Class<*>) - } - - override fun start(map: Map) { - this.config = Neo4jSinkConnectorConfig(map) - this.neo4jSinkService = Neo4jSinkService(this.config) - this.errorService = KafkaErrorService(this.config.kafkaBrokerProperties.asProperties(), - ErrorService.ErrorConfig.from(map.asProperties()), - log::error) - } - - override fun put(collection: Collection) { - if (collection.isEmpty()) { - return - } - try { - val data = EventBuilder() - .withBatchSize(config.batchSize) - .withSinkRecords(collection) - .build() - - neo4jSinkService.writeData(data) - } catch(e:Exception) { - errorService.report(collection.map { - ErrorData(it.topic(), it.timestamp(), it.key(), it.value(), it.kafkaPartition(), it.kafkaOffset(), this::class.java, this.config.database, e) - }) - } - } - - override fun stop() { - log.info("Stop() - Neo4j Sink Service") - StreamsUtils.ignoreExceptions({ neo4jSinkService.close() }, UninitializedPropertyAccessException::class.java) - } -} \ No newline at end of file diff --git a/kafka-connect-neo4j/src/main/kotlin/streams/kafka/connect/sink/Neo4jStrategyStorage.kt b/kafka-connect-neo4j/src/main/kotlin/streams/kafka/connect/sink/Neo4jStrategyStorage.kt deleted file mode 100644 index 2d028696..00000000 --- a/kafka-connect-neo4j/src/main/kotlin/streams/kafka/connect/sink/Neo4jStrategyStorage.kt +++ /dev/null @@ -1,34 +0,0 @@ -package streams.kafka.connect.sink - -import streams.service.StreamsStrategyStorage -import streams.service.TopicType -import streams.service.sink.strategy.CUDIngestionStrategy -import streams.service.sink.strategy.CypherTemplateStrategy -import streams.service.sink.strategy.IngestionStrategy -import streams.service.sink.strategy.NodePatternIngestionStrategy -import streams.service.sink.strategy.RelationshipPatternIngestionStrategy -import streams.service.sink.strategy.SchemaIngestionStrategy -import streams.service.sink.strategy.SourceIdIngestionStrategy -import streams.service.sink.strategy.SourceIdIngestionStrategyConfig - -class Neo4jStrategyStorage(val config: Neo4jSinkConnectorConfig): StreamsStrategyStorage() { - private val topicConfigMap = config.topics.asMap() - - override fun getTopicType(topic: String): TopicType? = TopicType.values().firstOrNull { topicType -> - when (val topicConfig = topicConfigMap.getOrDefault(topicType, emptyList())) { - is Collection<*> -> topicConfig.contains(topic) - is Map<*, *> -> topicConfig.containsKey(topic) - else -> false - } - } - - override fun getStrategy(topic: String): IngestionStrategy = when (val topicType = getTopicType(topic)) { - TopicType.CDC_SOURCE_ID -> config.strategyMap[topicType] as SourceIdIngestionStrategy - TopicType.CDC_SCHEMA -> SchemaIngestionStrategy() - TopicType.CUD -> CUDIngestionStrategy() - TopicType.PATTERN_NODE -> NodePatternIngestionStrategy(config.topics.nodePatternTopics.getValue(topic)) - TopicType.PATTERN_RELATIONSHIP -> RelationshipPatternIngestionStrategy(config.topics.relPatternTopics.getValue(topic)) - TopicType.CYPHER -> CypherTemplateStrategy(config.topics.cypherTopics.getValue(topic)) - null -> throw RuntimeException("Topic Type not Found") - } -} \ No newline at end of file diff --git a/kafka-connect-neo4j/src/main/kotlin/streams/kafka/connect/sink/converters/MapValueConverter.kt b/kafka-connect-neo4j/src/main/kotlin/streams/kafka/connect/sink/converters/MapValueConverter.kt deleted file mode 100644 index 13b925df..00000000 --- a/kafka-connect-neo4j/src/main/kotlin/streams/kafka/connect/sink/converters/MapValueConverter.kt +++ /dev/null @@ -1,105 +0,0 @@ -package streams.kafka.connect.sink.converters - -import com.github.jcustenborder.kafka.connect.utils.data.AbstractConverter -import org.apache.kafka.connect.data.Schema -import org.apache.kafka.connect.data.Struct -import java.math.BigDecimal -import java.util.* - -open class MapValueConverter: AbstractConverter>() { - - open fun setValue(result: MutableMap?, fieldName: String, value: Any?) { - if (result != null) { - result[fieldName] = value as T - } - } - - override fun newValue(): MutableMap { - return mutableMapOf() - } - - override fun setBytesField(result: MutableMap?, fieldName: String, value: ByteArray?) { - setValue(result, fieldName, value) - } - - override fun setStringField(result: MutableMap?, fieldName: String, value: String?) { - setValue(result, fieldName, value) - } - - override fun setFloat32Field(result: MutableMap?, fieldName: String, value: Float?) { - setValue(result, fieldName, value) - } - - override fun setInt32Field(result: MutableMap?, fieldName: String, value: Int?) { - setValue(result, fieldName, value) - } - - override fun setArray(result: MutableMap?, fieldName: String, schema: Schema?, array: MutableList?) { - val convertedArray = array?.map { convertInner(it) } - setValue(result, fieldName, convertedArray) - } - - override fun setTimestampField(result: MutableMap?, fieldName: String, value: Date) { - setValue(result, fieldName, value) - - } - - override fun setTimeField(result: MutableMap?, fieldName: String, value: Date) { - setValue(result, fieldName, value) - } - - override fun setInt8Field(result: MutableMap?, fieldName: String, value: Byte) { - setValue(result, fieldName, value) - } - - override fun setStructField(result: MutableMap?, fieldName: String, value: Struct) { - val converted = convert(value) as MutableMap - setMap(result, fieldName, null, converted) - } - - override fun setMap(result: MutableMap?, fieldName: String, schema: Schema?, value: MutableMap?) { - if (value != null) { - val converted = convert(value) as MutableMap - setValue(result, fieldName, converted) - } else { - setNullField(result, fieldName) - } - } - - override fun setNullField(result: MutableMap?, fieldName: String) { - setValue(result, fieldName, null) - } - - override fun setFloat64Field(result: MutableMap?, fieldName: String, value: Double) { - setValue(result, fieldName, value) - } - - override fun setInt16Field(result: MutableMap?, fieldName: String, value: Short) { - setValue(result, fieldName, value) - } - - override fun setInt64Field(result: MutableMap?, fieldName: String, value: Long) { - setValue(result, fieldName, value) - } - - override fun setBooleanField(result: MutableMap?, fieldName: String, value: Boolean) { - setValue(result, fieldName, value) - } - - override fun setDecimalField(result: MutableMap?, fieldName: String, value: BigDecimal) { - setValue(result, fieldName, value) - } - - override fun setDateField(result: MutableMap?, fieldName: String, value: Date) { - setValue(result, fieldName, value) - } - - open fun convertInner(value: Any?): Any? { - return when (value) { - is Struct, is Map<*, *> -> convert(value) - is Collection<*> -> value.map(::convertInner) - is Array<*> -> if (value.javaClass.componentType.isPrimitive) value else value.map(::convertInner) - else -> value - } - } -} \ No newline at end of file diff --git a/kafka-connect-neo4j/src/main/kotlin/streams/kafka/connect/sink/converters/Neo4jValueConverter.kt b/kafka-connect-neo4j/src/main/kotlin/streams/kafka/connect/sink/converters/Neo4jValueConverter.kt deleted file mode 100644 index 60ece516..00000000 --- a/kafka-connect-neo4j/src/main/kotlin/streams/kafka/connect/sink/converters/Neo4jValueConverter.kt +++ /dev/null @@ -1,63 +0,0 @@ -package streams.kafka.connect.sink.converters - -import org.apache.kafka.connect.data.Struct -import org.neo4j.driver.Value -import org.neo4j.driver.Values -import java.math.BigDecimal -import java.time.LocalTime -import java.time.ZoneId -import java.util.Date -import java.util.concurrent.TimeUnit - - -class Neo4jValueConverter: MapValueConverter() { - - companion object { - @JvmStatic private val UTC = ZoneId.of("UTC") - } - - override fun setValue(result: MutableMap?, fieldName: String, value: Any?) { - if (result != null) { - result[fieldName] = Values.value(value) ?: Values.NULL - } - } - - override fun newValue(): MutableMap { - return mutableMapOf() - } - - override fun setDecimalField(result: MutableMap?, fieldName: String, value: BigDecimal) { - val doubleValue = value.toDouble() - val fitsScale = doubleValue != Double.POSITIVE_INFINITY - && doubleValue != Double.NEGATIVE_INFINITY - && value.compareTo(doubleValue.let { BigDecimal.valueOf(it) }) == 0 - if (fitsScale) { - setValue(result, fieldName, doubleValue) - } else { - setValue(result, fieldName, value.toPlainString()) - } - } - - override fun setTimestampField(result: MutableMap?, fieldName: String, value: Date) { - val localDate = value.toInstant().atZone(UTC).toLocalDateTime() - setValue(result, fieldName, localDate) - - } - - override fun setTimeField(result: MutableMap?, fieldName: String, value: Date) { - val time = LocalTime.ofNanoOfDay(TimeUnit.MILLISECONDS.toNanos(value.time)) - setValue(result, fieldName, time) - } - - override fun setDateField(result: MutableMap?, fieldName: String, value: Date) { - val localDate = value.toInstant().atZone(UTC).toLocalDate() - setValue(result, fieldName, localDate) - } - - override fun setStructField(result: MutableMap?, fieldName: String, value: Struct) { - val converted = convert(value) - .mapValues { it.value?.asObject() } - .toMutableMap() as MutableMap - setMap(result, fieldName, null, converted) - } -} \ No newline at end of file diff --git a/kafka-connect-neo4j/src/main/kotlin/streams/kafka/connect/source/Neo4jSourceConnector.kt b/kafka-connect-neo4j/src/main/kotlin/streams/kafka/connect/source/Neo4jSourceConnector.kt deleted file mode 100644 index 4bf7dbd2..00000000 --- a/kafka-connect-neo4j/src/main/kotlin/streams/kafka/connect/source/Neo4jSourceConnector.kt +++ /dev/null @@ -1,38 +0,0 @@ -package streams.kafka.connect.source - -import com.github.jcustenborder.kafka.connect.utils.config.* -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.GlobalScope -import kotlinx.coroutines.Job -import kotlinx.coroutines.launch -import org.apache.kafka.common.config.ConfigDef -import org.apache.kafka.connect.connector.Task -import org.apache.kafka.connect.source.SourceConnector -import streams.kafka.connect.utils.PropertiesUtil - -@Title("Neo4j Source Connector") -@Description("The Neo4j Source connector reads data from Neo4j and and writes the data to a Kafka Topic") -class Neo4jSourceConnector: SourceConnector() { - private lateinit var settings: Map - private lateinit var config: Neo4jSourceConnectorConfig - - // TODO Add monitor thread when we want to have schema on LABELS and RELATIONSHIP query type - - // TODO: for now we support just one task we need to implement - // a SKIP/LIMIT mechanism in case we want parallelize - override fun taskConfigs(maxTasks: Int): List> = listOf(settings) - - override fun start(props: MutableMap?) { - settings = props!! - config = Neo4jSourceConnectorConfig(settings) - } - - override fun stop() {} - - override fun version(): String = PropertiesUtil.getVersion() - - override fun taskClass(): Class = Neo4jSourceTask::class.java - - override fun config(): ConfigDef = Neo4jSourceConnectorConfig.config() - -} \ No newline at end of file diff --git a/kafka-connect-neo4j/src/main/kotlin/streams/kafka/connect/source/Neo4jSourceConnectorConfig.kt b/kafka-connect-neo4j/src/main/kotlin/streams/kafka/connect/source/Neo4jSourceConnectorConfig.kt deleted file mode 100644 index 5a80f026..00000000 --- a/kafka-connect-neo4j/src/main/kotlin/streams/kafka/connect/source/Neo4jSourceConnectorConfig.kt +++ /dev/null @@ -1,117 +0,0 @@ -package streams.kafka.connect.source - -import com.github.jcustenborder.kafka.connect.utils.config.ConfigKeyBuilder -import com.github.jcustenborder.kafka.connect.utils.config.ValidEnum -import org.apache.kafka.common.config.ConfigDef -import org.apache.kafka.common.config.ConfigException -import streams.kafka.connect.common.ConnectorType -import streams.kafka.connect.common.Neo4jConnectorConfig -import streams.kafka.connect.utils.PropertiesUtil - -enum class SourceType { - QUERY, LABELS, RELATIONSHIP -} - -enum class StreamingFrom { - ALL, NOW, LAST_COMMITTED; - - fun value() = when (this) { - ALL -> -1 - else -> System.currentTimeMillis() - } -} - -class Neo4jSourceConnectorConfig(originals: Map<*, *>): Neo4jConnectorConfig(config(), originals, ConnectorType.SOURCE) { - - val topic: String = getString(TOPIC) - - val labels: Array - val relationship: String - val query: String - - val partitions: List = (1 .. getInt(PARTITIONS)).toList() - - val streamingFrom: StreamingFrom = StreamingFrom.valueOf(getString(STREAMING_FROM)) - val streamingProperty: String = getString(STREAMING_PROPERTY) - - val sourceType: SourceType = SourceType.valueOf(getString(SOURCE_TYPE)) - - val pollInterval: Int = getInt(STREAMING_POLL_INTERVAL) - - val enforceSchema: Boolean = getBoolean(ENFORCE_SCHEMA) - - init { - when (sourceType) { - SourceType.QUERY -> { - query = getString(SOURCE_TYPE_QUERY) - if (query.isNullOrBlank()) { - throw ConfigException("You need to define: $SOURCE_TYPE_QUERY") - } - labels = emptyArray() - relationship = "" - } - else -> { - throw ConfigException("Supported source query types are: ${SourceType.QUERY}") - } - } - } - - fun sourcePartition() = when (sourceType) { - SourceType.QUERY -> mapOf("database" to this.database, - "type" to "query", "query" to query, "partition" to 1) - else -> throw UnsupportedOperationException("Supported source query types are: ${SourceType.QUERY}") - } - - companion object { - const val PARTITIONS = "partitions" - const val TOPIC = "topic" - const val STREAMING_FROM = "neo4j.streaming.from" - const val ENFORCE_SCHEMA = "neo4j.enforce.schema" - const val STREAMING_PROPERTY = "neo4j.streaming.property" - const val STREAMING_POLL_INTERVAL = "neo4j.streaming.poll.interval.msecs" - const val SOURCE_TYPE = "neo4j.source.type" - const val SOURCE_TYPE_QUERY = "neo4j.source.query" - const val SOURCE_TYPE_LABELS = "neo4j.source.labels" - const val SOURCE_TYPE_RELATIONSHIP = "neo4j.source.relationship" - - fun config(): ConfigDef = Neo4jConnectorConfig.config() - .define(ConfigKeyBuilder.of(ENFORCE_SCHEMA, ConfigDef.Type.BOOLEAN) - .documentation(PropertiesUtil.getProperty(ENFORCE_SCHEMA)).importance(ConfigDef.Importance.HIGH) - .defaultValue(false) - .validator(ConfigDef.NonNullValidator()) - .build()) - .define(ConfigKeyBuilder.of(STREAMING_POLL_INTERVAL, ConfigDef.Type.INT) - .documentation(PropertiesUtil.getProperty(STREAMING_POLL_INTERVAL)).importance(ConfigDef.Importance.HIGH) - .defaultValue(10000) - .validator(ConfigDef.Range.atLeast(1)) - .build()) - .define(ConfigKeyBuilder.of(STREAMING_PROPERTY, ConfigDef.Type.STRING) - .documentation(PropertiesUtil.getProperty(STREAMING_PROPERTY)).importance(ConfigDef.Importance.HIGH) - .defaultValue("") -// .validator(ConfigDef.NonEmptyString()) - .build()) - .define(ConfigKeyBuilder.of(TOPIC, ConfigDef.Type.STRING) - .documentation(PropertiesUtil.getProperty(TOPIC)).importance(ConfigDef.Importance.HIGH) - .validator(ConfigDef.NonEmptyString()) - .build()) - .define(ConfigKeyBuilder.of(PARTITIONS, ConfigDef.Type.INT) - .documentation(PropertiesUtil.getProperty(PARTITIONS)).importance(ConfigDef.Importance.HIGH) - .defaultValue(1) - .validator(ConfigDef.Range.atLeast(1)) - .build()) - .define(ConfigKeyBuilder.of(STREAMING_FROM, ConfigDef.Type.STRING) - .documentation(PropertiesUtil.getProperty(STREAMING_FROM)).importance(ConfigDef.Importance.HIGH) - .defaultValue(StreamingFrom.NOW.toString()) - .validator(ValidEnum.of(StreamingFrom::class.java)) - .build()) - .define(ConfigKeyBuilder.of(SOURCE_TYPE, ConfigDef.Type.STRING) - .documentation(PropertiesUtil.getProperty(SOURCE_TYPE)).importance(ConfigDef.Importance.HIGH) - .defaultValue(SourceType.QUERY.toString()) - .validator(ValidEnum.of(SourceType::class.java)) - .build()) - .define(ConfigKeyBuilder.of(SOURCE_TYPE_QUERY, ConfigDef.Type.STRING) - .documentation(PropertiesUtil.getProperty(SOURCE_TYPE_QUERY)).importance(ConfigDef.Importance.HIGH) - .defaultValue("") - .build()) - } -} \ No newline at end of file diff --git a/kafka-connect-neo4j/src/main/kotlin/streams/kafka/connect/source/Neo4jSourceService.kt b/kafka-connect-neo4j/src/main/kotlin/streams/kafka/connect/source/Neo4jSourceService.kt deleted file mode 100644 index be33ad6c..00000000 --- a/kafka-connect-neo4j/src/main/kotlin/streams/kafka/connect/source/Neo4jSourceService.kt +++ /dev/null @@ -1,177 +0,0 @@ -package streams.kafka.connect.source - -import kotlinx.coroutines.CancellationException -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.GlobalScope -import kotlinx.coroutines.Job -import kotlinx.coroutines.cancelAndJoin -import kotlinx.coroutines.delay -import kotlinx.coroutines.isActive -import kotlinx.coroutines.launch -import kotlinx.coroutines.runBlocking -import org.apache.kafka.connect.errors.ConnectException -import org.apache.kafka.connect.source.SourceRecord -import org.apache.kafka.connect.storage.OffsetStorageReader -import org.neo4j.driver.Record -import org.neo4j.driver.Values -import org.slf4j.Logger -import org.slf4j.LoggerFactory -import streams.utils.StreamsUtils -import java.util.concurrent.BlockingQueue -import java.util.concurrent.LinkedBlockingQueue -import java.util.concurrent.TimeUnit -import java.util.concurrent.atomic.AtomicBoolean -import java.util.concurrent.atomic.AtomicLong -import java.util.concurrent.atomic.AtomicReference - - -class Neo4jSourceService(private val config: Neo4jSourceConnectorConfig, offsetStorageReader: OffsetStorageReader): AutoCloseable { - - private val log: Logger = LoggerFactory.getLogger(Neo4jSourceService::class.java) - - private val driver = config.createDriver() - - private val queue: BlockingQueue = LinkedBlockingQueue() - private val error: AtomicReference = AtomicReference(null) - - private val sourcePartition = config.sourcePartition() - - private val isClose = AtomicBoolean() - - private val lastCheck: AtomicLong by lazy { - val offset = offsetStorageReader.offset(sourcePartition) ?: emptyMap() - // if the user wants to recover from LAST_COMMITTED - val startValue = if (config.streamingFrom == StreamingFrom.LAST_COMMITTED - && offset["value"] != null && offset["property"] == config.streamingProperty) { - log.info("Resuming offset $offset, the ${Neo4jSourceConnectorConfig.STREAMING_FROM} value is ignored") - offset["value"] as Long - } else { - if (config.streamingFrom == StreamingFrom.LAST_COMMITTED) { - log.info("You provided ${Neo4jSourceConnectorConfig.STREAMING_FROM}: ${config.streamingFrom} but no offset has been found, we'll start to consume from NOW") - } else { - log.info("No offset to resume, we'll the provided value of ${Neo4jSourceConnectorConfig.STREAMING_FROM}: ${config.streamingFrom}") - } - config.streamingFrom.value() - } - AtomicLong(startValue) - } - - private val sessionConfig = config.createSessionConfig() - private val transactionConfig = config.createTransactionConfig() - - private val pollInterval = config.pollInterval.toLong() - private val isStreamingPropertyDefined = config.streamingProperty.isNotBlank() - private val streamingProperty = config.streamingProperty.ifBlank { "undefined" } - - private val job: Job = GlobalScope.launch(Dispatchers.IO) { - var lastCheckHadResult = false - while (isActive) { - try { - // if the user doesn't set the streaming property we fallback to an - // internal mechanism - if (!isStreamingPropertyDefined) { - // we update the lastCheck property only if the last loop round - // returned results otherwise we stick to the old value - if (lastCheckHadResult) { - lastCheck.set(System.currentTimeMillis() - pollInterval) - } - } - driver.session(sessionConfig).readTransaction({ tx -> - val result = tx.run(config.query, mapOf("lastCheck" to lastCheck.get())) - lastCheckHadResult = result.hasNext() - result.forEach { record -> - try { - val sourceRecord = toSourceRecord(record) - queue.put(sourceRecord) - } catch (e: Exception) { - setError(e) - } - } - }, transactionConfig) - delay(pollInterval) - } catch (e: Exception) { - setError(e) - } - } - } - - private fun toSourceRecord(record: Record): SourceRecord { - val thisValue = computeLastTimestamp(record) - return SourceRecordBuilder() - .withRecord(record) - .withTopic(config.topic) - .withSourcePartition(sourcePartition) - .withStreamingProperty(streamingProperty) - .withEnforceSchema(config.enforceSchema) - .withTimestamp(thisValue) - .build() - } - - private fun computeLastTimestamp(record: Record) = try { - if (isStreamingPropertyDefined) { - val value = record.get(config.streamingProperty, Values.value(-1L)).asLong() - lastCheck.getAndUpdate { oldValue -> - if (oldValue >= value) { - oldValue - } else { - value - } - } - value - } else { - lastCheck.get() - } - } catch (e: Throwable) { - lastCheck.get() - } - - private fun checkError() { - val fatalError = error.getAndSet(null) - if (fatalError != null) { - throw ConnectException(fatalError) - } - } - - fun poll(): List? { - if (isClose.get()) { - return null - } - checkError() - // Block until at least one item is available or until the - // courtesy timeout expires, giving the framework a chance - // to pause the connector. - val firstEvent = queue.poll(1, TimeUnit.SECONDS) - if (firstEvent == null) { - log.debug("Poll returns 0 results") - return null // Looks weird, but caller expects it. - } - - val events = mutableListOf() - return try { - events.add(firstEvent) - queue.drainTo(events, config.batchSize - 1) - log.info("Poll returns {} result(s)", events.size) - events - } catch (e: Exception) { - setError(e) - null - } - } - - private fun setError(e: Exception) { - if (e !is CancellationException) { - if (error.compareAndSet(null, e)) { - log.error("Error:", e) - } - } - } - - override fun close() { - isClose.set(true) - runBlocking { job.cancelAndJoin() } - StreamsUtils.closeSafetely(driver) { - log.info("Error while closing Driver instance:", it) - } - log.info("Neo4j Source Service closed successfully") - } -} \ No newline at end of file diff --git a/kafka-connect-neo4j/src/main/kotlin/streams/kafka/connect/source/Neo4jSourceTask.kt b/kafka-connect-neo4j/src/main/kotlin/streams/kafka/connect/source/Neo4jSourceTask.kt deleted file mode 100644 index e836d937..00000000 --- a/kafka-connect-neo4j/src/main/kotlin/streams/kafka/connect/source/Neo4jSourceTask.kt +++ /dev/null @@ -1,32 +0,0 @@ -package streams.kafka.connect.source - -import com.github.jcustenborder.kafka.connect.utils.VersionUtil -import org.apache.kafka.connect.source.SourceRecord -import org.apache.kafka.connect.source.SourceTask -import org.slf4j.Logger -import org.slf4j.LoggerFactory -import streams.kafka.connect.sink.Neo4jSinkTask -import streams.utils.StreamsUtils - -class Neo4jSourceTask: SourceTask() { - private lateinit var settings: Map - private lateinit var config: Neo4jSourceConnectorConfig - private lateinit var neo4jSourceService: Neo4jSourceService - - private val log: Logger = LoggerFactory.getLogger(Neo4jSinkTask::class.java) - - override fun version(): String = VersionUtil.version(this.javaClass as Class<*>) - - override fun start(props: MutableMap?) { - settings = props!! - config = Neo4jSourceConnectorConfig(settings) - neo4jSourceService = Neo4jSourceService(config, context.offsetStorageReader()) - } - - override fun stop() { - log.info("Stop() - Closing Neo4j Source Service.") - StreamsUtils.ignoreExceptions({ neo4jSourceService.close() }, UninitializedPropertyAccessException::class.java) - } - - override fun poll(): List? = neo4jSourceService.poll() -} \ No newline at end of file diff --git a/kafka-connect-neo4j/src/main/kotlin/streams/kafka/connect/source/SourceRecordBuilder.kt b/kafka-connect-neo4j/src/main/kotlin/streams/kafka/connect/source/SourceRecordBuilder.kt deleted file mode 100644 index 7ad4690b..00000000 --- a/kafka-connect-neo4j/src/main/kotlin/streams/kafka/connect/source/SourceRecordBuilder.kt +++ /dev/null @@ -1,67 +0,0 @@ -package streams.kafka.connect.source - -import org.apache.kafka.connect.data.Schema -import org.apache.kafka.connect.source.SourceRecord -import org.neo4j.driver.Record -import streams.kafka.connect.utils.asJsonString -import streams.kafka.connect.utils.asStruct -import kotlin.properties.Delegates - -class SourceRecordBuilder { - - private lateinit var topic: String - - private lateinit var streamingProperty: String - - private var timestamp by Delegates.notNull() - - private lateinit var sourcePartition: Map - - private lateinit var record: Record - - private var enforceSchema: Boolean = false - - fun withTopic(topic: String): SourceRecordBuilder { - this.topic = topic - return this - } - - fun withStreamingProperty(streamingProperty: String): SourceRecordBuilder { - this.streamingProperty = streamingProperty - return this - } - - fun withTimestamp(timestamp: Long): SourceRecordBuilder { - this.timestamp = timestamp - return this - } - - fun withSourcePartition(sourcePartition: Map): SourceRecordBuilder { - this.sourcePartition = sourcePartition - return this - } - - fun withRecord(record: Record): SourceRecordBuilder { - this.record = record - return this - } - - fun withEnforceSchema(enforceSchema: Boolean): SourceRecordBuilder { - this.enforceSchema = enforceSchema - return this - } - - fun build(): SourceRecord { - val sourceOffset = mapOf("property" to streamingProperty.ifBlank { "undefined" }, - "value" to timestamp) - val (struct, schema) = when (enforceSchema) { - true -> { - val st = record.asStruct() - val sc = st.schema() - st to sc - } - else -> record.asJsonString() to Schema.STRING_SCHEMA - } - return SourceRecord(sourcePartition, sourceOffset, topic, schema, struct, schema, struct) - } -} diff --git a/kafka-connect-neo4j/src/main/kotlin/streams/kafka/connect/utils/ConnectExtensionFunctions.kt b/kafka-connect-neo4j/src/main/kotlin/streams/kafka/connect/utils/ConnectExtensionFunctions.kt deleted file mode 100644 index 892385da..00000000 --- a/kafka-connect-neo4j/src/main/kotlin/streams/kafka/connect/utils/ConnectExtensionFunctions.kt +++ /dev/null @@ -1,126 +0,0 @@ -package streams.kafka.connect.utils - -import org.apache.kafka.connect.data.Schema -import org.apache.kafka.connect.data.SchemaBuilder -import org.apache.kafka.connect.data.Struct -import org.apache.kafka.connect.sink.SinkRecord -import org.neo4j.driver.Record -import org.neo4j.driver.types.Node -import org.neo4j.driver.types.Point -import org.neo4j.driver.types.Relationship -import streams.extensions.asStreamsMap -import streams.kafka.connect.sink.converters.Neo4jValueConverter -import streams.utils.JSONUtils -import streams.service.StreamsSinkEntity -import java.time.temporal.TemporalAccessor - -fun SinkRecord.toStreamsSinkEntity(): StreamsSinkEntity = StreamsSinkEntity( - convertData(this.key(),true), - convertData(this.value())) - -private val converter = Neo4jValueConverter() - -private fun convertData(data: Any?, stringWhenFailure :Boolean = false) = when (data) { - is Struct -> converter.convert(data) - null -> null - else -> JSONUtils.readValue(data, stringWhenFailure) -} - -fun Record.asJsonString(): String = JSONUtils.writeValueAsString(this.asMap()) - -fun Record.schema(asMap: Map = this.asMap()): Schema { - val structBuilder = SchemaBuilder.struct() - asMap.forEach { structBuilder.field(it.key, neo4jValueSchema(it.value)) } - return structBuilder.build() -} - -fun Record.asStruct(): Struct { - val asMap = this.asMap() - val schema = schema(asMap) - val struct = Struct(schema) - schema.fields().forEach { - struct.put(it, neo4jToKafka(it.schema(), asMap[it.name()])) - } - struct - return struct -} - -private fun neo4jToKafka(schema: Schema, value: Any?): Any? = when (schema.type()) { - Schema.Type.ARRAY -> when (value) { - is Collection<*> -> value.map { neo4jToKafka(neo4jValueSchema(it), it) } - is Array<*> -> value.map { neo4jToKafka(neo4jValueSchema(it), it) }.toTypedArray() - else -> throw IllegalArgumentException("For Schema.Type.ARRAY we support only Collection and Array") - } - Schema.Type.MAP -> when (value) { - is Map<*, *> -> value.mapValues { neo4jToKafka(neo4jValueSchema(it.value), it.value) } - else -> throw IllegalArgumentException("For Schema.Type.MAP we support only Map") - } - Schema.Type.STRUCT -> when (value) { - is Map<*, *> -> { - val struct = Struct(schema) - schema.fields().forEach { - struct.put(it, neo4jToKafka(it.schema(), value[it.name()])) - } - struct - } - is Point -> { - val map = JSONUtils.readValue>(value) - neo4jToKafka(schema, map) - } - is Node -> { - val map = value.asStreamsMap() - neo4jToKafka(schema, map) - } - is Relationship -> { - val map = value.asStreamsMap() - neo4jToKafka(schema, map) - } - else -> throw IllegalArgumentException("For Schema.Type.STRUCT we support only Map and Point") - } - else -> when (value) { - null -> null - is TemporalAccessor -> { - val temporalValue = JSONUtils.readValue(value) - neo4jToKafka(schema, temporalValue) - } - else -> when { - Schema.Type.STRING == schema.type() && value !is String -> value.toString() - else -> value - } - } -} - -private fun neo4jValueSchema(value: Any?): Schema = when (value) { - is Long -> Schema.OPTIONAL_INT64_SCHEMA - is Double -> Schema.OPTIONAL_FLOAT64_SCHEMA - is Boolean -> Schema.OPTIONAL_BOOLEAN_SCHEMA - is Collection<*> -> { - (value.firstOrNull()?.let { - SchemaBuilder.array(neo4jValueSchema(it)) - } ?: SchemaBuilder.array(Schema.OPTIONAL_STRING_SCHEMA)).build() - } - is Array<*> -> { - (value.firstOrNull()?.let { - SchemaBuilder.array(neo4jValueSchema(it)) - } ?: SchemaBuilder.array(Schema.OPTIONAL_STRING_SCHEMA)).build() - } - is Map<*, *> -> { - if (value.isEmpty()) { - SchemaBuilder.map(Schema.STRING_SCHEMA, Schema.OPTIONAL_STRING_SCHEMA).build() - } else { - val valueTypes = value.values.mapNotNull { elem -> elem?.let{ it::class.java.simpleName } } - .toSet() - if (valueTypes.size == 1) { - SchemaBuilder.map(Schema.STRING_SCHEMA, neo4jValueSchema(value.values.first())).build() - } else { - val structMap = SchemaBuilder.struct() - value.forEach { structMap.field(it.key.toString(), neo4jValueSchema(it.value)) } - structMap.build() - } - } - } - is Point -> neo4jValueSchema(JSONUtils.readValue>(value)) - is Node -> neo4jValueSchema(value.asStreamsMap()) - is Relationship -> neo4jValueSchema(value.asStreamsMap()) - else -> Schema.OPTIONAL_STRING_SCHEMA -} diff --git a/kafka-connect-neo4j/src/main/kotlin/streams/kafka/connect/utils/PropertiesUtil.kt b/kafka-connect-neo4j/src/main/kotlin/streams/kafka/connect/utils/PropertiesUtil.kt deleted file mode 100644 index e3a319dc..00000000 --- a/kafka-connect-neo4j/src/main/kotlin/streams/kafka/connect/utils/PropertiesUtil.kt +++ /dev/null @@ -1,33 +0,0 @@ -package streams.kafka.connect.utils - -import org.slf4j.LoggerFactory -import java.util.* - -class PropertiesUtil { - - companion object { - private val LOGGER = LoggerFactory.getLogger(PropertiesUtil::class.java) - private const val DEFAULT_VERSION = "unknown" - private var properties: Properties? = null - private var VERSION: String? = null - init { - properties = Properties() - properties!!.load(PropertiesUtil::class.java.getResourceAsStream("/kafka-connect-version.properties")) - properties!!.load(PropertiesUtil::class.java.getResourceAsStream("/kafka-connect-neo4j.properties")) - VERSION = try { - properties!!.getProperty("version", DEFAULT_VERSION).trim() - } catch (e: Exception) { - LOGGER.warn("error while loading version:", e) - DEFAULT_VERSION - } - } - - fun getVersion(): String { - return VERSION!! - } - - fun getProperty(key: String): String { - return properties!!.getProperty(key) - } - } -} \ No newline at end of file diff --git a/kafka-connect-neo4j/src/main/resources/kafka-connect-neo4j.properties b/kafka-connect-neo4j/src/main/resources/kafka-connect-neo4j.properties deleted file mode 100644 index e4ced21a..00000000 --- a/kafka-connect-neo4j/src/main/resources/kafka-connect-neo4j.properties +++ /dev/null @@ -1,62 +0,0 @@ -## -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -## - -## Common Properties -neo4j.database=Type: String;\nDescription: The neo4j database instance name (default neo4j) -neo4j.server.uri=Type: String;\nDescription: The Bolt URI (default bolt://localhost:7687) -neo4j.authentication.type=Type: enum[NONE, BASIC, KERBEROS];\nDescription: The authentication type (default BASIC) -neo4j.batch.size=Type: Int;\nDescription: The max number of events processed by the Cypher query for the Sink. \ - The max number of messages pushed for each poll cycle in case of the Source. (default 1000) -neo4j.batch.timeout.msecs=Type: Long;\nDescription: The execution timeout for the cypher query (default: 0, that is without timeout) -neo4j.authentication.basic.username=Type: String;\nDescription: The authentication username -neo4j.authentication.basic.password=Type: String;\nDescription: The authentication password -neo4j.authentication.basic.realm=Type: String;\nDescription: The authentication realm -neo4j.authentication.kerberos.ticket=Type: String;\nDescription: The Kerberos ticket -neo4j.encryption.enabled=Type: Boolean;\nDescription: If the encryption is enabled (default false) -neo4j.encryption.trust.strategy=Type: enum[TRUST_ALL_CERTIFICATES, TRUST_CUSTOM_CA_SIGNED_CERTIFICATES, TRUST_SYSTEM_CA_SIGNED_CERTIFICATES];\n\ - Description: The Neo4j trust strategy (default TRUST_ALL_CERTIFICATES) -neo4j.encryption.ca.certificate.path=Type: String;\nDescription: The path of the certificate -neo4j.connection.max.lifetime.msecs=Type: Long;\nDescription: The max Neo4j connection lifetime (default 1 hour) -neo4j.connection.acquisition.timeout.msecs=Type: Long;\nDescription: The max Neo4j acquisition timeout (default 1 hour) -neo4j.connection.liveness.check.timeout.msecs=Type: Long;\nDescription: The max Neo4j liveness check timeout (default 1 hour) -neo4j.connection.max.pool.size=Type: Int;\nDescription: The max pool size (default 100) -neo4j.retry.backoff.msecs=Type: Long;\nDescription: The time in milliseconds to wait following a transient error \ - before a retry attempt is made (default 30000). -neo4j.retry.max.attemps=Type: Int;\nDescription: The maximum number of times to retry on transient errors \ - (except for TimeoutException) before failing the task (default 5). - -## Sink Properties -neo4j.topic.cdc.sourceId=Type: String;\nDescription: The topic list (separated by semicolon) that manages CDC events with the `SourceId` strategy -neo4j.topic.cdc.sourceId.labelName=Type: String;\nDescription: The label name attached to the events with the `SourceId` strategy (default SourceEvent) -neo4j.topic.cdc.sourceId.idName=Type: String;\nDescription: The id property name attached to the events with the `SourceId` strategy (default sourceId) -neo4j.topic.cdc.schema=Type: String;\nDescription: The topic list (separated by semicolon) that manages CDC events with the `Schema` strategy -neo4j.batch.parallelize=Type: Boolean;\nDescription: If enabled messages are processed concurrently in the sink. \ - Non concurrent execution supports in-order processing, e.g. for CDC (default true) -neo4j.topic.cud=Type: String;\nDescription: The topic list (separated by semicolon) that manages CUD events - -## Source Properties -topic=Type: String;\nDescription: The topic where the Source will publish the data -partitions=Type: Int;\nDescription: The number of partition for the Source (default 1) -neo4j.streaming.from=Type: enum[ALL, NOW, LAST_COMMITTED];\nDescription: When start the Source. ALL means from the beginning. \ - LAST_COMMITTED will try to retrieve already committed offset, \ - in case it will not find one LAST_COMMITTED use NOW as fallback (default NOW) -neo4j.source.type=Type: enum[QUERY];\nDescription: The type of the Source strategy, with UERY you must set `neo4j.source.query` -neo4j.source.query=Type: String\nDescription: The Cypher query in order to extract the data from Neo4j you need to \ - define it if you use `neo4j.source.type=QUERY` -neo4j.streaming.property=Type: String;\nDescription: The name of the property that we need to consider in order to determinate \ - the last queried record. If not defined we use an internal value given from the last performed check -neo4j.streaming.poll.interval.msecs=Type Int;\nDescription: The polling interval in ms (Default: 10000) -neo4j.enforce.schema=Type: Boolean;\nApply a schema to each record (Default: false) - diff --git a/kafka-connect-neo4j/src/main/resources/kafka-connect-version.properties b/kafka-connect-neo4j/src/main/resources/kafka-connect-version.properties deleted file mode 100644 index 799c6344..00000000 --- a/kafka-connect-neo4j/src/main/resources/kafka-connect-version.properties +++ /dev/null @@ -1,16 +0,0 @@ -## -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -## - -version=${project.version} \ No newline at end of file diff --git a/kafka-connect-neo4j/src/test/kotlin/streams/kafka/connect/sink/EventBuilderTest.kt b/kafka-connect-neo4j/src/test/kotlin/streams/kafka/connect/sink/EventBuilderTest.kt deleted file mode 100644 index 10852f84..00000000 --- a/kafka-connect-neo4j/src/test/kotlin/streams/kafka/connect/sink/EventBuilderTest.kt +++ /dev/null @@ -1,62 +0,0 @@ -package streams.kafka.connect.sink - -import org.apache.kafka.connect.data.Schema -import org.apache.kafka.connect.data.SchemaBuilder -import org.apache.kafka.connect.data.Struct -import org.apache.kafka.connect.data.Timestamp -import org.apache.kafka.connect.sink.SinkRecord -import org.junit.Test -import java.util.* -import kotlin.test.assertEquals - -class EventBuilderTest { - private val PERSON_SCHEMA = SchemaBuilder.struct().name("com.example.Person") - .field("firstName", Schema.STRING_SCHEMA) - .field("lastName", Schema.STRING_SCHEMA) - .field("age", Schema.OPTIONAL_INT32_SCHEMA) - .field("bool", Schema.OPTIONAL_BOOLEAN_SCHEMA) - .field("short", Schema.OPTIONAL_INT16_SCHEMA) - .field("byte", Schema.OPTIONAL_INT8_SCHEMA) - .field("long", Schema.OPTIONAL_INT64_SCHEMA) - .field("float", Schema.OPTIONAL_FLOAT32_SCHEMA) - .field("double", Schema.OPTIONAL_FLOAT64_SCHEMA) - .field("modified", Timestamp.SCHEMA) - .build() - - @Test - fun `should create event map properly`() { - // Given - val firstTopic = "neotopic" - val secondTopic = "foo" - val batchSize = 2 - val struct= Struct(PERSON_SCHEMA) - .put("firstName", "Alex") - .put("lastName", "Smith") - .put("bool", true) - .put("short", 1234.toShort()) - .put("byte", (-32).toByte()) - .put("long", 12425436L) - .put("float", 2356.3.toFloat()) - .put("double", -2436546.56457) - .put("age", 21) - .put("modified", Date(1474661402123L)) - val input = listOf(SinkRecord(firstTopic, 1, null, null, PERSON_SCHEMA, struct, 42), - SinkRecord(firstTopic, 1, null, null, PERSON_SCHEMA, struct, 42), - SinkRecord(firstTopic, 1, null, null, PERSON_SCHEMA, struct, 43), - SinkRecord(firstTopic, 1, null, null, PERSON_SCHEMA, struct, 44), - SinkRecord(firstTopic, 1, null, null, PERSON_SCHEMA, struct, 45), - SinkRecord(secondTopic, 1, null, null, PERSON_SCHEMA, struct, 43)) // 5 records for topic "neotopic", 1 for topic "foo" - val topics = listOf(firstTopic, secondTopic) - - // When - val data = EventBuilder() - .withBatchSize(batchSize) - .withSinkRecords(input) - .build() - - // Then - assertEquals(topics, data.keys.toList()) - assertEquals(3, data[firstTopic]!!.size) // n° of chunks for "neotopic" - assertEquals(1, data[secondTopic]!!.size) // n° of chunks for "foo" - } -} \ No newline at end of file diff --git a/kafka-connect-neo4j/src/test/kotlin/streams/kafka/connect/sink/MapValueConverterTest.kt b/kafka-connect-neo4j/src/test/kotlin/streams/kafka/connect/sink/MapValueConverterTest.kt deleted file mode 100644 index 1d1bedcd..00000000 --- a/kafka-connect-neo4j/src/test/kotlin/streams/kafka/connect/sink/MapValueConverterTest.kt +++ /dev/null @@ -1,104 +0,0 @@ -package streams.kafka.connect.sink - -import org.apache.kafka.connect.data.Schema -import org.apache.kafka.connect.data.SchemaBuilder -import org.apache.kafka.connect.data.Struct -import org.junit.Test -import streams.kafka.connect.sink.converters.MapValueConverter -import kotlin.test.assertEquals - -class MapValueConverterTest { - - @Test - fun `should convert tree struct into map of String,Any?`() { - // given - // this test generates a simple tree structure like this - // body - // / \ - // p ul - // | - // li - val body = getTreeStruct() - - // when - val result = MapValueConverter().convert(body) as Map<*, *> - - // then - val expected = getTreeMap() - assertEquals(expected, result) - } - - @Test - fun `should convert tree simple map into map of String,Any?`() { - // given - // this test generates a simple tree structure like this - // body - // / \ - // p ul - // | - // li - val body = getTreeMap() - - // when - val result = MapValueConverter().convert(body) as Map<*, *> - - // then - val expected = getTreeMap() - assertEquals(expected, result) - } - - companion object { - private val LI_SCHEMA = SchemaBuilder.struct().name("org.neo4j.example.html.LI") - .field("value", Schema.OPTIONAL_STRING_SCHEMA) - .field("class", SchemaBuilder.array(Schema.STRING_SCHEMA).optional()) - .build() - - private val UL_SCHEMA = SchemaBuilder.struct().name("org.neo4j.example.html.UL") - .field("value", SchemaBuilder.array(LI_SCHEMA)) - .build() - - private val P_SCHEMA = SchemaBuilder.struct().name("org.neo4j.example.html.P") - .field("value", Schema.OPTIONAL_STRING_SCHEMA) - .build() - - private val BODY_SCHEMA = SchemaBuilder.struct().name("org.neo4j.example.html.BODY") - .field("ul", SchemaBuilder.array(UL_SCHEMA).optional()) - .field("p", SchemaBuilder.array(P_SCHEMA).optional()) - .build() - - fun getTreeStruct(): Struct? { - val firstUL = Struct(UL_SCHEMA).put("value", listOf( - Struct(LI_SCHEMA).put("value", "First UL - First Element"), - Struct(LI_SCHEMA).put("value", "First UL - Second Element") - .put("class", listOf("ClassA", "ClassB")) - )) - val secondUL = Struct(UL_SCHEMA).put("value", listOf( - Struct(LI_SCHEMA).put("value", "Second UL - First Element"), - Struct(LI_SCHEMA).put("value", "Second UL - Second Element") - )) - val ulList = listOf(firstUL, secondUL) - val pList = listOf( - Struct(P_SCHEMA).put("value", "First Paragraph"), - Struct(P_SCHEMA).put("value", "Second Paragraph") - ) - return Struct(BODY_SCHEMA) - .put("ul", ulList) - .put("p", pList) - } - - fun getTreeMap(): Map { - val firstULMap = mapOf("value" to listOf( - mapOf("value" to "First UL - First Element", "class" to null), - mapOf("value" to "First UL - Second Element", "class" to listOf("ClassA", "ClassB")))) - val secondULMap = mapOf("value" to listOf( - mapOf("value" to "Second UL - First Element", "class" to null), - mapOf("value" to "Second UL - Second Element", "class" to null))) - val ulListMap = listOf(firstULMap, secondULMap) - val pListMap = listOf(mapOf("value" to "First Paragraph"), - mapOf("value" to "Second Paragraph")) - return mapOf("ul" to ulListMap, "p" to pListMap) - } - } - -} - diff --git a/kafka-connect-neo4j/src/test/kotlin/streams/kafka/connect/sink/Neo4jSinkConnectorConfigTest.kt b/kafka-connect-neo4j/src/test/kotlin/streams/kafka/connect/sink/Neo4jSinkConnectorConfigTest.kt deleted file mode 100644 index 4d22d3a6..00000000 --- a/kafka-connect-neo4j/src/test/kotlin/streams/kafka/connect/sink/Neo4jSinkConnectorConfigTest.kt +++ /dev/null @@ -1,128 +0,0 @@ -package streams.kafka.connect.sink - -import org.apache.kafka.clients.CommonClientConfigs -import org.apache.kafka.clients.producer.ProducerConfig -import org.apache.kafka.common.config.ConfigException -import org.apache.kafka.connect.sink.SinkConnector -import org.junit.Test -import org.neo4j.driver.internal.async.pool.PoolSettings -import org.neo4j.driver.Config -import streams.kafka.connect.common.Neo4jConnectorConfig -import kotlin.test.assertEquals -import kotlin.test.assertFalse -import kotlin.test.assertNull - -class Neo4jSinkConnectorConfigTest { - - @Test(expected = ConfigException::class) - fun `should throw a ConfigException because of mismatch`() { - try { - val originals = mapOf(SinkConnector.TOPICS_CONFIG to "foo, bar", - "${Neo4jSinkConnectorConfig.TOPIC_CYPHER_PREFIX}foo" to "CREATE (p:Person{name: event.firstName})") - Neo4jSinkConnectorConfig(originals) - } catch (e: ConfigException) { - assertEquals("There is a mismatch between topics defined into the property `topics` ([bar, foo]) and configured topics ([foo])", e.message) - throw e - } - } - - @Test(expected = ConfigException::class) - fun `should throw a ConfigException because of cross defined topics`() { - try { - val originals = mapOf(SinkConnector.TOPICS_CONFIG to "foo, bar", - "${Neo4jSinkConnectorConfig.TOPIC_CYPHER_PREFIX}foo" to "CREATE (p:Person{name: event.firstName})", - "${Neo4jSinkConnectorConfig.TOPIC_CYPHER_PREFIX}bar" to "CREATE (p:Person{name: event.firstName})", - Neo4jSinkConnectorConfig.TOPIC_CDC_SOURCE_ID to "foo") - Neo4jSinkConnectorConfig(originals) - } catch (e: ConfigException) { - assertEquals("The following topics are cross defined: [foo]", e.message) - throw e - } - } - - @Test - fun `should return the configuration`() { - val a = "bolt://neo4j:7687" - val b = "bolt://neo4j2:7687" - - val originals = mapOf(SinkConnector.TOPICS_CONFIG to "foo", - "${Neo4jSinkConnectorConfig.TOPIC_CYPHER_PREFIX}foo" to "CREATE (p:Person{name: event.firstName})", - Neo4jConnectorConfig.SERVER_URI to "$a,$b", // Check for string trimming - Neo4jConnectorConfig.BATCH_SIZE to 10, - "kafka.${CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG}" to "broker:9093", - "kafka.${ProducerConfig.ACKS_CONFIG}" to 1, - Neo4jConnectorConfig.DATABASE to "customers", - Neo4jConnectorConfig.AUTHENTICATION_BASIC_USERNAME to "FOO", - Neo4jConnectorConfig.AUTHENTICATION_BASIC_PASSWORD to "BAR") - val config = Neo4jSinkConnectorConfig(originals) - - assertEquals(mapOf(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG to "broker:9093", - ProducerConfig.ACKS_CONFIG to 1), config.kafkaBrokerProperties) - assertEquals(originals["${Neo4jSinkConnectorConfig.TOPIC_CYPHER_PREFIX}foo"], config.topics.cypherTopics["foo"]) - assertFalse { config.encryptionEnabled } - assertEquals(a, config.serverUri.get(0).toString()) - assertEquals(b, config.serverUri.get(1).toString()) - assertEquals(originals[Neo4jConnectorConfig.BATCH_SIZE], config.batchSize) - assertEquals(Config.TrustStrategy.Strategy.TRUST_ALL_CERTIFICATES, config.encryptionTrustStrategy) - assertEquals(AuthenticationType.BASIC, config.authenticationType) - assertEquals(originals[Neo4jConnectorConfig.AUTHENTICATION_BASIC_USERNAME], config.authenticationUsername) - assertEquals(originals[Neo4jConnectorConfig.AUTHENTICATION_BASIC_PASSWORD], config.authenticationPassword) - assertEquals(originals[Neo4jConnectorConfig.DATABASE], config.database) - assertEquals("", config.authenticationKerberosTicket) - assertNull(config.encryptionCACertificateFile, "encryptionCACertificateFile should be null") - - assertEquals(Neo4jConnectorConfig.CONNECTION_MAX_CONNECTION_LIFETIME_MSECS_DEFAULT, config.connectionMaxConnectionLifetime) - assertEquals(Neo4jConnectorConfig.CONNECTION_LIVENESS_CHECK_TIMEOUT_MSECS_DEFAULT, config.connectionLivenessCheckTimeout) - assertEquals(Neo4jConnectorConfig.CONNECTION_POOL_MAX_SIZE_DEFAULT, config.connectionPoolMaxSize) - assertEquals(PoolSettings.DEFAULT_CONNECTION_ACQUISITION_TIMEOUT, config.connectionAcquisitionTimeout) - assertEquals(Neo4jConnectorConfig.BATCH_TIMEOUT_DEFAULT, config.batchTimeout) - assertEquals(Neo4jConnectorConfig.BATCH_TIMEOUT_DEFAULT, config.batchTimeout) - } - - @Test - fun `should return valid configuration with multiple URIs`() { - val a = "bolt://neo4j:7687" - val b = "bolt://neo4j2:7687" - val c = "bolt://neo4j3:7777" - - val originals = mapOf(SinkConnector.TOPICS_CONFIG to "foo", - "${Neo4jSinkConnectorConfig.TOPIC_CYPHER_PREFIX}foo" to "CREATE (p:Person{name: event.firstName})", - Neo4jConnectorConfig.SERVER_URI to "$a,$b,$c") - val config = Neo4jSinkConnectorConfig(originals) - - assertEquals(a, config.serverUri[0].toString()) - assertEquals(b, config.serverUri[1].toString()) - assertEquals(c, config.serverUri[2].toString()) - } - - @Test - fun `should return the configuration with shuffled topic order`() { - val originals = mapOf(SinkConnector.TOPICS_CONFIG to "bar,foo", - "${Neo4jSinkConnectorConfig.TOPIC_PATTERN_NODE_PREFIX}foo" to "(:Foo{!fooId,fooName})", - "${Neo4jSinkConnectorConfig.TOPIC_PATTERN_NODE_PREFIX}bar" to "(:Bar{!barId,barName})", - Neo4jConnectorConfig.SERVER_URI to "bolt://neo4j:7687", - Neo4jConnectorConfig.BATCH_SIZE to 10, - Neo4jConnectorConfig.AUTHENTICATION_BASIC_USERNAME to "FOO", - Neo4jConnectorConfig.AUTHENTICATION_BASIC_PASSWORD to "BAR") - val config = Neo4jSinkConnectorConfig(originals) - - assertEquals(originals["${Neo4jSinkConnectorConfig.TOPIC_CYPHER_PREFIX}foo"], config.topics.cypherTopics["foo"]) - assertFalse { config.encryptionEnabled } - assertEquals(originals[Neo4jConnectorConfig.SERVER_URI], config.serverUri.get(0).toString()) - assertEquals(originals[Neo4jConnectorConfig.BATCH_SIZE], config.batchSize) - assertEquals(Config.TrustStrategy.Strategy.TRUST_ALL_CERTIFICATES, config.encryptionTrustStrategy) - assertEquals(AuthenticationType.BASIC, config.authenticationType) - assertEquals(originals[Neo4jConnectorConfig.AUTHENTICATION_BASIC_USERNAME], config.authenticationUsername) - assertEquals(originals[Neo4jConnectorConfig.AUTHENTICATION_BASIC_PASSWORD], config.authenticationPassword) - assertEquals(originals[Neo4jConnectorConfig.AUTHENTICATION_BASIC_PASSWORD], config.authenticationPassword) - assertEquals("", config.authenticationKerberosTicket) - assertNull(config.encryptionCACertificateFile, "encryptionCACertificateFile should be null") - - assertEquals(Neo4jConnectorConfig.CONNECTION_MAX_CONNECTION_LIFETIME_MSECS_DEFAULT, config.connectionMaxConnectionLifetime) - assertEquals(Neo4jConnectorConfig.CONNECTION_LIVENESS_CHECK_TIMEOUT_MSECS_DEFAULT, config.connectionLivenessCheckTimeout) - assertEquals(Neo4jConnectorConfig.CONNECTION_POOL_MAX_SIZE_DEFAULT, config.connectionPoolMaxSize) - assertEquals(PoolSettings.DEFAULT_CONNECTION_ACQUISITION_TIMEOUT, config.connectionAcquisitionTimeout) - assertEquals(Neo4jConnectorConfig.BATCH_TIMEOUT_DEFAULT, config.batchTimeout) - } - -} \ No newline at end of file diff --git a/kafka-connect-neo4j/src/test/kotlin/streams/kafka/connect/sink/Neo4jSinkTaskAuraTest.kt b/kafka-connect-neo4j/src/test/kotlin/streams/kafka/connect/sink/Neo4jSinkTaskAuraTest.kt deleted file mode 100644 index 00c698b8..00000000 --- a/kafka-connect-neo4j/src/test/kotlin/streams/kafka/connect/sink/Neo4jSinkTaskAuraTest.kt +++ /dev/null @@ -1,372 +0,0 @@ -package streams.kafka.connect.sink - -import org.apache.kafka.connect.data.Schema -import org.apache.kafka.connect.data.SchemaBuilder -import org.apache.kafka.connect.data.Struct -import org.apache.kafka.connect.sink.SinkRecord -import org.apache.kafka.connect.sink.SinkTask -import org.apache.kafka.connect.sink.SinkTaskContext -import org.junit.* -import org.junit.Assume.assumeTrue -import org.mockito.Mockito.mock -import org.neo4j.driver.* -import org.neo4j.driver.exceptions.ClientException -import streams.events.* -import streams.kafka.connect.common.Neo4jConnectorConfig -import streams.service.sink.strategy.CUDNode -import streams.service.sink.strategy.CUDOperations -import streams.utils.JSONUtils -import java.util.* -import kotlin.test.assertEquals -import kotlin.test.assertFailsWith -import kotlin.test.assertFalse -import kotlin.test.assertTrue - - -class Neo4jSinkTaskAuraTest { - - companion object { - - private val SIMPLE_SCHEMA = SchemaBuilder.struct().name("com.example.Person") - .field("name", Schema.STRING_SCHEMA) - .build() - - private val user: String? = System.getenv("AURA_USER") - private val password: String? = System.getenv("AURA_PASSWORD") - private val uri: String? = System.getenv("AURA_URI") - private var driver: Driver? = null - - private const val NAME_TOPIC = "neotopic" - private const val SHOW_CURRENT_USER = "SHOW CURRENT USER" - private const val DBMS_LIST_CONFIG = "CALL dbms.listConfig" - private const val NEO4J = "neo4j" - private const val SYSTEM = "system" - private const val ERROR_ADMIN_COMMAND = "Executing admin procedure is not allowed for user '$NEO4J' with roles [PUBLIC] overridden by READ restricted to ACCESS." - private const val LABEL_SINK_AURA = "SinkAuraTest" - private const val COUNT_NODES_SINK_AURA = "MATCH (s:$LABEL_SINK_AURA) RETURN count(s) as count" - - @BeforeClass - @JvmStatic - fun setUp() { - assumeTrue(user != null) - assumeTrue(password != null) - driver = GraphDatabase.driver(uri, AuthTokens.basic(user, password)) - } - - fun getMapSinkConnectorConfig() = mutableMapOf( - Neo4jConnectorConfig.AUTHENTICATION_BASIC_USERNAME to user!!, - Neo4jConnectorConfig.AUTHENTICATION_BASIC_PASSWORD to password!!, - Neo4jConnectorConfig.SERVER_URI to uri!!, - Neo4jConnectorConfig.AUTHENTICATION_TYPE to AuthenticationType.BASIC.toString(), - Neo4jSinkConnectorConfig.TOPIC_CDC_SOURCE_ID_LABEL_NAME to LABEL_SINK_AURA, - ) - - fun countEntitiesSinkAura(session: Session?, number: Int, query: String = COUNT_NODES_SINK_AURA) = session!!.run(query).let { - assertTrue { it!!.hasNext() } - assertEquals(number, it!!.next()["count"].asInt()) - assertFalse { it.hasNext() } - } - } - - @After - fun clearNodesAura() { - driver?.session()?.run("MATCH (n:$LABEL_SINK_AURA) DETACH DELETE n") - } - - - @Test - fun `test with struct in Aura 4`() { - driver?.session().use { countEntitiesSinkAura(it, 0) } - val props = getMapSinkConnectorConfig() - props["${Neo4jSinkConnectorConfig.TOPIC_CYPHER_PREFIX}$NAME_TOPIC"] = " CREATE (b:$LABEL_SINK_AURA)" - props[Neo4jConnectorConfig.BATCH_SIZE] = 2.toString() - props[SinkTask.TOPICS_CONFIG] = NAME_TOPIC - - val task = Neo4jSinkTask() - task.initialize(mock(SinkTaskContext::class.java)) - task.start(props) - val input = listOf(SinkRecord(NAME_TOPIC, 1, null, null, SIMPLE_SCHEMA, Struct(SIMPLE_SCHEMA).put("name", "Baz"), 42)) - task.put(input) - - driver?.session().use { countEntitiesSinkAura(it, 1) } - } - - @Test - fun `should insert data into Neo4j from CDC events in Aura 4`() { - - val props = getMapSinkConnectorConfig() - props[SinkTask.TOPICS_CONFIG] = NAME_TOPIC - props[Neo4jSinkConnectorConfig.TOPIC_CDC_SOURCE_ID] = NAME_TOPIC - - val cdcDataStart = StreamsTransactionEvent(meta = Meta(timestamp = System.currentTimeMillis(), - username = "user", - txId = 1, - txEventId = 0, - txEventsCount = 3, - operation = OperationType.created - ), - payload = NodePayload(id = "0", - before = null, - after = NodeChange(properties = mapOf("name" to "Pippo"), labels = listOf("User")) - ), - schema = Schema() - ) - val cdcDataEnd = StreamsTransactionEvent(meta = Meta(timestamp = System.currentTimeMillis(), - username = "user", - txId = 1, - txEventId = 1, - txEventsCount = 3, - operation = OperationType.created - ), - payload = NodePayload(id = "1", - before = null, - after = NodeChange(properties = mapOf("name" to "Pluto"), labels = listOf("User Ext")) - ), - schema = Schema() - ) - val cdcDataRelationship = StreamsTransactionEvent(meta = Meta(timestamp = System.currentTimeMillis(), - username = "user", - txId = 1, - txEventId = 2, - txEventsCount = 3, - operation = OperationType.created - ), - payload = RelationshipPayload( - id = "2", - start = RelationshipNodeChange(id = "0", labels = listOf("User"), ids = emptyMap()), - end = RelationshipNodeChange(id = "1", labels = listOf("User Ext"), ids = emptyMap()), - after = RelationshipChange(properties = mapOf("since" to 2014)), - before = null, - label = "HAS_REL" - ), - schema = Schema() - ) - - val task = Neo4jSinkTask() - task.initialize(mock(SinkTaskContext::class.java)) - task.start(props) - val input = listOf(SinkRecord(NAME_TOPIC, 1, null, null, null, cdcDataStart, 42), - SinkRecord(NAME_TOPIC, 1, null, null, null, cdcDataEnd, 43), - SinkRecord(NAME_TOPIC, 1, null, null, null, cdcDataRelationship, 44)) - task.put(input) - - driver?.session().use { - countEntitiesSinkAura(it, 1, "MATCH (:$LABEL_SINK_AURA)-[r:HAS_REL]->(:$LABEL_SINK_AURA) RETURN COUNT(r) as count") - } - } - - @Test - fun `should update data into Neo4j from CDC events in Aura 4`() { - driver?.session()?.run(""" - CREATE (s:User:OldLabel:$LABEL_SINK_AURA{name:'Pippo', sourceId:'0'}) - -[r:`KNOWS WHO`{since:2014, sourceId:'2'}]-> - (e:`User Ext`:$LABEL_SINK_AURA{name:'Pluto', sourceId:'1'}) - """.trimIndent() - ) - - val props = getMapSinkConnectorConfig() - props[Neo4jSinkConnectorConfig.TOPIC_CDC_SOURCE_ID] = NAME_TOPIC - props[SinkTask.TOPICS_CONFIG] = NAME_TOPIC - - val cdcDataStart = StreamsTransactionEvent(meta = Meta(timestamp = System.currentTimeMillis(), - username = "user", - txId = 1, - txEventId = 0, - txEventsCount = 3, - operation = OperationType.updated - ), - payload = NodePayload(id = "0", - before = NodeChange(properties = mapOf("name" to "Pippo"), - labels = listOf("User", "OldLabel")), - after = NodeChange(properties = mapOf("name" to "Pippo", "age" to 99), - labels = listOf("User")) - ), - schema = Schema() - ) - val cdcDataRelationship = StreamsTransactionEvent(meta = Meta(timestamp = System.currentTimeMillis(), - username = "user", - txId = 1, - txEventId = 2, - txEventsCount = 3, - operation = OperationType.updated - ), - payload = RelationshipPayload( - id = "2", - start = RelationshipNodeChange(id = "0", labels = listOf("User"), ids = emptyMap()), - end = RelationshipNodeChange(id = "1", labels = listOf("User Ext"), ids = emptyMap()), - after = RelationshipChange(properties = mapOf("since" to 1999, "foo" to "bar")), - before = RelationshipChange(properties = mapOf("since" to 2014)), - label = "KNOWS WHO" - ), - schema = Schema() - ) - - val task = Neo4jSinkTask() - task.initialize(mock(SinkTaskContext::class.java)) - task.start(props) - val input = listOf(SinkRecord(NAME_TOPIC, 1, null, null, null, cdcDataStart, 42), - SinkRecord(NAME_TOPIC, 1, null, null, null, cdcDataRelationship, 43)) - task.put(input) - - driver?.session().use { - countEntitiesSinkAura(it, 1, - "MATCH (:User {age:99})-[r:`KNOWS WHO`{since:1999, sourceId:'2', foo:'bar'}]->(:`User Ext`) RETURN COUNT(r) as count") - } - } - - @Test - fun `should delete data into Neo4j from CDC events in Aura 4`() { - - driver?.session().use { - it?.run("CREATE (s:User:OldLabel:$LABEL_SINK_AURA{name:'Andrea', `comp@ny`:'LARUS-BA', sourceId:'0'})") - it?.run("CREATE (s:User:OldLabel:$LABEL_SINK_AURA{name:'Andrea', `comp@ny`:'LARUS-BA', sourceId:'1'})") - } - - driver?.session().use { countEntitiesSinkAura(it, 2) } - - val props = getMapSinkConnectorConfig() - props[SinkTask.TOPICS_CONFIG] = NAME_TOPIC - props[Neo4jSinkConnectorConfig.TOPIC_CDC_SOURCE_ID] = NAME_TOPIC - - val cdcDataStart = StreamsTransactionEvent(meta = Meta(timestamp = System.currentTimeMillis(), - username = "user", - txId = 1, - txEventId = 0, - txEventsCount = 3, - operation = OperationType.deleted - ), - payload = NodePayload(id = "0", - before = NodeChange(properties = mapOf("name" to "Andrea", "comp@ny" to "LARUS-BA"), - labels = listOf("User", "OldLabel")), - after = null - ), - schema = Schema() - ) - val task = Neo4jSinkTask() - task.initialize(mock(SinkTaskContext::class.java)) - task.start(props) - val input = listOf(SinkRecord(NAME_TOPIC, 1, null, null, null, cdcDataStart, 42)) - task.put(input) - - driver?.session().use { countEntitiesSinkAura(it, 1) } - } - - @Test - fun `should work with node pattern topic in Aura 4`() { - val props = getMapSinkConnectorConfig() - props["${Neo4jSinkConnectorConfig.TOPIC_PATTERN_NODE_PREFIX}$NAME_TOPIC"] = "$LABEL_SINK_AURA{!userId,name,surname,address.city}" - props[SinkTask.TOPICS_CONFIG] = NAME_TOPIC - - val data = mapOf("userId" to 1, "name" to "Pippo", "surname" to "Pluto", - "address" to mapOf("city" to "Cerignola", "CAP" to "12345")) - - val task = Neo4jSinkTask() - task.initialize(mock(SinkTaskContext::class.java)) - task.start(props) - val input = listOf(SinkRecord(NAME_TOPIC, 1, null, null, null, data, 42)) - task.put(input) - - driver?.session().use { - countEntitiesSinkAura(it, 1, - "MATCH (n:$LABEL_SINK_AURA{name: 'Pippo', surname: 'Pluto', userId: 1, `address.city`: 'Cerignola'}) RETURN count(n) AS count") - } - } - - @Test - fun `should work with relationship pattern topic in Aura 4`() { - val props = getMapSinkConnectorConfig() - props["${Neo4jSinkConnectorConfig.TOPIC_PATTERN_RELATIONSHIP_PREFIX}$NAME_TOPIC"] = "(:$LABEL_SINK_AURA{!sourceId,sourceName,sourceSurname})-[:HAS_REL]->(:$LABEL_SINK_AURA{!targetId,targetName,targetSurname})" - props[SinkTask.TOPICS_CONFIG] = NAME_TOPIC - - val data = mapOf("sourceId" to 1, "sourceName" to "Pippo", "sourceSurname" to "Pluto", - "targetId" to 1, "targetName" to "Foo", "targetSurname" to "Bar") - - val task = Neo4jSinkTask() - task.initialize(mock(SinkTaskContext::class.java)) - task.start(props) - val input = listOf(SinkRecord(NAME_TOPIC, 1, null, null, null, data, 42)) - task.put(input) - driver?.session().use { - countEntitiesSinkAura(it, 1, "MATCH (:$LABEL_SINK_AURA{sourceId: 1})-[r:HAS_REL]->(:$LABEL_SINK_AURA{targetId: 1}) RETURN COUNT(r) as count") - } - } - - @Test - fun `should ingest node data from CUD Events in Aura 4`() { - val mergeMarkers = listOf(2, 5, 7) - val key = "key" - val topic = UUID.randomUUID().toString() - val data = (1..10).map { - val labels = if (it % 2 == 0) listOf(LABEL_SINK_AURA, "Bar") else listOf(LABEL_SINK_AURA, "Bar", "Label") - val properties = mapOf("foo" to "foo-value-$it", "id" to it) - val (op, ids) = when (it) { - in mergeMarkers -> CUDOperations.merge to mapOf(key to it) - else -> CUDOperations.create to emptyMap() - } - val cudNode = CUDNode(op = op, - labels = labels, - ids = ids, - properties = properties) - SinkRecord(topic, 1, null, null, null, JSONUtils.asMap(cudNode), it.toLong()) - } - - val props = getMapSinkConnectorConfig() - props[Neo4jSinkConnectorConfig.TOPIC_CUD] = topic - props[SinkTask.TOPICS_CONFIG] = topic - - val task = Neo4jSinkTask() - task.initialize(mock(SinkTaskContext::class.java)) - task.start(props) - task.put(data) - - driver?.session().use { - countEntitiesSinkAura(it, 5, "MATCH (n:$LABEL_SINK_AURA:Bar:Label) RETURN count(n) AS count") - countEntitiesSinkAura(it, 10, "MATCH (n:$LABEL_SINK_AURA:Bar) RETURN count(n) AS count") - } - } - - @Test - fun `neo4j user should not have the admin role in Aura 4`() { - - driver?.session(SessionConfig.forDatabase(SYSTEM)).use { session -> - session?.run(SHOW_CURRENT_USER).let { - assertTrue { it!!.hasNext() } - val roles = it!!.next().get("roles").asList() - assertFalse { roles.contains("admin") } - assertTrue { roles.contains("PUBLIC") } - assertFalse { it.hasNext() } - } - } - } - - @Test - fun `should fail if I try to run SHOW CURRENT USER commands on neo4j database in Aura 4`() { - - assertFailsWith(ClientException::class, - "This is an administration command and it should be executed against the system database: $SHOW_CURRENT_USER") - { - driver?.session(SessionConfig.forDatabase(NEO4J)).use { - it?.run(SHOW_CURRENT_USER) - } - } - } - - @Test - fun `should fail if I try to run admin commands with neo4j user in Aura 4`() { - - assertFailsWith(ClientException::class, ERROR_ADMIN_COMMAND) - { - driver?.session(SessionConfig.forDatabase(SYSTEM)).use { - it?.run(DBMS_LIST_CONFIG) - } - } - - assertFailsWith(ClientException::class, ERROR_ADMIN_COMMAND) - { - driver?.session(SessionConfig.forDatabase(NEO4J)).use { - it?.run(DBMS_LIST_CONFIG) - } - } - } - -} diff --git a/kafka-connect-neo4j/src/test/kotlin/streams/kafka/connect/sink/Neo4jSinkTaskTest.kt b/kafka-connect-neo4j/src/test/kotlin/streams/kafka/connect/sink/Neo4jSinkTaskTest.kt deleted file mode 100644 index 65f3a039..00000000 --- a/kafka-connect-neo4j/src/test/kotlin/streams/kafka/connect/sink/Neo4jSinkTaskTest.kt +++ /dev/null @@ -1,1372 +0,0 @@ -package streams.kafka.connect.sink - -import org.apache.kafka.connect.data.Schema -import org.apache.kafka.connect.data.SchemaBuilder -import org.apache.kafka.connect.data.Struct -import org.apache.kafka.connect.data.Timestamp -import org.apache.kafka.connect.sink.SinkRecord -import org.apache.kafka.connect.sink.SinkTask -import org.apache.kafka.connect.sink.SinkTaskContext -import org.junit.After -import org.junit.Before -import org.junit.Ignore -import org.junit.Rule -import org.junit.Test -import org.mockito.Mockito.mock -import org.neo4j.configuration.GraphDatabaseSettings -import org.neo4j.graphdb.Label -import org.neo4j.graphdb.Node -import org.neo4j.harness.junit.rule.Neo4jRule -import streams.events.* -import streams.kafka.connect.common.Neo4jConnectorConfig -import streams.utils.JSONUtils -import streams.service.errors.ErrorService -import streams.service.errors.ProcessingError -import streams.service.sink.strategy.CUDNode -import streams.service.sink.strategy.CUDNodeRel -import streams.service.sink.strategy.CUDOperations -import streams.service.sink.strategy.CUDRelationship -import java.util.Date -import java.util.UUID -import java.util.stream.Collectors -import java.util.stream.StreamSupport -import kotlin.test.assertEquals -import kotlin.test.assertFalse -import kotlin.test.assertTrue -import kotlin.test.fail - - -class Neo4jSinkTaskTest { - - @Rule @JvmField val db = Neo4jRule() - .withDisabledServer() - .withConfig(GraphDatabaseSettings.auth_enabled, false) - - private lateinit var task: SinkTask - - @After - fun after() { - db.defaultDatabaseService().beginTx().use { - it.execute("MATCH (n) DETACH DELETE n") - it.commit() - } - task.stop() - } - - @Before - fun before() { - task = Neo4jSinkTask() - task.initialize(mock(SinkTaskContext::class.java)) - } - - - private val PERSON_SCHEMA = SchemaBuilder.struct().name("com.example.Person") - .field("firstName", Schema.STRING_SCHEMA) - .field("lastName", Schema.STRING_SCHEMA) - .field("age", Schema.OPTIONAL_INT32_SCHEMA) - .field("bool", Schema.OPTIONAL_BOOLEAN_SCHEMA) - .field("short", Schema.OPTIONAL_INT16_SCHEMA) - .field("byte", Schema.OPTIONAL_INT8_SCHEMA) - .field("long", Schema.OPTIONAL_INT64_SCHEMA) - .field("float", Schema.OPTIONAL_FLOAT32_SCHEMA) - .field("double", Schema.OPTIONAL_FLOAT64_SCHEMA) - .field("modified", Timestamp.SCHEMA) - .build() - - private val PLACE_SCHEMA = SchemaBuilder.struct().name("com.example.Place") - .field("name", Schema.STRING_SCHEMA) - .field("latitude", Schema.FLOAT32_SCHEMA) - .field("longitude", Schema.FLOAT32_SCHEMA) - .build() - - @Test - fun `test array of struct`() { - val firstTopic = "neotopic" - val props = mutableMapOf() - props[Neo4jConnectorConfig.SERVER_URI] = db.boltURI().toString() - props["${Neo4jSinkConnectorConfig.TOPIC_CYPHER_PREFIX}$firstTopic"] = """ - CREATE (b:BODY) - WITH event.p AS paragraphList, event.ul AS ulList, b - FOREACH (paragraph IN paragraphList | CREATE (b)-[:HAS_P]->(p:P{value: paragraph.value})) - - WITH ulList, b - UNWIND ulList AS ulElem - CREATE (b)-[:HAS_UL]->(ul:UL) - - WITH ulElem, ul - UNWIND ulElem.value AS liElem - CREATE (ul)-[:HAS_LI]->(li:LI{value: liElem.value, class: liElem.class}) - """.trimIndent() - props[Neo4jConnectorConfig.AUTHENTICATION_TYPE] = AuthenticationType.NONE.toString() - props[Neo4jConnectorConfig.BATCH_SIZE] = 2.toString() - props[SinkTask.TOPICS_CONFIG] = firstTopic - - task.start(props) - val input = listOf(SinkRecord(firstTopic, 1, null, null, PERSON_SCHEMA, Neo4jValueConverterTest.getTreeStruct(), 42)) - task.put(input) - db.defaultDatabaseService().beginTx().use { - assertEquals(1, it.findNodes(Label.label("BODY")).stream().count()) - assertEquals(2, it.findNodes(Label.label("P")).stream().count()) - assertEquals(2, it.findNodes(Label.label("UL")).stream().count()) - assertEquals(4, it.findNodes(Label.label("LI")).stream().count()) - - assertEquals(2, it.execute("MATCH (b:BODY)-[r:HAS_P]->(p:P) RETURN COUNT(r) AS COUNT").columnAs("COUNT").next()) - assertEquals(2, it.execute("MATCH (b:BODY)-[r:HAS_UL]->(ul:UL) RETURN COUNT(r) AS COUNT").columnAs("COUNT").next()) - assertEquals(4, it.execute("MATCH (ul:UL)-[r:HAS_LI]->(li:LI) RETURN COUNT(r) AS COUNT").columnAs("COUNT").next()) - - assertEquals(1, it.execute("MATCH (li:LI{class:['ClassA', 'ClassB']}) RETURN COUNT(li) AS COUNT").columnAs("COUNT").next()) - } - } - - @Test - fun `should insert data into Neo4j`() { - val firstTopic = "neotopic" - val secondTopic = "foo" - val props = mutableMapOf() - props[Neo4jConnectorConfig.SERVER_URI] = db.boltURI().toString() - props["${Neo4jSinkConnectorConfig.TOPIC_CYPHER_PREFIX}$firstTopic"] = "CREATE (n:PersonExt {name: event.firstName, surname: event.lastName})" - props["${Neo4jSinkConnectorConfig.TOPIC_CYPHER_PREFIX}$secondTopic"] = "CREATE (n:Person {name: event.firstName})" - props[Neo4jConnectorConfig.AUTHENTICATION_TYPE] = AuthenticationType.NONE.toString() - props[Neo4jConnectorConfig.BATCH_SIZE] = 2.toString() - props[SinkTask.TOPICS_CONFIG] = "$firstTopic,$secondTopic" - - val struct= Struct(PERSON_SCHEMA) - .put("firstName", "Alex") - .put("lastName", "Smith") - .put("bool", true) - .put("short", 1234.toShort()) - .put("byte", (-32).toByte()) - .put("long", 12425436L) - .put("float", 2356.3.toFloat()) - .put("double", -2436546.56457) - .put("age", 21) - .put("modified", Date(1474661402123L)) - - task.start(props) - val input = listOf(SinkRecord(firstTopic, 1, null, null, PERSON_SCHEMA, struct, 42), - SinkRecord(firstTopic, 1, null, null, PERSON_SCHEMA, struct, 42), - SinkRecord(firstTopic, 1, null, null, PERSON_SCHEMA, struct, 42), - SinkRecord(firstTopic, 1, null, null, PERSON_SCHEMA, struct, 42), - SinkRecord(firstTopic, 1, null, null, PERSON_SCHEMA, struct, 42), - SinkRecord(secondTopic, 1, null, null, PERSON_SCHEMA, struct, 43)) - task.put(input) - db.defaultDatabaseService().beginTx().use { - val personCount = it.execute("MATCH (p:Person) RETURN COUNT(p) as COUNT").columnAs("COUNT").next() - val expectedPersonCount = input.filter { it.topic() == secondTopic }.size - assertEquals(expectedPersonCount, personCount.toInt()) - val personExtCount = it.execute("MATCH (p:PersonExt) RETURN COUNT(p) as COUNT").columnAs("COUNT").next() - val expectedPersonExtCount = input.filter { it.topic() == firstTopic }.size - assertEquals(expectedPersonExtCount, personExtCount.toInt()) - } - } - - @Test - fun `should insert data into Neo4j from CDC events`() { - val firstTopic = "neotopic" - val props = mapOf(Neo4jConnectorConfig.SERVER_URI to db.boltURI().toString(), - Neo4jSinkConnectorConfig.TOPIC_CDC_SOURCE_ID to firstTopic, - Neo4jConnectorConfig.AUTHENTICATION_TYPE to AuthenticationType.NONE.toString(), - SinkTask.TOPICS_CONFIG to firstTopic) - - val cdcDataStart = StreamsTransactionEvent(meta = Meta(timestamp = System.currentTimeMillis(), - username = "user", - txId = 1, - txEventId = 0, - txEventsCount = 3, - operation = OperationType.created - ), - payload = NodePayload(id = "0", - before = null, - after = NodeChange(properties = mapOf("name" to "Andrea", "comp@ny" to "LARUS-BA"), labels = listOf("User")) - ), - schema = Schema() - ) - val cdcDataEnd = StreamsTransactionEvent(meta = Meta(timestamp = System.currentTimeMillis(), - username = "user", - txId = 1, - txEventId = 1, - txEventsCount = 3, - operation = OperationType.created - ), - payload = NodePayload(id = "1", - before = null, - after = NodeChange(properties = mapOf("name" to "Michael", "comp@ny" to "Neo4j"), labels = listOf("User Ext")) - ), - schema = Schema() - ) - val cdcDataRelationship = StreamsTransactionEvent(meta = Meta(timestamp = System.currentTimeMillis(), - username = "user", - txId = 1, - txEventId = 2, - txEventsCount = 3, - operation = OperationType.created - ), - payload = RelationshipPayload( - id = "2", - start = RelationshipNodeChange(id = "0", labels = listOf("User"), ids = emptyMap()), - end = RelationshipNodeChange(id = "1", labels = listOf("User Ext"), ids = emptyMap()), - after = RelationshipChange(properties = mapOf("since" to 2014)), - before = null, - label = "KNOWS WHO" - ), - schema = Schema() - ) - - task.start(props) - val input = listOf(SinkRecord(firstTopic, 1, null, null, null, cdcDataStart, 42), - SinkRecord(firstTopic, 1, null, null, null, cdcDataEnd, 43), - SinkRecord(firstTopic, 1, null, null, null, cdcDataRelationship, 44)) - task.put(input) - db.defaultDatabaseService().beginTx().use { - it.execute(""" - MATCH p = (s:User{name:'Andrea', `comp@ny`:'LARUS-BA', sourceId:'0'}) - -[r:`KNOWS WHO`{since:2014, sourceId:'2'}]-> - (e:`User Ext`{name:'Michael', `comp@ny`:'Neo4j', sourceId:'1'}) - RETURN count(p) AS count - """.trimIndent()) - .columnAs("count").use { - assertTrue { it.hasNext() } - val count = it.next() - assertEquals(1, count) - assertFalse { it.hasNext() } - } - } - } - - @Test - fun `should update data into Neo4j from CDC events`() { - db.defaultDatabaseService().beginTx().use { - it.execute(""" - CREATE (s:User:OldLabel:SourceEvent{name:'Andrea', `comp@ny`:'LARUS-BA', sourceId:'0'}) - -[r:`KNOWS WHO`{since:2014, sourceId:'2'}]-> - (e:`User Ext`:SourceEvent{name:'Michael', `comp@ny`:'Neo4j', sourceId:'1'}) - """.trimIndent()).close() - it.commit() - } - val firstTopic = "neotopic" - val props = mapOf(Neo4jConnectorConfig.SERVER_URI to db.boltURI().toString(), - Neo4jSinkConnectorConfig.TOPIC_CDC_SOURCE_ID to firstTopic, - Neo4jConnectorConfig.AUTHENTICATION_TYPE to AuthenticationType.NONE.toString(), - SinkTask.TOPICS_CONFIG to firstTopic) - - val cdcDataStart = StreamsTransactionEvent(meta = Meta(timestamp = System.currentTimeMillis(), - username = "user", - txId = 1, - txEventId = 0, - txEventsCount = 3, - operation = OperationType.updated - ), - payload = NodePayload(id = "0", - before = NodeChange(properties = mapOf("name" to "Andrea", "comp@ny" to "LARUS-BA"), - labels = listOf("User", "OldLabel")), - after = NodeChange(properties = mapOf("name" to "Andrea", "comp@ny" to "LARUS-BA, Venice", "age" to 34), - labels = listOf("User")) - ), - schema = Schema() - ) - val cdcDataRelationship = StreamsTransactionEvent(meta = Meta(timestamp = System.currentTimeMillis(), - username = "user", - txId = 1, - txEventId = 2, - txEventsCount = 3, - operation = OperationType.updated - ), - payload = RelationshipPayload( - id = "2", - start = RelationshipNodeChange(id = "0", labels = listOf("User"), ids = emptyMap()), - end = RelationshipNodeChange(id = "1", labels = listOf("User Ext"), ids = emptyMap()), - after = RelationshipChange(properties = mapOf("since" to 2014, "foo" to "bar")), - before = RelationshipChange(properties = mapOf("since" to 2014)), - label = "KNOWS WHO" - ), - schema = Schema() - ) - - task.start(props) - val input = listOf(SinkRecord(firstTopic, 1, null, null, null, cdcDataStart, 42), - SinkRecord(firstTopic, 1, null, null, null, cdcDataRelationship, 43)) - task.put(input) - db.defaultDatabaseService().beginTx().use { - it.execute(""" - MATCH p = (s:User{name:'Andrea', `comp@ny`:'LARUS-BA, Venice', sourceId:'0', age:34}) - -[r:`KNOWS WHO`{since:2014, sourceId:'2', foo:'bar'}]-> - (e:`User Ext`{name:'Michael', `comp@ny`:'Neo4j', sourceId:'1'}) - RETURN count(p) AS count - """.trimIndent()) - .columnAs("count").use { - assertTrue { it.hasNext() } - val count = it.next() - assertEquals(1, count) - assertFalse { it.hasNext() } - } - } - } - - @Test - fun `should insert data into Neo4j from CDC events with schema strategy`() { - val firstTopic = "neotopic" - val props = mapOf(Neo4jConnectorConfig.SERVER_URI to db.boltURI().toString(), - Neo4jSinkConnectorConfig.TOPIC_CDC_SCHEMA to firstTopic, - Neo4jConnectorConfig.AUTHENTICATION_TYPE to AuthenticationType.NONE.toString(), - SinkTask.TOPICS_CONFIG to firstTopic) - - val constraints = listOf(Constraint(label = "User", type = StreamsConstraintType.UNIQUE, properties = setOf("name", "surname"))) - val relSchema = Schema(properties = mapOf("since" to "Long"), constraints = constraints) - val nodeSchema = Schema(properties = mapOf("name" to "String", "surname" to "String", "comp@ny" to "String"), - constraints = constraints) - val cdcDataStart = StreamsTransactionEvent( - meta = Meta(timestamp = System.currentTimeMillis(), - username = "user", - txId = 1, - txEventId = 0, - txEventsCount = 3, - operation = OperationType.created - ), - payload = NodePayload(id = "0", - before = null, - after = NodeChange(properties = mapOf("name" to "Andrea", "surname" to "Santurbano", "comp@ny" to "LARUS-BA"), labels = listOf("User")) - ), - schema = nodeSchema - ) - val cdcDataEnd = StreamsTransactionEvent( - meta = Meta(timestamp = System.currentTimeMillis(), - username = "user", - txId = 1, - txEventId = 1, - txEventsCount = 3, - operation = OperationType.created - ), - payload = NodePayload(id = "1", - before = null, - after = NodeChange(properties = mapOf("name" to "Michael", "surname" to "Hunger", "comp@ny" to "Neo4j"), labels = listOf("User")) - ), - schema = nodeSchema - ) - val cdcDataRelationship = StreamsTransactionEvent( - meta = Meta(timestamp = System.currentTimeMillis(), - username = "user", - txId = 1, - txEventId = 2, - txEventsCount = 3, - operation = OperationType.created - ), - payload = RelationshipPayload( - id = "2", - start = RelationshipNodeChange(id = "0", labels = listOf("User"), ids = mapOf("name" to "Andrea", "surname" to "Santurbano")), - end = RelationshipNodeChange(id = "1", labels = listOf("User"), ids = mapOf("name" to "Michael", "surname" to "Hunger")), - after = RelationshipChange(properties = mapOf("since" to 2014)), - before = null, - label = "KNOWS WHO" - ), - schema = relSchema - ) - - task.start(props) - val input = listOf(SinkRecord(firstTopic, 1, null, null, null, cdcDataStart, 42), - SinkRecord(firstTopic, 1, null, null, null, cdcDataEnd, 43), - SinkRecord(firstTopic, 1, null, null, null, cdcDataRelationship, 44)) - task.put(input) - db.defaultDatabaseService().beginTx().use { - val query = """ - |MATCH p = (s:User{name:'Andrea', surname:'Santurbano', `comp@ny`:'LARUS-BA'})-[r:`KNOWS WHO`{since:2014}]->(e:User{name:'Michael', surname:'Hunger', `comp@ny`:'Neo4j'}) - |RETURN count(p) AS count - |""".trimMargin() - it.execute(query) - .columnAs("count").use { - assertTrue { it.hasNext() } - val count = it.next() - assertEquals(1, count) - assertFalse { it.hasNext() } - } - - val labels = db.defaultDatabaseService().beginTx() - .use { StreamSupport.stream(it.allLabels.spliterator(), false).map { it.name() }.collect(Collectors.toSet()) } - assertEquals(setOf("User"), labels) - } - } - - @Test - fun `should insert data into Neo4j from CDC events with schema strategy, multiple constraints and labels`() { - val myTopic = UUID.randomUUID().toString() - val props = mapOf(Neo4jConnectorConfig.SERVER_URI to db.boltURI().toString(), - Neo4jSinkConnectorConfig.TOPIC_CDC_SCHEMA to myTopic, - Neo4jConnectorConfig.AUTHENTICATION_TYPE to AuthenticationType.NONE.toString(), - SinkTask.TOPICS_CONFIG to myTopic) - - val constraintsCharacter = listOf( - Constraint(label = "Character", type = StreamsConstraintType.UNIQUE, properties = setOf("surname")), - Constraint(label = "Character", type = StreamsConstraintType.UNIQUE, properties = setOf("name")), - Constraint(label = "Character", type = StreamsConstraintType.UNIQUE, properties = setOf("country", "address")), - ) - val constraintsWriter = listOf( - Constraint(label = "Writer", type = StreamsConstraintType.UNIQUE, properties = setOf("lastName")), - Constraint(label = "Writer", type = StreamsConstraintType.UNIQUE, properties = setOf("firstName")), - ) - val relSchema = Schema(properties = mapOf("since" to "Long"), constraints = constraintsCharacter.plus(constraintsWriter)) - val nodeSchemaCharacter = Schema(properties = mapOf("name" to "String", "surname" to "String", "country" to "String", "address" to "String"), constraints = constraintsCharacter) - val nodeSchemaWriter = Schema(properties = mapOf("firstName" to "String", "lastName" to "String"), constraints = constraintsWriter) - val cdcDataStart = StreamsTransactionEvent( - meta = Meta(timestamp = System.currentTimeMillis(), - username = "user", - txId = 1, - txEventId = 0, - txEventsCount = 3, - operation = OperationType.created - ), - payload = NodePayload(id = "0", - before = null, - after = NodeChange(properties = mapOf("name" to "Naruto", "surname" to "Uzumaki", "country" to "Japan", "address" to "Land of Leaf"), labels = listOf("Character")) - ), - schema = nodeSchemaCharacter - ) - val cdcDataEnd = StreamsTransactionEvent( - meta = Meta(timestamp = System.currentTimeMillis(), - username = "user", - txId = 1, - txEventId = 1, - txEventsCount = 3, - operation = OperationType.created - ), - payload = NodePayload(id = "1", - before = null, - after = NodeChange(properties = mapOf("firstName" to "Minato", "lastName" to "Namikaze"), labels = listOf("Writer")) - ), - schema = nodeSchemaWriter - ) - val cdcDataRelationship = StreamsTransactionEvent( - meta = Meta(timestamp = System.currentTimeMillis(), - username = "user", - txId = 1, - txEventId = 2, - txEventsCount = 3, - operation = OperationType.created - ), - payload = RelationshipPayload( - id = "2", - // leverage on first ids alphabetically, that is name, so we take the 2 previously created nodes - start = RelationshipNodeChange(id = "99", labels = listOf("Character"), ids = mapOf("name" to "Naruto", "surname" to "Osvaldo", "address" to "Land of Sand")), - end = RelationshipNodeChange(id = "88", labels = listOf("Writer"), ids = mapOf("firstName" to "Minato", "lastName" to "Franco", "address" to "Land of Fire")), - after = RelationshipChange(properties = mapOf("since" to 2014)), - before = null, - label = "KNOWS WHO" - ), - schema = relSchema - ) - - task.start(props) - val input = listOf(SinkRecord(myTopic, 1, null, null, null, cdcDataStart, 42), - SinkRecord(myTopic, 1, null, null, null, cdcDataEnd, 43), - SinkRecord(myTopic, 1, null, null, null, cdcDataRelationship, 44)) - - task.put(input) - - db.defaultDatabaseService().beginTx().use { - val query = """ - |MATCH p = (s:Character)-[r:`KNOWS WHO`{since: 2014}]->(e:Writer) - |RETURN count(p) AS count - |""".trimMargin() - it.execute(query) - .columnAs("count").use { - assertTrue { it.hasNext() } - val path = it.next() - assertEquals(1, path) - assertFalse { it.hasNext() } - } - - val countAllNodes = db.defaultDatabaseService().beginTx().use { it.allNodes.stream().count() } - assertEquals(2L, countAllNodes) - } - - // another CDC data, not matching the previously created nodes - val cdcDataRelationshipNotMatched = StreamsTransactionEvent( - meta = Meta(timestamp = System.currentTimeMillis(), - username = "user", - txId = 1, - txEventId = 2, - txEventsCount = 3, - operation = OperationType.created - ), - payload = RelationshipPayload( - id = "2", - // leverage on first ids alphabetically, that is name, so create 2 additional nodes - start = RelationshipNodeChange(id = "1", labels = listOf("Character"), ids = mapOf("name" to "Invalid", "surname" to "Uzumaki")), - end = RelationshipNodeChange(id = "2", labels = listOf("Writer"), ids = mapOf("firstName" to "AnotherInvalid", "surname" to "Namikaze")), - after = RelationshipChange(properties = mapOf("since" to 1998)), - before = null, - label = "HAS WRITTEN" - ), - schema = relSchema - ) - - val inputNotMatched = listOf(SinkRecord(myTopic, 1, null, null, null, cdcDataRelationshipNotMatched, 45)) - - task.put(inputNotMatched) - - db.defaultDatabaseService().beginTx().use { - val query = """ - |MATCH p = (s:Character)-[r:`HAS WRITTEN`{since: 1998}]->(e:Writer) - |RETURN count(p) AS count - |""".trimMargin() - it.execute(query) - .columnAs("count").use { - assertTrue { it.hasNext() } - val path = it.next() - assertEquals(1, path) - assertFalse { it.hasNext() } - } - - val countAllNodes = db.defaultDatabaseService().beginTx().use { it.allNodes.stream().count() } - assertEquals(4L, countAllNodes) - } - - } - - @Test - fun `should insert data into Neo4j from CDC events with schema strategy and multiple unique constraints merging previous nodes`() { - val myTopic = UUID.randomUUID().toString() - val props = mapOf(Neo4jConnectorConfig.SERVER_URI to db.boltURI().toString(), - Neo4jSinkConnectorConfig.TOPIC_CDC_SCHEMA to myTopic, - Neo4jConnectorConfig.AUTHENTICATION_TYPE to AuthenticationType.NONE.toString(), - SinkTask.TOPICS_CONFIG to myTopic) - - val constraints = listOf( - Constraint(label = "User", type = StreamsConstraintType.UNIQUE, properties = setOf("name")), - Constraint(label = "User", type = StreamsConstraintType.UNIQUE, properties = setOf("country", "address")), - Constraint(label = "User", type = StreamsConstraintType.UNIQUE, properties = setOf("surname")), - ) - val relSchema = Schema(properties = mapOf("since" to "Long"), constraints = constraints) - val nodeSchema = Schema(properties = mapOf("name" to "String", "surname" to "String", "country" to "String", "address" to "String"), - constraints = constraints) - val cdcDataStart = StreamsTransactionEvent( - meta = Meta(timestamp = System.currentTimeMillis(), - username = "user", - txId = 1, - txEventId = 0, - txEventsCount = 3, - operation = OperationType.created - ), - payload = NodePayload(id = "0", - before = null, - after = NodeChange(properties = mapOf("name" to "Naruto", "surname" to "Uzumaki", "country" to "Japan", "address" to "Land of Leaf"), labels = listOf("User")) - ), - schema = nodeSchema - ) - val cdcDataEnd = StreamsTransactionEvent( - meta = Meta(timestamp = System.currentTimeMillis(), - username = "user", - txId = 1, - txEventId = 1, - txEventsCount = 3, - operation = OperationType.created - ), - payload = NodePayload(id = "1", - before = null, - after = NodeChange(properties = mapOf("name" to "Minato", "surname" to "Namikaze", "country" to "Japan", "address" to "Land of Leaf"), labels = listOf("User")) - ), - schema = nodeSchema - ) - val cdcDataRelationship = StreamsTransactionEvent( - meta = Meta(timestamp = System.currentTimeMillis(), - username = "user", - txId = 1, - txEventId = 2, - txEventsCount = 3, - operation = OperationType.created - ), - payload = RelationshipPayload( - id = "2", - // leverage on first ids alphabetically, that is name, so we take the 2 previously created nodes - start = RelationshipNodeChange(id = "99", labels = listOf("User"), ids = mapOf("name" to "Naruto", "surname" to "Osvaldo", "address" to "Land of Sand")), - end = RelationshipNodeChange(id = "88", labels = listOf("User"), ids = mapOf("name" to "Minato", "surname" to "Franco", "address" to "Land of Fire")), - after = RelationshipChange(properties = mapOf("since" to 2014)), - before = null, - label = "KNOWS WHO" - ), - schema = relSchema - ) - - task.start(props) - val input = listOf(SinkRecord(myTopic, 1, null, null, null, cdcDataStart, 42), - SinkRecord(myTopic, 1, null, null, null, cdcDataEnd, 43), - SinkRecord(myTopic, 1, null, null, null, cdcDataRelationship, 44)) - - task.put(input) - - db.defaultDatabaseService().beginTx().use { - val query = """ - |MATCH p = (s:User)-[r:`KNOWS WHO` {since: 2014}]->(e:User) - |RETURN count(p) as count - |""".trimMargin() - it.execute(query) - .columnAs("count").use { - assertTrue { it.hasNext() } - val path = it.next() - assertEquals(1, path) - assertFalse { it.hasNext() } - } - - val labels = db.defaultDatabaseService().beginTx() - .use { StreamSupport.stream(it.allLabels.spliterator(), false).map { it.name() }.collect(Collectors.toSet()) } - assertEquals(setOf("User"), labels) - - val countUsers = db.defaultDatabaseService().beginTx().use { it.findNodes(Label.label("User")).stream().count() } - assertEquals(2L, countUsers) - } - - - // another CDC data, not matching the previously created nodes - val cdcDataRelationshipNotMatched = StreamsTransactionEvent( - meta = Meta(timestamp = System.currentTimeMillis(), - username = "user", - txId = 1, - txEventId = 2, - txEventsCount = 3, - operation = OperationType.created - ), - payload = RelationshipPayload( - id = "2", - // leverage on first ids alphabetically, that is name, so create 2 additional nodes - start = RelationshipNodeChange(id = "1", labels = listOf("User"), ids = mapOf("name" to "Invalid", "surname" to "Uzumaki")), - end = RelationshipNodeChange(id = "2", labels = listOf("User"), ids = mapOf("name" to "AnotherInvalid", "surname" to "Namikaze")), - after = RelationshipChange(properties = mapOf("since" to 2000)), - before = null, - label = "HAS WRITTEN" - ), - schema = relSchema - ) - - val inputNotMatched = listOf(SinkRecord(myTopic, 1, null, null, null, cdcDataRelationshipNotMatched, 45)) - - task.put(inputNotMatched) - - db.defaultDatabaseService().beginTx().use { - val query = """ - |MATCH p = (s:User)-[r:`HAS WRITTEN`{since: 2000}]->(e:User) - |RETURN count(p) AS count - |""".trimMargin() - it.execute(query) - .columnAs("count").use { - assertTrue { it.hasNext() } - val path = it.next() - assertEquals(1, path) - assertFalse { it.hasNext() } - } - - val labels = db.defaultDatabaseService().beginTx() - .use { StreamSupport.stream(it.allLabels.spliterator(), false).map { it.name() }.collect(Collectors.toSet()) } - assertEquals(setOf("User"), labels) - - val countUsers = db.defaultDatabaseService().beginTx().use { it.allNodes.stream().count() } - assertEquals(4L, countUsers) - } - } - - @Test - fun `should insert data into Neo4j from CDC events with schema strategy and multiple unique constraints`() { - val myTopic = UUID.randomUUID().toString() - val props = mapOf(Neo4jConnectorConfig.SERVER_URI to db.boltURI().toString(), - Neo4jSinkConnectorConfig.TOPIC_CDC_SCHEMA to myTopic, - Neo4jConnectorConfig.AUTHENTICATION_TYPE to AuthenticationType.NONE.toString(), - SinkTask.TOPICS_CONFIG to myTopic) - - val constraints = listOf( - Constraint(label = "User", type = StreamsConstraintType.UNIQUE, properties = setOf("name")), - Constraint(label = "User", type = StreamsConstraintType.UNIQUE, properties = setOf("surname")), - Constraint(label = "User", type = StreamsConstraintType.UNIQUE, properties = setOf("country", "address")), - ) - val relSchema = Schema(properties = mapOf("since" to "Long"), constraints = constraints) - val nodeSchema = Schema(properties = mapOf("name" to "String", "surname" to "String", "country" to "String", "address" to "String"), - constraints = constraints) - val cdcDataStart = StreamsTransactionEvent( - meta = Meta(timestamp = System.currentTimeMillis(), - username = "user", - txId = 1, - txEventId = 0, - txEventsCount = 3, - operation = OperationType.created - ), - payload = NodePayload(id = "0", - before = null, - after = NodeChange(properties = mapOf("name" to "Naruto", "surname" to "Uzumaki", "country" to "Japan", "address" to "Land of Leaf"), labels = listOf("User")) - ), - schema = nodeSchema - ) - val cdcDataEnd = StreamsTransactionEvent( - meta = Meta(timestamp = System.currentTimeMillis(), - username = "user", - txId = 1, - txEventId = 1, - txEventsCount = 3, - operation = OperationType.created - ), - payload = NodePayload(id = "1", - before = null, - after = NodeChange(properties = mapOf("name" to "Minato", "surname" to "Namikaze", "country" to "Japan", "address" to "Land of Leaf"), labels = listOf("User")) - ), - schema = nodeSchema - ) - val cdcDataRelationship = StreamsTransactionEvent( - meta = Meta(timestamp = System.currentTimeMillis(), - username = "user", - txId = 1, - txEventId = 2, - txEventsCount = 3, - operation = OperationType.created - ), - payload = RelationshipPayload( - id = "2", - // leverage on first ids alphabetically, that is name, so create 2 additional nodes - start = RelationshipNodeChange(id = "1", labels = listOf("User"), ids = mapOf("name" to "Invalid", "surname" to "Uzumaki")), - end = RelationshipNodeChange(id = "2", labels = listOf("User"), ids = mapOf("name" to "AnotherInvalid", "surname" to "Namikaze")), - after = RelationshipChange(properties = mapOf("since" to 2014)), - before = null, - label = "KNOWS WHO" - ), - schema = relSchema - ) - - val task = Neo4jSinkTask() - task.initialize(mock(SinkTaskContext::class.java)) - task.start(props) - val input = listOf( - SinkRecord(myTopic, 1, null, null, null, cdcDataStart, 42), - SinkRecord(myTopic, 1, null, null, null, cdcDataEnd, 43), - SinkRecord(myTopic, 1, null, null, null, cdcDataRelationship, 44), - ) - task.put(input) - - db.defaultDatabaseService().beginTx().use { - val query = """ - |MATCH p = (s:User)-[r:`KNOWS WHO` {since: 2014}]->(e:User) - |RETURN count(p) as count - |""".trimMargin() - it.execute(query) - .columnAs("count").use { - assertTrue { it.hasNext() } - val path = it.next() - assertEquals(1, path) - assertFalse { it.hasNext() } - } - - val labels = db.defaultDatabaseService().beginTx() - .use { StreamSupport.stream(it.allLabels.spliterator(), false).map { it.name() }.collect(Collectors.toSet()) } - assertEquals(setOf("User"), labels) - - val countUsers = db.defaultDatabaseService().beginTx().use { it.findNodes(Label.label("User")).stream().count() } - assertEquals(4L, countUsers) - } - } - - @Test - fun `should delete data into Neo4j from CDC events`() { - db.defaultDatabaseService().beginTx().use { - it.execute(""" - CREATE (s:User:OldLabel:SourceEvent{name:'Andrea', `comp@ny`:'LARUS-BA', sourceId:'0'}) - -[r:`KNOWS WHO`{since:2014, sourceId:'2'}]-> - (e:`User Ext`:SourceEvent{name:'Michael', `comp@ny`:'Neo4j', sourceId:'1'}) - """.trimIndent()).close() - it.commit() - } - val firstTopic = "neotopic" - val props = mapOf(Neo4jConnectorConfig.SERVER_URI to db.boltURI().toString(), - Neo4jSinkConnectorConfig.TOPIC_CDC_SOURCE_ID to firstTopic, - Neo4jConnectorConfig.AUTHENTICATION_TYPE to AuthenticationType.NONE.toString(), - SinkTask.TOPICS_CONFIG to firstTopic) - - val cdcDataStart = StreamsTransactionEvent(meta = Meta(timestamp = System.currentTimeMillis(), - username = "user", - txId = 1, - txEventId = 0, - txEventsCount = 3, - operation = OperationType.deleted - ), - payload = NodePayload(id = "0", - before = NodeChange(properties = mapOf("name" to "Andrea", "comp@ny" to "LARUS-BA"), - labels = listOf("User", "OldLabel")), - after = null - ), - schema = Schema() - ) - task.start(props) - val input = listOf(SinkRecord(firstTopic, 1, null, null, null, cdcDataStart, 42)) - task.put(input) - db.defaultDatabaseService().beginTx().use { - it.execute(""" - MATCH (s:SourceEvent) - RETURN count(s) as count - """.trimIndent()) - .columnAs("count").use { - assertTrue { it.hasNext() } - val count = it.next() - assertEquals(1, count) - assertFalse { it.hasNext() } - } - } - } - - @Test - fun `should not insert data into Neo4j`() { - val topic = "neotopic" - val props = mutableMapOf() - props[Neo4jConnectorConfig.SERVER_URI] = db.boltURI().toString() - props["${Neo4jSinkConnectorConfig.TOPIC_CYPHER_PREFIX}$topic"] = "CREATE (n:Person {name: event.firstName, surname: event.lastName})" - props[Neo4jConnectorConfig.AUTHENTICATION_TYPE] = AuthenticationType.NONE.toString() - props[SinkTask.TOPICS_CONFIG] = topic - - val struct= Struct(PLACE_SCHEMA) - .put("name", "San Mateo (CA)") - .put("latitude", 37.5629917.toFloat()) - .put("longitude", -122.3255254.toFloat()) - - task.start(props) - task.put(listOf(SinkRecord(topic, 1, null, null, PERSON_SCHEMA, struct, 42))) - db.defaultDatabaseService().beginTx().use { - val node: Node? = it.findNode(Label.label("Person"), "name", "Alex") - assertTrue { node == null } - } - } - - @Test - fun `should report but not fail parsing data`() { - val topic = "neotopic" - val props = mutableMapOf() - props[Neo4jConnectorConfig.SERVER_URI] = db.boltURI().toString() - props["${Neo4jSinkConnectorConfig.TOPIC_CYPHER_PREFIX}$topic"] = "CREATE (n:Person {name: event.firstName, surname: event.lastName})" - props[Neo4jConnectorConfig.AUTHENTICATION_TYPE] = AuthenticationType.NONE.toString() - props[SinkTask.TOPICS_CONFIG] = topic - props[ErrorService.ErrorConfig.TOLERANCE] = "all" - props[ErrorService.ErrorConfig.LOG] = true.toString() - - task.start(props) - task.put(listOf(SinkRecord(topic, 1, null, null, null, "a", 42))) - db.defaultDatabaseService().beginTx().use { - val node: Node? = it.findNode(Label.label("Person"), "name", "Alex") - assertTrue { node == null } - } - } - - @Test - fun `should report but not fail invalid schema`() { - val topic = "neotopic" - val props = mutableMapOf() - props[Neo4jConnectorConfig.SERVER_URI] = db.boltURI().toString() - props["${Neo4jSinkConnectorConfig.TOPIC_CYPHER_PREFIX}$topic"] = "CREATE (n:Person {name: event.firstName, surname: event.lastName})" - props[Neo4jConnectorConfig.AUTHENTICATION_TYPE] = AuthenticationType.NONE.toString() - props[ErrorService.ErrorConfig.TOLERANCE] = "all" - props[ErrorService.ErrorConfig.LOG] = true.toString() - props[SinkTask.TOPICS_CONFIG] = topic - - task.start(props) - task.put(listOf(SinkRecord(topic, 1, null, 42, null, "true", 42))) - db.defaultDatabaseService().beginTx().use { - val node: Node? = it.findNode(Label.label("Person"), "name", "Alex") - assertTrue { node == null } - } - } - - @Test - fun `should fail running invalid cypher`() { - val topic = "neotopic" - val props = mutableMapOf() - props[Neo4jConnectorConfig.SERVER_URI] = db.boltURI().toString() - props["${Neo4jSinkConnectorConfig.TOPIC_CYPHER_PREFIX}$topic"] = " No Valid Cypher " - props[Neo4jConnectorConfig.AUTHENTICATION_TYPE] = AuthenticationType.NONE.toString() - props[SinkTask.TOPICS_CONFIG] = topic - props[ErrorService.ErrorConfig.TOLERANCE] = "all" - props[ErrorService.ErrorConfig.LOG] = true.toString() - - task.start(props) - task.put(listOf(SinkRecord(topic, 1, null, 42, null, "{\"foo\":42}", 42))) - db.defaultDatabaseService().beginTx().use { - val node: Node? = it.findNode(Label.label("Person"), "name", "Alex") - assertTrue { node == null } - } - } - - @Test - fun `should work with node pattern topic`() { - val topic = "neotopic" - val props = mutableMapOf() - props[Neo4jConnectorConfig.SERVER_URI] = db.boltURI().toString() - props["${Neo4jSinkConnectorConfig.TOPIC_PATTERN_NODE_PREFIX}$topic"] = "User{!userId,name,surname,address.city}" - props[Neo4jConnectorConfig.AUTHENTICATION_TYPE] = AuthenticationType.NONE.toString() - props[SinkTask.TOPICS_CONFIG] = topic - - val data = mapOf("userId" to 1, "name" to "Andrea", "surname" to "Santurbano", - "address" to mapOf("city" to "Venice", "CAP" to "30100")) - - task.start(props) - val input = listOf(SinkRecord(topic, 1, null, null, null, data, 42)) - task.put(input) - db.defaultDatabaseService().beginTx().use { - it.execute("MATCH (n:User{name: 'Andrea', surname: 'Santurbano', userId: 1, `address.city`: 'Venice'}) RETURN count(n) AS count") - .columnAs("count").use { - assertTrue { it.hasNext() } - val count = it.next() - assertEquals(1, count) - assertFalse { it.hasNext() } - } - } - } - - @Test - fun `should work with relationship pattern topic`() { - val topic = "neotopic" - val props = mutableMapOf() - props[Neo4jConnectorConfig.SERVER_URI] = db.boltURI().toString() - props["${Neo4jSinkConnectorConfig.TOPIC_PATTERN_RELATIONSHIP_PREFIX}$topic"] = "(:User{!sourceId,sourceName,sourceSurname})-[:KNOWS]->(:User{!targetId,targetName,targetSurname})" - props[Neo4jConnectorConfig.AUTHENTICATION_TYPE] = AuthenticationType.NONE.toString() - props[SinkTask.TOPICS_CONFIG] = topic - - val data = mapOf("sourceId" to 1, "sourceName" to "Andrea", "sourceSurname" to "Santurbano", - "targetId" to 1, "targetName" to "Michael", "targetSurname" to "Hunger", "since" to 2014) - - task.start(props) - val input = listOf(SinkRecord(topic, 1, null, null, null, data, 42)) - task.put(input) - db.defaultDatabaseService().beginTx().use { - it.execute(""" - MATCH p = (s:User{sourceName: 'Andrea', sourceSurname: 'Santurbano', sourceId: 1})-[:KNOWS{since: 2014}]->(e:User{targetName: 'Michael', targetSurname: 'Hunger', targetId: 1}) - RETURN count(p) AS count - """.trimIndent()) - .columnAs("count").use { - assertTrue { it.hasNext() } - val count = it.next() - assertEquals(1, count) - assertFalse { it.hasNext() } - } - } - } - - @Test - fun `should work with node pattern topic for tombstone record`() { - db.defaultDatabaseService().beginTx().use { - it.execute("CREATE (u:User{userId: 1, name: 'Andrea', surname: 'Santurbano'})").close() - it.commit() - } - val count = db.defaultDatabaseService().beginTx().use { - it.execute("MATCH (n) RETURN count(n) AS count") - .columnAs("count") - .next() - } - assertEquals(1L, count) - val topic = "neotopic" - val props = mutableMapOf() - props[Neo4jConnectorConfig.SERVER_URI] = db.boltURI().toString() - props["${Neo4jSinkConnectorConfig.TOPIC_PATTERN_NODE_PREFIX}$topic"] = "User{!userId,name,surname,address.city}" - props[Neo4jConnectorConfig.AUTHENTICATION_TYPE] = AuthenticationType.NONE.toString() - props[SinkTask.TOPICS_CONFIG] = topic - - val data = mapOf("userId" to 1) - - task.start(props) - val input = listOf(SinkRecord(topic, 1, null, data, null, null, 42)) - task.put(input) - db.defaultDatabaseService().beginTx().use { - it.execute("MATCH (n) RETURN count(n) AS count") - .columnAs("count").use { - assertTrue { it.hasNext() } - val count = it.next() - assertEquals(0, count) - assertFalse { it.hasNext() } - } - } - } - - @Test - fun `should create and delete relationship from CUD event without properties field`() { - val relType = "MY_REL" - val key = "key" - val startNode = "SourceNode" - val endNode = "TargetNode" - val topic = UUID.randomUUID().toString() - - db.defaultDatabaseService().beginTx().use { - it.execute("CREATE (:$startNode {key: 1}) CREATE (:$endNode {key: 1})").close() - it.commit() - } - - val start = CUDNodeRel(ids = mapOf(key to 1), labels = listOf(startNode)) - val end = CUDNodeRel(ids = mapOf(key to 1), labels = listOf(endNode)) - val relMerge = CUDRelationship(op = CUDOperations.merge, from = start, to = end, rel_type = relType) - val sinkRecordMerge = SinkRecord(topic, 1, null, null, null, JSONUtils.asMap(relMerge), 0L) - - val props = mutableMapOf() - props[Neo4jConnectorConfig.SERVER_URI] = db.boltURI().toString() - props[Neo4jSinkConnectorConfig.TOPIC_CUD] = topic - props[Neo4jConnectorConfig.AUTHENTICATION_TYPE] = AuthenticationType.NONE.toString() - props[SinkTask.TOPICS_CONFIG] = topic - - task.start(props) - task.put(listOf(sinkRecordMerge)) - - val queryCount = "MATCH p = (:$startNode)-[:$relType]->(:$endNode) RETURN count(p) AS count" - - db.defaultDatabaseService().beginTx().use { - val countRels = it.execute(queryCount) - .columnAs("count") - .next() - assertEquals(1L, countRels) - } - - val relDelete = CUDRelationship(op = CUDOperations.delete, from = start, to = end, rel_type = relType) - val sinkRecordDelete = SinkRecord(topic, 1, null, null, null, JSONUtils.asMap(relDelete), 1L) - task.put(listOf(sinkRecordDelete)) - - db.defaultDatabaseService().beginTx().use { - val countRels = it.execute(queryCount) - .columnAs("count") - .next() - assertEquals(0L, countRels) - } - } - - @Test - fun `should ingest node data from CUD Events`() { - // given - val mergeMarkers = listOf(2, 5, 7) - val key = "key" - val topic = UUID.randomUUID().toString() - val data = (1..10).map { - val labels = if (it % 2 == 0) listOf("Foo", "Bar") else listOf("Foo", "Bar", "Label") - val properties = mapOf("foo" to "foo-value-$it", "id" to it) - val (op, ids) = when (it) { - in mergeMarkers -> CUDOperations.merge to mapOf(key to it) - else -> CUDOperations.create to emptyMap() - } - val cudNode = CUDNode(op = op, - labels = labels, - ids = ids, - properties = properties) - SinkRecord(topic, 1, null, null, null, JSONUtils.asMap(cudNode), it.toLong()) - } - val props = mutableMapOf() - props[Neo4jConnectorConfig.SERVER_URI] = db.boltURI().toString() - props[Neo4jSinkConnectorConfig.TOPIC_CUD] = topic - props[Neo4jConnectorConfig.AUTHENTICATION_TYPE] = AuthenticationType.NONE.toString() - props[SinkTask.TOPICS_CONFIG] = topic - - // when - task.start(props) - task.put(data) - - // then - db.defaultDatabaseService().beginTx().use { - val countFooBarLabel = it.execute("MATCH (n:Foo:Bar:Label) RETURN count(n) AS count") - .columnAs("count") - .next() - assertEquals(5L, countFooBarLabel) - val countFooBar = it.execute("MATCH (n:Foo:Bar) RETURN count(n) AS count") - .columnAs("count") - .next() - assertEquals(10L, countFooBar) - } - } - - @Test - fun `should ingest relationship data from CUD Events`() { - // given - val key = "key" - val topic = UUID.randomUUID().toString() - val rel_type = "MY_REL" - val data = (1..10).map { - val properties = mapOf("foo" to "foo-value-$it", "id" to it) - val start = CUDNodeRel(ids = mapOf(key to it), labels = listOf("Foo", "Bar")) - val end = CUDNodeRel(ids = mapOf(key to it), labels = listOf("FooBar")) - val rel = CUDRelationship(op = CUDOperations.create, properties = properties, from = start, to = end, rel_type = rel_type) - SinkRecord(topic, 1, null, null, null, JSONUtils.asMap(rel), it.toLong()) - } - val props = mutableMapOf() - props[Neo4jConnectorConfig.SERVER_URI] = db.boltURI().toString() - props[Neo4jSinkConnectorConfig.TOPIC_CUD] = topic - props[Neo4jConnectorConfig.AUTHENTICATION_TYPE] = AuthenticationType.NONE.toString() - props[SinkTask.TOPICS_CONFIG] = topic - db.defaultDatabaseService().beginTx().use { - it.execute(""" - UNWIND range(1, 10) AS id - CREATE (:Foo:Bar {key: id}) - CREATE (:FooBar {key: id}) - """.trimIndent()).close() - assertEquals(0, it.allRelationships.count()) - assertEquals(20, it.allNodes.count()) - it.commit() - } - - // when - task.start(props) - task.put(data) - - // then - db.defaultDatabaseService().beginTx().use { - val countFooBarLabel = it.execute(""" - MATCH (:Foo:Bar)-[r:$rel_type]->(:FooBar) - RETURN count(r) AS count - """.trimIndent()) - .columnAs("count").next() - assertEquals(10L, countFooBarLabel) - } - } - - @Test - fun `should create nodes and relationship, if one or both nodes doesn't exist from CUD Events`() { - // given - val key = "key" - val topic = UUID.randomUUID().toString() - val relType = "MY_REL" - val data = (1..10).map { - val properties = mapOf("foo" to "foo-value-$it", "id" to it) - val start = CUDNodeRel(ids = mapOf(key to it), labels = listOf("Foo", "Bar"), op = CUDOperations.merge) - val end = CUDNodeRel(ids = mapOf(key to it), labels = listOf("FooBar"), op = CUDOperations.merge) - val rel = CUDRelationship(op = CUDOperations.merge, properties = properties, from = start, to = end, rel_type = relType) - SinkRecord(topic, 1, null, null, null, JSONUtils.asMap(rel), it.toLong()) - } - val props = mutableMapOf() - props[Neo4jConnectorConfig.SERVER_URI] = db.boltURI().toString() - props[Neo4jSinkConnectorConfig.TOPIC_CUD] = topic - props[Neo4jConnectorConfig.AUTHENTICATION_TYPE] = AuthenticationType.NONE.toString() - props[SinkTask.TOPICS_CONFIG] = topic - - // when - task.start(props) - task.put(data) - - // then - db.defaultDatabaseService().beginTx().use { - val countFooBarLabel = it.execute(""" - MATCH (:Foo:Bar)-[r:$relType]->(:FooBar) - RETURN count(r) AS count - """.trimIndent()) - .columnAs("count").next() - assertEquals(10L, countFooBarLabel) - } - - // now, I create only start nodes - val dataWithStartPreset = (11..20).map { - val properties = mapOf("foo" to "foo-value-$it", "id" to it) - val start = CUDNodeRel(ids = mapOf(key to it), labels = listOf("Foo", "Bar")) - val end = CUDNodeRel(ids = mapOf(key to it), labels = listOf("FooBar"), op = CUDOperations.merge) - val rel = CUDRelationship(op = CUDOperations.merge, properties = properties, from = start, to = end, rel_type = relType) - SinkRecord(topic, 1, null, null, null, JSONUtils.asMap(rel), it.toLong()) - } - - db.defaultDatabaseService().beginTx().use { - it.execute(""" - UNWIND range(11, 20) AS id - CREATE (:Foo:Bar {key: id}) - """.trimIndent()).close() - assertEquals(10, it.allRelationships.count()) - assertEquals(30, it.allNodes.count()) - it.commit() - } - - task.put(dataWithStartPreset) - - db.defaultDatabaseService().beginTx().use { - val countFooBarLabel = it.execute(""" - MATCH (:Foo:Bar)-[r:$relType]->(:FooBar) - RETURN count(r) AS count - """.trimIndent()) - .columnAs("count").next() - assertEquals(20L, countFooBarLabel) - } - - // now, I create only end nodes - val dataWithEndPreset = (21..30).map { - val properties = mapOf("foo" to "foo-value-$it", "id" to it) - val start = CUDNodeRel(ids = mapOf(key to it), labels = listOf("Foo", "Bar"), op = CUDOperations.merge) - val end = CUDNodeRel(ids = mapOf(key to it), labels = listOf("FooBar")) - val rel = CUDRelationship(op = CUDOperations.merge, properties = properties, from = start, to = end, rel_type = relType) - SinkRecord(topic, 1, null, null, null, JSONUtils.asMap(rel), it.toLong()) - } - - db.defaultDatabaseService().beginTx().use { - it.execute(""" - UNWIND range(21, 30) AS id - CREATE (:FooBar {key: id}) - """.trimIndent()).close() - assertEquals(20, it.allRelationships.count()) - assertEquals(50, it.allNodes.count()) - it.commit() - } - - task.put(dataWithEndPreset) - - db.defaultDatabaseService().beginTx().use { - val countFooBarLabel = it.execute(""" - MATCH (:Foo:Bar)-[r:$relType]->(:FooBar) - RETURN count(r) AS count - """.trimIndent()) - .columnAs("count").next() - assertEquals(30L, countFooBarLabel) - } - - } - - @Test - fun `should create entities only with valid CUD operations`() { - // given - val invalidMarkers = listOf(3, 6, 9) - val key = "key" - val topic = UUID.randomUUID().toString() - val data = (1..10).map { - val labels = listOf("Foo", "Bar", "Label") - val properties = mapOf("foo" to "foo-value-$it", "id" to it) - val (op, ids) = when (it) { - in invalidMarkers -> CUDOperations.match to mapOf(key to it) - else -> CUDOperations.create to emptyMap() - } - val cudNode = CUDNode(op = op, - labels = labels, - ids = ids, - properties = properties) - SinkRecord(topic, 1, null, null, null, JSONUtils.asMap(cudNode), it.toLong()) - } - - val relType = "MY_REL" - val invalidRelMarkers = listOf(1, 4) - val invalidNodeRelMarkers = listOf(3, 6, 7) - val dataRel = (1..10).map { - val properties = mapOf("foo" to "foo-value-$it", "id" to it) - val opRelationship = if (it in invalidRelMarkers) CUDOperations.delete else CUDOperations.merge - val opStartNode = if (it in invalidNodeRelMarkers) CUDOperations.delete else CUDOperations.merge - val start = CUDNodeRel(ids = mapOf(key to it), labels = listOf("Foo", "Bar"), op = opStartNode) - val end = CUDNodeRel(ids = mapOf(key to it), labels = listOf("FooBar"), op = CUDOperations.merge) - val rel = CUDRelationship(op = opRelationship, properties = properties, from = start, to = end, rel_type = relType) - SinkRecord(topic, 1, null, null, null, JSONUtils.asMap(rel), it.toLong()) - } - - val props = mutableMapOf() - props[Neo4jConnectorConfig.SERVER_URI] = db.boltURI().toString() - props[Neo4jSinkConnectorConfig.TOPIC_CUD] = topic - props[Neo4jConnectorConfig.AUTHENTICATION_TYPE] = AuthenticationType.NONE.toString() - props[SinkTask.TOPICS_CONFIG] = topic - - // when - task.start(props) - task.put(data) - task.put(dataRel) - - // then - db.defaultDatabaseService().beginTx().use { - val countFooBarLabel = it.execute("MATCH (n:Foo:Bar:Label) RETURN count(n) AS count") - .columnAs("count") - .next() - assertEquals(7L, countFooBarLabel) - val countRelationships = it.execute(""" - MATCH (:Foo:Bar)-[r:$relType]->(:FooBar) - RETURN count(r) AS count - """.trimIndent()) - .columnAs("count").next() - assertEquals(5L, countRelationships) - } - } - - @Test - fun `should fail data insertion with ProcessingError`() { - // given - val topic = UUID.randomUUID().toString() - - val props = mutableMapOf() - props[Neo4jConnectorConfig.SERVER_URI] = db.boltURI().toString() - props["${Neo4jSinkConnectorConfig.TOPIC_PATTERN_RELATIONSHIP_PREFIX}$topic"] = "(:User{!sourceId,sourceName,sourceSurname})-[:KNOWS]->(:User{!targetId,targetName,targetSurname})" - props[Neo4jConnectorConfig.AUTHENTICATION_TYPE] = AuthenticationType.NONE.toString() - props[SinkTask.TOPICS_CONFIG] = topic - props[Neo4jConnectorConfig.DATABASE] = "notExistent" - - val data = mapOf("sourceId" to 1, "sourceName" to "Andrea", "sourceSurname" to "Santurbano", - "targetId" to 1, "targetName" to "Michael", "targetSurname" to "Hunger", "since" to 2014) - - task.start(props) - val input = listOf(SinkRecord(topic, 1, null, null, null, data, 42)) - - try { - task.put(input) - fail("It should fail with ProcessingError") - } catch (e: ProcessingError) { - val errorData = e.errorDatas.first() - assertTrue(errorData.databaseName == "notExistent" - && errorData.exception!!.javaClass.name == "org.neo4j.driver.exceptions.FatalDiscoveryException") - } - - props[Neo4jConnectorConfig.DATABASE] = "neo4j" - val taskNotValid = Neo4jSinkTask() - taskNotValid.initialize(mock(SinkTaskContext::class.java)) - taskNotValid.start(props) - - val dataNotValid = mapOf("sourceId" to null, "sourceName" to "Andrea", "sourceSurname" to "Santurbano", - "targetId" to 1, "targetName" to "Michael", "targetSurname" to "Hunger", "since" to 2014) - val inputNotValid = listOf(SinkRecord(topic, 1, null, null, null, dataNotValid, 43)) - - try { - taskNotValid.put(inputNotValid) - fail("It should fail with ProcessingError") - } catch (e: ProcessingError) { - val errorData = e.errorDatas.first() - assertTrue(errorData.databaseName == "neo4j" - && errorData.exception!!.javaClass.name == "org.neo4j.driver.exceptions.ClientException") - } - } - - @Test - @Ignore("flaky") - fun `should stop the query and fails with small timeout and vice versa`() { - val myTopic = "foo" - val props = mutableMapOf() - props[Neo4jConnectorConfig.SERVER_URI] = db.boltURI().toString() - props["${Neo4jSinkConnectorConfig.TOPIC_CYPHER_PREFIX}$myTopic"] = "CREATE (n:Person {name: event.name})" - props[Neo4jConnectorConfig.AUTHENTICATION_TYPE] = AuthenticationType.NONE.toString() - props[Neo4jSinkConnectorConfig.BATCH_PARALLELIZE] = true.toString() - val batchSize = 500000 - props[Neo4jConnectorConfig.BATCH_SIZE] = batchSize.toString() - props[Neo4jConnectorConfig.BATCH_TIMEOUT_MSECS] = 1.toString() - props[SinkTask.TOPICS_CONFIG] = myTopic - val input = (1..batchSize).map { - SinkRecord(myTopic, 1, null, null, null, mapOf("name" to it.toString()), it.toLong()) - } - // test timeout with parallel=true - assertFailsWithTimeout(props, input, batchSize) - countFooPersonEntities(0) - - // test timeout with parallel=false - props[Neo4jSinkConnectorConfig.BATCH_PARALLELIZE] = false.toString() - assertFailsWithTimeout(props, input, batchSize) - countFooPersonEntities(0) - - // test with large timeout - props[Neo4jConnectorConfig.BATCH_TIMEOUT_MSECS] = 30000.toString() - val taskValidParallelFalse = Neo4jSinkTask() - taskValidParallelFalse.initialize(mock(SinkTaskContext::class.java)) - taskValidParallelFalse.start(props) - taskValidParallelFalse.put(input) - countFooPersonEntities(batchSize) - - props[Neo4jSinkConnectorConfig.BATCH_PARALLELIZE] = true.toString() - val taskValidParallelTrue = Neo4jSinkTask() - taskValidParallelTrue.initialize(mock(SinkTaskContext::class.java)) - taskValidParallelTrue.start(props) - taskValidParallelTrue.put(input) - countFooPersonEntities(batchSize * 2) - } - - private fun assertFailsWithTimeout(props: MutableMap, input: List, expectedDataErrorSize: Int) { - try { - val taskInvalid = Neo4jSinkTask() - taskInvalid.initialize(mock(SinkTaskContext::class.java)) - taskInvalid.start(props) - taskInvalid.put(input) - fail("Should fail because of TimeoutException") - } catch (e: ProcessingError) { - val errors = e.errorDatas - assertEquals(expectedDataErrorSize, errors.size) - } - } - - private fun countFooPersonEntities(expected: Int) { - db.defaultDatabaseService().beginTx().use { - val personCount = it.execute("MATCH (p:Person) RETURN count(p) as count").columnAs("count").next() - assertEquals(expected, personCount.toInt()) - } - } - - -} \ No newline at end of file diff --git a/kafka-connect-neo4j/src/test/kotlin/streams/kafka/connect/sink/Neo4jValueConverterNestedStructTest.kt b/kafka-connect-neo4j/src/test/kotlin/streams/kafka/connect/sink/Neo4jValueConverterNestedStructTest.kt deleted file mode 100644 index 5c178c5a..00000000 --- a/kafka-connect-neo4j/src/test/kotlin/streams/kafka/connect/sink/Neo4jValueConverterNestedStructTest.kt +++ /dev/null @@ -1,167 +0,0 @@ -package streams.kafka.connect.sink - -import org.apache.kafka.connect.data.SchemaBuilder -import org.apache.kafka.connect.data.Struct -import org.junit.Test -import org.neo4j.driver.Value -import org.neo4j.driver.Values -import streams.kafka.connect.sink.converters.Neo4jValueConverter -import streams.utils.JSONUtils -import java.time.Instant -import java.time.ZonedDateTime -import java.util.* -import kotlin.test.assertEquals - -class Neo4jValueConverterNestedStructTest { - - @Test - fun `should convert nested map into map of neo4j values`() { - // given - val body = JSONUtils.readValue>(data).mapValues(::convertDate) - - // when - val result = Neo4jValueConverter().convert(body) as Map<*, *> - - // then - val expected = getExpectedMap() - assertEquals(expected, result) - } - - @Test - fun `should convert nested struct into map of neo4j values`() { - - val body = getTreeStruct() - - // when - val result = Neo4jValueConverter().convert(body) as Map<*, *> - - // then - val expected = getExpectedMap() - assertEquals(expected, result) - } - - companion object { - - private val PREF_SCHEMA = SchemaBuilder.struct().name("org.neo4j.example.email.Preference") - .field("preferenceType", SchemaBuilder.string()) - .field("endEffectiveDate", org.apache.kafka.connect.data.Timestamp.SCHEMA) - .build() - - private val EMAIL_SCHEMA = SchemaBuilder.struct().name("org.neo4j.example.email.Email") - .field("email", SchemaBuilder.string()) - .field("preferences", SchemaBuilder.array(PREF_SCHEMA)) - .build() - - private val TN_SCHEMA = SchemaBuilder.struct().name("org.neo4j.example.email.Transaction") - .field("tn", SchemaBuilder.string()) - .field("preferences", SchemaBuilder.array(PREF_SCHEMA)) - .build() - - private val EVENT_SCHEMA = SchemaBuilder.struct().name("org.neo4j.example.email.Event") - .field("eventId", SchemaBuilder.string()) - .field("eventTimestamp", org.apache.kafka.connect.data.Timestamp.SCHEMA) - .field("emails", SchemaBuilder.array(EMAIL_SCHEMA).optional()) - .field("tns", SchemaBuilder.array(TN_SCHEMA).optional()) - .build() - - fun getTreeStruct(): Struct? { - val source = JSONUtils.readValue>(data).mapValues(::convertDate) - - val emails = source["emails"] as List> - val email = Struct(EMAIL_SCHEMA) - .put("email",emails[0]["email"]) - .put("preferences", - (emails[0]["preferences"] as List>).map { Struct(PREF_SCHEMA).put("preferenceType", it["preferenceType"]).put("endEffectiveDate",it["endEffectiveDate"]) }) - - val emailList = listOf(email) - val tnsList = - (source["tns"] as List>).map { - Struct(TN_SCHEMA).put("tn",it["tn"]) - .put("preferences", (it["preferences"] as List>).map{ Struct(PREF_SCHEMA).put("preferenceType", it["preferenceType"]).put("endEffectiveDate",it["endEffectiveDate"]) }) } - - return Struct(EVENT_SCHEMA) - .put("eventId", source["eventId"]) - .put("eventTimestamp", source["eventTimestamp"]) - .put("emails", emailList) - .put("tns", tnsList) - } - - fun getExpectedMap(): Map { - return JSONUtils.readValue>(data).mapValues(::convertDateNew).mapValues { Values.value(it.value) } - } - - fun convertDate(it: Map.Entry) : Any? = - when { - it.value is Map<*,*> -> (it.value as Map).mapValues(::convertDate) - it.value is Collection<*> -> (it.value as Collection).map{ x-> convertDate(AbstractMap.SimpleEntry(it.key, x)) } - it.key.endsWith("Date") -> Date.from(Instant.parse(it.value.toString())) - it.key.endsWith("Timestamp") -> Date.from(Instant.parse(it.value.toString())) - else -> it.value - } - fun convertDateNew(it: Map.Entry) : Any? = - when { - it.value is Map<*,*> -> (it.value as Map).mapValues(::convertDateNew) - it.value is Collection<*> -> (it.value as Collection).map{ x-> convertDateNew(AbstractMap.SimpleEntry(it.key, x)) } - it.key.endsWith("Date") -> ZonedDateTime.parse(it.value.toString()).toLocalDateTime() - it.key.endsWith("Timestamp") -> ZonedDateTime.parse(it.value.toString()).toLocalDateTime() - else -> it.value - } - - val data : String = """ -{ - "eventId": "d70f306a-71d2-48d9-aea3-87b3808b764b", - "eventTimestamp": "2019-08-21T22:29:22.151Z", - "emails": [ - { - "email": "century@gmail.com", - "preferences": [ - { - "preferenceType": "repair_subscription", - "endEffectiveDate": "2019-05-08T14:51:26.116Z" - }, - { - "preferenceType": "ordering_subscription", - "endEffectiveDate": "2019-05-08T14:51:26.116Z" - }, - { - "preferenceType": "marketing_subscription", - "endEffectiveDate": "2019-05-08T14:51:26.116Z" - } - ] - } - ], - "tns": [ - { - "tn": "1122334455", - "preferences": [ - { - "preferenceType": "billing_subscription", - "endEffectiveDate": "2019-10-22T14:51:26.116Z" - }, - { - "preferenceType": "repair_subscription", - "endEffectiveDate": "2019-10-22T14:51:26.116Z" - }, - { - "preferenceType": "sms", - "endEffectiveDate": "2019-10-22T14:51:26.116Z" - } - ] - }, - { - "tn": "5544332211", - "preferences": [ - { - "preferenceType": "acct_lookup", - "endEffectiveDate": "2019-10-22T14:51:26.116Z" - } - ] - } - ] -} - """.trimIndent() - - } - -} - diff --git a/kafka-connect-neo4j/src/test/kotlin/streams/kafka/connect/sink/Neo4jValueConverterTest.kt b/kafka-connect-neo4j/src/test/kotlin/streams/kafka/connect/sink/Neo4jValueConverterTest.kt deleted file mode 100644 index 1583e948..00000000 --- a/kafka-connect-neo4j/src/test/kotlin/streams/kafka/connect/sink/Neo4jValueConverterTest.kt +++ /dev/null @@ -1,335 +0,0 @@ -package streams.kafka.connect.sink - -import org.apache.kafka.connect.data.* -import org.junit.Test -import org.neo4j.driver.Value -import org.neo4j.driver.Values -import org.neo4j.driver.internal.value.* -import streams.kafka.connect.sink.converters.Neo4jValueConverter -import java.math.BigDecimal -import java.time.Instant -import java.time.ZoneId -import java.util.Date -import kotlin.test.assertEquals -import kotlin.test.assertTrue - -class Neo4jValueConverterTest { - - @Test - fun `should convert tree struct into map of neo4j values`() { - // given - // this test generates a simple tree structure like this - // body - // / \ - // p ul - // | - // li - val body = getTreeStruct() - - // when - val result = Neo4jValueConverter().convert(body) as Map<*, *> - - // then - val expected = getExpectedMap() - assertEquals(expected, result) - } - - @Test - fun `should convert tree simple map into map of neo4j values`() { - // given - // this test generates a simple tree structure like this - // body - // / \ - // p ul - // | - // li - val body = getTreeMap() - - // when - val result = Neo4jValueConverter().convert(body) as Map<*, *> - - // then - val expected = getExpectedMap() - assertEquals(expected, result) - } - - @Test - fun `should convert tree with mixes types into map of neo4j values`() { - - val utc = ZoneId.of("UTC") - val result = Neo4jValueConverter().convert(Struct(TEST_SCHEMA)) as Map - - val target = result["target"] - assertTrue{ target is FloatValue } - assertEquals(123.4, target?.asDouble()) - - val largeDecimal = result["largeDecimal"] - assertTrue{ largeDecimal is StringValue } - assertEquals(BigDecimal.valueOf(Double.MAX_VALUE).pow(2).toPlainString(), largeDecimal?.asString()) - - val byteArray = result["byteArray"] - assertTrue{ byteArray is BytesValue } - assertEquals("Foo".toByteArray().map { it }, byteArray?.asByteArray()?.map { it }) - - val int64 = result["int64"] - assertTrue{ int64 is IntegerValue } - assertEquals(Long.MAX_VALUE, int64?.asLong()) - - val int64Timestamp = result["int64Timestamp"] - assertTrue{ int64Timestamp is LocalDateTimeValue } - assertEquals(Date.from(Instant.ofEpochMilli(789)).toInstant().atZone(utc).toLocalDateTime(), int64Timestamp?.asLocalDateTime()) - - val int32 = result["int32"] - assertTrue{ int32 is IntegerValue } - assertEquals(123, int32?.asInt()) - - val int32Date = result["int32Date"] - assertTrue{ int32Date is DateValue } - assertEquals(Date.from(Instant.ofEpochMilli(456)).toInstant().atZone(utc).toLocalDate(), int32Date?.asLocalDate()) - - val int32Time = result["int32Time"] - assertTrue{ int32Time is LocalTimeValue } - assertEquals(Date.from(Instant.ofEpochMilli(123)).toInstant().atZone(utc).toLocalTime(), int32Time?.asLocalTime()) - - val nullField = result["nullField"] - assertTrue{ nullField is NullValue } - - val nullFieldBytes = result["nullFieldBytes"] - assertTrue{ nullFieldBytes is NullValue } - - } - - @Test - fun `should convert BigDecimal into String neo4j value if is a positive less than Double MIN_VALUE`() { - - val number = BigDecimal.valueOf(Double.MIN_VALUE).pow(2) - val result = Neo4jValueConverter().convert(getItemElement(number)) - val item = result["item"] - - assertTrue{ item is StringValue } - assertEquals(number.toPlainString(), item?.asString()) - - val result2 = Neo4jValueConverter().convert(getItemElement(null)) - val item2 = result2["item"] - - assertTrue{ item2 is NullValue } - } - - @Test - fun `should convert BigDecimal into String neo4j value if is a negative less than Double MAX_VALUE`() { - - val number = - (BigDecimal.valueOf(Double.MAX_VALUE)).multiply(BigDecimal.valueOf(2)) - val result = Neo4jValueConverter().convert(getItemElement(number)) - val item = result["item"] - - assertTrue{ item is StringValue } - assertEquals(number.toPlainString(), item?.asString()) - } - - @Test - fun `should convert BigDecimal into String neo4j value if is greater than Double MAX_VALUE`() { - - val number = BigDecimal.valueOf(Double.MAX_VALUE).pow(2) - val result = Neo4jValueConverter().convert(getItemElement(number)) - val item = result["item"] - - assertTrue{ item is StringValue } - assertEquals(number.toPlainString(), item?.asString()) - } - - @Test - fun `should convert BigDecimal into Double neo4j value if is less than Double MAX_VALUE and greater than Double MIN_VALUE`() { - - val number = 3456.78 - val result = Neo4jValueConverter().convert(getItemElement(BigDecimal.valueOf(number))) - val item = result["item"] - - assertTrue{ item is FloatValue } - assertEquals(number, item?.asDouble()) - } - - @Test - fun `should convert BigDecimal into Double neo4j value if is equal to Double MAX_VALUE`() { - - val number = Double.MAX_VALUE - val result = Neo4jValueConverter().convert(getItemElement(BigDecimal.valueOf(number))) - val item = result["item"] - - assertTrue{ item is FloatValue } - assertEquals(number, item?.asDouble()) - } - - @Test - fun `should convert properly mixed items`() { - - val double = Double.MAX_VALUE - val long = Long.MAX_VALUE - val bigDouble = BigDecimal.valueOf(Double.MAX_VALUE).pow(2) - val string = "FooBar" - val date = Date() - val result = Neo4jValueConverter().convert(mapOf( - "double" to double, - "long" to long, - "bigDouble" to bigDouble, - "string" to string, - "date" to date)) - - assertEquals(double, result["double"]?.asDouble()) - assertEquals(long, result["long"]?.asLong()) - assertEquals(bigDouble.toPlainString(), result["bigDouble"]?.asString()) - assertEquals(string, result["string"]?.asString()) - assertEquals(date.toInstant().atZone(ZoneId.of("UTC")).toLocalDateTime(), result["date"]?.asLocalDateTime()) - } - - @Test - fun `should be able to process a nested AVRO structure`() { - val trainSchema = SchemaBuilder.struct() - .field("internationalTrainNumber", Schema.STRING_SCHEMA) - .field("trainDate", Schema.STRING_SCHEMA).build() - val mySchema = SchemaBuilder.struct() - .field("trainId", trainSchema) - .field("coreId", Schema.STRING_SCHEMA).build() - - val trainIdStruct = Struct(trainSchema) - .put("internationalTrainNumber", "46261") - .put("trainDate", "2021-05-20") - val rootStruct = Struct(mySchema) - .put("trainId", trainIdStruct) - .put("coreId", "000000046261") - - val result = Neo4jValueConverter().convert(rootStruct) as Map<*, *> - } - - companion object { - private val LI_SCHEMA = SchemaBuilder.struct().name("org.neo4j.example.html.LI") - .field("value", Schema.OPTIONAL_STRING_SCHEMA) - .field("class", SchemaBuilder.array(Schema.STRING_SCHEMA).optional()) - .build() - - private val UL_SCHEMA = SchemaBuilder.struct().name("org.neo4j.example.html.UL") - .field("value", SchemaBuilder.array(LI_SCHEMA)) - .build() - - private val P_SCHEMA = SchemaBuilder.struct().name("org.neo4j.example.html.P") - .field("value", Schema.OPTIONAL_STRING_SCHEMA) - .build() - - private val BODY_SCHEMA = SchemaBuilder.struct().name("org.neo4j.example.html.BODY") - .field("ul", SchemaBuilder.array(UL_SCHEMA).optional()) - .field("p", SchemaBuilder.array(P_SCHEMA).optional()) - .build() - - private val TEST_SCHEMA = SchemaBuilder.struct().name("test.schema") - .field("target", - ConnectSchema(Schema.Type.BYTES, - false, - BigDecimal.valueOf(123.4), - Decimal.LOGICAL_NAME, - null, null)) - .field("largeDecimal", - ConnectSchema(Schema.Type.BYTES, - false, - BigDecimal.valueOf(Double.MAX_VALUE).pow(2), - Decimal.LOGICAL_NAME, - null, null)) - .field("byteArray", - ConnectSchema(Schema.Type.BYTES, - false, - "Foo".toByteArray(), - "name.byteArray", - null, null)) - .field("int64", - ConnectSchema(Schema.Type.INT64, - false, - Long.MAX_VALUE, - "name.int64", - null, null)) - .field("int64Timestamp", - ConnectSchema(Schema.Type.INT64, - false, - Date.from(Instant.ofEpochMilli(789)), - Timestamp.LOGICAL_NAME, - null, null)) - .field("int32", - ConnectSchema(Schema.Type.INT32, - false, - 123, - "name.int32", - null, null)) - .field("int32Date", - ConnectSchema(Schema.Type.INT32, - false, - Date.from(Instant.ofEpochMilli(456)), - org.apache.kafka.connect.data.Date.LOGICAL_NAME, - null, null)) - .field("int32Time", - ConnectSchema(Schema.Type.INT32, - false, - Date.from(Instant.ofEpochMilli(123)), - Time.LOGICAL_NAME, - null, null)) - .field("nullField", - ConnectSchema(Schema.Type.INT64, - false, - null, - Time.LOGICAL_NAME, - null, null)) - .field("nullFieldBytes", - ConnectSchema(Schema.Type.BYTES, - false, - null, - Time.LOGICAL_NAME, - null, null)) - .build() - - fun getTreeStruct(): Struct? { - val firstUL = Struct(UL_SCHEMA).put("value", listOf( - Struct(LI_SCHEMA).put("value", "First UL - First Element"), - Struct(LI_SCHEMA).put("value", "First UL - Second Element") - .put("class", listOf("ClassA", "ClassB")) - )) - val secondUL = Struct(UL_SCHEMA).put("value", listOf( - Struct(LI_SCHEMA).put("value", "Second UL - First Element"), - Struct(LI_SCHEMA).put("value", "Second UL - Second Element") - )) - val ulList = listOf(firstUL, secondUL) - val pList = listOf( - Struct(P_SCHEMA).put("value", "First Paragraph"), - Struct(P_SCHEMA).put("value", "Second Paragraph") - ) - return Struct(BODY_SCHEMA) - .put("ul", ulList) - .put("p", pList) - } - - fun getExpectedMap(): Map { - val firstULMap = mapOf("value" to listOf( - mapOf("value" to Values.value("First UL - First Element"), "class" to Values.NULL), - mapOf("value" to Values.value("First UL - Second Element"), "class" to Values.value(listOf("ClassA", "ClassB"))))) - val secondULMap = mapOf("value" to listOf( - mapOf("value" to Values.value("Second UL - First Element"), "class" to Values.NULL), - mapOf("value" to Values.value("Second UL - Second Element"), "class" to Values.NULL))) - val ulListMap = Values.value(listOf(firstULMap, secondULMap)) - val pListMap = Values.value(listOf(mapOf("value" to Values.value("First Paragraph")), - mapOf("value" to Values.value("Second Paragraph")))) - return mapOf("ul" to ulListMap, "p" to pListMap) - } - - fun getTreeMap(): Map { - val firstULMap = mapOf("value" to listOf( - mapOf("value" to "First UL - First Element", "class" to null), - mapOf("value" to "First UL - Second Element", "class" to listOf("ClassA", "ClassB")))) - val secondULMap = mapOf("value" to listOf( - mapOf("value" to "Second UL - First Element", "class" to null), - mapOf("value" to "Second UL - Second Element", "class" to null))) - val ulListMap = listOf(firstULMap, secondULMap) - val pListMap = listOf(mapOf("value" to "First Paragraph"), - mapOf("value" to "Second Paragraph")) - return mapOf("ul" to ulListMap, "p" to pListMap) - } - - fun getItemElement(number: Any?): Map = mapOf("item" to number) - } - -} - diff --git a/kafka-connect-neo4j/src/test/kotlin/streams/kafka/connect/source/Neo4jSourceConnectorConfigTest.kt b/kafka-connect-neo4j/src/test/kotlin/streams/kafka/connect/source/Neo4jSourceConnectorConfigTest.kt deleted file mode 100644 index 570cb184..00000000 --- a/kafka-connect-neo4j/src/test/kotlin/streams/kafka/connect/source/Neo4jSourceConnectorConfigTest.kt +++ /dev/null @@ -1,64 +0,0 @@ -package streams.kafka.connect.source - -import org.apache.kafka.common.config.ConfigException -import org.junit.Test -import kotlin.test.assertEquals -import kotlin.test.assertNull - -class Neo4jSourceConnectorConfigTest { - - @Test(expected = ConfigException::class) - fun `should throw a ConfigException because of unsupported streaming type`() { - try { - val originals = mapOf(Neo4jSourceConnectorConfig.SOURCE_TYPE to SourceType.LABELS.toString(), - Neo4jSourceConnectorConfig.TOPIC to "topic", - Neo4jSourceConnectorConfig.STREAMING_FROM to StreamingFrom.NOW.toString(), - Neo4jSourceConnectorConfig.STREAMING_PROPERTY to "timestamp") - Neo4jSourceConnectorConfig(originals) - } catch (e: ConfigException) { - assertEquals("Supported source query types are: ${SourceType.QUERY}", e.message) - throw e - } - } - - @Test(expected = ConfigException::class) - fun `should throw a ConfigException because of empty query`() { - try { - val originals = mapOf(Neo4jSourceConnectorConfig.SOURCE_TYPE to SourceType.QUERY.toString(), - Neo4jSourceConnectorConfig.TOPIC to "topic", - Neo4jSourceConnectorConfig.STREAMING_FROM to StreamingFrom.NOW.toString(), - Neo4jSourceConnectorConfig.STREAMING_PROPERTY to "timestamp") - Neo4jSourceConnectorConfig(originals) - } catch (e: ConfigException) { - assertEquals("You need to define: ${Neo4jSourceConnectorConfig.SOURCE_TYPE_QUERY}", e.message) - throw e - } - } - - @Test - fun `should return config`() { - val originals = mapOf(Neo4jSourceConnectorConfig.SOURCE_TYPE to SourceType.QUERY.toString(), - Neo4jSourceConnectorConfig.SOURCE_TYPE_QUERY to "MATCH (n) RETURN n", - Neo4jSourceConnectorConfig.TOPIC to "topic", - Neo4jSourceConnectorConfig.STREAMING_POLL_INTERVAL to "10", - Neo4jSourceConnectorConfig.STREAMING_FROM to StreamingFrom.NOW.toString(), - Neo4jSourceConnectorConfig.STREAMING_PROPERTY to "timestamp") - val config = Neo4jSourceConnectorConfig(originals) - assertEquals(originals[Neo4jSourceConnectorConfig.TOPIC], config.topic) - assertEquals(originals[Neo4jSourceConnectorConfig.SOURCE_TYPE_QUERY], config.query) - assertEquals(originals[Neo4jSourceConnectorConfig.STREAMING_PROPERTY], config.streamingProperty) - assertEquals(originals[Neo4jSourceConnectorConfig.STREAMING_FROM], config.streamingFrom.toString()) - assertEquals(originals[Neo4jSourceConnectorConfig.STREAMING_POLL_INTERVAL]?.toInt(), config.pollInterval) - } - - @Test - fun `should return config null streaming property`() { - val originals = mapOf(Neo4jSourceConnectorConfig.SOURCE_TYPE to SourceType.QUERY.toString(), - Neo4jSourceConnectorConfig.SOURCE_TYPE_QUERY to "MATCH (n) RETURN n", - Neo4jSourceConnectorConfig.TOPIC to "topic", - Neo4jSourceConnectorConfig.STREAMING_POLL_INTERVAL to "10", - Neo4jSourceConnectorConfig.STREAMING_FROM to StreamingFrom.NOW.toString()) - val config = Neo4jSourceConnectorConfig(originals) - assertEquals("", config.streamingProperty) - } -} \ No newline at end of file diff --git a/kafka-connect-neo4j/src/test/kotlin/streams/kafka/connect/source/Neo4jSourceTaskTest.kt b/kafka-connect-neo4j/src/test/kotlin/streams/kafka/connect/source/Neo4jSourceTaskTest.kt deleted file mode 100644 index 7fa3b4aa..00000000 --- a/kafka-connect-neo4j/src/test/kotlin/streams/kafka/connect/source/Neo4jSourceTaskTest.kt +++ /dev/null @@ -1,282 +0,0 @@ -package streams.kafka.connect.source - -import org.apache.kafka.connect.data.Struct -import org.apache.kafka.connect.errors.ConnectException -import org.apache.kafka.connect.source.SourceRecord -import org.apache.kafka.connect.source.SourceTask -import org.apache.kafka.connect.source.SourceTaskContext -import org.apache.kafka.connect.storage.OffsetStorageReader -import org.hamcrest.Matchers -import org.junit.After -import org.junit.Before -import org.junit.Rule -import org.junit.Test -import org.mockito.Mockito -import org.neo4j.configuration.GraphDatabaseSettings -import org.neo4j.function.ThrowingSupplier -import org.neo4j.harness.junit.rule.Neo4jRule -import streams.Assert -import streams.extensions.labelNames -import streams.kafka.connect.common.Neo4jConnectorConfig -import streams.kafka.connect.sink.AuthenticationType -import streams.utils.JSONUtils -import java.util.UUID -import java.util.concurrent.TimeUnit - -class Neo4jSourceTaskTest { - - @Rule @JvmField val db = Neo4jRule() - .withDisabledServer() - .withConfig(GraphDatabaseSettings.auth_enabled, false) - - private lateinit var task: SourceTask - - @After - fun after() { - task.stop() - } - - @Before - fun before() { - db.defaultDatabaseService().beginTx().use { - it.execute("MATCH (n) DETACH DELETE n") - it.commit() - } - task = Neo4jSourceTask() - val sourceTaskContextMock = Mockito.mock(SourceTaskContext::class.java) - val offsetStorageReader = Mockito.mock(OffsetStorageReader::class.java) - Mockito.`when`(sourceTaskContextMock.offsetStorageReader()) - .thenReturn(offsetStorageReader) - Mockito.`when`(offsetStorageReader.offset(Mockito.anyMap())) - .thenReturn(emptyMap()) - task.initialize(sourceTaskContextMock) - } - - private fun structToMap(struct: Struct): Map = struct.schema().fields() - .map { - it.name() to when (val value = struct[it.name()]) { - is Struct -> structToMap(value) - else -> value - } - } - .toMap() - - fun Struct.toMap() = structToMap(this) - - @Test - fun `should source data from Neo4j with custom QUERY from NOW`() { - val props = mutableMapOf() - props[Neo4jConnectorConfig.SERVER_URI] = db.boltURI().toString() - props[Neo4jSourceConnectorConfig.TOPIC] = UUID.randomUUID().toString() - props[Neo4jSourceConnectorConfig.STREAMING_POLL_INTERVAL] = "10" - props[Neo4jSourceConnectorConfig.STREAMING_PROPERTY] = "timestamp" - props[Neo4jSourceConnectorConfig.SOURCE_TYPE_QUERY] = getSourceQuery() - props[Neo4jConnectorConfig.AUTHENTICATION_TYPE] = AuthenticationType.NONE.toString() - - task.start(props) - val totalRecords = 10 - val expected = insertRecords(totalRecords, true) - - val list = mutableListOf() - Assert.assertEventually(ThrowingSupplier { - task.poll()?.let { list.addAll(it) } - val actualList = list.map { JSONUtils.readValue>(it.value()) } - expected.containsAll(actualList) - }, Matchers.equalTo(true), 30, TimeUnit.SECONDS) - } - - @Test - fun `should source data from Neo4j with custom QUERY from NOW with Schema`() { - val props = mutableMapOf() - props[Neo4jConnectorConfig.SERVER_URI] = db.boltURI().toString() - props[Neo4jSourceConnectorConfig.TOPIC] = UUID.randomUUID().toString() - props[Neo4jSourceConnectorConfig.STREAMING_POLL_INTERVAL] = "10" - props[Neo4jSourceConnectorConfig.ENFORCE_SCHEMA] = "true" - props[Neo4jSourceConnectorConfig.STREAMING_PROPERTY] = "timestamp" - props[Neo4jSourceConnectorConfig.SOURCE_TYPE_QUERY] = getSourceQuery() - props[Neo4jConnectorConfig.AUTHENTICATION_TYPE] = AuthenticationType.NONE.toString() - - task.start(props) - val totalRecords = 10 - val expected = insertRecords(totalRecords) - - val list = mutableListOf() - Assert.assertEventually(ThrowingSupplier { - task.poll()?.let { list.addAll(it) } - val actualList = list.map { (it.value() as Struct).toMap() } - expected.containsAll(actualList) - }, Matchers.equalTo(true), 30, TimeUnit.SECONDS) - } - - @Test - fun `should source data from Neo4j with custom QUERY from ALL`() { - val props = mutableMapOf() - props[Neo4jConnectorConfig.SERVER_URI] = db.boltURI().toString() - props[Neo4jSourceConnectorConfig.TOPIC] = UUID.randomUUID().toString() - props[Neo4jSourceConnectorConfig.STREAMING_FROM] = "ALL" - props[Neo4jSourceConnectorConfig.STREAMING_POLL_INTERVAL] = "10" - props[Neo4jSourceConnectorConfig.STREAMING_PROPERTY] = "timestamp" - props[Neo4jSourceConnectorConfig.SOURCE_TYPE_QUERY] = getSourceQuery() - props[Neo4jConnectorConfig.AUTHENTICATION_TYPE] = AuthenticationType.NONE.toString() - - task.start(props) - val totalRecords = 10 - val expected = insertRecords(totalRecords, true) - - val list = mutableListOf() - Assert.assertEventually(ThrowingSupplier { - task.poll()?.let { list.addAll(it) } - val actualList = list.map { JSONUtils.readValue>(it.value()) } - expected == actualList - }, Matchers.equalTo(true), 30, TimeUnit.SECONDS) - } - - @Test - fun `should source data from Neo4j with custom QUERY from ALL with Schema`() { - val props = mutableMapOf() - props[Neo4jConnectorConfig.SERVER_URI] = db.boltURI().toString() - props[Neo4jSourceConnectorConfig.TOPIC] = UUID.randomUUID().toString() - props[Neo4jSourceConnectorConfig.STREAMING_FROM] = "ALL" - props[Neo4jSourceConnectorConfig.STREAMING_POLL_INTERVAL] = "10" - props[Neo4jSourceConnectorConfig.ENFORCE_SCHEMA] = "true" - props[Neo4jSourceConnectorConfig.STREAMING_PROPERTY] = "timestamp" - props[Neo4jSourceConnectorConfig.SOURCE_TYPE_QUERY] = getSourceQuery() - props[Neo4jConnectorConfig.AUTHENTICATION_TYPE] = AuthenticationType.NONE.toString() - - task.start(props) - val totalRecords = 10 - val expected = insertRecords(totalRecords) - - val list = mutableListOf() - Assert.assertEventually(ThrowingSupplier { - task.poll()?.let { list.addAll(it) } - val actualList = list.map { (it.value() as Struct).toMap() } - expected == actualList - }, Matchers.equalTo(true), 30, TimeUnit.SECONDS) - } - - private fun insertRecords(totalRecords: Int, longToInt: Boolean = false) = db.defaultDatabaseService().beginTx().use { tx -> - val elements = (1..totalRecords).map { - val result = tx.execute(""" - |CREATE (n:Test{ - | name: 'Name ' + $it, - | timestamp: timestamp(), - | point: point({longitude: 56.7, latitude: 12.78, height: 8}), - | array: [1,2,3], - | datetime: localdatetime(), - | boolean: true - |}) - |RETURN n.name AS name, n.timestamp AS timestamp, - | n.point AS point, - | n.array AS array, - | n.datetime AS datetime, - | n.boolean AS boolean, - | { - | key1: "value1", - | key2: "value2" - | } AS map, - | n AS node - """.trimMargin()) - val map = result.next() - map["array"] = (map["array"] as LongArray).toList() - .map { if (longToInt) it.toInt() else it } - map["point"] = JSONUtils.readValue>(map["point"]!!) - map["datetime"] = JSONUtils.readValue(map["datetime"]!!) - val node = map["node"] as org.neo4j.graphdb.Node - val nodeMap = node.allProperties - nodeMap[""] = if (longToInt) node.id.toInt() else node.id - nodeMap[""] = node.labelNames() - // are the same value as above - nodeMap["array"] = map["array"] - nodeMap["point"] = map["point"] - nodeMap["datetime"] = map["datetime"] - map["node"] = nodeMap - map - } - tx.commit() - elements - } - - @Test - fun `should source data from Neo4j with custom QUERY without streaming property`() { - val props = mutableMapOf() - props[Neo4jConnectorConfig.SERVER_URI] = db.boltURI().toString() - props[Neo4jSourceConnectorConfig.TOPIC] = UUID.randomUUID().toString() - props[Neo4jSourceConnectorConfig.STREAMING_POLL_INTERVAL] = "10" - props[Neo4jSourceConnectorConfig.SOURCE_TYPE_QUERY] = getSourceQuery() - props[Neo4jConnectorConfig.AUTHENTICATION_TYPE] = AuthenticationType.NONE.toString() - - task.start(props) - val totalRecords = 10 - insertRecords(totalRecords) - - val list = mutableListOf() - Assert.assertEventually(ThrowingSupplier { - task.poll()?.let { list.addAll(it) } - val actualList = list.map { JSONUtils.readValue>(it.value()) } - actualList.size >= 2 - }, Matchers.equalTo(true), 30, TimeUnit.SECONDS) - } - - @Test - fun `should source data from Neo4j with custom QUERY without streaming property with Schema`() { - val props = mutableMapOf() - props[Neo4jConnectorConfig.SERVER_URI] = db.boltURI().toString() - props[Neo4jSourceConnectorConfig.TOPIC] = UUID.randomUUID().toString() - props[Neo4jSourceConnectorConfig.STREAMING_POLL_INTERVAL] = "10" - props[Neo4jSourceConnectorConfig.ENFORCE_SCHEMA] = "true" - props[Neo4jSourceConnectorConfig.SOURCE_TYPE_QUERY] = getSourceQuery() - props[Neo4jConnectorConfig.AUTHENTICATION_TYPE] = AuthenticationType.NONE.toString() - - task.start(props) - val totalRecords = 10 - insertRecords(totalRecords) - - val list = mutableListOf() - Assert.assertEventually(ThrowingSupplier { - task.poll()?.let { list.addAll(it) } - val actualList = list.map { (it.value() as Struct).toMap() } - actualList.size >= 2 - }, Matchers.equalTo(true), 30, TimeUnit.SECONDS) - } - - private fun getSourceQuery() = """ - |MATCH (n:Test) - |WHERE n.timestamp > ${'$'}lastCheck - |RETURN n.name AS name, n.timestamp AS timestamp, - | n.point AS point, - | n.array AS array, - | n.datetime AS datetime, - | n.boolean AS boolean, - | { - | key1: "value1", - | key2: "value2" - | } AS map, - | n AS node - """.trimMargin() - - @Test(expected = ConnectException::class) - fun `should throw exception`() { - val props = mutableMapOf() - props[Neo4jConnectorConfig.SERVER_URI] = db.boltURI().toString() - props[Neo4jSourceConnectorConfig.TOPIC] = UUID.randomUUID().toString() - props[Neo4jSourceConnectorConfig.STREAMING_POLL_INTERVAL] = "10" - props[Neo4jSourceConnectorConfig.SOURCE_TYPE_QUERY] = "WRONG QUERY".trimMargin() - props[Neo4jConnectorConfig.AUTHENTICATION_TYPE] = AuthenticationType.NONE.toString() - - task.start(props) - val totalRecords = 10 - insertRecords(totalRecords) - var exception: ConnectException? = null - Assert.assertEventually(ThrowingSupplier { - try { - task.poll() - false - } catch (e: ConnectException) { - exception = e - true - } - }, Matchers.equalTo(true), 30, TimeUnit.SECONDS) - if (exception != null) throw exception as ConnectException - } -} \ No newline at end of file diff --git a/kafka-connect-neo4j/src/test/resources/logback.xml b/kafka-connect-neo4j/src/test/resources/logback.xml deleted file mode 100644 index 4b82edca..00000000 --- a/kafka-connect-neo4j/src/test/resources/logback.xml +++ /dev/null @@ -1,11 +0,0 @@ - - - - %d{HH:mm:ss.SSS} [%thread] %-5level %logger - %msg%n - - - - - - - \ No newline at end of file diff --git a/pom.xml b/pom.xml index 10be1f94..ef91a9a5 100644 --- a/pom.xml +++ b/pom.xml @@ -43,7 +43,6 @@ common test-support - kafka-connect-neo4j producer consumer distribution @@ -54,7 +53,7 @@ 1.8 1.6.10 1.6.0 - 4.4.25 + 4.4.3 2.6.3 2.14.3 true @@ -110,6 +109,11 @@ it-test-support ${neo4j.version} + + org.neo4j + neo4j-configuration + ${neo4j.version} + org.testcontainers testcontainers diff --git a/producer/src/test/kotlin/streams/integrations/KafkaEventRouterCompactionStrategyTSE.kt b/producer/src/test/kotlin/streams/integrations/KafkaEventRouterCompactionStrategyTSE.kt index 43ea84eb..f3f84efc 100644 --- a/producer/src/test/kotlin/streams/integrations/KafkaEventRouterCompactionStrategyTSE.kt +++ b/producer/src/test/kotlin/streams/integrations/KafkaEventRouterCompactionStrategyTSE.kt @@ -49,7 +49,7 @@ class KafkaEventRouterCompactionStrategyTSE : KafkaEventRouterBaseTSE() { // check if there is only one record with key 'test' and payload 'Compaction 4' assertTopicFilled(kafkaConsumer, true) { val compactedRecord = it.filter { JSONUtils.readValue(it.key()) == keyRecord } - it.count() == 500 && + it.count() != 0 && compactedRecord.count() == 1 && JSONUtils.readValue>(compactedRecord.first().value())["payload"] == "Compaction 4" } @@ -119,7 +119,7 @@ class KafkaEventRouterCompactionStrategyTSE : KafkaEventRouterBaseTSE() { // we check that there is only one tombstone record assertTopicFilled(kafkaConsumer, true) { val nullRecords = it.filter { it.value() == null } - it.count() == 500 + it.count() != 0 && nullRecords.count() == 1 && JSONUtils.readValue>(nullRecords.first().key()) == mapOf("start" to "0", "end" to "1", "label" to relType) } @@ -171,7 +171,7 @@ class KafkaEventRouterCompactionStrategyTSE : KafkaEventRouterBaseTSE() { assertTopicFilled(kafkaConsumer, true) { val nullRecords = it.filter { it.value() == null } val keyRecordExpected = mapOf("ids" to mapOf("name" to "Sherlock"), "labels" to listOf("Person")) - it.count() == 500 + it.count() != 0 && nullRecords.count() == 1 && keyRecordExpected == JSONUtils.readValue>(nullRecords.first().key()) } diff --git a/producer/src/test/kotlin/streams/integrations/KafkaEventRouterEnterpriseTSE.kt b/producer/src/test/kotlin/streams/integrations/KafkaEventRouterEnterpriseTSE.kt index b36c11ed..9cda6900 100644 --- a/producer/src/test/kotlin/streams/integrations/KafkaEventRouterEnterpriseTSE.kt +++ b/producer/src/test/kotlin/streams/integrations/KafkaEventRouterEnterpriseTSE.kt @@ -192,6 +192,7 @@ class KafkaEventRouterEnterpriseTSE { } @Test + @Ignore("flaky") fun `should stream the data from a specific instance with custom routing params`() = runBlocking { // given createPath("foo") diff --git a/producer/src/test/kotlin/streams/integrations/KafkaEventRouterSimpleTSE.kt b/producer/src/test/kotlin/streams/integrations/KafkaEventRouterSimpleTSE.kt index dc2f1ea1..37547918 100644 --- a/producer/src/test/kotlin/streams/integrations/KafkaEventRouterSimpleTSE.kt +++ b/producer/src/test/kotlin/streams/integrations/KafkaEventRouterSimpleTSE.kt @@ -5,6 +5,8 @@ import kotlinx.coroutines.runBlocking import org.hamcrest.Matchers import org.junit.Test import org.neo4j.function.ThrowingSupplier +import org.neo4j.values.storable.CoordinateReferenceSystem +import org.neo4j.values.storable.PointValue import streams.Assert import streams.events.EntityType import streams.events.NodeChange @@ -17,6 +19,7 @@ import streams.KafkaTestUtils import streams.utils.JSONUtils import streams.setConfig import streams.start +import streams.utils.toMapValue import java.util.* import java.util.concurrent.TimeUnit import kotlin.test.assertEquals @@ -44,6 +47,36 @@ class KafkaEventRouterSimpleTSE: KafkaEventRouterBaseTSE() { }) } + @Test + fun testCreateNodeWithPointValue() { + db.start() + kafkaConsumer.subscribe(listOf("neo4j")) + db.execute("CREATE (:Person {name:'John Doe', age:42, bornIn: point({longitude: 12.78, latitude: 56.7, height: 100})})") + val records = kafkaConsumer.poll(5000) + assertEquals(1, records.count()) + assertEquals(true, records.all { + JSONUtils.asStreamsTransactionEvent(it.value()).let { + val after = it.payload.after as NodeChange + val labels = after.labels + val propertiesAfter = after.properties + val expectedProperties = mapOf( + "name" to "John Doe", + "age" to 42, + "bornIn" to PointValue.fromMap(mapOf( + "crs" to "wgs-84-3d", + "latitude" to 12.78, + "longitude" to 56.7, + "height" to 100.0, + ).toMapValue()) + ) + labels == listOf("Person") && propertiesAfter == expectedProperties + && it.meta.operation == OperationType.created + && it.schema.properties == mapOf("name" to "String", "age" to "Long", "bornIn" to "PointValue") + && it.schema.constraints.isEmpty() + } + }) + } + @Test fun testCreateRelationshipWithRelRouting() { db.setConfig("streams.source.topic.relationships.knows", "KNOWS{*}").start() diff --git a/producer/src/test/kotlin/streams/integrations/KafkaEventRouterSuiteIT.kt b/producer/src/test/kotlin/streams/integrations/KafkaEventRouterSuiteIT.kt index f7a4d542..19db4248 100644 --- a/producer/src/test/kotlin/streams/integrations/KafkaEventRouterSuiteIT.kt +++ b/producer/src/test/kotlin/streams/integrations/KafkaEventRouterSuiteIT.kt @@ -44,14 +44,13 @@ class KafkaEventRouterSuiteIT { @BeforeClass @JvmStatic fun setUpContainer() { - var exists = false StreamsUtils.ignoreExceptions({ kafka = KafkaContainer(confluentPlatformVersion) .withNetwork(Network.newNetwork()) kafka.start() - exists = true + isRunning = kafka.isRunning }, IllegalStateException::class.java) - Assume.assumeTrue("Kafka container has to exist", exists) + Assume.assumeTrue("Kafka container has to exist", isRunning) Assume.assumeTrue("Kafka must be running", ::kafka.isInitialized && kafka.isRunning) } diff --git a/test-support/src/main/kotlin/streams/MavenUtils.kt b/test-support/src/main/kotlin/streams/MavenUtils.kt index e337ba56..f62ad0c1 100644 --- a/test-support/src/main/kotlin/streams/MavenUtils.kt +++ b/test-support/src/main/kotlin/streams/MavenUtils.kt @@ -11,7 +11,7 @@ object MavenUtils { val rt = Runtime.getRuntime() val mvnw = if (System.getProperty("os.name").startsWith("Windows")) "./mvnw.cmd" else "./mvnw" - val commands = arrayOf(mvnw, "-pl", "!kafka-connect-neo4j", "-DbuildSubDirectory=containerPlugins") + + val commands = arrayOf(mvnw, "-DbuildSubDirectory=containerPlugins") + args.let { if (it.isNullOrEmpty()) arrayOf("package", "-Dmaven.test.skip") else it } val proc = rt.exec(commands, null, File(path))