From 4de2aa389368d72db28a1094f3b59cc678151ad0 Mon Sep 17 00:00:00 2001 From: Kohei Watanabe Date: Thu, 31 Aug 2023 13:27:44 +0900 Subject: [PATCH 1/2] fix(datahub-system-update-job): enable SPRING_KAFKA_PROPERTIES_AUTO_REGISTER_SCHEMAS `SPRING_KAFKA_PROPERTIES_AUTO_REGISTER_SCHEMAS` should be true to solve the following error in datahub-system-update-job ```log 2023-08-28 01:50:42,180 [main] ERROR c.l.d.u.s.e.steps.DataHubStartupStep:40 - DataHubStartupStep failed. org.apache.kafka.common.errors.SerializationException: Error serializing Avro message Caused by: java.io.IOException: No schema registered under subject! at io.confluent.kafka.schemaregistry.client.MockSchemaRegistryClient.getLatestVersion(MockSchemaRegistryClient.java:261) at io.confluent.kafka.schemaregistry.client.MockSchemaRegistryClient.getLatestSchemaMetadata(MockSchemaRegistryClient.java:310) at io.confluent.kafka.serializers.AbstractKafkaSchemaSerDe.lookupLatestVersion(AbstractKafkaSchemaSerDe.java:181) at io.confluent.kafka.serializers.AbstractKafkaAvroSerializer.serializeImpl(AbstractKafkaAvroSerializer.java:77) at io.confluent.kafka.serializers.KafkaAvroSerializer.serialize(KafkaAvroSerializer.java:59) at org.apache.kafka.common.serialization.Serializer.serialize(Serializer.java:62) at org.apache.kafka.clients.producer.KafkaProducer.doSend(KafkaProducer.java:902) at org.apache.kafka.clients.producer.KafkaProducer.send(KafkaProducer.java:862) at com.linkedin.metadata.dao.producer.KafkaEventProducer.produceDataHubUpgradeHistoryEvent(KafkaEventProducer.java:171) at com.linkedin.datahub.upgrade.system.elasticsearch.steps.DataHubStartupStep.lambda$executable$0(DataHubStartupStep.java:37) at com.linkedin.datahub.upgrade.impl.DefaultUpgradeManager.executeStepInternal(DefaultUpgradeManager.java:110) at com.linkedin.datahub.upgrade.impl.DefaultUpgradeManager.executeInternal(DefaultUpgradeManager.java:68) at com.linkedin.datahub.upgrade.impl.DefaultUpgradeManager.executeInternal(DefaultUpgradeManager.java:42) at com.linkedin.datahub.upgrade.impl.DefaultUpgradeManager.execute(DefaultUpgradeManager.java:33) at com.linkedin.datahub.upgrade.UpgradeCli.run(UpgradeCli.java:80) at org.springframework.boot.SpringApplication.callRunner(SpringApplication.java:768) at org.springframework.boot.SpringApplication.callRunners(SpringApplication.java:752) at org.springframework.boot.SpringApplication.run(SpringApplication.java:314) at org.springframework.boot.builder.SpringApplicationBuilder.run(SpringApplicationBuilder.java:164) at com.linkedin.datahub.upgrade.UpgradeCliApplication.main(UpgradeCliApplication.java:23) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:566) at org.springframework.boot.loader.MainMethodRunner.run(MainMethodRunner.java:49) at org.springframework.boot.loader.Launcher.launch(Launcher.java:108) at org.springframework.boot.loader.Launcher.launch(Launcher.java:58) at org.springframework.boot.loader.JarLauncher.main(JarLauncher.java:65) ``` The related slack threads: - https://datahubspace.slack.com/archives/C029A3M079U/p1693269038664329?thread_ts=1692266022.927369&cid=C029A3M079U - https://datahubspace.slack.com/archives/CV2UVAPPG/p1693424025572079 --- .../templates/datahub-upgrade/datahub-system-update-job.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/datahub/templates/datahub-upgrade/datahub-system-update-job.yml b/charts/datahub/templates/datahub-upgrade/datahub-system-update-job.yml index e03e8b38f..261f0b921 100644 --- a/charts/datahub/templates/datahub-upgrade/datahub-system-update-job.yml +++ b/charts/datahub/templates/datahub-upgrade/datahub-system-update-job.yml @@ -70,7 +70,7 @@ spec: - name: SCHEMA_REGISTRY_SYSTEM_UPDATE value: "true" - name: SPRING_KAFKA_PROPERTIES_AUTO_REGISTER_SCHEMAS - value: "false" + value: "true" - name: SPRING_KAFKA_PROPERTIES_USE_LATEST_VERSION value: "true" {{- end }} From 0af5af6441e800df3286c2016ca60b0c604d6f34 Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Tue, 5 Sep 2023 22:38:48 -0500 Subject: [PATCH 2/2] Update Chart.yaml --- charts/datahub/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/datahub/Chart.yaml b/charts/datahub/Chart.yaml index 43e2cf266..ecd6dc00e 100644 --- a/charts/datahub/Chart.yaml +++ b/charts/datahub/Chart.yaml @@ -4,7 +4,7 @@ description: A Helm chart for LinkedIn DataHub type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.2.182 +version: 0.2.183 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. appVersion: 0.10.5