From eb63e1ffa1a99aaf489a18bbca830d6290c04bba Mon Sep 17 00:00:00 2001 From: Vova Kolmakov Date: Wed, 5 Jun 2024 01:04:54 +0700 Subject: [PATCH] [HUDI-7782] Task not serializable due to DynamoDBBasedLockProvider and HiveMetastoreBasedLockProvider in clean action (#11389) Co-authored-by: Vova Kolmakov --- .../aws/transaction/lock/DynamoDBBasedLockProvider.java | 5 +++-- .../transaction/lock/HiveMetastoreBasedLockProvider.java | 9 +++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/hudi-aws/src/main/java/org/apache/hudi/aws/transaction/lock/DynamoDBBasedLockProvider.java b/hudi-aws/src/main/java/org/apache/hudi/aws/transaction/lock/DynamoDBBasedLockProvider.java index 2b67a483f383..bbdcb4e7003d 100644 --- a/hudi-aws/src/main/java/org/apache/hudi/aws/transaction/lock/DynamoDBBasedLockProvider.java +++ b/hudi-aws/src/main/java/org/apache/hudi/aws/transaction/lock/DynamoDBBasedLockProvider.java @@ -47,6 +47,7 @@ import javax.annotation.concurrent.NotThreadSafe; +import java.io.Serializable; import java.net.URI; import java.util.ArrayList; import java.util.Collection; @@ -58,13 +59,13 @@ * using DynamoDB. Users need to have access to AWS DynamoDB to be able to use this lock. */ @NotThreadSafe -public class DynamoDBBasedLockProvider implements LockProvider { +public class DynamoDBBasedLockProvider implements LockProvider, Serializable { private static final Logger LOG = LoggerFactory.getLogger(DynamoDBBasedLockProvider.class); private static final String DYNAMODB_ATTRIBUTE_NAME = "key"; - private final AmazonDynamoDBLockClient client; + private final transient AmazonDynamoDBLockClient client; private final String tableName; private final String dynamoDBPartitionKey; protected final DynamoDbBasedLockConfig dynamoDBLockConfiguration; diff --git a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/transaction/lock/HiveMetastoreBasedLockProvider.java b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/transaction/lock/HiveMetastoreBasedLockProvider.java index b458df9a5796..4013a97688e9 100644 --- a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/transaction/lock/HiveMetastoreBasedLockProvider.java +++ b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/transaction/lock/HiveMetastoreBasedLockProvider.java @@ -44,6 +44,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.Serializable; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -75,18 +76,18 @@ * using hive metastore APIs. Users need to have a HiveMetastore & Zookeeper cluster deployed to be able to use this lock. * */ -public class HiveMetastoreBasedLockProvider implements LockProvider { +public class HiveMetastoreBasedLockProvider implements LockProvider, Serializable { private static final Logger LOG = LoggerFactory.getLogger(HiveMetastoreBasedLockProvider.class); private final String databaseName; private final String tableName; private final String hiveMetastoreUris; - private IMetaStoreClient hiveClient; + private transient IMetaStoreClient hiveClient; private volatile LockResponse lock = null; protected LockConfiguration lockConfiguration; - private ScheduledFuture future = null; - private final ScheduledExecutorService executor = Executors.newScheduledThreadPool(2); + private transient ScheduledFuture future = null; + private final transient ScheduledExecutorService executor = Executors.newScheduledThreadPool(2); public HiveMetastoreBasedLockProvider(final LockConfiguration lockConfiguration, final StorageConfiguration conf) { this(lockConfiguration);