From c80b5596c1de08dc25a096be663241abf5de1b6e Mon Sep 17 00:00:00 2001 From: Jon Vexler Date: Thu, 27 Jun 2024 04:06:54 -0400 Subject: [PATCH] [HUDI-7932] Fix Import ordering (#11524) --- .../aws/cloudwatch/CloudWatchReporter.java | 12 ++++----- ...nfigAWSAssumedRoleCredentialsProvider.java | 1 + .../HoodieConfigAWSCredentialsProvider.java | 8 +++--- .../aws/sync/AWSGlueCatalogSyncClient.java | 6 ++--- .../cloudwatch/TestCloudWatchReporter.java | 10 +++---- .../aws/sync/TestGluePartitionPushdown.java | 1 + .../ITTestDynamoDBBasedLockProvider.java | 17 ++++++------ .../org/apache/hudi/cli/HoodiePrompt.java | 3 +-- .../hudi/cli/commands/BootstrapCommand.java | 4 ++- .../hudi/cli/commands/CleansCommand.java | 4 ++- .../hudi/cli/commands/ClusteringCommand.java | 2 ++ .../commands/HoodieSyncValidateCommand.java | 1 + .../hudi/cli/commands/MarkersCommand.java | 1 + .../hudi/cli/commands/RestoresCommand.java | 1 + .../hudi/cli/commands/RollbacksCommand.java | 1 - .../hudi/cli/commands/SavepointsCommand.java | 1 + .../hudi/cli/commands/SparkEnvCommand.java | 1 + .../hudi/cli/commands/StatsCommand.java | 4 +-- .../hudi/cli/commands/TempViewCommand.java | 1 + .../hudi/cli/commands/TimelineCommand.java | 4 +-- .../commands/UpgradeOrDowngradeCommand.java | 1 + .../hudi/cli/commands/UtilsCommand.java | 1 + .../org/apache/hudi/cli/TestSparkUtil.java | 4 +-- .../cli/commands/TestSparkEnvCommand.java | 2 +- .../HoodieCLIIntegrationTestBase.java | 1 + .../HoodieCLIIntegrationTestHarness.java | 1 + .../heartbeat/HoodieHeartbeatClient.java | 2 +- .../transaction/ConcurrentOperation.java | 2 +- .../client/transaction/lock/LockInfo.java | 3 ++- .../utils/LegacyArchivedMetaEntryReader.java | 2 +- .../execution/CopyOnWriteInsertHandler.java | 1 + .../execution/HoodieLazyInsertIterable.java | 3 ++- .../index/bucket/HoodieSimpleBucketIndex.java | 9 +++---- .../apache/hudi/io/HoodieBootstrapHandle.java | 5 ++-- .../hudi/io/HoodieMergeHandleFactory.java | 1 + .../hudi/io/HoodieSortedMergeHandle.java | 3 ++- .../HoodieSortedMergeHandleWithChangeLog.java | 2 +- .../hudi/keygen/ComplexAvroKeyGenerator.java | 3 ++- .../keygen/GlobalAvroDeleteKeyGenerator.java | 3 ++- .../hudi/keygen/SimpleAvroKeyGenerator.java | 3 ++- .../HoodieAvroKeyGeneratorFactory.java | 2 +- .../metadata/HoodieMetadataWriteUtils.java | 2 +- .../HoodieTableMetadataKeyGenerator.java | 3 ++- .../BucketIndexBulkInsertPartitioner.java | 1 + .../ConsistentHashingUpdateStrategyUtils.java | 2 +- .../hudi/table/action/commit/SmallFile.java | 3 ++- .../compact/strategy/CompactionStrategy.java | 2 +- .../upgrade/ZeroToOneUpgradeHandler.java | 2 +- .../org/apache/hudi/DummyActiveAction.java | 2 +- .../heartbeat/TestHoodieHeartbeatClient.java | 2 +- ...eateAvroKeyGeneratorByTypeWithFactory.java | 1 + .../TestHoodieMetricsDatadogConfig.java | 2 +- .../hudi/table/action/TestCleanPlanner.java | 1 - .../utils/HoodieWriterClientTestHarness.java | 6 ++--- .../hudi/utils/TestConcatenatingIterator.java | 1 + .../execution/FlinkLazyInsertIterable.java | 3 ++- .../FlinkPartitionTTLActionExecutor.java | 1 + .../table/action/commit/FlinkWriteHelper.java | 3 ++- ...tFlinkSizeBasedClusteringPlanStrategy.java | 1 + .../execution/JavaLazyInsertIterable.java | 3 ++- ...JavaInsertPreppedCommitActionExecutor.java | 1 - ...JavaUpsertPreppedCommitActionExecutor.java | 1 - .../table/action/commit/JavaWriteHelper.java | 3 ++- .../run/strategy/ExecutionStrategyUtil.java | 9 ++++--- .../hudi/client/model/HoodieInternalRow.java | 1 + .../execution/SparkLazyInsertIterable.java | 2 +- .../bulkinsert/RDDBucketIndexPartitioner.java | 7 ++--- .../RDDSimpleBucketBulkInsertPartitioner.java | 1 + .../RDDSpatialCurveSortPartitioner.java | 7 ++--- .../RowSpatialCurveSortPartitioner.java | 1 + .../io/storage/HoodieSparkFileWriter.java | 3 ++- .../HoodieSparkParquetStreamWriter.java | 10 ++++--- .../row/HoodieRowParquetWriteSupport.java | 2 +- .../hudi/keygen/ComplexKeyGenerator.java | 3 ++- .../hudi/keygen/SimpleKeyGenerator.java | 3 ++- .../keygen/SparkKeyGeneratorInterface.java | 1 + .../keygen/TimestampBasedKeyGenerator.java | 3 ++- .../UTF8StringPartitionPathFormatter.java | 1 + ...rkHoodieMetadataBulkInsertPartitioner.java | 12 +++++---- .../hudi/metrics/DistributedRegistry.java | 9 ++++--- ...ketBulkInsertDataInternalWriterHelper.java | 1 + .../commit/SparkBucketIndexPartitioner.java | 26 +++++++++---------- .../action/commit/SparkHoodiePartitioner.java | 1 + .../org/apache/hudi/HoodieUnsafeRDD.scala | 1 - .../apache/hudi/SparkConversionUtils.scala | 1 + ...rkFileFormatInternalRowReaderContext.scala | 7 ++--- .../util/CloseableInternalRowIterator.scala | 1 + .../spark/sql/BucketPartitionUtils.scala | 1 + .../org/apache/spark/sql/DataFrameUtil.scala | 1 + .../sql/HoodieCatalystExpressionUtils.scala | 1 + .../apache/spark/sql/HoodieUnsafeUtils.scala | 1 + .../sql/execution/SQLConfInjectingRDD.scala | 2 +- .../datasources/SparkParsePartitionUtil.scala | 5 ++-- .../parquet/SparkParquetReader.scala | 3 ++- .../apache/spark/sql/hudi/SparkAdapter.scala | 1 + .../sql/hudi/execution/RangeSample.scala | 1 + ...rkHoodieMetadataBulkInsertPartitioner.java | 11 ++++---- .../TestUniformBootstrapModeSelector.java | 1 + ...tSparkSizeBasedClusteringPlanStrategy.java | 1 - .../TestBoundedInMemoryExecutorInSpark.java | 2 +- .../execution/TestBoundedInMemoryQueue.java | 2 +- .../TestDisruptorExecutionInSpark.java | 1 + .../execution/TestDisruptorMessageQueue.java | 8 +++--- .../execution/TestSimpleExecutionInSpark.java | 2 +- .../hbase/TestHBaseQPSResourceAllocator.java | 2 +- .../hudi/table/TestConsistencyGuard.java | 2 +- .../action/rollback/TestRollbackUtils.java | 2 +- .../HoodieSparkWriteableTestTable.java | 2 +- .../DirectMarkerBasedDetectionStrategy.java | 4 +-- .../common/fs/OptimisticConsistencyGuard.java | 2 +- .../hudi/common/model/BootstrapIndexType.java | 2 +- .../common/model/FirstValueAvroPayload.java | 5 ++-- .../common/model/HoodieAvroRecordMerger.java | 6 ++--- .../hudi/common/model/HoodieLogFile.java | 4 +-- .../common/model/HoodieRecordPayload.java | 5 ++-- .../model/PartialUpdateAvroPayload.java | 10 +++---- .../log/HoodieUnMergedLogRecordScanner.java | 2 +- .../util/HoodieCommonKryoRegistrar.java | 5 ++-- .../hudi/common/util/SerializationUtils.java | 3 ++- .../common/util/collection/FlatLists.java | 10 +++---- .../util/queue/DisruptorMessageQueue.java | 2 +- .../util/queue/WaitStrategyFactory.java | 3 ++- .../org/apache/hudi/expression/Literal.java | 1 + .../hudi/io/storage/HoodieFileWriter.java | 4 +-- .../io/storage/HoodieSeekingFileReader.java | 3 ++- .../HoodieMetadataLogRecordReader.java | 2 +- .../org/apache/hudi/metrics/MetricUtils.java | 1 + .../hudi/metrics/m3/M3MetricsReporter.java | 14 +++++----- .../metrics/m3/M3ScopeReporterAdaptor.java | 4 ++- .../java/org/apache/hudi/util/Transient.java | 1 + .../model/TestFirstValueAvroPayload.java | 3 ++- .../model/TestHoodieLSMTimelineManifest.java | 2 +- .../table/timeline/TestLSMTimeline.java | 2 +- .../common/util/TestHoodieRecordUtils.java | 3 +-- .../hudi/common/util/TestRetryHelper.java | 2 +- .../expression/TestPartialBindVisitor.java | 1 + .../quickstart/HoodieFlinkQuickstart.java | 25 ++++++++++-------- .../factory/ContinuousFileSourceFactory.java | 5 ++-- .../utils/QuickstartConfigurations.java | 22 +++++++++------- .../quickstart/TestHoodieFlinkQuickstart.java | 3 ++- .../common/ExampleDataSchemaProvider.java | 3 ++- .../examples/common/IdentityTransformer.java | 1 + .../spark/HoodieDataSourceExample.scala | 1 + .../spark/HoodieMorCompactionJob.scala | 1 + .../apache/hudi/sink/StreamWriteFunction.java | 2 +- .../bucket/BucketStreamWriteFunction.java | 2 +- .../hudi/sink/bulk/sort/SortOperator.java | 3 ++- .../partitioner/BucketIndexPartitioner.java | 2 +- .../hudi/source/ExpressionPredicates.java | 3 ++- .../apache/hudi/table/HoodieTableSource.java | 6 ++--- .../hudi/table/catalog/HoodieHiveCatalog.java | 8 +++--- .../hudi/table/format/RecordIterators.java | 2 +- .../format/mor/MergeOnReadInputFormat.java | 4 +-- .../hudi/util/RowDataCastProjection.java | 1 + .../apache/hudi/util/RowDataProjection.java | 1 + .../org/apache/hudi/util/StreamerUtil.java | 8 +++--- .../hudi/sink/TestWriterWithPartitionTTl.java | 3 ++- ...TTestFlinkConsistentHashingClustering.java | 2 +- .../cluster/ITTestHoodieFlinkClustering.java | 2 +- .../hudi/sink/compact/TestCustomSerDe.java | 17 ++++++------ .../hudi/source/TestExpressionEvaluators.java | 3 +-- .../hudi/source/TestExpressionPredicates.java | 4 +-- .../format/cow/TestBlockLocationSort.java | 9 ++++--- .../apache/hudi/utils/FlinkMiniCluster.java | 2 -- .../apache/hudi/utils/TestClusteringUtil.java | 3 +-- .../utils/TestRowDataToAvroConverters.java | 8 +++--- .../bigquery/HoodieBigQuerySyncClient.java | 2 +- .../bigquery/TestBigQuerySyncToolArgs.java | 2 +- .../TestHoodieBigQuerySyncClient.java | 9 +++---- .../fs/HoodieSerializableFileStatus.java | 2 +- .../table/timeline/TestHoodieInstant.java | 1 + .../testutils/HoodieCommonTestHarness.java | 4 +-- .../util/collection/TestBitCaskDiskMap.java | 2 +- .../collection/TestExternalSpillableMap.java | 2 +- .../util/collection/TestRocksDbDiskMap.java | 2 +- .../hudi/hadoop/HiveHoodieTableFileIndex.java | 2 +- .../HoodieCopyOnWriteTableInputFormat.java | 1 - .../hudi/hadoop/HoodieHFileInputFormat.java | 5 ++-- .../hadoop/HoodieParquetInputFormatBase.java | 3 ++- .../hudi/hadoop/RealtimeFileStatus.java | 5 ++-- .../hive/HoodieCombineRealtimeHiveSplit.java | 8 +++--- .../HoodieRealtimeBootstrapBaseFileSplit.java | 3 ++- .../hadoop/realtime/HoodieRealtimePath.java | 3 ++- .../hudi/hadoop/utils/HiveAvroSerializer.java | 1 - .../hudi/hadoop/utils/HoodieHiveUtils.java | 1 - .../apache/hudi/hadoop/TestAnnotation.java | 4 ++- .../hadoop/utils/TestHiveAvroSerializer.java | 2 +- .../TestConnectTransactionParticipant.java | 2 +- .../hudi/connect/TestHdfsConfiguration.java | 7 ++--- .../java/org/apache/hudi/DataSourceUtils.java | 2 +- ...DatasetBulkInsertCommitActionExecutor.java | 1 + ...DatasetBulkInsertCommitActionExecutor.java | 1 + ...lkInsertOverwriteCommitActionExecutor.java | 1 + .../org/apache/hudi/AvroProjection.scala | 3 ++- .../org/apache/hudi/BucketIndexSupport.scala | 9 ++++--- .../org/apache/hudi/HoodieBaseRelation.scala | 1 + .../apache/hudi/HoodieBootstrapMORRDD.scala | 4 +-- .../hudi/HoodieBootstrapMORRelation.scala | 1 - .../org/apache/hudi/HoodieBootstrapRDD.scala | 1 + .../apache/hudi/HoodieBootstrapRelation.scala | 1 - .../org/apache/hudi/HoodieCatalystUtils.scala | 1 + .../apache/hudi/HoodieCreateRecordUtils.scala | 3 ++- .../hudi/HoodieHadoopFsRelationFactory.scala | 9 +++---- .../hudi/HoodieIncrementalFileIndex.scala | 4 +-- .../apache/hudi/HoodieMergeOnReadRDD.scala | 5 ++-- .../HoodiePartitionCDCFileGroupMapping.scala | 1 - .../HoodiePartitionFileSliceMapping.scala | 3 --- .../org/apache/hudi/HoodieSchemaUtils.scala | 1 + .../apache/hudi/HoodieSparkConfUtils.scala | 1 + .../apache/hudi/HoodieSparkSqlWriter.scala | 4 ++- .../scala/org/apache/hudi/Iterators.scala | 2 ++ .../hudi/MergeOnReadIncrementalRelation.scala | 7 +++-- .../hudi/MergeOnReadSnapshotRelation.scala | 1 - .../apache/hudi/RecordLevelIndexSupport.scala | 1 + .../apache/hudi/SecondaryIndexSupport.scala | 1 + .../apache/hudi/SparkBaseIndexSupport.scala | 1 + .../hudi/SparkHoodieTableFileIndex.scala | 10 +++---- .../InternalRowToJsonStringConverter.scala | 3 ++- .../apache/spark/HoodieHadoopFSUtils.scala | 3 ++- .../datasources/HoodieInMemoryFileIndex.scala | 5 ++-- .../FileFormatUtilsForFileGroupReader.scala | 6 ++--- .../spark/sql/avro/SchemaConverters.scala | 4 +-- .../catalyst/catalog/HoodieCatalogTable.scala | 8 +++--- .../HoodieMultipleBaseFileFormat.scala | 9 +++---- ...ileGroupReaderBasedParquetFileFormat.scala | 4 +-- .../LegacyHoodieParquetFileFormat.scala | 4 +-- .../spark/sql/hudi/DataSkippingUtils.scala | 4 +-- .../spark/sql/hudi/HoodieSqlCommonUtils.scala | 13 +++++----- .../spark/sql/hudi/ProvidesHoodieConfig.scala | 9 ++++--- .../AlterHoodieTableAddColumnsCommand.scala | 6 ++--- .../AlterHoodieTableAddPartitionCommand.scala | 1 - ...AlterHoodieTableDropPartitionCommand.scala | 2 +- .../AlterHoodieTableRenameCommand.scala | 3 +-- .../command/CreateHoodieTableCommand.scala | 15 +++++------ .../hudi/command/DropHoodieTableCommand.scala | 3 +-- ...parkValidateDuplicateKeyRecordMerger.scala | 2 +- .../command/RepairHoodieTableCommand.scala | 7 +++-- .../ShowHoodieTablePartitionsCommand.scala | 1 - .../command/TruncateHoodieTableCommand.scala | 5 ++-- .../command/ValidateDuplicateKeyPayload.scala | 1 - .../hudi/streaming/HoodieMetadataLog.scala | 6 ++--- .../hudi/streaming/HoodieSourceOffset.scala | 1 - .../hudi/streaming/HoodieStreamSource.scala | 11 ++++---- .../TestHoodieInMemoryFileIndex.scala | 3 +-- .../spark/sql/hive/TestHiveClientUtils.scala | 2 +- .../sql/hudi/TestProvidesHoodieConfig.scala | 1 - .../java/org/apache/hudi/QuickstartUtils.java | 7 ++--- .../hudi/payload/AWSDmsAvroPayload.java | 3 ++- .../spark/sql/hudi/DedupeSparkJob.scala | 2 +- .../hudi/HoodieSparkSessionExtension.scala | 3 ++- .../sql/hudi/analysis/HoodieAnalysis.scala | 2 ++ .../HoodiePruneFileSourcePartitions.scala | 1 + .../command/CallProcedureHoodieCommand.scala | 1 + .../CompactionHoodieTableCommand.scala | 1 + .../CompactionShowHoodieTableCommand.scala | 1 + .../CreateHoodieTableAsSelectCommand.scala | 5 ++-- .../command/DeleteHoodieTableCommand.scala | 3 ++- .../sql/hudi/command/IndexCommands.scala | 1 + .../InsertIntoHoodieTableCommand.scala | 1 + .../command/MergeIntoHoodieTableCommand.scala | 4 ++- .../command/UpdateHoodieTableCommand.scala | 3 ++- .../sql/hudi/command/UuidKeyGenerator.scala | 6 +++-- .../command/payload/ExpressionPayload.scala | 8 +++--- .../procedures/ArchiveCommitsProcedure.scala | 1 + .../procedures/CommitsCompareProcedure.scala | 1 + .../procedures/CopyToTableProcedure.scala | 1 + .../procedures/CopyToTempViewProcedure.scala | 1 + .../procedures/DeleteMarkerProcedure.scala | 2 ++ .../procedures/ExportInstantsProcedure.scala | 3 ++- .../HdfsParquetImportProcedure.scala | 2 ++ .../command/procedures/HelpProcedure.scala | 1 + .../procedures/HiveSyncProcedure.scala | 3 ++- .../procedures/HoodieProcedureUtils.scala | 1 + .../hudi/command/procedures/Procedure.scala | 4 --- .../RepairAddpartitionmetaProcedure.scala | 1 + .../RepairCorruptedCleanFilesProcedure.scala | 1 + .../RepairDeduplicateProcedure.scala | 1 + .../RollbackToInstantTimeProcedure.scala | 1 - .../procedures/RunBootstrapProcedure.scala | 2 +- .../procedures/RunCleanProcedure.scala | 2 +- .../procedures/RunClusteringProcedure.scala | 1 + .../procedures/RunCompactionProcedure.scala | 1 + .../command/procedures/RunTTLProcedure.scala | 2 ++ .../ShowArchivedCommitsProcedure.scala | 1 + .../ShowBootstrapMappingProcedure.scala | 1 + .../procedures/ShowClusteringProcedure.scala | 1 + .../procedures/ShowCommitFilesProcedure.scala | 1 + .../ShowCommitPartitionsProcedure.scala | 1 + .../ShowCommitWriteStatsProcedure.scala | 1 + .../procedures/ShowCommitsProcedure.scala | 1 + .../procedures/ShowCompactionProcedure.scala | 1 + .../ShowFileSystemViewProcedure.scala | 2 +- .../ShowFsPathDetailProcedure.scala | 1 - .../ShowHoodieLogFileMetadataProcedure.scala | 3 +-- .../ShowHoodieLogFileRecordsProcedure.scala | 4 ++- ...ShowMetadataTablePartitionsProcedure.scala | 1 - .../ShowTablePropertiesProcedure.scala | 1 + .../procedures/StatsFileSizeProcedure.scala | 1 + .../StatsWriteAmplificationProcedure.scala | 1 + .../UpgradeOrDowngradeProcedure.scala | 5 ++-- .../ValidateHoodieSyncProcedure.scala | 1 + .../sql/parser/HoodieCommonSqlParser.scala | 11 ++++---- .../parser/HoodieSqlCommonAstBuilder.scala | 6 ++--- .../org/apache/hudi/TestQuickstartUtils.java | 1 + .../hudi/keygen/TestComplexKeyGenerator.java | 3 ++- .../TestGlobalDeleteRecordGenerator.java | 3 ++- .../TestNonpartitionedKeyGenerator.java | 3 ++- .../hudi/keygen/TestSimpleKeyGenerator.java | 3 ++- ...stCreateKeyGeneratorByTypeWithFactory.java | 1 + .../testutils/HoodieSparkClientTestBase.java | 1 + .../hudi/HoodieSparkWriterTestBase.scala | 3 ++- .../org/apache/hudi/SparkDatasetMixin.scala | 3 ++- .../apache/hudi/TestAvroConversionUtils.scala | 5 ++-- .../TestAvroSchemaResolutionSupport.scala | 1 + .../apache/hudi/TestDataSkippingUtils.scala | 4 ++- .../apache/hudi/TestDataSourceDefaults.scala | 7 ++--- .../apache/hudi/TestDataSourceOptions.scala | 1 + .../TestGenericRecordAndRowConsistency.scala | 3 ++- .../apache/hudi/TestHoodieParquetBloom.scala | 12 +++------ .../org/apache/hudi/TestHoodieRelations.scala | 3 ++- .../hudi/TestHoodieSparkSqlWriter.scala | 2 +- ...TestHoodieSparkSqlWriterPartitionTTL.scala | 5 ++-- .../hudi/TestHoodieSparkSqlWriterUtc.scala | 3 ++- .../apache/hudi/TestHoodieSparkUtils.scala | 3 ++- .../hudi/TestParquetReaderCompatibility.scala | 14 +++++----- .../model/TestHoodieRecordSerialization.scala | 3 ++- .../TestHoodieFileGroupReaderOnSpark.scala | 8 +++--- ...tSpark35RecordPositionMetadataColumn.scala | 3 ++- .../functional/ColumnStatIndexTestBase.scala | 1 + .../hudi/functional/CommonOptionUtils.scala | 2 +- .../functional/RecordLevelIndexTestBase.scala | 7 +++-- .../functional/SecondaryIndexTestBase.scala | 1 + .../TestBloomFiltersIndexSupport.scala | 2 ++ .../functional/TestBucketIndexSupport.scala | 9 ++++--- .../hudi/functional/TestCOWDataSource.scala | 2 +- .../TestDataSourceForBootstrap.scala | 5 ++-- .../hudi/functional/TestEmptyCommit.scala | 4 +-- .../functional/TestHoodieActiveTimeline.scala | 1 - .../functional/TestLayoutOptimization.scala | 1 - .../functional/TestMORDataSourceStorage.scala | 1 + .../functional/TestMetadataRecordIndex.scala | 4 +-- .../hudi/functional/TestMetricsReporter.scala | 4 +-- .../TestPartialUpdateAvroPayload.scala | 5 ++-- .../TestPartitionStatsIndexWithSql.scala | 1 + .../TestRecordLevelIndexWithSQL.scala | 2 +- .../TestSecondaryIndexPruning.scala | 1 + .../TestSparkDataSourceDAGExecution.scala | 7 +++-- .../functional/TestSparkSqlCoreFlow.scala | 1 + ...estSparkSqlWithTimestampKeyGenerator.scala | 1 + .../hudi/functional/TestSqlStatement.scala | 1 + .../cdc/TestCDCDataFrameSuite.scala | 1 + .../apache/hudi/util/JavaConversions.scala | 3 ++- .../org/apache/hudi/util/TestPathUtils.scala | 1 - .../TestSparkInternalSchemaConverter.scala | 3 ++- .../hudi/benchmark/HoodieBenchmark.scala | 9 +++---- .../hudi/benchmark/HoodieBenchmarkBase.scala | 1 + .../hudi/benchmark/HoodieBenchmarks.scala | 4 ++- .../apache/spark/sql/avro/TestAvroSerDe.scala | 4 +-- .../spark/sql/avro/TestSchemaConverters.scala | 1 + .../benchmark/AvroSerDerBenchmark.scala | 3 ++- .../BoundInMemoryExecutorBenchmark.scala | 3 ++- .../benchmark/CowTableReadBenchmark.scala | 9 ++++--- .../benchmark/LSMTimelineReadBenchmark.scala | 5 ++-- .../ReadAndWriteWithoutAvroBenchmark.scala | 5 ++-- .../SpaceCurveOptimizeBenchmark.scala | 3 ++- .../TestHoodiePruneFileSourcePartitions.scala | 1 + .../command/index/TestFunctionalIndex.scala | 1 + .../hudi/command/index/TestIndexSyntax.scala | 1 + .../command/index/TestSecondaryIndex.scala | 1 + .../common/TestHoodieInternalRowUtils.scala | 5 ++-- .../hudi/common/TestHoodieOptionConfig.scala | 1 + .../TestNestedSchemaPruningOptimization.scala | 1 + .../spark/sql/hudi/common/TestSqlConf.scala | 1 + .../spark/sql/hudi/dml/TestDeleteTable.scala | 1 + .../spark/sql/hudi/dml/TestDropTable.scala | 3 ++- .../dml/TestHoodieTableValuedFunction.scala | 1 + .../spark/sql/hudi/dml/TestInsertTable.scala | 5 ++-- .../hudi/dml/TestMergeIntoLogOnlyTable.scala | 1 + .../sql/hudi/dml/TestMergeIntoTable.scala | 1 + ...tMergeIntoTableWithNonRecordKeyField.scala | 1 + .../spark/sql/hudi/dml/TestRepairTable.scala | 1 + .../sql/hudi/dml/TestShowPartitions.scala | 1 + .../sql/hudi/dml/TestTruncateTable.scala | 1 + .../procedure/TestCallCommandParser.scala | 2 ++ .../procedure/TestClusteringProcedure.scala | 4 ++- .../procedure/TestCompactionProcedure.scala | 3 ++- .../TestCopyToTempViewProcedure.scala | 1 + .../TestShowInvalidParquetProcedure.scala | 1 - .../hudi/spark3/internal/DefaultSource.java | 1 + .../internal/HoodieWriterCommitMessage.java | 1 + .../sql/HoodieSpark3CatalystPlanUtils.scala | 4 +-- .../spark/sql/adapter/BaseSpark3Adapter.scala | 9 +++---- .../Spark3ParquetSchemaEvolutionUtils.scala | 9 +++---- .../Spark3ParsePartitionUtil.scala | 6 ++--- .../sql/HoodieSpark33CatalystPlanUtils.scala | 3 +-- .../spark/sql/adapter/Spark3_3Adapter.scala | 5 ++-- .../spark/sql/avro/AvroDeserializer.scala | 1 + .../spark/sql/avro/AvroSerializer.scala | 11 ++++---- .../org/apache/spark/sql/avro/AvroUtils.scala | 11 ++++---- .../Spark33NestedSchemaPruning.scala | 1 + ...Spark33LegacyHoodieParquetFileFormat.scala | 1 + .../Spark33ResolveHudiAlterTableCommand.scala | 1 + .../hudi/analysis/HoodieSpark33Analysis.scala | 2 +- .../HoodieSpark3_3ExtendedSqlAstBuilder.scala | 6 +++-- .../HoodieSpark3_3ExtendedSqlParser.scala | 5 ++-- .../org/apache/hudi/hive/HiveSyncTool.java | 2 +- .../hive/MultiPartKeysValueExtractor.java | 2 +- .../hudi/hive/transaction/lock/Heartbeat.java | 3 ++- .../hudi/hive/util/FilterGenVisitor.java | 2 +- .../hive/util/PartitionFilterGenerator.java | 8 +++--- .../apache/hudi/hive/TestHiveSyncTool.java | 2 +- .../hive/TestMultiPartKeysValueExtractor.java | 3 ++- .../hive/TestParquet2SparkSchemaUtils.java | 3 ++- .../util/TestPartitionFilterGenerator.java | 1 + .../common/util/Parquet2SparkSchemaUtils.java | 1 + .../handlers/marker/MarkerDirState.java | 2 +- .../apache/hudi/utilities/HoodieCleaner.java | 2 +- .../utilities/HoodieSnapshotExporter.java | 2 +- .../hudi/utilities/schema/SchemaSet.java | 4 +-- .../schema/SimpleSchemaProvider.java | 3 ++- .../ChainedSchemaPostProcessor.java | 2 +- .../hudi/utilities/sources/ORCDFSSource.java | 1 + .../utilities/sources/S3EventsSource.java | 4 +-- .../sources/SnapshotLoadQuerySplitter.java | 1 + .../sources/helpers/AvroConvertor.java | 2 +- .../sources/helpers/QueryRunner.java | 2 +- .../sources/helpers/S3EventsMetaSelector.java | 4 +-- .../callback/TestKafkaCallbackProvider.java | 1 + ...TestErrorTableAwareChainedTransformer.java | 6 ++--- .../TestHoodieSnapshotExporter.java | 2 +- .../utilities/sources/TestAvroDFSSource.java | 2 ++ .../utilities/sources/TestS3EventsSource.java | 2 +- .../TestSnapshotQuerySplitterImpl.java | 2 ++ ...tFunctionalTestHarnessWithHiveSupport.java | 1 + 434 files changed, 826 insertions(+), 612 deletions(-) diff --git a/hudi-aws/src/main/java/org/apache/hudi/aws/cloudwatch/CloudWatchReporter.java b/hudi-aws/src/main/java/org/apache/hudi/aws/cloudwatch/CloudWatchReporter.java index f8fefbc69348..5f09743a73b8 100644 --- a/hudi-aws/src/main/java/org/apache/hudi/aws/cloudwatch/CloudWatchReporter.java +++ b/hudi-aws/src/main/java/org/apache/hudi/aws/cloudwatch/CloudWatchReporter.java @@ -21,12 +21,6 @@ import org.apache.hudi.aws.credentials.HoodieAWSCredentialsProviderFactory; import org.apache.hudi.common.util.Option; -import software.amazon.awssdk.services.cloudwatch.CloudWatchAsyncClient; -import software.amazon.awssdk.services.cloudwatch.model.Dimension; -import software.amazon.awssdk.services.cloudwatch.model.MetricDatum; -import software.amazon.awssdk.services.cloudwatch.model.PutMetricDataRequest; -import software.amazon.awssdk.services.cloudwatch.model.PutMetricDataResponse; -import software.amazon.awssdk.services.cloudwatch.model.StandardUnit; import com.codahale.metrics.Clock; import com.codahale.metrics.Counter; import com.codahale.metrics.Counting; @@ -39,6 +33,12 @@ import com.codahale.metrics.Timer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import software.amazon.awssdk.services.cloudwatch.CloudWatchAsyncClient; +import software.amazon.awssdk.services.cloudwatch.model.Dimension; +import software.amazon.awssdk.services.cloudwatch.model.MetricDatum; +import software.amazon.awssdk.services.cloudwatch.model.PutMetricDataRequest; +import software.amazon.awssdk.services.cloudwatch.model.PutMetricDataResponse; +import software.amazon.awssdk.services.cloudwatch.model.StandardUnit; import java.time.Instant; import java.util.ArrayList; diff --git a/hudi-aws/src/main/java/org/apache/hudi/aws/credentials/HoodieConfigAWSAssumedRoleCredentialsProvider.java b/hudi-aws/src/main/java/org/apache/hudi/aws/credentials/HoodieConfigAWSAssumedRoleCredentialsProvider.java index 0333e973da7c..292f4722da5b 100644 --- a/hudi-aws/src/main/java/org/apache/hudi/aws/credentials/HoodieConfigAWSAssumedRoleCredentialsProvider.java +++ b/hudi-aws/src/main/java/org/apache/hudi/aws/credentials/HoodieConfigAWSAssumedRoleCredentialsProvider.java @@ -20,6 +20,7 @@ import org.apache.hudi.common.util.StringUtils; import org.apache.hudi.config.HoodieAWSConfig; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.awssdk.auth.credentials.AwsCredentials; diff --git a/hudi-aws/src/main/java/org/apache/hudi/aws/credentials/HoodieConfigAWSCredentialsProvider.java b/hudi-aws/src/main/java/org/apache/hudi/aws/credentials/HoodieConfigAWSCredentialsProvider.java index 271149f71b9f..105a4ffbb649 100644 --- a/hudi-aws/src/main/java/org/apache/hudi/aws/credentials/HoodieConfigAWSCredentialsProvider.java +++ b/hudi-aws/src/main/java/org/apache/hudi/aws/credentials/HoodieConfigAWSCredentialsProvider.java @@ -18,15 +18,15 @@ package org.apache.hudi.aws.credentials; -import org.apache.hudi.config.HoodieAWSConfig; import org.apache.hudi.common.util.StringUtils; +import org.apache.hudi.config.HoodieAWSConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.AwsCredentials; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; -import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.AwsSessionCredentials; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.Properties; diff --git a/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java b/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java index e511fa7bf99d..2a293007062b 100644 --- a/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java +++ b/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java @@ -24,12 +24,12 @@ import org.apache.hudi.common.model.HoodieFileFormat; import org.apache.hudi.common.table.TableSchemaResolver; import org.apache.hudi.common.util.CollectionUtils; -import org.apache.hudi.common.util.MapUtils; import org.apache.hudi.common.util.CustomizedThreadFactory; import org.apache.hudi.common.util.HoodieTimer; +import org.apache.hudi.common.util.MapUtils; import org.apache.hudi.common.util.Option; -import org.apache.hudi.hadoop.utils.HoodieInputFormatUtils; import org.apache.hudi.config.GlueCatalogSyncClientConfig; +import org.apache.hudi.hadoop.utils.HoodieInputFormatUtils; import org.apache.hudi.hive.HiveSyncConfig; import org.apache.hudi.sync.common.HoodieSyncClient; import org.apache.hudi.sync.common.model.FieldSchema; @@ -111,9 +111,9 @@ import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_SUPPORT_TIMESTAMP_TYPE; import static org.apache.hudi.hive.util.HiveSchemaUtil.getPartitionKeyType; import static org.apache.hudi.hive.util.HiveSchemaUtil.parquetSchemaToMapSchema; +import static org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_BASE_FILE_FORMAT; import static org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_DATABASE_NAME; import static org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_PARTITION_FIELDS; -import static org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_BASE_FILE_FORMAT; import static org.apache.hudi.sync.common.util.TableUtils.tableId; /** diff --git a/hudi-aws/src/test/java/org/apache/hudi/aws/cloudwatch/TestCloudWatchReporter.java b/hudi-aws/src/test/java/org/apache/hudi/aws/cloudwatch/TestCloudWatchReporter.java index 600dd5786959..800245c4e4d2 100644 --- a/hudi-aws/src/test/java/org/apache/hudi/aws/cloudwatch/TestCloudWatchReporter.java +++ b/hudi-aws/src/test/java/org/apache/hudi/aws/cloudwatch/TestCloudWatchReporter.java @@ -18,11 +18,6 @@ package org.apache.hudi.aws.cloudwatch; -import software.amazon.awssdk.services.cloudwatch.CloudWatchAsyncClient; -import software.amazon.awssdk.services.cloudwatch.model.Dimension; -import software.amazon.awssdk.services.cloudwatch.model.MetricDatum; -import software.amazon.awssdk.services.cloudwatch.model.PutMetricDataRequest; -import software.amazon.awssdk.services.cloudwatch.model.PutMetricDataResponse; import com.codahale.metrics.Clock; import com.codahale.metrics.Counter; import com.codahale.metrics.ExponentiallyDecayingReservoir; @@ -42,6 +37,11 @@ import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; +import software.amazon.awssdk.services.cloudwatch.CloudWatchAsyncClient; +import software.amazon.awssdk.services.cloudwatch.model.Dimension; +import software.amazon.awssdk.services.cloudwatch.model.MetricDatum; +import software.amazon.awssdk.services.cloudwatch.model.PutMetricDataRequest; +import software.amazon.awssdk.services.cloudwatch.model.PutMetricDataResponse; import java.util.List; import java.util.SortedMap; diff --git a/hudi-aws/src/test/java/org/apache/hudi/aws/sync/TestGluePartitionPushdown.java b/hudi-aws/src/test/java/org/apache/hudi/aws/sync/TestGluePartitionPushdown.java index d0fe7bf2922d..9f9203570b3f 100644 --- a/hudi-aws/src/test/java/org/apache/hudi/aws/sync/TestGluePartitionPushdown.java +++ b/hudi-aws/src/test/java/org/apache/hudi/aws/sync/TestGluePartitionPushdown.java @@ -21,6 +21,7 @@ import org.apache.hudi.aws.sync.util.GluePartitionFilterGenerator; import org.apache.hudi.hive.HiveSyncConfig; import org.apache.hudi.sync.common.model.FieldSchema; + import org.junit.jupiter.api.Test; import java.util.ArrayList; diff --git a/hudi-aws/src/test/java/org/apache/hudi/aws/transaction/integ/ITTestDynamoDBBasedLockProvider.java b/hudi-aws/src/test/java/org/apache/hudi/aws/transaction/integ/ITTestDynamoDBBasedLockProvider.java index b874f4f3c3cc..10d8b068279f 100644 --- a/hudi-aws/src/test/java/org/apache/hudi/aws/transaction/integ/ITTestDynamoDBBasedLockProvider.java +++ b/hudi-aws/src/test/java/org/apache/hudi/aws/transaction/integ/ITTestDynamoDBBasedLockProvider.java @@ -18,25 +18,24 @@ package org.apache.hudi.aws.transaction.integ; -import org.junit.jupiter.api.Disabled; -import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; -import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; -import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; -import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.services.dynamodb.DynamoDbClient; -import software.amazon.awssdk.services.dynamodb.model.BillingMode; - import org.apache.hudi.aws.transaction.lock.DynamoDBBasedLockProvider; import org.apache.hudi.common.config.LockConfiguration; import org.apache.hudi.config.DynamoDbBasedLockConfig; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.dynamodb.DynamoDbClient; +import software.amazon.awssdk.services.dynamodb.model.BillingMode; import java.net.URI; -import java.util.UUID; import java.util.Properties; +import java.util.UUID; import java.util.concurrent.TimeUnit; import static org.apache.hudi.common.config.LockConfiguration.LOCK_ACQUIRE_WAIT_TIMEOUT_MS_PROP_KEY; diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/HoodiePrompt.java b/hudi-cli/src/main/java/org/apache/hudi/cli/HoodiePrompt.java index 347b81cdbf49..03e37b5ea9bc 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/HoodiePrompt.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/HoodiePrompt.java @@ -18,13 +18,12 @@ package org.apache.hudi.cli; +import org.jline.utils.AttributedString; import org.springframework.core.Ordered; import org.springframework.core.annotation.Order; import org.springframework.shell.jline.PromptProvider; import org.springframework.stereotype.Component; -import org.jline.utils.AttributedString; - /** * This class deals with displaying prompt on CLI based on the state. */ diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/BootstrapCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/BootstrapCommand.java index c0615793a184..5d2eb0bfb239 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/BootstrapCommand.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/BootstrapCommand.java @@ -30,12 +30,12 @@ import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.exception.HoodieException; import org.apache.hudi.utilities.UtilHelpers; + import org.apache.spark.launcher.SparkLauncher; import org.apache.spark.util.Utils; import org.springframework.shell.standard.ShellComponent; import org.springframework.shell.standard.ShellMethod; import org.springframework.shell.standard.ShellOption; -import scala.collection.JavaConverters; import java.io.IOException; import java.net.URISyntaxException; @@ -45,6 +45,8 @@ import java.util.List; import java.util.stream.Collectors; +import scala.collection.JavaConverters; + /** * CLI command to perform bootstrap action & display bootstrap index. */ diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/CleansCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/CleansCommand.java index 3095e9ca242f..31bd708e4a2e 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/CleansCommand.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/CleansCommand.java @@ -34,12 +34,12 @@ import org.apache.hudi.common.table.timeline.HoodieTimeline; import org.apache.hudi.common.table.timeline.TimelineMetadataUtils; import org.apache.hudi.utilities.UtilHelpers; + import org.apache.spark.launcher.SparkLauncher; import org.apache.spark.util.Utils; import org.springframework.shell.standard.ShellComponent; import org.springframework.shell.standard.ShellMethod; import org.springframework.shell.standard.ShellOption; -import scala.collection.JavaConverters; import java.io.IOException; import java.net.URISyntaxException; @@ -49,6 +49,8 @@ import java.util.Map; import java.util.stream.Collectors; +import scala.collection.JavaConverters; + /** * CLI command to show cleans options. */ diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/ClusteringCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/ClusteringCommand.java index 3898892e9ede..616ea6bb1c2a 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/ClusteringCommand.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/ClusteringCommand.java @@ -24,11 +24,13 @@ import org.apache.hudi.cli.utils.SparkUtil; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.utilities.UtilHelpers; + import org.apache.spark.launcher.SparkLauncher; import org.apache.spark.util.Utils; import org.springframework.shell.standard.ShellComponent; import org.springframework.shell.standard.ShellMethod; import org.springframework.shell.standard.ShellOption; + import scala.collection.JavaConverters; @ShellComponent diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/HoodieSyncValidateCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/HoodieSyncValidateCommand.java index f24e16559719..4e3219d18dbb 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/HoodieSyncValidateCommand.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/HoodieSyncValidateCommand.java @@ -24,6 +24,7 @@ import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.HoodieTimeline; import org.apache.hudi.exception.HoodieException; + import org.springframework.shell.standard.ShellComponent; import org.springframework.shell.standard.ShellMethod; import org.springframework.shell.standard.ShellOption; diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/MarkersCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/MarkersCommand.java index 1bc3f1d05085..c0671c33ff02 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/MarkersCommand.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/MarkersCommand.java @@ -21,6 +21,7 @@ import org.apache.hudi.cli.HoodieCLI; import org.apache.hudi.cli.utils.InputStreamConsumer; import org.apache.hudi.cli.utils.SparkUtil; + import org.apache.spark.launcher.SparkLauncher; import org.springframework.shell.standard.ShellComponent; import org.springframework.shell.standard.ShellMethod; diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RestoresCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RestoresCommand.java index fb6c4b7a66c0..550166ac0fa1 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RestoresCommand.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RestoresCommand.java @@ -29,6 +29,7 @@ import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.TimelineMetadataUtils; import org.apache.hudi.common.util.Option; + import org.springframework.shell.standard.ShellComponent; import org.springframework.shell.standard.ShellMethod; import org.springframework.shell.standard.ShellOption; diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RollbacksCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RollbacksCommand.java index f9c38a3bd3be..db902d3a0478 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RollbacksCommand.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RollbacksCommand.java @@ -33,7 +33,6 @@ import org.apache.hudi.common.util.collection.Pair; import org.apache.spark.launcher.SparkLauncher; - import org.springframework.shell.standard.ShellComponent; import org.springframework.shell.standard.ShellMethod; import org.springframework.shell.standard.ShellOption; diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/SavepointsCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/SavepointsCommand.java index 73aaabfc670b..bb4451f6d3fb 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/SavepointsCommand.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/SavepointsCommand.java @@ -28,6 +28,7 @@ import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.HoodieTimeline; import org.apache.hudi.exception.HoodieException; + import org.apache.spark.launcher.SparkLauncher; import org.springframework.shell.standard.ShellComponent; import org.springframework.shell.standard.ShellMethod; diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/SparkEnvCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/SparkEnvCommand.java index 02778ac2cff6..ae3fdab01fcb 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/SparkEnvCommand.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/SparkEnvCommand.java @@ -19,6 +19,7 @@ package org.apache.hudi.cli.commands; import org.apache.hudi.cli.HoodiePrintHelper; + import org.springframework.shell.standard.ShellComponent; import org.springframework.shell.standard.ShellMethod; import org.springframework.shell.standard.ShellOption; diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/StatsCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/StatsCommand.java index 9f859bf72bfc..ac6c9536a0dc 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/StatsCommand.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/StatsCommand.java @@ -28,9 +28,9 @@ import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.HoodieTimeline; import org.apache.hudi.common.util.NumericUtils; -import org.apache.hudi.storage.StoragePathInfo; -import org.apache.hudi.storage.StoragePath; import org.apache.hudi.storage.HoodieStorage; +import org.apache.hudi.storage.StoragePath; +import org.apache.hudi.storage.StoragePathInfo; import com.codahale.metrics.Histogram; import com.codahale.metrics.Snapshot; diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/TempViewCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/TempViewCommand.java index 3f88532b568e..54566576ce3d 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/TempViewCommand.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/TempViewCommand.java @@ -20,6 +20,7 @@ import org.apache.hudi.cli.HoodieCLI; import org.apache.hudi.exception.HoodieException; + import org.springframework.shell.standard.ShellComponent; import org.springframework.shell.standard.ShellMethod; import org.springframework.shell.standard.ShellOption; diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/TimelineCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/TimelineCommand.java index 8cb6fb72180c..1f4089bf1a83 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/TimelineCommand.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/TimelineCommand.java @@ -32,9 +32,9 @@ import org.apache.hudi.common.table.timeline.TimelineMetadataUtils; import org.apache.hudi.common.util.Option; import org.apache.hudi.metadata.HoodieTableMetadata; -import org.apache.hudi.storage.StoragePathInfo; -import org.apache.hudi.storage.StoragePath; import org.apache.hudi.storage.HoodieStorage; +import org.apache.hudi.storage.StoragePath; +import org.apache.hudi.storage.StoragePathInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/UpgradeOrDowngradeCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/UpgradeOrDowngradeCommand.java index 29e1056d6e80..e121ff808b2c 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/UpgradeOrDowngradeCommand.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/UpgradeOrDowngradeCommand.java @@ -24,6 +24,7 @@ import org.apache.hudi.cli.utils.SparkUtil; import org.apache.hudi.common.table.HoodieTableVersion; import org.apache.hudi.common.util.StringUtils; + import org.apache.spark.launcher.SparkLauncher; import org.springframework.shell.standard.ShellComponent; import org.springframework.shell.standard.ShellMethod; diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/UtilsCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/UtilsCommand.java index 2861b05b44d0..9bacaff3fcbe 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/UtilsCommand.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/UtilsCommand.java @@ -19,6 +19,7 @@ package org.apache.hudi.cli.commands; import org.apache.hudi.common.util.StringUtils; + import org.springframework.shell.standard.ShellComponent; import org.springframework.shell.standard.ShellMethod; import org.springframework.shell.standard.ShellOption; diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/TestSparkUtil.java b/hudi-cli/src/test/java/org/apache/hudi/cli/TestSparkUtil.java index 4dcd15156baf..452b162bcd5f 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/TestSparkUtil.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/TestSparkUtil.java @@ -18,10 +18,10 @@ package org.apache.hudi.cli; -import org.apache.hudi.common.util.Option; import org.apache.hudi.cli.utils.SparkUtil; -import org.apache.spark.SparkConf; +import org.apache.hudi.common.util.Option; +import org.apache.spark.SparkConf; import org.apache.spark.launcher.SparkLauncher; import org.junit.jupiter.api.Test; diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestSparkEnvCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestSparkEnvCommand.java index 09f5bd0576a6..31b6945d45eb 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestSparkEnvCommand.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestSparkEnvCommand.java @@ -20,8 +20,8 @@ import org.apache.hudi.cli.HoodiePrintHelper; import org.apache.hudi.cli.functional.CLIFunctionalTestHarness; - import org.apache.hudi.cli.testutils.ShellEvaluationResultUtil; + import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/testutils/HoodieCLIIntegrationTestBase.java b/hudi-cli/src/test/java/org/apache/hudi/cli/testutils/HoodieCLIIntegrationTestBase.java index 86b618d50229..9d75bd17c3d6 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/testutils/HoodieCLIIntegrationTestBase.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/testutils/HoodieCLIIntegrationTestBase.java @@ -19,6 +19,7 @@ package org.apache.hudi.cli.testutils; import org.apache.hudi.common.model.HoodieTableType; + import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/testutils/HoodieCLIIntegrationTestHarness.java b/hudi-cli/src/test/java/org/apache/hudi/cli/testutils/HoodieCLIIntegrationTestHarness.java index fbd516c050fc..2e0271a07ddc 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/testutils/HoodieCLIIntegrationTestHarness.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/testutils/HoodieCLIIntegrationTestHarness.java @@ -19,6 +19,7 @@ package org.apache.hudi.cli.testutils; import org.apache.hudi.testutils.HoodieSparkClientTestHarness; + import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/heartbeat/HoodieHeartbeatClient.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/heartbeat/HoodieHeartbeatClient.java index 0238f6e7f45f..44f323b2e05f 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/heartbeat/HoodieHeartbeatClient.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/heartbeat/HoodieHeartbeatClient.java @@ -22,8 +22,8 @@ import org.apache.hudi.common.util.ValidationUtils; import org.apache.hudi.exception.HoodieException; import org.apache.hudi.exception.HoodieHeartbeatException; -import org.apache.hudi.storage.StoragePath; import org.apache.hudi.storage.HoodieStorage; +import org.apache.hudi.storage.StoragePath; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/ConcurrentOperation.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/ConcurrentOperation.java index 5c02592d329d..cba0f516e889 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/ConcurrentOperation.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/ConcurrentOperation.java @@ -19,13 +19,13 @@ package org.apache.hudi.client.transaction; import org.apache.hudi.avro.model.HoodieRequestedReplaceMetadata; -import org.apache.hudi.common.table.timeline.MetadataConversionUtils; import org.apache.hudi.common.model.HoodieCommitMetadata; import org.apache.hudi.common.model.HoodieMetadataWrapper; import org.apache.hudi.common.model.HoodieReplaceCommitMetadata; import org.apache.hudi.common.model.WriteOperationType; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.table.timeline.MetadataConversionUtils; import org.apache.hudi.common.util.CommitUtils; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.collection.Pair; diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/LockInfo.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/LockInfo.java index 8d7df08f22f1..7b9a0dee727e 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/LockInfo.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/LockInfo.java @@ -19,9 +19,10 @@ package org.apache.hudi.client.transaction.lock; -import com.fasterxml.jackson.core.JsonProcessingException; import org.apache.hudi.common.util.JsonUtils; +import com.fasterxml.jackson.core.JsonProcessingException; + import java.util.ArrayList; public class LockInfo { diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/utils/LegacyArchivedMetaEntryReader.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/utils/LegacyArchivedMetaEntryReader.java index 8ecdb9dccc8a..26ccc2a456e7 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/utils/LegacyArchivedMetaEntryReader.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/utils/LegacyArchivedMetaEntryReader.java @@ -36,8 +36,8 @@ import org.apache.hudi.common.util.collection.ClosableIterator; import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.exception.HoodieIOException; -import org.apache.hudi.storage.StoragePathInfo; import org.apache.hudi.storage.StoragePath; +import org.apache.hudi.storage.StoragePathInfo; import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.IndexedRecord; diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/execution/CopyOnWriteInsertHandler.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/execution/CopyOnWriteInsertHandler.java index 0191b8f9d3a8..9303a304ebb4 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/execution/CopyOnWriteInsertHandler.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/execution/CopyOnWriteInsertHandler.java @@ -27,6 +27,7 @@ import org.apache.hudi.io.HoodieWriteHandle; import org.apache.hudi.io.WriteHandleFactory; import org.apache.hudi.table.HoodieTable; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/execution/HoodieLazyInsertIterable.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/execution/HoodieLazyInsertIterable.java index 84fea62604a2..5dbd46729265 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/execution/HoodieLazyInsertIterable.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/execution/HoodieLazyInsertIterable.java @@ -18,7 +18,6 @@ package org.apache.hudi.execution; -import org.apache.avro.Schema; import org.apache.hudi.client.WriteStatus; import org.apache.hudi.client.utils.LazyIterableIterator; import org.apache.hudi.common.engine.TaskContextSupplier; @@ -29,6 +28,8 @@ import org.apache.hudi.table.HoodieTable; import org.apache.hudi.util.ExecutorFactory; +import org.apache.avro.Schema; + import java.util.Iterator; import java.util.List; import java.util.function.Function; diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bucket/HoodieSimpleBucketIndex.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bucket/HoodieSimpleBucketIndex.java index 630c267011e0..298403250a23 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bucket/HoodieSimpleBucketIndex.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bucket/HoodieSimpleBucketIndex.java @@ -36,20 +36,19 @@ import org.apache.hudi.storage.StoragePath; import org.apache.hudi.storage.StoragePathInfo; import org.apache.hudi.table.HoodieTable; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; -import java.util.Collections; -import java.util.stream.Collectors; -import java.util.stream.Stream; - import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; - +import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.apache.hudi.index.HoodieIndexUtils.tagAsNewRecordIfNeeded; diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieBootstrapHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieBootstrapHandle.java index e4985907e2e3..185d9976439b 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieBootstrapHandle.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieBootstrapHandle.java @@ -18,14 +18,15 @@ package org.apache.hudi.io; -import org.apache.avro.JsonProperties; -import org.apache.avro.Schema; import org.apache.hudi.common.engine.TaskContextSupplier; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.util.Option; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.table.HoodieTable; +import org.apache.avro.JsonProperties; +import org.apache.avro.Schema; + import java.util.List; import java.util.stream.Collectors; diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieMergeHandleFactory.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieMergeHandleFactory.java index 5c36eb3e8c1d..ac8a08c96ad1 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieMergeHandleFactory.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieMergeHandleFactory.java @@ -26,6 +26,7 @@ import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.keygen.BaseKeyGenerator; import org.apache.hudi.table.HoodieTable; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieSortedMergeHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieSortedMergeHandle.java index ee0ee914e197..b5b0bd6e7be5 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieSortedMergeHandle.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieSortedMergeHandle.java @@ -18,7 +18,6 @@ package org.apache.hudi.io; -import org.apache.avro.Schema; import org.apache.hudi.client.WriteStatus; import org.apache.hudi.common.engine.TaskContextSupplier; import org.apache.hudi.common.model.HoodieBaseFile; @@ -29,6 +28,8 @@ import org.apache.hudi.keygen.BaseKeyGenerator; import org.apache.hudi.table.HoodieTable; +import org.apache.avro.Schema; + import javax.annotation.concurrent.NotThreadSafe; import java.io.IOException; diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieSortedMergeHandleWithChangeLog.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieSortedMergeHandleWithChangeLog.java index 819cfd0754f6..4d16876ff5b4 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieSortedMergeHandleWithChangeLog.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieSortedMergeHandleWithChangeLog.java @@ -30,9 +30,9 @@ import org.apache.avro.Schema; import java.io.IOException; -import java.util.Properties; import java.util.Iterator; import java.util.Map; +import java.util.Properties; /** * A sorted merge handle that supports logging change logs. diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/ComplexAvroKeyGenerator.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/ComplexAvroKeyGenerator.java index 743aef1174a7..a08b4616043b 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/ComplexAvroKeyGenerator.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/ComplexAvroKeyGenerator.java @@ -17,10 +17,11 @@ package org.apache.hudi.keygen; -import org.apache.avro.generic.GenericRecord; import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.keygen.constant.KeyGeneratorOptions; +import org.apache.avro.generic.GenericRecord; + import java.util.Arrays; import java.util.stream.Collectors; diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/GlobalAvroDeleteKeyGenerator.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/GlobalAvroDeleteKeyGenerator.java index 517798e7e7ce..9fb55ae09baa 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/GlobalAvroDeleteKeyGenerator.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/GlobalAvroDeleteKeyGenerator.java @@ -17,9 +17,10 @@ package org.apache.hudi.keygen; -import org.apache.avro.generic.GenericRecord; import org.apache.hudi.common.config.TypedProperties; +import org.apache.avro.generic.GenericRecord; + import java.util.ArrayList; import java.util.List; diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/SimpleAvroKeyGenerator.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/SimpleAvroKeyGenerator.java index 82a137f7cb1a..5546d821adf0 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/SimpleAvroKeyGenerator.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/SimpleAvroKeyGenerator.java @@ -17,11 +17,12 @@ package org.apache.hudi.keygen; -import org.apache.avro.generic.GenericRecord; import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.common.util.Option; import org.apache.hudi.keygen.constant.KeyGeneratorOptions; +import org.apache.avro.generic.GenericRecord; + import java.util.Collections; /** diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/factory/HoodieAvroKeyGeneratorFactory.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/factory/HoodieAvroKeyGeneratorFactory.java index f68e3232753a..d3cfb7899ffb 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/factory/HoodieAvroKeyGeneratorFactory.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/factory/HoodieAvroKeyGeneratorFactory.java @@ -21,11 +21,11 @@ import org.apache.hudi.common.util.StringUtils; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.exception.HoodieKeyGeneratorException; +import org.apache.hudi.keygen.AutoRecordGenWrapperAvroKeyGenerator; import org.apache.hudi.keygen.BaseKeyGenerator; import org.apache.hudi.keygen.ComplexAvroKeyGenerator; import org.apache.hudi.keygen.CustomAvroKeyGenerator; import org.apache.hudi.keygen.GlobalAvroDeleteKeyGenerator; -import org.apache.hudi.keygen.AutoRecordGenWrapperAvroKeyGenerator; import org.apache.hudi.keygen.KeyGenUtils; import org.apache.hudi.keygen.KeyGenerator; import org.apache.hudi.keygen.NonpartitionedAvroKeyGenerator; diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataWriteUtils.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataWriteUtils.java index 3caff83eb05c..1143220e34d6 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataWriteUtils.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataWriteUtils.java @@ -35,11 +35,11 @@ import org.apache.hudi.config.HoodieCompactionConfig; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.config.metrics.HoodieMetricsConfig; +import org.apache.hudi.config.metrics.HoodieMetricsDatadogConfig; import org.apache.hudi.config.metrics.HoodieMetricsGraphiteConfig; import org.apache.hudi.config.metrics.HoodieMetricsJmxConfig; import org.apache.hudi.config.metrics.HoodieMetricsM3Config; import org.apache.hudi.config.metrics.HoodieMetricsPrometheusConfig; -import org.apache.hudi.config.metrics.HoodieMetricsDatadogConfig; import org.apache.hudi.exception.HoodieMetadataException; import org.apache.hudi.table.action.compact.strategy.UnBoundedCompactionStrategy; diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataKeyGenerator.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataKeyGenerator.java index 332be73b14f5..94d5a07a21db 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataKeyGenerator.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataKeyGenerator.java @@ -19,11 +19,12 @@ package org.apache.hudi.metadata; -import org.apache.avro.generic.GenericRecord; import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.keygen.BaseKeyGenerator; import org.apache.hudi.keygen.KeyGenUtils; +import org.apache.avro.generic.GenericRecord; + /** * Custom key generator for the Hoodie table metadata. The metadata table record payload * has an internal schema with a known key field HoodieMetadataPayload.SCHEMA_FIELD_ID_KEY. diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/BucketIndexBulkInsertPartitioner.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/BucketIndexBulkInsertPartitioner.java index df21d034a6aa..df50877a410e 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/BucketIndexBulkInsertPartitioner.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/BucketIndexBulkInsertPartitioner.java @@ -24,6 +24,7 @@ import org.apache.hudi.io.AppendHandleFactory; import org.apache.hudi.io.SingleFileHandleCreateFactory; import org.apache.hudi.io.WriteHandleFactory; + import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/util/ConsistentHashingUpdateStrategyUtils.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/util/ConsistentHashingUpdateStrategyUtils.java index 873badbd1852..9e50c6056a49 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/util/ConsistentHashingUpdateStrategyUtils.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/util/ConsistentHashingUpdateStrategyUtils.java @@ -26,8 +26,8 @@ import org.apache.hudi.common.table.timeline.HoodieTimeline; import org.apache.hudi.common.util.ClusteringUtils; import org.apache.hudi.common.util.Option; -import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.common.util.ValidationUtils; +import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.exception.HoodieException; import org.apache.hudi.index.bucket.ConsistentBucketIdentifier; import org.apache.hudi.index.bucket.ConsistentBucketIndexUtils; diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/SmallFile.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/SmallFile.java index ccea6afa680a..e495d28e10bd 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/SmallFile.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/SmallFile.java @@ -18,9 +18,10 @@ package org.apache.hudi.table.action.commit; -import java.io.Serializable; import org.apache.hudi.common.model.HoodieRecordLocation; +import java.io.Serializable; + /** * Helper class for a small file's location and its actual size on disk. */ diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/CompactionStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/CompactionStrategy.java index ff2dfbd4c1ec..5b2dba7ab7a0 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/CompactionStrategy.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/strategy/CompactionStrategy.java @@ -20,9 +20,9 @@ import org.apache.hudi.avro.model.HoodieCompactionOperation; import org.apache.hudi.avro.model.HoodieCompactionPlan; +import org.apache.hudi.client.utils.FileSliceMetricUtils; import org.apache.hudi.common.model.FileSlice; import org.apache.hudi.common.util.CompactionUtils; -import org.apache.hudi.client.utils.FileSliceMetricUtils; import org.apache.hudi.config.HoodieWriteConfig; import java.io.Serializable; diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/ZeroToOneUpgradeHandler.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/ZeroToOneUpgradeHandler.java index 0037e3b301d8..9fdc707a9090 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/ZeroToOneUpgradeHandler.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/ZeroToOneUpgradeHandler.java @@ -31,8 +31,8 @@ import org.apache.hudi.common.util.Option; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.exception.HoodieRollbackException; -import org.apache.hudi.storage.StoragePathInfo; import org.apache.hudi.storage.StoragePath; +import org.apache.hudi.storage.StoragePathInfo; import org.apache.hudi.table.HoodieTable; import org.apache.hudi.table.action.rollback.BaseRollbackHelper; import org.apache.hudi.table.action.rollback.ListingBasedRollbackStrategy; diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/DummyActiveAction.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/DummyActiveAction.java index 0e9ca44bdd61..66070ee818b8 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/DummyActiveAction.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/DummyActiveAction.java @@ -18,8 +18,8 @@ package org.apache.hudi; -import org.apache.hudi.common.table.timeline.ActiveAction; import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.timeline.ActiveAction; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.util.Option; diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/heartbeat/TestHoodieHeartbeatClient.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/heartbeat/TestHoodieHeartbeatClient.java index c48fcfb8bcd0..7778fa1c677e 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/heartbeat/TestHoodieHeartbeatClient.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/heartbeat/TestHoodieHeartbeatClient.java @@ -19,8 +19,8 @@ package org.apache.hudi.client.heartbeat; import org.apache.hudi.common.testutils.HoodieCommonTestHarness; -import org.apache.hudi.storage.StoragePathInfo; import org.apache.hudi.storage.StoragePath; +import org.apache.hudi.storage.StoragePathInfo; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/keygen/factory/TestCreateAvroKeyGeneratorByTypeWithFactory.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/keygen/factory/TestCreateAvroKeyGeneratorByTypeWithFactory.java index 0c12547fcbdf..f2246261a5f3 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/keygen/factory/TestCreateAvroKeyGeneratorByTypeWithFactory.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/keygen/factory/TestCreateAvroKeyGeneratorByTypeWithFactory.java @@ -31,6 +31,7 @@ import org.apache.hudi.keygen.TimestampBasedAvroKeyGenerator; import org.apache.hudi.keygen.constant.KeyGeneratorOptions; import org.apache.hudi.keygen.constant.KeyGeneratorType; + import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/datadog/TestHoodieMetricsDatadogConfig.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/datadog/TestHoodieMetricsDatadogConfig.java index aa486e9b9524..4d50e4f1938d 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/datadog/TestHoodieMetricsDatadogConfig.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/datadog/TestHoodieMetricsDatadogConfig.java @@ -19,8 +19,8 @@ package org.apache.hudi.metrics.datadog; -import org.apache.hudi.config.metrics.HoodieMetricsDatadogConfig; import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.config.metrics.HoodieMetricsDatadogConfig; import org.junit.jupiter.api.Test; diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/action/TestCleanPlanner.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/action/TestCleanPlanner.java index 519f3ea705d0..179454e2063b 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/action/TestCleanPlanner.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/action/TestCleanPlanner.java @@ -75,7 +75,6 @@ import static org.apache.hudi.table.action.clean.CleanPlanner.SAVEPOINTED_TIMESTAMPS; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.mock; - import static org.mockito.Mockito.when; public class TestCleanPlanner { diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/utils/HoodieWriterClientTestHarness.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/utils/HoodieWriterClientTestHarness.java index a466a615d7c6..809f01b5576b 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/utils/HoodieWriterClientTestHarness.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/utils/HoodieWriterClientTestHarness.java @@ -18,9 +18,6 @@ package org.apache.hudi.utils; -import org.apache.avro.generic.GenericRecord; -import org.apache.hadoop.fs.Path; - import org.apache.hudi.avro.model.HoodieCleanMetadata; import org.apache.hudi.avro.model.HoodieClusteringPlan; import org.apache.hudi.client.BaseHoodieWriteClient; @@ -86,6 +83,9 @@ import org.apache.hudi.table.action.commit.HoodieWriteHelper; import org.apache.hudi.table.marker.WriteMarkersFactory; import org.apache.hudi.testutils.MetadataMergeWriteStatus; + +import org.apache.avro.generic.GenericRecord; +import org.apache.hadoop.fs.Path; import org.jetbrains.annotations.NotNull; import java.io.IOException; diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/utils/TestConcatenatingIterator.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/utils/TestConcatenatingIterator.java index fc591edd761e..7350b9c10e7a 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/utils/TestConcatenatingIterator.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/utils/TestConcatenatingIterator.java @@ -19,6 +19,7 @@ package org.apache.hudi.utils; import org.apache.hudi.client.utils.ConcatenatingIterator; + import org.junit.jupiter.api.Test; import java.util.ArrayList; diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/execution/FlinkLazyInsertIterable.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/execution/FlinkLazyInsertIterable.java index 3a088cc51d21..ac024481c15f 100644 --- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/execution/FlinkLazyInsertIterable.java +++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/execution/FlinkLazyInsertIterable.java @@ -18,7 +18,6 @@ package org.apache.hudi.execution; -import org.apache.avro.Schema; import org.apache.hudi.client.WriteStatus; import org.apache.hudi.common.engine.TaskContextSupplier; import org.apache.hudi.common.model.HoodieRecord; @@ -30,6 +29,8 @@ import org.apache.hudi.table.HoodieTable; import org.apache.hudi.util.ExecutorFactory; +import org.apache.avro.Schema; + import java.util.Iterator; import java.util.List; diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/action/commit/FlinkPartitionTTLActionExecutor.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/action/commit/FlinkPartitionTTLActionExecutor.java index f167fb5a9165..7f525cb05a88 100644 --- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/action/commit/FlinkPartitionTTLActionExecutor.java +++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/action/commit/FlinkPartitionTTLActionExecutor.java @@ -30,6 +30,7 @@ import org.apache.hudi.table.action.HoodieWriteMetadata; import org.apache.hudi.table.action.ttl.strategy.HoodiePartitionTTLStrategyFactory; import org.apache.hudi.table.action.ttl.strategy.PartitionTTLStrategy; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/action/commit/FlinkWriteHelper.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/action/commit/FlinkWriteHelper.java index b6f8541f50ef..f7de794398ab 100644 --- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/action/commit/FlinkWriteHelper.java +++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/action/commit/FlinkWriteHelper.java @@ -18,7 +18,6 @@ package org.apache.hudi.table.action.commit; -import org.apache.avro.Schema; import org.apache.hudi.client.WriteStatus; import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.common.data.HoodieListData; @@ -34,6 +33,8 @@ import org.apache.hudi.table.HoodieTable; import org.apache.hudi.table.action.HoodieWriteMetadata; +import org.apache.avro.Schema; + import java.io.IOException; import java.time.Duration; import java.time.Instant; diff --git a/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/table/action/cluster/strategy/TestFlinkSizeBasedClusteringPlanStrategy.java b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/table/action/cluster/strategy/TestFlinkSizeBasedClusteringPlanStrategy.java index 50a3233bf370..36cb5ebdf900 100644 --- a/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/table/action/cluster/strategy/TestFlinkSizeBasedClusteringPlanStrategy.java +++ b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/table/action/cluster/strategy/TestFlinkSizeBasedClusteringPlanStrategy.java @@ -30,6 +30,7 @@ import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.table.HoodieFlinkCopyOnWriteTable; import org.apache.hudi.table.action.cluster.ClusteringPlanPartitionFilterMode; + import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mock; diff --git a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/execution/JavaLazyInsertIterable.java b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/execution/JavaLazyInsertIterable.java index d3612fbf8686..f3e0432eb111 100644 --- a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/execution/JavaLazyInsertIterable.java +++ b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/execution/JavaLazyInsertIterable.java @@ -18,7 +18,6 @@ package org.apache.hudi.execution; -import org.apache.avro.Schema; import org.apache.hudi.client.WriteStatus; import org.apache.hudi.common.engine.TaskContextSupplier; import org.apache.hudi.common.model.HoodieRecord; @@ -29,6 +28,8 @@ import org.apache.hudi.table.HoodieTable; import org.apache.hudi.util.ExecutorFactory; +import org.apache.avro.Schema; + import java.util.Iterator; import java.util.List; diff --git a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/commit/JavaInsertPreppedCommitActionExecutor.java b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/commit/JavaInsertPreppedCommitActionExecutor.java index 7f3bcc4d8d70..454bc16fd469 100644 --- a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/commit/JavaInsertPreppedCommitActionExecutor.java +++ b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/commit/JavaInsertPreppedCommitActionExecutor.java @@ -24,7 +24,6 @@ import org.apache.hudi.common.model.WriteOperationType; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.table.HoodieTable; - import org.apache.hudi.table.action.HoodieWriteMetadata; import java.util.List; diff --git a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/commit/JavaUpsertPreppedCommitActionExecutor.java b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/commit/JavaUpsertPreppedCommitActionExecutor.java index 23fcf1f9ca7a..bb95b917ca89 100644 --- a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/commit/JavaUpsertPreppedCommitActionExecutor.java +++ b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/commit/JavaUpsertPreppedCommitActionExecutor.java @@ -24,7 +24,6 @@ import org.apache.hudi.common.model.WriteOperationType; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.table.HoodieTable; - import org.apache.hudi.table.action.HoodieWriteMetadata; import java.util.List; diff --git a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/commit/JavaWriteHelper.java b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/commit/JavaWriteHelper.java index beb1d14f329c..1ab944b94feb 100644 --- a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/commit/JavaWriteHelper.java +++ b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/commit/JavaWriteHelper.java @@ -18,7 +18,6 @@ package org.apache.hudi.table.action.commit; -import org.apache.avro.Schema; import org.apache.hudi.client.WriteStatus; import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.common.data.HoodieListData; @@ -31,6 +30,8 @@ import org.apache.hudi.index.HoodieIndex; import org.apache.hudi.table.HoodieTable; +import org.apache.avro.Schema; + import java.io.IOException; import java.util.List; import java.util.Map; diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/ExecutionStrategyUtil.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/ExecutionStrategyUtil.java index 5fd2cb65d69f..9789067e5f38 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/ExecutionStrategyUtil.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/ExecutionStrategyUtil.java @@ -18,19 +18,20 @@ package org.apache.hudi.client.clustering.run.strategy; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.generic.IndexedRecord; -import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieAvroRecord; import org.apache.hudi.common.model.HoodieKey; +import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieRecordPayload; import org.apache.hudi.common.model.RewriteAvroPayload; -import org.apache.hudi.common.model.HoodieAvroRecord; import org.apache.hudi.common.util.Option; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.keygen.BaseKeyGenerator; import org.apache.hudi.keygen.KeyGenUtils; import org.apache.hudi.keygen.factory.HoodieSparkKeyGeneratorFactory; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.generic.IndexedRecord; + public class ExecutionStrategyUtil { /** diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/model/HoodieInternalRow.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/model/HoodieInternalRow.java index 7370ee0f088a..2ca61d2823cb 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/model/HoodieInternalRow.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/model/HoodieInternalRow.java @@ -19,6 +19,7 @@ package org.apache.hudi.client.model; import org.apache.hudi.common.model.HoodieRecord; + import org.apache.spark.sql.catalyst.InternalRow; import org.apache.spark.sql.catalyst.expressions.UnsafeRow; import org.apache.spark.sql.catalyst.util.ArrayData; diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/SparkLazyInsertIterable.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/SparkLazyInsertIterable.java index 1a0dcc09ffc2..98eef39dd008 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/SparkLazyInsertIterable.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/SparkLazyInsertIterable.java @@ -27,9 +27,9 @@ import org.apache.hudi.exception.HoodieException; import org.apache.hudi.io.WriteHandleFactory; import org.apache.hudi.table.HoodieTable; +import org.apache.hudi.util.ExecutorFactory; import org.apache.avro.Schema; -import org.apache.hudi.util.ExecutorFactory; import java.util.Iterator; import java.util.List; diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/RDDBucketIndexPartitioner.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/RDDBucketIndexPartitioner.java index 98a0a1efd5d7..57962d2e697b 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/RDDBucketIndexPartitioner.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/RDDBucketIndexPartitioner.java @@ -18,24 +18,25 @@ package org.apache.hudi.execution.bulkinsert; -import org.apache.avro.Schema; import org.apache.hudi.avro.HoodieAvroUtils; import org.apache.hudi.common.config.SerializableSchema; import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.util.collection.FlatLists; import org.apache.hudi.table.BucketIndexBulkInsertPartitioner; - import org.apache.hudi.table.HoodieTable; + +import org.apache.avro.Schema; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.spark.Partitioner; import org.apache.spark.api.java.JavaRDD; -import scala.Tuple2; import java.io.Serializable; import java.util.Comparator; +import scala.Tuple2; + /** * Abstract of bucket index bulk_insert partitioner */ diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/RDDSimpleBucketBulkInsertPartitioner.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/RDDSimpleBucketBulkInsertPartitioner.java index bbb9eaf4f4e3..8304c1031cb5 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/RDDSimpleBucketBulkInsertPartitioner.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/RDDSimpleBucketBulkInsertPartitioner.java @@ -27,6 +27,7 @@ import org.apache.hudi.index.bucket.BucketIdentifier; import org.apache.hudi.index.bucket.HoodieSimpleBucketIndex; import org.apache.hudi.table.HoodieTable; + import org.apache.spark.Partitioner; import org.apache.spark.api.java.JavaRDD; diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/RDDSpatialCurveSortPartitioner.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/RDDSpatialCurveSortPartitioner.java index 0512042d93f0..bd2897007105 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/RDDSpatialCurveSortPartitioner.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/RDDSpatialCurveSortPartitioner.java @@ -18,22 +18,23 @@ package org.apache.hudi.execution.bulkinsert; -import org.apache.avro.Schema; -import org.apache.avro.generic.GenericRecord; import org.apache.hudi.AvroConversionUtils; import org.apache.hudi.HoodieSparkUtils; import org.apache.hudi.SparkConversionUtils; import org.apache.hudi.client.common.HoodieSparkEngineContext; -import org.apache.hudi.common.model.HoodieSparkRecord; import org.apache.hudi.common.config.SerializableSchema; import org.apache.hudi.common.model.HoodieAvroRecord; import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieRecord.HoodieMetadataField; import org.apache.hudi.common.model.HoodieRecord.HoodieRecordType; +import org.apache.hudi.common.model.HoodieSparkRecord; import org.apache.hudi.common.model.RewriteAvroPayload; import org.apache.hudi.common.util.Option; import org.apache.hudi.config.HoodieClusteringConfig; + +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericRecord; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.HoodieInternalRowUtils; diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/RowSpatialCurveSortPartitioner.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/RowSpatialCurveSortPartitioner.java index 1217477c9d81..9e5b06f47ec6 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/RowSpatialCurveSortPartitioner.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/RowSpatialCurveSortPartitioner.java @@ -20,6 +20,7 @@ import org.apache.hudi.config.HoodieClusteringConfig; import org.apache.hudi.config.HoodieWriteConfig; + import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkFileWriter.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkFileWriter.java index 3c69f0ab5c2e..97ec88ff7c41 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkFileWriter.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkFileWriter.java @@ -18,9 +18,10 @@ package org.apache.hudi.io.storage; -import org.apache.avro.Schema; import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.model.HoodieRecord; + +import org.apache.avro.Schema; import org.apache.spark.sql.catalyst.InternalRow; import java.io.IOException; diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkParquetStreamWriter.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkParquetStreamWriter.java index 5c6f85af290d..4374227d8b20 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkParquetStreamWriter.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkParquetStreamWriter.java @@ -18,14 +18,14 @@ package org.apache.hudi.io.storage; -import java.io.IOException; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.Path; import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.io.storage.row.HoodieRowParquetConfig; import org.apache.hudi.io.storage.row.HoodieRowParquetWriteSupport; import org.apache.hudi.parquet.io.OutputStreamBackedOutputFile; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.Path; import org.apache.parquet.hadoop.ParquetFileWriter; import org.apache.parquet.hadoop.ParquetWriter; import org.apache.parquet.hadoop.api.WriteSupport; @@ -33,6 +33,8 @@ import org.apache.spark.sql.catalyst.InternalRow; import org.apache.spark.unsafe.types.UTF8String; +import java.io.IOException; + public class HoodieSparkParquetStreamWriter implements HoodieSparkFileWriter, AutoCloseable { private final ParquetWriter writer; private final HoodieRowParquetWriteSupport writeSupport; diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowParquetWriteSupport.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowParquetWriteSupport.java index 99102c309223..d05abb5a5a0a 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowParquetWriteSupport.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowParquetWriteSupport.java @@ -18,7 +18,6 @@ package org.apache.hudi.io.storage.row; -import org.apache.hadoop.conf.Configuration; import org.apache.hudi.avro.HoodieBloomFilterWriteSupport; import org.apache.hudi.common.bloom.BloomFilter; import org.apache.hudi.common.config.HoodieConfig; @@ -26,6 +25,7 @@ import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.ReflectionUtils; +import org.apache.hadoop.conf.Configuration; import org.apache.parquet.hadoop.api.WriteSupport; import org.apache.spark.sql.execution.datasources.parquet.ParquetWriteSupport; import org.apache.spark.sql.types.StructType; diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/ComplexKeyGenerator.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/ComplexKeyGenerator.java index d00ca066cedd..11e0a8adbfbe 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/ComplexKeyGenerator.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/ComplexKeyGenerator.java @@ -17,9 +17,10 @@ package org.apache.hudi.keygen; -import org.apache.avro.generic.GenericRecord; import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.keygen.constant.KeyGeneratorOptions; + +import org.apache.avro.generic.GenericRecord; import org.apache.spark.sql.Row; import org.apache.spark.sql.catalyst.InternalRow; import org.apache.spark.sql.types.StructType; diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/SimpleKeyGenerator.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/SimpleKeyGenerator.java index c897d6b657e9..09bf48f2bdfa 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/SimpleKeyGenerator.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/SimpleKeyGenerator.java @@ -18,10 +18,11 @@ package org.apache.hudi.keygen; -import org.apache.avro.generic.GenericRecord; import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.common.util.Option; import org.apache.hudi.keygen.constant.KeyGeneratorOptions; + +import org.apache.avro.generic.GenericRecord; import org.apache.spark.sql.Row; import org.apache.spark.sql.catalyst.InternalRow; import org.apache.spark.sql.types.StructType; diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/SparkKeyGeneratorInterface.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/SparkKeyGeneratorInterface.java index 977ff709bb1a..9f2054f3c2e9 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/SparkKeyGeneratorInterface.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/SparkKeyGeneratorInterface.java @@ -20,6 +20,7 @@ import org.apache.hudi.ApiMaturityLevel; import org.apache.hudi.PublicAPIMethod; + import org.apache.spark.sql.Row; import org.apache.spark.sql.catalyst.InternalRow; import org.apache.spark.sql.types.StructType; diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/TimestampBasedKeyGenerator.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/TimestampBasedKeyGenerator.java index 470af0454857..70d6ed720cc2 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/TimestampBasedKeyGenerator.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/TimestampBasedKeyGenerator.java @@ -18,11 +18,12 @@ package org.apache.hudi.keygen; -import org.apache.avro.generic.GenericRecord; import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.common.util.Option; import org.apache.hudi.exception.HoodieKeyGeneratorException; import org.apache.hudi.keygen.constant.KeyGeneratorOptions; + +import org.apache.avro.generic.GenericRecord; import org.apache.spark.sql.Row; import org.apache.spark.sql.catalyst.InternalRow; import org.apache.spark.sql.types.StructType; diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/UTF8StringPartitionPathFormatter.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/UTF8StringPartitionPathFormatter.java index 10d90a7a5e92..dec07f34eaa6 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/UTF8StringPartitionPathFormatter.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/UTF8StringPartitionPathFormatter.java @@ -18,6 +18,7 @@ package org.apache.hudi.keygen; import org.apache.hudi.common.util.PartitionPathEncodeUtils; + import org.apache.spark.unsafe.types.UTF8String; import java.util.function.Supplier; diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metadata/SparkHoodieMetadataBulkInsertPartitioner.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metadata/SparkHoodieMetadataBulkInsertPartitioner.java index e005f0ef7afc..3b36a7f92e9f 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metadata/SparkHoodieMetadataBulkInsertPartitioner.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metadata/SparkHoodieMetadataBulkInsertPartitioner.java @@ -18,17 +18,19 @@ package org.apache.hudi.metadata; -import java.io.Serializable; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; - import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.util.StringUtils; import org.apache.hudi.common.util.ValidationUtils; import org.apache.hudi.table.BulkInsertPartitioner; + import org.apache.spark.Partitioner; import org.apache.spark.api.java.JavaRDD; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; + import scala.Tuple2; /** diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metrics/DistributedRegistry.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metrics/DistributedRegistry.java index ca01def80366..673211cfa0ae 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metrics/DistributedRegistry.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metrics/DistributedRegistry.java @@ -18,15 +18,16 @@ package org.apache.hudi.metrics; +import org.apache.hudi.common.metrics.Registry; + +import org.apache.spark.api.java.JavaSparkContext; +import org.apache.spark.util.AccumulatorV2; + import java.io.Serializable; import java.util.HashMap; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import org.apache.hudi.common.metrics.Registry; -import org.apache.spark.api.java.JavaSparkContext; -import org.apache.spark.util.AccumulatorV2; - /** * Lightweight Metrics Registry to track Hudi events. */ diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/BucketBulkInsertDataInternalWriterHelper.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/BucketBulkInsertDataInternalWriterHelper.java index e0f4fb450569..b4b1f03473fc 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/BucketBulkInsertDataInternalWriterHelper.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/BucketBulkInsertDataInternalWriterHelper.java @@ -26,6 +26,7 @@ import org.apache.hudi.io.storage.row.HoodieRowCreateHandle; import org.apache.hudi.keygen.constant.KeyGeneratorOptions; import org.apache.hudi.table.HoodieTable; + import org.apache.spark.sql.catalyst.InternalRow; import org.apache.spark.sql.types.StructType; import org.apache.spark.unsafe.types.UTF8String; diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkBucketIndexPartitioner.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkBucketIndexPartitioner.java index 0817d8cb7b1a..3b3b6667b9c2 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkBucketIndexPartitioner.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkBucketIndexPartitioner.java @@ -18,31 +18,31 @@ package org.apache.hudi.table.action.commit; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; - -import org.apache.hudi.common.model.WriteOperationType; -import org.apache.hudi.index.bucket.BucketIdentifier; -import scala.Tuple2; - import org.apache.hudi.common.engine.HoodieEngineContext; import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.model.HoodieRecordLocation; +import org.apache.hudi.common.model.WriteOperationType; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.exception.HoodieException; +import org.apache.hudi.index.bucket.BucketIdentifier; import org.apache.hudi.index.bucket.HoodieBucketIndex; import org.apache.hudi.table.HoodieTable; import org.apache.hudi.table.WorkloadProfile; import org.apache.hudi.table.WorkloadStat; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; + +import scala.Tuple2; + import static org.apache.hudi.common.model.WriteOperationType.INSERT_OVERWRITE; import static org.apache.hudi.common.model.WriteOperationType.INSERT_OVERWRITE_TABLE; diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkHoodiePartitioner.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkHoodiePartitioner.java index 3a74578edf0c..5642af09c834 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkHoodiePartitioner.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkHoodiePartitioner.java @@ -20,6 +20,7 @@ import org.apache.hudi.table.HoodieTable; import org.apache.hudi.table.WorkloadProfile; + import org.apache.spark.Partitioner; /** diff --git a/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/HoodieUnsafeRDD.scala b/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/HoodieUnsafeRDD.scala index 51b03a0024ef..f203ca9b3cf8 100644 --- a/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/HoodieUnsafeRDD.scala +++ b/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/HoodieUnsafeRDD.scala @@ -21,7 +21,6 @@ package org.apache.hudi import org.apache.spark.rdd.RDD import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.{Partition, SparkContext, TaskContext} /** * !!! PLEASE READ CAREFULLY !!! diff --git a/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/SparkConversionUtils.scala b/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/SparkConversionUtils.scala index 799bda2f8e5b..368c49c6c32d 100644 --- a/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/SparkConversionUtils.scala +++ b/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/SparkConversionUtils.scala @@ -19,6 +19,7 @@ package org.apache.hudi import org.apache.hudi.common.model.HoodieRecord + import org.apache.spark.rdd.RDD import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.types.StructType diff --git a/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/SparkFileFormatInternalRowReaderContext.scala b/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/SparkFileFormatInternalRowReaderContext.scala index 92389571c13d..074e41b6fef7 100644 --- a/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/SparkFileFormatInternalRowReaderContext.scala +++ b/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/SparkFileFormatInternalRowReaderContext.scala @@ -19,9 +19,6 @@ package org.apache.hudi -import org.apache.avro.Schema -import org.apache.avro.generic.IndexedRecord -import org.apache.hadoop.conf.Configuration import org.apache.hudi.SparkFileFormatInternalRowReaderContext.{filterIsSafeForBootstrap, getAppliedRequiredSchema} import org.apache.hudi.avro.{AvroSchemaUtils, HoodieAvroUtils} import org.apache.hudi.common.engine.HoodieReaderContext @@ -33,6 +30,10 @@ import org.apache.hudi.common.util.collection.{CachingIterator, ClosableIterator import org.apache.hudi.io.storage.{HoodieSparkFileReaderFactory, HoodieSparkParquetReader} import org.apache.hudi.storage.{HoodieStorage, StorageConfiguration, StoragePath} import org.apache.hudi.util.CloseableInternalRowIterator + +import org.apache.avro.Schema +import org.apache.avro.generic.IndexedRecord +import org.apache.hadoop.conf.Configuration import org.apache.spark.sql.HoodieInternalRowUtils import org.apache.spark.sql.avro.HoodieAvroDeserializer import org.apache.spark.sql.catalyst.InternalRow diff --git a/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/util/CloseableInternalRowIterator.scala b/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/util/CloseableInternalRowIterator.scala index bf71a9c6a415..e759b60bff26 100644 --- a/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/util/CloseableInternalRowIterator.scala +++ b/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/util/CloseableInternalRowIterator.scala @@ -20,6 +20,7 @@ package org.apache.hudi.util import org.apache.hudi.common.util.collection.ClosableIterator + import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.vectorized.ColumnarBatch diff --git a/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/BucketPartitionUtils.scala b/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/BucketPartitionUtils.scala index d5b840d7ffcb..41b091e6ecf4 100644 --- a/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/BucketPartitionUtils.scala +++ b/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/BucketPartitionUtils.scala @@ -20,6 +20,7 @@ package org.apache.spark.sql import org.apache.hudi.common.model.HoodieRecord import org.apache.hudi.index.bucket.BucketIdentifier + import org.apache.spark.Partitioner import org.apache.spark.sql.catalyst.InternalRow diff --git a/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/DataFrameUtil.scala b/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/DataFrameUtil.scala index 11ccc59388eb..729163f1bb4e 100644 --- a/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/DataFrameUtil.scala +++ b/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/DataFrameUtil.scala @@ -19,6 +19,7 @@ package org.apache.spark.sql import org.apache.hudi.SparkAdapterSupport + import org.apache.spark.rdd.RDD import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.execution.LogicalRDD diff --git a/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/HoodieCatalystExpressionUtils.scala b/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/HoodieCatalystExpressionUtils.scala index df55a19db441..7eb52e58daa9 100644 --- a/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/HoodieCatalystExpressionUtils.scala +++ b/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/HoodieCatalystExpressionUtils.scala @@ -18,6 +18,7 @@ package org.apache.spark.sql import org.apache.hudi.SparkAdapterSupport + import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.analysis.{UnresolvedAttribute, UnresolvedFunction} import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder diff --git a/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/HoodieUnsafeUtils.scala b/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/HoodieUnsafeUtils.scala index 138815bc9c84..393250b87d0d 100644 --- a/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/HoodieUnsafeUtils.scala +++ b/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/HoodieUnsafeUtils.scala @@ -19,6 +19,7 @@ package org.apache.spark.sql import org.apache.hudi.{HoodieUnsafeRDD, SparkAdapterSupport} + import org.apache.spark.rdd.RDD import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, LogicalPlan} diff --git a/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/execution/SQLConfInjectingRDD.scala b/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/execution/SQLConfInjectingRDD.scala index 1a44fd1af1e5..31b2305fcf4a 100644 --- a/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/execution/SQLConfInjectingRDD.scala +++ b/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/execution/SQLConfInjectingRDD.scala @@ -18,10 +18,10 @@ package org.apache.spark.sql.execution -import org.apache.spark.{Partition, TaskContext} import org.apache.spark.rdd.RDD import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.{Partition, TaskContext} import scala.reflect.ClassTag diff --git a/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/execution/datasources/SparkParsePartitionUtil.scala b/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/execution/datasources/SparkParsePartitionUtil.scala index 2279e5a13f6f..ecf775f715a8 100644 --- a/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/execution/datasources/SparkParsePartitionUtil.scala +++ b/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/execution/datasources/SparkParsePartitionUtil.scala @@ -17,13 +17,12 @@ package org.apache.spark.sql.execution.datasources -import java.util.TimeZone - import org.apache.hadoop.fs.Path - import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.types.DataType +import java.util.TimeZone + trait SparkParsePartitionUtil extends Serializable { def parsePartition(path: Path, diff --git a/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/SparkParquetReader.scala b/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/SparkParquetReader.scala index 4ed4b0cd1d2f..d1f6826a2e1e 100644 --- a/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/SparkParquetReader.scala +++ b/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/SparkParquetReader.scala @@ -19,8 +19,9 @@ package org.apache.spark.sql.execution.datasources.parquet -import org.apache.hadoop.conf.Configuration import org.apache.hudi.storage.StorageConfiguration + +import org.apache.hadoop.conf.Configuration import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.execution.datasources.PartitionedFile import org.apache.spark.sql.sources.Filter diff --git a/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/hudi/SparkAdapter.scala b/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/hudi/SparkAdapter.scala index 40f64e410291..93e20031f3dc 100644 --- a/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/hudi/SparkAdapter.scala +++ b/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/hudi/SparkAdapter.scala @@ -21,6 +21,7 @@ package org.apache.spark.sql.hudi import org.apache.hudi.client.utils.SparkRowSerDe import org.apache.hudi.common.table.HoodieTableMetaClient import org.apache.hudi.storage.StoragePath + import org.apache.avro.Schema import org.apache.hadoop.conf.Configuration import org.apache.spark.sql._ diff --git a/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/hudi/execution/RangeSample.scala b/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/hudi/execution/RangeSample.scala index 898c8dc82094..d82987133fbb 100644 --- a/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/hudi/execution/RangeSample.scala +++ b/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/hudi/execution/RangeSample.scala @@ -22,6 +22,7 @@ import org.apache.hudi.common.util.BinaryUtil import org.apache.hudi.config.HoodieClusteringConfig import org.apache.hudi.config.HoodieClusteringConfig.LayoutOptimizationStrategy import org.apache.hudi.optimize.HilbertCurveUtils + import org.apache.spark.rdd.{PartitionPruningRDD, RDD} import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.codegen.LazilyGeneratedOrdering diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestSparkHoodieMetadataBulkInsertPartitioner.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestSparkHoodieMetadataBulkInsertPartitioner.java index 140c67d1d8c9..415b367bb6a1 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestSparkHoodieMetadataBulkInsertPartitioner.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestSparkHoodieMetadataBulkInsertPartitioner.java @@ -25,14 +25,10 @@ import org.apache.hudi.metadata.MetadataPartitionType; import org.apache.hudi.metadata.SparkHoodieMetadataBulkInsertPartitioner; import org.apache.hudi.testutils.SparkClientFunctionalTestHarness; + import org.apache.spark.api.java.JavaRDD; import org.junit.jupiter.api.Test; -import scala.Tuple2; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -42,6 +38,11 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; +import scala.Tuple2; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + class TestSparkHoodieMetadataBulkInsertPartitioner extends SparkClientFunctionalTestHarness { @Test public void testPartitioner() { diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/bootstrap/TestUniformBootstrapModeSelector.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/bootstrap/TestUniformBootstrapModeSelector.java index d1425f6fab70..fe1e496032a0 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/bootstrap/TestUniformBootstrapModeSelector.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/bootstrap/TestUniformBootstrapModeSelector.java @@ -24,6 +24,7 @@ import org.apache.hudi.client.bootstrap.selector.UniformBootstrapModeSelector; import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.config.HoodieWriteConfig; + import org.junit.jupiter.api.Test; import java.util.ArrayList; diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/clustering/plan/strategy/TestSparkSizeBasedClusteringPlanStrategy.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/clustering/plan/strategy/TestSparkSizeBasedClusteringPlanStrategy.java index 99cafdceea42..ad9c5b45a0f9 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/clustering/plan/strategy/TestSparkSizeBasedClusteringPlanStrategy.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/clustering/plan/strategy/TestSparkSizeBasedClusteringPlanStrategy.java @@ -29,7 +29,6 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; - import org.mockito.Mock; import java.util.ArrayList; diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/execution/TestBoundedInMemoryExecutorInSpark.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/execution/TestBoundedInMemoryExecutorInSpark.java index cea3107746fe..9035c10a83fe 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/execution/TestBoundedInMemoryExecutorInSpark.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/execution/TestBoundedInMemoryExecutorInSpark.java @@ -19,8 +19,8 @@ package org.apache.hudi.execution; import org.apache.hudi.common.model.HoodieRecord; -import org.apache.hudi.common.testutils.InProcessTimeGenerator; import org.apache.hudi.common.testutils.HoodieTestDataGenerator; +import org.apache.hudi.common.testutils.InProcessTimeGenerator; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.queue.BoundedInMemoryExecutor; import org.apache.hudi.common.util.queue.ExecutorType; diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/execution/TestBoundedInMemoryQueue.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/execution/TestBoundedInMemoryQueue.java index e456ca80eea3..4b5feee8ddb0 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/execution/TestBoundedInMemoryQueue.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/execution/TestBoundedInMemoryQueue.java @@ -20,8 +20,8 @@ import org.apache.hudi.common.model.HoodieAvroRecord; import org.apache.hudi.common.model.HoodieRecord; -import org.apache.hudi.common.testutils.InProcessTimeGenerator; import org.apache.hudi.common.testutils.HoodieTestDataGenerator; +import org.apache.hudi.common.testutils.InProcessTimeGenerator; import org.apache.hudi.common.util.DefaultSizeEstimator; import org.apache.hudi.common.util.FileIOUtils; import org.apache.hudi.common.util.Option; diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/execution/TestDisruptorExecutionInSpark.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/execution/TestDisruptorExecutionInSpark.java index cba0ebafded9..49e25c5db700 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/execution/TestDisruptorExecutionInSpark.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/execution/TestDisruptorExecutionInSpark.java @@ -27,6 +27,7 @@ import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.exception.HoodieException; import org.apache.hudi.testutils.HoodieSparkClientTestHarness; + import org.apache.spark.TaskContext; import org.apache.spark.TaskContext$; import org.junit.jupiter.api.AfterEach; diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/execution/TestDisruptorMessageQueue.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/execution/TestDisruptorMessageQueue.java index 03ec64c33860..95408521219d 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/execution/TestDisruptorMessageQueue.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/execution/TestDisruptorMessageQueue.java @@ -18,11 +18,10 @@ package org.apache.hudi.execution; -import org.apache.avro.generic.IndexedRecord; import org.apache.hudi.common.model.HoodieAvroRecord; import org.apache.hudi.common.model.HoodieRecord; -import org.apache.hudi.common.testutils.InProcessTimeGenerator; import org.apache.hudi.common.testutils.HoodieTestDataGenerator; +import org.apache.hudi.common.testutils.InProcessTimeGenerator; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.common.util.queue.DisruptorExecutor; @@ -36,13 +35,14 @@ import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.exception.HoodieException; import org.apache.hudi.testutils.HoodieSparkClientTestHarness; + +import org.apache.avro.generic.IndexedRecord; import org.apache.spark.TaskContext; import org.apache.spark.TaskContext$; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; -import scala.Tuple2; import java.io.IOException; import java.lang.reflect.Method; @@ -56,6 +56,8 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; +import scala.Tuple2; + import static org.apache.hudi.exception.ExceptionUtil.getRootCause; import static org.apache.hudi.execution.HoodieLazyInsertIterable.getTransformerInternal; import static org.junit.jupiter.api.Assertions.assertEquals; diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/execution/TestSimpleExecutionInSpark.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/execution/TestSimpleExecutionInSpark.java index 53e86da16c54..d92381f334bf 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/execution/TestSimpleExecutionInSpark.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/execution/TestSimpleExecutionInSpark.java @@ -20,8 +20,8 @@ import org.apache.hudi.common.model.HoodieAvroRecord; import org.apache.hudi.common.model.HoodieRecord; -import org.apache.hudi.common.testutils.InProcessTimeGenerator; import org.apache.hudi.common.testutils.HoodieTestDataGenerator; +import org.apache.hudi.common.testutils.InProcessTimeGenerator; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.queue.HoodieConsumer; import org.apache.hudi.common.util.queue.SimpleExecutor; diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/index/hbase/TestHBaseQPSResourceAllocator.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/index/hbase/TestHBaseQPSResourceAllocator.java index a60ecee1a96a..eb449473ca9c 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/index/hbase/TestHBaseQPSResourceAllocator.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/index/hbase/TestHBaseQPSResourceAllocator.java @@ -18,12 +18,12 @@ package org.apache.hudi.index.hbase; +import org.apache.hudi.common.config.HoodieStorageConfig; import org.apache.hudi.common.testutils.HoodieTestDataGenerator; import org.apache.hudi.common.util.Option; import org.apache.hudi.config.HoodieCompactionConfig; import org.apache.hudi.config.HoodieHBaseIndexConfig; import org.apache.hudi.config.HoodieIndexConfig; -import org.apache.hudi.common.config.HoodieStorageConfig; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.index.HoodieIndex; diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestConsistencyGuard.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestConsistencyGuard.java index 9461e7819009..b635cb1f975c 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestConsistencyGuard.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestConsistencyGuard.java @@ -18,12 +18,12 @@ package org.apache.hudi.table; +import org.apache.hudi.common.fs.ConsistencyGuard; import org.apache.hudi.common.fs.ConsistencyGuardConfig; import org.apache.hudi.common.fs.FailSafeConsistencyGuard; import org.apache.hudi.common.fs.OptimisticConsistencyGuard; import org.apache.hudi.common.table.HoodieTableConfig; import org.apache.hudi.common.testutils.FileCreateUtils; -import org.apache.hudi.common.fs.ConsistencyGuard; import org.apache.hudi.storage.StoragePath; import org.apache.hudi.testutils.HoodieSparkClientTestHarness; diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/rollback/TestRollbackUtils.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/rollback/TestRollbackUtils.java index bff86d25c9a8..2104f37e78fb 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/rollback/TestRollbackUtils.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/rollback/TestRollbackUtils.java @@ -24,8 +24,8 @@ import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.HoodieTimeline; import org.apache.hudi.common.util.CollectionUtils; -import org.apache.hudi.storage.StoragePathInfo; import org.apache.hudi.storage.StoragePath; +import org.apache.hudi.storage.StoragePathInfo; import org.junit.jupiter.api.Test; diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieSparkWriteableTestTable.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieSparkWriteableTestTable.java index b78b178f416e..5244dec11470 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieSparkWriteableTestTable.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieSparkWriteableTestTable.java @@ -29,8 +29,8 @@ import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.util.Option; import org.apache.hudi.metadata.HoodieTableMetadataWriter; -import org.apache.hudi.storage.StoragePath; import org.apache.hudi.storage.HoodieStorage; +import org.apache.hudi.storage.StoragePath; import org.apache.hudi.table.HoodieTable; import org.apache.avro.Schema; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/conflict/detection/DirectMarkerBasedDetectionStrategy.java b/hudi-common/src/main/java/org/apache/hudi/common/conflict/detection/DirectMarkerBasedDetectionStrategy.java index 40503f4d7139..efaed790088a 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/conflict/detection/DirectMarkerBasedDetectionStrategy.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/conflict/detection/DirectMarkerBasedDetectionStrategy.java @@ -26,9 +26,9 @@ import org.apache.hudi.common.util.MarkerUtils; import org.apache.hudi.common.util.StringUtils; import org.apache.hudi.exception.HoodieIOException; -import org.apache.hudi.storage.StoragePathInfo; -import org.apache.hudi.storage.StoragePath; import org.apache.hudi.storage.HoodieStorage; +import org.apache.hudi.storage.StoragePath; +import org.apache.hudi.storage.StoragePathInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/fs/OptimisticConsistencyGuard.java b/hudi-common/src/main/java/org/apache/hudi/common/fs/OptimisticConsistencyGuard.java index dfe7047999f1..dfc58c6dd11a 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/fs/OptimisticConsistencyGuard.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/fs/OptimisticConsistencyGuard.java @@ -18,8 +18,8 @@ package org.apache.hudi.common.fs; -import org.apache.hudi.storage.StoragePath; import org.apache.hudi.storage.HoodieStorage; +import org.apache.hudi.storage.StoragePath; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/BootstrapIndexType.java b/hudi-common/src/main/java/org/apache/hudi/common/model/BootstrapIndexType.java index a21a2a1c6982..ed5a5502e3fc 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/model/BootstrapIndexType.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/BootstrapIndexType.java @@ -19,8 +19,8 @@ package org.apache.hudi.common.model; -import org.apache.hudi.common.bootstrap.index.hfile.HFileBootstrapIndex; import org.apache.hudi.common.bootstrap.index.NoOpBootstrapIndex; +import org.apache.hudi.common.bootstrap.index.hfile.HFileBootstrapIndex; import org.apache.hudi.common.config.EnumDescription; import org.apache.hudi.common.config.EnumFieldDescription; import org.apache.hudi.common.config.HoodieConfig; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/FirstValueAvroPayload.java b/hudi-common/src/main/java/org/apache/hudi/common/model/FirstValueAvroPayload.java index 33da44e3bccd..d3f9c430c391 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/model/FirstValueAvroPayload.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/FirstValueAvroPayload.java @@ -18,13 +18,14 @@ package org.apache.hudi.common.model; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.generic.IndexedRecord; import org.apache.hudi.avro.HoodieAvroUtils; import org.apache.hudi.common.util.ConfigUtils; import org.apache.hudi.common.util.Option; import org.apache.hudi.keygen.constant.KeyGeneratorOptions; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.generic.IndexedRecord; + import java.util.Properties; /** diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieAvroRecordMerger.java b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieAvroRecordMerger.java index a7bba4ebd9c0..72d27a221d26 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieAvroRecordMerger.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieAvroRecordMerger.java @@ -18,14 +18,14 @@ package org.apache.hudi.common.model; -import org.apache.avro.Schema; -import org.apache.avro.generic.IndexedRecord; - import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.common.model.HoodieRecord.HoodieRecordType; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.collection.Pair; +import org.apache.avro.Schema; +import org.apache.avro.generic.IndexedRecord; + import java.io.IOException; import java.util.Properties; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieLogFile.java b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieLogFile.java index 4a89ec3a7cdc..f55c5cc90a68 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieLogFile.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieLogFile.java @@ -21,9 +21,9 @@ import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.table.cdc.HoodieCDCUtils; import org.apache.hudi.exception.InvalidHoodiePathException; -import org.apache.hudi.storage.StoragePathInfo; -import org.apache.hudi.storage.StoragePath; import org.apache.hudi.storage.HoodieStorage; +import org.apache.hudi.storage.StoragePath; +import org.apache.hudi.storage.StoragePathInfo; import java.io.Serializable; import java.util.Comparator; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieRecordPayload.java b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieRecordPayload.java index 5885c9702ab4..d4e61da9bbf6 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieRecordPayload.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieRecordPayload.java @@ -18,13 +18,14 @@ package org.apache.hudi.common.model; -import org.apache.avro.Schema; -import org.apache.avro.generic.IndexedRecord; import org.apache.hudi.ApiMaturityLevel; import org.apache.hudi.PublicAPIClass; import org.apache.hudi.PublicAPIMethod; import org.apache.hudi.common.util.Option; +import org.apache.avro.Schema; +import org.apache.avro.generic.IndexedRecord; + import java.io.IOException; import java.io.Serializable; import java.util.Map; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/PartialUpdateAvroPayload.java b/hudi-common/src/main/java/org/apache/hudi/common/model/PartialUpdateAvroPayload.java index 91b66e004e55..98104a5a8361 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/model/PartialUpdateAvroPayload.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/PartialUpdateAvroPayload.java @@ -18,17 +18,17 @@ package org.apache.hudi.common.model; -import org.apache.avro.Schema; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.generic.GenericRecordBuilder; -import org.apache.avro.generic.IndexedRecord; - import org.apache.hudi.avro.HoodieAvroUtils; import org.apache.hudi.common.util.ConfigUtils; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.ReflectionUtils; import org.apache.hudi.common.util.StringUtils; import org.apache.hudi.keygen.constant.KeyGeneratorOptions; + +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.generic.GenericRecordBuilder; +import org.apache.avro.generic.IndexedRecord; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieUnMergedLogRecordScanner.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieUnMergedLogRecordScanner.java index 2b70ae579bbc..1ce3dea58cbf 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieUnMergedLogRecordScanner.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieUnMergedLogRecordScanner.java @@ -29,9 +29,9 @@ import org.apache.hudi.common.util.ValidationUtils; import org.apache.hudi.internal.schema.InternalSchema; import org.apache.hudi.storage.HoodieStorage; +import org.apache.hudi.storage.StoragePath; import org.apache.avro.Schema; -import org.apache.hudi.storage.StoragePath; import java.util.List; import java.util.stream.Collectors; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/HoodieCommonKryoRegistrar.java b/hudi-common/src/main/java/org/apache/hudi/common/util/HoodieCommonKryoRegistrar.java index 42a16b7723a5..e8c88873d034 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/HoodieCommonKryoRegistrar.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/HoodieCommonKryoRegistrar.java @@ -18,7 +18,6 @@ package org.apache.hudi.common.util; -import com.esotericsoftware.kryo.Kryo; import org.apache.hudi.common.HoodieJsonPayload; import org.apache.hudi.common.model.AWSDmsAvroPayload; import org.apache.hudi.common.model.DefaultHoodieRecordPayload; @@ -27,9 +26,9 @@ import org.apache.hudi.common.model.HoodieAvroPayload; import org.apache.hudi.common.model.HoodieAvroRecord; import org.apache.hudi.common.model.HoodieEmptyRecord; +import org.apache.hudi.common.model.HoodieRecordDelegate; import org.apache.hudi.common.model.HoodieRecordGlobalLocation; import org.apache.hudi.common.model.HoodieRecordLocation; -import org.apache.hudi.common.model.HoodieRecordDelegate; import org.apache.hudi.common.model.OverwriteNonDefaultsWithLatestAvroPayload; import org.apache.hudi.common.model.OverwriteWithLatestAvroPayload; import org.apache.hudi.common.model.PartialUpdateAvroPayload; @@ -38,6 +37,8 @@ import org.apache.hudi.common.model.debezium.PostgresDebeziumAvroPayload; import org.apache.hudi.metadata.HoodieMetadataPayload; +import com.esotericsoftware.kryo.Kryo; + import java.util.Arrays; /** diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/SerializationUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/SerializationUtils.java index de5df5c73b76..dbc0c436378c 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/SerializationUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/SerializationUtils.java @@ -18,12 +18,13 @@ package org.apache.hudi.common.util; +import org.apache.hudi.avro.GenericAvroSerializer; + import com.esotericsoftware.kryo.Kryo; import com.esotericsoftware.kryo.Serializer; import com.esotericsoftware.kryo.io.Input; import com.esotericsoftware.kryo.io.Output; import org.apache.avro.generic.GenericData; -import org.apache.hudi.avro.GenericAvroSerializer; import org.apache.avro.util.Utf8; import org.objenesis.strategy.StdInstantiatorStrategy; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/collection/FlatLists.java b/hudi-common/src/main/java/org/apache/hudi/common/util/collection/FlatLists.java index ece503c0cd2d..645628ef288a 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/collection/FlatLists.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/collection/FlatLists.java @@ -17,17 +17,17 @@ package org.apache.hudi.common.util.collection; +import com.esotericsoftware.kryo.Kryo; +import com.esotericsoftware.kryo.KryoSerializable; +import com.esotericsoftware.kryo.io.Input; +import com.esotericsoftware.kryo.io.Output; + import java.util.AbstractList; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; -import com.esotericsoftware.kryo.Kryo; -import com.esotericsoftware.kryo.KryoSerializable; -import com.esotericsoftware.kryo.io.Input; -import com.esotericsoftware.kryo.io.Output; - /** * Space-efficient, comparable, immutable lists, copied from calcite core. */ diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/queue/DisruptorMessageQueue.java b/hudi-common/src/main/java/org/apache/hudi/common/util/queue/DisruptorMessageQueue.java index ea0efab5386c..1c8ae245cc8a 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/queue/DisruptorMessageQueue.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/queue/DisruptorMessageQueue.java @@ -18,13 +18,13 @@ package org.apache.hudi.common.util.queue; -import com.lmax.disruptor.TimeoutException; import org.apache.hudi.common.util.CustomizedThreadFactory; import org.apache.hudi.common.util.Option; import org.apache.hudi.exception.HoodieException; import com.lmax.disruptor.EventTranslator; import com.lmax.disruptor.RingBuffer; +import com.lmax.disruptor.TimeoutException; import com.lmax.disruptor.WaitStrategy; import com.lmax.disruptor.dsl.Disruptor; import com.lmax.disruptor.dsl.ProducerType; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/queue/WaitStrategyFactory.java b/hudi-common/src/main/java/org/apache/hudi/common/util/queue/WaitStrategyFactory.java index 5c32e7f835ee..a5a66ce7f9c6 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/util/queue/WaitStrategyFactory.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/queue/WaitStrategyFactory.java @@ -18,12 +18,13 @@ package org.apache.hudi.common.util.queue; +import org.apache.hudi.exception.HoodieException; + import com.lmax.disruptor.BlockingWaitStrategy; import com.lmax.disruptor.BusySpinWaitStrategy; import com.lmax.disruptor.SleepingWaitStrategy; import com.lmax.disruptor.WaitStrategy; import com.lmax.disruptor.YieldingWaitStrategy; -import org.apache.hudi.exception.HoodieException; import static org.apache.hudi.common.util.queue.DisruptorWaitStrategyType.BLOCKING_WAIT; diff --git a/hudi-common/src/main/java/org/apache/hudi/expression/Literal.java b/hudi-common/src/main/java/org/apache/hudi/expression/Literal.java index 01fbdb1a1c87..14df7668c5dc 100644 --- a/hudi-common/src/main/java/org/apache/hudi/expression/Literal.java +++ b/hudi-common/src/main/java/org/apache/hudi/expression/Literal.java @@ -22,6 +22,7 @@ import org.apache.hudi.internal.schema.Types; import javax.xml.bind.DatatypeConverter; + import java.math.BigDecimal; import java.nio.ByteBuffer; import java.util.UUID; diff --git a/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieFileWriter.java b/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieFileWriter.java index 4370d7b9c672..7ca159e69be2 100644 --- a/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieFileWriter.java +++ b/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieFileWriter.java @@ -18,11 +18,11 @@ package org.apache.hudi.io.storage; -import org.apache.avro.Schema; - import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.model.HoodieRecord; +import org.apache.avro.Schema; + import java.io.IOException; import java.util.Properties; diff --git a/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieSeekingFileReader.java b/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieSeekingFileReader.java index cfb27370ff66..e7547d2f02cd 100644 --- a/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieSeekingFileReader.java +++ b/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieSeekingFileReader.java @@ -18,10 +18,11 @@ package org.apache.hudi.io.storage; -import org.apache.avro.Schema; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.util.collection.ClosableIterator; +import org.apache.avro.Schema; + import java.io.IOException; import java.io.UnsupportedEncodingException; import java.util.List; diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataLogRecordReader.java b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataLogRecordReader.java index 3d64ebde0d44..cee3b3066e98 100644 --- a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataLogRecordReader.java +++ b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataLogRecordReader.java @@ -28,9 +28,9 @@ import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.collection.ExternalSpillableMap; import org.apache.hudi.storage.HoodieStorage; +import org.apache.hudi.storage.StoragePath; import org.apache.avro.Schema; -import org.apache.hudi.storage.StoragePath; import javax.annotation.concurrent.ThreadSafe; diff --git a/hudi-common/src/main/java/org/apache/hudi/metrics/MetricUtils.java b/hudi-common/src/main/java/org/apache/hudi/metrics/MetricUtils.java index e119760883f1..3411f7a4f6ee 100644 --- a/hudi-common/src/main/java/org/apache/hudi/metrics/MetricUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/metrics/MetricUtils.java @@ -21,6 +21,7 @@ import org.apache.hudi.common.util.StringUtils; import org.apache.hudi.common.util.ValidationUtils; import org.apache.hudi.common.util.collection.Pair; + import java.util.Arrays; import java.util.List; import java.util.Map; diff --git a/hudi-common/src/main/java/org/apache/hudi/metrics/m3/M3MetricsReporter.java b/hudi-common/src/main/java/org/apache/hudi/metrics/m3/M3MetricsReporter.java index 869b721f4d86..ce0925593394 100644 --- a/hudi-common/src/main/java/org/apache/hudi/metrics/m3/M3MetricsReporter.java +++ b/hudi-common/src/main/java/org/apache/hudi/metrics/m3/M3MetricsReporter.java @@ -18,21 +18,23 @@ package org.apache.hudi.metrics.m3; +import org.apache.hudi.config.metrics.HoodieMetricsConfig; +import org.apache.hudi.metrics.MetricsReporter; + import com.codahale.metrics.MetricRegistry; +import com.uber.m3.tally.RootScopeBuilder; +import com.uber.m3.tally.Scope; import com.uber.m3.tally.m3.M3Reporter; import com.uber.m3.util.Duration; import com.uber.m3.util.ImmutableMap; -import com.uber.m3.tally.RootScopeBuilder; -import com.uber.m3.tally.Scope; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.net.InetSocketAddress; import java.util.Arrays; import java.util.HashMap; import java.util.Map; import java.util.concurrent.TimeUnit; -import org.apache.hudi.config.metrics.HoodieMetricsConfig; -import org.apache.hudi.metrics.MetricsReporter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * Implementation of M3 Metrics reporter, which can report metrics to a https://m3db.io/ service diff --git a/hudi-common/src/main/java/org/apache/hudi/metrics/m3/M3ScopeReporterAdaptor.java b/hudi-common/src/main/java/org/apache/hudi/metrics/m3/M3ScopeReporterAdaptor.java index ae66914400b9..2cc8d9cd37fa 100644 --- a/hudi-common/src/main/java/org/apache/hudi/metrics/m3/M3ScopeReporterAdaptor.java +++ b/hudi-common/src/main/java/org/apache/hudi/metrics/m3/M3ScopeReporterAdaptor.java @@ -18,6 +18,8 @@ package org.apache.hudi.metrics.m3; +import org.apache.hudi.common.util.collection.Pair; + import com.codahale.metrics.Counter; import com.codahale.metrics.Gauge; import com.codahale.metrics.Histogram; @@ -29,13 +31,13 @@ import com.codahale.metrics.Snapshot; import com.codahale.metrics.Timer; import com.uber.m3.tally.Scope; + import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.SortedMap; import java.util.concurrent.TimeUnit; -import org.apache.hudi.common.util.collection.Pair; /** * Implementation of com.codahale.metrics.ScheduledReporter, to emit metrics from diff --git a/hudi-common/src/main/java/org/apache/hudi/util/Transient.java b/hudi-common/src/main/java/org/apache/hudi/util/Transient.java index 0d8f6ad6565e..a66aa5f6b108 100644 --- a/hudi-common/src/main/java/org/apache/hudi/util/Transient.java +++ b/hudi-common/src/main/java/org/apache/hudi/util/Transient.java @@ -22,6 +22,7 @@ import org.apache.hudi.common.function.ThrowingConsumer; import javax.annotation.concurrent.ThreadSafe; + import java.io.Serializable; import static org.apache.hudi.common.util.ValidationUtils.checkArgument; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/model/TestFirstValueAvroPayload.java b/hudi-common/src/test/java/org/apache/hudi/common/model/TestFirstValueAvroPayload.java index a0b7eb86b488..de1bcc65d65f 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/model/TestFirstValueAvroPayload.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/model/TestFirstValueAvroPayload.java @@ -18,10 +18,11 @@ package org.apache.hudi.common.model; +import org.apache.hudi.common.testutils.PreCombineTestUtils; + import org.apache.avro.Schema; import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericRecord; -import org.apache.hudi.common.testutils.PreCombineTestUtils; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieLSMTimelineManifest.java b/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieLSMTimelineManifest.java index a5210626f42f..598e98ce901a 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieLSMTimelineManifest.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieLSMTimelineManifest.java @@ -23,8 +23,8 @@ import java.util.Arrays; import java.util.stream.Collectors; -import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; /** * Test cases for {@link HoodieLSMTimelineManifest}. diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/timeline/TestLSMTimeline.java b/hudi-common/src/test/java/org/apache/hudi/common/table/timeline/TestLSMTimeline.java index 743a9f01e264..009c926fa226 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/timeline/TestLSMTimeline.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/timeline/TestLSMTimeline.java @@ -23,8 +23,8 @@ import org.junit.jupiter.api.Test; -import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.jupiter.api.Assertions.assertThrows; /** diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/TestHoodieRecordUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestHoodieRecordUtils.java index f06670cc76d2..8af1ea7b4c94 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/TestHoodieRecordUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestHoodieRecordUtils.java @@ -18,14 +18,13 @@ package org.apache.hudi.common.util; -import org.apache.avro.generic.GenericRecord; - import org.apache.hudi.common.model.DefaultHoodieRecordPayload; import org.apache.hudi.common.model.HoodieAvroRecordMerger; import org.apache.hudi.common.model.HoodieRecordMerger; import org.apache.hudi.common.model.HoodieRecordPayload; import org.apache.hudi.exception.HoodieException; +import org.apache.avro.generic.GenericRecord; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/TestRetryHelper.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestRetryHelper.java index f022afbbaf88..e675aabb6059 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/TestRetryHelper.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestRetryHelper.java @@ -23,8 +23,8 @@ import java.io.IOException; import java.lang.reflect.Method; -import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; /** * Test retry helper. diff --git a/hudi-common/src/test/java/org/apache/hudi/expression/TestPartialBindVisitor.java b/hudi-common/src/test/java/org/apache/hudi/expression/TestPartialBindVisitor.java index c7e757118239..ca7f2db73c11 100644 --- a/hudi-common/src/test/java/org/apache/hudi/expression/TestPartialBindVisitor.java +++ b/hudi-common/src/test/java/org/apache/hudi/expression/TestPartialBindVisitor.java @@ -19,6 +19,7 @@ package org.apache.hudi.expression; import org.apache.hudi.internal.schema.Types; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; diff --git a/hudi-examples/hudi-examples-flink/src/main/java/org/apache/hudi/examples/quickstart/HoodieFlinkQuickstart.java b/hudi-examples/hudi-examples-flink/src/main/java/org/apache/hudi/examples/quickstart/HoodieFlinkQuickstart.java index 3b794bb2caca..a8e45b27a22d 100644 --- a/hudi-examples/hudi-examples-flink/src/main/java/org/apache/hudi/examples/quickstart/HoodieFlinkQuickstart.java +++ b/hudi-examples/hudi-examples-flink/src/main/java/org/apache/hudi/examples/quickstart/HoodieFlinkQuickstart.java @@ -18,12 +18,12 @@ package org.apache.hudi.examples.quickstart; -import static org.apache.hudi.examples.quickstart.utils.QuickstartConfigurations.sql; -import java.util.Collection; -import java.util.List; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; +import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.configuration.FlinkOptions; +import org.apache.hudi.examples.quickstart.factory.CollectSinkTableFactory; +import org.apache.hudi.examples.quickstart.utils.QuickstartConfigurations; + import org.apache.flink.configuration.Configuration; import org.apache.flink.core.execution.JobClient; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; @@ -39,13 +39,16 @@ import org.apache.flink.table.catalog.ResolvedSchema; import org.apache.flink.table.catalog.exceptions.TableNotExistException; import org.apache.flink.types.Row; -import org.apache.hudi.common.model.HoodieTableType; -import org.apache.hudi.config.HoodieWriteConfig; -import org.apache.hudi.configuration.FlinkOptions; -import org.apache.hudi.examples.quickstart.factory.CollectSinkTableFactory; -import org.apache.hudi.examples.quickstart.utils.QuickstartConfigurations; import org.jetbrains.annotations.NotNull; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import static org.apache.hudi.examples.quickstart.utils.QuickstartConfigurations.sql; + public final class HoodieFlinkQuickstart { private EnvironmentSettings settings = null; private TableEnvironment streamTableEnv = null; diff --git a/hudi-examples/hudi-examples-flink/src/main/java/org/apache/hudi/examples/quickstart/factory/ContinuousFileSourceFactory.java b/hudi-examples/hudi-examples-flink/src/main/java/org/apache/hudi/examples/quickstart/factory/ContinuousFileSourceFactory.java index 834fa9f252fd..967fd088f33e 100644 --- a/hudi-examples/hudi-examples-flink/src/main/java/org/apache/hudi/examples/quickstart/factory/ContinuousFileSourceFactory.java +++ b/hudi-examples/hudi-examples-flink/src/main/java/org/apache/hudi/examples/quickstart/factory/ContinuousFileSourceFactory.java @@ -18,6 +18,9 @@ package org.apache.hudi.examples.quickstart.factory; +import org.apache.hudi.configuration.FlinkOptions; +import org.apache.hudi.examples.quickstart.source.ContinuousFileSource; + import org.apache.flink.configuration.ConfigOption; import org.apache.flink.configuration.ConfigOptions; import org.apache.flink.configuration.Configuration; @@ -29,8 +32,6 @@ import java.util.Collections; import java.util.Set; -import org.apache.hudi.configuration.FlinkOptions; -import org.apache.hudi.examples.quickstart.source.ContinuousFileSource; /** * Factory for ContinuousFileSource. diff --git a/hudi-examples/hudi-examples-flink/src/main/java/org/apache/hudi/examples/quickstart/utils/QuickstartConfigurations.java b/hudi-examples/hudi-examples-flink/src/main/java/org/apache/hudi/examples/quickstart/utils/QuickstartConfigurations.java index 01dd7e1c024b..47a5009ade59 100644 --- a/hudi-examples/hudi-examples-flink/src/main/java/org/apache/hudi/examples/quickstart/utils/QuickstartConfigurations.java +++ b/hudi-examples/hudi-examples-flink/src/main/java/org/apache/hudi/examples/quickstart/utils/QuickstartConfigurations.java @@ -18,12 +18,11 @@ package org.apache.hudi.examples.quickstart.utils; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.stream.Collectors; +import org.apache.hudi.configuration.FlinkOptions; +import org.apache.hudi.examples.quickstart.factory.CollectSinkTableFactory; +import org.apache.hudi.examples.quickstart.factory.ContinuousFileSourceFactory; +import org.apache.hudi.streamer.FlinkStreamerConfig; + import org.apache.flink.configuration.ConfigOption; import org.apache.flink.configuration.Configuration; import org.apache.flink.table.api.DataTypes; @@ -32,10 +31,13 @@ import org.apache.flink.table.runtime.typeutils.RowDataSerializer; import org.apache.flink.table.types.DataType; import org.apache.flink.table.types.logical.RowType; -import org.apache.hudi.configuration.FlinkOptions; -import org.apache.hudi.examples.quickstart.factory.CollectSinkTableFactory; -import org.apache.hudi.examples.quickstart.factory.ContinuousFileSourceFactory; -import org.apache.hudi.streamer.FlinkStreamerConfig; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; /** * Configurations for the test. diff --git a/hudi-examples/hudi-examples-flink/src/test/java/org/apache/hudi/examples/quickstart/TestHoodieFlinkQuickstart.java b/hudi-examples/hudi-examples-flink/src/test/java/org/apache/hudi/examples/quickstart/TestHoodieFlinkQuickstart.java index 4a2768119bf8..17eb3fce6cd1 100644 --- a/hudi-examples/hudi-examples-flink/src/test/java/org/apache/hudi/examples/quickstart/TestHoodieFlinkQuickstart.java +++ b/hudi-examples/hudi-examples-flink/src/test/java/org/apache/hudi/examples/quickstart/TestHoodieFlinkQuickstart.java @@ -18,9 +18,10 @@ package org.apache.hudi.examples.quickstart; +import org.apache.hudi.common.model.HoodieTableType; + import org.apache.flink.test.util.AbstractTestBase; import org.apache.flink.types.Row; -import org.apache.hudi.common.model.HoodieTableType; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; diff --git a/hudi-examples/hudi-examples-spark/src/main/java/org/apache/hudi/examples/common/ExampleDataSchemaProvider.java b/hudi-examples/hudi-examples-spark/src/main/java/org/apache/hudi/examples/common/ExampleDataSchemaProvider.java index c974d9ad7331..e19c13967e22 100644 --- a/hudi-examples/hudi-examples-spark/src/main/java/org/apache/hudi/examples/common/ExampleDataSchemaProvider.java +++ b/hudi-examples/hudi-examples-spark/src/main/java/org/apache/hudi/examples/common/ExampleDataSchemaProvider.java @@ -18,9 +18,10 @@ package org.apache.hudi.examples.common; -import org.apache.avro.Schema; import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.utilities.schema.SchemaProvider; + +import org.apache.avro.Schema; import org.apache.spark.api.java.JavaSparkContext; /** diff --git a/hudi-examples/hudi-examples-spark/src/main/java/org/apache/hudi/examples/common/IdentityTransformer.java b/hudi-examples/hudi-examples-spark/src/main/java/org/apache/hudi/examples/common/IdentityTransformer.java index 1f27c9c9f95e..6661f35944df 100644 --- a/hudi-examples/hudi-examples-spark/src/main/java/org/apache/hudi/examples/common/IdentityTransformer.java +++ b/hudi-examples/hudi-examples-spark/src/main/java/org/apache/hudi/examples/common/IdentityTransformer.java @@ -20,6 +20,7 @@ import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.utilities.transform.Transformer; + import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; diff --git a/hudi-examples/hudi-examples-spark/src/main/scala/org/apache/hudi/examples/spark/HoodieDataSourceExample.scala b/hudi-examples/hudi-examples-spark/src/main/scala/org/apache/hudi/examples/spark/HoodieDataSourceExample.scala index 432c7c065310..463cb0e01398 100644 --- a/hudi-examples/hudi-examples-spark/src/main/scala/org/apache/hudi/examples/spark/HoodieDataSourceExample.scala +++ b/hudi-examples/hudi-examples-spark/src/main/scala/org/apache/hudi/examples/spark/HoodieDataSourceExample.scala @@ -24,6 +24,7 @@ import org.apache.hudi.QuickstartUtils.getQuickstartWriteConfigs import org.apache.hudi.common.model.HoodieAvroPayload import org.apache.hudi.config.HoodieWriteConfig.TBL_NAME import org.apache.hudi.examples.common.{HoodieExampleDataGenerator, HoodieExampleSparkUtils} + import org.apache.spark.sql.SaveMode.{Append, Overwrite} import org.apache.spark.sql.SparkSession diff --git a/hudi-examples/hudi-examples-spark/src/main/scala/org/apache/hudi/examples/spark/HoodieMorCompactionJob.scala b/hudi-examples/hudi-examples-spark/src/main/scala/org/apache/hudi/examples/spark/HoodieMorCompactionJob.scala index d9517b2b7531..de9b52b08205 100644 --- a/hudi-examples/hudi-examples-spark/src/main/scala/org/apache/hudi/examples/spark/HoodieMorCompactionJob.scala +++ b/hudi-examples/hudi-examples-spark/src/main/scala/org/apache/hudi/examples/spark/HoodieMorCompactionJob.scala @@ -28,6 +28,7 @@ import org.apache.hudi.common.util.Option import org.apache.hudi.config.HoodieWriteConfig.TBL_NAME import org.apache.hudi.config.{HoodieCompactionConfig, HoodieWriteConfig} import org.apache.hudi.examples.common.{HoodieExampleDataGenerator, HoodieExampleSparkUtils} + import org.apache.spark.sql.SaveMode.{Append, Overwrite} import org.apache.spark.sql.SparkSession diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/StreamWriteFunction.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/StreamWriteFunction.java index f203756867f4..be6e901f1715 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/StreamWriteFunction.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/StreamWriteFunction.java @@ -23,8 +23,8 @@ import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.model.HoodieOperation; import org.apache.hudi.common.model.HoodieRecord; -import org.apache.hudi.common.model.HoodieRecordMerger; import org.apache.hudi.common.model.HoodieRecordLocation; +import org.apache.hudi.common.model.HoodieRecordMerger; import org.apache.hudi.common.model.HoodieRecordPayload; import org.apache.hudi.common.model.WriteOperationType; import org.apache.hudi.common.util.HoodieRecordUtils; diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/bucket/BucketStreamWriteFunction.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/bucket/BucketStreamWriteFunction.java index b36ddfed4e8f..5229cc591fc7 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/bucket/BucketStreamWriteFunction.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/bucket/BucketStreamWriteFunction.java @@ -26,12 +26,12 @@ import org.apache.hudi.configuration.OptionsResolver; import org.apache.hudi.index.bucket.BucketIdentifier; import org.apache.hudi.sink.StreamWriteFunction; +import org.apache.hudi.sink.utils.BucketIndexUtil; import org.apache.flink.configuration.Configuration; import org.apache.flink.runtime.state.FunctionInitializationContext; import org.apache.flink.streaming.api.functions.ProcessFunction; import org.apache.flink.util.Collector; -import org.apache.hudi.sink.utils.BucketIndexUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/bulk/sort/SortOperator.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/bulk/sort/SortOperator.java index 357bc07160d3..5a898760a282 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/bulk/sort/SortOperator.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/bulk/sort/SortOperator.java @@ -18,6 +18,8 @@ package org.apache.hudi.sink.bulk.sort; +import org.apache.hudi.adapter.Utils; + import org.apache.flink.configuration.Configuration; import org.apache.flink.metrics.Gauge; import org.apache.flink.runtime.memory.MemoryManager; @@ -36,7 +38,6 @@ import org.apache.flink.table.runtime.typeutils.BinaryRowDataSerializer; import org.apache.flink.table.runtime.util.StreamRecordCollector; import org.apache.flink.util.MutableObjectIterator; -import org.apache.hudi.adapter.Utils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/BucketIndexPartitioner.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/BucketIndexPartitioner.java index 0ac54e9a4a9f..d80ebf650c90 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/BucketIndexPartitioner.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/BucketIndexPartitioner.java @@ -21,9 +21,9 @@ import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.util.Functions; import org.apache.hudi.index.bucket.BucketIdentifier; +import org.apache.hudi.sink.utils.BucketIndexUtil; import org.apache.flink.api.common.functions.Partitioner; -import org.apache.hudi.sink.utils.BucketIndexUtil; /** * Bucket index input partitioner. diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/ExpressionPredicates.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/ExpressionPredicates.java index 58ee59a81766..d3ee2c59a49c 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/ExpressionPredicates.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/ExpressionPredicates.java @@ -18,6 +18,8 @@ package org.apache.hudi.source; +import org.apache.hudi.util.ImplicitTypeConverter; + import org.apache.flink.table.expressions.CallExpression; import org.apache.flink.table.expressions.Expression; import org.apache.flink.table.expressions.FieldReferenceExpression; @@ -26,7 +28,6 @@ import org.apache.flink.table.functions.BuiltInFunctionDefinitions; import org.apache.flink.table.functions.FunctionDefinition; import org.apache.flink.table.types.logical.LogicalType; -import org.apache.hudi.util.ImplicitTypeConverter; import org.apache.parquet.filter2.predicate.FilterPredicate; import org.apache.parquet.filter2.predicate.Operators; import org.slf4j.Logger; diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/HoodieTableSource.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/HoodieTableSource.java index c0a87d43c9e3..bd941a951229 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/HoodieTableSource.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/HoodieTableSource.java @@ -46,13 +46,13 @@ import org.apache.hudi.source.IncrementalInputSplits; import org.apache.hudi.source.StreamReadMonitoringFunction; import org.apache.hudi.source.StreamReadOperator; +import org.apache.hudi.source.prune.DataPruner; +import org.apache.hudi.source.prune.PartitionPruners; +import org.apache.hudi.source.prune.PrimaryKeyPruners; import org.apache.hudi.source.rebalance.partitioner.StreamReadAppendPartitioner; import org.apache.hudi.source.rebalance.partitioner.StreamReadBucketIndexPartitioner; import org.apache.hudi.source.rebalance.selector.StreamReadAppendKeySelector; import org.apache.hudi.source.rebalance.selector.StreamReadBucketIndexKeySelector; -import org.apache.hudi.source.prune.DataPruner; -import org.apache.hudi.source.prune.PartitionPruners; -import org.apache.hudi.source.prune.PrimaryKeyPruners; import org.apache.hudi.storage.StorageConfiguration; import org.apache.hudi.storage.StoragePath; import org.apache.hudi.storage.StoragePathInfo; diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/catalog/HoodieHiveCatalog.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/catalog/HoodieHiveCatalog.java index 24ba6799995d..1a021e110fde 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/catalog/HoodieHiveCatalog.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/catalog/HoodieHiveCatalog.java @@ -102,6 +102,10 @@ import java.util.List; import java.util.Map; +import static org.apache.flink.table.factories.FactoryUtil.CONNECTOR; +import static org.apache.flink.util.Preconditions.checkArgument; +import static org.apache.flink.util.Preconditions.checkNotNull; +import static org.apache.flink.util.StringUtils.isNullOrWhitespaceOnly; import static org.apache.hudi.adapter.HiveCatalogConstants.ALTER_DATABASE_OP; import static org.apache.hudi.adapter.HiveCatalogConstants.DATABASE_LOCATION_URI; import static org.apache.hudi.adapter.HiveCatalogConstants.DATABASE_OWNER_NAME; @@ -112,10 +116,6 @@ import static org.apache.hudi.table.catalog.TableOptionProperties.COMMENT; import static org.apache.hudi.table.catalog.TableOptionProperties.PK_CONSTRAINT_NAME; import static org.apache.hudi.table.catalog.TableOptionProperties.SPARK_SOURCE_PROVIDER; -import static org.apache.flink.table.factories.FactoryUtil.CONNECTOR; -import static org.apache.flink.util.Preconditions.checkArgument; -import static org.apache.flink.util.Preconditions.checkNotNull; -import static org.apache.flink.util.StringUtils.isNullOrWhitespaceOnly; /** * A catalog implementation for Hoodie based on MetaStore. diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/RecordIterators.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/RecordIterators.java index 711ed4467134..06862005f329 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/RecordIterators.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/RecordIterators.java @@ -18,8 +18,8 @@ package org.apache.hudi.table.format; -import org.apache.hudi.common.util.collection.ClosableIterator; import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.collection.ClosableIterator; import org.apache.hudi.internal.schema.InternalSchema; import org.apache.hudi.source.ExpressionPredicates.Predicate; import org.apache.hudi.table.format.cow.ParquetSplitReaderUtil; diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/mor/MergeOnReadInputFormat.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/mor/MergeOnReadInputFormat.java index 3690fc911d8b..9946eae930c1 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/mor/MergeOnReadInputFormat.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/mor/MergeOnReadInputFormat.java @@ -27,9 +27,9 @@ import org.apache.hudi.common.model.HoodieRecordMerger; import org.apache.hudi.common.table.log.HoodieMergedLogRecordScanner; import org.apache.hudi.common.table.log.InstantRange; -import org.apache.hudi.common.util.collection.ClosableIterator; import org.apache.hudi.common.util.HoodieRecordUtils; import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.collection.ClosableIterator; import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.configuration.FlinkOptions; import org.apache.hudi.configuration.HadoopConfigurations; @@ -68,13 +68,13 @@ import java.io.IOException; import java.util.Arrays; -import java.util.stream.Collectors; import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Set; import java.util.function.Function; +import java.util.stream.Collectors; import java.util.stream.IntStream; import static org.apache.hudi.hadoop.utils.HoodieInputFormatUtils.HOODIE_COMMIT_TIME_COL_POS; diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/RowDataCastProjection.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/RowDataCastProjection.java index 55e85aa1f602..6d27f65b2160 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/RowDataCastProjection.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/RowDataCastProjection.java @@ -23,6 +23,7 @@ import org.apache.flink.table.types.logical.LogicalType; import javax.annotation.Nullable; + import java.util.stream.IntStream; /** diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/RowDataProjection.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/RowDataProjection.java index 2434359c3dc5..967d337ba9f6 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/RowDataProjection.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/RowDataProjection.java @@ -26,6 +26,7 @@ import org.apache.flink.table.types.logical.RowType; import javax.annotation.Nullable; + import java.io.Serializable; import java.util.Arrays; import java.util.List; diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/StreamerUtil.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/StreamerUtil.java index 86dd5ad2074b..4edc3c04bacd 100644 --- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/StreamerUtil.java +++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/StreamerUtil.java @@ -48,15 +48,15 @@ import org.apache.hudi.exception.HoodieIOException; import org.apache.hudi.exception.HoodieValidationException; import org.apache.hudi.hadoop.fs.HadoopFSUtils; -import org.apache.hudi.storage.StoragePathInfo; -import org.apache.hudi.storage.StoragePath; -import org.apache.hudi.storage.HoodieStorage; -import org.apache.hudi.storage.HoodieStorageUtils; import org.apache.hudi.keygen.ComplexAvroKeyGenerator; import org.apache.hudi.keygen.SimpleAvroKeyGenerator; import org.apache.hudi.schema.FilebasedSchemaProvider; import org.apache.hudi.sink.transform.ChainedTransformer; import org.apache.hudi.sink.transform.Transformer; +import org.apache.hudi.storage.HoodieStorage; +import org.apache.hudi.storage.HoodieStorageUtils; +import org.apache.hudi.storage.StoragePath; +import org.apache.hudi.storage.StoragePathInfo; import org.apache.hudi.streamer.FlinkStreamerConfig; import org.apache.avro.Schema; diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestWriterWithPartitionTTl.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestWriterWithPartitionTTl.java index 95e87b410ff8..9bedb32f1394 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestWriterWithPartitionTTl.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestWriterWithPartitionTTl.java @@ -20,7 +20,6 @@ package org.apache.hudi.sink; -import org.apache.flink.configuration.Configuration; import org.apache.hudi.avro.model.HoodieReplaceCommitMetadata; import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; import org.apache.hudi.common.table.timeline.HoodieInstant; @@ -31,6 +30,8 @@ import org.apache.hudi.table.action.ttl.strategy.KeepByTimeStrategy; import org.apache.hudi.util.StreamerUtil; import org.apache.hudi.utils.TestData; + +import org.apache.flink.configuration.Configuration; import org.junit.jupiter.api.Test; import static org.apache.hudi.common.table.timeline.HoodieInstantTimeGenerator.fixInstantTimeCompatibility; diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/cluster/ITTestFlinkConsistentHashingClustering.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/cluster/ITTestFlinkConsistentHashingClustering.java index f2684d698097..a00cceaabf0e 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/cluster/ITTestFlinkConsistentHashingClustering.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/cluster/ITTestFlinkConsistentHashingClustering.java @@ -20,6 +20,7 @@ import org.apache.hudi.avro.model.HoodieClusteringPlan; import org.apache.hudi.client.HoodieFlinkWriteClient; +import org.apache.hudi.client.clustering.plan.strategy.FlinkConsistentBucketClusteringPlanStrategy; import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.model.WriteOperationType; import org.apache.hudi.common.table.HoodieTableMetaClient; @@ -33,7 +34,6 @@ import org.apache.hudi.index.HoodieIndex; import org.apache.hudi.sink.clustering.FlinkClusteringConfig; import org.apache.hudi.table.HoodieFlinkTable; -import org.apache.hudi.client.clustering.plan.strategy.FlinkConsistentBucketClusteringPlanStrategy; import org.apache.hudi.util.CompactionUtil; import org.apache.hudi.util.FlinkWriteClients; import org.apache.hudi.util.StreamerUtil; diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/cluster/ITTestHoodieFlinkClustering.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/cluster/ITTestHoodieFlinkClustering.java index 61d9469bd9bc..dff930e3fbb2 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/cluster/ITTestHoodieFlinkClustering.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/cluster/ITTestHoodieFlinkClustering.java @@ -81,8 +81,8 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; -import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/compact/TestCustomSerDe.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/compact/TestCustomSerDe.java index 89962bf83478..c6fa0e7108d5 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/compact/TestCustomSerDe.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/compact/TestCustomSerDe.java @@ -18,12 +18,6 @@ package org.apache.hudi.sink.compact; -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; - -import org.apache.avro.LogicalTypes; -import org.apache.avro.Schema; -import org.apache.avro.generic.GenericData; -import org.apache.avro.generic.GenericFixed; import org.apache.hudi.common.model.EventTimeAvroPayload; import org.apache.hudi.common.model.HoodieAvroRecord; import org.apache.hudi.common.model.HoodieKey; @@ -31,13 +25,20 @@ import org.apache.hudi.common.testutils.HoodieCommonTestHarness; import org.apache.hudi.common.util.collection.BitCaskDiskMap; import org.apache.hudi.common.util.collection.RocksDbDiskMap; + +import org.apache.avro.LogicalTypes; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericFixed; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; - -import java.io.IOException; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; +import java.io.IOException; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; + /** * Tests for custom SerDe of non-primitive avro types when using Avro versions > 1.10.0. * The avro version used by hudi-flink module is 1.10.0, these tests are placed here so that avro 1.10.0 is used, diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/TestExpressionEvaluators.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/TestExpressionEvaluators.java index 887cbbe8e47d..e3dc0a836c9b 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/TestExpressionEvaluators.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/TestExpressionEvaluators.java @@ -33,7 +33,6 @@ import org.apache.flink.table.functions.FunctionDefinition; import org.apache.flink.table.types.DataType; import org.apache.flink.table.types.logical.RowType; - import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; @@ -44,8 +43,8 @@ import java.util.Map; import java.util.stream.Stream; -import static org.apache.hudi.source.prune.DataPruner.convertColumnStats; import static org.apache.hudi.source.ExpressionEvaluators.fromExpression; +import static org.apache.hudi.source.prune.DataPruner.convertColumnStats; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/TestExpressionPredicates.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/TestExpressionPredicates.java index 869b69a1a2db..c3c4b23bd166 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/TestExpressionPredicates.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/TestExpressionPredicates.java @@ -18,7 +18,6 @@ package org.apache.hudi.source; -import org.apache.flink.table.types.DataType; import org.apache.hudi.source.ExpressionPredicates.And; import org.apache.hudi.source.ExpressionPredicates.Equals; import org.apache.hudi.source.ExpressionPredicates.GreaterThan; @@ -37,6 +36,7 @@ import org.apache.flink.table.expressions.ResolvedExpression; import org.apache.flink.table.expressions.ValueLiteralExpression; import org.apache.flink.table.functions.BuiltInFunctionDefinitions; +import org.apache.flink.table.types.DataType; import org.apache.parquet.filter2.predicate.Operators.Eq; import org.apache.parquet.filter2.predicate.Operators.Gt; import org.apache.parquet.filter2.predicate.Operators.IntColumn; @@ -46,10 +46,10 @@ import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; +import java.math.BigDecimal; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.LocalTime; -import java.math.BigDecimal; import java.util.Arrays; import java.util.Collections; import java.util.List; diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/format/cow/TestBlockLocationSort.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/format/cow/TestBlockLocationSort.java index d868dce4d915..b77d12b724dc 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/format/cow/TestBlockLocationSort.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/format/cow/TestBlockLocationSort.java @@ -18,13 +18,14 @@ package org.apache.hudi.table.format.cow; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.core.IsEqual.equalTo; +import org.apache.hadoop.fs.BlockLocation; +import org.junit.jupiter.api.Test; import java.util.Arrays; import java.util.Comparator; -import org.apache.hadoop.fs.BlockLocation; -import org.junit.jupiter.api.Test; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.core.IsEqual.equalTo; public class TestBlockLocationSort { diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/FlinkMiniCluster.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/FlinkMiniCluster.java index 96d07cd6565b..bfddb43578c6 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/FlinkMiniCluster.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/FlinkMiniCluster.java @@ -22,12 +22,10 @@ import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration; import org.apache.flink.test.util.AbstractTestBase; import org.apache.flink.test.util.MiniClusterWithClientResource; - import org.junit.jupiter.api.extension.AfterAllCallback; import org.junit.jupiter.api.extension.AfterEachCallback; import org.junit.jupiter.api.extension.BeforeAllCallback; import org.junit.jupiter.api.extension.ExtensionContext; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/TestClusteringUtil.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/TestClusteringUtil.java index ca8718289d96..2abd51e3a51e 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/TestClusteringUtil.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/TestClusteringUtil.java @@ -39,12 +39,11 @@ import org.apache.hudi.util.FlinkWriteClients; import org.apache.hudi.util.StreamerUtil; +import org.apache.flink.configuration.Configuration; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -import org.apache.flink.configuration.Configuration; - import java.io.File; import java.io.IOException; import java.util.Collections; diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/TestRowDataToAvroConverters.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/TestRowDataToAvroConverters.java index 0ab0626d0345..471c0686b60f 100644 --- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/TestRowDataToAvroConverters.java +++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/TestRowDataToAvroConverters.java @@ -18,6 +18,9 @@ package org.apache.hudi.utils; +import org.apache.hudi.util.AvroSchemaConverter; +import org.apache.hudi.util.RowDataToAvroConverters; + import org.apache.avro.generic.GenericRecord; import org.apache.flink.formats.common.TimestampFormat; import org.apache.flink.formats.json.JsonToRowDataConverters; @@ -26,9 +29,6 @@ import org.apache.flink.table.api.DataTypes; import org.apache.flink.table.types.DataType; import org.apache.flink.table.types.logical.RowType; -import org.apache.hudi.util.AvroSchemaConverter; -import org.apache.hudi.util.RowDataToAvroConverters; - import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -37,8 +37,8 @@ import java.time.ZoneId; import java.time.format.DateTimeFormatter; -import static org.apache.flink.table.api.DataTypes.ROW; import static org.apache.flink.table.api.DataTypes.FIELD; +import static org.apache.flink.table.api.DataTypes.ROW; import static org.apache.flink.table.api.DataTypes.TIMESTAMP; class TestRowDataToAvroConverters { diff --git a/hudi-gcp/src/main/java/org/apache/hudi/gcp/bigquery/HoodieBigQuerySyncClient.java b/hudi-gcp/src/main/java/org/apache/hudi/gcp/bigquery/HoodieBigQuerySyncClient.java index 7681a2d38360..5143e0af28b8 100644 --- a/hudi-gcp/src/main/java/org/apache/hudi/gcp/bigquery/HoodieBigQuerySyncClient.java +++ b/hudi-gcp/src/main/java/org/apache/hudi/gcp/bigquery/HoodieBigQuerySyncClient.java @@ -40,11 +40,11 @@ import com.google.cloud.bigquery.QueryJobConfiguration; import com.google.cloud.bigquery.Schema; import com.google.cloud.bigquery.StandardSQLTypeName; +import com.google.cloud.bigquery.StandardTableDefinition; import com.google.cloud.bigquery.Table; import com.google.cloud.bigquery.TableId; import com.google.cloud.bigquery.TableInfo; import com.google.cloud.bigquery.ViewDefinition; -import com.google.cloud.bigquery.StandardTableDefinition; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hudi-gcp/src/test/java/org/apache/hudi/gcp/bigquery/TestBigQuerySyncToolArgs.java b/hudi-gcp/src/test/java/org/apache/hudi/gcp/bigquery/TestBigQuerySyncToolArgs.java index bfda76eb7061..a57b574d3aea 100644 --- a/hudi-gcp/src/test/java/org/apache/hudi/gcp/bigquery/TestBigQuerySyncToolArgs.java +++ b/hudi-gcp/src/test/java/org/apache/hudi/gcp/bigquery/TestBigQuerySyncToolArgs.java @@ -29,10 +29,10 @@ import static org.apache.hudi.gcp.bigquery.BigQuerySyncConfig.BIGQUERY_SYNC_PARTITION_FIELDS; import static org.apache.hudi.gcp.bigquery.BigQuerySyncConfig.BIGQUERY_SYNC_PROJECT_ID; import static org.apache.hudi.gcp.bigquery.BigQuerySyncConfig.BIGQUERY_SYNC_REQUIRE_PARTITION_FILTER; -import static org.apache.hudi.gcp.bigquery.BigQuerySyncConfig.BIGQUERY_SYNC_USE_BQ_MANIFEST_FILE; import static org.apache.hudi.gcp.bigquery.BigQuerySyncConfig.BIGQUERY_SYNC_SOURCE_URI; import static org.apache.hudi.gcp.bigquery.BigQuerySyncConfig.BIGQUERY_SYNC_SOURCE_URI_PREFIX; import static org.apache.hudi.gcp.bigquery.BigQuerySyncConfig.BIGQUERY_SYNC_TABLE_NAME; +import static org.apache.hudi.gcp.bigquery.BigQuerySyncConfig.BIGQUERY_SYNC_USE_BQ_MANIFEST_FILE; import static org.apache.hudi.gcp.bigquery.BigQuerySyncConfig.BIGQUERY_SYNC_USE_FILE_LISTING_FROM_METADATA; import static org.junit.jupiter.api.Assertions.assertEquals; diff --git a/hudi-gcp/src/test/java/org/apache/hudi/gcp/bigquery/TestHoodieBigQuerySyncClient.java b/hudi-gcp/src/test/java/org/apache/hudi/gcp/bigquery/TestHoodieBigQuerySyncClient.java index d623f011c634..6394f36225eb 100644 --- a/hudi-gcp/src/test/java/org/apache/hudi/gcp/bigquery/TestHoodieBigQuerySyncClient.java +++ b/hudi-gcp/src/test/java/org/apache/hudi/gcp/bigquery/TestHoodieBigQuerySyncClient.java @@ -45,21 +45,20 @@ import org.junit.jupiter.api.io.TempDir; import org.mockito.ArgumentCaptor; -import java.util.ArrayList; import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Properties; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.ArgumentMatchers.any; -import java.util.Collections; - import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; -import static org.mockito.Mockito.when; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; public class TestHoodieBigQuerySyncClient { private static final String PROJECT_ID = "test_project"; diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop/fs/HoodieSerializableFileStatus.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop/fs/HoodieSerializableFileStatus.java index d9b0d10163c4..64b827d2613e 100644 --- a/hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop/fs/HoodieSerializableFileStatus.java +++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop/fs/HoodieSerializableFileStatus.java @@ -23,8 +23,8 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; -import java.io.Serializable; import java.io.IOException; +import java.io.Serializable; import java.util.Arrays; import java.util.stream.Collectors; diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/timeline/TestHoodieInstant.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/timeline/TestHoodieInstant.java index b5dff87b5a7c..a3f2fc8de956 100644 --- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/timeline/TestHoodieInstant.java +++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/timeline/TestHoodieInstant.java @@ -21,6 +21,7 @@ import org.apache.hudi.common.model.HoodieTimelineTimeZone; import org.apache.hudi.common.testutils.HoodieCommonTestHarness; import org.apache.hudi.common.util.Option; + import org.junit.jupiter.api.Test; import java.io.IOException; diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/HoodieCommonTestHarness.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/HoodieCommonTestHarness.java index 87ce14f5c846..e2d63d692703 100644 --- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/HoodieCommonTestHarness.java +++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/HoodieCommonTestHarness.java @@ -18,7 +18,6 @@ package org.apache.hudi.common.testutils; -import org.apache.hadoop.conf.Configuration; import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.table.HoodieTableConfig; import org.apache.hudi.common.table.HoodieTableMetaClient; @@ -26,9 +25,10 @@ import org.apache.hudi.common.table.view.HoodieTableFileSystemView; import org.apache.hudi.common.table.view.SyncableFileSystemView; import org.apache.hudi.exception.HoodieIOException; - import org.apache.hudi.storage.HoodieStorage; import org.apache.hudi.storage.StorageConfiguration; + +import org.apache.hadoop.conf.Configuration; import org.junit.jupiter.api.io.TempDir; import java.io.IOException; diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/collection/TestBitCaskDiskMap.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/collection/TestBitCaskDiskMap.java index 670facf10863..e6080519f5ee 100755 --- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/collection/TestBitCaskDiskMap.java +++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/collection/TestBitCaskDiskMap.java @@ -24,9 +24,9 @@ import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieRecordPayload; -import org.apache.hudi.common.testutils.InProcessTimeGenerator; import org.apache.hudi.common.testutils.AvroBinaryTestPayload; import org.apache.hudi.common.testutils.HoodieCommonTestHarness; +import org.apache.hudi.common.testutils.InProcessTimeGenerator; import org.apache.hudi.common.testutils.SchemaTestUtil; import org.apache.hudi.common.testutils.SpillableMapTestUtils; import org.apache.hudi.common.util.HoodieRecordSizeEstimator; diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/collection/TestExternalSpillableMap.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/collection/TestExternalSpillableMap.java index ed64ee002878..9d7f29a4fe97 100644 --- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/collection/TestExternalSpillableMap.java +++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/collection/TestExternalSpillableMap.java @@ -25,8 +25,8 @@ import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieRecordLocation; import org.apache.hudi.common.model.HoodieRecordPayload; -import org.apache.hudi.common.testutils.InProcessTimeGenerator; import org.apache.hudi.common.testutils.HoodieCommonTestHarness; +import org.apache.hudi.common.testutils.InProcessTimeGenerator; import org.apache.hudi.common.testutils.SchemaTestUtil; import org.apache.hudi.common.testutils.SpillableMapTestUtils; import org.apache.hudi.common.util.DefaultSizeEstimator; diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/collection/TestRocksDbDiskMap.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/collection/TestRocksDbDiskMap.java index 0cbaaac7a258..69bd193a1a76 100644 --- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/collection/TestRocksDbDiskMap.java +++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/collection/TestRocksDbDiskMap.java @@ -24,8 +24,8 @@ import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieRecordPayload; -import org.apache.hudi.common.testutils.InProcessTimeGenerator; import org.apache.hudi.common.testutils.HoodieCommonTestHarness; +import org.apache.hudi.common.testutils.InProcessTimeGenerator; import org.apache.hudi.common.testutils.SchemaTestUtil; import org.apache.hudi.common.testutils.SpillableMapTestUtils; import org.apache.hudi.common.util.Option; diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HiveHoodieTableFileIndex.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HiveHoodieTableFileIndex.java index 8e446f78681f..90a053a66883 100644 --- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HiveHoodieTableFileIndex.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HiveHoodieTableFileIndex.java @@ -25,8 +25,8 @@ import org.apache.hudi.common.model.HoodieTableQueryType; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.util.Option; -import org.apache.hudi.storage.StoragePathInfo; import org.apache.hudi.storage.StoragePath; +import org.apache.hudi.storage.StoragePathInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieCopyOnWriteTableInputFormat.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieCopyOnWriteTableInputFormat.java index 0e94a1601f8d..90d44b79f86e 100644 --- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieCopyOnWriteTableInputFormat.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieCopyOnWriteTableInputFormat.java @@ -65,7 +65,6 @@ import java.util.stream.Collectors; import static org.apache.hudi.common.config.HoodieMetadataConfig.ENABLE; -import static org.apache.hudi.hadoop.fs.HadoopFSUtils.convertToStoragePath; /** * Base implementation of the Hive's {@link FileInputFormat} allowing for reading of Hudi's diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieHFileInputFormat.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieHFileInputFormat.java index 6eb1663a0d12..cddf7fbb86a8 100644 --- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieHFileInputFormat.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieHFileInputFormat.java @@ -18,6 +18,9 @@ package org.apache.hudi.hadoop; +import org.apache.hudi.common.table.timeline.HoodieDefaultTimeline; +import org.apache.hudi.hadoop.utils.HoodieInputFormatUtils; + import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.ArrayWritable; @@ -26,8 +29,6 @@ import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.RecordReader; import org.apache.hadoop.mapred.Reporter; -import org.apache.hudi.common.table.timeline.HoodieDefaultTimeline; -import org.apache.hudi.hadoop.utils.HoodieInputFormatUtils; import java.io.IOException; diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieParquetInputFormatBase.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieParquetInputFormatBase.java index ed88acacb4d2..22f4b1527ead 100644 --- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieParquetInputFormatBase.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieParquetInputFormatBase.java @@ -18,6 +18,8 @@ package org.apache.hudi.hadoop; +import org.apache.hudi.hadoop.realtime.HoodieMergeOnReadTableInputFormat; + import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; @@ -28,7 +30,6 @@ import org.apache.hadoop.mapred.FileSplit; import org.apache.hadoop.mapred.InputSplit; import org.apache.hadoop.mapred.JobConf; -import org.apache.hudi.hadoop.realtime.HoodieMergeOnReadTableInputFormat; import java.io.IOException; diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/RealtimeFileStatus.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/RealtimeFileStatus.java index 641aa2759ff2..e11dc8d7122d 100644 --- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/RealtimeFileStatus.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/RealtimeFileStatus.java @@ -18,13 +18,14 @@ package org.apache.hudi.hadoop; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.Path; import org.apache.hudi.common.model.HoodieLogFile; import org.apache.hudi.common.util.Option; import org.apache.hudi.hadoop.realtime.HoodieRealtimePath; import org.apache.hudi.hadoop.realtime.HoodieVirtualKeyInfo; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; + import java.io.IOException; import java.util.List; diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/hive/HoodieCombineRealtimeHiveSplit.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/hive/HoodieCombineRealtimeHiveSplit.java index c29e51b8744d..eb71525bcbc6 100644 --- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/hive/HoodieCombineRealtimeHiveSplit.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/hive/HoodieCombineRealtimeHiveSplit.java @@ -18,14 +18,16 @@ package org.apache.hudi.hadoop.hive; -import java.io.IOException; -import java.util.Map; +import org.apache.hudi.hadoop.hive.HoodieCombineHiveInputFormat.CombineHiveInputSplit; + import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.ql.plan.PartitionDesc; import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.lib.CombineFileSplit; -import org.apache.hudi.hadoop.hive.HoodieCombineHiveInputFormat.CombineHiveInputSplit; + +import java.io.IOException; +import java.util.Map; /** * Represents a CombineHiveInputSplit for realtime tables. diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieRealtimeBootstrapBaseFileSplit.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieRealtimeBootstrapBaseFileSplit.java index c7022c98ad3c..f98d3d7eeb44 100644 --- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieRealtimeBootstrapBaseFileSplit.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieRealtimeBootstrapBaseFileSplit.java @@ -18,11 +18,12 @@ package org.apache.hudi.hadoop.realtime; -import org.apache.hadoop.mapred.FileSplit; import org.apache.hudi.common.model.HoodieLogFile; import org.apache.hudi.common.util.Option; import org.apache.hudi.hadoop.BootstrapBaseFileSplit; +import org.apache.hadoop.mapred.FileSplit; + import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieRealtimePath.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieRealtimePath.java index 37b59a962750..8123fe6006d2 100644 --- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieRealtimePath.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieRealtimePath.java @@ -18,11 +18,12 @@ package org.apache.hudi.hadoop.realtime; -import org.apache.hadoop.fs.Path; import org.apache.hudi.common.model.HoodieLogFile; import org.apache.hudi.common.util.Option; import org.apache.hudi.hadoop.PathWithBootstrapFileStatus; +import org.apache.hadoop.fs.Path; + import java.util.List; /** diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HiveAvroSerializer.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HiveAvroSerializer.java index 22116283d121..278fd57d71bd 100644 --- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HiveAvroSerializer.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HiveAvroSerializer.java @@ -49,7 +49,6 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.UnionTypeInfo; import org.apache.hadoop.io.ArrayWritable; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieHiveUtils.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieHiveUtils.java index e257f96e44b2..63ba2f74f6b1 100644 --- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieHiveUtils.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieHiveUtils.java @@ -28,7 +28,6 @@ import org.apache.hadoop.io.Writable; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapreduce.JobContext; - import org.apache.hive.common.util.HiveVersionInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestAnnotation.java b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestAnnotation.java index 7a4f260ea432..3b56e53afb1a 100644 --- a/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestAnnotation.java +++ b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestAnnotation.java @@ -18,8 +18,10 @@ package org.apache.hudi.hadoop; -import org.junit.jupiter.api.Test; import org.apache.hudi.hadoop.realtime.HoodieParquetRealtimeInputFormat; + +import org.junit.jupiter.api.Test; + import java.lang.annotation.Annotation; import static org.junit.jupiter.api.Assertions.assertTrue; diff --git a/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/utils/TestHiveAvroSerializer.java b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/utils/TestHiveAvroSerializer.java index 1ea333272756..31a9c1781b47 100644 --- a/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/utils/TestHiveAvroSerializer.java +++ b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/utils/TestHiveAvroSerializer.java @@ -18,11 +18,11 @@ package org.apache.hudi.hadoop.utils; -import org.apache.avro.generic.GenericArray; import org.apache.hudi.avro.HoodieAvroUtils; import org.apache.avro.LogicalTypes; import org.apache.avro.Schema; +import org.apache.avro.generic.GenericArray; import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericRecord; import org.apache.hadoop.hive.ql.io.parquet.serde.ArrayWritableObjectInspector; diff --git a/hudi-kafka-connect/src/test/java/org/apache/hudi/connect/TestConnectTransactionParticipant.java b/hudi-kafka-connect/src/test/java/org/apache/hudi/connect/TestConnectTransactionParticipant.java index 36da6ac32e01..a1c1c7ac849c 100644 --- a/hudi-kafka-connect/src/test/java/org/apache/hudi/connect/TestConnectTransactionParticipant.java +++ b/hudi-kafka-connect/src/test/java/org/apache/hudi/connect/TestConnectTransactionParticipant.java @@ -24,9 +24,9 @@ import org.apache.hudi.connect.transaction.TransactionCoordinator; import org.apache.hudi.connect.writers.KafkaConnectConfigs; import org.apache.hudi.exception.HoodieException; +import org.apache.hudi.helper.MockKafkaConnect; import org.apache.hudi.helper.MockKafkaControlAgent; import org.apache.hudi.helper.TestHudiWriterProvider; -import org.apache.hudi.helper.MockKafkaConnect; import org.apache.kafka.common.TopicPartition; import org.junit.jupiter.api.BeforeEach; diff --git a/hudi-kafka-connect/src/test/java/org/apache/hudi/connect/TestHdfsConfiguration.java b/hudi-kafka-connect/src/test/java/org/apache/hudi/connect/TestHdfsConfiguration.java index dca8f577bde8..0099449583f5 100644 --- a/hudi-kafka-connect/src/test/java/org/apache/hudi/connect/TestHdfsConfiguration.java +++ b/hudi-kafka-connect/src/test/java/org/apache/hudi/connect/TestHdfsConfiguration.java @@ -19,15 +19,16 @@ package org.apache.hudi.connect; import org.apache.hudi.connect.utils.KafkaConnectUtils; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - import org.apache.hudi.connect.writers.KafkaConnectConfigs; + import org.junit.jupiter.api.Test; import java.nio.file.Path; import java.util.List; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + public class TestHdfsConfiguration { private boolean checkFiles(List paths) { diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/DataSourceUtils.java b/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/DataSourceUtils.java index 47f12218b1ea..c86e2c908c3f 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/DataSourceUtils.java +++ b/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/DataSourceUtils.java @@ -40,8 +40,8 @@ import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.exception.HoodieException; import org.apache.hudi.exception.TableNotFoundException; -import org.apache.hudi.storage.StoragePath; import org.apache.hudi.storage.HoodieStorage; +import org.apache.hudi.storage.StoragePath; import org.apache.hudi.table.BulkInsertPartitioner; import org.apache.avro.generic.GenericRecord; diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/commit/BaseDatasetBulkInsertCommitActionExecutor.java b/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/commit/BaseDatasetBulkInsertCommitActionExecutor.java index 2a225a7fc60a..2a91cc16adac 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/commit/BaseDatasetBulkInsertCommitActionExecutor.java +++ b/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/commit/BaseDatasetBulkInsertCommitActionExecutor.java @@ -40,6 +40,7 @@ import org.apache.hudi.table.BulkInsertPartitioner; import org.apache.hudi.table.HoodieTable; import org.apache.hudi.table.action.HoodieWriteMetadata; + import org.apache.spark.api.java.JavaRDD; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/commit/DatasetBulkInsertCommitActionExecutor.java b/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/commit/DatasetBulkInsertCommitActionExecutor.java index 4ebdce2e2830..b9d3c8961c25 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/commit/DatasetBulkInsertCommitActionExecutor.java +++ b/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/commit/DatasetBulkInsertCommitActionExecutor.java @@ -29,6 +29,7 @@ import org.apache.hudi.exception.HoodieException; import org.apache.hudi.internal.DataSourceInternalWriterHelper; import org.apache.hudi.table.action.HoodieWriteMetadata; + import org.apache.spark.api.java.JavaRDD; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/commit/DatasetBulkInsertOverwriteCommitActionExecutor.java b/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/commit/DatasetBulkInsertOverwriteCommitActionExecutor.java index 67ba2027cbd9..01e7315cb529 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/commit/DatasetBulkInsertOverwriteCommitActionExecutor.java +++ b/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/commit/DatasetBulkInsertOverwriteCommitActionExecutor.java @@ -31,6 +31,7 @@ import org.apache.hudi.config.HoodieInternalConfig; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.data.HoodieJavaPairRDD; + import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/AvroProjection.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/AvroProjection.scala index a99a0213c569..b045f13ec680 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/AvroProjection.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/AvroProjection.scala @@ -19,9 +19,10 @@ package org.apache.hudi +import org.apache.hudi.avro.HoodieAvroUtils.rewriteRecordWithNewSchema + import org.apache.avro.Schema import org.apache.avro.generic.GenericRecord -import org.apache.hudi.avro.HoodieAvroUtils.rewriteRecordWithNewSchema abstract class AvroProjection extends (GenericRecord => GenericRecord) diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/BucketIndexSupport.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/BucketIndexSupport.scala index def32f9ee946..0dd75cf23255 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/BucketIndexSupport.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/BucketIndexSupport.scala @@ -17,22 +17,23 @@ package org.apache.hudi -import org.apache.avro.generic.GenericData import org.apache.hudi.common.config.HoodieMetadataConfig import org.apache.hudi.common.fs.FSUtils import org.apache.hudi.common.model.FileSlice import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient, TableSchemaResolver} +import org.apache.hudi.common.util.collection.Pair import org.apache.hudi.config.HoodieIndexConfig import org.apache.hudi.index.HoodieIndex import org.apache.hudi.index.HoodieIndex.IndexType import org.apache.hudi.index.bucket.BucketIdentifier import org.apache.hudi.keygen.KeyGenerator import org.apache.hudi.keygen.factory.HoodieSparkKeyGeneratorFactory + +import org.apache.avro.generic.GenericData +import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.expressions import org.apache.spark.sql.catalyst.expressions.{And, Attribute, EmptyRow, Expression, Literal} -import org.apache.hudi.common.util.collection.Pair -import org.apache.spark.sql.SparkSession -import org.apache.spark.sql.types.{DoubleType, FloatType, StructType} +import org.apache.spark.sql.types.{DoubleType, FloatType} import org.apache.spark.util.collection.BitSet import org.slf4j.LoggerFactory diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieBaseRelation.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieBaseRelation.scala index f50e47b9b46b..1920b56ecab3 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieBaseRelation.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieBaseRelation.scala @@ -87,6 +87,7 @@ case class HoodieTableState(tablePath: String, recordMergerImpls: List[String], recordMergerStrategy: String) + /** * Hoodie BaseRelation which extends [[PrunedFilteredScan]] */ diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieBootstrapMORRDD.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieBootstrapMORRDD.scala index f298ca849107..f29b2a2b84d8 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieBootstrapMORRDD.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieBootstrapMORRDD.scala @@ -19,16 +19,16 @@ package org.apache.hudi import org.apache.hudi.HoodieBaseRelation.BaseFileReader -import org.apache.hudi.HoodieBootstrapMORRDD.{getPartitionPath, CONFIG_INSTANTIATION_LOCK} +import org.apache.hudi.HoodieBootstrapMORRDD.{CONFIG_INSTANTIATION_LOCK, getPartitionPath} import org.apache.hudi.hadoop.utils.HoodieRealtimeRecordReaderUtils.getMaxCompactionMemoryInBytes import org.apache.hudi.storage.StoragePath import org.apache.hadoop.conf.Configuration import org.apache.hadoop.mapred.JobConf -import org.apache.spark.{Partition, SerializableWritable, TaskContext} import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.execution.datasources.PartitionedFile +import org.apache.spark.{Partition, SerializableWritable, TaskContext} class HoodieBootstrapMORRDD(@transient spark: SparkSession, @transient config: Configuration, diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieBootstrapMORRelation.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieBootstrapMORRelation.scala index 8241034b2899..a06a216c661c 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieBootstrapMORRelation.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieBootstrapMORRelation.scala @@ -21,7 +21,6 @@ package org.apache.hudi import org.apache.hudi.common.model.{FileSlice, HoodieLogFile} import org.apache.hudi.common.table.HoodieTableMetaClient import org.apache.hudi.storage.StoragePath - import org.apache.spark.rdd.RDD import org.apache.spark.sql.SQLContext import org.apache.spark.sql.catalyst.InternalRow diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieBootstrapRDD.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieBootstrapRDD.scala index f67c8820ebc0..6804edf40d28 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieBootstrapRDD.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieBootstrapRDD.scala @@ -20,6 +20,7 @@ package org.apache.hudi import org.apache.hudi.HoodieBaseRelation.BaseFileReader import org.apache.hudi.common.util.ValidationUtils.checkState + import org.apache.spark.rdd.RDD import org.apache.spark.sql.HoodieCatalystExpressionUtils.generateUnsafeProjection import org.apache.spark.sql.SparkSession diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieBootstrapRelation.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieBootstrapRelation.scala index ffa21e40263a..cda0bafa17f8 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieBootstrapRelation.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieBootstrapRelation.scala @@ -25,7 +25,6 @@ import org.apache.hudi.common.table.HoodieTableMetaClient import org.apache.hudi.common.util.ValidationUtils.checkState import org.apache.hudi.storage.StoragePath -import org.apache.hadoop.fs.Path import org.apache.spark.rdd.RDD import org.apache.spark.sql.SQLContext import org.apache.spark.sql.catalyst.InternalRow diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieCatalystUtils.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieCatalystUtils.scala index 0f41dc1fff3f..3a54000b59dc 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieCatalystUtils.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieCatalystUtils.scala @@ -19,6 +19,7 @@ package org.apache.hudi import org.apache.hudi.common.data.HoodieData + import org.apache.spark.sql.Dataset import org.apache.spark.storage.StorageLevel import org.apache.spark.storage.StorageLevel._ diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieCreateRecordUtils.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieCreateRecordUtils.scala index 5fa0cc8d1fff..1b4c600d5487 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieCreateRecordUtils.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieCreateRecordUtils.scala @@ -23,13 +23,14 @@ import org.apache.hudi.avro.HoodieAvroUtils import org.apache.hudi.common.config.TypedProperties import org.apache.hudi.common.fs.FSUtils import org.apache.hudi.common.model._ +import org.apache.hudi.common.util.StringUtils import org.apache.hudi.config.HoodieWriteConfig import org.apache.hudi.keygen.constant.KeyGeneratorOptions import org.apache.hudi.keygen.factory.HoodieSparkKeyGeneratorFactory import org.apache.hudi.keygen.{BaseKeyGenerator, KeyGenUtils, SparkKeyGeneratorInterface} + import org.apache.avro.Schema import org.apache.avro.generic.GenericRecord -import org.apache.hudi.common.util.StringUtils import org.apache.spark.TaskContext import org.apache.spark.api.java.JavaRDD import org.apache.spark.rdd.RDD diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieHadoopFsRelationFactory.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieHadoopFsRelationFactory.scala index 27306783eeb9..c3faa339b70e 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieHadoopFsRelationFactory.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieHadoopFsRelationFactory.scala @@ -18,6 +18,9 @@ package org.apache.hudi +import org.apache.avro.Schema +import org.apache.hadoop.conf.Configuration +import org.apache.hadoop.mapred.JobConf import org.apache.hudi.HoodieBaseRelation.{convertToAvroSchema, isSchemaEvolutionEnabledOnRead} import org.apache.hudi.HoodieConversionUtils.toScalaOption import org.apache.hudi.HoodieFileIndex.getConfigProperties @@ -26,17 +29,13 @@ import org.apache.hudi.common.config.HoodieMetadataConfig.{DEFAULT_METADATA_ENAB import org.apache.hudi.common.model.HoodieRecord import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient, TableSchemaResolver} import org.apache.hudi.common.table.timeline.HoodieTimeline -import org.apache.hudi.common.util.ValidationUtils.checkState import org.apache.hudi.common.util.{ConfigUtils, StringUtils} +import org.apache.hudi.common.util.ValidationUtils.checkState import org.apache.hudi.config.HoodieWriteConfig import org.apache.hudi.internal.schema.InternalSchema import org.apache.hudi.internal.schema.convert.AvroInternalSchemaConverter import org.apache.hudi.metadata.HoodieTableMetadataUtil import org.apache.hudi.storage.StoragePath - -import org.apache.avro.Schema -import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.mapred.JobConf import org.apache.spark.sql.{SparkSession, SQLContext} import org.apache.spark.sql.catalyst.analysis.Resolver import org.apache.spark.sql.catalyst.catalog.BucketSpec diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieIncrementalFileIndex.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieIncrementalFileIndex.scala index 6d7174174a7b..7b39257c4d27 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieIncrementalFileIndex.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieIncrementalFileIndex.scala @@ -17,12 +17,11 @@ package org.apache.hudi +import org.apache.hadoop.fs.{FileStatus, Path} import org.apache.hudi.common.model.{FileSlice, HoodieLogFile} import org.apache.hudi.common.table.HoodieTableMetaClient import org.apache.hudi.storage.StoragePathInfo import org.apache.hudi.util.JFunction - -import org.apache.hadoop.fs.{FileStatus, Path} import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.Expression @@ -31,6 +30,7 @@ import org.apache.spark.sql.sources.Filter import org.apache.spark.sql.types.StructType import java.util.stream.Collectors + import scala.collection.JavaConverters._ class HoodieIncrementalFileIndex(override val spark: SparkSession, diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieMergeOnReadRDD.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieMergeOnReadRDD.scala index db538f110c90..2fe3e892c1e2 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieMergeOnReadRDD.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieMergeOnReadRDD.scala @@ -18,8 +18,6 @@ package org.apache.hudi -import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.mapred.JobConf import org.apache.hudi.HoodieBaseRelation.{BaseFileReader, projectReader} import org.apache.hudi.HoodieMergeOnReadRDD.CONFIG_INSTANTIATION_LOCK import org.apache.hudi.MergeOnReadSnapshotRelation.isProjectionCompatible @@ -27,6 +25,9 @@ import org.apache.hudi.common.model.HoodieRecord import org.apache.hudi.common.util.StringUtils import org.apache.hudi.exception.HoodieException import org.apache.hudi.hadoop.utils.HoodieRealtimeRecordReaderUtils.getMaxCompactionMemoryInBytes + +import org.apache.hadoop.conf.Configuration +import org.apache.hadoop.mapred.JobConf import org.apache.spark.rdd.RDD import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.{Partition, SerializableWritable, SparkContext, TaskContext} diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodiePartitionCDCFileGroupMapping.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodiePartitionCDCFileGroupMapping.scala index 418037fcd22d..46b176885288 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodiePartitionCDCFileGroupMapping.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodiePartitionCDCFileGroupMapping.scala @@ -19,7 +19,6 @@ package org.apache.hudi -import org.apache.hudi.common.model.HoodieFileGroupId import org.apache.hudi.common.table.cdc.HoodieCDCFileSplit import org.apache.spark.sql.catalyst.InternalRow diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodiePartitionFileSliceMapping.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodiePartitionFileSliceMapping.scala index 4121a41b8d82..736f61951b2c 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodiePartitionFileSliceMapping.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodiePartitionFileSliceMapping.scala @@ -21,9 +21,6 @@ package org.apache.hudi import org.apache.hudi.common.model.FileSlice import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.util.{ArrayData, MapData} -import org.apache.spark.sql.types.{DataType, Decimal} -import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String} class HoodiePartitionFileSliceMapping(values: InternalRow, slices: Map[String, FileSlice]) diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSchemaUtils.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSchemaUtils.scala index 934ba8d4704f..5ab32338706c 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSchemaUtils.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSchemaUtils.scala @@ -32,6 +32,7 @@ import org.apache.hudi.internal.schema.InternalSchema import org.apache.hudi.internal.schema.convert.AvroInternalSchemaConverter import org.apache.hudi.internal.schema.utils.AvroSchemaEvolutionUtils import org.apache.hudi.internal.schema.utils.AvroSchemaEvolutionUtils.reconcileSchemaRequirements + import org.apache.avro.Schema import org.slf4j.LoggerFactory diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSparkConfUtils.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSparkConfUtils.scala index b4bc632d614b..cca1730d7a78 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSparkConfUtils.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSparkConfUtils.scala @@ -21,6 +21,7 @@ package org.apache.hudi import org.apache.hudi.DataSourceReadOptions.INCREMENTAL_READ_HANDLE_HOLLOW_COMMIT import org.apache.hudi.common.table.timeline.TimelineUtils.HollowCommitHandling + import org.apache.spark.sql.internal.SQLConf /** diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala index b07ea629eabb..27e64d5a88af 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala @@ -24,6 +24,7 @@ import org.apache.hudi.DataSourceUtils.tryOverrideParquetWriteLegacyFormatProper import org.apache.hudi.DataSourceWriteOptions._ import org.apache.hudi.HoodieConversionUtils.{toProperties, toScalaOption} import org.apache.hudi.HoodieSparkSqlWriter.StreamingWriteParams +import org.apache.hudi.HoodieSparkUtils.sparkAdapter import org.apache.hudi.HoodieWriterUtils._ import org.apache.hudi.avro.AvroSchemaUtils.resolveNullableSchema import org.apache.hudi.avro.HoodieAvroUtils @@ -59,13 +60,13 @@ import org.apache.hudi.sync.common.HoodieSyncConfig import org.apache.hudi.sync.common.util.SyncUtilHelpers import org.apache.hudi.sync.common.util.SyncUtilHelpers.getHoodieMetaSyncException import org.apache.hudi.util.SparkKeyGenUtils + import org.apache.avro.Schema import org.apache.avro.generic.GenericData import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileSystem, Path} import org.apache.hadoop.hive.conf.HiveConf import org.apache.hadoop.hive.shims.ShimLoader -import org.apache.hudi.HoodieSparkUtils.sparkAdapter import org.apache.spark.api.java.JavaSparkContext import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.TableIdentifier @@ -77,6 +78,7 @@ import org.apache.spark.{SPARK_VERSION, SparkContext} import org.slf4j.LoggerFactory import java.util.function.BiConsumer + import scala.collection.JavaConverters._ import scala.collection.mutable import scala.util.{Failure, Success, Try} diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/Iterators.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/Iterators.scala index 6cba28685360..52e406699994 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/Iterators.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/Iterators.scala @@ -37,6 +37,7 @@ import org.apache.hudi.metadata.HoodieTableMetadata.getDataTableBasePathFromMeta import org.apache.hudi.metadata.{HoodieBackedTableMetadata, HoodieTableMetadata} import org.apache.hudi.storage.{HoodieStorageUtils, StoragePath} import org.apache.hudi.util.CachingIterator + import org.apache.avro.Schema import org.apache.avro.generic.GenericRecord import org.apache.hadoop.conf.Configuration @@ -48,6 +49,7 @@ import org.apache.spark.sql.catalyst.expressions.Projection import org.apache.spark.sql.types.StructType import java.io.Closeable + import scala.annotation.tailrec import scala.collection.JavaConverters._ import scala.collection.mutable diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/MergeOnReadIncrementalRelation.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/MergeOnReadIncrementalRelation.scala index ed591bf6b613..c4dd7c1ae84e 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/MergeOnReadIncrementalRelation.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/MergeOnReadIncrementalRelation.scala @@ -17,21 +17,20 @@ package org.apache.hudi +import org.apache.hadoop.fs.GlobPattern import org.apache.hudi.HoodieConversionUtils.toScalaOption import org.apache.hudi.HoodieSparkConfUtils.getHollowCommitHandling import org.apache.hudi.common.model.{FileSlice, HoodieRecord} import org.apache.hudi.common.table.HoodieTableMetaClient -import org.apache.hudi.common.table.timeline.TimelineUtils.HollowCommitHandling.USE_TRANSITION_TIME -import org.apache.hudi.common.table.timeline.TimelineUtils.{HollowCommitHandling, concatTimeline, getCommitMetadata, handleHollowCommitIfNeeded} import org.apache.hudi.common.table.timeline.{HoodieInstant, HoodieTimeline} +import org.apache.hudi.common.table.timeline.TimelineUtils.{concatTimeline, getCommitMetadata, handleHollowCommitIfNeeded, HollowCommitHandling} +import org.apache.hudi.common.table.timeline.TimelineUtils.HollowCommitHandling.USE_TRANSITION_TIME import org.apache.hudi.common.table.view.HoodieTableFileSystemView import org.apache.hudi.common.util.StringUtils import org.apache.hudi.exception.HoodieException import org.apache.hudi.hadoop.utils.HoodieInputFormatUtils.listAffectedFilesForCommits import org.apache.hudi.metadata.HoodieTableMetadataUtil.getWritePartitionPaths import org.apache.hudi.storage.StoragePathInfo - -import org.apache.hadoop.fs.GlobPattern import org.apache.spark.rdd.RDD import org.apache.spark.sql.SQLContext import org.apache.spark.sql.catalyst.InternalRow diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/MergeOnReadSnapshotRelation.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/MergeOnReadSnapshotRelation.scala index 772e90d63c18..4f932b75746d 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/MergeOnReadSnapshotRelation.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/MergeOnReadSnapshotRelation.scala @@ -23,7 +23,6 @@ import org.apache.hudi.MergeOnReadSnapshotRelation.{createPartitionedFile, isPro import org.apache.hudi.common.model.{FileSlice, HoodieLogFile, OverwriteWithLatestAvroPayload} import org.apache.hudi.common.table.HoodieTableMetaClient import org.apache.hudi.storage.StoragePath - import org.apache.spark.rdd.RDD import org.apache.spark.sql.SQLContext import org.apache.spark.sql.catalyst.InternalRow diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/RecordLevelIndexSupport.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/RecordLevelIndexSupport.scala index a0e51e410e47..4b79b96be762 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/RecordLevelIndexSupport.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/RecordLevelIndexSupport.scala @@ -24,6 +24,7 @@ import org.apache.hudi.common.model.HoodieRecord.HoodieMetadataField import org.apache.hudi.common.table.HoodieTableMetaClient import org.apache.hudi.metadata.HoodieTableMetadataUtil import org.apache.hudi.storage.StoragePath + import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.expressions.{AttributeReference, EqualTo, Expression, In, Literal} diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/SecondaryIndexSupport.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/SecondaryIndexSupport.scala index c7e630ea33ff..81a2e9fcb385 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/SecondaryIndexSupport.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/SecondaryIndexSupport.scala @@ -27,6 +27,7 @@ import org.apache.hudi.common.model.FileSlice import org.apache.hudi.common.table.HoodieTableMetaClient import org.apache.hudi.metadata.HoodieTableMetadataUtil.PARTITION_NAME_SECONDARY_INDEX import org.apache.hudi.storage.StoragePath + import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.expressions.Expression diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/SparkBaseIndexSupport.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/SparkBaseIndexSupport.scala index 4698f147746f..bc29354d7f88 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/SparkBaseIndexSupport.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/SparkBaseIndexSupport.scala @@ -24,6 +24,7 @@ import org.apache.hudi.common.model.FileSlice import org.apache.hudi.common.table.HoodieTableMetaClient import org.apache.hudi.metadata.{HoodieMetadataPayload, HoodieTableMetadata} import org.apache.hudi.util.JFunction + import org.apache.spark.api.java.JavaSparkContext import org.apache.spark.sql.catalyst.expressions.{And, Expression} import org.apache.spark.sql.hudi.DataSkippingUtils.translateIntoColumnStatsIndexFilterExpr diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/SparkHoodieTableFileIndex.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/SparkHoodieTableFileIndex.scala index 58903113b59b..c5581f116be3 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/SparkHoodieTableFileIndex.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/SparkHoodieTableFileIndex.scala @@ -17,14 +17,15 @@ package org.apache.hudi +import org.apache.hadoop.fs.{FileStatus, Path} import org.apache.hudi.BaseHoodieTableFileIndex.PartitionPath import org.apache.hudi.DataSourceReadOptions._ import org.apache.hudi.HoodieConversionUtils.toJavaOption import org.apache.hudi.SparkHoodieTableFileIndex.{deduceQueryType, extractEqualityPredicatesLiteralValues, generateFieldMap, haveProperPartitionValues, shouldListLazily, shouldUsePartitionPathPrefixAnalysis, shouldValidatePartitionColumns} import org.apache.hudi.client.common.HoodieSparkEngineContext import org.apache.hudi.common.config.TypedProperties -import org.apache.hudi.common.model.HoodieRecord.HOODIE_META_COLUMNS_WITH_OPERATION import org.apache.hudi.common.model.{FileSlice, HoodieTableQueryType} +import org.apache.hudi.common.model.HoodieRecord.HOODIE_META_COLUMNS_WITH_OPERATION import org.apache.hudi.common.table.{HoodieTableMetaClient, TableSchemaResolver} import org.apache.hudi.common.util.ValidationUtils.checkState import org.apache.hudi.config.HoodieBootstrapConfig.DATA_QUERIES_ONLY @@ -34,21 +35,20 @@ import org.apache.hudi.internal.schema.utils.Conversions import org.apache.hudi.keygen.{StringPartitionPathFormatter, TimestampBasedAvroKeyGenerator, TimestampBasedKeyGenerator} import org.apache.hudi.storage.{StoragePath, StoragePathInfo} import org.apache.hudi.util.JFunction - -import org.apache.hadoop.fs.{FileStatus, Path} import org.apache.spark.api.java.JavaSparkContext import org.apache.spark.internal.Logging import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.{expressions, InternalRow} import org.apache.spark.sql.catalyst.expressions.{AttributeReference, BoundReference, EmptyRow, EqualTo, Expression, InterpretedPredicate, Literal} import org.apache.spark.sql.catalyst.util.DateTimeUtils -import org.apache.spark.sql.catalyst.{InternalRow, expressions} import org.apache.spark.sql.execution.datasources.{FileStatusCache, NoopCache} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{ByteType, DateType, IntegerType, LongType, ShortType, StringType, StructField, StructType} -import java.util.Collections import javax.annotation.concurrent.NotThreadSafe +import java.util.Collections + import scala.collection.JavaConverters._ import scala.language.implicitConversions import scala.util.{Success, Try} diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/cdc/InternalRowToJsonStringConverter.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/cdc/InternalRowToJsonStringConverter.scala index c0f52a7ca121..2bdb07418c9f 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/cdc/InternalRowToJsonStringConverter.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/cdc/InternalRowToJsonStringConverter.scala @@ -18,10 +18,11 @@ package org.apache.hudi.cdc +import org.apache.hudi.HoodieTableSchema + import com.fasterxml.jackson.annotation.JsonInclude.Include import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper} import com.fasterxml.jackson.module.scala.DefaultScalaModule -import org.apache.hudi.HoodieTableSchema import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.types.StringType import org.apache.spark.unsafe.types.UTF8String diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/HoodieHadoopFSUtils.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/HoodieHadoopFSUtils.scala index 353d94a7c105..7dd5e619e699 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/HoodieHadoopFSUtils.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/HoodieHadoopFSUtils.scala @@ -18,14 +18,15 @@ package org.apache.spark import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.fs.viewfs.ViewFileSystem import org.apache.hadoop.fs._ +import org.apache.hadoop.fs.viewfs.ViewFileSystem import org.apache.hadoop.hdfs.DistributedFileSystem import org.apache.spark.internal.Logging import org.apache.spark.metrics.source.HiveCatalogMetrics import org.apache.spark.util.SerializableConfiguration import java.io.FileNotFoundException + import scala.collection.mutable /** diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/execution/datasources/HoodieInMemoryFileIndex.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/execution/datasources/HoodieInMemoryFileIndex.scala index 722cd74408f5..ed039e67ef9b 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/execution/datasources/HoodieInMemoryFileIndex.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/execution/datasources/HoodieInMemoryFileIndex.scala @@ -17,12 +17,11 @@ package org.apache.spark.execution.datasources -import org.apache.hudi.SparkAdapterSupport -import org.apache.hudi.storage.StoragePath - import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileStatus, Path, PathFilter} import org.apache.hadoop.mapred.{FileInputFormat, JobConf} +import org.apache.hudi.SparkAdapterSupport +import org.apache.hudi.storage.StoragePath import org.apache.spark.HoodieHadoopFSUtils import org.apache.spark.metrics.source.HiveCatalogMetrics import org.apache.spark.sql.SparkSession diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/FileFormatUtilsForFileGroupReader.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/FileFormatUtilsForFileGroupReader.scala index 0587f135467e..593ec15ff4de 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/FileFormatUtilsForFileGroupReader.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/FileFormatUtilsForFileGroupReader.scala @@ -19,11 +19,11 @@ package org.apache.spark.sql import org.apache.hudi.{HoodieCDCFileIndex, SparkAdapterSupport, SparkHoodieTableFileIndex} -import org.apache.spark.sql.catalyst.expressions.{And, Attribute, AttributeSet, Contains, EndsWith, EqualNullSafe, EqualTo, Expression, GreaterThan, GreaterThanOrEqual, In, IsNotNull, IsNull, LessThan, LessThanOrEqual, Literal, NamedExpression, Not, Or, StartsWith} -import org.apache.spark.sql.catalyst.plans.logical.{Filter, Join, LogicalPlan, Project} +import org.apache.spark.sql.catalyst.expressions.{And, Attribute, Contains, EndsWith, EqualNullSafe, EqualTo, Expression, GreaterThan, GreaterThanOrEqual, In, IsNotNull, IsNull, LessThan, LessThanOrEqual, Literal, NamedExpression, Not, Or, StartsWith} +import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan, Project} import org.apache.spark.sql.execution.datasources.HadoopFsRelation import org.apache.spark.sql.execution.datasources.parquet.{HoodieFormatTrait, ParquetFileFormat} -import org.apache.spark.sql.types.{BooleanType, StructType} +import org.apache.spark.sql.types.StructType import scala.util.Try diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/avro/SchemaConverters.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/avro/SchemaConverters.scala index b4e09f6d1f65..9430fcd08ce3 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/avro/SchemaConverters.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/avro/SchemaConverters.scala @@ -17,12 +17,12 @@ package org.apache.spark.sql.avro +import org.apache.avro.{LogicalTypes, Schema, SchemaBuilder} import org.apache.avro.LogicalTypes.{Date, Decimal, TimestampMicros, TimestampMillis} import org.apache.avro.Schema.Type._ -import org.apache.avro.{LogicalTypes, Schema, SchemaBuilder} import org.apache.spark.annotation.DeveloperApi -import org.apache.spark.sql.types.Decimal.minBytesForPrecision import org.apache.spark.sql.types._ +import org.apache.spark.sql.types.Decimal.minBytesForPrecision import scala.collection.JavaConverters._ diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/catalyst/catalog/HoodieCatalogTable.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/catalyst/catalog/HoodieCatalogTable.scala index 2867078aadbf..1d8ab23e184d 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/catalyst/catalog/HoodieCatalogTable.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/catalyst/catalog/HoodieCatalogTable.scala @@ -17,31 +17,31 @@ package org.apache.spark.sql.catalyst.catalog +import org.apache.hudi.{AvroConversionUtils, DataSourceOptionsHelper} import org.apache.hudi.DataSourceWriteOptions.OPERATION import org.apache.hudi.HoodieWriterUtils._ import org.apache.hudi.avro.AvroSchemaUtils import org.apache.hudi.common.config.{DFSPropertiesConfiguration, TypedProperties} import org.apache.hudi.common.model.HoodieTableType +import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient} import org.apache.hudi.common.table.HoodieTableConfig.URL_ENCODE_PARTITIONING import org.apache.hudi.common.table.timeline.TimelineUtils -import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient} import org.apache.hudi.common.util.StringUtils import org.apache.hudi.common.util.ValidationUtils.checkArgument import org.apache.hudi.hadoop.fs.HadoopFSUtils import org.apache.hudi.keygen.constant.KeyGeneratorType import org.apache.hudi.keygen.factory.HoodieSparkKeyGeneratorFactory -import org.apache.hudi.{AvroConversionUtils, DataSourceOptionsHelper} - import org.apache.spark.internal.Logging +import org.apache.spark.sql.{AnalysisException, SparkSession} import org.apache.spark.sql.avro.SchemaConverters import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.hudi.HoodieOptionConfig import org.apache.spark.sql.hudi.HoodieOptionConfig._ import org.apache.spark.sql.hudi.HoodieSqlCommonUtils._ import org.apache.spark.sql.types.{StructField, StructType} -import org.apache.spark.sql.{AnalysisException, SparkSession} import java.util.Locale + import scala.collection.JavaConverters._ import scala.collection.mutable diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/HoodieMultipleBaseFileFormat.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/HoodieMultipleBaseFileFormat.scala index f7da4c8fd6af..25696b5b560d 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/HoodieMultipleBaseFileFormat.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/HoodieMultipleBaseFileFormat.scala @@ -19,16 +19,15 @@ package org.apache.spark.sql.execution.datasources +import org.apache.hadoop.conf.Configuration +import org.apache.hadoop.fs.{FileStatus, Path} +import org.apache.hadoop.mapreduce.Job +import org.apache.hudi.{HoodieBaseRelation, HoodiePartitionFileSliceMapping, HoodieTableSchema, HoodieTableState, LogFileIterator, MergeOnReadSnapshotRelation, RecordMergingFileIterator, SparkAdapterSupport} import org.apache.hudi.DataSourceReadOptions.{REALTIME_PAYLOAD_COMBINE_OPT_VAL, REALTIME_SKIP_MERGE_OPT_VAL} import org.apache.hudi.MergeOnReadSnapshotRelation.createPartitionedFile import org.apache.hudi.common.fs.FSUtils import org.apache.hudi.common.model.{FileSlice, HoodieLogFile} import org.apache.hudi.storage.StoragePath -import org.apache.hudi.{HoodieBaseRelation, HoodiePartitionFileSliceMapping, HoodieTableSchema, HoodieTableState, LogFileIterator, MergeOnReadSnapshotRelation, RecordMergingFileIterator, SparkAdapterSupport} - -import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.fs.{FileStatus, Path} -import org.apache.hadoop.mapreduce.Job import org.apache.spark.broadcast.Broadcast import org.apache.spark.sql.HoodieCatalystExpressionUtils.generateUnsafeProjection import org.apache.spark.sql.SparkSession diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/HoodieFileGroupReaderBasedParquetFileFormat.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/HoodieFileGroupReaderBasedParquetFileFormat.scala index 71dbf9ae0170..0ad2bda9cf2d 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/HoodieFileGroupReaderBasedParquetFileFormat.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/HoodieFileGroupReaderBasedParquetFileFormat.scala @@ -19,19 +19,19 @@ package org.apache.spark.sql.execution.datasources.parquet import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path +import org.apache.hudi.{AvroConversionUtils, HoodieFileIndex, HoodiePartitionCDCFileGroupMapping, HoodiePartitionFileSliceMapping, HoodieTableSchema, HoodieTableState, SparkAdapterSupport, SparkFileFormatInternalRowReaderContext} import org.apache.hudi.MergeOnReadSnapshotRelation.createPartitionedFile import org.apache.hudi.avro.AvroSchemaUtils import org.apache.hudi.cdc.{CDCFileGroupIterator, CDCRelation, HoodieCDCFileGroupSplit} import org.apache.hudi.client.utils.SparkInternalSchemaConverter import org.apache.hudi.common.config.TypedProperties import org.apache.hudi.common.fs.FSUtils -import org.apache.hudi.common.table.read.HoodieFileGroupReader import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient} +import org.apache.hudi.common.table.read.HoodieFileGroupReader import org.apache.hudi.internal.schema.InternalSchema import org.apache.hudi.internal.schema.utils.SerDeHelper import org.apache.hudi.storage.StorageConfiguration import org.apache.hudi.storage.hadoop.{HadoopStorageConfiguration, HoodieHadoopStorage} -import org.apache.hudi.{AvroConversionUtils, HoodieFileIndex, HoodiePartitionCDCFileGroupMapping, HoodiePartitionFileSliceMapping, HoodieTableSchema, HoodieTableState, SparkAdapterSupport, SparkFileFormatInternalRowReaderContext} import org.apache.spark.sql.HoodieCatalystExpressionUtils.generateUnsafeProjection import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.InternalRow diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/LegacyHoodieParquetFileFormat.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/LegacyHoodieParquetFileFormat.scala index d579c9052a4b..686afccb72cd 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/LegacyHoodieParquetFileFormat.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/LegacyHoodieParquetFileFormat.scala @@ -19,13 +19,13 @@ package org.apache.spark.sql.execution.datasources.parquet import org.apache.hadoop.conf.Configuration -import org.apache.hudi.{DataSourceReadOptions, HoodieSparkUtils, SparkAdapterSupport} +import org.apache.hudi.{DataSourceReadOptions, SparkAdapterSupport} import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.execution.datasources.PartitionedFile import org.apache.spark.sql.execution.datasources.parquet.LegacyHoodieParquetFileFormat.FILE_FORMAT_ID import org.apache.spark.sql.sources.Filter -import org.apache.spark.sql.types.{AtomicType, StructType} +import org.apache.spark.sql.types.StructType /** * This legacy parquet file format implementation to support Hudi will be replaced by diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/DataSkippingUtils.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/DataSkippingUtils.scala index cfd8d1351d8d..b9252eb831da 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/DataSkippingUtils.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/DataSkippingUtils.scala @@ -21,13 +21,13 @@ import org.apache.hudi.ColumnStatsIndexSupport.{getMaxColumnNameFor, getMinColum import org.apache.hudi.SparkAdapterSupport import org.apache.hudi.common.util.ValidationUtils.checkState import org.apache.spark.internal.Logging +import org.apache.spark.sql.{AnalysisException, HoodieCatalystExpressionUtils} import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute -import org.apache.spark.sql.catalyst.expressions.Literal.TrueLiteral import org.apache.spark.sql.catalyst.expressions.{Alias, And, Attribute, AttributeReference, EqualNullSafe, EqualTo, Expression, ExtractValue, GetStructField, GreaterThan, GreaterThanOrEqual, In, InSet, IsNotNull, IsNull, LessThan, LessThanOrEqual, Literal, Not, Or, StartsWith, SubqueryExpression} +import org.apache.spark.sql.catalyst.expressions.Literal.TrueLiteral import org.apache.spark.sql.functions.col import org.apache.spark.sql.hudi.ColumnStatsExpressionUtils._ import org.apache.spark.sql.types.StructType -import org.apache.spark.sql.{AnalysisException, HoodieCatalystExpressionUtils} import org.apache.spark.unsafe.types.UTF8String object DataSkippingUtils extends Logging { diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/HoodieSqlCommonUtils.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/HoodieSqlCommonUtils.scala index 719327cc42c6..153a4cd65aab 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/HoodieSqlCommonUtils.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/HoodieSqlCommonUtils.scala @@ -17,29 +17,28 @@ package org.apache.spark.sql.hudi +import org.apache.hadoop.conf.Configuration +import org.apache.hadoop.fs.Path +import org.apache.hudi.{AvroConversionUtils, DataSourceReadOptions, SparkAdapterSupport} import org.apache.hudi.DataSourceWriteOptions.COMMIT_METADATA_KEYPREFIX import org.apache.hudi.client.common.HoodieSparkEngineContext import org.apache.hudi.common.config.{HoodieMetadataConfig, TypedProperties} import org.apache.hudi.common.fs.FSUtils import org.apache.hudi.common.model.HoodieRecord -import org.apache.hudi.common.table.timeline.HoodieActiveTimeline.parseDateFromInstantTime -import org.apache.hudi.common.table.timeline.{HoodieActiveTimeline, HoodieInstantTimeGenerator, HoodieTimeline} import org.apache.hudi.common.table.{HoodieTableMetaClient, TableSchemaResolver} +import org.apache.hudi.common.table.timeline.{HoodieActiveTimeline, HoodieInstantTimeGenerator, HoodieTimeline} +import org.apache.hudi.common.table.timeline.HoodieActiveTimeline.parseDateFromInstantTime import org.apache.hudi.common.util.PartitionPathEncodeUtils import org.apache.hudi.exception.HoodieException import org.apache.hudi.storage.{HoodieStorage, StoragePathInfo} -import org.apache.hudi.{AvroConversionUtils, DataSourceReadOptions, SparkAdapterSupport} - -import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.fs.Path import org.apache.spark.api.java.JavaSparkContext +import org.apache.spark.sql.{AnalysisException, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.analysis.Resolver import org.apache.spark.sql.catalyst.catalog.{CatalogTable, HoodieCatalogTable} import org.apache.spark.sql.catalyst.expressions.{Attribute, Cast, Expression, Literal} import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf} import org.apache.spark.sql.types._ -import org.apache.spark.sql.{AnalysisException, SparkSession} import java.net.URI import java.text.SimpleDateFormat diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/ProvidesHoodieConfig.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/ProvidesHoodieConfig.scala index 718c6c08dc57..070f507a2520 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/ProvidesHoodieConfig.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/ProvidesHoodieConfig.scala @@ -17,6 +17,7 @@ package org.apache.spark.sql.hudi +import org.apache.hudi.{DataSourceWriteOptions, HoodieFileIndex} import org.apache.hudi.AutoRecordKeyGenerationUtils.shouldAutoGenerateRecordKeys import org.apache.hudi.DataSourceWriteOptions._ import org.apache.hudi.HoodieConversionUtils.toProperties @@ -24,15 +25,15 @@ import org.apache.hudi.common.config.{DFSPropertiesConfiguration, TypedPropertie import org.apache.hudi.common.model.{DefaultHoodieRecordPayload, WriteOperationType} import org.apache.hudi.common.table.HoodieTableConfig import org.apache.hudi.common.util.{ReflectionUtils, StringUtils} -import org.apache.hudi.config.HoodieWriteConfig.TBL_NAME import org.apache.hudi.config.{HoodieIndexConfig, HoodieInternalConfig, HoodieWriteConfig} -import org.apache.hudi.hive.ddl.HiveSyncMode +import org.apache.hudi.config.HoodieWriteConfig.TBL_NAME import org.apache.hudi.hive.{HiveSyncConfig, HiveSyncConfigHolder, MultiPartKeysValueExtractor} +import org.apache.hudi.hive.ddl.HiveSyncMode import org.apache.hudi.keygen.{ComplexKeyGenerator, CustomAvroKeyGenerator, CustomKeyGenerator} import org.apache.hudi.sql.InsertMode import org.apache.hudi.sync.common.HoodieSyncConfig -import org.apache.hudi.{DataSourceWriteOptions, HoodieFileIndex} import org.apache.spark.internal.Logging +import org.apache.spark.sql.{SaveMode, SparkSession} import org.apache.spark.sql.catalyst.catalog.HoodieCatalogTable import org.apache.spark.sql.catalyst.expressions.{AttributeReference, EqualTo, Literal} import org.apache.spark.sql.execution.datasources.FileStatusCache @@ -44,10 +45,10 @@ import org.apache.spark.sql.hudi.command.{SqlKeyGenerator, ValidateDuplicateKeyP import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.internal.SQLConf.PARTITION_OVERWRITE_MODE import org.apache.spark.sql.types.StructType -import org.apache.spark.sql.{SaveMode, SparkSession} import org.slf4j.LoggerFactory import java.util.Locale + import scala.collection.JavaConverters._ trait ProvidesHoodieConfig extends Logging { diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/AlterHoodieTableAddColumnsCommand.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/AlterHoodieTableAddColumnsCommand.scala index 47ad11e17bc7..3c8c63bed372 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/AlterHoodieTableAddColumnsCommand.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/AlterHoodieTableAddColumnsCommand.scala @@ -18,21 +18,21 @@ package org.apache.spark.sql.hudi.command import org.apache.avro.Schema +import org.apache.hudi.{AvroConversionUtils, DataSourceUtils, HoodieWriterUtils, SparkAdapterSupport} import org.apache.hudi.avro.HoodieAvroUtils import org.apache.hudi.common.model.{HoodieCommitMetadata, HoodieTableType, WriteOperationType} +import org.apache.hudi.common.table.timeline.HoodieInstant import org.apache.hudi.common.table.timeline.HoodieInstant.State import org.apache.hudi.common.table.timeline.TimelineMetadataUtils.serializeCommitMetadata -import org.apache.hudi.common.table.timeline.HoodieInstant import org.apache.hudi.common.util.CommitUtils import org.apache.hudi.table.HoodieSparkTable -import org.apache.hudi.{AvroConversionUtils, DataSourceUtils, HoodieWriterUtils, SparkAdapterSupport} import org.apache.spark.api.java.JavaSparkContext import org.apache.spark.internal.Logging +import org.apache.spark.sql.{AnalysisException, Row, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.catalog.HoodieCatalogTable import org.apache.spark.sql.hudi.HoodieOptionConfig import org.apache.spark.sql.types.{StructField, StructType} -import org.apache.spark.sql.{AnalysisException, Row, SparkSession} import scala.collection.JavaConverters._ import scala.util.control.NonFatal diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/AlterHoodieTableAddPartitionCommand.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/AlterHoodieTableAddPartitionCommand.scala index 8e45ff094aa2..1f76f4517c9b 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/AlterHoodieTableAddPartitionCommand.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/AlterHoodieTableAddPartitionCommand.scala @@ -21,7 +21,6 @@ import org.apache.hudi.common.fs.FSUtils import org.apache.hudi.common.model.HoodiePartitionMetadata import org.apache.hudi.common.table.timeline.HoodieTimeline import org.apache.hudi.storage.StoragePath - import org.apache.spark.sql.{AnalysisException, Row, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.catalog.{CatalogTablePartition, HoodieCatalogTable} diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/AlterHoodieTableDropPartitionCommand.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/AlterHoodieTableDropPartitionCommand.scala index c6aa2e7aedac..a739d55953cf 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/AlterHoodieTableDropPartitionCommand.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/AlterHoodieTableDropPartitionCommand.scala @@ -19,13 +19,13 @@ package org.apache.spark.sql.hudi.command import org.apache.hudi.HoodieSparkSqlWriter import org.apache.hudi.exception.HoodieException +import org.apache.spark.sql.{AnalysisException, Row, SaveMode, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec import org.apache.spark.sql.catalyst.catalog.HoodieCatalogTable import org.apache.spark.sql.execution.command.DDLUtils import org.apache.spark.sql.hudi.HoodieSqlCommonUtils._ import org.apache.spark.sql.hudi.ProvidesHoodieConfig -import org.apache.spark.sql.{AnalysisException, Row, SaveMode, SparkSession} case class AlterHoodieTableDropPartitionCommand( tableIdentifier: TableIdentifier, diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/AlterHoodieTableRenameCommand.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/AlterHoodieTableRenameCommand.scala index 990c9863a388..84ca4c3c00b1 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/AlterHoodieTableRenameCommand.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/AlterHoodieTableRenameCommand.scala @@ -19,11 +19,10 @@ package org.apache.spark.sql.hudi.command import org.apache.hudi.common.table.HoodieTableMetaClient import org.apache.hudi.hadoop.fs.HadoopFSUtils - +import org.apache.spark.sql.{Row, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.catalog.HoodieCatalogTable import org.apache.spark.sql.execution.command.{AlterTableRenameCommand, AlterTableSetPropertiesCommand} -import org.apache.spark.sql.{Row, SparkSession} /** * Command for alter hudi table's table name. diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/CreateHoodieTableCommand.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/CreateHoodieTableCommand.scala index a857c3a5ded0..a3e94506b2f7 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/CreateHoodieTableCommand.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/CreateHoodieTableCommand.scala @@ -18,27 +18,24 @@ package org.apache.spark.sql.hudi.command import org.apache.hadoop.fs.Path -import org.apache.hudi.common.model.{HoodieFileFormat, HoodieTableType} +import org.apache.hudi.{DataSourceWriteOptions, SparkAdapterSupport} +import org.apache.hudi.common.model.HoodieTableType import org.apache.hudi.common.table.HoodieTableConfig import org.apache.hudi.common.util.ConfigUtils import org.apache.hudi.exception.HoodieException -import org.apache.hudi.hadoop.HoodieParquetInputFormat -import org.apache.hudi.hadoop.realtime.HoodieParquetRealtimeInputFormat import org.apache.hudi.hadoop.utils.HoodieInputFormatUtils -import org.apache.hudi.{DataSourceWriteOptions, SparkAdapterSupport} +import org.apache.spark.{SPARK_VERSION, SparkConf} +import org.apache.spark.sql.{AnalysisException, Row, SparkSession} import org.apache.spark.sql.catalyst.analysis.NoSuchDatabaseException -import org.apache.spark.sql.catalyst.catalog.HoodieCatalogTable.needFilterProps import org.apache.spark.sql.catalyst.catalog._ +import org.apache.spark.sql.catalyst.catalog.HoodieCatalogTable.needFilterProps import org.apache.spark.sql.hive.HiveClientUtils import org.apache.spark.sql.hive.HiveExternalCatalog._ -import org.apache.spark.sql.hudi.HoodieSqlCommonUtils.isUsingHiveCatalog import org.apache.spark.sql.hudi.{HoodieOptionConfig, HoodieSqlCommonUtils} +import org.apache.spark.sql.hudi.HoodieSqlCommonUtils.isUsingHiveCatalog import org.apache.spark.sql.internal.StaticSQLConf.SCHEMA_STRING_LENGTH_THRESHOLD import org.apache.spark.sql.types.StructType -import org.apache.spark.sql.{AnalysisException, Row, SparkSession} -import org.apache.spark.{SPARK_VERSION, SparkConf} -import java.io.{PrintWriter, StringWriter} import scala.collection.JavaConverters._ import scala.collection.mutable import scala.util.control.NonFatal diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/DropHoodieTableCommand.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/DropHoodieTableCommand.scala index 63c4875e3371..09e05ff79235 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/DropHoodieTableCommand.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/DropHoodieTableCommand.scala @@ -23,10 +23,9 @@ import org.apache.hudi.common.model.HoodieTableType import org.apache.hudi.common.util.ConfigUtils import org.apache.hudi.hadoop.fs.HadoopFSUtils import org.apache.hudi.storage.{HoodieStorageUtils, StoragePath} - import org.apache.spark.sql._ -import org.apache.spark.sql.catalyst.catalog._ import org.apache.spark.sql.catalyst.{QualifiedTableName, TableIdentifier} +import org.apache.spark.sql.catalyst.catalog._ /** * Physical plan node for dropping a table. diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/HoodieSparkValidateDuplicateKeyRecordMerger.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/HoodieSparkValidateDuplicateKeyRecordMerger.scala index 8127cb734143..8a64007f0533 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/HoodieSparkValidateDuplicateKeyRecordMerger.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/HoodieSparkValidateDuplicateKeyRecordMerger.scala @@ -21,7 +21,7 @@ import org.apache.avro.Schema import org.apache.hudi.HoodieSparkRecordMerger import org.apache.hudi.common.config.TypedProperties import org.apache.hudi.common.model.{HoodieRecord, HoodieRecordMerger, OperationModeAwareness} -import org.apache.hudi.common.util.{HoodieRecordUtils, collection, Option => HOption} +import org.apache.hudi.common.util.{collection, HoodieRecordUtils, Option => HOption} import org.apache.hudi.exception.HoodieDuplicateKeyException /** diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/RepairHoodieTableCommand.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/RepairHoodieTableCommand.scala index 53d131d78a52..8014c463ea37 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/RepairHoodieTableCommand.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/RepairHoodieTableCommand.scala @@ -17,16 +17,15 @@ package org.apache.spark.sql.hudi.command +import org.apache.hadoop.fs.Path import org.apache.hudi.hadoop.fs.HadoopFSUtils import org.apache.hudi.storage.HoodieStorageUtils - -import org.apache.hadoop.fs.Path +import org.apache.spark.sql.{AnalysisException, Row, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier -import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec import org.apache.spark.sql.catalyst.catalog._ +import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec import org.apache.spark.sql.execution.command.PartitionStatistics import org.apache.spark.sql.hudi.HoodieSqlCommonUtils -import org.apache.spark.sql.{AnalysisException, Row, SparkSession} import org.apache.spark.util.ThreadUtils import java.util.concurrent.TimeUnit.MILLISECONDS diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/ShowHoodieTablePartitionsCommand.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/ShowHoodieTablePartitionsCommand.scala index f43b10fcafc4..09851321eef2 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/ShowHoodieTablePartitionsCommand.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/ShowHoodieTablePartitionsCommand.scala @@ -18,7 +18,6 @@ package org.apache.spark.sql.hudi.command import org.apache.hudi.common.util.PartitionPathEncodeUtils - import org.apache.spark.sql.{Row, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/TruncateHoodieTableCommand.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/TruncateHoodieTableCommand.scala index b2bbf8f2ccf5..06450e60ce31 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/TruncateHoodieTableCommand.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/TruncateHoodieTableCommand.scala @@ -24,13 +24,12 @@ import org.apache.hudi.common.table.HoodieTableMetaClient import org.apache.hudi.exception.HoodieException import org.apache.hudi.hadoop.fs.HadoopFSUtils import org.apache.hudi.storage.{HoodieStorageUtils, StoragePath} - +import org.apache.spark.sql.{AnalysisException, Row, SaveMode, SparkSession} import org.apache.spark.sql.catalyst.TableIdentifier -import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec import org.apache.spark.sql.catalyst.catalog.{CatalogTableType, HoodieCatalogTable} +import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec import org.apache.spark.sql.hudi.HoodieSqlCommonUtils.{getPartitionPathToDrop, normalizePartitionSpec} import org.apache.spark.sql.hudi.ProvidesHoodieConfig -import org.apache.spark.sql.{AnalysisException, Row, SaveMode, SparkSession} /** * Command for truncate hudi table. diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/ValidateDuplicateKeyPayload.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/ValidateDuplicateKeyPayload.scala index 2619d1d9fe15..1a8297e20ce3 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/ValidateDuplicateKeyPayload.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/ValidateDuplicateKeyPayload.scala @@ -23,7 +23,6 @@ import org.apache.hudi.common.model.{DefaultHoodieRecordPayload, HoodieRecord} import org.apache.hudi.common.util.{Option => HOption} import org.apache.hudi.exception.HoodieDuplicateKeyException - import java.util.Properties /** diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/streaming/HoodieMetadataLog.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/streaming/HoodieMetadataLog.scala index 3675e6c05e55..45b8d96ccf9e 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/streaming/HoodieMetadataLog.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/streaming/HoodieMetadataLog.scala @@ -17,13 +17,13 @@ package org.apache.spark.sql.hudi.streaming -import java.io.{BufferedWriter, InputStream, OutputStream, OutputStreamWriter} -import java.nio.charset.StandardCharsets - import org.apache.hudi.common.util.FileIOUtils import org.apache.spark.sql.SparkSession import org.apache.spark.sql.execution.streaming.HDFSMetadataLog +import java.io.{BufferedWriter, InputStream, OutputStream, OutputStreamWriter} +import java.nio.charset.StandardCharsets + /** * Hoodie type metadata log that uses the specified path as the metadata storage. */ diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/streaming/HoodieSourceOffset.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/streaming/HoodieSourceOffset.scala index a5561a65a015..f5d73c55416b 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/streaming/HoodieSourceOffset.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/streaming/HoodieSourceOffset.scala @@ -20,7 +20,6 @@ package org.apache.spark.sql.hudi.streaming import com.fasterxml.jackson.annotation.JsonInclude.Include import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper} import com.fasterxml.jackson.module.scala.DefaultScalaModule -import com.fasterxml.jackson.module.scala.experimental.ScalaObjectMapper import org.apache.hudi.common.table.timeline.HoodieTimeline import org.apache.spark.sql.execution.streaming.{Offset, SerializedOffset} diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/streaming/HoodieStreamSource.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/streaming/HoodieStreamSource.scala index 57eefbfa5e0b..bddc17d91003 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/streaming/HoodieStreamSource.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/streaming/HoodieStreamSource.scala @@ -17,27 +17,26 @@ package org.apache.spark.sql.hudi.streaming +import org.apache.hudi.{AvroConversionUtils, DataSourceReadOptions, IncrementalRelation, MergeOnReadIncrementalRelation, SparkAdapterSupport} import org.apache.hudi.DataSourceReadOptions.INCREMENTAL_READ_HANDLE_HOLLOW_COMMIT import org.apache.hudi.cdc.CDCRelation import org.apache.hudi.common.model.HoodieTableType +import org.apache.hudi.common.table.{HoodieTableMetaClient, TableSchemaResolver} import org.apache.hudi.common.table.cdc.HoodieCDCUtils +import org.apache.hudi.common.table.timeline.TimelineUtils.{handleHollowCommitIfNeeded, HollowCommitHandling} import org.apache.hudi.common.table.timeline.TimelineUtils.HollowCommitHandling._ -import org.apache.hudi.common.table.timeline.TimelineUtils.{HollowCommitHandling, handleHollowCommitIfNeeded} -import org.apache.hudi.common.table.{HoodieTableMetaClient, TableSchemaResolver} import org.apache.hudi.common.util.TablePathUtils import org.apache.hudi.hadoop.fs.HadoopFSUtils +import org.apache.hudi.storage.StoragePath import org.apache.hudi.storage.hadoop.HoodieHadoopStorage -import org.apache.hudi.storage.{HoodieStorageUtils, StoragePath} -import org.apache.hudi.{AvroConversionUtils, DataSourceReadOptions, IncrementalRelation, MergeOnReadIncrementalRelation, SparkAdapterSupport} - import org.apache.spark.internal.Logging import org.apache.spark.rdd.RDD +import org.apache.spark.sql.{DataFrame, SQLContext} import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.execution.streaming.{Offset, Source} import org.apache.spark.sql.hudi.streaming.HoodieSourceOffset.INIT_OFFSET import org.apache.spark.sql.sources.Filter import org.apache.spark.sql.types.StructType -import org.apache.spark.sql.{DataFrame, SQLContext} /** * The Struct Stream Source for Hudi to consume the data by streaming job. diff --git a/hudi-spark-datasource/hudi-spark-common/src/test/scala/org/apache/spark/execution/datasources/TestHoodieInMemoryFileIndex.scala b/hudi-spark-datasource/hudi-spark-common/src/test/scala/org/apache/spark/execution/datasources/TestHoodieInMemoryFileIndex.scala index d370b9e1dd70..e27c728a8e5f 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/test/scala/org/apache/spark/execution/datasources/TestHoodieInMemoryFileIndex.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/test/scala/org/apache/spark/execution/datasources/TestHoodieInMemoryFileIndex.scala @@ -17,9 +17,8 @@ package org.apache.spark.execution.datasources -import org.apache.hudi.storage.{StoragePath, HoodieStorageUtils} - import org.apache.hadoop.fs.Path +import org.apache.hudi.storage.StoragePath import org.apache.hudi.testutils.HoodieClientTestUtils.getSparkConfForTest import org.apache.spark.sql.SparkSession import org.junit.jupiter.api.Assertions.assertEquals diff --git a/hudi-spark-datasource/hudi-spark-common/src/test/scala/org/apache/spark/sql/hive/TestHiveClientUtils.scala b/hudi-spark-datasource/hudi-spark-common/src/test/scala/org/apache/spark/sql/hive/TestHiveClientUtils.scala index 814975ffff87..1fa106fe3d63 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/test/scala/org/apache/spark/sql/hive/TestHiveClientUtils.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/test/scala/org/apache/spark/sql/hive/TestHiveClientUtils.scala @@ -23,8 +23,8 @@ import org.apache.spark.sql.hive.client.HiveClient import org.apache.spark.sql.hive.test.{TestHive, TestHiveContext} import org.apache.spark.sql.internal.StaticSQLConf.CATALOG_IMPLEMENTATION import org.junit.Assume -import org.junit.jupiter.api.TestInstance.Lifecycle import org.junit.jupiter.api.{BeforeAll, Test, TestInstance} +import org.junit.jupiter.api.TestInstance.Lifecycle @TestInstance(Lifecycle.PER_CLASS) class TestHiveClientUtils { diff --git a/hudi-spark-datasource/hudi-spark-common/src/test/scala/org/apache/spark/sql/hudi/TestProvidesHoodieConfig.scala b/hudi-spark-datasource/hudi-spark-common/src/test/scala/org/apache/spark/sql/hudi/TestProvidesHoodieConfig.scala index 8414e41ca6c8..a55729984095 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/test/scala/org/apache/spark/sql/hudi/TestProvidesHoodieConfig.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/test/scala/org/apache/spark/sql/hudi/TestProvidesHoodieConfig.scala @@ -21,7 +21,6 @@ package org.apache.spark.sql.hudi import org.apache.hudi.DataSourceWriteOptions.PARTITIONPATH_FIELD import org.apache.hudi.keygen.{ComplexKeyGenerator, CustomKeyGenerator} - import org.apache.spark.sql.catalyst.catalog.HoodieCatalogTable import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test diff --git a/hudi-spark-datasource/hudi-spark/src/main/java/org/apache/hudi/QuickstartUtils.java b/hudi-spark-datasource/hudi-spark/src/main/java/org/apache/hudi/QuickstartUtils.java index 59674b928fdf..f7eb35090e0c 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/java/org/apache/hudi/QuickstartUtils.java +++ b/hudi-spark-datasource/hudi-spark/src/main/java/org/apache/hudi/QuickstartUtils.java @@ -18,9 +18,6 @@ package org.apache.hudi; -import org.apache.avro.Schema; -import org.apache.avro.generic.GenericData; -import org.apache.avro.generic.GenericRecord; import org.apache.hudi.common.model.HoodieAvroRecord; import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.model.HoodieRecord; @@ -28,6 +25,10 @@ import org.apache.hudi.common.util.Option; import org.apache.hudi.exception.HoodieException; import org.apache.hudi.exception.HoodieIOException; + +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericRecord; import org.apache.spark.sql.Row; import java.io.IOException; diff --git a/hudi-spark-datasource/hudi-spark/src/main/java/org/apache/hudi/payload/AWSDmsAvroPayload.java b/hudi-spark-datasource/hudi-spark/src/main/java/org/apache/hudi/payload/AWSDmsAvroPayload.java index 1411d4f4796c..b3f6e3eef493 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/java/org/apache/hudi/payload/AWSDmsAvroPayload.java +++ b/hudi-spark-datasource/hudi-spark/src/main/java/org/apache/hudi/payload/AWSDmsAvroPayload.java @@ -18,9 +18,10 @@ package org.apache.hudi.payload; -import org.apache.avro.generic.GenericRecord; import org.apache.hudi.common.util.Option; +import org.apache.avro.generic.GenericRecord; + /** * Provides support for seamlessly applying changes captured via Amazon Database Migration Service onto S3. * diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/DedupeSparkJob.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/DedupeSparkJob.scala index 761f2ae49b92..ebc9b3e7e195 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/DedupeSparkJob.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/DedupeSparkJob.scala @@ -23,11 +23,11 @@ import org.apache.hudi.common.table.HoodieTableMetaClient import org.apache.hudi.common.table.view.HoodieTableFileSystemView import org.apache.hudi.common.util.FileIOUtils import org.apache.hudi.exception.HoodieException +import org.apache.hudi.hadoop.fs.HadoopFSUtils.convertToStoragePath import org.apache.hudi.storage.{HoodieStorage, StorageConfiguration, StoragePath} import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileSystem, Path} -import org.apache.hudi.hadoop.fs.HadoopFSUtils.convertToStoragePath import org.apache.spark.sql.{DataFrame, Row, SQLContext} import org.slf4j.LoggerFactory diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/HoodieSparkSessionExtension.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/HoodieSparkSessionExtension.scala index 0e80aca505cd..2cbf479bdead 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/HoodieSparkSessionExtension.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/HoodieSparkSessionExtension.scala @@ -17,7 +17,8 @@ package org.apache.spark.sql.hudi -import org.apache.hudi.{HoodieSparkUtils, SparkAdapterSupport} +import org.apache.hudi.SparkAdapterSupport + import org.apache.spark.sql.SparkSessionExtensions import org.apache.spark.sql.hudi.analysis.HoodieAnalysis import org.apache.spark.sql.parser.HoodieCommonSqlParser diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieAnalysis.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieAnalysis.scala index f19558a1dc47..aef98b4e91d8 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieAnalysis.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieAnalysis.scala @@ -20,6 +20,7 @@ package org.apache.spark.sql.hudi.analysis import org.apache.hudi.common.util.ReflectionUtils import org.apache.hudi.common.util.ReflectionUtils.loadClass import org.apache.hudi.{HoodieSparkUtils, SparkAdapterSupport} + import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable} @@ -36,6 +37,7 @@ import org.apache.spark.sql.hudi.command.procedures.{HoodieProcedures, Procedure import org.apache.spark.sql.{AnalysisException, SparkSession} import java.util + import scala.collection.mutable.ListBuffer object HoodieAnalysis extends SparkAdapterSupport { diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodiePruneFileSourcePartitions.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodiePruneFileSourcePartitions.scala index bec41a2479eb..08b5685f46a6 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodiePruneFileSourcePartitions.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodiePruneFileSourcePartitions.scala @@ -19,6 +19,7 @@ package org.apache.spark.sql.hudi.analysis import org.apache.hudi.SparkAdapterSupport.sparkAdapter import org.apache.hudi.{HoodieBaseRelation, HoodieFileIndex} + import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.catalog.CatalogStatistics import org.apache.spark.sql.catalyst.expressions.{And, AttributeReference, AttributeSet, Expression, ExpressionSet, NamedExpression, PredicateHelper, SubqueryExpression} diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/CallProcedureHoodieCommand.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/CallProcedureHoodieCommand.scala index f18509696193..e5f448eebaaa 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/CallProcedureHoodieCommand.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/CallProcedureHoodieCommand.scala @@ -18,6 +18,7 @@ package org.apache.spark.sql.hudi.command import org.apache.hudi.SparkAdapterSupport + import org.apache.spark.sql.catalyst.expressions.Attribute import org.apache.spark.sql.hudi.command.procedures.{Procedure, ProcedureArgs} import org.apache.spark.sql.{Row, SparkSession} diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/CompactionHoodieTableCommand.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/CompactionHoodieTableCommand.scala index 426d6f27720b..f0c1bd48bc63 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/CompactionHoodieTableCommand.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/CompactionHoodieTableCommand.scala @@ -18,6 +18,7 @@ package org.apache.spark.sql.hudi.command import org.apache.hudi.SparkAdapterSupport + import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.catalyst.expressions.Attribute import org.apache.spark.sql.catalyst.plans.logical.CompactionOperation.CompactionOperation diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/CompactionShowHoodieTableCommand.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/CompactionShowHoodieTableCommand.scala index 070e93912aba..d97be7120cf6 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/CompactionShowHoodieTableCommand.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/CompactionShowHoodieTableCommand.scala @@ -18,6 +18,7 @@ package org.apache.spark.sql.hudi.command import org.apache.hudi.SparkAdapterSupport + import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.catalyst.expressions.Attribute import org.apache.spark.sql.hudi.HoodieSqlCommonUtils.getTableLocation diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/CreateHoodieTableAsSelectCommand.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/CreateHoodieTableAsSelectCommand.scala index bd5f9b7f901d..e3521f288dbd 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/CreateHoodieTableAsSelectCommand.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/CreateHoodieTableAsSelectCommand.scala @@ -17,14 +17,15 @@ package org.apache.spark.sql.hudi.command -import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.fs.Path import org.apache.hudi.DataSourceWriteOptions import org.apache.hudi.common.util.ConfigUtils import org.apache.hudi.common.util.ValidationUtils.checkState import org.apache.hudi.config.HoodieWriteConfig import org.apache.hudi.hive.HiveSyncConfigHolder import org.apache.hudi.sql.InsertMode + +import org.apache.hadoop.conf.Configuration +import org.apache.hadoop.fs.Path import org.apache.spark.sql.catalyst.catalog.HoodieCatalogTable.needFilterProps import org.apache.spark.sql.catalyst.catalog.{CatalogTable, CatalogTableType, HoodieCatalogTable} import org.apache.spark.sql.catalyst.plans.QueryPlan diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/DeleteHoodieTableCommand.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/DeleteHoodieTableCommand.scala index 55f2ebb8ac67..b06374496396 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/DeleteHoodieTableCommand.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/DeleteHoodieTableCommand.scala @@ -17,8 +17,9 @@ package org.apache.spark.sql.hudi.command -import org.apache.hudi.DataSourceWriteOptions.{SPARK_SQL_WRITES_PREPPED_KEY, SPARK_SQL_OPTIMIZED_WRITES} +import org.apache.hudi.DataSourceWriteOptions.{SPARK_SQL_OPTIMIZED_WRITES, SPARK_SQL_WRITES_PREPPED_KEY} import org.apache.hudi.SparkAdapterSupport + import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.catalog.HoodieCatalogTable import org.apache.spark.sql.catalyst.plans.logical.{DeleteFromTable, Filter} diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/IndexCommands.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/IndexCommands.scala index 5de3f705e3ef..b4b35595052a 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/IndexCommands.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/IndexCommands.scala @@ -34,6 +34,7 @@ import org.apache.spark.sql.hudi.HoodieSqlCommonUtils.getTableLocation import org.apache.spark.sql.{Row, SparkSession} import java.util + import scala.collection.JavaConverters.{collectionAsScalaIterableConverter, mapAsJavaMapConverter} case class CreateIndexCommand(table: CatalogTable, diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/InsertIntoHoodieTableCommand.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/InsertIntoHoodieTableCommand.scala index cf8abfd9afc8..8b07d9bda716 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/InsertIntoHoodieTableCommand.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/InsertIntoHoodieTableCommand.scala @@ -19,6 +19,7 @@ package org.apache.spark.sql.hudi.command import org.apache.hudi.exception.HoodieException import org.apache.hudi.{HoodieSparkSqlWriter, SparkAdapterSupport} + import org.apache.spark.internal.Logging import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.catalog.{CatalogTable, HoodieCatalogTable} diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/MergeIntoHoodieTableCommand.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/MergeIntoHoodieTableCommand.scala index 79cd2646e081..a549372be5c3 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/MergeIntoHoodieTableCommand.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/MergeIntoHoodieTableCommand.scala @@ -17,7 +17,6 @@ package org.apache.spark.sql.hudi.command -import org.apache.avro.Schema import org.apache.hudi.AvroConversionUtils.convertStructTypeToAvroSchema import org.apache.hudi.DataSourceWriteOptions._ import org.apache.hudi.HoodieSparkSqlWriter.CANONICALIZE_SCHEMA @@ -31,6 +30,8 @@ import org.apache.hudi.hive.HiveSyncConfigHolder import org.apache.hudi.sync.common.HoodieSyncConfig import org.apache.hudi.util.JFunction.scalaFunction1Noop import org.apache.hudi.{AvroConversionUtils, DataSourceWriteOptions, HoodieSparkSqlWriter, HoodieSparkUtils, SparkAdapterSupport} + +import org.apache.avro.Schema import org.apache.spark.sql.HoodieCatalystExpressionUtils.{MatchCast, attributeEquals} import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.catalog.HoodieCatalogTable @@ -49,6 +50,7 @@ import org.apache.spark.sql.hudi.command.payload.ExpressionPayload._ import org.apache.spark.sql.types.{BooleanType, StructField, StructType} import java.util.Base64 + import scala.collection.JavaConverters._ /** diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/UpdateHoodieTableCommand.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/UpdateHoodieTableCommand.scala index e35e4939f049..1e0479d657ff 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/UpdateHoodieTableCommand.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/UpdateHoodieTableCommand.scala @@ -17,8 +17,9 @@ package org.apache.spark.sql.hudi.command -import org.apache.hudi.DataSourceWriteOptions.{SPARK_SQL_WRITES_PREPPED_KEY, SPARK_SQL_OPTIMIZED_WRITES} +import org.apache.hudi.DataSourceWriteOptions.{SPARK_SQL_OPTIMIZED_WRITES, SPARK_SQL_WRITES_PREPPED_KEY} import org.apache.hudi.SparkAdapterSupport + import org.apache.spark.sql.HoodieCatalystExpressionUtils.attributeEquals import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.catalog.HoodieCatalogTable diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/UuidKeyGenerator.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/UuidKeyGenerator.scala index 14a0074fef85..1ed8108b5c56 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/UuidKeyGenerator.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/UuidKeyGenerator.scala @@ -17,11 +17,13 @@ package org.apache.spark.sql.hudi.command -import java.util.UUID -import org.apache.avro.generic.GenericRecord import org.apache.hudi.common.config.TypedProperties + +import org.apache.avro.generic.GenericRecord import org.apache.spark.sql.Row +import java.util.UUID + /** * A KeyGenerator which use the uuid as the record key. */ diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/payload/ExpressionPayload.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/payload/ExpressionPayload.scala index 404bcf4ff622..04783d0e0406 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/payload/ExpressionPayload.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/payload/ExpressionPayload.scala @@ -17,9 +17,6 @@ package org.apache.spark.sql.hudi.command.payload -import com.github.benmanes.caffeine.cache.{Cache, Caffeine} -import org.apache.avro.Schema -import org.apache.avro.generic.{GenericData, GenericRecord, IndexedRecord} import org.apache.hudi.AvroConversionUtils.{convertAvroSchemaToStructType, convertStructTypeToAvroSchema} import org.apache.hudi.DataSourceWriteOptions._ import org.apache.hudi.SparkAdapterSupport.sparkAdapter @@ -31,6 +28,10 @@ import org.apache.hudi.common.util.ValidationUtils.checkState import org.apache.hudi.common.util.{BinaryUtil, ConfigUtils, StringUtils, ValidationUtils, Option => HOption} import org.apache.hudi.config.HoodieWriteConfig import org.apache.hudi.exception.HoodieException + +import com.github.benmanes.caffeine.cache.{Cache, Caffeine} +import org.apache.avro.Schema +import org.apache.avro.generic.{GenericData, GenericRecord, IndexedRecord} import org.apache.spark.internal.Logging import org.apache.spark.serializer.{KryoSerializer, SerializerInstance} import org.apache.spark.sql.avro.{HoodieAvroDeserializer, HoodieAvroSerializer} @@ -43,6 +44,7 @@ import org.apache.spark.{SparkConf, SparkEnv} import java.nio.ByteBuffer import java.util.function.{Function, Supplier} import java.util.{Base64, Objects, Properties} + import scala.collection.JavaConverters._ /** diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ArchiveCommitsProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ArchiveCommitsProcedure.scala index 51c6e8edc8d9..efc5a0cc5c2a 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ArchiveCommitsProcedure.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ArchiveCommitsProcedure.scala @@ -19,6 +19,7 @@ package org.apache.spark.sql.hudi.command.procedures import org.apache.hudi.SparkAdapterSupport import org.apache.hudi.cli.ArchiveExecutorUtils + import org.apache.spark.internal.Logging import org.apache.spark.sql.Row import org.apache.spark.sql.types._ diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/CommitsCompareProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/CommitsCompareProcedure.scala index d51c58289f50..3092df255251 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/CommitsCompareProcedure.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/CommitsCompareProcedure.scala @@ -24,6 +24,7 @@ import org.apache.spark.sql.Row import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType} import java.util.function.Supplier + import scala.collection.JavaConverters._ class CommitsCompareProcedure() extends BaseProcedure with ProcedureBuilder { diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/CopyToTableProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/CopyToTableProcedure.scala index e661f0a35ae0..579e0e1ba676 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/CopyToTableProcedure.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/CopyToTableProcedure.scala @@ -18,6 +18,7 @@ package org.apache.spark.sql.hudi.command.procedures import org.apache.hudi.DataSourceReadOptions + import org.apache.spark.internal.Logging import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType} import org.apache.spark.sql.{Row, SaveMode} diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/CopyToTempViewProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/CopyToTempViewProcedure.scala index a23eea1363ef..bfa43407003b 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/CopyToTempViewProcedure.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/CopyToTempViewProcedure.scala @@ -18,6 +18,7 @@ package org.apache.spark.sql.hudi.command.procedures import org.apache.hudi.DataSourceReadOptions + import org.apache.spark.internal.Logging import org.apache.spark.sql.Row import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType} diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/DeleteMarkerProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/DeleteMarkerProcedure.scala index 8d73a753cf4c..c37053eef9b4 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/DeleteMarkerProcedure.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/DeleteMarkerProcedure.scala @@ -21,11 +21,13 @@ import org.apache.hudi.HoodieCLIUtils import org.apache.hudi.client.SparkRDDWriteClient import org.apache.hudi.table.HoodieSparkTable import org.apache.hudi.table.marker.WriteMarkersFactory + import org.apache.spark.internal.Logging import org.apache.spark.sql.Row import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType} import java.util.function.Supplier + import scala.util.{Failure, Success, Try} class DeleteMarkerProcedure extends BaseProcedure with ProcedureBuilder with Logging { diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ExportInstantsProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ExportInstantsProcedure.scala index 3992e43d1d27..5d1be4159233 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ExportInstantsProcedure.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ExportInstantsProcedure.scala @@ -28,12 +28,12 @@ import org.apache.hudi.common.table.log.block.HoodieAvroDataBlock import org.apache.hudi.common.table.timeline.{HoodieInstant, HoodieTimeline, TimelineMetadataUtils} import org.apache.hudi.exception.HoodieException import org.apache.hudi.hadoop.fs.HadoopFSUtils +import org.apache.hudi.hadoop.fs.HadoopFSUtils.convertToStoragePath import org.apache.hudi.storage.{HoodieStorage, HoodieStorageUtils, StoragePath} import org.apache.avro.generic.GenericRecord import org.apache.avro.specific.SpecificData import org.apache.hadoop.fs.{FileStatus, Path} -import org.apache.hudi.hadoop.fs.HadoopFSUtils.convertToStoragePath import org.apache.spark.internal.Logging import org.apache.spark.sql.Row import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType} @@ -42,6 +42,7 @@ import java.io.File import java.util import java.util.Collections import java.util.function.Supplier + import scala.collection.JavaConverters._ import scala.util.control.Breaks.break diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/HdfsParquetImportProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/HdfsParquetImportProcedure.scala index addfaca049fe..29a800505cac 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/HdfsParquetImportProcedure.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/HdfsParquetImportProcedure.scala @@ -18,11 +18,13 @@ package org.apache.spark.sql.hudi.command.procedures import org.apache.hudi.cli.HDFSParquetImporterUtils + import org.apache.spark.internal.Logging import org.apache.spark.sql.Row import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType} import java.util.function.Supplier + import scala.language.higherKinds class HdfsParquetImportProcedure extends BaseProcedure with ProcedureBuilder with Logging { diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/HelpProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/HelpProcedure.scala index 43b7fcfc6753..13c9fc888adc 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/HelpProcedure.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/HelpProcedure.scala @@ -18,6 +18,7 @@ package org.apache.spark.sql.hudi.command.procedures import org.apache.hudi.exception.HoodieException + import org.apache.spark.internal.Logging import org.apache.spark.sql.Row import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType} diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/HiveSyncProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/HiveSyncProcedure.scala index f53f4f7d776f..975b2a8a30da 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/HiveSyncProcedure.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/HiveSyncProcedure.scala @@ -17,11 +17,12 @@ package org.apache.spark.sql.hudi.command.procedures -import org.apache.hadoop.hive.conf.HiveConf import org.apache.hudi.HoodieCLIUtils import org.apache.hudi.exception.HoodieException import org.apache.hudi.hive.{HiveSyncConfig, HiveSyncConfigHolder, HiveSyncTool} import org.apache.hudi.sync.common.HoodieSyncConfig + +import org.apache.hadoop.hive.conf.HiveConf import org.apache.spark.internal.Logging import org.apache.spark.sql.Row import org.apache.spark.sql.hudi.ProvidesHoodieConfig diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/HoodieProcedureUtils.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/HoodieProcedureUtils.scala index b7675ebbd37a..f433c58179b5 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/HoodieProcedureUtils.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/HoodieProcedureUtils.scala @@ -25,6 +25,7 @@ import org.apache.hudi.exception.HoodieException import org.apache.spark.sql.catalyst.expressions.GenericInternalRow import java.util + import scala.collection.JavaConverters._ object HoodieProcedureUtils { diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/Procedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/Procedure.scala index f34e30615982..5ef5c7c8902c 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/Procedure.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/Procedure.scala @@ -18,12 +18,8 @@ package org.apache.spark.sql.hudi.command.procedures import org.apache.spark.sql.Row -import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.types.StructType -import java.util -import scala.collection.mutable - /** * An interface representing a stored procedure available for execution. */ diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairAddpartitionmetaProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairAddpartitionmetaProcedure.scala index 1f523aabc993..1eadad0187d5 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairAddpartitionmetaProcedure.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairAddpartitionmetaProcedure.scala @@ -27,6 +27,7 @@ import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType} import java.util import java.util.function.Supplier + import scala.collection.JavaConverters._ class RepairAddpartitionmetaProcedure extends BaseProcedure with ProcedureBuilder with Logging { diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairCorruptedCleanFilesProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairCorruptedCleanFilesProcedure.scala index e0e0db63a83e..b0bc57ab7c17 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairCorruptedCleanFilesProcedure.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairCorruptedCleanFilesProcedure.scala @@ -28,6 +28,7 @@ import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType} import java.io.IOException import java.util.function.Supplier + import scala.collection.JavaConverters.asScalaIteratorConverter class RepairCorruptedCleanFilesProcedure extends BaseProcedure with ProcedureBuilder with Logging { diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairDeduplicateProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairDeduplicateProcedure.scala index e1317151a0d9..f1e462e6b94a 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairDeduplicateProcedure.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairDeduplicateProcedure.scala @@ -27,6 +27,7 @@ import org.apache.spark.sql.hudi.{DeDupeType, DedupeSparkJob} import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType} import java.util.function.Supplier + import scala.util.{Failure, Success, Try} class RepairDeduplicateProcedure extends BaseProcedure with ProcedureBuilder with Logging { diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RollbackToInstantTimeProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RollbackToInstantTimeProcedure.scala index 2aba13a8948c..6af5412f4da6 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RollbackToInstantTimeProcedure.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RollbackToInstantTimeProcedure.scala @@ -20,7 +20,6 @@ package org.apache.spark.sql.hudi.command.procedures import org.apache.hudi.HoodieCLIUtils import org.apache.hudi.client.SparkRDDWriteClient import org.apache.hudi.common.table.HoodieTableMetaClient -import org.apache.hudi.common.table.timeline.HoodieTimeline import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion import org.apache.hudi.common.util.Option import org.apache.hudi.config.HoodieWriteConfig.ROLLBACK_USING_MARKERS_ENABLE diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RunBootstrapProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RunBootstrapProcedure.scala index de257017cd9c..065b312b9fa1 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RunBootstrapProcedure.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RunBootstrapProcedure.scala @@ -17,7 +17,6 @@ package org.apache.spark.sql.hudi.command.procedures -import org.apache.hudi.{DataSourceWriteOptions, HoodieCLIUtils} import org.apache.hudi.cli.BootstrapExecutorUtils import org.apache.hudi.cli.HDFSParquetImporterUtils.{buildProperties, readConfig} import org.apache.hudi.common.config.TypedProperties @@ -26,6 +25,7 @@ import org.apache.hudi.config.{HoodieBootstrapConfig, HoodieWriteConfig} import org.apache.hudi.hadoop.fs.HadoopFSUtils import org.apache.hudi.keygen.constant.KeyGeneratorType import org.apache.hudi.storage.StoragePath +import org.apache.hudi.{DataSourceWriteOptions, HoodieCLIUtils} import org.apache.spark.internal.Logging import org.apache.spark.sql.Row diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RunCleanProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RunCleanProcedure.scala index c666773c4bdb..21139320d59f 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RunCleanProcedure.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RunCleanProcedure.scala @@ -19,9 +19,9 @@ package org.apache.spark.sql.hudi.command.procedures import org.apache.hudi.HoodieCLIUtils import org.apache.hudi.client.SparkRDDWriteClient -import org.apache.hudi.common.table.timeline.HoodieActiveTimeline import org.apache.hudi.common.util.JsonUtils import org.apache.hudi.config.HoodieCleanConfig + import org.apache.spark.internal.Logging import org.apache.spark.sql.Row import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType} diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RunClusteringProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RunClusteringProcedure.scala index e60898db0469..f2d553a13d35 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RunClusteringProcedure.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RunClusteringProcedure.scala @@ -35,6 +35,7 @@ import org.apache.spark.sql.execution.datasources.FileStatusCache import org.apache.spark.sql.types._ import java.util.function.Supplier + import scala.collection.JavaConverters._ class RunClusteringProcedure extends BaseProcedure diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RunCompactionProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RunCompactionProcedure.scala index 837d573b8033..cdebf7cf9f78 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RunCompactionProcedure.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RunCompactionProcedure.scala @@ -30,6 +30,7 @@ import org.apache.spark.sql.Row import org.apache.spark.sql.types._ import java.util.function.Supplier + import scala.collection.JavaConverters._ class RunCompactionProcedure extends BaseProcedure with ProcedureBuilder with SparkAdapterSupport with Logging { diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RunTTLProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RunTTLProcedure.scala index 2d3e704ad129..d3f9606fecfa 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RunTTLProcedure.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RunTTLProcedure.scala @@ -20,11 +20,13 @@ package org.apache.spark.sql.hudi.command.procedures import org.apache.hudi.HoodieCLIUtils import org.apache.hudi.client.SparkRDDWriteClient import org.apache.hudi.config.HoodieTTLConfig + import org.apache.spark.internal.Logging import org.apache.spark.sql.Row import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType} import java.util.function.Supplier + import scala.collection.JavaConverters._ class RunTTLProcedure extends BaseProcedure with ProcedureBuilder with Logging { diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowArchivedCommitsProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowArchivedCommitsProcedure.scala index 646547debe5e..5a76a0fd44ad 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowArchivedCommitsProcedure.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowArchivedCommitsProcedure.scala @@ -29,6 +29,7 @@ import java.time.ZonedDateTime import java.util import java.util.function.Supplier import java.util.{Collections, Date} + import scala.collection.JavaConverters._ class ShowArchivedCommitsProcedure(includeExtraMetadata: Boolean) extends BaseProcedure with ProcedureBuilder { diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowBootstrapMappingProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowBootstrapMappingProcedure.scala index 19ec7711fade..f9783fcf008c 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowBootstrapMappingProcedure.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowBootstrapMappingProcedure.scala @@ -27,6 +27,7 @@ import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType} import java.util import java.util.function.Supplier + import scala.collection.JavaConverters._ class ShowBootstrapMappingProcedure extends BaseProcedure with ProcedureBuilder { diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowClusteringProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowClusteringProcedure.scala index ad92c34ea9ee..2f8a92e5ee6b 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowClusteringProcedure.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowClusteringProcedure.scala @@ -26,6 +26,7 @@ import org.apache.spark.sql.Row import org.apache.spark.sql.types._ import java.util.function.Supplier + import scala.collection.JavaConverters._ class ShowClusteringProcedure extends BaseProcedure with ProcedureBuilder with SparkAdapterSupport with Logging { diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowCommitFilesProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowCommitFilesProcedure.scala index b99a6694a330..1801ed647f4b 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowCommitFilesProcedure.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowCommitFilesProcedure.scala @@ -28,6 +28,7 @@ import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType} import java.util import java.util.List import java.util.function.Supplier + import scala.collection.JavaConverters._ class ShowCommitFilesProcedure() extends BaseProcedure with ProcedureBuilder { diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowCommitPartitionsProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowCommitPartitionsProcedure.scala index 7aead8f0c855..b7180810e4c7 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowCommitPartitionsProcedure.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowCommitPartitionsProcedure.scala @@ -28,6 +28,7 @@ import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType} import java.util import java.util.List import java.util.function.Supplier + import scala.collection.JavaConverters._ class ShowCommitPartitionsProcedure() extends BaseProcedure with ProcedureBuilder { diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowCommitWriteStatsProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowCommitWriteStatsProcedure.scala index 4f6358a73ee7..f5f58966f734 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowCommitWriteStatsProcedure.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowCommitWriteStatsProcedure.scala @@ -27,6 +27,7 @@ import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType} import java.util import java.util.function.Supplier + import scala.collection.JavaConverters._ class ShowCommitWriteStatsProcedure() extends BaseProcedure with ProcedureBuilder { diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowCommitsProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowCommitsProcedure.scala index af1d5d89f105..6e1df2a9b68f 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowCommitsProcedure.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowCommitsProcedure.scala @@ -27,6 +27,7 @@ import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType} import java.util import java.util.Collections import java.util.function.Supplier + import scala.collection.JavaConverters._ class ShowCommitsProcedure(includeExtraMetadata: Boolean) extends BaseProcedure with ProcedureBuilder { diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowCompactionProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowCompactionProcedure.scala index 6a0a8d1a1aec..2fb629d56f64 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowCompactionProcedure.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowCompactionProcedure.scala @@ -27,6 +27,7 @@ import org.apache.spark.sql.Row import org.apache.spark.sql.types._ import java.util.function.Supplier + import scala.collection.JavaConverters._ class ShowCompactionProcedure extends BaseProcedure with ProcedureBuilder with SparkAdapterSupport with Logging { diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowFileSystemViewProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowFileSystemViewProcedure.scala index f19cd105c81d..aa4e11304ebf 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowFileSystemViewProcedure.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowFileSystemViewProcedure.scala @@ -19,11 +19,11 @@ package org.apache.spark.sql.hudi.command.procedures import org.apache.hudi.common.fs.FSUtils import org.apache.hudi.common.model.{FileSlice, HoodieLogFile} +import org.apache.hudi.common.table.HoodieTableMetaClient import org.apache.hudi.common.table.timeline.{CompletionTimeQueryView, HoodieDefaultTimeline, HoodieInstant, HoodieTimeline} import org.apache.hudi.common.table.view.HoodieTableFileSystemView import org.apache.hudi.common.util import org.apache.hudi.exception.HoodieException -import org.apache.hudi.common.table.HoodieTableMetaClient import org.apache.hudi.storage.StoragePath import org.apache.spark.sql.Row diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowFsPathDetailProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowFsPathDetailProcedure.scala index b98a1309a06b..29a8d4e62604 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowFsPathDetailProcedure.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowFsPathDetailProcedure.scala @@ -17,7 +17,6 @@ package org.apache.spark.sql.hudi.command.procedures -import org.apache.hudi.common.fs.FSUtils import org.apache.hudi.hadoop.fs.HadoopFSUtils import org.apache.hadoop.fs.{ContentSummary, FileStatus, Path} diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowHoodieLogFileMetadataProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowHoodieLogFileMetadataProcedure.scala index 05ea6ae4548a..891da52a05a2 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowHoodieLogFileMetadataProcedure.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowHoodieLogFileMetadataProcedure.scala @@ -27,13 +27,12 @@ import org.apache.hudi.common.table.log.block.{HoodieCorruptBlock, HoodieDataBlo import org.apache.hudi.storage.StoragePath import com.fasterxml.jackson.databind.ObjectMapper -import org.apache.parquet.avro.AvroSchemaConverter import org.apache.spark.sql.Row import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType} -import java.util.Objects import java.util.concurrent.atomic.AtomicInteger import java.util.function.Supplier + import scala.collection.JavaConverters.{asScalaBufferConverter, asScalaIteratorConverter, mapAsScalaMapConverter} class ShowHoodieLogFileMetadataProcedure extends BaseProcedure with ProcedureBuilder { diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowHoodieLogFileRecordsProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowHoodieLogFileRecordsProcedure.scala index 1c5221081e25..41a8fd433fcf 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowHoodieLogFileRecordsProcedure.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowHoodieLogFileRecordsProcedure.scala @@ -17,7 +17,6 @@ package org.apache.spark.sql.hudi.command.procedures -import org.apache.avro.generic.IndexedRecord import org.apache.hudi.common.config.{HoodieCommonConfig, HoodieMemoryConfig, HoodieReaderConfig} import org.apache.hudi.common.fs.FSUtils import org.apache.hudi.common.model.HoodieRecord.HoodieRecordType @@ -27,11 +26,14 @@ import org.apache.hudi.common.table.log.block.HoodieDataBlock import org.apache.hudi.common.table.log.{HoodieLogFormat, HoodieMergedLogRecordScanner} import org.apache.hudi.common.util.{FileIOUtils, ValidationUtils} import org.apache.hudi.storage.StoragePath + +import org.apache.avro.generic.IndexedRecord import org.apache.spark.sql.Row import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType} import java.util.Objects import java.util.function.Supplier + import scala.collection.JavaConverters._ class ShowHoodieLogFileRecordsProcedure extends BaseProcedure with ProcedureBuilder { diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowMetadataTablePartitionsProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowMetadataTablePartitionsProcedure.scala index 9f0e23a86fe2..6767b027206b 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowMetadataTablePartitionsProcedure.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowMetadataTablePartitionsProcedure.scala @@ -21,7 +21,6 @@ import org.apache.hudi.client.common.HoodieSparkEngineContext import org.apache.hudi.common.config.HoodieMetadataConfig import org.apache.hudi.common.util.HoodieTimer import org.apache.hudi.exception.HoodieException -import org.apache.hudi.hadoop.fs.HadoopFSUtils import org.apache.hudi.metadata.HoodieBackedTableMetadata import org.apache.hudi.storage.hadoop.HoodieHadoopStorage diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowTablePropertiesProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowTablePropertiesProcedure.scala index a8872b75678f..987769eb2c17 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowTablePropertiesProcedure.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowTablePropertiesProcedure.scala @@ -22,6 +22,7 @@ import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType} import java.util import java.util.function.Supplier + import scala.collection.JavaConverters._ class ShowTablePropertiesProcedure() extends BaseProcedure with ProcedureBuilder { diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/StatsFileSizeProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/StatsFileSizeProcedure.scala index cb5c0d67b668..3ffb23f8ca1c 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/StatsFileSizeProcedure.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/StatsFileSizeProcedure.scala @@ -27,6 +27,7 @@ import org.apache.spark.sql.hudi.command.procedures.StatsFileSizeProcedure.MAX_F import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType} import java.util.function.Supplier + import scala.collection.JavaConverters.{asScalaBufferConverter, mapAsScalaMapConverter} class StatsFileSizeProcedure extends BaseProcedure with ProcedureBuilder { diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/StatsWriteAmplificationProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/StatsWriteAmplificationProcedure.scala index 5556fd93b33e..433687fb0502 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/StatsWriteAmplificationProcedure.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/StatsWriteAmplificationProcedure.scala @@ -24,6 +24,7 @@ import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType} import java.text.DecimalFormat import java.util.function.Supplier + import scala.collection.JavaConverters.asScalaIteratorConverter class StatsWriteAmplificationProcedure extends BaseProcedure with ProcedureBuilder { diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/UpgradeOrDowngradeProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/UpgradeOrDowngradeProcedure.scala index 63eadb740e1a..971bac743ed1 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/UpgradeOrDowngradeProcedure.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/UpgradeOrDowngradeProcedure.scala @@ -17,22 +17,23 @@ package org.apache.spark.sql.hudi.command.procedures +import org.apache.hudi.HoodieCLIUtils import org.apache.hudi.client.common.HoodieSparkEngineContext import org.apache.hudi.common.model.HoodieFailedWritesCleaningPolicy import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient, HoodieTableVersion} import org.apache.hudi.common.util.Option import org.apache.hudi.config.{HoodieCleanConfig, HoodieIndexConfig, HoodieWriteConfig} +import org.apache.hudi.hadoop.fs.HadoopFSUtils import org.apache.hudi.index.HoodieIndex import org.apache.hudi.table.upgrade.{SparkUpgradeDowngradeHelper, UpgradeDowngrade} -import org.apache.hudi.HoodieCLIUtils -import org.apache.hudi.hadoop.fs.HadoopFSUtils import org.apache.spark.internal.Logging import org.apache.spark.sql.Row import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType} import java.util.function.Supplier + import scala.collection.JavaConverters._ import scala.util.{Failure, Success, Try} diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ValidateHoodieSyncProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ValidateHoodieSyncProcedure.scala index 57a17b213b88..05192dbb051c 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ValidateHoodieSyncProcedure.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ValidateHoodieSyncProcedure.scala @@ -30,6 +30,7 @@ import org.joda.time.DateTime import java.io.IOException import java.sql.{Connection, DriverManager, ResultSet, SQLException} import java.util.function.Supplier + import scala.collection.JavaConverters._ class ValidateHoodieSyncProcedure extends BaseProcedure with ProcedureBuilder with Logging { diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/parser/HoodieCommonSqlParser.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/parser/HoodieCommonSqlParser.scala index 6f78423fd10e..8d820696b3bf 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/parser/HoodieCommonSqlParser.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/parser/HoodieCommonSqlParser.scala @@ -17,19 +17,20 @@ package org.apache.spark.sql.parser -import org.antlr.v4.runtime.{CharStream, CharStreams, CodePointCharStream, CommonTokenStream, IntStream} -import org.antlr.v4.runtime.atn.PredictionMode -import org.antlr.v4.runtime.misc.{Interval, ParseCancellationException} import org.apache.hudi.SparkAdapterSupport import org.apache.hudi.spark.sql.parser.{HoodieSqlCommonLexer, HoodieSqlCommonParser} + +import org.antlr.v4.runtime.atn.PredictionMode +import org.antlr.v4.runtime.misc.{Interval, ParseCancellationException} +import org.antlr.v4.runtime.{CharStream, CharStreams, CodePointCharStream, CommonTokenStream, IntStream} import org.apache.spark.internal.Logging -import org.apache.spark.sql.{AnalysisException, SparkSession} -import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier} import org.apache.spark.sql.catalyst.expressions.Expression import org.apache.spark.sql.catalyst.parser.{ParseErrorListener, ParseException, ParserInterface} import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.catalyst.trees.Origin +import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier} import org.apache.spark.sql.types.{DataType, StructType} +import org.apache.spark.sql.{AnalysisException, SparkSession} class HoodieCommonSqlParser(session: SparkSession, delegate: ParserInterface) extends ParserInterface with Logging with SparkAdapterSupport { diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/parser/HoodieSqlCommonAstBuilder.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/parser/HoodieSqlCommonAstBuilder.scala index b6bd7f977684..9026fe1f8814 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/parser/HoodieSqlCommonAstBuilder.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/parser/HoodieSqlCommonAstBuilder.scala @@ -17,11 +17,12 @@ package org.apache.spark.sql.parser -import org.antlr.v4.runtime.ParserRuleContext -import org.antlr.v4.runtime.tree.{ParseTree, RuleNode, TerminalNode} import org.apache.hudi.SparkAdapterSupport import org.apache.hudi.spark.sql.parser.HoodieSqlCommonBaseVisitor import org.apache.hudi.spark.sql.parser.HoodieSqlCommonParser._ + +import org.antlr.v4.runtime.ParserRuleContext +import org.antlr.v4.runtime.tree.{ParseTree, RuleNode, TerminalNode} import org.apache.spark.internal.Logging import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.TableIdentifier @@ -30,7 +31,6 @@ import org.apache.spark.sql.catalyst.expressions.{Expression, Literal} import org.apache.spark.sql.catalyst.parser.{ParserInterface, ParserUtils} import org.apache.spark.sql.catalyst.plans.logical._ -import java.util.Locale import scala.collection.JavaConverters._ class HoodieSqlCommonAstBuilder(session: SparkSession, delegate: ParserInterface) diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/TestQuickstartUtils.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/TestQuickstartUtils.java index 2042249ecd78..6f45a0793f2d 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/TestQuickstartUtils.java +++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/TestQuickstartUtils.java @@ -19,6 +19,7 @@ package org.apache.hudi; import org.apache.hudi.exception.HoodieException; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/keygen/TestComplexKeyGenerator.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/keygen/TestComplexKeyGenerator.java index 2fa09861d25c..1b7890ad6187 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/keygen/TestComplexKeyGenerator.java +++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/keygen/TestComplexKeyGenerator.java @@ -18,7 +18,6 @@ package org.apache.hudi.keygen; -import org.apache.avro.generic.GenericRecord; import org.apache.hudi.AvroConversionUtils; import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.common.model.HoodieKey; @@ -26,6 +25,8 @@ import org.apache.hudi.exception.HoodieKeyException; import org.apache.hudi.keygen.constant.KeyGeneratorOptions; import org.apache.hudi.testutils.KeyGeneratorTestUtilities; + +import org.apache.avro.generic.GenericRecord; import org.apache.spark.sql.Row; import org.apache.spark.sql.catalyst.InternalRow; import org.apache.spark.unsafe.types.UTF8String; diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/keygen/TestGlobalDeleteRecordGenerator.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/keygen/TestGlobalDeleteRecordGenerator.java index 4c9fc1c9ddaa..12ccba0feae1 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/keygen/TestGlobalDeleteRecordGenerator.java +++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/keygen/TestGlobalDeleteRecordGenerator.java @@ -18,12 +18,13 @@ package org.apache.hudi.keygen; -import org.apache.avro.generic.GenericRecord; import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.exception.HoodieKeyException; import org.apache.hudi.keygen.constant.KeyGeneratorOptions; import org.apache.hudi.testutils.KeyGeneratorTestUtilities; + +import org.apache.avro.generic.GenericRecord; import org.apache.spark.sql.Row; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/keygen/TestNonpartitionedKeyGenerator.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/keygen/TestNonpartitionedKeyGenerator.java index 187f96197b1d..2b2bd1797495 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/keygen/TestNonpartitionedKeyGenerator.java +++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/keygen/TestNonpartitionedKeyGenerator.java @@ -18,13 +18,14 @@ package org.apache.hudi.keygen; -import org.apache.avro.generic.GenericRecord; import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.testutils.HoodieTestDataGenerator; import org.apache.hudi.exception.HoodieKeyException; import org.apache.hudi.keygen.constant.KeyGeneratorOptions; import org.apache.hudi.testutils.KeyGeneratorTestUtilities; + +import org.apache.avro.generic.GenericRecord; import org.apache.spark.sql.Row; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/keygen/TestSimpleKeyGenerator.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/keygen/TestSimpleKeyGenerator.java index adf522f8354b..e67cce3d3133 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/keygen/TestSimpleKeyGenerator.java +++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/keygen/TestSimpleKeyGenerator.java @@ -18,13 +18,14 @@ package org.apache.hudi.keygen; -import org.apache.avro.generic.GenericRecord; import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.exception.HoodieException; import org.apache.hudi.exception.HoodieKeyException; import org.apache.hudi.keygen.constant.KeyGeneratorOptions; import org.apache.hudi.testutils.KeyGeneratorTestUtilities; + +import org.apache.avro.generic.GenericRecord; import org.apache.spark.sql.Row; import org.apache.spark.sql.catalyst.InternalRow; import org.apache.spark.unsafe.types.UTF8String; diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/keygen/factory/TestCreateKeyGeneratorByTypeWithFactory.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/keygen/factory/TestCreateKeyGeneratorByTypeWithFactory.java index dc597df2cf5c..94ba5acc03a0 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/keygen/factory/TestCreateKeyGeneratorByTypeWithFactory.java +++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/keygen/factory/TestCreateKeyGeneratorByTypeWithFactory.java @@ -32,6 +32,7 @@ import org.apache.hudi.keygen.TimestampBasedKeyGenerator; import org.apache.hudi.keygen.constant.KeyGeneratorOptions; import org.apache.hudi.keygen.constant.KeyGeneratorType; + import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/testutils/HoodieSparkClientTestBase.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/testutils/HoodieSparkClientTestBase.java index ed96df17544c..811564e76115 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/testutils/HoodieSparkClientTestBase.java +++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/testutils/HoodieSparkClientTestBase.java @@ -19,6 +19,7 @@ package org.apache.hudi.testutils; import org.apache.hudi.common.util.Option; + import org.apache.spark.sql.SparkSessionExtensions; import org.apache.spark.sql.hudi.HoodieSparkSessionExtension; diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/HoodieSparkWriterTestBase.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/HoodieSparkWriterTestBase.scala index c0c1c2c12bd4..3dcb9b436952 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/HoodieSparkWriterTestBase.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/HoodieSparkWriterTestBase.scala @@ -18,10 +18,11 @@ package org.apache.hudi -import org.apache.commons.io.FileUtils import org.apache.hudi.common.model.{HoodieRecord, HoodieTableType} import org.apache.hudi.config.HoodieWriteConfig import org.apache.hudi.testutils.HoodieClientTestUtils + +import org.apache.commons.io.FileUtils import org.apache.spark.SparkContext import org.apache.spark.sql.hudi.HoodieSparkSessionExtension import org.apache.spark.sql.{Dataset, Row, SQLContext, SparkSession} diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/SparkDatasetMixin.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/SparkDatasetMixin.scala index 9d648cd47817..c36f79c14ccd 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/SparkDatasetMixin.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/SparkDatasetMixin.scala @@ -17,9 +17,10 @@ package org.apache.hudi -import org.apache.avro.generic.GenericRecord import org.apache.hudi.common.model.{HoodieRecord, HoodieRecordPayload} import org.apache.hudi.common.testutils.HoodieTestDataGenerator + +import org.apache.avro.generic.GenericRecord import org.apache.spark.rdd.RDD import org.apache.spark.sql.SparkSession diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestAvroConversionUtils.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestAvroConversionUtils.scala index 5cd6ac3954ee..01de5013b829 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestAvroConversionUtils.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestAvroConversionUtils.scala @@ -18,8 +18,6 @@ package org.apache.hudi -import java.nio.ByteBuffer -import java.util.Objects import org.apache.avro.Schema import org.apache.avro.generic.GenericData import org.apache.spark.sql.catalyst.InternalRow @@ -27,6 +25,9 @@ import org.apache.spark.sql.catalyst.util.{ArrayData, MapData} import org.apache.spark.sql.types.{ArrayType, BinaryType, DataType, DataTypes, MapType, StringType, StructField, StructType} import org.scalatest.{FunSuite, Matchers} +import java.nio.ByteBuffer +import java.util.Objects + class TestAvroConversionUtils extends FunSuite with Matchers { diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestAvroSchemaResolutionSupport.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestAvroSchemaResolutionSupport.scala index 73c07331c1fc..86c5ba48ef2a 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestAvroSchemaResolutionSupport.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestAvroSchemaResolutionSupport.scala @@ -23,6 +23,7 @@ import org.apache.hudi.common.model.HoodieTableType import org.apache.hudi.config.HoodieWriteConfig import org.apache.hudi.exception.SchemaCompatibilityException import org.apache.hudi.testutils.HoodieClientTestBase + import org.apache.spark.SparkException import org.apache.spark.sql.types._ import org.apache.spark.sql.{DataFrame, Row, SparkSession} diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingUtils.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingUtils.scala index cd1846285ffe..f9f63d4193ea 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingUtils.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingUtils.scala @@ -19,6 +19,7 @@ package org.apache.hudi import org.apache.hudi.ColumnStatsIndexSupport.composeIndexSchema import org.apache.hudi.testutils.HoodieSparkClientTestBase + import org.apache.spark.sql.HoodieCatalystExpressionUtils.resolveExpr import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute import org.apache.spark.sql.catalyst.encoders.DummyExpressionHolder @@ -30,7 +31,7 @@ import org.apache.spark.sql.functions.{col, lower} import org.apache.spark.sql.hudi.DataSkippingUtils import org.apache.spark.sql.internal.SQLConf.SESSION_LOCAL_TIMEZONE import org.apache.spark.sql.types._ -import org.apache.spark.sql.{Column, HoodieCatalystExpressionUtils, Row, SparkSession} +import org.apache.spark.sql.{Column, Row, SparkSession} import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.params.ParameterizedTest @@ -38,6 +39,7 @@ import org.junit.jupiter.params.provider.Arguments.arguments import org.junit.jupiter.params.provider.{Arguments, MethodSource} import java.sql.Timestamp + import scala.collection.JavaConverters._ import scala.collection.immutable.HashSet diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSourceDefaults.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSourceDefaults.scala index 784ddd6c883b..5f0cd4983381 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSourceDefaults.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSourceDefaults.scala @@ -17,18 +17,19 @@ package org.apache.hudi -import org.apache.avro.Schema -import org.apache.avro.generic.GenericRecord import org.apache.hudi.avro.HoodieAvroUtils import org.apache.hudi.common.config.TypedProperties import org.apache.hudi.common.model._ -import org.apache.hudi.common.testutils.{SchemaTestUtil, PreCombineTestUtils} +import org.apache.hudi.common.testutils.{PreCombineTestUtils, SchemaTestUtil} import org.apache.hudi.common.util.Option import org.apache.hudi.common.util.PartitionPathEncodeUtils.DEFAULT_PARTITION_PATH import org.apache.hudi.config.HoodiePayloadConfig import org.apache.hudi.exception.{HoodieException, HoodieKeyException} import org.apache.hudi.keygen._ import org.apache.hudi.testutils.KeyGeneratorTestUtilities + +import org.apache.avro.Schema +import org.apache.avro.generic.GenericRecord import org.apache.spark.sql.Row import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.types.StructType diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSourceOptions.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSourceOptions.scala index 302ddf38682c..f5457783d714 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSourceOptions.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSourceOptions.scala @@ -21,6 +21,7 @@ import org.apache.hudi.DataSourceWriteOptions._ import org.apache.hudi.hive.{HiveStylePartitionValueExtractor, MultiPartKeysValueExtractor} import org.apache.hudi.keygen.{ComplexKeyGenerator, SimpleKeyGenerator} import org.apache.hudi.sync.common.HoodieSyncConfig + import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestGenericRecordAndRowConsistency.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestGenericRecordAndRowConsistency.scala index 9a557a343ef0..3f2c2a303ca9 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestGenericRecordAndRowConsistency.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestGenericRecordAndRowConsistency.scala @@ -19,8 +19,9 @@ package org.apache.hudi import org.apache.hudi.config.HoodieWriteConfig import org.apache.hudi.testutils.HoodieSparkClientTestBase + import org.apache.spark.sql.{DataFrame, SparkSession} -import org.junit.jupiter.api.Assertions.{assertArrayEquals, assertEquals} +import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.{AfterEach, BeforeEach, Test} import java.sql.{Date, Timestamp} diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieParquetBloom.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieParquetBloom.scala index a6f3a0e7368b..438a95e4072e 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieParquetBloom.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieParquetBloom.scala @@ -17,17 +17,13 @@ package org.apache.hudi -import org.apache.spark.sql._ -import org.apache.spark.sql.hudi.HoodieSparkSessionExtension -import org.apache.spark.util.AccumulatorV2 -import org.apache.spark.SparkContext -import org.apache.hudi.testutils.HoodieClientTestUtils.getSparkConfForTest -import org.apache.hudi.DataSourceWriteOptions -import org.apache.hudi.config.HoodieWriteConfig import org.apache.hudi.common.model.{HoodieTableType, WriteOperationType} +import org.apache.hudi.config.HoodieWriteConfig import org.apache.hudi.testutils.HoodieSparkClientTestBase + +import org.apache.spark.sql._ +import org.apache.spark.util.AccumulatorV2 import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.EnumSource diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieRelations.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieRelations.scala index ce9a5d571eb4..abdd0f97547e 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieRelations.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieRelations.scala @@ -17,8 +17,9 @@ package org.apache.hudi -import org.apache.avro.Schema import org.apache.hudi.AvroConversionUtils.convertAvroSchemaToStructType + +import org.apache.avro.Schema import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriter.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriter.scala index d8a6c9379a37..b4d8da7b131b 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriter.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriter.scala @@ -19,8 +19,8 @@ package org.apache.hudi import org.apache.hudi.client.SparkRDDWriteClient import org.apache.hudi.common.model.{HoodieFileFormat, HoodieRecord, HoodieRecordPayload, HoodieReplaceCommitMetadata, HoodieTableType, WriteOperationType} -import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient, TableSchemaResolver} import org.apache.hudi.common.table.timeline.TimelineUtils +import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient, TableSchemaResolver} import org.apache.hudi.common.testutils.HoodieTestDataGenerator import org.apache.hudi.config.{HoodieBootstrapConfig, HoodieIndexConfig, HoodieWriteConfig} import org.apache.hudi.exception.{HoodieException, SchemaCompatibilityException} diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriterPartitionTTL.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriterPartitionTTL.scala index 495f9d39413c..5d891cfa37d5 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriterPartitionTTL.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriterPartitionTTL.scala @@ -22,16 +22,17 @@ package org.apache.hudi import org.apache.hudi.DataSourceWriteOptions.MOR_TABLE_TYPE_OPT_VAL import org.apache.hudi.common.model.HoodieFileFormat +import org.apache.hudi.common.table.HoodieTableConfig import org.apache.hudi.common.table.timeline.HoodieInstantTimeGenerator.{fixInstantTimeCompatibility, instantTimePlusMillis} import org.apache.hudi.common.table.timeline.TimelineMetadataUtils -import org.apache.hudi.common.table.HoodieTableConfig import org.apache.hudi.common.testutils.HoodieTestUtils import org.apache.hudi.config.{HoodieTTLConfig, HoodieWriteConfig} +import org.apache.hudi.table.HoodieTable import org.apache.hudi.table.action.ttl.strategy.KeepByTimeStrategy import org.apache.hudi.testutils.DataSourceTestUtils + import org.apache.spark.sql.SaveMode import org.junit.jupiter.api.Test -import org.apache.hudi.table.HoodieTable /** diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriterUtc.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriterUtc.scala index ca4d23f719d7..52979dc4b9b4 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriterUtc.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriterUtc.scala @@ -21,13 +21,14 @@ import org.apache.hudi.common.model.HoodieTimelineTimeZone import org.apache.hudi.common.table.HoodieTableConfig import org.apache.hudi.common.table.timeline.HoodieInstantTimeGenerator import org.apache.hudi.testutils.DataSourceTestUtils + import org.apache.spark.sql.SaveMode import org.junit.jupiter.api.Assertions.{assertEquals, assertTrue} import org.junit.jupiter.api.Test -import java.time.{Instant, ZoneId} import java.time.format.DateTimeFormatterBuilder import java.time.temporal.ChronoField +import java.time.{Instant, ZoneId} import java.util.TimeZone /** diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkUtils.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkUtils.scala index 5a0f919c9fb2..1783c286c59d 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkUtils.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkUtils.scala @@ -18,10 +18,11 @@ package org.apache.hudi -import org.apache.avro.generic.GenericRecord import org.apache.hudi.common.model.HoodieRecord import org.apache.hudi.testutils.DataSourceTestUtils import org.apache.hudi.testutils.HoodieClientTestUtils.getSparkConfForTest + +import org.apache.avro.generic.GenericRecord import org.apache.spark.sql.types.{ArrayType, StructField, StructType} import org.apache.spark.sql.{DataFrame, Row, SparkSession} import org.junit.jupiter.api.Assertions._ diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestParquetReaderCompatibility.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestParquetReaderCompatibility.scala index c5f91657f123..dc6c25983d70 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestParquetReaderCompatibility.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestParquetReaderCompatibility.scala @@ -18,7 +18,6 @@ package org.apache.hudi -import org.apache.hadoop.conf.Configuration import org.apache.hudi.TestParquetReaderCompatibility.NullabilityEnum.{NotNullable, Nullability, Nullable} import org.apache.hudi.TestParquetReaderCompatibility.{SparkSetting, TestScenario, ThreeLevel, TwoLevel} import org.apache.hudi.client.common.HoodieSparkEngineContext @@ -26,23 +25,26 @@ import org.apache.hudi.common.config.HoodieMetadataConfig import org.apache.hudi.common.model.HoodieRecord.HoodieRecordType import org.apache.hudi.common.table.ParquetTableSchemaResolver import org.apache.hudi.common.testutils.HoodieTestUtils +import org.apache.hudi.common.util.ConfigUtils.DEFAULT_HUDI_CONFIG_FOR_READER import org.apache.hudi.config.HoodieWriteConfig import org.apache.hudi.io.storage.HoodieIOFactory import org.apache.hudi.metadata.HoodieBackedTableMetadata import org.apache.hudi.storage.StoragePath -import org.apache.hudi.testutils.HoodieClientTestUtils -import org.apache.spark.sql.{Row, SaveMode, SparkSession} -import org.apache.spark.sql.types.{ArrayType, LongType, StringType, StructField, StructType} -import org.apache.hudi.common.util.ConfigUtils.DEFAULT_HUDI_CONFIG_FOR_READER import org.apache.hudi.storage.hadoop.{HadoopStorageConfiguration, HoodieHadoopStorage} +import org.apache.hudi.testutils.HoodieClientTestUtils + +import org.apache.hadoop.conf.Configuration import org.apache.parquet.schema.OriginalType import org.apache.spark.SparkConf +import org.apache.spark.sql.types.{ArrayType, LongType, StringType, StructField, StructType} +import org.apache.spark.sql.{Row, SaveMode, SparkSession} import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.MethodSource import java.util.Collections -import scala.collection.mutable + import scala.collection.JavaConverters._ +import scala.collection.mutable object TestParquetReaderCompatibility { val listFieldName = "internal_list" diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/common/model/TestHoodieRecordSerialization.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/common/model/TestHoodieRecordSerialization.scala index 1ce1b3e8fca0..a5cc768ffbe8 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/common/model/TestHoodieRecordSerialization.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/common/model/TestHoodieRecordSerialization.scala @@ -18,13 +18,14 @@ package org.apache.hudi.common.model -import org.apache.avro.generic.GenericRecord import org.apache.hudi.AvroConversionUtils.{convertStructTypeToAvroSchema, createInternalRowToAvroConverter} import org.apache.hudi.client.model.HoodieInternalRow import org.apache.hudi.common.model.HoodieRecord.HoodieRecordType import org.apache.hudi.common.model.TestHoodieRecordSerialization.{OverwriteWithLatestAvroPayloadWithEquality, cloneUsingKryo, convertToAvroRecord, toUnsafeRow} import org.apache.hudi.testutils.SparkClientFunctionalTestHarness import org.apache.hudi.{HoodieSparkUtils, SparkAdapterSupport} + +import org.apache.avro.generic.GenericRecord import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.objects.SerializerSupport import org.apache.spark.sql.catalyst.expressions.{GenericRowWithSchema, UnsafeRow} diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/common/table/read/TestHoodieFileGroupReaderOnSpark.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/common/table/read/TestHoodieFileGroupReaderOnSpark.scala index c2dc9091f452..23013db5d336 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/common/table/read/TestHoodieFileGroupReaderOnSpark.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/common/table/read/TestHoodieFileGroupReaderOnSpark.scala @@ -20,15 +20,16 @@ package org.apache.hudi.common.table.read import org.apache.hudi.common.config.HoodieReaderConfig.FILE_GROUP_READER_ENABLED +import org.apache.hudi.common.config.RecordMergeMode import org.apache.hudi.common.engine.HoodieReaderContext -import org.apache.hudi.common.model.{DefaultHoodieRecordPayload, HoodieRecord, HoodieRecordMerger, OverwriteWithLatestAvroPayload, WriteOperationType} +import org.apache.hudi.common.model.{DefaultHoodieRecordPayload, HoodieRecord, OverwriteWithLatestAvroPayload, WriteOperationType} import org.apache.hudi.common.table.HoodieTableMetaClient import org.apache.hudi.common.testutils.HoodieTestUtils import org.apache.hudi.storage.StorageConfiguration -import org.apache.hudi.{HoodieSparkRecordMerger, OverwriteWithLatestSparkMerger, SparkAdapterSupport, SparkFileFormatInternalRowReaderContext} +import org.apache.hudi.{HoodieSparkRecordMerger, SparkAdapterSupport, SparkFileFormatInternalRowReaderContext} + import org.apache.avro.Schema import org.apache.hadoop.conf.Configuration -import org.apache.hudi.common.config.RecordMergeMode import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.functions.col import org.apache.spark.sql.{Dataset, HoodieInternalRowUtils, HoodieUnsafeUtils, Row, SaveMode, SparkSession} @@ -38,6 +39,7 @@ import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.{AfterEach, BeforeEach} import java.util + import scala.collection.JavaConverters._ /** diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/common/table/read/TestSpark35RecordPositionMetadataColumn.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/common/table/read/TestSpark35RecordPositionMetadataColumn.scala index 61b7b0ded042..4b6196761adb 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/common/table/read/TestSpark35RecordPositionMetadataColumn.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/common/table/read/TestSpark35RecordPositionMetadataColumn.scala @@ -19,7 +19,6 @@ package org.apache.hudi.common.table.read -import org.apache.hadoop.conf.Configuration import org.apache.hudi.SparkAdapterSupport.sparkAdapter import org.apache.hudi.common.config.{HoodieReaderConfig, HoodieStorageConfig} import org.apache.hudi.common.model.HoodieTableType @@ -30,6 +29,8 @@ import org.apache.hudi.storage.hadoop.HadoopStorageConfiguration import org.apache.hudi.testutils.SparkClientFunctionalTestHarness import org.apache.hudi.util.CloseableInternalRowIterator import org.apache.hudi.{DataSourceWriteOptions, HoodieSparkUtils, SparkFileFormatInternalRowReaderContext} + +import org.apache.hadoop.conf.Configuration import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.types.{IntegerType, StringType, StructField, StructType} import org.junit.jupiter.api.Assertions.{assertArrayEquals, assertEquals, assertFalse} diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/ColumnStatIndexTestBase.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/ColumnStatIndexTestBase.scala index 031964d297fa..2b81715c2d17 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/ColumnStatIndexTestBase.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/ColumnStatIndexTestBase.scala @@ -37,6 +37,7 @@ import org.junit.jupiter.params.provider.Arguments import java.math.BigInteger import java.sql.{Date, Timestamp} + import scala.collection.JavaConverters._ import scala.util.Random diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/CommonOptionUtils.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/CommonOptionUtils.scala index 6748c82d1307..dffe06d46119 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/CommonOptionUtils.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/CommonOptionUtils.scala @@ -20,10 +20,10 @@ package org.apache.hudi.functional import org.apache.hudi.common.config.HoodieMetadataConfig -import org.apache.hudi.{DataSourceReadOptions, DataSourceWriteOptions, HoodieSparkRecordMerger} import org.apache.hudi.common.model.HoodieRecord.HoodieRecordType import org.apache.hudi.common.table.HoodieTableConfig import org.apache.hudi.config.HoodieWriteConfig +import org.apache.hudi.{DataSourceReadOptions, DataSourceWriteOptions, HoodieSparkRecordMerger} object CommonOptionUtils { diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/RecordLevelIndexTestBase.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/RecordLevelIndexTestBase.scala index 1b543c59795b..9c8bdd86647c 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/RecordLevelIndexTestBase.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/RecordLevelIndexTestBase.scala @@ -23,8 +23,8 @@ import org.apache.hudi.client.SparkRDDWriteClient import org.apache.hudi.client.common.HoodieSparkEngineContext import org.apache.hudi.common.config.{HoodieMetadataConfig, TypedProperties} import org.apache.hudi.common.model._ -import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient} import org.apache.hudi.common.table.timeline.{HoodieInstant, MetadataConversionUtils} +import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient} import org.apache.hudi.common.testutils.RawTripTestPayload.recordsToStrings import org.apache.hudi.config.HoodieWriteConfig import org.apache.hudi.metadata.{HoodieBackedTableMetadata, HoodieTableMetadataUtil, MetadataPartitionType} @@ -33,16 +33,15 @@ import org.apache.hudi.testutils.HoodieSparkClientTestBase import org.apache.hudi.util.JavaConversions import org.apache.spark.sql._ -import org.apache.spark.sql.{DataFrame, _} import org.apache.spark.sql.functions.{col, not} -import org.junit.jupiter.api._ import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertTrue} +import org.junit.jupiter.api._ import java.util.concurrent.atomic.AtomicInteger import java.util.stream.Collectors -import scala.collection.{mutable, JavaConverters} import scala.collection.JavaConverters._ +import scala.collection.{JavaConverters, mutable} class RecordLevelIndexTestBase extends HoodieSparkClientTestBase { var spark: SparkSession = _ diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/SecondaryIndexTestBase.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/SecondaryIndexTestBase.scala index 9b7a14a56441..bba806f70780 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/SecondaryIndexTestBase.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/SecondaryIndexTestBase.scala @@ -28,6 +28,7 @@ import org.apache.hudi.metadata.HoodieMetadataFileSystemView import org.apache.hudi.testutils.HoodieSparkClientTestBase import org.apache.hudi.util.JFunction import org.apache.hudi.{DataSourceReadOptions, HoodieFileIndex} + import org.apache.spark.sql.catalyst.expressions.{AttributeReference, EqualTo, Expression, Literal} import org.apache.spark.sql.types.StringType import org.apache.spark.sql.{DataFrame, SparkSession} diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestBloomFiltersIndexSupport.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestBloomFiltersIndexSupport.scala index cc46f40523d1..b3de9fcbbdfe 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestBloomFiltersIndexSupport.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestBloomFiltersIndexSupport.scala @@ -29,6 +29,7 @@ import org.apache.hudi.metadata.HoodieMetadataFileSystemView import org.apache.hudi.testutils.HoodieSparkClientTestBase import org.apache.hudi.util.{JFunction, JavaConversions} import org.apache.hudi.{DataSourceReadOptions, DataSourceWriteOptions, HoodieFileIndex} + import org.apache.spark.sql.catalyst.expressions.{AttributeReference, EqualTo, Expression, Literal} import org.apache.spark.sql.functions.{col, not} import org.apache.spark.sql.types.StringType @@ -40,6 +41,7 @@ import org.junit.jupiter.params.provider.EnumSource import java.util.concurrent.atomic.AtomicInteger import java.util.stream.Collectors + import scala.collection.JavaConverters._ import scala.collection.{JavaConverters, mutable} diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestBucketIndexSupport.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestBucketIndexSupport.scala index 8222030f8c6b..9608c459ddf5 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestBucketIndexSupport.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestBucketIndexSupport.scala @@ -17,8 +17,7 @@ package org.apache.hudi.functional -import org.apache.avro.Schema -import org.apache.avro.generic.GenericData +import org.apache.hudi.BucketIndexSupport import org.apache.hudi.common.config.HoodieMetadataConfig.ENABLE import org.apache.hudi.common.config.{HoodieMetadataConfig, TypedProperties} import org.apache.hudi.common.fs.FSUtils @@ -29,11 +28,13 @@ import org.apache.hudi.index.bucket.BucketIdentifier import org.apache.hudi.keygen.constant.KeyGeneratorOptions import org.apache.hudi.keygen.{ComplexKeyGenerator, NonpartitionedKeyGenerator} import org.apache.hudi.testutils.HoodieSparkClientTestBase -import org.apache.hudi.BucketIndexSupport -import org.apache.spark.sql.{HoodieCatalystExpressionUtils, SparkSession} + +import org.apache.avro.Schema +import org.apache.avro.generic.GenericData import org.apache.spark.sql.catalyst.encoders.DummyExpressionHolder import org.apache.spark.sql.catalyst.expressions.PredicateHelper import org.apache.spark.sql.types._ +import org.apache.spark.sql.{HoodieCatalystExpressionUtils, SparkSession} import org.junit.jupiter.api.{AfterEach, BeforeEach, Tag, Test} @Tag("functional") diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestCOWDataSource.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestCOWDataSource.scala index fa53b908bd0d..0c7f578107af 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestCOWDataSource.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestCOWDataSource.scala @@ -43,7 +43,7 @@ import org.apache.hudi.hive.HiveSyncConfigHolder import org.apache.hudi.keygen.constant.KeyGeneratorOptions import org.apache.hudi.keygen.{ComplexKeyGenerator, CustomKeyGenerator, GlobalDeleteKeyGenerator, NonpartitionedKeyGenerator, SimpleKeyGenerator, TimestampBasedKeyGenerator} import org.apache.hudi.metrics.{Metrics, MetricsReporterType} -import org.apache.hudi.storage.{HoodieStorageUtils, StoragePath, StoragePathFilter} +import org.apache.hudi.storage.{StoragePath, StoragePathFilter} import org.apache.hudi.table.HoodieSparkTable import org.apache.hudi.testutils.HoodieSparkClientTestBase import org.apache.hudi.util.JFunction diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestDataSourceForBootstrap.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestDataSourceForBootstrap.scala index 7381a78ec1fe..fe3373f1a9fc 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestDataSourceForBootstrap.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestDataSourceForBootstrap.scala @@ -25,11 +25,10 @@ import org.apache.hudi.common.model.HoodieRecord.HoodieRecordType import org.apache.hudi.common.table.timeline.HoodieTimeline import org.apache.hudi.config.{HoodieBootstrapConfig, HoodieClusteringConfig, HoodieCompactionConfig, HoodieWriteConfig} import org.apache.hudi.functional.TestDataSourceForBootstrap.{dropMetaCols, sort} +import org.apache.hudi.hadoop.fs.HadoopFSUtils import org.apache.hudi.keygen.{NonpartitionedKeyGenerator, SimpleKeyGenerator} import org.apache.hudi.testutils.HoodieClientTestUtils import org.apache.hudi.{DataSourceReadOptions, DataSourceWriteOptions, HoodieDataSourceHelpers, HoodieSparkRecordMerger} -import org.apache.hudi.common.fs.FSUtils -import org.apache.hudi.hadoop.fs.HadoopFSUtils import org.apache.hadoop.fs.{FileSystem, Path} import org.apache.spark.api.java.JavaSparkContext @@ -39,7 +38,7 @@ import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.io.TempDir import org.junit.jupiter.api.{AfterEach, BeforeEach, Test} import org.junit.jupiter.params.ParameterizedTest -import org.junit.jupiter.params.provider.{CsvSource, EnumSource, ValueSource} +import org.junit.jupiter.params.provider.{CsvSource, EnumSource} import java.time.Instant import java.util.Collections diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestEmptyCommit.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestEmptyCommit.scala index c9e1c970f98c..c0d5de7e1031 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestEmptyCommit.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestEmptyCommit.scala @@ -17,13 +17,13 @@ package org.apache.hudi.functional -import org.apache.hudi.{DataSourceWriteOptions, HoodieDataSourceHelpers} import org.apache.hudi.config.HoodieWriteConfig import org.apache.hudi.testutils.HoodieSparkClientTestBase +import org.apache.hudi.{DataSourceWriteOptions, HoodieDataSourceHelpers} import org.apache.spark.sql.{SaveMode, SparkSession} -import org.junit.jupiter.api.{AfterEach, BeforeEach} import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.{AfterEach, BeforeEach} import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.ValueSource diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestHoodieActiveTimeline.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestHoodieActiveTimeline.scala index 1ccd46ad7138..2f83eeca449c 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestHoodieActiveTimeline.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestHoodieActiveTimeline.scala @@ -19,7 +19,6 @@ package org.apache.hudi.functional import org.apache.hudi.common.model.HoodieFileFormat import org.apache.hudi.common.table.HoodieTableMetaClient -import org.apache.hudi.common.testutils.HoodieTestUtils import org.apache.hudi.common.testutils.RawTripTestPayload.recordsToStrings import org.apache.hudi.config.HoodieWriteConfig import org.apache.hudi.testutils.HoodieSparkClientTestBase diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestLayoutOptimization.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestLayoutOptimization.scala index 91da4abe7d9b..0e2f058ee20a 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestLayoutOptimization.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestLayoutOptimization.scala @@ -21,7 +21,6 @@ package org.apache.hudi.functional import org.apache.hudi.HoodieFileIndex.DataSkippingFailureMode import org.apache.hudi.common.config.HoodieMetadataConfig import org.apache.hudi.common.table.timeline.{HoodieInstant, HoodieTimeline} -import org.apache.hudi.common.testutils.HoodieTestUtils import org.apache.hudi.common.testutils.RawTripTestPayload.recordsToStrings import org.apache.hudi.config.{HoodieClusteringConfig, HoodieWriteConfig} import org.apache.hudi.testutils.HoodieSparkClientTestBase diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestMORDataSourceStorage.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestMORDataSourceStorage.scala index 7f557e7e1d59..9e0c933c01e3 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestMORDataSourceStorage.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestMORDataSourceStorage.scala @@ -29,6 +29,7 @@ import org.apache.hudi.storage.hadoop.HadoopStorageConfiguration import org.apache.hudi.testutils.SparkClientFunctionalTestHarness import org.apache.hudi.testutils.SparkClientFunctionalTestHarness.getSparkSqlConf import org.apache.hudi.{DataSourceReadOptions, DataSourceWriteOptions, HoodieDataSourceHelpers} + import org.apache.spark.SparkConf import org.apache.spark.sql._ import org.apache.spark.sql.functions.{col, lit} diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestMetadataRecordIndex.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestMetadataRecordIndex.scala index d40da13d3c8a..4253f425a58e 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestMetadataRecordIndex.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestMetadataRecordIndex.scala @@ -21,8 +21,8 @@ package org.apache.hudi.functional import org.apache.hudi.DataSourceWriteOptions._ import org.apache.hudi.common.config.{HoodieMetadataConfig, TypedProperties} import org.apache.hudi.common.model.HoodieTableType -import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient} import org.apache.hudi.common.table.timeline.HoodieInstant +import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient} import org.apache.hudi.common.testutils.RawTripTestPayload.recordsToStrings import org.apache.hudi.common.util.Option import org.apache.hudi.config.{HoodieClusteringConfig, HoodieWriteConfig} @@ -30,8 +30,8 @@ import org.apache.hudi.metadata.{HoodieBackedTableMetadata, HoodieTableMetadataU import org.apache.hudi.testutils.HoodieSparkClientTestBase import org.apache.spark.sql._ -import org.junit.jupiter.api._ import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertTrue} +import org.junit.jupiter.api._ import java.util.concurrent.atomic.AtomicInteger diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestMetricsReporter.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestMetricsReporter.scala index 5f6b86662af3..83f01d288b53 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestMetricsReporter.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestMetricsReporter.scala @@ -17,7 +17,6 @@ package org.apache.hudi.functional -import org.apache.hudi.{DataSourceWriteOptions, SparkDatasetMixin} import org.apache.hudi.HoodieConversionUtils.toJavaOption import org.apache.hudi.common.config.HoodieMetadataConfig import org.apache.hudi.common.testutils.RawTripTestPayload.recordsToStrings @@ -26,11 +25,12 @@ import org.apache.hudi.config.HoodieWriteConfig import org.apache.hudi.config.metrics.{HoodieMetricsConfig, HoodieMetricsDatadogConfig} import org.apache.hudi.testutils.HoodieSparkClientTestBase import org.apache.hudi.util.JFunction +import org.apache.hudi.{DataSourceWriteOptions, SparkDatasetMixin} import org.apache.spark.sql._ import org.apache.spark.sql.hudi.HoodieSparkSessionExtension -import org.junit.jupiter.api.{AfterEach, Assertions, BeforeEach, Test} import org.junit.jupiter.api.function.Executable +import org.junit.jupiter.api.{AfterEach, Assertions, BeforeEach, Test} import org.slf4j.LoggerFactory import java.util.function.Consumer diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestPartialUpdateAvroPayload.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestPartialUpdateAvroPayload.scala index fc1a19e5453f..3ff095ad089b 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestPartialUpdateAvroPayload.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestPartialUpdateAvroPayload.scala @@ -18,24 +18,23 @@ package org.apache.hudi.functional -import org.apache.hudi.{DataSourceWriteOptions, QuickstartUtils} import org.apache.hudi.HoodieConversionUtils.toJavaOption import org.apache.hudi.QuickstartUtils.{convertToStringList, getQuickstartWriteConfigs} import org.apache.hudi.common.config.HoodieReaderConfig import org.apache.hudi.common.model.HoodieTableType -import org.apache.hudi.common.util import org.apache.hudi.common.util.Option import org.apache.hudi.config.HoodieWriteConfig import org.apache.hudi.testutils.HoodieClientTestBase import org.apache.hudi.util.JFunction +import org.apache.hudi.{DataSourceWriteOptions, QuickstartUtils} import org.apache.hadoop.fs.FileSystem import org.apache.spark.sql._ import org.apache.spark.sql.functions.{lit, typedLit} import org.apache.spark.sql.hudi.HoodieSparkSessionExtension import org.apache.spark.sql.types.{DoubleType, StringType} -import org.junit.jupiter.api.{AfterEach, BeforeEach} import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.{AfterEach, BeforeEach} import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.CsvSource diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestPartitionStatsIndexWithSql.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestPartitionStatsIndexWithSql.scala index dabddb99410e..0e337cc3995d 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestPartitionStatsIndexWithSql.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestPartitionStatsIndexWithSql.scala @@ -22,6 +22,7 @@ package org.apache.hudi.functional import org.apache.hudi.common.model.WriteOperationType import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient} import org.apache.hudi.common.testutils.HoodieTestUtils + import org.apache.spark.sql.hudi.common.HoodieSparkSqlTestBase import org.junit.jupiter.api.Tag diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestRecordLevelIndexWithSQL.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestRecordLevelIndexWithSQL.scala index 97fdc1e10b21..a51c74415e41 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestRecordLevelIndexWithSQL.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestRecordLevelIndexWithSQL.scala @@ -22,11 +22,11 @@ import org.apache.hudi.common.table.HoodieTableMetaClient import org.apache.hudi.metadata.HoodieMetadataFileSystemView import org.apache.hudi.util.JFunction import org.apache.hudi.{DataSourceReadOptions, DataSourceWriteOptions, HoodieFileIndex} + import org.apache.spark.sql.SaveMode import org.apache.spark.sql.catalyst.expressions.{AttributeReference, EqualTo, Expression, GreaterThan, GreaterThanOrEqual, In, Literal, Or} import org.apache.spark.sql.types.StringType import org.junit.jupiter.api.Assertions.{assertEquals, assertTrue} -import org.junit.jupiter.api.io.TempDir import org.junit.jupiter.api.{Tag, Test} import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.ValueSource diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSecondaryIndexPruning.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSecondaryIndexPruning.scala index 9de8b065f59d..9db6f6b9d999 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSecondaryIndexPruning.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSecondaryIndexPruning.scala @@ -23,6 +23,7 @@ import org.apache.hudi.common.model.HoodieTableType import org.apache.hudi.common.table.HoodieTableMetaClient import org.apache.hudi.common.testutils.HoodieTestUtils import org.apache.hudi.{DataSourceReadOptions, DataSourceWriteOptions, HoodieSparkUtils} + import org.apache.spark.sql.Row import org.junit.jupiter.api.{Tag, Test} import org.scalatest.Assertions.assertResult diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSparkDataSourceDAGExecution.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSparkDataSourceDAGExecution.scala index 154a2b65d75e..fff0046d56f4 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSparkDataSourceDAGExecution.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSparkDataSourceDAGExecution.scala @@ -17,24 +17,23 @@ package org.apache.hudi.functional -import org.apache.hudi.{DataSourceWriteOptions, HoodieSparkRecordMerger, ScalaAssertionSupport} import org.apache.hudi.HoodieConversionUtils.toJavaOption import org.apache.hudi.common.config.HoodieMetadataConfig import org.apache.hudi.common.table.HoodieTableConfig import org.apache.hudi.common.testutils.RawTripTestPayload.recordsToStrings -import org.apache.hudi.common.util import org.apache.hudi.common.util.Option import org.apache.hudi.config.HoodieWriteConfig import org.apache.hudi.testutils.HoodieSparkClientTestBase import org.apache.hudi.util.JFunction +import org.apache.hudi.{DataSourceWriteOptions, HoodieSparkRecordMerger, ScalaAssertionSupport} import org.apache.hadoop.fs.FileSystem import org.apache.spark.scheduler.{SparkListener, SparkListenerStageCompleted} -import org.apache.spark.sql.{SaveMode, SparkSession, SparkSessionExtensions} import org.apache.spark.sql.hudi.HoodieSparkSessionExtension import org.apache.spark.sql.types.StructType -import org.junit.jupiter.api.{AfterEach, BeforeEach, Test} +import org.apache.spark.sql.{SaveMode, SparkSession, SparkSessionExtensions} import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.{AfterEach, BeforeEach, Test} import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.CsvSource diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSparkSqlCoreFlow.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSparkSqlCoreFlow.scala index 2ab67c7e87d4..bf5cb404393b 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSparkSqlCoreFlow.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSparkSqlCoreFlow.scala @@ -39,6 +39,7 @@ import org.junit.jupiter.api.Assertions.{assertEquals, assertTrue} import org.scalatest.Inspectors.forAll import java.io.File + import scala.collection.JavaConverters._ @SparkSQLCoreFlow diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSparkSqlWithTimestampKeyGenerator.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSparkSqlWithTimestampKeyGenerator.scala index 92c3dac6832c..6c10cd11b03f 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSparkSqlWithTimestampKeyGenerator.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSparkSqlWithTimestampKeyGenerator.scala @@ -19,6 +19,7 @@ package org.apache.hudi.functional import org.apache.hudi.functional.TestSparkSqlWithTimestampKeyGenerator._ + import org.apache.spark.sql.hudi.common.HoodieSparkSqlTestBase import org.slf4j.LoggerFactory diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSqlStatement.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSqlStatement.scala index 607b99e87b85..016d755cb9d3 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSqlStatement.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSqlStatement.scala @@ -18,6 +18,7 @@ package org.apache.hudi.functional import org.apache.hudi.common.util.FileIOUtils + import org.apache.spark.sql.hudi.common.HoodieSparkSqlTestBase class TestSqlStatement extends HoodieSparkSqlTestBase { diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/cdc/TestCDCDataFrameSuite.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/cdc/TestCDCDataFrameSuite.scala index cfc0735b2fd3..0a904f75c3f7 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/cdc/TestCDCDataFrameSuite.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/cdc/TestCDCDataFrameSuite.scala @@ -28,6 +28,7 @@ import org.apache.hudi.common.table.{HoodieTableConfig, TableSchemaResolver} import org.apache.hudi.common.testutils.HoodieTestDataGenerator import org.apache.hudi.common.testutils.RawTripTestPayload.{deleteRecordsToStrings, recordsToStrings} import org.apache.hudi.config.HoodieWriteConfig + import org.apache.avro.generic.GenericRecord import org.apache.spark.sql.types.{StringType, StructField, StructType} import org.apache.spark.sql.{Row, SaveMode} diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/util/JavaConversions.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/util/JavaConversions.scala index 52333e726282..c7c0df20bd93 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/util/JavaConversions.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/util/JavaConversions.scala @@ -18,9 +18,10 @@ package org.apache.hudi.util -import org.apache.spark.sql.execution.datasources.parquet.TestSparkParquetReaderFormat import org.apache.hudi.SparkAdapterSupport + import org.apache.spark.sql.execution.datasources.DataSource +import org.apache.spark.sql.execution.datasources.parquet.TestSparkParquetReaderFormat import org.apache.spark.sql.{DataFrame, SparkSession} import java.util.function.Predicate diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/util/TestPathUtils.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/util/TestPathUtils.scala index bdb10cd4afbc..39119d988fa7 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/util/TestPathUtils.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/util/TestPathUtils.scala @@ -21,7 +21,6 @@ package org.apache.hudi.util import org.apache.hudi.common.testutils.HoodieTestUtils import org.apache.hudi.storage.{HoodieStorageUtils, StoragePath} -import org.apache.hadoop.conf.Configuration import org.junit.jupiter.api.Assertions._ import org.junit.jupiter.api.Test import org.junit.jupiter.api.io.TempDir diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/util/TestSparkInternalSchemaConverter.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/util/TestSparkInternalSchemaConverter.scala index d2ba8c3582c9..6da4b317960c 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/util/TestSparkInternalSchemaConverter.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/util/TestSparkInternalSchemaConverter.scala @@ -19,11 +19,12 @@ package org.apache.hudi.util -import org.apache.avro.Schema import org.apache.hudi.SparkAdapterSupport import org.apache.hudi.client.utils.SparkInternalSchemaConverter.collectColNamesFromSparkStruct import org.apache.hudi.internal.schema.convert.TestAvroInternalSchemaConverter._ import org.apache.hudi.testutils.HoodieSparkClientTestHarness + +import org.apache.avro.Schema import org.apache.spark.sql.types._ import org.junit.jupiter.api.Assertions.{assertEquals, assertTrue} import org.junit.jupiter.api.Test diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/hudi/benchmark/HoodieBenchmark.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/hudi/benchmark/HoodieBenchmark.scala index 6d4317a8135e..6d1d3bcbfd98 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/hudi/benchmark/HoodieBenchmark.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/hudi/benchmark/HoodieBenchmark.scala @@ -18,6 +18,10 @@ package org.apache.spark.hudi.benchmark +import org.apache.commons.io.output.TeeOutputStream +import org.apache.commons.lang3.SystemUtils +import org.apache.spark.util.Utils + import java.io.{OutputStream, PrintStream} import scala.collection.mutable @@ -25,11 +29,6 @@ import scala.collection.mutable.ArrayBuffer import scala.concurrent.duration._ import scala.util.Try -import org.apache.commons.io.output.TeeOutputStream -import org.apache.commons.lang3.SystemUtils - -import org.apache.spark.util.Utils - /** * Reference from spark. * Utility class to benchmark components. An example of how to use this is: diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/hudi/benchmark/HoodieBenchmarkBase.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/hudi/benchmark/HoodieBenchmarkBase.scala index 510ff44120ed..e70c76ffd639 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/hudi/benchmark/HoodieBenchmarkBase.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/hudi/benchmark/HoodieBenchmarkBase.scala @@ -18,6 +18,7 @@ package org.apache.spark.hudi.benchmark import org.apache.hudi.common.util.StringUtils.getUTF8Bytes + import org.apache.spark.util.Utils import java.io.{File, FileOutputStream, OutputStream} diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/hudi/benchmark/HoodieBenchmarks.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/hudi/benchmark/HoodieBenchmarks.scala index 872991002f8b..448654814746 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/hudi/benchmark/HoodieBenchmarks.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/hudi/benchmark/HoodieBenchmarks.scala @@ -16,13 +16,15 @@ */ package org.apache.spark.hudi.benchmark +import org.apache.hbase.thirdparty.com.google.common.reflect.ClassPath + import java.io.File import java.lang.reflect.Modifier import java.nio.file.{FileSystems, Paths} import java.util.Locale + import scala.collection.JavaConverters._ import scala.util.Try -import org.apache.hbase.thirdparty.com.google.common.reflect.ClassPath /** * Reference from spark. diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/avro/TestAvroSerDe.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/avro/TestAvroSerDe.scala index bf68e34fb648..4a43d7bf8eee 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/avro/TestAvroSerDe.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/avro/TestAvroSerDe.scala @@ -17,10 +17,10 @@ package org.apache.spark.sql.avro -import org.apache.avro.generic.GenericData import org.apache.hudi.SparkAdapterSupport import org.apache.hudi.avro.model.{HoodieMetadataColumnStats, IntWrapper} -import org.apache.spark.internal.Logging + +import org.apache.avro.generic.GenericData import org.apache.spark.sql.avro.SchemaConverters.SchemaType import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/avro/TestSchemaConverters.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/avro/TestSchemaConverters.scala index 3d172fecdf62..45da6ad4d052 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/avro/TestSchemaConverters.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/avro/TestSchemaConverters.scala @@ -18,6 +18,7 @@ package org.apache.spark.sql.avro import org.apache.hudi.avro.model.HoodieMetadataColumnStats + import org.apache.spark.sql.avro.SchemaConverters.SchemaType import org.junit.Test import org.junit.jupiter.api.Assertions.assertEquals diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/execution/benchmark/AvroSerDerBenchmark.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/execution/benchmark/AvroSerDerBenchmark.scala index 5e092bdb51c3..842276c0d40e 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/execution/benchmark/AvroSerDerBenchmark.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/execution/benchmark/AvroSerDerBenchmark.scala @@ -18,8 +18,9 @@ package org.apache.spark.sql.execution.benchmark -import org.apache.avro.generic.GenericRecord import org.apache.hudi.{AvroConversionUtils, HoodieSparkUtils} + +import org.apache.avro.generic.GenericRecord import org.apache.spark.hudi.benchmark.{HoodieBenchmark, HoodieBenchmarkBase} import org.apache.spark.sql.functions.lit import org.apache.spark.sql.{DataFrame, SparkSession} diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/execution/benchmark/BoundInMemoryExecutorBenchmark.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/execution/benchmark/BoundInMemoryExecutorBenchmark.scala index eba322c16cf7..74837c0007f7 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/execution/benchmark/BoundInMemoryExecutorBenchmark.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/execution/benchmark/BoundInMemoryExecutorBenchmark.scala @@ -18,8 +18,9 @@ package org.apache.spark.sql.execution.benchmark -import org.apache.hadoop.fs.Path import org.apache.hudi.HoodieSparkUtils + +import org.apache.hadoop.fs.Path import org.apache.spark.SparkConf import org.apache.spark.hudi.benchmark.{HoodieBenchmark, HoodieBenchmarkBase} import org.apache.spark.sql.hudi.HoodieSparkSessionExtension diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/execution/benchmark/CowTableReadBenchmark.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/execution/benchmark/CowTableReadBenchmark.scala index 13d1746fd51a..2d9525c934e0 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/execution/benchmark/CowTableReadBenchmark.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/execution/benchmark/CowTableReadBenchmark.scala @@ -18,15 +18,16 @@ package org.apache.spark.sql.execution.benchmark -import org.apache.hudi.{HoodieFileIndex, HoodieSparkUtils} +import org.apache.hudi.HoodieSparkUtils + +import org.apache.hadoop.fs.Path import org.apache.spark.SparkConf import org.apache.spark.hudi.benchmark.{HoodieBenchmark, HoodieBenchmarkBase} -import org.apache.spark.sql.{DataFrame, RowFactory, SparkSession} import org.apache.spark.sql.hudi.HoodieSparkSessionExtension import org.apache.spark.sql.types._ -import java.sql.{Date, Timestamp} +import org.apache.spark.sql.{DataFrame, RowFactory, SparkSession} -import org.apache.hadoop.fs.Path +import java.sql.{Date, Timestamp} import scala.util.Random diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/execution/benchmark/LSMTimelineReadBenchmark.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/execution/benchmark/LSMTimelineReadBenchmark.scala index e5dda1c33749..caa3fe3ae969 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/execution/benchmark/LSMTimelineReadBenchmark.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/execution/benchmark/LSMTimelineReadBenchmark.scala @@ -18,8 +18,6 @@ package org.apache.spark.sql.execution.benchmark -import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.fs.Path import org.apache.hudi.DummyActiveAction import org.apache.hudi.client.common.HoodieJavaEngineContext import org.apache.hudi.client.timeline.LSMTimelineWriter @@ -30,9 +28,12 @@ import org.apache.hudi.common.testutils.{HoodieTestTable, HoodieTestUtils} import org.apache.hudi.config.{HoodieIndexConfig, HoodieWriteConfig} import org.apache.hudi.index.HoodieIndex.IndexType import org.apache.hudi.table.HoodieJavaTable + +import org.apache.hadoop.fs.Path import org.apache.spark.hudi.benchmark.{HoodieBenchmark, HoodieBenchmarkBase} import java.util + import scala.collection.JavaConverters._ object LSMTimelineReadBenchmark extends HoodieBenchmarkBase { diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/execution/benchmark/ReadAndWriteWithoutAvroBenchmark.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/execution/benchmark/ReadAndWriteWithoutAvroBenchmark.scala index ef7e606077f4..4d02d3ed482f 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/execution/benchmark/ReadAndWriteWithoutAvroBenchmark.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/execution/benchmark/ReadAndWriteWithoutAvroBenchmark.scala @@ -18,13 +18,12 @@ package org.apache.spark.sql.execution.benchmark -import org.apache.hadoop.fs.Path -import org.apache.hudi.config.HoodieWriteConfig import org.apache.hudi.common.config.HoodieStorageConfig import org.apache.hudi.common.model.HoodieAvroRecordMerger -import org.apache.hudi.config.HoodieCompactionConfig +import org.apache.hudi.config.{HoodieCompactionConfig, HoodieWriteConfig} import org.apache.hudi.{HoodieSparkRecordMerger, HoodieSparkUtils} +import org.apache.hadoop.fs.Path import org.apache.spark.SparkConf import org.apache.spark.hudi.benchmark.{HoodieBenchmark, HoodieBenchmarkBase} import org.apache.spark.sql.functions._ diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/execution/benchmark/SpaceCurveOptimizeBenchmark.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/execution/benchmark/SpaceCurveOptimizeBenchmark.scala index 5440b5392568..98b077d43910 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/execution/benchmark/SpaceCurveOptimizeBenchmark.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/execution/benchmark/SpaceCurveOptimizeBenchmark.scala @@ -18,10 +18,11 @@ package org.apache.spark.sql.execution.benchmark -import org.apache.hadoop.fs.Path import org.apache.hudi.ColumnStatsIndexHelper.buildColumnStatsTableFor import org.apache.hudi.config.HoodieClusteringConfig.LayoutOptimizationStrategy import org.apache.hudi.sort.SpaceCurveSortingHelper + +import org.apache.hadoop.fs.Path import org.apache.spark.sql.DataFrame import org.apache.spark.sql.hudi.common.HoodieSparkSqlTestBase import org.apache.spark.sql.types.{IntegerType, StructField} diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/analysis/TestHoodiePruneFileSourcePartitions.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/analysis/TestHoodiePruneFileSourcePartitions.scala index df783d7f2a40..4b94dd602933 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/analysis/TestHoodiePruneFileSourcePartitions.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/analysis/TestHoodiePruneFileSourcePartitions.scala @@ -21,6 +21,7 @@ import org.apache.hudi.HoodieConversionUtils.toJavaOption import org.apache.hudi.ScalaAssertionSupport import org.apache.hudi.testutils.HoodieClientTestBase import org.apache.hudi.util.JFunction + import org.apache.spark.sql.catalyst.expressions.{And, AttributeReference, EqualTo, IsNotNull, Literal} import org.apache.spark.sql.catalyst.plans.logical.Filter import org.apache.spark.sql.execution.datasources.LogicalRelation diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/command/index/TestFunctionalIndex.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/command/index/TestFunctionalIndex.scala index 83dd66c329ed..0148b8028de8 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/command/index/TestFunctionalIndex.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/command/index/TestFunctionalIndex.scala @@ -29,6 +29,7 @@ import org.apache.hudi.hive.{HiveSyncTool, HoodieHiveSyncClient} import org.apache.hudi.metadata.MetadataPartitionType import org.apache.hudi.sync.common.HoodieSyncConfig.{META_SYNC_BASE_PATH, META_SYNC_DATABASE_NAME, META_SYNC_NO_PARTITION_METADATA, META_SYNC_TABLE_NAME} import org.apache.hudi.testutils.HoodieClientTestUtils.createMetaClient + import org.apache.spark.sql.catalyst.analysis.Analyzer import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.catalyst.parser.ParserInterface diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/command/index/TestIndexSyntax.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/command/index/TestIndexSyntax.scala index 158d8ca4f018..87146b5c0e29 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/command/index/TestIndexSyntax.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/command/index/TestIndexSyntax.scala @@ -20,6 +20,7 @@ package org.apache.spark.sql.hudi.command.index import org.apache.hudi.HoodieSparkUtils + import org.apache.spark.sql.catalyst.analysis.Analyzer import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.catalyst.parser.ParserInterface diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/command/index/TestSecondaryIndex.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/command/index/TestSecondaryIndex.scala index de76c4a59a5f..da7a3ad348f0 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/command/index/TestSecondaryIndex.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/command/index/TestSecondaryIndex.scala @@ -22,6 +22,7 @@ package org.apache.spark.sql.hudi.command.index import org.apache.hudi.HoodieSparkUtils import org.apache.hudi.common.table.HoodieTableMetaClient import org.apache.hudi.common.testutils.HoodieTestUtils + import org.apache.spark.sql.hudi.common.HoodieSparkSqlTestBase class TestSecondaryIndex extends HoodieSparkSqlTestBase { diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/common/TestHoodieInternalRowUtils.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/common/TestHoodieInternalRowUtils.scala index 2ce4393c6a8c..7d7a84982788 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/common/TestHoodieInternalRowUtils.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/common/TestHoodieInternalRowUtils.scala @@ -18,8 +18,6 @@ package org.apache.spark.sql.hudi.common -import org.apache.avro.generic.GenericData -import org.apache.avro.{LogicalTypes, Schema} import org.apache.hudi.AvroConversionUtils import org.apache.hudi.SparkAdapterSupport.sparkAdapter import org.apache.hudi.avro.HoodieAvroUtils @@ -28,6 +26,9 @@ import org.apache.hudi.internal.schema.action.TableChanges import org.apache.hudi.internal.schema.convert.AvroInternalSchemaConverter import org.apache.hudi.internal.schema.utils.SchemaChangeUtils import org.apache.hudi.testutils.HoodieClientTestUtils + +import org.apache.avro.generic.GenericData +import org.apache.avro.{LogicalTypes, Schema} import org.apache.spark.api.java.JavaSparkContext import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.util.{ArrayData, MapData} diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/common/TestHoodieOptionConfig.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/common/TestHoodieOptionConfig.scala index 42db138671e9..1271277c08f9 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/common/TestHoodieOptionConfig.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/common/TestHoodieOptionConfig.scala @@ -21,6 +21,7 @@ import org.apache.hudi.DataSourceWriteOptions import org.apache.hudi.common.model.{DefaultHoodieRecordPayload, HoodieRecordMerger, OverwriteWithLatestAvroPayload} import org.apache.hudi.common.table.HoodieTableConfig import org.apache.hudi.testutils.SparkClientFunctionalTestHarness + import org.apache.spark.sql.hudi.HoodieOptionConfig import org.apache.spark.sql.types._ import org.junit.jupiter.api.Assertions.assertTrue diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/common/TestNestedSchemaPruningOptimization.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/common/TestNestedSchemaPruningOptimization.scala index 62b2352d9c7f..d9307b8213a1 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/common/TestNestedSchemaPruningOptimization.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/common/TestNestedSchemaPruningOptimization.scala @@ -20,6 +20,7 @@ package org.apache.spark.sql.hudi.common import org.apache.hudi.common.config.HoodieCommonConfig import org.apache.hudi.config.HoodieWriteConfig import org.apache.hudi.{HoodieSparkUtils, SparkAdapterSupport} + import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.execution.{FileSourceScanExec, ProjectExec, RowDataSourceScanExec, SparkPlan} import org.apache.spark.sql.internal.SQLConf diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/common/TestSqlConf.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/common/TestSqlConf.scala index adce16e7193f..b0d57aceab63 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/common/TestSqlConf.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/common/TestSqlConf.scala @@ -24,6 +24,7 @@ import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient} import org.apache.hudi.common.testutils.HoodieTestUtils import org.apache.hudi.storage.{HoodieStorageUtils, StoragePath} import org.apache.hudi.testutils.HoodieClientTestUtils.createMetaClient + import org.scalatest.BeforeAndAfter import java.io.File diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestDeleteTable.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestDeleteTable.scala index c157091d94d1..6c116aac9c31 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestDeleteTable.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestDeleteTable.scala @@ -20,6 +20,7 @@ package org.apache.spark.sql.hudi.dml import org.apache.hudi.DataSourceWriteOptions._ import org.apache.hudi.HoodieSparkUtils.isSpark2 import org.apache.hudi.config.HoodieWriteConfig + import org.apache.spark.sql.SaveMode import org.apache.spark.sql.hudi.common.HoodieSparkSqlTestBase diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestDropTable.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestDropTable.scala index 743abc5b2fd0..a77cdce3c9bc 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestDropTable.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestDropTable.scala @@ -17,8 +17,9 @@ package org.apache.spark.sql.hudi.dml -import org.apache.hadoop.fs.Path import org.apache.hudi.hadoop.fs.HadoopFSUtils + +import org.apache.hadoop.fs.Path import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.catalog.SessionCatalog diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestHoodieTableValuedFunction.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestHoodieTableValuedFunction.scala index 17b5b558b0a0..9090b1c8a5dc 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestHoodieTableValuedFunction.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestHoodieTableValuedFunction.scala @@ -20,6 +20,7 @@ package org.apache.spark.sql.hudi.dml import org.apache.hudi.DataSourceWriteOptions.SPARK_SQL_INSERT_INTO_OPERATION import org.apache.hudi.HoodieSparkUtils import org.apache.hudi.metadata.HoodieMetadataPayload.getPartitionStatsIndexKey + import org.apache.spark.sql.functions.{col, from_json} import org.apache.spark.sql.hudi.common.HoodieSparkSqlTestBase diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestInsertTable.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestInsertTable.scala index ec9e90be7c1a..b46723e97bf5 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestInsertTable.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestInsertTable.scala @@ -21,8 +21,8 @@ import org.apache.hudi.DataSourceWriteOptions._ import org.apache.hudi.client.common.HoodieSparkEngineContext import org.apache.hudi.common.model.HoodieRecord.HoodieRecordType import org.apache.hudi.common.model.{HoodieRecord, WriteOperationType} -import org.apache.hudi.common.table.{HoodieTableConfig, TableSchemaResolver} import org.apache.hudi.common.table.timeline.HoodieInstant +import org.apache.hudi.common.table.{HoodieTableConfig, TableSchemaResolver} import org.apache.hudi.common.util.{Option => HOption} import org.apache.hudi.config.{HoodieClusteringConfig, HoodieIndexConfig, HoodieWriteConfig} import org.apache.hudi.exception.{HoodieDuplicateKeyException, HoodieException} @@ -30,8 +30,9 @@ import org.apache.hudi.execution.bulkinsert.BulkInsertSortMode import org.apache.hudi.index.HoodieIndex.IndexType import org.apache.hudi.testutils.HoodieClientTestUtils.createMetaClient import org.apache.hudi.{DataSourceWriteOptions, HoodieCLIUtils, HoodieSparkUtils} + import org.apache.spark.scheduler.{SparkListener, SparkListenerStageSubmitted} -import org.apache.spark.sql.{Row, SaveMode} +import org.apache.spark.sql.SaveMode import org.apache.spark.sql.hudi.HoodieSqlCommonUtils import org.apache.spark.sql.hudi.command.HoodieSparkValidateDuplicateKeyRecordMerger import org.apache.spark.sql.hudi.common.HoodieSparkSqlTestBase diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestMergeIntoLogOnlyTable.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestMergeIntoLogOnlyTable.scala index d25b9752e35b..b8ab95e7653d 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestMergeIntoLogOnlyTable.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestMergeIntoLogOnlyTable.scala @@ -18,6 +18,7 @@ package org.apache.spark.sql.hudi.dml import org.apache.hudi.testutils.DataSourceTestUtils + import org.apache.spark.sql.hudi.common.HoodieSparkSqlTestBase class TestMergeIntoLogOnlyTable extends HoodieSparkSqlTestBase { diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestMergeIntoTable.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestMergeIntoTable.scala index d712e2df2597..cfb0475b36bb 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestMergeIntoTable.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestMergeIntoTable.scala @@ -21,6 +21,7 @@ import org.apache.hudi.DataSourceWriteOptions.SPARK_SQL_OPTIMIZED_WRITES import org.apache.hudi.config.HoodieWriteConfig.MERGE_SMALL_FILE_GROUP_CANDIDATES_LIMIT import org.apache.hudi.hadoop.fs.HadoopFSUtils import org.apache.hudi.{DataSourceReadOptions, HoodieDataSourceHelpers, HoodieSparkUtils, ScalaAssertionSupport} + import org.apache.spark.sql.hudi.common.HoodieSparkSqlTestBase import org.apache.spark.sql.internal.SQLConf diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestMergeIntoTableWithNonRecordKeyField.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestMergeIntoTableWithNonRecordKeyField.scala index 8e06995475b8..2a9bad788c70 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestMergeIntoTableWithNonRecordKeyField.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestMergeIntoTableWithNonRecordKeyField.scala @@ -19,6 +19,7 @@ package org.apache.spark.sql.hudi.dml import org.apache.hudi.DataSourceWriteOptions.SPARK_SQL_OPTIMIZED_WRITES import org.apache.hudi.{HoodieSparkUtils, ScalaAssertionSupport} + import org.apache.spark.sql.hudi.common.HoodieSparkSqlTestBase class TestMergeIntoTableWithNonRecordKeyField extends HoodieSparkSqlTestBase with ScalaAssertionSupport { diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestRepairTable.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestRepairTable.scala index fccc7b61f1f5..e4191157e6c8 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestRepairTable.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestRepairTable.scala @@ -22,6 +22,7 @@ import org.apache.hudi.DataSourceWriteOptions.{PARTITIONPATH_FIELD, PRECOMBINE_F import org.apache.hudi.HoodieSparkUtils import org.apache.hudi.common.table.HoodieTableConfig.HIVE_STYLE_PARTITIONING_ENABLE import org.apache.hudi.config.HoodieWriteConfig.TBL_NAME + import org.apache.spark.sql.SaveMode import org.apache.spark.sql.hudi.common.HoodieSparkSqlTestBase diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestShowPartitions.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestShowPartitions.scala index 1afb0f0974fc..2670b13f6468 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestShowPartitions.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestShowPartitions.scala @@ -19,6 +19,7 @@ package org.apache.spark.sql.hudi.dml import org.apache.hudi.HoodieSparkUtils.isSpark2 import org.apache.hudi.common.util.PartitionPathEncodeUtils.DEFAULT_PARTITION_PATH + import org.apache.spark.sql.hudi.common.HoodieSparkSqlTestBase class TestShowPartitions extends HoodieSparkSqlTestBase { diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestTruncateTable.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestTruncateTable.scala index 411562c35583..0f599aa2b9ac 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestTruncateTable.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestTruncateTable.scala @@ -20,6 +20,7 @@ package org.apache.spark.sql.hudi.dml import org.apache.hudi.DataSourceWriteOptions._ import org.apache.hudi.config.HoodieWriteConfig + import org.apache.spark.sql.SaveMode import org.apache.spark.sql.hudi.common.HoodieSparkSqlTestBase diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestCallCommandParser.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestCallCommandParser.scala index 3d07286ca190..7f90d9654d22 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestCallCommandParser.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestCallCommandParser.scala @@ -19,12 +19,14 @@ package org.apache.spark.sql.hudi.procedure import org.apache.hudi.HoodieSparkUtils import org.apache.hudi.common.util.CollectionUtils.createImmutableList + import org.apache.spark.sql.catalyst.expressions.Literal import org.apache.spark.sql.catalyst.plans.logical.{CallCommand, NamedArgument, PositionalArgument} import org.apache.spark.sql.hudi.common.HoodieSparkSqlTestBase import org.apache.spark.sql.types.{DataType, DataTypes} import java.math.BigDecimal + import scala.collection.JavaConverters class TestCallCommandParser extends HoodieSparkSqlTestBase { diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestClusteringProcedure.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestClusteringProcedure.scala index e3989428c877..55355ae0b04f 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestClusteringProcedure.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestClusteringProcedure.scala @@ -26,15 +26,17 @@ import org.apache.hudi.common.table.timeline.{HoodieInstant, HoodieTimeline} import org.apache.hudi.common.testutils.HoodieTestUtils import org.apache.hudi.common.util.collection.Pair import org.apache.hudi.common.util.{Option => HOption} +import org.apache.hudi.storage.hadoop.HadoopStorageConfiguration import org.apache.hudi.{DataSourceReadOptions, HoodieCLIUtils, HoodieDataSourceHelpers, HoodieFileIndex} + import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path -import org.apache.hudi.storage.hadoop.HadoopStorageConfiguration import org.apache.spark.sql.catalyst.expressions.{AttributeReference, EqualTo, Literal} import org.apache.spark.sql.types.{DataTypes, Metadata, StringType, StructField, StructType} import org.apache.spark.sql.{Dataset, Row} import java.util + import scala.collection.JavaConverters.asScalaIteratorConverter class TestClusteringProcedure extends HoodieSparkProcedureTestBase { diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestCompactionProcedure.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestCompactionProcedure.scala index 147bdfdb23a5..05fcc3f2a87e 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestCompactionProcedure.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestCompactionProcedure.scala @@ -22,9 +22,10 @@ package org.apache.spark.sql.hudi.procedure import org.apache.hudi.common.table.timeline.HoodieInstant import org.apache.hudi.common.testutils.HoodieTestUtils import org.apache.hudi.common.testutils.HoodieTestUtils.createMetaClient -import org.apache.hadoop.conf.Configuration import org.apache.hudi.storage.hadoop.HadoopStorageConfiguration +import org.apache.hadoop.conf.Configuration + class TestCompactionProcedure extends HoodieSparkProcedureTestBase { test("Test Call run_compaction Procedure by Table") { diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestCopyToTempViewProcedure.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestCopyToTempViewProcedure.scala index 6f54dfb5094c..86f6c80cb8b5 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestCopyToTempViewProcedure.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestCopyToTempViewProcedure.scala @@ -18,6 +18,7 @@ package org.apache.spark.sql.hudi.procedure import org.apache.hudi.HoodieSparkUtils + import org.apache.spark.sql.hudi.common.HoodieSparkSqlTestBase class TestCopyToTempViewProcedure extends HoodieSparkSqlTestBase { diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestShowInvalidParquetProcedure.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestShowInvalidParquetProcedure.scala index 118d34e10366..f7f83233a962 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestShowInvalidParquetProcedure.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestShowInvalidParquetProcedure.scala @@ -17,7 +17,6 @@ package org.apache.spark.sql.hudi.procedure -import org.apache.hudi.common.fs.FSUtils import org.apache.hudi.hadoop.fs.HadoopFSUtils import org.apache.hadoop.fs.Path diff --git a/hudi-spark-datasource/hudi-spark3-common/src/main/java/org/apache/hudi/spark3/internal/DefaultSource.java b/hudi-spark-datasource/hudi-spark3-common/src/main/java/org/apache/hudi/spark3/internal/DefaultSource.java index c0425aa8f861..233fb20354da 100644 --- a/hudi-spark-datasource/hudi-spark3-common/src/main/java/org/apache/hudi/spark3/internal/DefaultSource.java +++ b/hudi-spark-datasource/hudi-spark3-common/src/main/java/org/apache/hudi/spark3/internal/DefaultSource.java @@ -24,6 +24,7 @@ import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.internal.BaseDefaultSource; import org.apache.hudi.internal.DataSourceInternalWriterHelper; + import org.apache.spark.sql.HoodieDataTypeUtils; import org.apache.spark.sql.connector.catalog.Table; import org.apache.spark.sql.connector.catalog.TableProvider; diff --git a/hudi-spark-datasource/hudi-spark3-common/src/main/java/org/apache/hudi/spark3/internal/HoodieWriterCommitMessage.java b/hudi-spark-datasource/hudi-spark3-common/src/main/java/org/apache/hudi/spark3/internal/HoodieWriterCommitMessage.java index bc2904f1ba13..086cb2f75f15 100644 --- a/hudi-spark-datasource/hudi-spark3-common/src/main/java/org/apache/hudi/spark3/internal/HoodieWriterCommitMessage.java +++ b/hudi-spark-datasource/hudi-spark3-common/src/main/java/org/apache/hudi/spark3/internal/HoodieWriterCommitMessage.java @@ -20,6 +20,7 @@ import org.apache.hudi.client.WriteStatus; import org.apache.hudi.internal.BaseWriterCommitMessage; + import org.apache.spark.sql.connector.write.WriterCommitMessage; import java.util.List; diff --git a/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/HoodieSpark3CatalystPlanUtils.scala b/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/HoodieSpark3CatalystPlanUtils.scala index 399e05334e01..a0938b94671b 100644 --- a/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/HoodieSpark3CatalystPlanUtils.scala +++ b/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/HoodieSpark3CatalystPlanUtils.scala @@ -23,10 +23,10 @@ import org.apache.spark.sql.catalyst.analysis.TableOutputResolver import org.apache.spark.sql.catalyst.catalog.CatalogStorageFormat import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeSet, Expression, ProjectionOverSchema} import org.apache.spark.sql.catalyst.plans.JoinType -import org.apache.spark.sql.catalyst.plans.logical.{InsertIntoStatement, Join, JoinHint, LeafNode, LogicalPlan} +import org.apache.spark.sql.catalyst.plans.logical.{InsertIntoStatement, Join, JoinHint, LogicalPlan} import org.apache.spark.sql.connector.catalog.{Identifier, Table, TableCatalog} -import org.apache.spark.sql.execution.command.{CreateTableLikeCommand, ExplainCommand} import org.apache.spark.sql.execution.{ExtendedMode, SimpleMode} +import org.apache.spark.sql.execution.command.{CreateTableLikeCommand, ExplainCommand} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.StructType diff --git a/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/adapter/BaseSpark3Adapter.scala b/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/adapter/BaseSpark3Adapter.scala index 5b7106123f70..9e123bd84b03 100644 --- a/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/adapter/BaseSpark3Adapter.scala +++ b/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/adapter/BaseSpark3Adapter.scala @@ -17,15 +17,15 @@ package org.apache.spark.sql.adapter +import org.apache.avro.Schema +import org.apache.hudi.{AvroConversionUtils, DefaultSource, HoodieSparkUtils, Spark3RowSerDe} import org.apache.hudi.client.utils.SparkRowSerDe import org.apache.hudi.common.table.HoodieTableMetaClient import org.apache.hudi.common.util.JsonUtils import org.apache.hudi.spark3.internal.ReflectUtil import org.apache.hudi.storage.StoragePath -import org.apache.hudi.{AvroConversionUtils, DefaultSource, HoodieSparkUtils, Spark3RowSerDe} - -import org.apache.avro.Schema import org.apache.spark.internal.Logging +import org.apache.spark.sql.{HoodieSpark3CatalogUtils, SparkSession, SQLContext} import org.apache.spark.sql.avro.{HoodieAvroSchemaConverters, HoodieSparkAvroSchemaConverters} import org.apache.spark.sql.catalyst.expressions.{Expression, InterpretedPredicate, Predicate} import org.apache.spark.sql.catalyst.util.DateFormatter @@ -34,8 +34,7 @@ import org.apache.spark.sql.execution.datasources._ import org.apache.spark.sql.hudi.SparkAdapter import org.apache.spark.sql.sources.{BaseRelation, Filter} import org.apache.spark.sql.types.StructType -import org.apache.spark.sql.vectorized.{ColumnVector, ColumnarBatch} -import org.apache.spark.sql.{HoodieSpark3CatalogUtils, SQLContext, SparkSession} +import org.apache.spark.sql.vectorized.{ColumnarBatch, ColumnVector} import org.apache.spark.storage.StorageLevel import java.time.ZoneId diff --git a/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/execution/datasources/Spark3ParquetSchemaEvolutionUtils.scala b/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/execution/datasources/Spark3ParquetSchemaEvolutionUtils.scala index 967c647b284f..d2b2bfad3cf8 100644 --- a/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/execution/datasources/Spark3ParquetSchemaEvolutionUtils.scala +++ b/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/execution/datasources/Spark3ParquetSchemaEvolutionUtils.scala @@ -18,6 +18,8 @@ package org.apache.spark.sql.execution.datasources +import org.apache.hadoop.conf.Configuration +import org.apache.hadoop.fs.Path import org.apache.hudi.SparkAdapterSupport import org.apache.hudi.client.utils.SparkInternalSchemaConverter import org.apache.hudi.common.fs.FSUtils @@ -25,19 +27,14 @@ import org.apache.hudi.common.util import org.apache.hudi.common.util.InternalSchemaCache import org.apache.hudi.common.util.StringUtils.isNullOrEmpty import org.apache.hudi.common.util.collection.Pair -import org.apache.hudi.hadoop.fs.HadoopFSUtils import org.apache.hudi.internal.schema.InternalSchema import org.apache.hudi.internal.schema.action.InternalSchemaMerger import org.apache.hudi.internal.schema.utils.{InternalSchemaUtils, SerDeHelper} -import org.apache.hudi.storage.StoragePath import org.apache.hudi.storage.hadoop.HoodieHadoopStorage - -import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.fs.Path import org.apache.parquet.hadoop.metadata.FileMetaData import org.apache.spark.sql.HoodieSchemaUtils -import org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Cast, UnsafeProjection} +import org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection import org.apache.spark.sql.execution.datasources.Spark3ParquetSchemaEvolutionUtils.pruneInternalSchema import org.apache.spark.sql.execution.datasources.parquet.{HoodieParquetFileFormatHelper, ParquetReadSupport} import org.apache.spark.sql.sources._ diff --git a/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/execution/datasources/Spark3ParsePartitionUtil.scala b/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/execution/datasources/Spark3ParsePartitionUtil.scala index d204512a6ceb..2632a37d0176 100644 --- a/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/execution/datasources/Spark3ParsePartitionUtil.scala +++ b/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/execution/datasources/Spark3ParsePartitionUtil.scala @@ -17,10 +17,9 @@ package org.apache.spark.sql.execution.datasources +import org.apache.hadoop.fs.Path import org.apache.hudi.common.util.PartitionPathEncodeUtils.DEFAULT_PARTITION_PATH import org.apache.hudi.spark3.internal.ReflectUtil - -import org.apache.hadoop.fs.Path import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.catalog.ExternalCatalogUtils.unescapePathName import org.apache.spark.sql.catalyst.expressions.{Cast, Literal} @@ -32,10 +31,9 @@ import org.apache.spark.unsafe.types.UTF8String import java.lang.{Double => JDouble, Long => JLong} import java.math.{BigDecimal => JBigDecimal} import java.time.ZoneId -import java.util.concurrent.ConcurrentHashMap import java.util.{Locale, TimeZone} +import java.util.concurrent.ConcurrentHashMap -import scala.collection.JavaConverters._ import scala.collection.mutable.ArrayBuffer import scala.util.Try import scala.util.control.NonFatal diff --git a/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/HoodieSpark33CatalystPlanUtils.scala b/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/HoodieSpark33CatalystPlanUtils.scala index 54dbaa0db7bb..2d9250bbe9ea 100644 --- a/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/HoodieSpark33CatalystPlanUtils.scala +++ b/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/HoodieSpark33CatalystPlanUtils.scala @@ -19,8 +19,7 @@ package org.apache.spark.sql import org.apache.spark.sql.catalyst.TableIdentifier -import org.apache.spark.sql.catalyst.analysis.AnalysisErrorAt -import org.apache.spark.sql.catalyst.analysis.ResolvedTable +import org.apache.spark.sql.catalyst.analysis.{AnalysisErrorAt, ResolvedTable} import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeSet, Expression, ProjectionOverSchema} import org.apache.spark.sql.catalyst.planning.ScanOperation import org.apache.spark.sql.catalyst.plans.logical._ diff --git a/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/adapter/Spark3_3Adapter.scala b/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/adapter/Spark3_3Adapter.scala index c11c404c33a0..a053646c13ee 100644 --- a/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/adapter/Spark3_3Adapter.scala +++ b/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/adapter/Spark3_3Adapter.scala @@ -17,9 +17,10 @@ package org.apache.spark.sql.adapter +import org.apache.hudi.Spark33HoodieFileScanRDD + import org.apache.avro.Schema import org.apache.hadoop.conf.Configuration -import org.apache.hudi.Spark33HoodieFileScanRDD import org.apache.spark.sql._ import org.apache.spark.sql.avro._ import org.apache.spark.sql.catalyst.InternalRow @@ -31,9 +32,9 @@ import org.apache.spark.sql.catalyst.planning.PhysicalOperation import org.apache.spark.sql.catalyst.plans.logical._ import org.apache.spark.sql.catalyst.util.METADATA_COL_ATTR_KEY import org.apache.spark.sql.connector.catalog.V2TableWithV1Fallback +import org.apache.spark.sql.execution.datasources._ import org.apache.spark.sql.execution.datasources.parquet.{ParquetFileFormat, Spark33LegacyHoodieParquetFileFormat, Spark33ParquetReader, SparkParquetReader} import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation -import org.apache.spark.sql.execution.datasources._ import org.apache.spark.sql.hudi.analysis.TableValuedFunctions import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.parser.{HoodieExtendedParserInterface, HoodieSpark3_3ExtendedSqlParser} diff --git a/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala b/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala index f3b22d4d6f36..caf86e75c9ed 100644 --- a/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala +++ b/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala @@ -37,6 +37,7 @@ import org.apache.spark.unsafe.types.UTF8String import java.math.BigDecimal import java.nio.ByteBuffer import java.util.TimeZone + import scala.collection.JavaConverters._ /** diff --git a/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala b/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala index 277b8d0528af..1a04ace3aaaa 100644 --- a/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala +++ b/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala @@ -17,17 +17,13 @@ package org.apache.spark.sql.avro -import java.nio.ByteBuffer -import scala.collection.JavaConverters._ import org.apache.avro.Conversions.DecimalConversion -import org.apache.avro.LogicalTypes import org.apache.avro.LogicalTypes.{LocalTimestampMicros, LocalTimestampMillis, TimestampMicros, TimestampMillis} -import org.apache.avro.Schema import org.apache.avro.Schema.Type import org.apache.avro.Schema.Type._ -import org.apache.avro.generic.GenericData.{EnumSymbol, Fixed} -import org.apache.avro.generic.GenericData.Record +import org.apache.avro.generic.GenericData.{EnumSymbol, Fixed, Record} import org.apache.avro.util.Utf8 +import org.apache.avro.{LogicalTypes, Schema} import org.apache.spark.internal.Logging import org.apache.spark.sql.avro.AvroSerializer.{createDateRebaseFuncInWrite, createTimestampRebaseFuncInWrite} import org.apache.spark.sql.avro.AvroUtils.{AvroMatchedField, toFieldStr} @@ -39,8 +35,11 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy import org.apache.spark.sql.types._ +import java.nio.ByteBuffer import java.util.TimeZone +import scala.collection.JavaConverters._ + /** * A serializer to serialize data in catalyst format to data in avro format. * diff --git a/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala b/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala index b9845c491dc0..028ebebf0bf5 100644 --- a/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala +++ b/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala @@ -17,19 +17,18 @@ package org.apache.spark.sql.avro -import java.util.Locale - -import scala.collection.JavaConverters._ - import org.apache.avro.Schema -import org.apache.avro.file. FileReader +import org.apache.avro.file.FileReader import org.apache.avro.generic.GenericRecord - import org.apache.spark.internal.Logging import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ +import java.util.Locale + +import scala.collection.JavaConverters._ + /** * NOTE: This code is borrowed from Spark 3.3.0 * This code is borrowed, so that we can better control compatibility w/in Spark minor diff --git a/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/execution/datasources/Spark33NestedSchemaPruning.scala b/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/execution/datasources/Spark33NestedSchemaPruning.scala index cb390642bb59..787ed26fb885 100644 --- a/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/execution/datasources/Spark33NestedSchemaPruning.scala +++ b/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/execution/datasources/Spark33NestedSchemaPruning.scala @@ -18,6 +18,7 @@ package org.apache.spark.sql.execution.datasources import org.apache.hudi.{HoodieBaseRelation, SparkAdapterSupport} + import org.apache.spark.sql.HoodieSpark3CatalystPlanUtils import org.apache.spark.sql.catalyst.expressions.{And, AttributeReference, AttributeSet, Expression, NamedExpression, ProjectionOverSchema} import org.apache.spark.sql.catalyst.planning.PhysicalOperation diff --git a/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark33LegacyHoodieParquetFileFormat.scala b/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark33LegacyHoodieParquetFileFormat.scala index 2e779100df3f..2654b044c194 100644 --- a/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark33LegacyHoodieParquetFileFormat.scala +++ b/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark33LegacyHoodieParquetFileFormat.scala @@ -53,6 +53,7 @@ import org.apache.spark.sql.types.{AtomicType, DataType, StructField, StructType import org.apache.spark.util.SerializableConfiguration import java.net.URI + import scala.collection.convert.ImplicitConversions.`collection AsScalaIterable` /** diff --git a/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/hudi/Spark33ResolveHudiAlterTableCommand.scala b/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/hudi/Spark33ResolveHudiAlterTableCommand.scala index 84cf7c36d5e1..c8078147838a 100644 --- a/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/hudi/Spark33ResolveHudiAlterTableCommand.scala +++ b/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/hudi/Spark33ResolveHudiAlterTableCommand.scala @@ -19,6 +19,7 @@ package org.apache.spark.sql.hudi import org.apache.hudi.common.config.HoodieCommonConfig import org.apache.hudi.internal.schema.action.TableChange.ColumnChangeID + import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.analysis.ResolvedTable import org.apache.spark.sql.catalyst.plans.logical._ diff --git a/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSpark33Analysis.scala b/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSpark33Analysis.scala index 3273d23e7c89..8478fed98d1b 100644 --- a/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSpark33Analysis.scala +++ b/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSpark33Analysis.scala @@ -21,8 +21,8 @@ import org.apache.hudi.DefaultSource import org.apache.spark.sql.catalyst.plans.logical._ import org.apache.spark.sql.catalyst.rules.Rule -import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation import org.apache.spark.sql.execution.datasources.LogicalRelation +import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation import org.apache.spark.sql.hudi.ProvidesHoodieConfig import org.apache.spark.sql.hudi.catalog.HoodieInternalV2Table import org.apache.spark.sql.{SQLContext, SparkSession} diff --git a/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_3ExtendedSqlAstBuilder.scala b/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_3ExtendedSqlAstBuilder.scala index 4e5e32e76fea..ed55d08a175c 100644 --- a/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_3ExtendedSqlAstBuilder.scala +++ b/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_3ExtendedSqlAstBuilder.scala @@ -16,10 +16,11 @@ */ package org.apache.spark.sql.parser -import org.antlr.v4.runtime.tree.{ParseTree, RuleNode, TerminalNode} -import org.antlr.v4.runtime.{ParserRuleContext, Token} import org.apache.hudi.spark.sql.parser.HoodieSqlBaseParser._ import org.apache.hudi.spark.sql.parser.{HoodieSqlBaseBaseVisitor, HoodieSqlBaseParser} + +import org.antlr.v4.runtime.tree.{ParseTree, RuleNode, TerminalNode} +import org.antlr.v4.runtime.{ParserRuleContext, Token} import org.apache.spark.internal.Logging import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.catalyst.analysis._ @@ -46,6 +47,7 @@ import org.apache.spark.util.random.RandomSampler import java.util.Locale import java.util.concurrent.TimeUnit import javax.xml.bind.DatatypeConverter + import scala.collection.JavaConverters._ import scala.collection.mutable.ArrayBuffer diff --git a/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_3ExtendedSqlParser.scala b/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_3ExtendedSqlParser.scala index 24b665c8a37f..e5057f4bebaf 100644 --- a/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_3ExtendedSqlParser.scala +++ b/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_3ExtendedSqlParser.scala @@ -17,12 +17,13 @@ package org.apache.spark.sql.parser +import org.apache.hudi.spark.sql.parser.HoodieSqlBaseParser.{NonReservedContext, QuotedIdentifierContext} +import org.apache.hudi.spark.sql.parser.{HoodieSqlBaseBaseListener, HoodieSqlBaseLexer, HoodieSqlBaseParser} + import org.antlr.v4.runtime._ import org.antlr.v4.runtime.atn.PredictionMode import org.antlr.v4.runtime.misc.{Interval, ParseCancellationException} import org.antlr.v4.runtime.tree.TerminalNodeImpl -import org.apache.hudi.spark.sql.parser.HoodieSqlBaseParser.{NonReservedContext, QuotedIdentifierContext} -import org.apache.hudi.spark.sql.parser.{HoodieSqlBaseBaseListener, HoodieSqlBaseLexer, HoodieSqlBaseParser} import org.apache.spark.internal.Logging import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.parser.{ParseErrorListener, ParseException, ParserInterface} diff --git a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncTool.java b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncTool.java index 2c32e2a6ca18..c2d2dd26fe8b 100644 --- a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncTool.java +++ b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncTool.java @@ -49,8 +49,8 @@ import static org.apache.hudi.hadoop.utils.HoodieInputFormatUtils.getInputFormatClassName; import static org.apache.hudi.hadoop.utils.HoodieInputFormatUtils.getOutputFormatClassName; import static org.apache.hudi.hadoop.utils.HoodieInputFormatUtils.getSerDeClassName; -import static org.apache.hudi.hive.HiveSyncConfig.RECREATE_HIVE_TABLE_ON_ERROR; import static org.apache.hudi.hive.HiveSyncConfig.HIVE_SYNC_FILTER_PUSHDOWN_ENABLED; +import static org.apache.hudi.hive.HiveSyncConfig.RECREATE_HIVE_TABLE_ON_ERROR; import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_AUTO_CREATE_DATABASE; import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_IGNORE_EXCEPTIONS; import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_SKIP_RO_SUFFIX_FOR_READ_OPTIMIZED_TABLE; diff --git a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/MultiPartKeysValueExtractor.java b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/MultiPartKeysValueExtractor.java index 14d4c9099eeb..dd356638a47e 100644 --- a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/MultiPartKeysValueExtractor.java +++ b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/MultiPartKeysValueExtractor.java @@ -19,11 +19,11 @@ package org.apache.hudi.hive; -import java.util.Collections; import org.apache.hudi.common.util.ValidationUtils; import org.apache.hudi.sync.common.model.PartitionValueExtractor; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.stream.Collectors; diff --git a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/transaction/lock/Heartbeat.java b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/transaction/lock/Heartbeat.java index 14398af2c742..f91b66038044 100644 --- a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/transaction/lock/Heartbeat.java +++ b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/transaction/lock/Heartbeat.java @@ -19,9 +19,10 @@ package org.apache.hudi.hive.transaction.lock; -import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hudi.exception.HoodieLockException; +import org.apache.hadoop.hive.metastore.IMetaStoreClient; + class Heartbeat implements Runnable { private final IMetaStoreClient client; private final long lockId; diff --git a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/util/FilterGenVisitor.java b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/util/FilterGenVisitor.java index d0bc5d9e05bf..3187b1767b6a 100644 --- a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/util/FilterGenVisitor.java +++ b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/util/FilterGenVisitor.java @@ -18,11 +18,11 @@ package org.apache.hudi.hive.util; -import org.apache.hudi.expression.NameReference; import org.apache.hudi.expression.BoundReference; import org.apache.hudi.expression.Expression; import org.apache.hudi.expression.ExpressionVisitor; import org.apache.hudi.expression.Literal; +import org.apache.hudi.expression.NameReference; import org.apache.hudi.expression.Predicate; import org.apache.hudi.expression.Predicates; import org.apache.hudi.internal.schema.Types; diff --git a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/util/PartitionFilterGenerator.java b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/util/PartitionFilterGenerator.java index 55354818598d..7026da59a5de 100644 --- a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/util/PartitionFilterGenerator.java +++ b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/util/PartitionFilterGenerator.java @@ -20,13 +20,13 @@ import org.apache.hudi.common.util.ReflectionUtils; import org.apache.hudi.common.util.collection.Pair; -import org.apache.hudi.expression.Predicates; -import org.apache.hudi.hive.HiveSyncConfig; -import org.apache.hudi.hive.HoodieHiveSyncException; -import org.apache.hudi.expression.NameReference; import org.apache.hudi.expression.BinaryExpression; import org.apache.hudi.expression.Expression; import org.apache.hudi.expression.Literal; +import org.apache.hudi.expression.NameReference; +import org.apache.hudi.expression.Predicates; +import org.apache.hudi.hive.HiveSyncConfig; +import org.apache.hudi.hive.HoodieHiveSyncException; import org.apache.hudi.internal.schema.Types; import org.apache.hudi.sync.common.model.FieldSchema; import org.apache.hudi.sync.common.model.Partition; diff --git a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java index e9418ca93749..f8b067014c37 100644 --- a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java +++ b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java @@ -88,8 +88,8 @@ import static org.apache.hudi.common.table.timeline.HoodieTimeline.COMMIT_ACTION; import static org.apache.hudi.common.table.timeline.HoodieTimeline.DELTA_COMMIT_ACTION; import static org.apache.hudi.hadoop.fs.HadoopFSUtils.getRelativePartitionPath; -import static org.apache.hudi.hive.HiveSyncConfig.RECREATE_HIVE_TABLE_ON_ERROR; import static org.apache.hudi.hive.HiveSyncConfig.HIVE_SYNC_FILTER_PUSHDOWN_ENABLED; +import static org.apache.hudi.hive.HiveSyncConfig.RECREATE_HIVE_TABLE_ON_ERROR; import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_AUTO_CREATE_DATABASE; import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_CREATE_MANAGED_TABLE; import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_IGNORE_EXCEPTIONS; diff --git a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestMultiPartKeysValueExtractor.java b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestMultiPartKeysValueExtractor.java index 5cfc654cd25b..d8b9100309e5 100644 --- a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestMultiPartKeysValueExtractor.java +++ b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestMultiPartKeysValueExtractor.java @@ -18,9 +18,10 @@ package org.apache.hudi.hive; +import org.junit.jupiter.api.Test; + import java.util.ArrayList; import java.util.List; -import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; diff --git a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestParquet2SparkSchemaUtils.java b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestParquet2SparkSchemaUtils.java index b6940629af3d..c769dd7f484c 100644 --- a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestParquet2SparkSchemaUtils.java +++ b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestParquet2SparkSchemaUtils.java @@ -19,13 +19,14 @@ package org.apache.hudi.hive; import org.apache.hudi.sync.common.util.Parquet2SparkSchemaUtils; + import org.apache.spark.sql.execution.SparkSqlParser; import org.apache.spark.sql.execution.datasources.parquet.SparkToParquetSchemaConverter; import org.apache.spark.sql.internal.SQLConf; import org.apache.spark.sql.types.ArrayType; +import org.apache.spark.sql.types.IntegerType$; import org.apache.spark.sql.types.MapType; import org.apache.spark.sql.types.Metadata; -import org.apache.spark.sql.types.IntegerType$; import org.apache.spark.sql.types.StringType$; import org.apache.spark.sql.types.StructField; import org.apache.spark.sql.types.StructType; diff --git a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/util/TestPartitionFilterGenerator.java b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/util/TestPartitionFilterGenerator.java index a142020c6863..b607e7f6948c 100644 --- a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/util/TestPartitionFilterGenerator.java +++ b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/util/TestPartitionFilterGenerator.java @@ -20,6 +20,7 @@ import org.apache.hudi.hive.HiveSyncConfig; import org.apache.hudi.sync.common.model.FieldSchema; + import org.junit.jupiter.api.Test; import java.util.ArrayList; diff --git a/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/util/Parquet2SparkSchemaUtils.java b/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/util/Parquet2SparkSchemaUtils.java index c5b98c17eb4a..02729a2c791d 100644 --- a/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/util/Parquet2SparkSchemaUtils.java +++ b/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/util/Parquet2SparkSchemaUtils.java @@ -19,6 +19,7 @@ package org.apache.hudi.sync.common.util; import org.apache.hudi.common.util.ValidationUtils; + import org.apache.parquet.schema.GroupType; import org.apache.parquet.schema.OriginalType; import org.apache.parquet.schema.PrimitiveType; diff --git a/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/handlers/marker/MarkerDirState.java b/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/handlers/marker/MarkerDirState.java index b56d4193d29c..4204a06876f6 100644 --- a/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/handlers/marker/MarkerDirState.java +++ b/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/handlers/marker/MarkerDirState.java @@ -29,8 +29,8 @@ import org.apache.hudi.exception.HoodieEarlyConflictDetectionException; import org.apache.hudi.exception.HoodieException; import org.apache.hudi.exception.HoodieIOException; -import org.apache.hudi.storage.StoragePath; import org.apache.hudi.storage.HoodieStorage; +import org.apache.hudi.storage.StoragePath; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieCleaner.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieCleaner.java index 49eeeeaed0b2..b09c056c6fe5 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieCleaner.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieCleaner.java @@ -22,11 +22,11 @@ import org.apache.hudi.client.common.HoodieSparkEngineContext; import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.exception.HoodieException; import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; import org.apache.hadoop.fs.Path; -import org.apache.hudi.exception.HoodieException; import org.apache.spark.api.java.JavaSparkContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieSnapshotExporter.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieSnapshotExporter.java index 8255b5628dad..e229ae08141b 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieSnapshotExporter.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieSnapshotExporter.java @@ -45,6 +45,7 @@ import org.apache.hudi.util.JavaScalaConverters; import org.apache.hudi.utilities.config.SqlTransformerConfig; import org.apache.hudi.utilities.exception.HoodieSnapshotExporterException; +import org.apache.hudi.utilities.transform.Transformer; import com.beust.jcommander.IValueValidator; import com.beust.jcommander.JCommander; @@ -55,7 +56,6 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; -import org.apache.hudi.utilities.transform.Transformer; import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.sql.Column; diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/SchemaSet.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/SchemaSet.java index f06bb0e6ce1f..fe9340ad2e7c 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/SchemaSet.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/SchemaSet.java @@ -18,11 +18,11 @@ package org.apache.hudi.utilities.schema; -import java.io.Serializable; -import java.util.HashSet; import org.apache.avro.Schema; import org.apache.avro.SchemaNormalization; +import java.io.Serializable; +import java.util.HashSet; import java.util.Set; /** diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/SimpleSchemaProvider.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/SimpleSchemaProvider.java index f87469c01f74..6d721a04875a 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/SimpleSchemaProvider.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/SimpleSchemaProvider.java @@ -18,8 +18,9 @@ package org.apache.hudi.utilities.schema; -import org.apache.avro.Schema; import org.apache.hudi.common.config.TypedProperties; + +import org.apache.avro.Schema; import org.apache.spark.api.java.JavaSparkContext; public class SimpleSchemaProvider extends SchemaProvider { diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/postprocessor/ChainedSchemaPostProcessor.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/postprocessor/ChainedSchemaPostProcessor.java index 0295e80bed8b..352708ae7b6e 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/postprocessor/ChainedSchemaPostProcessor.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/postprocessor/ChainedSchemaPostProcessor.java @@ -19,9 +19,9 @@ package org.apache.hudi.utilities.schema.postprocessor; import org.apache.hudi.common.config.TypedProperties; +import org.apache.hudi.utilities.schema.SchemaPostProcessor; import org.apache.avro.Schema; -import org.apache.hudi.utilities.schema.SchemaPostProcessor; import org.apache.spark.api.java.JavaSparkContext; import java.util.List; diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/ORCDFSSource.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/ORCDFSSource.java index 942bae89eec4..a6ac5d57c9cb 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/ORCDFSSource.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/ORCDFSSource.java @@ -23,6 +23,7 @@ import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.utilities.schema.SchemaProvider; import org.apache.hudi.utilities.sources.helpers.DFSPathSelector; + import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/S3EventsSource.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/S3EventsSource.java index f9ba0c46b971..051176ace4b3 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/S3EventsSource.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/S3EventsSource.java @@ -26,14 +26,14 @@ import org.apache.hudi.utilities.schema.SchemaProvider; import org.apache.hudi.utilities.sources.helpers.S3EventsMetaSelector; -import software.amazon.awssdk.services.sqs.SqsClient; -import software.amazon.awssdk.services.sqs.model.Message; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Encoders; import org.apache.spark.sql.Row; import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.types.StructType; +import software.amazon.awssdk.services.sqs.SqsClient; +import software.amazon.awssdk.services.sqs.model.Message; import java.io.Closeable; import java.io.IOException; diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/SnapshotLoadQuerySplitter.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/SnapshotLoadQuerySplitter.java index f0fd1fed9042..e3b55c232f62 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/SnapshotLoadQuerySplitter.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/SnapshotLoadQuerySplitter.java @@ -26,6 +26,7 @@ import org.apache.hudi.common.util.ReflectionUtils; import org.apache.hudi.utilities.sources.helpers.QueryInfo; import org.apache.hudi.utilities.streamer.SourceProfileSupplier; + import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/AvroConvertor.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/AvroConvertor.java index f9c35bd3b6e1..a6b6d76a1855 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/AvroConvertor.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/AvroConvertor.java @@ -39,10 +39,10 @@ import static org.apache.hudi.utilities.config.HoodieStreamerConfig.SANITIZE_SCHEMA_FIELD_NAMES; import static org.apache.hudi.utilities.config.HoodieStreamerConfig.SCHEMA_FIELD_NAME_INVALID_CHAR_MASK; +import static org.apache.hudi.utilities.schema.KafkaOffsetPostProcessor.KAFKA_SOURCE_KEY_COLUMN; import static org.apache.hudi.utilities.schema.KafkaOffsetPostProcessor.KAFKA_SOURCE_OFFSET_COLUMN; import static org.apache.hudi.utilities.schema.KafkaOffsetPostProcessor.KAFKA_SOURCE_PARTITION_COLUMN; import static org.apache.hudi.utilities.schema.KafkaOffsetPostProcessor.KAFKA_SOURCE_TIMESTAMP_COLUMN; -import static org.apache.hudi.utilities.schema.KafkaOffsetPostProcessor.KAFKA_SOURCE_KEY_COLUMN; /** * Convert a variety of datum into Avro GenericRecords. Has a bunch of lazy fields to circumvent issues around diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/QueryRunner.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/QueryRunner.java index e2571bd54f88..142125b05c2d 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/QueryRunner.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/QueryRunner.java @@ -25,8 +25,8 @@ import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.exception.HoodieException; import org.apache.hudi.utilities.config.HoodieIncrSourceConfig; - import org.apache.hudi.utilities.sources.SnapshotLoadQuerySplitter; + import org.apache.spark.sql.Column; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/S3EventsMetaSelector.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/S3EventsMetaSelector.java index c12b95e9b481..4cc5f72f9cac 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/S3EventsMetaSelector.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/S3EventsMetaSelector.java @@ -26,11 +26,11 @@ import org.apache.hudi.exception.HoodieException; import org.apache.hudi.utilities.config.DFSPathSelectorConfig; -import software.amazon.awssdk.services.sqs.SqsClient; -import software.amazon.awssdk.services.sqs.model.Message; import com.fasterxml.jackson.databind.ObjectMapper; import org.json.JSONException; import org.json.JSONObject; +import software.amazon.awssdk.services.sqs.SqsClient; +import software.amazon.awssdk.services.sqs.model.Message; import java.io.IOException; import java.time.Instant; diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/callback/TestKafkaCallbackProvider.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/callback/TestKafkaCallbackProvider.java index e2c3c86cd5bf..ce058e8f4477 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/callback/TestKafkaCallbackProvider.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/callback/TestKafkaCallbackProvider.java @@ -28,6 +28,7 @@ import org.apache.hudi.utilities.callback.kafka.HoodieWriteCommitKafkaCallback; import org.apache.hudi.utilities.callback.kafka.HoodieWriteCommitKafkaCallbackConfig; import org.apache.hudi.utilities.testutils.UtilitiesTestBase; + import org.apache.spark.streaming.kafka010.KafkaTestUtils; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestErrorTableAwareChainedTransformer.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestErrorTableAwareChainedTransformer.java index 08074e6d6789..b2e9414417ba 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestErrorTableAwareChainedTransformer.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestErrorTableAwareChainedTransformer.java @@ -26,15 +26,15 @@ import org.apache.hudi.utilities.exception.HoodieTransformException; import org.apache.hudi.utilities.transform.ErrorTableAwareChainedTransformer; import org.apache.hudi.utilities.transform.Transformer; + import org.apache.spark.sql.Column; import org.apache.spark.sql.Dataset; -import org.apache.spark.sql.functions; import org.apache.spark.sql.Row; import org.apache.spark.sql.RowFactory; +import org.apache.spark.sql.functions; import org.apache.spark.sql.types.DataTypes; import org.apache.spark.sql.types.StructField; import org.apache.spark.sql.types.StructType; - import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -45,13 +45,13 @@ import static junit.framework.Assert.assertEquals; import static junit.framework.Assert.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertThrows; import static org.apache.hudi.config.HoodieErrorTableConfig.ERROR_TABLE_ENABLED; import static org.apache.hudi.utilities.streamer.BaseErrorTableWriter.ERROR_TABLE_CURRUPT_RECORD_COL_NAME; import static org.apache.spark.sql.types.DataTypes.IntegerType; import static org.apache.spark.sql.types.DataTypes.StringType; import static org.apache.spark.sql.types.DataTypes.createStructField; import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; @Tag("functional") public class TestErrorTableAwareChainedTransformer extends SparkClientFunctionalTestHarness { diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieSnapshotExporter.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieSnapshotExporter.java index 4de0ebeb2dde..329bb22340db 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieSnapshotExporter.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieSnapshotExporter.java @@ -42,8 +42,8 @@ import org.apache.hudi.utilities.HoodieSnapshotExporter.Partitioner; import org.apache.hudi.utilities.config.SqlTransformerConfig; import org.apache.hudi.utilities.exception.HoodieSnapshotExporterException; - import org.apache.hudi.utilities.testutils.UtilitiesTestBase; + import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.function.ForeachFunction; import org.apache.spark.sql.Column; diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestAvroDFSSource.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestAvroDFSSource.java index 808a4ca57cea..89d522675ba1 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestAvroDFSSource.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestAvroDFSSource.java @@ -21,8 +21,10 @@ import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.utilities.testutils.sources.AbstractDFSSourceTestBase; + import org.apache.hadoop.fs.Path; import org.junit.jupiter.api.BeforeEach; + import java.io.IOException; import java.util.List; diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestS3EventsSource.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestS3EventsSource.java index 5ed332a142dd..04caeef40316 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestS3EventsSource.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestS3EventsSource.java @@ -21,8 +21,8 @@ import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.util.Option; -import org.apache.hudi.utilities.streamer.SourceFormatAdapter; import org.apache.hudi.utilities.schema.FilebasedSchemaProvider; +import org.apache.hudi.utilities.streamer.SourceFormatAdapter; import org.apache.hudi.utilities.testutils.sources.AbstractCloudObjectsSourceTestBase; import org.apache.avro.generic.GenericRecord; diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/helpers/TestSnapshotQuerySplitterImpl.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/helpers/TestSnapshotQuerySplitterImpl.java index 16c07d15f3a6..e11522c20b9e 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/helpers/TestSnapshotQuerySplitterImpl.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/helpers/TestSnapshotQuerySplitterImpl.java @@ -23,8 +23,10 @@ import org.apache.hudi.common.util.Option; import org.apache.hudi.utilities.sources.SnapshotLoadQuerySplitter; import org.apache.hudi.utilities.streamer.SourceProfileSupplier; + import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; + import java.util.List; import static org.apache.spark.sql.functions.col; diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/SparkClientFunctionalTestHarnessWithHiveSupport.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/SparkClientFunctionalTestHarnessWithHiveSupport.java index fd59d633e7ae..1a52c1e309df 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/SparkClientFunctionalTestHarnessWithHiveSupport.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/SparkClientFunctionalTestHarnessWithHiveSupport.java @@ -20,6 +20,7 @@ package org.apache.hudi.utilities.testutils; import org.apache.hudi.testutils.SparkClientFunctionalTestHarness; + import org.apache.spark.SparkConf; import java.util.Collections;