From df178a7ba390a02184b235a1ab34c12c79786ac9 Mon Sep 17 00:00:00 2001 From: Dariusz Seweryn Date: Fri, 13 Dec 2024 14:20:16 +0100 Subject: [PATCH] Fixed formatting. --- .../snowflake/ingest/SimpleIngestManager.java | 8 ++-- .../ingest/connection/OAuthClient.java | 10 ++-- .../ingest/connection/OAuthManager.java | 2 +- .../ingest/connection/RequestBuilder.java | 8 ++-- .../connection/ServiceResponseHandler.java | 2 +- .../ingest/connection/TelemetryService.java | 6 +-- .../internal/DataValidationUtil.java | 2 +- .../streaming/internal/FlushService.java | 2 +- .../streaming/internal/InternalStage.java | 16 +++---- .../streaming/internal/ParquetRowBuffer.java | 2 +- .../internal/SnowflakeServiceClient.java | 2 +- ...nowflakeStreamingIngestClientInternal.java | 4 +- .../internal/StreamingIngestUtils.java | 6 +-- .../fileTransferAgent/IcebergAzureClient.java | 20 ++++---- .../fileTransferAgent/IcebergGCSClient.java | 14 +++--- .../fileTransferAgent/IcebergS3Client.java | 46 +++++++++---------- .../IcebergS3ObjectMetadata.java | 2 +- .../IcebergStorageClientFactory.java | 2 +- .../net/snowflake/ingest/utils/HttpUtil.java | 2 +- .../net/snowflake/ingest/utils/Utils.java | 3 +- .../collections/map/UnmodifiableMap.java | 6 +-- .../net/snowflake/ingest/SimpleIngestIT.java | 6 +-- .../java/net/snowflake/ingest/TestUtils.java | 6 +-- .../ingest/connection/MockOAuthClient.java | 2 +- .../streaming/internal/ChannelCacheTest.java | 2 +- .../internal/InsertRowsBenchmarkTest.java | 2 +- .../streaming/internal/InternalStageTest.java | 14 +++--- .../internal/MockSnowflakeServiceClient.java | 8 ++-- .../internal/RegisterServiceTest.java | 2 +- .../SnowflakeStreamingIngestChannelTest.java | 16 +++---- .../SnowflakeStreamingIngestClientTest.java | 18 ++++---- .../internal/StreamingIngestUtilsIT.java | 2 +- .../internal/StreamingIngestUtilsTest.java | 4 +- 33 files changed, 121 insertions(+), 126 deletions(-) diff --git a/src/main/java/net/snowflake/ingest/SimpleIngestManager.java b/src/main/java/net/snowflake/ingest/SimpleIngestManager.java index f9ba34128..441cd4b81 100644 --- a/src/main/java/net/snowflake/ingest/SimpleIngestManager.java +++ b/src/main/java/net/snowflake/ingest/SimpleIngestManager.java @@ -18,10 +18,6 @@ import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.impl.client.CloseableHttpClient; import net.snowflake.ingest.connection.HistoryRangeResponse; import net.snowflake.ingest.connection.HistoryResponse; import net.snowflake.ingest.connection.IngestResponse; @@ -32,6 +28,10 @@ import net.snowflake.ingest.utils.HttpUtil; import net.snowflake.ingest.utils.StagedFileWrapper; import net.snowflake.ingest.utils.Utils; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.impl.client.CloseableHttpClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/net/snowflake/ingest/connection/OAuthClient.java b/src/main/java/net/snowflake/ingest/connection/OAuthClient.java index d1f8384bd..f008b43ce 100644 --- a/src/main/java/net/snowflake/ingest/connection/OAuthClient.java +++ b/src/main/java/net/snowflake/ingest/connection/OAuthClient.java @@ -4,6 +4,8 @@ package net.snowflake.ingest.connection; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.HttpURLConnection; @@ -12,8 +14,9 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; +import net.snowflake.ingest.utils.ErrorCode; +import net.snowflake.ingest.utils.HttpUtil; +import net.snowflake.ingest.utils.SFException; import org.apache.http.HttpHeaders; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpPost; @@ -23,9 +26,6 @@ import org.apache.http.entity.StringEntity; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.util.EntityUtils; -import net.snowflake.ingest.utils.ErrorCode; -import net.snowflake.ingest.utils.HttpUtil; -import net.snowflake.ingest.utils.SFException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/net/snowflake/ingest/connection/OAuthManager.java b/src/main/java/net/snowflake/ingest/connection/OAuthManager.java index a33d18da3..01cbcd846 100644 --- a/src/main/java/net/snowflake/ingest/connection/OAuthManager.java +++ b/src/main/java/net/snowflake/ingest/connection/OAuthManager.java @@ -6,10 +6,10 @@ import java.io.IOException; import java.util.concurrent.TimeUnit; -import org.apache.http.client.utils.URIBuilder; import net.snowflake.ingest.utils.Constants; import net.snowflake.ingest.utils.ErrorCode; import net.snowflake.ingest.utils.SFException; +import org.apache.http.client.utils.URIBuilder; /** This class manages creating and automatically refresh the OAuth token */ public final class OAuthManager extends SecurityManager { diff --git a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java index acfa42b0d..74bc0b90f 100644 --- a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java +++ b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java @@ -15,6 +15,10 @@ import java.util.List; import java.util.Map; import java.util.UUID; +import net.snowflake.ingest.utils.ErrorCode; +import net.snowflake.ingest.utils.SFException; +import net.snowflake.ingest.utils.SnowflakeURL; +import net.snowflake.ingest.utils.StagedFileWrapper; import org.apache.http.HttpHeaders; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; @@ -23,10 +27,6 @@ import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; import org.apache.http.impl.client.CloseableHttpClient; -import net.snowflake.ingest.utils.ErrorCode; -import net.snowflake.ingest.utils.SFException; -import net.snowflake.ingest.utils.SnowflakeURL; -import net.snowflake.ingest.utils.StagedFileWrapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/net/snowflake/ingest/connection/ServiceResponseHandler.java b/src/main/java/net/snowflake/ingest/connection/ServiceResponseHandler.java index 50d7015f3..a3763615f 100644 --- a/src/main/java/net/snowflake/ingest/connection/ServiceResponseHandler.java +++ b/src/main/java/net/snowflake/ingest/connection/ServiceResponseHandler.java @@ -8,6 +8,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; import java.util.UUID; +import net.snowflake.ingest.utils.BackOffException; import org.apache.http.HttpEntity; import org.apache.http.HttpResponse; import org.apache.http.HttpStatus; @@ -17,7 +18,6 @@ import org.apache.http.client.methods.HttpUriRequest; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.util.EntityUtils; -import net.snowflake.ingest.utils.BackOffException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/net/snowflake/ingest/connection/TelemetryService.java b/src/main/java/net/snowflake/ingest/connection/TelemetryService.java index 1f3ccf4c5..8821b6a24 100644 --- a/src/main/java/net/snowflake/ingest/connection/TelemetryService.java +++ b/src/main/java/net/snowflake/ingest/connection/TelemetryService.java @@ -10,16 +10,16 @@ import com.codahale.metrics.Meter; import com.codahale.metrics.Snapshot; import com.codahale.metrics.Timer; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.util.concurrent.RateLimiter; import java.util.HashMap; import java.util.Map; import java.util.concurrent.TimeUnit; -import org.apache.http.impl.client.CloseableHttpClient; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; import net.snowflake.client.jdbc.telemetry.TelemetryClient; import net.snowflake.client.jdbc.telemetry.TelemetryUtil; import net.snowflake.ingest.utils.Logging; +import org.apache.http.impl.client.CloseableHttpClient; /** * Telemetry service to collect logs in the SDK and send them to Snowflake through the JDBC client diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java index a1efd3e16..0582b5315 100644 --- a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java +++ b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java @@ -16,6 +16,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.module.SimpleModule; import com.fasterxml.jackson.databind.ser.std.ToStringSerializer; +import com.google.common.collect.Sets; import java.io.IOException; import java.math.BigDecimal; import java.math.BigInteger; @@ -37,7 +38,6 @@ import java.util.Optional; import java.util.Set; import java.util.function.Supplier; -import com.google.common.collect.Sets; import net.snowflake.client.jdbc.internal.snowflake.common.core.SnowflakeDateTimeFormat; import net.snowflake.client.jdbc.internal.snowflake.common.util.Power10; import net.snowflake.ingest.streaming.internal.serialization.ByteArraySerializer; diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java index 8d685dc20..b83274adc 100644 --- a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java +++ b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java @@ -12,6 +12,7 @@ import com.codahale.metrics.Timer; import com.google.common.annotations.VisibleForTesting; +import com.google.common.util.concurrent.ThreadFactoryBuilder; import java.io.IOException; import java.lang.management.ManagementFactory; import java.security.InvalidAlgorithmParameterException; @@ -38,7 +39,6 @@ import javax.crypto.BadPaddingException; import javax.crypto.IllegalBlockSizeException; import javax.crypto.NoSuchPaddingException; -import com.google.common.util.concurrent.ThreadFactoryBuilder; import net.snowflake.ingest.utils.Constants; import net.snowflake.ingest.utils.ErrorCode; import net.snowflake.ingest.utils.Logging; diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/InternalStage.java b/src/main/java/net/snowflake/ingest/streaming/internal/InternalStage.java index 53da0771b..21aa1c906 100644 --- a/src/main/java/net/snowflake/ingest/streaming/internal/InternalStage.java +++ b/src/main/java/net/snowflake/ingest/streaming/internal/InternalStage.java @@ -46,9 +46,8 @@ class InternalStage implements IStorage { * expects a different version of json object than {@link StreamingIngestResponse}. TODO: * SNOW-1493470 Align Jackson version */ - private static final com.fasterxml.jackson.databind.ObjectMapper - parseConfigureResponseMapper = - new com.fasterxml.jackson.databind.ObjectMapper(); + private static final com.fasterxml.jackson.databind.ObjectMapper parseConfigureResponseMapper = + new com.fasterxml.jackson.databind.ObjectMapper(); private static final long REFRESH_THRESHOLD_IN_MS = TimeUnit.MILLISECONDS.convert(1, TimeUnit.MINUTES); @@ -258,9 +257,7 @@ private synchronized void setFileLocationInfo(FileLocationInfo fileLocationInfo) } static SnowflakeFileTransferMetadataWithAge createFileTransferMetadataWithAge( - FileLocationInfo fileLocationInfo) - throws JsonProcessingException, - SnowflakeSQLException { + FileLocationInfo fileLocationInfo) throws JsonProcessingException, SnowflakeSQLException { final SnowflakeFileTransferMetadataWithAge fileTransferMetadataWithAge; if (fileLocationInfo @@ -323,10 +320,9 @@ SnowflakeFileTransferMetadataV1 fetchSignedURL(String fileName) return metadata; } - static com.fasterxml.jackson.databind.JsonNode - parseFileLocationInfo(FileLocationInfo fileLocationInfo) - throws JsonProcessingException, - com.fasterxml.jackson.core.JsonProcessingException { + static com.fasterxml.jackson.databind.JsonNode parseFileLocationInfo( + FileLocationInfo fileLocationInfo) + throws JsonProcessingException, com.fasterxml.jackson.core.JsonProcessingException { JsonNode fileLocationInfoNode = mapper.valueToTree(fileLocationInfo); // Currently there are a few mismatches between the client/configure response and what diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java index 014f1fd6a..f1f0af47d 100644 --- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java +++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java @@ -6,6 +6,7 @@ import static net.snowflake.ingest.utils.Utils.concatDotPath; +import com.google.common.collect.Sets; import java.math.BigDecimal; import java.math.BigInteger; import java.nio.charset.StandardCharsets; @@ -19,7 +20,6 @@ import java.util.Optional; import java.util.Set; import java.util.function.Consumer; -import com.google.common.collect.Sets; import net.snowflake.ingest.connection.RequestBuilder; import net.snowflake.ingest.connection.TelemetryService; import net.snowflake.ingest.streaming.InsertValidationResponse; diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeServiceClient.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeServiceClient.java index ae7578b8d..d3d423dbb 100644 --- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeServiceClient.java +++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeServiceClient.java @@ -22,13 +22,13 @@ import static net.snowflake.ingest.utils.Constants.RESPONSE_SUCCESS; import java.io.IOException; -import org.apache.http.impl.client.CloseableHttpClient; import net.snowflake.ingest.connection.IngestResponseException; import net.snowflake.ingest.connection.RequestBuilder; import net.snowflake.ingest.connection.ServiceResponseHandler; import net.snowflake.ingest.utils.ErrorCode; import net.snowflake.ingest.utils.Logging; import net.snowflake.ingest.utils.SFException; +import org.apache.http.impl.client.CloseableHttpClient; /** * The SnowflakeServiceClient class is responsible for making API requests to the Snowflake service. diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java index 5f07b1f37..0d7dd75b6 100644 --- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java +++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java @@ -53,8 +53,6 @@ import javax.management.MalformedObjectNameException; import javax.management.ObjectName; import net.snowflake.client.core.SFSessionProperty; -import org.apache.http.client.utils.URIBuilder; -import org.apache.http.impl.client.CloseableHttpClient; import net.snowflake.ingest.connection.IngestResponseException; import net.snowflake.ingest.connection.OAuthCredential; import net.snowflake.ingest.connection.RequestBuilder; @@ -72,6 +70,8 @@ import net.snowflake.ingest.utils.SFException; import net.snowflake.ingest.utils.SnowflakeURL; import net.snowflake.ingest.utils.Utils; +import org.apache.http.client.utils.URIBuilder; +import org.apache.http.impl.client.CloseableHttpClient; import org.apache.parquet.column.ParquetProperties; /** diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtils.java b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtils.java index 666ddb18a..3c5234a6c 100644 --- a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtils.java +++ b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtils.java @@ -11,15 +11,15 @@ import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; import java.util.function.Function; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpUriRequest; -import org.apache.http.impl.client.CloseableHttpClient; import net.snowflake.ingest.connection.IngestResponseException; import net.snowflake.ingest.connection.RequestBuilder; import net.snowflake.ingest.connection.ServiceResponseHandler; import net.snowflake.ingest.utils.ErrorCode; import net.snowflake.ingest.utils.Logging; import net.snowflake.ingest.utils.SFException; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.impl.client.CloseableHttpClient; public class StreamingIngestUtils { diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergAzureClient.java b/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergAzureClient.java index 061a1a23d..09ebf284e 100644 --- a/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergAzureClient.java +++ b/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergAzureClient.java @@ -3,6 +3,16 @@ import static net.snowflake.client.core.Constants.CLOUD_STORAGE_CREDENTIALS_EXPIRED; import static net.snowflake.client.core.HttpUtil.setSessionlessProxyForAzure; +import com.microsoft.azure.storage.OperationContext; +import com.microsoft.azure.storage.StorageCredentials; +import com.microsoft.azure.storage.StorageCredentialsAnonymous; +import com.microsoft.azure.storage.StorageCredentialsSharedAccessSignature; +import com.microsoft.azure.storage.StorageException; +import com.microsoft.azure.storage.StorageExtendedErrorInformation; +import com.microsoft.azure.storage.blob.BlobRequestOptions; +import com.microsoft.azure.storage.blob.CloudBlobClient; +import com.microsoft.azure.storage.blob.CloudBlobContainer; +import com.microsoft.azure.storage.blob.CloudBlockBlob; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; @@ -24,16 +34,6 @@ import net.snowflake.client.jdbc.SnowflakeUtil; import net.snowflake.client.jdbc.cloud.storage.StageInfo; import net.snowflake.client.jdbc.cloud.storage.StorageObjectMetadata; -import com.microsoft.azure.storage.OperationContext; -import com.microsoft.azure.storage.StorageCredentials; -import com.microsoft.azure.storage.StorageCredentialsAnonymous; -import com.microsoft.azure.storage.StorageCredentialsSharedAccessSignature; -import com.microsoft.azure.storage.StorageException; -import com.microsoft.azure.storage.StorageExtendedErrorInformation; -import com.microsoft.azure.storage.blob.BlobRequestOptions; -import com.microsoft.azure.storage.blob.CloudBlobClient; -import com.microsoft.azure.storage.blob.CloudBlobContainer; -import com.microsoft.azure.storage.blob.CloudBlockBlob; import net.snowflake.client.jdbc.internal.snowflake.common.core.SqlState; import net.snowflake.client.util.SFPair; import net.snowflake.client.util.Stopwatch; diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergGCSClient.java b/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergGCSClient.java index cee2d9437..d94a728f1 100644 --- a/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergGCSClient.java +++ b/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergGCSClient.java @@ -3,6 +3,13 @@ import static net.snowflake.client.jdbc.SnowflakeUtil.getRootCause; import static net.snowflake.client.jdbc.SnowflakeUtil.isBlank; +import com.google.api.gax.rpc.FixedHeaderProvider; +import com.google.cloud.storage.Blob; +import com.google.cloud.storage.BlobId; +import com.google.cloud.storage.BlobInfo; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.StorageException; +import com.google.cloud.storage.StorageOptions; import com.google.common.base.Strings; import java.io.File; import java.io.FileInputStream; @@ -22,13 +29,6 @@ import net.snowflake.client.jdbc.SnowflakeSQLLoggedException; import net.snowflake.client.jdbc.cloud.storage.StageInfo; import net.snowflake.client.jdbc.cloud.storage.StorageObjectMetadata; -import com.google.api.gax.rpc.FixedHeaderProvider; -import com.google.cloud.storage.Blob; -import com.google.cloud.storage.BlobId; -import com.google.cloud.storage.BlobInfo; -import com.google.cloud.storage.Storage; -import com.google.cloud.storage.StorageException; -import com.google.cloud.storage.StorageOptions; import net.snowflake.client.jdbc.internal.snowflake.common.core.SqlState; import net.snowflake.client.util.SFPair; import net.snowflake.client.util.Stopwatch; diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergS3Client.java b/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergS3Client.java index 7a8f1849b..2cb93470a 100644 --- a/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergS3Client.java +++ b/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergS3Client.java @@ -4,6 +4,26 @@ import static net.snowflake.client.jdbc.SnowflakeUtil.createDefaultExecutorService; import static net.snowflake.client.jdbc.SnowflakeUtil.getRootCause; +import com.amazonaws.AmazonClientException; +import com.amazonaws.AmazonServiceException; +import com.amazonaws.ClientConfiguration; +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.auth.BasicSessionCredentials; +import com.amazonaws.client.builder.AwsClientBuilder; +import com.amazonaws.client.builder.ExecutorFactory; +import com.amazonaws.regions.Region; +import com.amazonaws.regions.RegionUtils; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3Builder; +import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.model.AmazonS3Exception; +import com.amazonaws.services.s3.model.ObjectMetadata; +import com.amazonaws.services.s3.model.PutObjectRequest; +import com.amazonaws.services.s3.transfer.TransferManager; +import com.amazonaws.services.s3.transfer.TransferManagerBuilder; +import com.amazonaws.services.s3.transfer.Upload; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; @@ -27,34 +47,14 @@ import net.snowflake.client.jdbc.SnowflakeSQLLoggedException; import net.snowflake.client.jdbc.cloud.storage.S3HttpUtil; import net.snowflake.client.jdbc.cloud.storage.StorageObjectMetadata; -import com.amazonaws.AmazonClientException; -import com.amazonaws.AmazonServiceException; -import com.amazonaws.ClientConfiguration; -import com.amazonaws.auth.AWSCredentials; -import com.amazonaws.auth.AWSStaticCredentialsProvider; -import com.amazonaws.auth.BasicAWSCredentials; -import com.amazonaws.auth.BasicSessionCredentials; -import com.amazonaws.client.builder.AwsClientBuilder; -import com.amazonaws.client.builder.ExecutorFactory; -import com.amazonaws.regions.Region; -import com.amazonaws.regions.RegionUtils; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3Builder; -import com.amazonaws.services.s3.AmazonS3Client; -import com.amazonaws.services.s3.model.AmazonS3Exception; -import com.amazonaws.services.s3.model.ObjectMetadata; -import com.amazonaws.services.s3.model.PutObjectRequest; -import com.amazonaws.services.s3.transfer.TransferManager; -import com.amazonaws.services.s3.transfer.TransferManagerBuilder; -import com.amazonaws.services.s3.transfer.Upload; -import org.apache.http.HttpStatus; -import org.apache.http.conn.ssl.SSLConnectionSocketFactory; -import org.apache.http.conn.ssl.SSLInitializationException; import net.snowflake.client.jdbc.internal.snowflake.common.core.SqlState; import net.snowflake.client.util.SFPair; import net.snowflake.client.util.Stopwatch; import net.snowflake.ingest.utils.Logging; import org.apache.commons.io.IOUtils; +import org.apache.http.HttpStatus; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.conn.ssl.SSLInitializationException; class IcebergS3Client implements IcebergStorageClient { private static final Logging logger = new Logging(IcebergS3Client.class); diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergS3ObjectMetadata.java b/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergS3ObjectMetadata.java index 69a65e9ab..03f77c4d5 100644 --- a/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergS3ObjectMetadata.java +++ b/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergS3ObjectMetadata.java @@ -4,10 +4,10 @@ package net.snowflake.ingest.streaming.internal.fileTransferAgent; +import com.amazonaws.services.s3.model.ObjectMetadata; import java.util.Map; import net.snowflake.client.jdbc.SnowflakeUtil; import net.snowflake.client.jdbc.cloud.storage.StorageObjectMetadata; -import com.amazonaws.services.s3.model.ObjectMetadata; /** * s3 implementation of platform independent StorageObjectMetadata interface, wraps an S3 diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergStorageClientFactory.java b/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergStorageClientFactory.java index b50658857..c53439dea 100644 --- a/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergStorageClientFactory.java +++ b/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergStorageClientFactory.java @@ -1,12 +1,12 @@ package net.snowflake.ingest.streaming.internal.fileTransferAgent; +import com.amazonaws.ClientConfiguration; import java.util.Map; import java.util.Properties; import net.snowflake.client.core.HttpUtil; import net.snowflake.client.jdbc.SnowflakeSQLException; import net.snowflake.client.jdbc.cloud.storage.StageInfo; import net.snowflake.client.jdbc.cloud.storage.StorageObjectMetadata; -import com.amazonaws.ClientConfiguration; import net.snowflake.ingest.utils.Logging; /** diff --git a/src/main/java/net/snowflake/ingest/utils/HttpUtil.java b/src/main/java/net/snowflake/ingest/utils/HttpUtil.java index 0a24230f3..601ca7a0a 100644 --- a/src/main/java/net/snowflake/ingest/utils/HttpUtil.java +++ b/src/main/java/net/snowflake/ingest/utils/HttpUtil.java @@ -17,6 +17,7 @@ import java.util.regex.Pattern; import javax.net.ssl.SSLContext; import net.snowflake.client.core.SFSessionProperty; +import net.snowflake.ingest.streaming.internal.StreamingIngestUtils; import org.apache.http.HttpHost; import org.apache.http.HttpRequest; import org.apache.http.HttpResponse; @@ -40,7 +41,6 @@ import org.apache.http.pool.PoolStats; import org.apache.http.protocol.HttpContext; import org.apache.http.ssl.SSLContexts; -import net.snowflake.ingest.streaming.internal.StreamingIngestUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/net/snowflake/ingest/utils/Utils.java b/src/main/java/net/snowflake/ingest/utils/Utils.java index 2c3f913a7..9b584d011 100644 --- a/src/main/java/net/snowflake/ingest/utils/Utils.java +++ b/src/main/java/net/snowflake/ingest/utils/Utils.java @@ -353,8 +353,7 @@ public static void showMemory() { Runtime runtime = Runtime.getRuntime(); logger.logInfo( - "Max max runtime memory={}, total runtime memory={}, free runtime" - + " memory={}", + "Max max runtime memory={}, total runtime memory={}, free runtime" + " memory={}", runtime.maxMemory(), runtime.totalMemory(), runtime.freeMemory()); diff --git a/src/main/java/org/apache/commons/collections/map/UnmodifiableMap.java b/src/main/java/org/apache/commons/collections/map/UnmodifiableMap.java index 54df85840..d53167738 100644 --- a/src/main/java/org/apache/commons/collections/map/UnmodifiableMap.java +++ b/src/main/java/org/apache/commons/collections/map/UnmodifiableMap.java @@ -11,7 +11,7 @@ */ public class UnmodifiableMap { - public static Map decorate(final Map map) { - return Collections.unmodifiableMap(map); - } + public static Map decorate(final Map map) { + return Collections.unmodifiableMap(map); + } } diff --git a/src/test/java/net/snowflake/ingest/SimpleIngestIT.java b/src/test/java/net/snowflake/ingest/SimpleIngestIT.java index 1e1a9036a..189f0be76 100644 --- a/src/test/java/net/snowflake/ingest/SimpleIngestIT.java +++ b/src/test/java/net/snowflake/ingest/SimpleIngestIT.java @@ -19,13 +19,13 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; -import org.apache.http.Header; -import org.apache.http.HttpHeaders; -import org.apache.http.client.methods.HttpPost; import net.snowflake.ingest.connection.HistoryRangeResponse; import net.snowflake.ingest.connection.HistoryResponse; import net.snowflake.ingest.connection.IngestResponse; import net.snowflake.ingest.utils.StagedFileWrapper; +import org.apache.http.Header; +import org.apache.http.HttpHeaders; +import org.apache.http.client.methods.HttpPost; import org.junit.After; import org.junit.Before; import org.junit.Ignore; diff --git a/src/test/java/net/snowflake/ingest/TestUtils.java b/src/test/java/net/snowflake/ingest/TestUtils.java index 99a5f42c0..c121f80bf 100644 --- a/src/test/java/net/snowflake/ingest/TestUtils.java +++ b/src/test/java/net/snowflake/ingest/TestUtils.java @@ -21,6 +21,8 @@ import static net.snowflake.ingest.utils.ParameterProvider.BLOB_FORMAT_VERSION; import static net.snowflake.ingest.utils.ParameterProvider.ENABLE_ICEBERG_STREAMING; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; import java.io.IOException; import java.math.BigDecimal; import java.net.URI; @@ -46,9 +48,6 @@ import java.util.Properties; import java.util.Random; import java.util.function.Supplier; -import org.apache.http.client.utils.URIBuilder; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; import net.snowflake.ingest.streaming.InsertValidationResponse; import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel; import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClient; @@ -59,6 +58,7 @@ import net.snowflake.ingest.utils.SnowflakeURL; import net.snowflake.ingest.utils.Utils; import org.apache.commons.codec.binary.Base64; +import org.apache.http.client.utils.URIBuilder; import org.junit.Assert; public class TestUtils { diff --git a/src/test/java/net/snowflake/ingest/connection/MockOAuthClient.java b/src/test/java/net/snowflake/ingest/connection/MockOAuthClient.java index 2b1ba91c8..4188735a9 100644 --- a/src/test/java/net/snowflake/ingest/connection/MockOAuthClient.java +++ b/src/test/java/net/snowflake/ingest/connection/MockOAuthClient.java @@ -2,9 +2,9 @@ import java.util.UUID; import java.util.concurrent.atomic.AtomicReference; -import org.apache.http.client.utils.URIBuilder; import net.snowflake.ingest.utils.ErrorCode; import net.snowflake.ingest.utils.SFException; +import org.apache.http.client.utils.URIBuilder; /** Mock implementation of {@link OAuthClient}, only use for test */ public class MockOAuthClient extends OAuthClient { diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/ChannelCacheTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/ChannelCacheTest.java index 2862870b6..b59c628f7 100644 --- a/src/test/java/net/snowflake/ingest/streaming/internal/ChannelCacheTest.java +++ b/src/test/java/net/snowflake/ingest/streaming/internal/ChannelCacheTest.java @@ -11,10 +11,10 @@ import java.util.Map; import java.util.Properties; import java.util.concurrent.ConcurrentHashMap; -import org.apache.http.impl.client.CloseableHttpClient; import net.snowflake.ingest.connection.RequestBuilder; import net.snowflake.ingest.streaming.OpenChannelRequest; import net.snowflake.ingest.utils.ParameterProvider; +import org.apache.http.impl.client.CloseableHttpClient; import org.apache.parquet.column.ParquetProperties; import org.junit.Assert; import org.junit.Before; diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/InsertRowsBenchmarkTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/InsertRowsBenchmarkTest.java index c116b6ad7..ab65c72bb 100644 --- a/src/test/java/net/snowflake/ingest/streaming/internal/InsertRowsBenchmarkTest.java +++ b/src/test/java/net/snowflake/ingest/streaming/internal/InsertRowsBenchmarkTest.java @@ -11,12 +11,12 @@ import java.util.Map; import java.util.Properties; import java.util.concurrent.TimeUnit; -import org.apache.http.impl.client.CloseableHttpClient; import net.snowflake.ingest.connection.RequestBuilder; import net.snowflake.ingest.streaming.InsertValidationResponse; import net.snowflake.ingest.streaming.OpenChannelRequest; import net.snowflake.ingest.utils.ParameterProvider; import net.snowflake.ingest.utils.Utils; +import org.apache.http.impl.client.CloseableHttpClient; import org.apache.parquet.column.ParquetProperties; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/InternalStageTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/InternalStageTest.java index da92c8b98..93ce0d6d6 100644 --- a/src/test/java/net/snowflake/ingest/streaming/internal/InternalStageTest.java +++ b/src/test/java/net/snowflake/ingest/streaming/internal/InternalStageTest.java @@ -16,6 +16,9 @@ import static net.snowflake.ingest.utils.HttpUtil.shouldBypassProxy; import static org.mockito.Mockito.times; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.util.concurrent.ThreadFactoryBuilder; import java.io.ByteArrayInputStream; import java.io.InputStream; import java.nio.charset.StandardCharsets; @@ -29,7 +32,6 @@ import java.util.concurrent.Executors; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; -import com.google.common.util.concurrent.ThreadFactoryBuilder; import net.snowflake.client.core.HttpUtil; import net.snowflake.client.core.OCSPMode; import net.snowflake.client.core.SFSessionProperty; @@ -38,17 +40,15 @@ import net.snowflake.client.jdbc.SnowflakeFileTransferMetadataV1; import net.snowflake.client.jdbc.SnowflakeSQLException; import net.snowflake.client.jdbc.cloud.storage.StageInfo; +import net.snowflake.ingest.TestUtils; +import net.snowflake.ingest.connection.RequestBuilder; +import net.snowflake.ingest.utils.ErrorCode; +import net.snowflake.ingest.utils.SFException; import org.apache.http.HttpEntity; import org.apache.http.StatusLine; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.entity.BasicHttpEntity; import org.apache.http.impl.client.CloseableHttpClient; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import net.snowflake.ingest.TestUtils; -import net.snowflake.ingest.connection.RequestBuilder; -import net.snowflake.ingest.utils.ErrorCode; -import net.snowflake.ingest.utils.SFException; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/MockSnowflakeServiceClient.java b/src/test/java/net/snowflake/ingest/streaming/internal/MockSnowflakeServiceClient.java index 8505e0e6e..d67d3d3a8 100644 --- a/src/test/java/net/snowflake/ingest/streaming/internal/MockSnowflakeServiceClient.java +++ b/src/test/java/net/snowflake/ingest/streaming/internal/MockSnowflakeServiceClient.java @@ -17,7 +17,11 @@ import java.util.List; import java.util.Map; import java.util.function.Function; +import net.snowflake.ingest.TestUtils; +import net.snowflake.ingest.connection.RequestBuilder; +import net.snowflake.ingest.utils.Logging; import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.tuple.Pair; import org.apache.http.HttpEntity; import org.apache.http.HttpStatus; import org.apache.http.HttpVersion; @@ -26,10 +30,6 @@ import org.apache.http.client.methods.HttpUriRequest; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.message.BasicStatusLine; -import net.snowflake.ingest.TestUtils; -import net.snowflake.ingest.connection.RequestBuilder; -import net.snowflake.ingest.utils.Logging; -import org.apache.commons.lang3.tuple.Pair; import org.mockito.Mockito; import org.mockito.stubbing.Answer; diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RegisterServiceTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RegisterServiceTest.java index 4ce4f3efa..fe15253aa 100644 --- a/src/test/java/net/snowflake/ingest/streaming/internal/RegisterServiceTest.java +++ b/src/test/java/net/snowflake/ingest/streaming/internal/RegisterServiceTest.java @@ -16,10 +16,10 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import org.apache.http.impl.client.CloseableHttpClient; import net.snowflake.ingest.connection.RequestBuilder; import net.snowflake.ingest.utils.Pair; import net.snowflake.ingest.utils.ParameterProvider; +import org.apache.http.impl.client.CloseableHttpClient; import org.junit.After; import org.junit.Assert; import org.junit.Before; diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java index b7ec55d91..192727d3c 100644 --- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java +++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java @@ -29,14 +29,6 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; import net.snowflake.client.core.SFSessionProperty; -import org.apache.commons.io.IOUtils; -import org.apache.http.HttpEntity; -import org.apache.http.HttpHeaders; -import org.apache.http.HttpStatus; -import org.apache.http.StatusLine; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.impl.client.CloseableHttpClient; import net.snowflake.ingest.TestUtils; import net.snowflake.ingest.connection.RequestBuilder; import net.snowflake.ingest.streaming.InsertValidationResponse; @@ -48,7 +40,15 @@ import net.snowflake.ingest.utils.SFException; import net.snowflake.ingest.utils.SnowflakeURL; import net.snowflake.ingest.utils.Utils; +import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.tuple.Pair; +import org.apache.http.HttpEntity; +import org.apache.http.HttpHeaders; +import org.apache.http.HttpStatus; +import org.apache.http.StatusLine; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.impl.client.CloseableHttpClient; import org.apache.parquet.column.ParquetProperties; import org.junit.After; import org.junit.Assert; diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java index e295166a1..d887b92f2 100644 --- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java +++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.PropertyAccessor; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Sets; import java.io.IOException; import java.io.StringWriter; import java.security.KeyPair; @@ -38,16 +39,7 @@ import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; -import com.google.common.collect.Sets; import net.snowflake.client.core.SFSessionProperty; -import org.apache.commons.io.IOUtils; -import org.apache.http.HttpEntity; -import org.apache.http.HttpHeaders; -import org.apache.http.HttpStatus; -import org.apache.http.StatusLine; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.impl.client.CloseableHttpClient; import net.snowflake.ingest.TestUtils; import net.snowflake.ingest.connection.RequestBuilder; import net.snowflake.ingest.streaming.DropChannelRequest; @@ -60,7 +52,15 @@ import net.snowflake.ingest.utils.SFException; import net.snowflake.ingest.utils.SnowflakeURL; import net.snowflake.ingest.utils.Utils; +import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.tuple.Pair; +import org.apache.http.HttpEntity; +import org.apache.http.HttpHeaders; +import org.apache.http.HttpStatus; +import org.apache.http.StatusLine; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.impl.client.CloseableHttpClient; import org.apache.parquet.column.ParquetProperties; import org.apache.parquet.schema.PrimitiveType; import org.apache.parquet.schema.Types; diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsIT.java index 22a8a1ea4..bf79e39fa 100644 --- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsIT.java +++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsIT.java @@ -9,13 +9,13 @@ import static net.snowflake.ingest.utils.Constants.CLIENT_CONFIGURE_ENDPOINT; import static net.snowflake.ingest.utils.Constants.RESPONSE_SUCCESS; -import org.apache.http.impl.client.CloseableHttpClient; import net.snowflake.ingest.TestUtils; import net.snowflake.ingest.connection.IngestResponseException; import net.snowflake.ingest.connection.JWTManager; import net.snowflake.ingest.connection.RequestBuilder; import net.snowflake.ingest.utils.HttpUtil; import net.snowflake.ingest.utils.SnowflakeURL; +import org.apache.http.impl.client.CloseableHttpClient; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsTest.java index 711b13091..854ac66ef 100644 --- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsTest.java +++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsTest.java @@ -10,13 +10,13 @@ import com.fasterxml.jackson.databind.ObjectMapper; import java.io.InputStream; import java.util.ArrayList; +import net.snowflake.ingest.TestUtils; +import net.snowflake.ingest.connection.RequestBuilder; import org.apache.commons.io.IOUtils; import org.apache.http.HttpEntity; import org.apache.http.StatusLine; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.impl.client.CloseableHttpClient; -import net.snowflake.ingest.TestUtils; -import net.snowflake.ingest.connection.RequestBuilder; import org.junit.Assert; import org.junit.Test; import org.mockito.Mockito;