From ae74f90d6ec8848726a705e63dc64fbe7d466bde Mon Sep 17 00:00:00 2001 From: Lukas Sembera Date: Mon, 22 Jul 2024 18:40:49 +0200 Subject: [PATCH 1/2] SNOW-1512935 Reduce client socket timeout from 5 minutes to 1 minute (#795) --- src/main/java/net/snowflake/ingest/utils/HttpUtil.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/net/snowflake/ingest/utils/HttpUtil.java b/src/main/java/net/snowflake/ingest/utils/HttpUtil.java index 1be382797..0da6370f7 100644 --- a/src/main/java/net/snowflake/ingest/utils/HttpUtil.java +++ b/src/main/java/net/snowflake/ingest/utils/HttpUtil.java @@ -79,7 +79,7 @@ public class HttpUtil { private static final ReentrantLock idleConnectionMonitorThreadLock = new ReentrantLock(true); private static final int DEFAULT_CONNECTION_TIMEOUT_MINUTES = 1; - private static final int DEFAULT_HTTP_CLIENT_SOCKET_TIMEOUT_MINUTES = 5; + private static final int DEFAULT_HTTP_CLIENT_SOCKET_TIMEOUT_MINUTES = 1; /** * After how many seconds of inactivity should be idle connections evicted from the connection From f69259a81d849948dabf72af88d48f7e1d94e7f6 Mon Sep 17 00:00:00 2001 From: Lukas Sembera Date: Mon, 22 Jul 2024 19:53:52 +0200 Subject: [PATCH 2/2] SNOW-1545879 Reduce the max channel/chunk sizes (#796) * SNOW-1545879 Reduce the max channel/chunk sizes * Fix flaky test * Trigger build --- .../java/net/snowflake/ingest/utils/ParameterProvider.java | 4 ++-- .../ingest/streaming/internal/StreamingIngestIT.java | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java b/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java index b98972a7d..9d9db5356 100644 --- a/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java +++ b/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java @@ -50,8 +50,8 @@ public class ParameterProvider { public static final int IO_TIME_CPU_RATIO_DEFAULT = 2; public static final int BLOB_UPLOAD_MAX_RETRY_COUNT_DEFAULT = 24; public static final long MAX_MEMORY_LIMIT_IN_BYTES_DEFAULT = -1L; - public static final long MAX_CHANNEL_SIZE_IN_BYTES_DEFAULT = 128 * 1024 * 1024; - public static final long MAX_CHUNK_SIZE_IN_BYTES_DEFAULT = 512 * 1024 * 1024; + public static final long MAX_CHANNEL_SIZE_IN_BYTES_DEFAULT = 64 * 1024 * 1024; + public static final long MAX_CHUNK_SIZE_IN_BYTES_DEFAULT = 256 * 1024 * 1024; // Lag related parameters public static final long MAX_CLIENT_LAG_DEFAULT = 1000; // 1 second diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java index 2ecaa4e01..9426d4dc1 100644 --- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java +++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java @@ -8,6 +8,7 @@ import static net.snowflake.ingest.utils.ParameterProvider.BDEC_PARQUET_COMPRESSION_ALGORITHM; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.core.Is.is; +import static org.mockito.Mockito.atLeastOnce; import java.math.BigDecimal; import java.math.BigInteger; @@ -187,7 +188,7 @@ public void testSimpleIngest() throws Exception { // verify expected request sent to server String[] expectedPayloadParams = {"request_id", "blobs", "role", "blob_stats"}; for (String expectedParam : expectedPayloadParams) { - Mockito.verify(requestBuilder) + Mockito.verify(requestBuilder, atLeastOnce()) .generateStreamingIngestPostRequest( ArgumentMatchers.contains(expectedParam), ArgumentMatchers.refEq(REGISTER_BLOB_ENDPOINT),