diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java index 4a13f5adb..1561a3f55 100644 --- a/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java +++ b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java @@ -202,7 +202,6 @@ public InsertValidationResponse insertRows( tempRowsSizeInBytes += addTempRow(row, tempRowCount, rowBuffer.tempStatsMap, inputColumnNames, tempRowCount); tempRowCount++; - checkBatchSizeEnforcedMaximum(tempRowsSizeInBytes); if ((long) rowBuffer.bufferedRowCount + tempRowCount >= Integer.MAX_VALUE) { throw new SFException(ErrorCode.INTERNAL_ERROR, "Row count reaches MAX value"); } @@ -261,7 +260,6 @@ public InsertValidationResponse insertRows( response.addError(error); } rowIndex++; - checkBatchSizeEnforcedMaximum(tempRowsSizeInBytes); if ((long) rowBuffer.bufferedRowCount + rowIndex >= Integer.MAX_VALUE) { throw new SFException(ErrorCode.INTERNAL_ERROR, "Row count reaches MAX value"); } @@ -673,15 +671,6 @@ static AbstractRowBuffer createRowBuffer( } } - private void checkBatchSizeEnforcedMaximum(float batchSizeInBytes) { - if (batchSizeInBytes > clientBufferParameters.getMaxChunkSizeInBytes()) { - throw new SFException( - ErrorCode.MAX_BATCH_SIZE_EXCEEDED, - clientBufferParameters.getMaxChunkSizeInBytes(), - INSERT_ROWS_RECOMMENDED_MAX_BATCH_SIZE_IN_BYTES); - } - } - private void checkBatchSizeRecommendedMaximum(float batchSizeInBytes) { if (batchSizeInBytes > INSERT_ROWS_RECOMMENDED_MAX_BATCH_SIZE_IN_BYTES) { logger.logWarn( diff --git a/src/main/java/net/snowflake/ingest/utils/Constants.java b/src/main/java/net/snowflake/ingest/utils/Constants.java index 096bb4ed4..3579e4d24 100644 --- a/src/main/java/net/snowflake/ingest/utils/Constants.java +++ b/src/main/java/net/snowflake/ingest/utils/Constants.java @@ -42,7 +42,7 @@ public class Constants { 7L; // Don't change, should match server side public static final int BLOB_UPLOAD_TIMEOUT_IN_SEC = 5; public static final int INSERT_THROTTLE_MAX_RETRY_COUNT = 60; - public static final long MAX_BLOB_SIZE_IN_BYTES = 256000000L; + public static final long MAX_BLOB_SIZE_IN_BYTES = 1024 * 1024 * 1024; public static final int BLOB_TAG_SIZE_IN_BYTES = 4; public static final int BLOB_VERSION_SIZE_IN_BYTES = 1; public static final int BLOB_FILE_SIZE_SIZE_IN_BYTES = 8; diff --git a/src/main/java/net/snowflake/ingest/utils/ErrorCode.java b/src/main/java/net/snowflake/ingest/utils/ErrorCode.java index 46c11a485..b863717e9 100644 --- a/src/main/java/net/snowflake/ingest/utils/ErrorCode.java +++ b/src/main/java/net/snowflake/ingest/utils/ErrorCode.java @@ -40,9 +40,8 @@ public enum ErrorCode { MAKE_URI_FAILURE("0032"), OAUTH_REFRESH_TOKEN_ERROR("0033"), INVALID_CONFIG_PARAMETER("0034"), - MAX_BATCH_SIZE_EXCEEDED("0035"), - CRYPTO_PROVIDER_ERROR("0036"), - DROP_CHANNEL_FAILURE("0037"); + CRYPTO_PROVIDER_ERROR("0035"), + DROP_CHANNEL_FAILURE("0036"); public static final String errorMessageResource = "net.snowflake.ingest.ingest_error_messages"; diff --git a/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java b/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java index 7967f1fdd..b98972a7d 100644 --- a/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java +++ b/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java @@ -50,8 +50,8 @@ public class ParameterProvider { public static final int IO_TIME_CPU_RATIO_DEFAULT = 2; public static final int BLOB_UPLOAD_MAX_RETRY_COUNT_DEFAULT = 24; public static final long MAX_MEMORY_LIMIT_IN_BYTES_DEFAULT = -1L; - public static final long MAX_CHANNEL_SIZE_IN_BYTES_DEFAULT = 32000000L; - public static final long MAX_CHUNK_SIZE_IN_BYTES_DEFAULT = 128000000L; + public static final long MAX_CHANNEL_SIZE_IN_BYTES_DEFAULT = 128 * 1024 * 1024; + public static final long MAX_CHUNK_SIZE_IN_BYTES_DEFAULT = 512 * 1024 * 1024; // Lag related parameters public static final long MAX_CLIENT_LAG_DEFAULT = 1000; // 1 second diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java index 5e34bc90b..2a3bb7edd 100644 --- a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java +++ b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java @@ -1021,17 +1021,8 @@ private void testMaxInsertRowsBatchSizeHelper(OpenChannelRequest.OnErrorOption o } // Insert rows should succeed - innerBuffer.insertRows(rows, "", ""); - - // After adding another row, it should fail due to too large batch of rows passed to - // insertRows() in one go rows.add(Collections.singletonMap("COLBINARY", arr)); - try { - innerBuffer.insertRows(rows, "", ""); - Assert.fail("Inserting rows should have failed"); - } catch (SFException e) { - Assert.assertEquals(ErrorCode.MAX_BATCH_SIZE_EXCEEDED.getMessageCode(), e.getVendorCode()); - } + innerBuffer.insertRows(rows, "", ""); } @Test