SNOW-1258064 Remove/Relax the limitations in order to generate bigger…

… files (#730) * Increase chunk/blob/channel size limitation.
snowflakedb · Apr 3, 2024 · f75124c · f75124c
1 parent a1259cc
commit f75124c
Show file tree

Hide file tree

Showing 5 changed files with 6 additions and 27 deletions.
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
@@ -202,7 +202,6 @@ public InsertValidationResponse insertRows(
         tempRowsSizeInBytes +=
             addTempRow(row, tempRowCount, rowBuffer.tempStatsMap, inputColumnNames, tempRowCount);
         tempRowCount++;
-        checkBatchSizeEnforcedMaximum(tempRowsSizeInBytes);
         if ((long) rowBuffer.bufferedRowCount + tempRowCount >= Integer.MAX_VALUE) {
           throw new SFException(ErrorCode.INTERNAL_ERROR, "Row count reaches MAX value");
         }
@@ -261,7 +260,6 @@ public InsertValidationResponse insertRows(
           response.addError(error);
         }
         rowIndex++;
-        checkBatchSizeEnforcedMaximum(tempRowsSizeInBytes);
         if ((long) rowBuffer.bufferedRowCount + rowIndex >= Integer.MAX_VALUE) {
           throw new SFException(ErrorCode.INTERNAL_ERROR, "Row count reaches MAX value");
         }
@@ -673,15 +671,6 @@ static <T> AbstractRowBuffer<T> createRowBuffer(
     }
   }
 
-  private void checkBatchSizeEnforcedMaximum(float batchSizeInBytes) {
-    if (batchSizeInBytes > clientBufferParameters.getMaxChunkSizeInBytes()) {
-      throw new SFException(
-          ErrorCode.MAX_BATCH_SIZE_EXCEEDED,
-          clientBufferParameters.getMaxChunkSizeInBytes(),
-          INSERT_ROWS_RECOMMENDED_MAX_BATCH_SIZE_IN_BYTES);
-    }
-  }
-
   private void checkBatchSizeRecommendedMaximum(float batchSizeInBytes) {
     if (batchSizeInBytes > INSERT_ROWS_RECOMMENDED_MAX_BATCH_SIZE_IN_BYTES) {
       logger.logWarn(

diff --git a/src/main/java/net/snowflake/ingest/utils/Constants.java b/src/main/java/net/snowflake/ingest/utils/Constants.java
@@ -42,7 +42,7 @@ public class Constants {
       7L; // Don't change, should match server side
   public static final int BLOB_UPLOAD_TIMEOUT_IN_SEC = 5;
   public static final int INSERT_THROTTLE_MAX_RETRY_COUNT = 60;
-  public static final long MAX_BLOB_SIZE_IN_BYTES = 256000000L;
+  public static final long MAX_BLOB_SIZE_IN_BYTES = 1024 * 1024 * 1024;
   public static final int BLOB_TAG_SIZE_IN_BYTES = 4;
   public static final int BLOB_VERSION_SIZE_IN_BYTES = 1;
   public static final int BLOB_FILE_SIZE_SIZE_IN_BYTES = 8;

diff --git a/src/main/java/net/snowflake/ingest/utils/ErrorCode.java b/src/main/java/net/snowflake/ingest/utils/ErrorCode.java
@@ -40,9 +40,8 @@ public enum ErrorCode {
   MAKE_URI_FAILURE("0032"),
   OAUTH_REFRESH_TOKEN_ERROR("0033"),
   INVALID_CONFIG_PARAMETER("0034"),
-  MAX_BATCH_SIZE_EXCEEDED("0035"),
-  CRYPTO_PROVIDER_ERROR("0036"),
-  DROP_CHANNEL_FAILURE("0037");
+  CRYPTO_PROVIDER_ERROR("0035"),
+  DROP_CHANNEL_FAILURE("0036");
 
   public static final String errorMessageResource = "net.snowflake.ingest.ingest_error_messages";
 

diff --git a/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java b/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java
@@ -50,8 +50,8 @@ public class ParameterProvider {
   public static final int IO_TIME_CPU_RATIO_DEFAULT = 2;
   public static final int BLOB_UPLOAD_MAX_RETRY_COUNT_DEFAULT = 24;
   public static final long MAX_MEMORY_LIMIT_IN_BYTES_DEFAULT = -1L;
-  public static final long MAX_CHANNEL_SIZE_IN_BYTES_DEFAULT = 32000000L;
-  public static final long MAX_CHUNK_SIZE_IN_BYTES_DEFAULT = 128000000L;
+  public static final long MAX_CHANNEL_SIZE_IN_BYTES_DEFAULT = 128 * 1024 * 1024;
+  public static final long MAX_CHUNK_SIZE_IN_BYTES_DEFAULT = 512 * 1024 * 1024;
 
   // Lag related parameters
   public static final long MAX_CLIENT_LAG_DEFAULT = 1000; // 1 second

diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
@@ -1021,17 +1021,8 @@ private void testMaxInsertRowsBatchSizeHelper(OpenChannelRequest.OnErrorOption o
     }
 
     // Insert rows should succeed
-    innerBuffer.insertRows(rows, "", "");
-
-    // After adding another row, it should fail due to too large batch of rows passed to
-    // insertRows() in one go
     rows.add(Collections.singletonMap("COLBINARY", arr));
-    try {
-      innerBuffer.insertRows(rows, "", "");
-      Assert.fail("Inserting rows should have failed");
-    } catch (SFException e) {
-      Assert.assertEquals(ErrorCode.MAX_BATCH_SIZE_EXCEEDED.getMessageCode(), e.getVendorCode());
-    }
+    innerBuffer.insertRows(rows, "", "");
   }
 
   @Test