Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

NO-SNOW: Fix two issues in insertRows API #602

Merged
merged 40 commits into from
Oct 17, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
40 commits
Select commit Hold shift + click to select a range
327d367
fixes
sfc-gh-tzhang Feb 7, 2023
9b22e75
minor change
sfc-gh-tzhang Feb 7, 2023
ed7ed11
Merge branch 'master' into tzhang-si-random
sfc-gh-tzhang Feb 7, 2023
a9d0e97
Merge branch 'master' into tzhang-si-random
sfc-gh-tzhang Feb 28, 2023
2ac72fc
add logging
sfc-gh-tzhang Feb 28, 2023
a26cd89
Merge branch 'master' into tzhang-si-random
sfc-gh-tzhang Apr 14, 2023
3af27c1
update behavior change
sfc-gh-tzhang Apr 14, 2023
c6a3edf
fix tests
sfc-gh-tzhang Apr 14, 2023
a4de40c
changes
sfc-gh-tzhang Apr 19, 2023
f6aa8e8
changes
sfc-gh-tzhang Apr 19, 2023
688e2b1
Merge branch 'master' into tzhang-si-random
sfc-gh-tzhang Apr 19, 2023
9b6a05c
small update
sfc-gh-tzhang Apr 19, 2023
0cd3e63
Merge branch 'master' into tzhang-si-random
sfc-gh-tzhang Apr 19, 2023
31e952f
Merge branch 'master' into tzhang-si-random
sfc-gh-tzhang Apr 20, 2023
f9df346
Merge branch 'master' into tzhang-si-random
sfc-gh-tzhang Apr 28, 2023
2894620
update
sfc-gh-tzhang May 5, 2023
cf97417
Merge branch 'tzhang-si-random' of github.com:snowflakedb/snowflake-i…
sfc-gh-tzhang May 5, 2023
e231414
fix tests
sfc-gh-tzhang May 5, 2023
64a3e1c
add comments
sfc-gh-tzhang May 5, 2023
73fc7f3
Merge branch 'master' into tzhang-si-random
sfc-gh-tzhang May 5, 2023
224a9c0
merge master
sfc-gh-tzhang Jun 2, 2023
b3d3b9b
add tests
sfc-gh-tzhang Jun 2, 2023
2049c19
Merge branch 'master' into tzhang-si-random
sfc-gh-tzhang Aug 23, 2023
2c6906e
add change
sfc-gh-tzhang Aug 23, 2023
a823e7c
format
sfc-gh-tzhang Aug 23, 2023
e3b5f57
Merge branch 'master' into tzhang-si-random
sfc-gh-tzhang Sep 7, 2023
572e875
save progress
sfc-gh-tzhang Sep 7, 2023
f1f85d7
save progress
sfc-gh-tzhang Sep 8, 2023
3e2408c
Merge branch 'master' into tzhang-si-random
sfc-gh-tzhang Sep 19, 2023
654afeb
Merge branch 'master' into tzhang-si-random
sfc-gh-tzhang Oct 4, 2023
bb014dc
save progress
sfc-gh-tzhang Oct 4, 2023
6e20f47
save progress
sfc-gh-tzhang Oct 4, 2023
825dd23
address comment + add tests
sfc-gh-tzhang Oct 4, 2023
b269cb9
Merge branch 'master' into tzhang-si-random
sfc-gh-tzhang Oct 12, 2023
123b972
address comments
sfc-gh-tzhang Oct 12, 2023
b6e7d3e
format
sfc-gh-tzhang Oct 12, 2023
3f7fcd1
fix test
sfc-gh-tzhang Oct 13, 2023
8200563
Merge branch 'master' into tzhang-si-random
sfc-gh-tzhang Oct 16, 2023
8235f67
fix behavior
sfc-gh-tzhang Oct 17, 2023
b306c34
nit
sfc-gh-tzhang Oct 17, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,9 @@ public InsertValidationResponse insertRows(
InsertValidationResponse.InsertError error =
new InsertValidationResponse.InsertError(row, rowIndex);
try {
if (rowBuffer.bufferedRowCount == Integer.MAX_VALUE) {
throw new SFException(ErrorCode.INTERNAL_ERROR, "Row count reaches MAX value");
}
Set<String> inputColumnNames = verifyInputColumns(row, error, rowIndex);
rowsSizeInBytes +=
addRow(
Expand All @@ -163,11 +166,7 @@ public InsertValidationResponse insertRows(
error.setException(new SFException(e, ErrorCode.INTERNAL_ERROR, e.getMessage()));
response.addError(error);
}
checkBatchSizeEnforcedMaximum(rowsSizeInBytes);
rowIndex++;
if (rowBuffer.bufferedRowCount == Integer.MAX_VALUE) {
throw new SFException(ErrorCode.INTERNAL_ERROR, "Row count reaches MAX value");
}
}
checkBatchSizeRecommendedMaximum(rowsSizeInBytes);
rowBuffer.channelState.setOffsetToken(offsetToken);
Expand All @@ -191,17 +190,17 @@ public InsertValidationResponse insertRows(
Set<String> inputColumnNames = verifyInputColumns(row, null, tempRowCount);
tempRowsSizeInBytes +=
addTempRow(row, tempRowCount, rowBuffer.tempStatsMap, inputColumnNames, tempRowCount);
checkBatchSizeEnforcedMaximum(tempRowsSizeInBytes);
tempRowCount++;
checkBatchSizeEnforcedMaximum(tempRowsSizeInBytes);
if ((long) rowBuffer.bufferedRowCount + tempRowCount >= Integer.MAX_VALUE) {
throw new SFException(ErrorCode.INTERNAL_ERROR, "Row count reaches MAX value");
}
}
checkBatchSizeRecommendedMaximum(tempRowsSizeInBytes);

moveTempRowsToActualBuffer(tempRowCount);

rowsSizeInBytes = tempRowsSizeInBytes;
if ((long) rowBuffer.bufferedRowCount + tempRowCount >= Integer.MAX_VALUE) {
throw new SFException(ErrorCode.INTERNAL_ERROR, "Row count reaches MAX value");
}
rowBuffer.bufferedRowCount += tempRowCount;
rowBuffer.statsMap.forEach(
(colName, stats) ->
Expand Down Expand Up @@ -241,25 +240,25 @@ public InsertValidationResponse insertRows(
response.addError(error);
}
checkBatchSizeEnforcedMaximum(tempRowsSizeInBytes);
if ((long) rowBuffer.bufferedRowCount + tempRowCount >= Integer.MAX_VALUE) {
throw new SFException(ErrorCode.INTERNAL_ERROR, "Row count reaches MAX value");
}
}

if (!response.hasErrors()) {
checkBatchSizeRecommendedMaximum(tempRowsSizeInBytes);
moveTempRowsToActualBuffer(tempRowCount);
rowsSizeInBytes = tempRowsSizeInBytes;
if ((long) rowBuffer.bufferedRowCount + tempRowCount >= Integer.MAX_VALUE) {
throw new SFException(ErrorCode.INTERNAL_ERROR, "Row count reaches MAX value");
}
rowBuffer.bufferedRowCount += tempRowCount;
rowBuffer.statsMap.forEach(
(colName, stats) ->
rowBuffer.statsMap.put(
colName,
RowBufferStats.getCombinedStats(stats, rowBuffer.tempStatsMap.get(colName))));
rowBuffer.channelState.setOffsetToken(offsetToken);
rowBuffer.bufferSize += rowsSizeInBytes;
rowBuffer.rowSizeMetric.accept(rowsSizeInBytes);
}
rowBuffer.bufferSize += rowsSizeInBytes;
rowBuffer.rowSizeMetric.accept(rowsSizeInBytes);
return response;
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -945,7 +945,6 @@ private void testE2ETimeHelper(OpenChannelRequest.OnErrorOption onErrorOption) {

@Test
public void testMaxInsertRowsBatchSize() {
testMaxInsertRowsBatchSizeHelper(OpenChannelRequest.OnErrorOption.CONTINUE);
testMaxInsertRowsBatchSizeHelper(OpenChannelRequest.OnErrorOption.ABORT);
testMaxInsertRowsBatchSizeHelper(OpenChannelRequest.OnErrorOption.SKIP_BATCH);
}
Expand Down
Loading