merge master

snowflakedb · Nov 2, 2023 · 97d7854 · 97d7854
2 parents b306c34 + d0fd1d8
commit 97d7854
Show file tree

Hide file tree

Showing 23 changed files with 802 additions and 146 deletions.
diff --git a/.github/workflows/snyk-issue.yml b/.github/workflows/snyk-issue.yml
@@ -4,6 +4,11 @@ on:
   schedule:
     - cron: '* */12 * * *'
 
+permissions:
+  contents: read
+  issues: write
+  pull-requests: write
+
 concurrency: snyk-issue
 
 jobs:

diff --git a/.github/workflows/snyk-pr.yml b/.github/workflows/snyk-pr.yml
@@ -3,6 +3,12 @@ on:
   pull_request:
     branches:
       - master
+
+permissions:
+  contents: read
+  issues: write
+  pull-requests: write
+
 jobs:
   snyk:
     runs-on: ubuntu-latest

diff --git a/e2e-jar-test/pom.xml b/e2e-jar-test/pom.xml
@@ -1,59 +1,65 @@
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-  <modelVersion>4.0.0</modelVersion>
-
-  <groupId>net.snowflake.snowflake-ingest-java-e2e-jar-test</groupId>
-  <artifactId>parent</artifactId>
-  <version>1.0-SNAPSHOT</version>
-  <packaging>pom</packaging>
-
-  <name>snowflake-ingest-sdk-e2e-test</name>
-
-  <modules>
-    <module>standard</module>
-    <module>fips</module>
-    <module>core</module>
-  </modules>
-
-  <dependencyManagement>
-
-    <dependencies>
-      <dependency>
-        <groupId>net.snowflake.snowflake-ingest-java-e2e-jar-test</groupId>
-        <artifactId>core</artifactId>
-        <version>${project.version}</version>
-      </dependency>
-
-      <dependency>
-        <groupId>net.snowflake</groupId>
-        <artifactId>snowflake-ingest-sdk</artifactId>
-        <version>2.0.4-SNAPSHOT</version>
-      </dependency>
-
-      <dependency>
-        <groupId>net.snowflake</groupId>
-        <artifactId>snowflake-jdbc-fips</artifactId>
-        <version>3.13.30</version>
-      </dependency>
-
-      <dependency>
-        <groupId>org.slf4j</groupId>
-        <artifactId>slf4j-simple</artifactId>
-        <version>1.7.36</version>
-      </dependency>
-
-      <dependency>
-        <groupId>com.fasterxml.jackson.core</groupId>
-        <artifactId>jackson-databind</artifactId>
-        <version>2.14.0</version>
-      </dependency>
-
-      <dependency>
-        <groupId>junit</groupId>
-        <artifactId>junit</artifactId>
-        <version>4.13.2</version>
-        <scope>test</scope>
-      </dependency>
-    </dependencies>
-  </dependencyManagement>
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>net.snowflake.snowflake-ingest-java-e2e-jar-test</groupId>
+    <artifactId>parent</artifactId>
+    <version>1.0-SNAPSHOT</version>
+    <packaging>pom</packaging>
+
+    <name>snowflake-ingest-sdk-e2e-test</name>
+
+    <modules>
+        <module>standard</module>
+        <module>fips</module>
+        <module>core</module>
+    </modules>
+
+    <dependencyManagement>
+
+        <dependencies>
+            <dependency>
+                <groupId>net.snowflake.snowflake-ingest-java-e2e-jar-test</groupId>
+                <artifactId>core</artifactId>
+                <version>${project.version}</version>
+            </dependency>
+
+            <dependency>
+                <groupId>net.snowflake</groupId>
+                <artifactId>snowflake-ingest-sdk</artifactId>
+                <version>2.0.4</version>
+            </dependency>
+
+            <dependency>
+                <groupId>net.snowflake</groupId>
+                <artifactId>snowflake-jdbc-fips</artifactId>
+                <version>3.13.30</version>
+            </dependency>
+
+            <dependency>
+                <groupId>org.bouncycastle</groupId>
+                <artifactId>bc-fips</artifactId>
+                <version>1.0.2.4</version>
+            </dependency>
+
+            <dependency>
+                <groupId>org.slf4j</groupId>
+                <artifactId>slf4j-simple</artifactId>
+                <version>1.7.36</version>
+            </dependency>
+
+            <dependency>
+                <groupId>com.fasterxml.jackson.core</groupId>
+                <artifactId>jackson-databind</artifactId>
+                <version>2.14.0</version>
+            </dependency>
+
+            <dependency>
+                <groupId>junit</groupId>
+                <artifactId>junit</artifactId>
+                <version>4.13.2</version>
+                <scope>test</scope>
+            </dependency>
+        </dependencies>
+    </dependencyManagement>
 </project>
diff --git a/format.sh b/format.sh
@@ -19,3 +19,6 @@ if ! command -v java > /dev/null; then
 fi
 echo "Running Google Java Format"
 find ./src -type f -name "*.java" -print0 | xargs -0 java -jar "${JAR_FILE}" --replace --set-exit-if-changed && echo "OK"
+
+echo "Sorting pom.xml"
+mvn com.github.ekryd.sortpom:sortpom-maven-plugin:sort
diff --git a/pom.xml b/pom.xml
@@ -6,7 +6,7 @@
   <!-- Arifact name and version information -->
   <groupId>net.snowflake</groupId>
   <artifactId>snowflake-ingest-sdk</artifactId>
-  <version>2.0.4-SNAPSHOT</version>
+  <version>2.0.4</version>
   <packaging>jar</packaging>
   <name>Snowflake Ingest SDK</name>
   <description>Snowflake Ingest SDK</description>
@@ -473,6 +473,17 @@
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-api</artifactId>
     </dependency>
+    <dependency>
+      <groupId>com.google.protobuf</groupId>
+      <artifactId>protobuf-java</artifactId>
+      <scope>runtime</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.xerial.snappy</groupId>
+      <artifactId>snappy-java</artifactId>
+      <scope>runtime</scope>
+    </dependency>
+
     <!-- JUnit so that we can make some basic unit tests -->
     <dependency>
       <groupId>junit</groupId>
@@ -768,9 +779,9 @@
         <configuration>
           <errorRemedy>failFast</errorRemedy>
           <!--
-          The list of allowed licenses. If you see the build failing due to "There are some forbidden licenses used, please
-          check your dependencies", verify the conditions of the license and add the reference to it here.
-          -->
+                    The list of allowed licenses. If you see the build failing due to "There are some forbidden licenses used, please
+                    check your dependencies", verify the conditions of the license and add the reference to it here.
+                    -->
           <includedLicenses>
             <includedLicense>Apache License 2.0</includedLicense>
             <includedLicense>BSD 2-Clause License</includedLicense>
@@ -888,9 +899,9 @@
       <build>
         <plugins>
           <!--
-          Copy all project dependencies to target/dependency-jars. License processing Python script will look here for
-          license files of SDK dependencies.
-          -->
+                    Copy all project dependencies to target/dependency-jars. License processing Python script will look here for
+                    license files of SDK dependencies.
+                    -->
           <plugin>
             <groupId>org.apache.maven.plugins</groupId>
             <artifactId>maven-dependency-plugin</artifactId>
@@ -912,9 +923,9 @@
           </plugin>
 
           <!--
-          Compile the list of SDK dependencies in 'compile' and 'runtime' scopes.
-          This list is an entry point for the license processing python script.
-          -->
+                    Compile the list of SDK dependencies in 'compile' and 'runtime' scopes.
+                    This list is an entry point for the license processing python script.
+                    -->
           <plugin>
             <groupId>org.apache.maven.plugins</groupId>
             <artifactId>maven-dependency-plugin</artifactId>
@@ -1097,9 +1108,9 @@
             </executions>
           </plugin>
           <!--
-          Plugin executes license processing Python script, which copies third party license files into the directory
-          target/generated-licenses-info/META-INF/third-party-licenses, which is then included in the shaded JAR. 
-          -->
+                    Plugin executes license processing Python script, which copies third party license files into the directory
+                    target/generated-licenses-info/META-INF/third-party-licenses, which is then included in the shaded JAR.
+                    -->
           <plugin>
             <groupId>org.codehaus.mojo</groupId>
             <artifactId>exec-maven-plugin</artifactId>

diff --git a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
@@ -110,7 +110,7 @@ public class RequestBuilder {
   // Don't change!
   public static final String CLIENT_NAME = "SnowpipeJavaSDK";
 
-  public static final String DEFAULT_VERSION = "2.0.4-SNAPSHOT";
+  public static final String DEFAULT_VERSION = "2.0.4";
 
   public static final String JAVA_USER_AGENT = "JAVA";
 
@@ -351,6 +351,7 @@ private static String getDefaultUserAgent() {
   private static String buildCustomUserAgent(String additionalUserAgentInfo) {
     return USER_AGENT.trim() + " " + additionalUserAgentInfo;
   }
+
   /** A simple POJO for generating our POST body to the insert endpoint */
   private static class IngestRequest {
     // the list of files we're loading

diff --git a/src/main/java/net/snowflake/ingest/streaming/OpenChannelRequest.java b/src/main/java/net/snowflake/ingest/streaming/OpenChannelRequest.java
@@ -40,6 +40,9 @@ public enum OnErrorOption {
   // Default timezone for TIMESTAMP_LTZ and TIMESTAMP_TZ columns
   private final ZoneId defaultTimezone;
 
+  private final String offsetToken;
+  private final boolean isOffsetTokenProvided;
+
   public static OpenChannelRequestBuilder builder(String channelName) {
     return new OpenChannelRequestBuilder(channelName);
   }
@@ -53,6 +56,9 @@ public static class OpenChannelRequestBuilder {
     private OnErrorOption onErrorOption;
     private ZoneId defaultTimezone;
 
+    private String offsetToken;
+    private boolean isOffsetTokenProvided = false;
+
     public OpenChannelRequestBuilder(String channelName) {
       this.channelName = channelName;
       this.defaultTimezone = DEFAULT_DEFAULT_TIMEZONE;
@@ -83,6 +89,12 @@ public OpenChannelRequestBuilder setDefaultTimezone(ZoneId defaultTimezone) {
       return this;
     }
 
+    public OpenChannelRequestBuilder setOffsetToken(String offsetToken){
+      this.offsetToken = offsetToken;
+      this.isOffsetTokenProvided = true;
+      return  this;
+    }
+
     public OpenChannelRequest build() {
       return new OpenChannelRequest(this);
     }
@@ -102,6 +114,8 @@ private OpenChannelRequest(OpenChannelRequestBuilder builder) {
     this.tableName = builder.tableName;
     this.onErrorOption = builder.onErrorOption;
     this.defaultTimezone = builder.defaultTimezone;
+    this.offsetToken = builder.offsetToken;
+    this.isOffsetTokenProvided = builder.isOffsetTokenProvided;
   }
 
   public String getDBName() {
@@ -131,4 +145,12 @@ public String getFullyQualifiedTableName() {
   public OnErrorOption getOnErrorOption() {
     return this.onErrorOption;
   }
+
+  public String getOffsetToken() {
+    return this.offsetToken;
+  }
+
+  public boolean isOffsetTokenProvided() {
+    return this.isOffsetTokenProvided;
+  }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
@@ -223,13 +223,14 @@ public InsertValidationResponse insertRows(
       float rowsSizeInBytes = 0F;
       float tempRowsSizeInBytes = 0F;
       int tempRowCount = 0;
+      int rowIndex = 0;
       for (Map<String, Object> row : rows) {
         InsertValidationResponse.InsertError error =
-            new InsertValidationResponse.InsertError(row, tempRowCount);
+            new InsertValidationResponse.InsertError(row, rowIndex);
         try {
-          Set<String> inputColumnNames = verifyInputColumns(row, error, tempRowCount);
+          Set<String> inputColumnNames = verifyInputColumns(row, error, rowIndex);
           tempRowsSizeInBytes +=
-              addTempRow(row, tempRowCount, rowBuffer.tempStatsMap, inputColumnNames, tempRowCount);
+              addTempRow(row, tempRowCount, rowBuffer.tempStatsMap, inputColumnNames, rowIndex);
           tempRowCount++;
         } catch (SFException e) {
           error.setException(e);
@@ -239,8 +240,9 @@ public InsertValidationResponse insertRows(
           error.setException(new SFException(e, ErrorCode.INTERNAL_ERROR, e.getMessage()));
           response.addError(error);
         }
+        rowIndex++;
         checkBatchSizeEnforcedMaximum(tempRowsSizeInBytes);
-        if ((long) rowBuffer.bufferedRowCount + tempRowCount >= Integer.MAX_VALUE) {
+        if ((long) rowBuffer.bufferedRowCount + rowIndex >= Integer.MAX_VALUE) {
           throw new SFException(ErrorCode.INTERNAL_ERROR, "Row count reaches MAX value");
         }
       }