DO NOT MERGE: Make binary string encoding configurable, add support for base64

e2e-jar-test/pom.xml

@@ -29,7 +29,7 @@
       <dependency>
         <groupId>net.snowflake</groupId>
         <artifactId>snowflake-ingest-sdk</artifactId>
-        <version>2.1.2-SNAPSHOT</version>
+        <version>2.2.0</version>
       </dependency>
 
       <dependency>

pom.xml

@@ -6,7 +6,7 @@
   <!-- Arifact name and version information -->
   <groupId>net.snowflake</groupId>
   <artifactId>snowflake-ingest-sdk</artifactId>
-  <version>2.1.2-SNAPSHOT</version>
+  <version>2.2.0</version>
   <packaging>jar</packaging>
   <name>Snowflake Ingest SDK</name>
   <description>Snowflake Ingest SDK</description>
@@ -45,7 +45,7 @@
     <commonslang3.version>3.14.0</commonslang3.version>
     <commonslogging.version>1.3.1</commonslogging.version>
     <commonstext.version>1.11.0</commonstext.version>
-    <fasterxml.version>2.16.1</fasterxml.version>
+    <fasterxml.version>2.17.0</fasterxml.version>
     <guava.version>32.0.1-jre</guava.version>
     <hadoop.version>3.3.6</hadoop.version>
     <jacoco.skip.instrument>true</jacoco.skip.instrument>
@@ -60,13 +60,13 @@
     <netty.version>4.1.94.Final</netty.version>
     <nimbusds.version>9.37.3</nimbusds.version>
     <objenesis.version>3.1</objenesis.version>
-    <parquet.version>1.13.1</parquet.version>
+    <parquet.version>1.14.1</parquet.version>
     <powermock.version>2.0.9</powermock.version>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-    <protobuf.version>3.19.6</protobuf.version>
+    <protobuf.version>4.27.2</protobuf.version>
     <shadeBase>net.snowflake.ingest.internal</shadeBase>
     <slf4j.version>1.7.36</slf4j.version>
-    <snappy.version>1.1.10.4</snappy.version>
+    <snappy.version>1.1.10.5</snappy.version>
     <snowjdbc.version>3.16.1</snowjdbc.version>
     <yetus.version>0.13.0</yetus.version>
   </properties>
@@ -343,13 +343,13 @@
       <dependency>
         <groupId>net.bytebuddy</groupId>
         <artifactId>byte-buddy</artifactId>
-        <version>1.10.19</version>
+        <version>1.14.9</version>
         <scope>test</scope>
       </dependency>
       <dependency>
         <groupId>net.bytebuddy</groupId>
         <artifactId>byte-buddy-agent</artifactId>
-        <version>1.10.19</version>
+        <version>1.14.9</version>
         <scope>test</scope>
       </dependency>
       <dependency>
@@ -358,6 +358,18 @@
         <version>3.7.7</version>
         <scope>test</scope>
       </dependency>
+      <dependency>
+        <groupId>org.openjdk.jmh</groupId>
+        <artifactId>jmh-core</artifactId>
+        <version>1.34</version>
+        <scope>test</scope>
+      </dependency>
+      <dependency>
+        <groupId>org.openjdk.jmh</groupId>
+        <artifactId>jmh-generator-annprocess</artifactId>
+        <version>1.34</version>
+        <scope>test</scope>
+      </dependency>
     </dependencies>
   </dependencyManagement>
 
@@ -470,6 +482,13 @@
     <dependency>
       <groupId>org.apache.parquet</groupId>
       <artifactId>parquet-common</artifactId>
+      <exclusions>
+        <!-- Dependencies are excluded because CDDL + GPLv2 with classpath exception license is not approved -->
+        <exclusion>
+          <groupId>javax.annotation</groupId>
+          <artifactId>javax.annotation-api</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.apache.parquet</groupId>
@@ -491,7 +510,7 @@
     <dependency>
       <groupId>com.github.luben</groupId>
       <artifactId>zstd-jni</artifactId>
-      <version>1.5.0-1</version>
+      <version>1.5.6-2</version>
       <scope>runtime</scope>
     </dependency>
     <dependency>
@@ -527,6 +546,16 @@
       <artifactId>mockito-core</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.openjdk.jmh</groupId>
+      <artifactId>jmh-core</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.openjdk.jmh</groupId>
+      <artifactId>jmh-generator-annprocess</artifactId>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>org.powermock</groupId>
       <artifactId>powermock-api-mockito2</artifactId>
@@ -723,8 +752,8 @@
               <ignoreNonCompile>true</ignoreNonCompile>
               <ignoredDependencies>
                 <!-- We defined these as direct dependencies (as opposed to just declaring it in dependencyManagement)
-                to workaround https://issues.apache.org/jira/browse/MNG-7982. Now the dependency analyzer complains that
-                the dependency is unused, so we ignore it here-->
+                                                                                                                                                                                                to workaround https://issues.apache.org/jira/browse/MNG-7982. Now the dependency analyzer complains that
+                                                                                                                                                                                                the dependency is unused, so we ignore it here-->
                 <ignoredDependency>org.apache.commons:commons-compress</ignoredDependency>
                 <ignoredDependency>org.apache.commons:commons-configuration2</ignoredDependency>
               </ignoredDependencies>
@@ -819,9 +848,9 @@
         <configuration>
           <errorRemedy>failFast</errorRemedy>
           <!--
-                                                   The list of allowed licenses. If you see the build failing due to "There are some forbidden licenses used, please
-                                                  check your dependencies", verify the conditions of the license and add the reference to it here.
-                                                -->
+                                                                                                                                                                 The list of allowed licenses. If you see the build failing due to "There are some forbidden licenses used, please
+                                                                                                                                                                check your dependencies", verify the conditions of the license and add the reference to it here.
+                                                                                                                                                              -->
           <includedLicenses>
             <includedLicense>Apache License 2.0</includedLicense>
             <includedLicense>BSD 2-Clause License</includedLicense>
@@ -844,6 +873,7 @@
             <licenseMerge>BSD 2-Clause License
                             |The BSD License</licenseMerge>
             <licenseMerge>The MIT License|MIT License</licenseMerge>
+            <licenseMerge>3-Clause BSD License|BSD-3-Clause</licenseMerge>
           </licenseMerges>
         </configuration>
         <executions>
@@ -1133,9 +1163,9 @@
             </executions>
           </plugin>
           <!--
-                                                            Plugin executes license processing Python script, which copies third party license files into the directory
-                                                            target/generated-licenses-info/META-INF/third-party-licenses, which is then included in the shaded JAR.
-                                                            -->
+                                                                                                                                                                          Plugin executes license processing Python script, which copies third party license files into the directory
+                                                                                                                                                                          target/generated-licenses-info/META-INF/third-party-licenses, which is then included in the shaded JAR.
+                                                                                                                                                                          -->
           <plugin>
             <groupId>org.codehaus.mojo</groupId>
             <artifactId>exec-maven-plugin</artifactId>

scripts/process_licenses.py

@@ -132,18 +132,22 @@ def main():
                     dependency_without_license_count += 1
                     missing_licenses_str += f"{dependency_lookup_key}: {license_name}\n"
                 else:
-                    raise Exception(f"The dependency {dependency_lookup_key} does not ship a license file, but neither is it not defined in ADDITIONAL_LICENSES_MAP")
+                    raise Exception(
+                        f"The dependency {dependency_lookup_key} does not ship a license file, but neither is it not "
+                        f"defined in ADDITIONAL_LICENSES_MAP")
 
     with open(Path(target_dir, "ADDITIONAL_LICENCES"), "w") as additional_licenses_handle:
         additional_licenses_handle.write(missing_licenses_str)
 
     if dependency_count < 30:
-        raise Exception(f"Suspiciously low number of dependency JARs detected in {dependency_jars_path}: {dependency_count}")
+        raise Exception(
+            f"Suspiciously low number of dependency JARs detected in {dependency_jars_path}: {dependency_count}")
     print("License generation finished")
     print(f"\tTotal dependencies: {dependency_count}")
     print(f"\tTotal dependencies (with license): {dependency_with_license_count}")
     print(f"\tTotal dependencies (without license): {dependency_without_license_count}")
     print(f"\tIgnored dependencies: {dependency_ignored_count}")
 
+
 if __name__ == "__main__":
     main()

src/main/java/net/snowflake/ingest/connection/OAuthClient.java

@@ -93,6 +93,7 @@ public void refreshToken() throws IOException {
 
   /** Helper method for making refresh request */
   private HttpUriRequest makeRefreshTokenRequest() {
+    // TODO SNOW-1538108 Use SnowflakeServiceClient to make the request
     HttpPost post = new HttpPost(oAuthCredential.get().getOAuthTokenEndpoint());
     post.addHeader(HttpHeaders.CONTENT_TYPE, OAUTH_CONTENT_TYPE_HEADER);
     post.addHeader(HttpHeaders.AUTHORIZATION, oAuthCredential.get().getAuthHeader());

src/main/java/net/snowflake/ingest/connection/RequestBuilder.java

@@ -110,7 +110,7 @@ public class RequestBuilder {
   // Don't change!
   public static final String CLIENT_NAME = "SnowpipeJavaSDK";
 
-  public static final String DEFAULT_VERSION = "2.1.2-SNAPSHOT";
+  public static final String DEFAULT_VERSION = "2.2.0";
 
   public static final String JAVA_USER_AGENT = "JAVA";
 
@@ -678,12 +678,23 @@ public HttpGet generateHistoryRangeRequest(
    */
   public HttpPost generateStreamingIngestPostRequest(
       String payload, String endPoint, String message) {
-    LOGGER.debug("Generate Snowpipe streaming request: endpoint={}, payload={}", endPoint, payload);
+    final String requestId = UUID.randomUUID().toString();
+    LOGGER.debug(
+        "Generate Snowpipe streaming request: endpoint={}, payload={}, requestId={}",
+        endPoint,
+        payload,
+        requestId);
     // Make the corresponding URI
     URI uri = null;
     try {
       uri =
-          new URIBuilder().setScheme(scheme).setHost(host).setPort(port).setPath(endPoint).build();
+          new URIBuilder()
+              .setScheme(scheme)
+              .setHost(host)
+              .setPort(port)
+              .setPath(endPoint)
+              .setParameter(REQUEST_ID, requestId)
+              .build();
     } catch (URISyntaxException e) {
       throw new SFException(e, ErrorCode.BUILD_REQUEST_FAILURE, message);
     }

src/main/java/net/snowflake/ingest/streaming/OpenChannelRequest.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021 Snowflake Computing Inc. All rights reserved.
+ * Copyright (c) 2021-2024 Snowflake Computing Inc. All rights reserved.
  */
 
 package net.snowflake.ingest.streaming;
@@ -150,7 +150,7 @@ public ZoneId getDefaultTimezone() {
   }
 
   public String getFullyQualifiedTableName() {
-    return String.format("%s.%s.%s", this.dbName, this.schemaName, this.tableName);
+    return Utils.getFullyQualifiedTableName(this.dbName, this.schemaName, this.tableName);
   }
 
   public OnErrorOption getOnErrorOption() {

src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java

@@ -16,7 +16,6 @@
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
 import java.util.function.Consumer;
-import java.util.stream.Collectors;
 import net.snowflake.ingest.connection.TelemetryService;
 import net.snowflake.ingest.streaming.InsertValidationResponse;
 import net.snowflake.ingest.streaming.OffsetTokenVerificationFunction;
@@ -400,10 +399,10 @@ public float getSize() {
   Set<String> verifyInputColumns(
       Map<String, Object> row, InsertValidationResponse.InsertError error, int rowIndex) {
     // Map of unquoted column name -> original column name
-    Map<String, String> inputColNamesMap =
-        row.keySet().stream()
-            .collect(Collectors.toMap(LiteralQuoteUtils::unquoteColumnName, value -> value));
-
+    Set<String> originalKeys = row.keySet();
+    Map<String, String> inputColNamesMap = new HashMap<>();
+    originalKeys.forEach(
+        key -> inputColNamesMap.put(LiteralQuoteUtils.unquoteColumnName(key), key));
     // Check for extra columns in the row
     List<String> extraCols = new ArrayList<>();
     for (String columnName : inputColNamesMap.keySet()) {