SNOW-1820372: (from 947) when proxy is set in Connection, driver does send traffic through the proxy to S3, but not to Azure blob / GCS bucket (only Snowflake). Works with proxy envvar (GCP)

index.d.ts

@@ -512,6 +512,11 @@ declare module 'snowflake-sdk' {
          *  The option to pass passcode from DUO.
          */
         passcode?: string;
+
+        /**
+         *  The option to override the environment proxy value (HTTPS_PROXY) if the connection proxy is configured in GCS
+         */
+        overrideEnvProxy?: string;
     }
 
     export interface Connection {

lib/connection/connection_config.js

@@ -67,6 +67,7 @@ const DEFAULT_PARAMS =
   'credentialCacheDir',
   'passcodeInPassword',
   'passcode',
+  'overrideEnvProxy'
 ];
 
 function consolidateHostAndAccount(options) {
@@ -514,6 +515,13 @@ function ConnectionConfig(options, validateCredentials, qaMode, clientInfo) {
     passcode = options.passcode;
   }
 
+  let overrideEnvProxy = true;
+  if (Util.exists(options.overrideEnvProxy)) {
+    Errors.checkArgumentValid(Util.isBoolean(options.overrideEnvProxy), 
+      ErrorCodes.ERR_CONN_CREATE_INVALID_OVERRIDE_ENV_PROXY);
+
+    overrideEnvProxy = options.overrideEnvProxy;
+  } 
 
   if (validateDefaultParameters) {
     for (const [key] of Object.entries(options)) {
@@ -837,6 +845,10 @@ function ConnectionConfig(options, validateCredentials, qaMode, clientInfo) {
     return passcode;
   };
 
+  this.getOverrideEnvProxy = function () {
+    return overrideEnvProxy;
+  };
+
   /**
    * Returns attributes of Connection Config object that can be used to identify
    * the connection, when ID is not available in the scope. This is not sufficient set,

lib/constants/error_messages.js

@@ -84,6 +84,7 @@ exports[404053] = 'A host must be specified.';
 exports[404054] = 'Invalid host. The specified value must be a string.';
 exports[404055] = 'Invalid passcodeInPassword. The specified value must be a boolean';
 exports[404056] = 'Invalid passcode. The specified value must be a string';
+exports[404057] = 'Invalid overrideEnvProxy. The specified value must be a boolean';
 
 // 405001
 exports[405001] = 'Invalid callback. The specified value must be a function.';

lib/errors.js

@@ -88,6 +88,7 @@ codes.ERR_CONN_CREATE_MISSING_HOST = 404053;
 codes.ERR_CONN_CREATE_INVALID_HOST = 404054;
 codes.ERR_CONN_CREATE_INVALID_PASSCODE_IN_PASSWORD = 404055;
 codes.ERR_CONN_CREATE_INVALID_PASSCODE = 404056;
+codes.ERR_CONN_CREATE_INVALID_OVERRIDE_ENV_PROXY = 404057;
 
 // 405001
 codes.ERR_CONN_CONNECT_INVALID_CALLBACK = 405001;

lib/file_transfer_agent/gcs_util.js

@@ -4,6 +4,10 @@
 
 const EncryptionMetadata = require('./encrypt_util').EncryptionMetadata;
 const FileHeader = require('./file_util').FileHeader;
+const getProxyAgent = require('../http/node').getProxyAgent;
+const ProxyUtil = require('../proxy_util');
+const Util = require('../util');
+const Logger = require('../logger');
 const { shouldPerformGCPBucket } = require('../util');
 
 const GCS_METADATA_PREFIX = 'x-goog-meta-';
@@ -19,8 +23,10 @@
 
 const HTTP_HEADER_CONTENT_ENCODING = 'Content-Encoding';
 const resultStatus = require('./file_util').resultStatus;
+const HTTPS_PROXY = 'HTTPS_PROXY';
 
 const { Storage } = require('@google-cloud/storage');
+const { isBypassProxy } = require('../http/node');
 
 const EXPIRED_TOKEN = 'ExpiredToken';
 
@@ -37,15 +43,19 @@
 /**
  * Creates an GCS utility object.
  *
- * @param {module} httpclient
- * @param {module} filestream
+ * @param {module} httpClient
+ * @param {module} fileStream
  *
  * @returns {Object}
  * @constructor
  */
-function GCSUtil(httpclient, filestream) {
-  const axios = typeof httpclient !== 'undefined' ? httpclient : require('axios');
-  const fs = typeof filestream !== 'undefined' ? filestream : require('fs');
+function GCSUtil(connectionConfig, httpClient, fileStream) {
+  let axios = httpClient;
+  const fs = typeof fileStream !== 'undefined' ? fileStream : require('fs');
+  const connectionProxy = connectionConfig.getProxy();
+  const originalHttpsProxy = process.env[HTTPS_PROXY];
+  const proxyString = ProxyUtil.stringifyProxy(connectionProxy);
+  let isEnvProxyOverridden = false;
 
   /**
   * Retrieve the GCS token from the stage info metadata.
@@ -57,7 +67,15 @@
   this.createClient = function (stageInfo) {
     const stageCredentials = stageInfo['creds'];
     const gcsToken = stageCredentials['GCS_ACCESS_TOKEN'];
-
+    //TODO: SNOW-1789759 the value is hardcoded now, but it should be server driven
+    const isRegionalUrlEnabled = (stageInfo.region).toLowerCase() === 'me-central2' || stageInfo.useRegionalUrl;
+    let endPoint = null;
+    if (stageInfo['endPoint']) {
+      endPoint = stageInfo['endPoint'];
+    } else if (isRegionalUrlEnabled) {
+      endPoint = `storage.${stageInfo.region.toLowerCase()}.rep.googleapis.com`;
+    }
+
     let client;
     if (gcsToken) {
       const interceptors = [];
@@ -68,21 +86,38 @@
           return requestConfig;
         }
       });
-
-      //TODO: SNOW-1789759 hardcoded region will be replaced in the future
-      const isRegionalUrlEnabled = (stageInfo.region).toLowerCase() === 'me-central2' || stageInfo.useRegionalUrl;
-      let endPoint = null;
-      if (stageInfo['endPoint']) {
-        endPoint = stageInfo['endPoint'];
-      } else if (isRegionalUrlEnabled) {
-        endPoint = `storage.${stageInfo.region.toLowerCase()}.rep.googleapis.com`;
-      }
-      const storage = endPoint ? new Storage({ interceptors_: interceptors, apiEndpoint: endPoint }) : new Storage({ interceptors_: interceptors });
+      const storage = Util.exists(endPoint) ? new Storage({ interceptors_: interceptors, apiEndpoint: endPoint }) : new Storage({ interceptors_: interceptors });
       client = { gcsToken: gcsToken, gcsClient: storage };
     } else {
       client = null;
     }
 
+    const isByPassed = isBypassProxy(connectionProxy, endPoint || `storage.${stageInfo.region.toLowerCase()}.rep.googleapis.com`);
+    if (Util.isEmptyObject(connectionProxy) || isByPassed) {
+      isEnvProxyOverridden = false;
+    } else if (!Util.exists(originalHttpsProxy)) {
+      isEnvProxyOverridden = true;
+    } else if (connectionConfig.getOverrideEnvProxy()) {
+      if (proxyString !==  originalHttpsProxy) {
+        Logger.getInstance().debug(`The destination host is: ${ProxyUtil.getHostFromURL(endPoint || `https://storage.${stageInfo.region.toLowerCase()}.rep.googleapis.com`)} and the proxy host is: ${connectionProxy.host}`);
+        Logger.getInstance().warn(`The environment variable ${HTTPS_PROXY} value will be overwritten during the Storage HTTP calls`);
+        isEnvProxyOverridden = true;
+      }
+    }
+
+    if (isEnvProxyOverridden) {
+      Logger.getInstance().trace(`Initializing the proxy information for the GCS Client: ${ProxyUtil.describeProxy(connectionProxy)}`);
+    }
+
+    if (typeof httpClient === 'undefined') {
+      const proxyAgent = getProxyAgent(connectionProxy, new URL(connectionConfig.accessUrl), endPoint || `storage.${stageInfo.region.toLowerCase()}.rep.googleapis.com` );
+      axios = require('axios').create({
+        proxy: false,
+        httpAgent: proxyAgent,
+        httpsAgent: proxyAgent,
+      });
+    }
+
     return client;
   };
 
@@ -148,10 +183,20 @@
           if (shouldPerformGCPBucket(accessToken)) {
             const gcsLocation = this.extractBucketNameAndPath(meta['stageInfo']['location']);
 
-            const metadata = await meta['client'].gcsClient
-              .bucket(gcsLocation.bucketName)
-              .file(gcsLocation.path + filename)
-              .getMetadata();
+            let metadata;
+            if (isEnvProxyOverridden) {
+              process.env[HTTPS_PROXY] = proxyString;
+              metadata = await meta['client'].gcsClient
+                .bucket(gcsLocation.bucketName)
+                .file(gcsLocation.path + filename)
+                .getMetadata();
+              Util.restoreEnvVar(HTTPS_PROXY, originalHttpsProxy);
+            } else {
+              metadata = await meta['client'].gcsClient
+                .bucket(gcsLocation.bucketName)
+                .file(gcsLocation.path + filename)
+                .getMetadata();
+            }
 
             digest = metadata[0].metadata[SFC_DIGEST];
             contentLength = metadata[0].size;
@@ -287,20 +332,37 @@
     try {
       if (shouldPerformGCPBucket(accessToken)) {
         const gcsLocation = this.extractBucketNameAndPath(meta['stageInfo']['location']);
-
-        await meta['client'].gcsClient
-          .bucket(gcsLocation.bucketName)
-          .file(gcsLocation.path + meta['dstFileName'])
-          .save(fileStream, {
-            resumable: false,
-            metadata: {
+        if (isEnvProxyOverridden){
+          process.env[HTTPS_PROXY] = proxyString;
+          await meta['client'].gcsClient
+            .bucket(gcsLocation.bucketName)
+            .file(gcsLocation.path + meta['dstFileName'])
+            .save(fileStream, {
+              resumable: false,
               metadata: {
-                [ENCRYPTIONDATAPROP]: gcsHeaders[GCS_METADATA_ENCRYPTIONDATAPROP],
-                [MATDESC_KEY]: gcsHeaders[GCS_METADATA_MATDESC_KEY],
-                [SFC_DIGEST]: gcsHeaders[GCS_METADATA_SFC_DIGEST]
+                metadata: {
+                  [ENCRYPTIONDATAPROP]: gcsHeaders[GCS_METADATA_ENCRYPTIONDATAPROP],
+                  [MATDESC_KEY]: gcsHeaders[GCS_METADATA_MATDESC_KEY],
+                  [SFC_DIGEST]: gcsHeaders[GCS_METADATA_SFC_DIGEST]
+                }
               }
-            }
-          });
+            });
+          Util.restoreEnvVar(HTTPS_PROXY, originalHttpsProxy);
+        } else {
+          await meta['client'].gcsClient
+            .bucket(gcsLocation.bucketName)
+            .file(gcsLocation.path + meta['dstFileName'])
+            .save(fileStream, {
+              resumable: false,
+              metadata: {
+                metadata: {
+                  [ENCRYPTIONDATAPROP]: gcsHeaders[GCS_METADATA_ENCRYPTIONDATAPROP],
+                  [MATDESC_KEY]: gcsHeaders[GCS_METADATA_MATDESC_KEY],
+                  [SFC_DIGEST]: gcsHeaders[GCS_METADATA_SFC_DIGEST]
+                }
+              }
+            });
+        }
       } else {
         // Set maxBodyLength to allow large file uploading
         await axios.put(uploadUrl, fileStream, { maxBodyLength: Infinity, headers: gcsHeaders });
@@ -360,28 +422,41 @@
     try {
       if (shouldPerformGCPBucket(accessToken)) {
         const gcsLocation = this.extractBucketNameAndPath(meta['stageInfo']['location']);
+        let metadata;
+        if (isEnvProxyOverridden){
+          process.env[HTTPS_PROXY] = proxyString;
+          await meta['client'].gcsClient
+            .bucket(gcsLocation.bucketName)
+            .file(gcsLocation.path + meta['srcFileName'])
+            .download({
+              destination: fullDstPath
+            });
 
-        await meta['client'].gcsClient
-          .bucket(gcsLocation.bucketName)
-          .file(gcsLocation.path + meta['srcFileName'])
-          .download({
-            destination: fullDstPath
-          });
-
-        const metadata = await meta['client'].gcsClient
-          .bucket(gcsLocation.bucketName)
-          .file(gcsLocation.path + meta['srcFileName'])
-          .getMetadata();
+          metadata = await meta['client'].gcsClient
+            .bucket(gcsLocation.bucketName)
+            .file(gcsLocation.path + meta['srcFileName'])
+            .getMetadata();
+          Util.restoreEnvVar(HTTPS_PROXY, originalHttpsProxy);
+        } else {
+          await meta['client'].gcsClient
+            .bucket(gcsLocation.bucketName)
+            .file(gcsLocation.path + meta['srcFileName'])
+            .download({
+              destination: fullDstPath
+            });
 
+          metadata = await meta['client'].gcsClient
+            .bucket(gcsLocation.bucketName)
+            .file(gcsLocation.path + meta['srcFileName'])
+            .getMetadata();
+        }
         encryptionDataprop = metadata[0].metadata[ENCRYPTIONDATAPROP];
         matDescKey = metadata[0].metadata[MATDESC_KEY];
         sfcDigest = metadata[0].metadata[SFC_DIGEST];
         size = metadata[0].size;
       } else {
         let response;
-        await axios({
-          method: 'get',
-          url: downloadUrl,
+        await axios.get(downloadUrl, {
           headers: gcsHeaders,
           responseType: 'stream'
         }).then(async (res) => {
@@ -457,6 +532,11 @@
     return link;
   };
 
+  //getIsEnvProxyOverridden are for testing purpose
+  this.getIsEnvProxyOverridden = function () {
+    return isEnvProxyOverridden;
+  };
+
   /**
 * Left strip the specified character from a string.
 *

lib/file_transfer_agent/remote_storage_util.js

@@ -33,6 +33,8 @@ exports.SnowflakeFileEncryptionMaterial = SnowflakeFileEncryptionMaterial;
  * @constructor
  */
 function RemoteStorageUtil(connectionConfig) {
+  let client = null;
+
   /**
   * Get storage type based on location type.
   *
@@ -41,15 +43,17 @@ function RemoteStorageUtil(connectionConfig) {
   * @returns {Object}
   */
   this.getForStorageType = function (type) {
+    if (client) {
+      return client;
+    }
     if (type === 'S3') {
-      return new SnowflakeS3Util(connectionConfig);
+      client = new SnowflakeS3Util(connectionConfig);
     } else if (type === 'AZURE') {
-      return new SnowflakeAzureUtil(connectionConfig);
+      client = new SnowflakeAzureUtil(connectionConfig);
     } else if (type === 'GCS') {
-      return new SnowflakeGCSUtil();
-    } else {
-      return null;
+      client = new SnowflakeGCSUtil(connectionConfig);
     }
+    return client;
   };
 
   /**