SNOW-1820372: (from 947) when proxy is set in Connection, driver does send traffic through the proxy to S3, but not to Azure blob / GCS bucket (only Snowflake). Works with proxy envvar (GCP)

index.d.ts

@@ -512,6 +512,11 @@ declare module 'snowflake-sdk' {
          *  The option to pass passcode from DUO.
          */
         passcode?: string;
+
+        /**
+         *  The option to override the environment proxy value (HTTPS_PROXY) if the connection proxy is configured in GCS
+         */
+        overrideGCSEnvProxy?: boolean;
     }
 
     export interface Connection {

lib/connection/connection_config.js

@@ -17,7 +17,6 @@ const levenshtein = require('fastest-levenshtein');
 const RowMode = require('./../constants/row_mode');
 const DataTypes = require('./result/data_types');
 const Logger = require('../logger');
-const LoggingUtil = require('../logger/logging_util');
 const WAIT_FOR_BROWSER_ACTION_TIMEOUT = 120000;
 const DEFAULT_PARAMS =
 [
@@ -67,6 +66,7 @@ const DEFAULT_PARAMS =
   'credentialCacheDir',
   'passcodeInPassword',
   'passcode',
+  'overrideGCSEnvProxy'
 ];
 
 function consolidateHostAndAccount(options) {
@@ -514,6 +514,13 @@ function ConnectionConfig(options, validateCredentials, qaMode, clientInfo) {
     passcode = options.passcode;
   }
 
+  let overrideGCSEnvProxy = true;
+  if (Util.exists(options.overrideGCSEnvProxy)) {
+    Errors.checkArgumentValid(Util.isBoolean(options.overrideGCSEnvProxy), 
+      ErrorCodes.ERR_CONN_CREATE_INVALID_OVERRIDE_GCS_ENV_PROXY);
+
+    overrideGCSEnvProxy = options.overrideGCSEnvProxy;
+  } 
 
   if (validateDefaultParameters) {
     for (const [key] of Object.entries(options)) {
@@ -837,6 +844,10 @@ function ConnectionConfig(options, validateCredentials, qaMode, clientInfo) {
     return passcode;
   };
 
+  this.getOverrideGCSEnvProxy = function () {
+    return overrideGCSEnvProxy;
+  };
+
   /**
    * Returns attributes of Connection Config object that can be used to identify
    * the connection, when ID is not available in the scope. This is not sufficient set,
@@ -847,21 +858,7 @@ function ConnectionConfig(options, validateCredentials, qaMode, clientInfo) {
   this.describeIdentityAttributes = function () {
     return `host: ${this.host}, account: ${this.account}, accessUrl: ${this.accessUrl}, `
         + `user: ${this.username}, role: ${this.getRole()}, database: ${this.getDatabase()}, `
-        + `schema: ${this.getSchema()}, warehouse: ${this.getWarehouse()}, ` + this.describeProxy();
-  };
-
-  /**
-   * @returns {string}
-   */
-  this.describeProxy = function () {
-    const proxy = this.getProxy();
-    if (Util.exists(proxy)) {
-      return `proxyHost: ${proxy.host}, proxyPort: ${proxy.port}, proxyUser: ${proxy.user}, `
-      + `proxyPassword is ${LoggingUtil.describePresence(proxy.password)}, `
-      + `proxyProtocol: ${proxy.protocol}, noProxy: ${proxy.noProxy}`;
-    } else {
-      return 'proxy was not configured';
-    }
+        + `schema: ${this.getSchema()}, warehouse: ${this.getWarehouse()}, ` + ProxyUtil.describeProxy(this.getProxy());
   };
 
   // save config options

lib/constants/error_messages.js

@@ -84,6 +84,7 @@ exports[404053] = 'A host must be specified.';
 exports[404054] = 'Invalid host. The specified value must be a string.';
 exports[404055] = 'Invalid passcodeInPassword. The specified value must be a boolean';
 exports[404056] = 'Invalid passcode. The specified value must be a string';
+exports[404057] = 'Invalid overrideGCSEnvProxy. The specified value must be a boolean';
 
 // 405001
 exports[405001] = 'Invalid callback. The specified value must be a function.';

lib/errors.js

@@ -88,6 +88,7 @@ codes.ERR_CONN_CREATE_MISSING_HOST = 404053;
 codes.ERR_CONN_CREATE_INVALID_HOST = 404054;
 codes.ERR_CONN_CREATE_INVALID_PASSCODE_IN_PASSWORD = 404055;
 codes.ERR_CONN_CREATE_INVALID_PASSCODE = 404056;
+codes.ERR_CONN_CREATE_INVALID_OVERRIDE_GCS_ENV_PROXY = 404057;
 
 // 405001
 codes.ERR_CONN_CONNECT_INVALID_CALLBACK = 405001;

lib/file_transfer_agent/gcs_util.js

@@ -4,6 +4,10 @@
 
 const EncryptionMetadata = require('./encrypt_util').EncryptionMetadata;
 const FileHeader = require('./file_util').FileHeader;
+const getProxyAgent = require('../http/node').getProxyAgent;
+const ProxyUtil = require('../proxy_util');
+const Util = require('../util');
+const Logger = require('../logger');
 const { shouldPerformGCPBucket } = require('../util');
 
 const GCS_METADATA_PREFIX = 'x-goog-meta-';
@@ -19,8 +23,8 @@ const GCS_FILE_HEADER_ENCRYPTION_METADATA = 'gcs-file-header-encryption-metadata
 
 const HTTP_HEADER_CONTENT_ENCODING = 'Content-Encoding';
 const resultStatus = require('./file_util').resultStatus;
-
 const { Storage } = require('@google-cloud/storage');
+const { isBypassProxy } = require('../http/node');
 
 const EXPIRED_TOKEN = 'ExpiredToken';
 
@@ -37,15 +41,24 @@ function GCSLocation(bucketName, path) {
 /**
  * Creates an GCS utility object.
  *
- * @param {module} httpclient
- * @param {module} filestream
+ * @param {module} httpClient
+ * @param {module} fileStream
  *
  * @returns {Object}
  * @constructor
  */
-function GCSUtil(httpclient, filestream) {
-  const axios = typeof httpclient !== 'undefined' ? httpclient : require('axios');
-  const fs = typeof filestream !== 'undefined' ? filestream : require('fs');
+function GCSUtil(connectionConfig, httpClient, fileStream) {
+  let axios = httpClient;
+  const fs = typeof fileStream !== 'undefined' ? fileStream : require('fs');
+  const connectionProxy = connectionConfig.getProxy();
+  const proxyString = ProxyUtil.stringifyProxy(connectionProxy);
+  let isEnvProxyOverridden = false;
+  let envProxy = 'HTTP_PROXY';
+  let originalEnvProxy =  process.env[envProxy];
+  if (!Util.exists(originalEnvProxy)) {
+    envProxy = 'HTTPS_PROXY';
+    originalEnvProxy =  process.env[envProxy];
+  }
 
   /**
   * Retrieve the GCS token from the stage info metadata.
@@ -57,7 +70,15 @@ function GCSUtil(httpclient, filestream) {
   this.createClient = function (stageInfo) {
     const stageCredentials = stageInfo['creds'];
     const gcsToken = stageCredentials['GCS_ACCESS_TOKEN'];
-
+    //TODO: SNOW-1789759 the value is hardcoded now, but it should be server driven
+    const isRegionalUrlEnabled = (stageInfo.region).toLowerCase() === 'me-central2' || stageInfo.useRegionalUrl;
+    let endPoint = null;
+    if (stageInfo['endPoint']) {
+      endPoint = stageInfo['endPoint'];
+    } else if (isRegionalUrlEnabled) {
+      endPoint = `storage.${stageInfo.region.toLowerCase()}.rep.googleapis.com`;
+    }
+
     let client;
     if (gcsToken) {
       const interceptors = [];
@@ -68,21 +89,38 @@ function GCSUtil(httpclient, filestream) {
           return requestConfig;
         }
       });
-
-      //TODO: SNOW-1789759 hardcoded region will be replaced in the future
-      const isRegionalUrlEnabled = (stageInfo.region).toLowerCase() === 'me-central2' || stageInfo.useRegionalUrl;
-      let endPoint = null;
-      if (stageInfo['endPoint']) {
-        endPoint = stageInfo['endPoint'];
-      } else if (isRegionalUrlEnabled) {
-        endPoint = `storage.${stageInfo.region.toLowerCase()}.rep.googleapis.com`;
-      }
-      const storage = endPoint ? new Storage({ interceptors_: interceptors, apiEndpoint: endPoint }) : new Storage({ interceptors_: interceptors });
+      const storage = Util.exists(endPoint) ? new Storage({ interceptors_: interceptors, apiEndpoint: endPoint }) : new Storage({ interceptors_: interceptors });
       client = { gcsToken: gcsToken, gcsClient: storage };
     } else {
       client = null;
     }
 
+    const isByPassed = isBypassProxy(connectionProxy, endPoint || `storage.${stageInfo.region.toLowerCase()}.rep.googleapis.com`);
+    if (Util.isEmptyObject(connectionProxy) || isByPassed) {
+      isEnvProxyOverridden = false;
+    } else if (!Util.exists(originalEnvProxy)) {
+      isEnvProxyOverridden = true;
+    } else if (connectionConfig.getOverrideGCSEnvProxy()) {
+      if (proxyString !==  originalEnvProxy) {
+        Logger.getInstance().debug(`The destination host is: ${ProxyUtil.getHostFromURL(endPoint || `https://storage.${stageInfo.region.toLowerCase()}.rep.googleapis.com`)} and the proxy host is: ${connectionProxy.host}`);
+        Logger.getInstance().warn(`The environment variable ${envProxy} value will be overwritten during the Storage HTTP calls`);
+        isEnvProxyOverridden = true;
+      }
+    }
+
+    if (isEnvProxyOverridden) {
+      Logger.getInstance().trace(`Initializing the proxy information for the GCS Client: ${ProxyUtil.describeProxy(connectionProxy)}`);
+    }
+
+    if (typeof httpClient === 'undefined') {
+      const proxyAgent = getProxyAgent(connectionProxy, new URL(connectionConfig.accessUrl), endPoint || `storage.${stageInfo.region.toLowerCase()}.rep.googleapis.com` );
+      axios = require('axios').create({
+        proxy: false,
+        httpAgent: proxyAgent,
+        httpsAgent: proxyAgent,
+      });
+    }
+
     return client;
   };
 
@@ -147,11 +185,13 @@ function GCSUtil(httpclient, filestream) {
         try {
           if (shouldPerformGCPBucket(accessToken)) {
             const gcsLocation = this.extractBucketNameAndPath(meta['stageInfo']['location']);
-
-            const metadata = await meta['client'].gcsClient
+            overrideEnvProxy();
+            const  metadata = await meta['client'].gcsClient
               .bucket(gcsLocation.bucketName)
               .file(gcsLocation.path + filename)
               .getMetadata();
+            restoreEnvProxy();
+
 
             digest = metadata[0].metadata[SFC_DIGEST];
             contentLength = metadata[0].size;
@@ -287,7 +327,8 @@ function GCSUtil(httpclient, filestream) {
     try {
       if (shouldPerformGCPBucket(accessToken)) {
         const gcsLocation = this.extractBucketNameAndPath(meta['stageInfo']['location']);
-
+
+        overrideEnvProxy();
         await meta['client'].gcsClient
           .bucket(gcsLocation.bucketName)
           .file(gcsLocation.path + meta['dstFileName'])
@@ -301,6 +342,7 @@ function GCSUtil(httpclient, filestream) {
               }
             }
           });
+        restoreEnvProxy();
       } else {
         // Set maxBodyLength to allow large file uploading
         await axios.put(uploadUrl, fileStream, { maxBodyLength: Infinity, headers: gcsHeaders });
@@ -360,7 +402,7 @@ function GCSUtil(httpclient, filestream) {
     try {
       if (shouldPerformGCPBucket(accessToken)) {
         const gcsLocation = this.extractBucketNameAndPath(meta['stageInfo']['location']);
-
+        overrideEnvProxy();
         await meta['client'].gcsClient
           .bucket(gcsLocation.bucketName)
           .file(gcsLocation.path + meta['srcFileName'])
@@ -372,16 +414,15 @@ function GCSUtil(httpclient, filestream) {
           .bucket(gcsLocation.bucketName)
           .file(gcsLocation.path + meta['srcFileName'])
           .getMetadata();
+        restoreEnvProxy();
 
         encryptionDataprop = metadata[0].metadata[ENCRYPTIONDATAPROP];
         matDescKey = metadata[0].metadata[MATDESC_KEY];
         sfcDigest = metadata[0].metadata[SFC_DIGEST];
         size = metadata[0].size;
       } else {
         let response;
-        await axios({
-          method: 'get',
-          url: downloadUrl,
+        await axios.get(downloadUrl, {
           headers: gcsHeaders,
           responseType: 'stream'
         }).then(async (res) => {
@@ -457,6 +498,18 @@ function GCSUtil(httpclient, filestream) {
     return link;
   };
 
+  //getIsEnvProxyOverridden and getEnvProxiesDuringProcess are for testing purpose
+  this.getIsEnvProxyOverridden = function () {
+    return isEnvProxyOverridden;
+  };
+
+  this.getEnvProxiesDuringProcess = function () {
+    if (isEnvProxyOverridden) {
+      return envProxy === 'HTTP_PROXY' ? [proxyString, process.env.HTTPS_PROXY] : [undefined, proxyString];
+    }
+    return [process.env.HTTP_PROXY, process.env.HTTPS_PROXY];
+  };
+
   /**
 * Left strip the specified character from a string.
 *
@@ -471,6 +524,20 @@ function GCSUtil(httpclient, filestream) {
     }
     return str;
   }
+
+  function overrideEnvProxy() {
+    if (!isEnvProxyOverridden) {
+      return;
+    }
+    process.env[envProxy] = proxyString;
+  }
+
+  function restoreEnvProxy() {
+    if (!isEnvProxyOverridden) {
+      return;
+    }
+    Util.restoreEnvVar(envProxy, originalEnvProxy);
+  }
 }
 
 module.exports = GCSUtil;

lib/file_transfer_agent/remote_storage_util.js

@@ -33,6 +33,8 @@ exports.SnowflakeFileEncryptionMaterial = SnowflakeFileEncryptionMaterial;
  * @constructor
  */
 function RemoteStorageUtil(connectionConfig) {
+  let client = null;
+
   /**
   * Get storage type based on location type.
   *
@@ -41,15 +43,17 @@ function RemoteStorageUtil(connectionConfig) {
   * @returns {Object}
   */
   this.getForStorageType = function (type) {
+    if (client) {
+      return client;
+    }
     if (type === 'S3') {
-      return new SnowflakeS3Util(connectionConfig);
+      client = new SnowflakeS3Util(connectionConfig);
     } else if (type === 'AZURE') {
-      return new SnowflakeAzureUtil(connectionConfig);
+      client = new SnowflakeAzureUtil(connectionConfig);
     } else if (type === 'GCS') {
-      return new SnowflakeGCSUtil();
-    } else {
-      return null;
+      client = new SnowflakeGCSUtil(connectionConfig);
     }
+    return client;
   };
 
   /**