SNOW-734920: Cache all types of HTTPS agent

lib/http/base.js

@@ -86,21 +86,17 @@ HttpClient.prototype.request = function (options)
       };
 
     let mock;
-    if (this._connectionConfig.agentClass)
-    {
+    if (this._connectionConfig.agentClass) {
       mock = {
         agentClass: this._connectionConfig.agentClass
       }
     }
 
     // add the agent and proxy options
-    const agentAndProxyOptions = this.getAgentAndProxyOptions(
-      url, this._connectionConfig.getProxy(),
-      mock);
+    const agent = this.getAgent(url, this._connectionConfig.getProxy(), mock);
 
     requestOptions.data = requestOptions.body;
-    requestOptions.httpsAgent = agentAndProxyOptions.agentClass(agentAndProxyOptions.agentOptions);
-    requestOptions.httpsAgent.keepAlive = agentAndProxyOptions.agentOptions.keepAlive;
+    requestOptions.httpsAgent = agent;
     requestOptions.retryDelay = this.constructExponentialBackoffStrategy();
 
     request = axios.request(requestOptions).then(response => {
@@ -210,7 +206,7 @@ HttpClient.prototype.getRequestModule = function ()
  *
  * @returns {*}
  */
-HttpClient.prototype.getAgentAndProxyOptions = function (url, proxy, mock)
+HttpClient.prototype.getAgent = function (url, proxy, mock)
 {
   return null;
 };

lib/http/node.js

@@ -41,92 +41,76 @@ function NodeHttpClient(connectionConfig)
 
 Util.inherits(NodeHttpClient, Base);
 
-/**
- * @inheritDoc
- */
-NodeHttpClient.prototype.getAgentAndProxyOptions = function (url, proxy, mock)
-{
-  const isHttps = Url.parse(url).protocol === 'https:';
-  let agentClass;
-  let agentOptions = { keepAlive: true };
-  let options;
-  var bypassProxy = false;
+const httpsAgentCache = new Map();
+
+function getFromCacheOrCreate(agentClass, options, url) {
+  const parsed = Url.parse(url);
+  const protocol = parsed.protocol || 'http:';
+  const port = parsed.port || (protocol === 'http:' ? '80' : '443');
+  const agentId = `${protocol}//${parsed.hostname}:${port}`
+  if (httpsAgentCache.has(agentId)) {
+    Logger.getInstance().trace(`Get agent with id: ${agentId} from cache`);
+    return httpsAgentCache.get(agentId);
+  } else {
+    agent = agentClass(options)
+    httpsAgentCache.set(agentId, agent);
+    Logger.getInstance().trace(`Create and add to cache new agent ${agentId}`);
+    return agent;
+  }
+}
+
+function prepareProxyAgentOptions(agentOptions, proxy) {
+  Logger.getInstance().info(`Use proxy: ${JSON.stringify(proxy)}`);
+  agentOptions.host = proxy.host;
+  agentOptions.port = proxy.port;
+  agentOptions.protocol = proxy.protocol;
+  if (proxy.user && proxy.password) {
+    agentOptions.user = proxy.user;
+    agentOptions.password = proxy.password;
+  }
+}
 
-  if (proxy && proxy.noProxy)
-  {
+function isBypassProxy(proxy, url, bypassProxy) {
+  if (proxy && proxy.noProxy) {
     var bypassList = proxy.noProxy.split("|");
-    for (let i = 0; i < bypassList.length; i++)
-    {
+    for (let i = 0; i < bypassList.length; i++) {
       host = bypassList[i].trim();
       host = host.replace("*", ".*?");
       var matches = url.match(host);
       if (matches) {
         Logger.getInstance().debug("bypassing proxy for %s", url);
-        bypassProxy = true;
+        return true;
       }
     }
   }
+  return false;
+}
 
-  if (isHttps && mock)
-  {
-    agentClass = mock.agentClass;
-    options = {
-      agentClass: agentClass,
-      agentOptions: agentOptions
-    };
-  }
-  else if (isHttps)
-  {
-    if (proxy && !bypassProxy)
-    {
-      agentClass = HttpsProxyAgent;
-      agentOptions.host = proxy.host;
-      agentOptions.port = proxy.port;
-      agentOptions.user = proxy.user;
-      agentOptions.password = proxy.password;
-      agentOptions.protocol = proxy.protocol;
-    }
-    else
-    {
-      agentClass = HttpsAgent;
-    }
+/**
+ * @inheritDoc
+ */
+NodeHttpClient.prototype.getAgent = function (url, proxy, mock) {
+  const isHttps = Url.parse(url).protocol === 'https:';
+  let agentOptions = {keepAlive: false};
+  var bypassProxy = isBypassProxy(proxy, url);
 
-    options =
-      {
-        agentClass: agentClass,
-        agentOptions: agentOptions
-      };
+  if (mock) {
+    return mock.agentClass(agentOptions)
   }
-  else if (proxy && !bypassProxy)
-  {
-    options =
-    {
-      proxy:
-      {
-        protocol: 'http:',
-        hostname: proxy.host,
-        port: proxy.port
-      },
-      agentClass: HttpAgent,
-      agentOptions: agentOptions
-    };
-    if (proxy.user && proxy.password)
-    {
-      options.proxy.user = proxy.user;
-      options.proxy.password = proxy.password;
+
+  if (isHttps) {
+    if (proxy && !bypassProxy) {
+      const proxyAgentOptions = prepareProxyAgentOptions(agentOptions, proxy)
+      return getFromCacheOrCreate(HttpsProxyAgent, proxyAgentOptions, url);
+    } else {
+      return getFromCacheOrCreate(HttpsAgent, agentOptions, url);
     }
+  } else if (proxy && !bypassProxy) {
+    let proxyAgentOptions = prepareProxyAgentOptions();
+    return getFromCacheOrCreate(HttpAgent, proxyAgentOptions, url);
+  } else {
+    return getFromCacheOrCreate(HttpAgent, agentOptions, url);
   }
-  else
-  {
-    options =
-      {
-        agentClass: HttpAgent,
-        agentOptions: agentOptions
-      };
-  }
-  // otherwise, just use the default agent used by request.js
-
-  return options || {};
 };
 
 module.exports = NodeHttpClient;

test/integration/ocsp_mock/testConnectionOcspMock.js

@@ -9,6 +9,8 @@ const connOption = require('../connectionOptions');
 const Errors = require('../../../lib/errors');
 const ErrorCodes = Errors.codes;
 const HttpsMockAgent = require('./https_ocsp_mock_agent');
+const {configureLogger} = require("../../configureLogger");
+const testUtil = require("../testUtil");
 
 function cloneConnOption(connOption)
 {
@@ -24,6 +26,11 @@ describe('Connection test with OCSP Mock', function ()
 {
   const valid = cloneConnOption(connOption.valid);
   const isHttps = valid.accessUrl.startsWith("https");
+  before(async () =>
+  {
+    configureLogger('TRACE');
+  });
+
 
   function connect(errcode, connection, callback)
   {
@@ -76,37 +83,36 @@ describe('Connection test with OCSP Mock', function ()
     );
   });
 
-  it('Connection failure with OCSP unknown error', function (done)
-  {
+  it('Connection failure with OCSP unknown error', function (done) {
     valid.agentClass = HttpsMockAgent.HttpsMockAgentOcspUnkwown;
     const connection = snowflake.createConnection(valid);
 
     async.series([
-        function (callback)
-        {
-          connect(ErrorCodes.ERR_OCSP_UNKNOWN, connection, callback);
+        function (callback) {
+      try {
+        connect(ErrorCodes.ERR_OCSP_UNKNOWN, connection, callback);
+      } catch (err){
+        console.log(`err ${err}`);
+      }
+
         },
-        function (callback)
-        {
+        function (callback) {
           destroy(connection, callback);
         }
       ],
       done
     );
   });
 
-  it('Connection failure with invalid validity OCSP error', function (done)
-  {
+  it('Connection failure with invalid validity OCSP error', function (done) {
     valid.agentClass = HttpsMockAgent.HttpsMockAgentOcspInvalid;
     const connection = snowflake.createConnection(valid);
 
     async.series([
-        function (callback)
-        {
+        function (callback) {
           connect(ErrorCodes.ERR_OCSP_INVALID_VALIDITY, connection, callback);
         },
-        function (callback)
-        {
+        function (callback) {
           destroy(connection, callback);
         }
       ],

test/integration/testKeepAlive.js

@@ -56,7 +56,7 @@ describe.skip('keepAlive perf test', function ()
             {
               var start = performance.now();
               var statement = connection.execute({
-                sqlText: "SELECT L_COMMENT from SNOWFLAKE_SAMPLE_DATA.TPCH_SF100.LINEITEM limit 10000;",
+                sqlText: "SELECT VALUE from VARIANT_TABLE2 limit 10000;",
                 complete: function (err, stmt, rows)
                 {
                   var stream = statement.streamRows();

test/integration/testLargeResultSet.js

@@ -263,13 +263,13 @@ describe('SNOW-743920: Large result set with ~35 chunks', function () {
 
   before(async () => {
     connection = testUtil.createConnection();
+    configureLogger('TRACE');
     await testUtil.connectAsync(connection);
     // setting ROWS_PER_RESULTSET causes invalid, not encoded chunks from GCP
     // await testUtil.executeCmdAsync(connection, 'alter session set ROWS_PER_RESULTSET = 1000000');
     await testUtil.executeCmdAsync(connection, 'alter session set USE_CACHED_RESULT = false;');
     await testUtil.executeCmdAsync(connection, createTable);
     await testUtil.executeCmdAsync(connection, populateData);
-    configureLogger('TRACE');
   });
 
   after(async () => {