From cb7540794908badc8cf416846cf4a64d018762b5 Mon Sep 17 00:00:00 2001 From: Dominik Przybysz Date: Tue, 4 Jul 2023 13:06:53 +0200 Subject: [PATCH] SNOW-734920: Cache all types of HTTPS agent --- lib/http/base.js | 12 +- lib/http/node.js | 126 ++++++++---------- .../ocsp_mock/testConnectionOcspMock.js | 32 +++-- test/integration/testKeepAlive.js | 4 +- test/integration/testLargeResultSet.js | 2 +- 5 files changed, 81 insertions(+), 95 deletions(-) diff --git a/lib/http/base.js b/lib/http/base.js index 4e48a9056..c14348954 100644 --- a/lib/http/base.js +++ b/lib/http/base.js @@ -86,21 +86,17 @@ HttpClient.prototype.request = function (options) }; let mock; - if (this._connectionConfig.agentClass) - { + if (this._connectionConfig.agentClass) { mock = { agentClass: this._connectionConfig.agentClass } } // add the agent and proxy options - const agentAndProxyOptions = this.getAgentAndProxyOptions( - url, this._connectionConfig.getProxy(), - mock); + const agent = this.getAgent(url, this._connectionConfig.getProxy(), mock); requestOptions.data = requestOptions.body; - requestOptions.httpsAgent = agentAndProxyOptions.agentClass(agentAndProxyOptions.agentOptions); - requestOptions.httpsAgent.keepAlive = agentAndProxyOptions.agentOptions.keepAlive; + requestOptions.httpsAgent = agent; requestOptions.retryDelay = this.constructExponentialBackoffStrategy(); request = axios.request(requestOptions).then(response => { @@ -210,7 +206,7 @@ HttpClient.prototype.getRequestModule = function () * * @returns {*} */ -HttpClient.prototype.getAgentAndProxyOptions = function (url, proxy, mock) +HttpClient.prototype.getAgent = function (url, proxy, mock) { return null; }; diff --git a/lib/http/node.js b/lib/http/node.js index 7d196f21e..49cf8062a 100644 --- a/lib/http/node.js +++ b/lib/http/node.js @@ -41,92 +41,76 @@ function NodeHttpClient(connectionConfig) Util.inherits(NodeHttpClient, Base); -/** - * @inheritDoc - */ -NodeHttpClient.prototype.getAgentAndProxyOptions = function (url, proxy, mock) -{ - const isHttps = Url.parse(url).protocol === 'https:'; - let agentClass; - let agentOptions = { keepAlive: true }; - let options; - var bypassProxy = false; +const httpsAgentCache = new Map(); + +function getFromCacheOrCreate(agentClass, options, url) { + const parsed = Url.parse(url); + const protocol = parsed.protocol || 'http:'; + const port = parsed.port || (protocol === 'http:' ? '80' : '443'); + const agentId = `${protocol}//${parsed.hostname}:${port}` + if (httpsAgentCache.has(agentId)) { + Logger.getInstance().trace(`Get agent with id: ${agentId} from cache`); + return httpsAgentCache.get(agentId); + } else { + agent = agentClass(options) + httpsAgentCache.set(agentId, agent); + Logger.getInstance().trace(`Create and add to cache new agent ${agentId}`); + return agent; + } +} + +function prepareProxyAgentOptions(agentOptions, proxy) { + Logger.getInstance().info(`Use proxy: ${JSON.stringify(proxy)}`); + agentOptions.host = proxy.host; + agentOptions.port = proxy.port; + agentOptions.protocol = proxy.protocol; + if (proxy.user && proxy.password) { + agentOptions.user = proxy.user; + agentOptions.password = proxy.password; + } +} - if (proxy && proxy.noProxy) - { +function isBypassProxy(proxy, url, bypassProxy) { + if (proxy && proxy.noProxy) { var bypassList = proxy.noProxy.split("|"); - for (let i = 0; i < bypassList.length; i++) - { + for (let i = 0; i < bypassList.length; i++) { host = bypassList[i].trim(); host = host.replace("*", ".*?"); var matches = url.match(host); if (matches) { Logger.getInstance().debug("bypassing proxy for %s", url); - bypassProxy = true; + return true; } } } + return false; +} - if (isHttps && mock) - { - agentClass = mock.agentClass; - options = { - agentClass: agentClass, - agentOptions: agentOptions - }; - } - else if (isHttps) - { - if (proxy && !bypassProxy) - { - agentClass = HttpsProxyAgent; - agentOptions.host = proxy.host; - agentOptions.port = proxy.port; - agentOptions.user = proxy.user; - agentOptions.password = proxy.password; - agentOptions.protocol = proxy.protocol; - } - else - { - agentClass = HttpsAgent; - } +/** + * @inheritDoc + */ +NodeHttpClient.prototype.getAgent = function (url, proxy, mock) { + const isHttps = Url.parse(url).protocol === 'https:'; + let agentOptions = {keepAlive: false}; + var bypassProxy = isBypassProxy(proxy, url); - options = - { - agentClass: agentClass, - agentOptions: agentOptions - }; + if (mock) { + return mock.agentClass(agentOptions) } - else if (proxy && !bypassProxy) - { - options = - { - proxy: - { - protocol: 'http:', - hostname: proxy.host, - port: proxy.port - }, - agentClass: HttpAgent, - agentOptions: agentOptions - }; - if (proxy.user && proxy.password) - { - options.proxy.user = proxy.user; - options.proxy.password = proxy.password; + + if (isHttps) { + if (proxy && !bypassProxy) { + const proxyAgentOptions = prepareProxyAgentOptions(agentOptions, proxy) + return getFromCacheOrCreate(HttpsProxyAgent, proxyAgentOptions, url); + } else { + return getFromCacheOrCreate(HttpsAgent, agentOptions, url); } + } else if (proxy && !bypassProxy) { + let proxyAgentOptions = prepareProxyAgentOptions(); + return getFromCacheOrCreate(HttpAgent, proxyAgentOptions, url); + } else { + return getFromCacheOrCreate(HttpAgent, agentOptions, url); } - else - { - options = - { - agentClass: HttpAgent, - agentOptions: agentOptions - }; - } - // otherwise, just use the default agent used by request.js - - return options || {}; }; module.exports = NodeHttpClient; diff --git a/test/integration/ocsp_mock/testConnectionOcspMock.js b/test/integration/ocsp_mock/testConnectionOcspMock.js index d9b78aef2..ac88c771e 100644 --- a/test/integration/ocsp_mock/testConnectionOcspMock.js +++ b/test/integration/ocsp_mock/testConnectionOcspMock.js @@ -9,6 +9,8 @@ const connOption = require('../connectionOptions'); const Errors = require('../../../lib/errors'); const ErrorCodes = Errors.codes; const HttpsMockAgent = require('./https_ocsp_mock_agent'); +const {configureLogger} = require("../../configureLogger"); +const testUtil = require("../testUtil"); function cloneConnOption(connOption) { @@ -24,6 +26,11 @@ describe('Connection test with OCSP Mock', function () { const valid = cloneConnOption(connOption.valid); const isHttps = valid.accessUrl.startsWith("https"); + before(async () => + { + configureLogger('TRACE'); + }); + function connect(errcode, connection, callback) { @@ -76,18 +83,20 @@ describe('Connection test with OCSP Mock', function () ); }); - it('Connection failure with OCSP unknown error', function (done) - { + it('Connection failure with OCSP unknown error', function (done) { valid.agentClass = HttpsMockAgent.HttpsMockAgentOcspUnkwown; const connection = snowflake.createConnection(valid); async.series([ - function (callback) - { - connect(ErrorCodes.ERR_OCSP_UNKNOWN, connection, callback); + function (callback) { + try { + connect(ErrorCodes.ERR_OCSP_UNKNOWN, connection, callback); + } catch (err){ + console.log(`err ${err}`); + } + }, - function (callback) - { + function (callback) { destroy(connection, callback); } ], @@ -95,18 +104,15 @@ describe('Connection test with OCSP Mock', function () ); }); - it('Connection failure with invalid validity OCSP error', function (done) - { + it('Connection failure with invalid validity OCSP error', function (done) { valid.agentClass = HttpsMockAgent.HttpsMockAgentOcspInvalid; const connection = snowflake.createConnection(valid); async.series([ - function (callback) - { + function (callback) { connect(ErrorCodes.ERR_OCSP_INVALID_VALIDITY, connection, callback); }, - function (callback) - { + function (callback) { destroy(connection, callback); } ], diff --git a/test/integration/testKeepAlive.js b/test/integration/testKeepAlive.js index 05f4f6d64..6f1237169 100644 --- a/test/integration/testKeepAlive.js +++ b/test/integration/testKeepAlive.js @@ -8,7 +8,7 @@ const async = require('async'); const testUtil = require('./testUtil'); const { performance } = require('perf_hooks'); -describe.skip('keepAlive perf test', function () +describe('keepAlive perf test', function () { this.timeout(1000000); @@ -56,7 +56,7 @@ describe.skip('keepAlive perf test', function () { var start = performance.now(); var statement = connection.execute({ - sqlText: "SELECT L_COMMENT from SNOWFLAKE_SAMPLE_DATA.TPCH_SF100.LINEITEM limit 10000;", + sqlText: "SELECT VALUE from VARIANT_TABLE2 limit 10000;", complete: function (err, stmt, rows) { var stream = statement.streamRows(); diff --git a/test/integration/testLargeResultSet.js b/test/integration/testLargeResultSet.js index 9b82518b9..973a1163e 100644 --- a/test/integration/testLargeResultSet.js +++ b/test/integration/testLargeResultSet.js @@ -263,13 +263,13 @@ describe('SNOW-743920: Large result set with ~35 chunks', function () { before(async () => { connection = testUtil.createConnection(); + configureLogger('TRACE'); await testUtil.connectAsync(connection); // setting ROWS_PER_RESULTSET causes invalid, not encoded chunks from GCP // await testUtil.executeCmdAsync(connection, 'alter session set ROWS_PER_RESULTSET = 1000000'); await testUtil.executeCmdAsync(connection, 'alter session set USE_CACHED_RESULT = false;'); await testUtil.executeCmdAsync(connection, createTable); await testUtil.executeCmdAsync(connection, populateData); - configureLogger('TRACE'); }); after(async () => {