From 6868a2bb46cda9a87bcb5516c7addafe35899147 Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-jy Date: Wed, 6 Sep 2023 08:56:11 -0700 Subject: [PATCH 01/30] Query Context Cache --- lib/connection/connection_config.js | 44 ++- lib/connection/result/result.js | 5 + lib/connection/statement.js | 5 + lib/constants/error_messages.js | 2 + lib/errors.js | 6 +- lib/parameters.js | 7 + lib/queryContextCache.js | 392 ++++++++++++++++++++++++++ lib/services/sf.js | 32 ++- test/unit/mock/mock_http_client.js | 32 ++- test/unit/query_context_cache_test.js | 216 ++++++++++++++ 10 files changed, 724 insertions(+), 17 deletions(-) create mode 100644 lib/queryContextCache.js create mode 100644 test/unit/query_context_cache_test.js diff --git a/lib/connection/connection_config.js b/lib/connection/connection_config.js index ede1870c2..802fd0b78 100644 --- a/lib/connection/connection_config.js +++ b/lib/connection/connection_config.js @@ -48,7 +48,8 @@ const DEFAULT_PARAMS = 'validateDefaultParameters', 'arrayBindingThreshold', 'gcsUseDownscopedCredential', - 'forceStageBindError' + 'forceStageBindError', + 'disableQueryContextCache' ]; function consolidateHostAndAccount(options) @@ -484,6 +485,20 @@ function ConnectionConfig(options, validateCredentials, qaMode, clientInfo) } } + var disableQueryContextCache = false; + if (Util.exists(options.disableQueryContextCache)) { + Errors.checkArgumentValid(Util.isBoolean(options.disableQueryContextCache), + ErrorCodes.ERR_CONN_CREATE_INVALID_DIABLED_QUERY_CONTEXT_CACHE); + disableQueryContextCache = options.disableQueryContextCache; + } + + var queryContextCacheSize = 5; + if(Util.exists(options.queryContextCacheSize)){ + Errors.checkArgumentValid(Util.isNumber(options.queryContextCacheSize), + ErrorCodes.ERR_CONN_CREATE_INVALID_QUERY_CONTEXT_CACHE_SIZE); + queryContextCacheSize = options.queryContextCacheSize; + } + /** * Returns an object that contains information about the proxy hostname, port, * etc. for when http requests are made. @@ -747,6 +762,24 @@ function ConnectionConfig(options, validateCredentials, qaMode, clientInfo) return forceStageBindError; }; + /** + * Returns the force stage bind error + * + * @returns {Number} + */ + this.getForceStageBindError = function () { + return forceStageBindError; + }; + /** + * Returns the status of the query contextcache. + * + * @returns {Number} + */ + this.getDisableQueryContextCache = function () { + return disableQueryContextCache; + } + + // save config options this.username = options.username; this.password = options.password; @@ -758,6 +791,8 @@ function ConnectionConfig(options, validateCredentials, qaMode, clientInfo) this.masterToken = options.masterToken; this.masterTokenExpirationTime = options.masterTokenExpirationTime; this.sessionTokenExpirationTime = options.sessionTokenExpirationTime; + this.disableQueryContextCache = disableQueryContextCache; + this.queryContextCacheSize = queryContextCacheSize; // create the parameters array var parameters = createParameters(); @@ -834,6 +869,7 @@ const PARAM_RETRY_SF_MAX_LOGIN_RETRIES = 'sfRetryMaxLoginRetries'; const PARAM_RETRY_SF_MAX_NUM_RETRIES = 'sfRetryMaxNumRetries'; const PARAM_RETRY_SF_STARTING_SLEEP_TIME = 'sfRetryStartingSleepTime'; const PARAM_RETRY_SF_MAX_SLEEP_TIME = 'sfRetryMaxSleepTime'; +const PARAM_QUERY_CONTEXT_CAHCE_SIZE = 'queryContextCacheSize'; /** * Creates the list of known parameters. If a parameter is marked as external, @@ -919,7 +955,7 @@ function createParameters() name: PARAM_RETRY_SF_MAX_SLEEP_TIME, defaultValue: 16, validate: isNonNegativeNumber - } + }, ]; } @@ -988,6 +1024,10 @@ ConnectionConfig.prototype.getRetrySfMaxSleepTime = function () return this._getParameterValue(PARAM_RETRY_SF_MAX_SLEEP_TIME); }; + + + + /** * Returns the value of a given connection config parameter. * diff --git a/lib/connection/result/result.js b/lib/connection/result/result.js index f71f57b97..655404352 100644 --- a/lib/connection/result/result.js +++ b/lib/connection/result/result.js @@ -53,6 +53,7 @@ function Result(options) { this._returnedRows = data.returned; this._totalRows = data.total; this._statementTypeId = data.statementTypeId; + this._queryContext = data.queryContext; // if no chunk headers were specified, but a query-result-master-key (qrmk) // was specified, build the chunk headers from the qrmk @@ -125,6 +126,10 @@ function Result(options) { this._statement, this._services); + this.getQueryContext = function() { + return this._queryContext; + } + /* Disable the ChunkCache until the implementation is complete. * * // create a chunk cache and save a reference to it in case we need to diff --git a/lib/connection/statement.js b/lib/connection/statement.js index 6adbca25e..a93363dbf 100644 --- a/lib/connection/statement.js +++ b/lib/connection/statement.js @@ -808,6 +808,7 @@ function createOnStatementRequestSuccRow(statement, context) }); // save the query id context.queryId = context.result.getQueryId(); + this.services.sf.deserializeQueryContext(context.result.getQueryContext()); } } else @@ -1334,6 +1335,10 @@ function sendRequestPreExec(statementContext, onResultAvailable) json.isInternal = statementContext.internal; } + if(!statementContext.disableQueryContextCache){ + json.queryContextDTO = statementContext.services.sf.getQueryContextDTO(); + } + // use the snowflake service to issue the request sendSfRequest(statementContext, { diff --git a/lib/constants/error_messages.js b/lib/constants/error_messages.js index 0b2887103..5195ad20c 100644 --- a/lib/constants/error_messages.js +++ b/lib/constants/error_messages.js @@ -64,6 +64,8 @@ exports[404037] = 'Invalid arrayBindingThreshold. The specified value must be a exports[404038] = 'Invalid gcsUseDownscopedCredential. The specified value must be a boolean.'; exports[404039] = 'Invalid forceStageBindError. The specified value must be a number.'; exports[404040] = 'Invalid browser timeout value. The specified value must be a positive number.'; +exports[404041] = 'Invalid disablQueryContextCache. The specified value must be a boolean.'; +exports[404042] = 'Invalid queryContextCacheSize. The specified value must be a positive number.' // 405001 exports[405001] = 'Invalid callback. The specified value must be a function.'; diff --git a/lib/errors.js b/lib/errors.js index 5361aedb0..3432b6a3b 100644 --- a/lib/errors.js +++ b/lib/errors.js @@ -69,6 +69,8 @@ codes.ERR_CONN_CREATE_INVALID_ARRAY_BINDING_THRESHOLD = 404037; codes.ERR_CONN_CREATE_INVALID_GCS_USE_DOWNSCOPED_CREDENTIAL = 404038; codes.ERR_CONN_CREATE_INVALID_FORCE_STAGE_BIND_ERROR = 404039; codes.ERR_CONN_CREATE_INVALID_BROWSER_TIMEOUT = 404040; +codes.ERR_CONN_CREATE_INVALID_DIABLED_QUERY_CONTEXT_CACHE = 404041 +codes.ERR_CONN_CREATE_INVALID_QUERY_CONTEXT_CACHE_SIZE =404042 // 405001 codes.ERR_CONN_CONNECT_INVALID_CALLBACK = 405001; @@ -113,8 +115,8 @@ codes.ERR_CONN_EXEC_STMT_INVALID_REQUEST_ID = 409013; // 410001 codes.ERR_CONN_FETCH_RESULT_MISSING_OPTIONS = 410001; codes.ERR_CONN_FETCH_RESULT_INVALID_OPTIONS = 410002; -codes.ERR_CONN_FETCH_RESULT_MISSING_QUERY_ID = 410003; -codes.ERR_CONN_FETCH_RESULT_INVALID_QUERY_ID = 410004; +codes.ERR_CONN_FETCH_RESULT_MISSING_STATEMENT_ID = 410003; +codes.ERR_CONN_FETCH_RESULT_INVALID_STATEMENT_ID = 410004; codes.ERR_CONN_FETCH_RESULT_INVALID_COMPLETE = 410005; codes.ERR_CONN_FETCH_RESULT_INVALID_STREAM_RESULT = 410006; codes.ERR_CONN_FETCH_RESULT_INVALID_FETCH_AS_STRING = 410007; diff --git a/lib/parameters.js b/lib/parameters.js index af0dd46f1..276b81c66 100644 --- a/lib/parameters.js +++ b/lib/parameters.js @@ -60,6 +60,7 @@ names.CLIENT_SESSION_KEEP_ALIVE_HEARTBEAT_FREQUENCY = 'CLIENT_SESSION_KEEP_ALIVE names.JS_TREAT_INTEGER_AS_BIGINT = 'JS_TREAT_INTEGER_AS_BIGINT'; names.CLIENT_STAGE_ARRAY_BINDING_THRESHOLD = 'CLIENT_STAGE_ARRAY_BINDING_THRESHOLD'; names.MULTI_STATEMENT_COUNT = 'MULTI_STATEMENT_COUNT'; +names.QUERY_CONTEXT_CACHE_SIZE = 'QUERY_CONTEXT_CACHE_SIZE'; var parameters = [ @@ -106,6 +107,12 @@ var parameters = value: 1, desc: 'When 1, multi statement is disable, when 0, multi statement is unlimited' }), + new Parameter( + { + name: names.QUERY_CONTEXT_CACHE_SIZE, + value: 5, + desc: '' + }), ]; // put all the parameters in a map so they're easy to retrieve and update diff --git a/lib/queryContextCache.js b/lib/queryContextCache.js new file mode 100644 index 000000000..e9c77fa5b --- /dev/null +++ b/lib/queryContextCache.js @@ -0,0 +1,392 @@ +/* + * Copyright (c) 2012-2022 Snowflake Computing Inc. All rights reserved. + */ +const Logger = require('./logger'); +const crypto = require('crypto'); +/** + * Constructor. + * + * @param id database id + * @param timestamp Server time when this entry read + * @param priority Priority of this entry w.r.t other ids + * @param context Opaque query context, used by query processor in the server. + */ + +/** Query context information. */ +function QueryContextElement(id,timestamp,priority,context) { + this.id = id; + this.timestamp = timestamp; + this.priority = priority; + this.context = context; + +} + +QueryContextElement.prototype.getId = function() { + return this.id; +}; + +QueryContextElement.prototype.getReadTimestamp = function() { + return this.timestamp; +}; + +QueryContextElement.prototype.getPriority = function() { + return this.priority; +}; + +QueryContextElement.prototype.getContext = function() { + return this.context; +}; + +QueryContextElement.prototype.setId = function(id) { + this.id = id; +}; + +QueryContextElement.prototype.setReadTimestamp = function(timestamp) { + this.timestamp = timestamp; +}; + +QueryContextElement.prototype.setPriority = function(priority) { + this.priority = priority; +}; + +QueryContextElement.prototype.setContext = function(context) { + this.context = context; +}; + + +QueryContextElement.prototype.equals = function(obj) { + if (!(obj instanceof QueryContextElement)) { + return false; + } + + return (this.id == obj.id + && this.timestamp == obj.timestamp + && this.priority == obj.priority + && this.context.equals(obj.context)); +}; + +QueryContextElement.prototype.hashCode=function() { + let hash = 31; + + hash = hash * 31 + parseInt(this.id); + hash += (hash * 31) + parseInt(this.timestamp); + hash += (hash * 31) + parseInt(this.priority); + hash += (hash * 31) + parseInt(crypto.createHash('md5').update(Buffer.from(this.context,'utf-8').toString()).digest(),2); + + return hash; +}; + +/** + * Most Recently Used and Priority based cache. A separate cache for each connection in the driver. + */ + +/** + * Constructor. + * + * @param capacity Maximum capacity of the cache. + */ + +function QueryContextCache(capacity) { + this.capacity = capacity; + this.idMap = new Map(); // Map for id and QCC + this.treeSet = new Set(); // Order data as per priority + this.priorityMap = new Map(); // Map for priority and QCC + this.newPriorityMap = new Map(); // Intermediate map for priority and QCC for current round of merging +} + + +/** + * Add an element in the cache. + * + * @param qce element to add + */ +QueryContextCache.prototype.sortTreeSet = function(){ + this.treeSet = new Set(Array.from(this.treeSet).sort((a,b)=>a.getPriority()-b.getPriority())); +}; + +QueryContextCache.prototype.addQCE= function(qce) { + this.idMap.set(qce.id,qce); + this.priorityMap.set(qce.priority,qce); + this.treeSet.add(qce); + this.sortTreeSet(); +}; + +/** + * Remove an element from the cache. + * + * @param qce element to remove. + */ +QueryContextCache.prototype.removeQCE = function(qce) { + + this.idMap.delete(qce.id); + this.priorityMap.delete(qce.priority); + this.treeSet.delete(qce); +}; + +/** + * Replace the cache element with a new response element. Remove old element exist in the cache + * and add a new element received. + * + * @param oldQCE an element exist in the cache + * @param newQCE a new element just received. + */ +QueryContextCache.prototype.replaceQCE = function(oldQCE, newQCE) { + // Remove old element from the cache + this.removeQCE(oldQCE); + // Add new element in the cache + this.addQCE(newQCE); +}; + +/** + * Merge a new element comes from the server with the existing cache. Merge is based on read time + * stamp for the same id and based on priority for two different ids. + * + * @param id Database id. + * @param timestamp Last time read metadata from FDB. + * @param priority 0 to N number, where 0 is the highest priority. Eviction policy is based on + * priority. + * @param context Opaque query context. + * + * + */ +QueryContextCache.prototype.merge = function(id, timestamp, priority, context) { + if (this.idMap.has(id)) { + // ID found in the cache + const qce = this.idMap.get(id); + if (timestamp > qce.getReadTimestamp()) { + if (qce.getPriority() === priority) { + + // Same priority, overwrite new data at same place + qce.setReadTimestamp(timestamp); + qce.setContext(context); + } else { + + // Change in priority + const newQCE = + new QueryContextElement(id, timestamp, priority, context); + + this.replaceQCE(qce, newQCE); + } // new priority + } // new data is recent + else if (timestamp === qce.getReadTimestamp() && qce.getPriority() !== priority) { + // Same read timestamp but change in priority + const newQCE = new QueryContextElement(id, timestamp, priority, context); + this.replaceQCE(qce, newQCE); + + } + } // id found + else { + // new id + if (this.priorityMap.has(priority)) { + + // Same priority with different id + const qce = this.priorityMap.get(priority); + // Replace with new data + const newQCE = new QueryContextElement(id, timestamp, priority, context); + + this.replaceQCE(qce, newQCE); + + } else { + // new priority + // Add new element in the cache + const newQCE = new QueryContextElement(id, timestamp, priority, context); + this.addQCE(newQCE); + + } + } +}; + +/** Sync the newPriorityMap with the priorityMap at the end of current round of merge */ +QueryContextCache.prototype.syncPriorityMap = function() { + Logger.getInstance().debug( + `syncPriorityMap called priorityMap size = ${this.priorityMap.size}, newPrioirtyMap size = ${this.newPriorityMap.size}` + ); + this.newPriorityMap.forEach((value,key,map)=>{ + this.priorityMap.set(map[key],map[value]); + }); + // clear the newPriorityMap for next round of QCC merge(a round consists of multiple entries) + this.newPriorityMap.clear(); +}; + + + + +/** + * After the merge, loop through priority list and make sure cache is at most capacity. Remove all + * other elements from the list based on priority. + */ +QueryContextCache.prototype.checkCacheCapacity = function() { + Logger.getInstance().debug( + `checkCacheCapacity() called. treeSet size ${this.treeSet.size} cache capacity ${this.capacity}` ); + // remove elements based on priority + while (this.treeSet.size > this.capacity) { + const qce = Array.from(this.treeSet).pop(); + this.removeQCE(qce); + } + Logger.getInstance().debug( + `checkCacheCapacity() returns. treeSet size ${this.treeSet.size} cache capacity ${this.capacity}`, + ); +}; + +/** Clear the cache. */ +QueryContextCache.prototype.clearCache = function() { + Logger.getInstance().debug('clearCache() called'); + this.idMap.clear(); + this.priorityMap.clear(); + this.treeSet.clear(); + Logger.getInstance().debug(`clearCache() returns. Number of entries in cache now ${this.treeSet.size}`,); +}; + + +QueryContextCache.prototype.getElements = function(){ + return this.treeSet; +}; + +/** + * @param data: the QueryContext Object serialized as a JSON format string + */ +QueryContextCache.prototype.deserializeQueryContext = function(data) { + Logger.getInstance().debug(`deserializeQueryContext() called: data from server: ${data}`); + this.logCacheEntries(); + + if (data == null || data.length === 0) { + // Clear the cache + this.clearCache(); + Logger.getInstance().debug('deserializeQueryContext() returns'); + this.logCacheEntries(); + return; + } + + try { + // Deserialize the entries. The first entry with priority is the main entry. On JDBC side, + // we save all entries into one list to simplify the logic. An example JSON is: + // { + // "entries": [ + // { + // "id": 0, + // "readtimestamp": 123456789, + // "priority": 0, + // "context": "base64 encoded context" + // }, + // { + // "id": 1, + // "readtimestamp": 123456789, + // "priority": 1, + // "context": "base64 encoded context" + // }, + // { + // "id": 2, + // "readtimestamp": 123456789, + // "priority": 2, + // "context": "base64 encoded context" + // } + // ] + + const entries = data.entries; + if (entries !== null && Array.isArray(entries)) { + for (const entryNode of entries) { + const entry = this.deserializeQueryContextElement(entryNode); + if (entry != null) { + this.merge(entry.getId(), entry.getReadTimestamp(), entry.getPriority(), entry.getContext()); + } else { + Logger.getInstance().warn( + 'deserializeQueryContextJson: deserializeQueryContextElement meets mismatch field type. Clear the QueryContextCache.'); + this.clearCache(); + return; + } + } + // after merging all entries, sync the internal priority map to priority map. Because of + // priority swicth from GS side, + // there could be priority key conflict if we directly operating on the priorityMap during + // a round of merge. + this.syncPriorityMap(); + } + } catch (e) { + Logger.getInstance().debug(`deserializeQueryContextJson: Exception = ${e.getMessage}`, ); + // Not rethrowing. clear the cache as incomplete merge can lead to unexpected behavior. + this.clearCache(); + } + + // After merging all entries, truncate to capacity + this.checkCacheCapacity(); + + // Log existing cache entries + this.logCacheEntries(); +}; // Synchronized + +QueryContextCache.prototype.deserializeQueryContextElement = function(node) { + const entry = new QueryContextElement(); + const idNode = node.id; + if (typeof idNode === 'number') { + entry.setId(idNode); + } else { + Logger.getInstance().warn('deserializeQueryContextElement: `id` field is not Number type'); + return null; + } + + const timestampNode = node.timestamp; + if (typeof timestampNode === 'number') { + entry.setReadTimestamp(timestampNode); + } else { + Logger.getInstance().warn('deserializeQueryContextElement: `timestamp` field is not Long type'); + return null; + } + + + const priorityNode = node.priority; + if (typeof priorityNode === 'number') { + entry.setPriority(priorityNode); + } else { + Logger.getInstance().warn('deserializeQueryContextElement: `priority` field is not Long type'); + return null; + } + + + const contextNode = node.context; + if (typeof contextNode === 'string') { + entry.setContext(contextNode); + + } else if (contextNode === null||contextNode === undefined) { + // Currenly the OpaqueContext field is empty in the JSON received from GS. In the future, it + // will + // be filled with OpaqueContext object in base64 format. + entry.setContext(null); + Logger.getInstance().debug('deserializeQueryContextElement `context` field is empty'); + } else { + Logger.getInstance().warn('deserializeQueryContextElement: `context` field is not String type'); + return null; + } + return entry; +}; + +QueryContextCache.prototype.logCacheEntries = function() { + if(Logger.getInstance().getLevel()===3){; + this.treeSet.forEach(function(elem) { + Logger.getInstance().debug( + `Cache Entry: id: ${elem.getId()} timestamp: ${elem.getReadTimestamp()} priority: ${elem.getPriority()}`, + ); + }); +} +}; + +QueryContextCache.prototype.getSize = function() { + return this.treeSet.size; +}; + +QueryContextCache.prototype.getQueryContextDTO = function (){ + + const arr = []; + const querycontexts =Array.from(this.getElements()); + for(let i=0; i10));' + sqlText: 'select count(*) from table(generator(timelimit=>10));', + queryContextDTO: { entries: [] } } }, output: @@ -1018,7 +1024,7 @@ function buildRequestOutputMappings(clientInfo) }, json: { - requestId: 'b97fee20-a805-11e5-a0ab-ddd3321ed586' + requestId: 'b97fee20-a805-11e5-a0ab-ddd3321ed586', } }, output: @@ -1052,7 +1058,8 @@ function buildRequestOutputMappings(clientInfo) json: { disableOfflineChunks: false, - sqlText: 'select \'too many concurrent queries\';' + sqlText: 'select \'too many concurrent queries\';', + queryContextDTO: { entries: [] } } }, output: @@ -1094,7 +1101,7 @@ function buildRequestOutputMappings(clientInfo) CLIENT_APP_VERSION: clientInfo.version, CLIENT_ENVIRONMENT: clientInfo.environment, SESSION_PARAMETERS: {} - } + }, } }, output: @@ -1192,7 +1199,8 @@ function buildRequestOutputMappings(clientInfo) json: { disableOfflineChunks: false, - sqlText: 'select * from faketable' + sqlText: 'select * from faketable', + queryContextDTO: { entries: [] } } }, output: @@ -1677,7 +1685,7 @@ function buildRequestOutputMappings(clientInfo) CLIENT_APP_VERSION: clientInfo.version, CLIENT_ENVIRONMENT: clientInfo.environment, SESSION_PARAMETERS: {} - } + }, } }, output: diff --git a/test/unit/query_context_cache_test.js b/test/unit/query_context_cache_test.js new file mode 100644 index 000000000..1eb699af6 --- /dev/null +++ b/test/unit/query_context_cache_test.js @@ -0,0 +1,216 @@ +const QueryContextCache = require('../../lib/queryContextCache.js'); +const assert = require('assert'); + +function TestingQCC(){ + this.qcc = null; + this.BASE_READ_TIMESTAMP = 1668727958; + this.CONTEXT = 'Some query Context'; + this.BASE_ID = 0; + this.BASE_PRIORITY = 0; + this.MAX_CAPACITY = 5; + this.expectedIDs; + this.expectedReadTimestamp; + this.expectedPriority; + + this.initCache = function() { + this.qcc = new QueryContextCache(this.MAX_CAPACITY); + }; + + this.initCacheWithData = function() { + this.initCacheWithDataWithContext(this.CONTEXT); + }; + + this.initCacheWithDataWithContext = function(Context) { + this.qcc = new QueryContextCache(this.MAX_CAPACITY); + this.expectedIDs = []; + this.expectedReadTimestamp = []; + this.expectedPriority = []; + for (let i = 0; i < this.MAX_CAPACITY; i++) { + this.expectedIDs[i] = this.BASE_ID + i; + this.expectedReadTimestamp[i] = this.BASE_READ_TIMESTAMP + i; + this.expectedPriority[i] = this.BASE_PRIORITY + i; + this.qcc.merge(this.expectedIDs[i], this.expectedReadTimestamp[i], this.expectedPriority[i], Context); + } + this.qcc.syncPriorityMap(); + }; + + this.initCacheWithDataInRandomOrder = function() { + this.qcc = new QueryContextCache(this.MAX_CAPACITY); + this.expectedIDs = []; + this.expectedReadTimestamp = []; + this.expectedPriority = []; + for (let i = 0; i < this.MAX_CAPACITY; i++) { + this.expectedIDs[i] = this.BASE_ID + i; + this.expectedReadTimestamp[i] = this.BASE_READ_TIMESTAMP + i; + this.expectedPriority[i] = this.BASE_PRIORITY + i; + } + + this.qcc.merge(this.expectedIDs[3], this.expectedReadTimestamp[3], this.expectedPriority[3], this.CONTEXT); + this.qcc.merge(this.expectedIDs[2], this.expectedReadTimestamp[2], this.expectedPriority[2], this.CONTEXT); + this.qcc.merge(this.expectedIDs[4], this.expectedReadTimestamp[4], this.expectedPriority[4], this.CONTEXT); + this.qcc.merge(this.expectedIDs[0], this.expectedReadTimestamp[0], this.expectedPriority[0], this.CONTEXT); + this.qcc.merge(this.expectedIDs[1], this.expectedReadTimestamp[1], this.expectedPriority[1], this.CONTEXT); + this.qcc.syncPriorityMap(); + }; + + this.assertCacheData = function() { + this.assertCacheDataWithContext(this.CONTEXT); + }; + + this.assertCacheDataWithContext = function(Context) { + const size = this.qcc.getSize(); + assert.strictEqual(size,this.MAX_CAPACITY); + const elements = Array.from(this.qcc.getElements()); + for (let i = 0; i < size; i++) { + assert.strictEqual(this.expectedIDs[i], elements[i].getId()); + assert.strictEqual(this.expectedReadTimestamp[i], elements[i].getReadTimestamp()); + assert.strictEqual(this.expectedPriority[i], elements[i].getPriority()); + assert.strictEqual(Context, elements[i].getContext()); + } + }; +} + +describe('QueryContextCacheTest', function() { + const testingQcc = new TestingQCC(); + + /** Test for empty cache */ + it('testIsEmpty',function(){ + testingQcc.initCache(); + assert.strictEqual(testingQcc.qcc.getSize(), 0); + }); + + it('testWithSomeData',function(){ + testingQcc.initCacheWithData(); + // Compare elements + + testingQcc.assertCacheData(); + }); + + it('testWithSomeDataInRandomOrder',function(){ + testingQcc.initCacheWithDataInRandomOrder(); + // Compare elements + testingQcc.assertCacheData(); + }); + it('testMoreThanCapacity',function() { + testingQcc.initCacheWithData(); + + // Add one more element at the end + const i = testingQcc.MAX_CAPACITY; + testingQcc.qcc.merge(testingQcc.BASE_ID + i, testingQcc.BASE_READ_TIMESTAMP + i, testingQcc.BASE_PRIORITY + i, testingQcc.CONTEXT); + testingQcc.qcc.syncPriorityMap(); + testingQcc.qcc.checkCacheCapacity(); + + // Compare elements + testingQcc.assertCacheData(); + }); + + it('testUpdateTimestamp',function() { + testingQcc.initCacheWithData(); + // Add one more element with new TS with existing id + const updatedID = 1; + testingQcc.expectedReadTimestamp[updatedID] = testingQcc.BASE_READ_TIMESTAMP + updatedID + 10; + testingQcc.qcc.merge( + testingQcc.BASE_ID + updatedID, testingQcc.expectedReadTimestamp[updatedID], testingQcc.BASE_PRIORITY + updatedID, testingQcc.CONTEXT); + testingQcc.qcc.syncPriorityMap(); + testingQcc.qcc.checkCacheCapacity(); + + // Compare elements + testingQcc.assertCacheData(); + }); + + it('testUpdatePriority', function() { + testingQcc.initCacheWithData(); + // Add one more element with new priority with existing id + const updatedID = 3; + const updatedPriority = testingQcc.BASE_PRIORITY + updatedID + 7; + + testingQcc.expectedPriority[updatedID] = updatedPriority; + testingQcc.qcc.merge( + testingQcc.BASE_ID + updatedID, testingQcc.BASE_READ_TIMESTAMP + updatedID, testingQcc.expectedPriority[updatedID], testingQcc.CONTEXT); + testingQcc.qcc.syncPriorityMap(); + testingQcc.qcc.checkCacheCapacity(); + + for (let i = updatedID; i < testingQcc.MAX_CAPACITY - 1; i++) { + testingQcc.expectedIDs[i] = testingQcc.expectedIDs[i + 1]; + testingQcc.expectedReadTimestamp[i] = testingQcc.expectedReadTimestamp[i + 1]; + testingQcc.expectedPriority[i] = testingQcc.expectedPriority[i + 1]; + } + + testingQcc.expectedIDs[testingQcc.MAX_CAPACITY - 1] = testingQcc.BASE_ID + updatedID; + testingQcc.expectedReadTimestamp[testingQcc.MAX_CAPACITY - 1] = testingQcc.BASE_READ_TIMESTAMP + updatedID; + testingQcc.expectedPriority[testingQcc.MAX_CAPACITY - 1] = updatedPriority; + testingQcc.assertCacheData(); + }); + + it('testAddSamePriority',function() { + testingQcc.initCacheWithData(); + + // Add one more element with same priority + const i = testingQcc.MAX_CAPACITY; + const UpdatedPriority = testingQcc.BASE_PRIORITY + 1; + testingQcc.qcc.merge(testingQcc.BASE_ID + i, testingQcc.BASE_READ_TIMESTAMP + i, UpdatedPriority, testingQcc.CONTEXT); + testingQcc.qcc.syncPriorityMap(); + testingQcc.qcc.checkCacheCapacity(); + testingQcc.expectedIDs[1] = testingQcc.BASE_ID + i; + testingQcc.expectedReadTimestamp[1] = testingQcc.BASE_READ_TIMESTAMP + i; + + // Compare elements + testingQcc.assertCacheData(); + }); + + it('testAddSameIDButStaleTimestamp', function(){ + testingQcc.initCacheWithData(); + // Add one more element with same priority + const i = 2; + testingQcc.qcc.merge(testingQcc.BASE_ID + i, testingQcc.BASE_READ_TIMESTAMP + i - 10, testingQcc.BASE_PRIORITY + i, testingQcc.CONTEXT); + testingQcc.qcc.syncPriorityMap(); + testingQcc.qcc.checkCacheCapacity(); + + // Compare elements + testingQcc.assertCacheData(); + }); + + it('testEmptyCacheWithNullData', function() { + testingQcc.initCacheWithData(); + testingQcc.qcc.deserializeQueryContext(null); + assert.strictEqual(testingQcc.qcc.getSize(),0,'Empty cache'); + }); + + it('testEmptyCacheWithEmptyResponseData',function() { + testingQcc.initCacheWithData(); + testingQcc.qcc.deserializeQueryContext(''); + assert.strictEqual(testingQcc.qcc.getSize(),0,'Empty cache'); + }); + + it('testSerializeRequestAndDeserializeResponseData', function() { + testingQcc.initCacheWithData(); + testingQcc.assertCacheData(); + + const queryContextDTO = testingQcc.qcc.getQueryContextDTO(); + // Clear testingQcc.qcc + testingQcc.qcc.clearCache(); + assert.strictEqual(testingQcc.qcc.getSize(),0,'Empty cache'); + + + testingQcc.qcc.deserializeQueryContext(queryContextDTO); + + testingQcc.assertCacheData(); + }); + + it('testSerializeRequestAndDeserializeResponseDataWithNulltestingCONTEXT', function() { + // Init testingQcc.qcc + testingQcc.initCacheWithDataWithContext(null); + testingQcc.assertCacheDataWithContext(null); + const queryContextDTO = testingQcc.qcc.getQueryContextDTO(); + //Clear testingQcc.qcc + testingQcc.qcc.clearCache(); + assert.strictEqual(testingQcc.qcc.getSize(), 0,'Empty cache'); + + testingQcc.qcc.deserializeQueryContext(queryContextDTO); + testingQcc.assertCacheDataWithContext(null); + }); +}); + + + + \ No newline at end of file From 050afdfe600b4c5986dbfcde69abf0e6d23bb7c5 Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-jy Date: Wed, 6 Sep 2023 15:33:28 -0700 Subject: [PATCH 02/30] fixed Parameter for Query Context Cache, and make serializeQueryContext function for the testing --- lib/connection/connection_config.js | 23 +++++++++++----- lib/queryContextCache.js | 38 +++++++++++++++++++-------- lib/services/sf.js | 2 +- test/unit/query_context_cache_test.js | 15 ++++++----- 4 files changed, 53 insertions(+), 25 deletions(-) diff --git a/lib/connection/connection_config.js b/lib/connection/connection_config.js index 802fd0b78..2f3857ea0 100644 --- a/lib/connection/connection_config.js +++ b/lib/connection/connection_config.js @@ -492,12 +492,12 @@ function ConnectionConfig(options, validateCredentials, qaMode, clientInfo) disableQueryContextCache = options.disableQueryContextCache; } - var queryContextCacheSize = 5; - if(Util.exists(options.queryContextCacheSize)){ - Errors.checkArgumentValid(Util.isNumber(options.queryContextCacheSize), - ErrorCodes.ERR_CONN_CREATE_INVALID_QUERY_CONTEXT_CACHE_SIZE); - queryContextCacheSize = options.queryContextCacheSize; - } + // var queryContextCacheSize = 5; + // if(Util.exists(options.queryContextCacheSize)){ + // Errors.checkArgumentValid(Util.isNumber(options.queryContextCacheSize), + // ErrorCodes.ERR_CONN_CREATE_INVALID_QUERY_CONTEXT_CACHE_SIZE); + // queryContextCacheSize = options.queryContextCacheSize; + // } /** * Returns an object that contains information about the proxy hostname, port, @@ -792,7 +792,6 @@ function ConnectionConfig(options, validateCredentials, qaMode, clientInfo) this.masterTokenExpirationTime = options.masterTokenExpirationTime; this.sessionTokenExpirationTime = options.sessionTokenExpirationTime; this.disableQueryContextCache = disableQueryContextCache; - this.queryContextCacheSize = queryContextCacheSize; // create the parameters array var parameters = createParameters(); @@ -956,6 +955,11 @@ function createParameters() defaultValue: 16, validate: isNonNegativeNumber }, + { + name: PARAM_QUERY_CONTEXT_CAHCE_SIZE, + defaultValue: 5, + validate: isNonNegativeNumber + }, ]; } @@ -1024,6 +1028,11 @@ ConnectionConfig.prototype.getRetrySfMaxSleepTime = function () return this._getParameterValue(PARAM_RETRY_SF_MAX_SLEEP_TIME); }; +ConnectionConfig.prototype.getQueryContextCacheSize = function () +{ + return this._getParameterValue(PARAM_QUERY_CONTEXT_CAHCE_SIZE); +}; + diff --git a/lib/queryContextCache.js b/lib/queryContextCache.js index e9c77fa5b..0a913a9cf 100644 --- a/lib/queryContextCache.js +++ b/lib/queryContextCache.js @@ -106,7 +106,7 @@ QueryContextCache.prototype.sortTreeSet = function(){ QueryContextCache.prototype.addQCE= function(qce) { this.idMap.set(qce.id,qce); - this.priorityMap.set(qce.priority,qce); + this.newPriorityMap.set(qce.priority,qce); this.treeSet.add(qce); this.sortTreeSet(); }; @@ -150,6 +150,7 @@ QueryContextCache.prototype.replaceQCE = function(oldQCE, newQCE) { * */ QueryContextCache.prototype.merge = function(id, timestamp, priority, context) { + if (this.idMap.has(id)) { // ID found in the cache const qce = this.idMap.get(id); @@ -201,8 +202,8 @@ QueryContextCache.prototype.syncPriorityMap = function() { Logger.getInstance().debug( `syncPriorityMap called priorityMap size = ${this.priorityMap.size}, newPrioirtyMap size = ${this.newPriorityMap.size}` ); - this.newPriorityMap.forEach((value,key,map)=>{ - this.priorityMap.set(map[key],map[value]); + this.newPriorityMap.forEach((value,key)=>{ + this.priorityMap.set(key,value); }); // clear the newPriorityMap for next round of QCC merge(a round consists of multiple entries) this.newPriorityMap.clear(); @@ -245,18 +246,22 @@ QueryContextCache.prototype.getElements = function(){ /** * @param data: the QueryContext Object serialized as a JSON format string */ -QueryContextCache.prototype.deserializeQueryContext = function(data) { +QueryContextCache.prototype.deserializeQueryContext = function(response) { + let data + try{ + data = JSON.parse(response); + }catch(e){ + data = null; + } Logger.getInstance().debug(`deserializeQueryContext() called: data from server: ${data}`); - this.logCacheEntries(); - - if (data == null || data.length === 0) { + if (!data||data.entries === null||typeof data === 'string') { // Clear the cache this.clearCache(); Logger.getInstance().debug('deserializeQueryContext() returns'); this.logCacheEntries(); return; } - +console.log("Type test pass"); try { // Deserialize the entries. The first entry with priority is the main entry. On JDBC side, // we save all entries into one list to simplify the logic. An example JSON is: @@ -344,12 +349,11 @@ QueryContextCache.prototype.deserializeQueryContextElement = function(node) { const contextNode = node.context; if (typeof contextNode === 'string') { + entry.setContext(contextNode); } else if (contextNode === null||contextNode === undefined) { - // Currenly the OpaqueContext field is empty in the JSON received from GS. In the future, it - // will - // be filled with OpaqueContext object in base64 format. + entry.setContext(null); Logger.getInstance().debug('deserializeQueryContextElement `context` field is empty'); } else { @@ -387,6 +391,18 @@ QueryContextCache.prototype.getQueryContextDTO = function (){ }; }; +QueryContextCache.prototype.getSerializeQueryContext = function(){ + const arr = []; + const querycontexts =Array.from(this.getElements()); + for(let i=0; i Date: Wed, 6 Sep 2023 16:47:02 -0700 Subject: [PATCH 03/30] Edited Unit testing for the serializeQueryContext and fix some erros --- lib/queryContextCache.js | 38 +++++++++++---------------- test/unit/query_context_cache_test.js | 7 ++--- 2 files changed, 18 insertions(+), 27 deletions(-) diff --git a/lib/queryContextCache.js b/lib/queryContextCache.js index 0a913a9cf..176edab04 100644 --- a/lib/queryContextCache.js +++ b/lib/queryContextCache.js @@ -246,22 +246,17 @@ QueryContextCache.prototype.getElements = function(){ /** * @param data: the QueryContext Object serialized as a JSON format string */ -QueryContextCache.prototype.deserializeQueryContext = function(response) { - let data - try{ - data = JSON.parse(response); - }catch(e){ - data = null; - } +QueryContextCache.prototype.deserializeQueryContext = function(data) { + Logger.getInstance().debug(`deserializeQueryContext() called: data from server: ${data}`); - if (!data||data.entries === null||typeof data === 'string') { + if (!data||JSON.stringify(data)==='{}'||data.entries === null) { // Clear the cache this.clearCache(); Logger.getInstance().debug('deserializeQueryContext() returns'); this.logCacheEntries(); return; } -console.log("Type test pass"); + try { // Deserialize the entries. The first entry with priority is the main entry. On JDBC side, // we save all entries into one list to simplify the logic. An example JSON is: @@ -364,30 +359,29 @@ QueryContextCache.prototype.deserializeQueryContextElement = function(node) { }; QueryContextCache.prototype.logCacheEntries = function() { - if(Logger.getInstance().getLevel()===3){; - this.treeSet.forEach(function(elem) { - Logger.getInstance().debug( - `Cache Entry: id: ${elem.getId()} timestamp: ${elem.getReadTimestamp()} priority: ${elem.getPriority()}`, - ); - }); -} + if(Logger.getInstance().getLevel()===3){ + this.treeSet.forEach(function(elem) { + Logger.getInstance().debug( + `Cache Entry: id: ${elem.getId()} timestamp: ${elem.getReadTimestamp()} priority: ${elem.getPriority()}`, + ); + }); + } }; QueryContextCache.prototype.getSize = function() { return this.treeSet.size; }; -QueryContextCache.prototype.getQueryContextDTO = function (){ +QueryContextCache.prototype.getQueryContextDTO = function () { const arr = []; const querycontexts =Array.from(this.getElements()); for(let i=0; i Date: Wed, 6 Sep 2023 16:49:38 -0700 Subject: [PATCH 04/30] Removed Error message for Query Context Cache because it is not a connection property --- lib/constants/error_messages.js | 1 - lib/errors.js | 1 - 2 files changed, 2 deletions(-) diff --git a/lib/constants/error_messages.js b/lib/constants/error_messages.js index 5195ad20c..248c3975d 100644 --- a/lib/constants/error_messages.js +++ b/lib/constants/error_messages.js @@ -65,7 +65,6 @@ exports[404038] = 'Invalid gcsUseDownscopedCredential. The specified value must exports[404039] = 'Invalid forceStageBindError. The specified value must be a number.'; exports[404040] = 'Invalid browser timeout value. The specified value must be a positive number.'; exports[404041] = 'Invalid disablQueryContextCache. The specified value must be a boolean.'; -exports[404042] = 'Invalid queryContextCacheSize. The specified value must be a positive number.' // 405001 exports[405001] = 'Invalid callback. The specified value must be a function.'; diff --git a/lib/errors.js b/lib/errors.js index 3432b6a3b..b2a941c0b 100644 --- a/lib/errors.js +++ b/lib/errors.js @@ -70,7 +70,6 @@ codes.ERR_CONN_CREATE_INVALID_GCS_USE_DOWNSCOPED_CREDENTIAL = 404038; codes.ERR_CONN_CREATE_INVALID_FORCE_STAGE_BIND_ERROR = 404039; codes.ERR_CONN_CREATE_INVALID_BROWSER_TIMEOUT = 404040; codes.ERR_CONN_CREATE_INVALID_DIABLED_QUERY_CONTEXT_CACHE = 404041 -codes.ERR_CONN_CREATE_INVALID_QUERY_CONTEXT_CACHE_SIZE =404042 // 405001 codes.ERR_CONN_CONNECT_INVALID_CALLBACK = 405001; From 3e059befc8b8ed5bae5d84a34669e36839ee9cfe Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-jy Date: Thu, 7 Sep 2023 18:00:14 -0700 Subject: [PATCH 05/30] Moved the point of initializing the querycontext cache, and fixed some errors --- lib/connection/connection_config.js | 24 +----------------------- lib/errors.js | 6 +++--- lib/parameters.js | 2 +- lib/queryContextCache.js | 4 ++-- lib/services/sf.js | 26 +++++++++++++++----------- 5 files changed, 22 insertions(+), 40 deletions(-) diff --git a/lib/connection/connection_config.js b/lib/connection/connection_config.js index 2f3857ea0..780820acb 100644 --- a/lib/connection/connection_config.js +++ b/lib/connection/connection_config.js @@ -488,17 +488,10 @@ function ConnectionConfig(options, validateCredentials, qaMode, clientInfo) var disableQueryContextCache = false; if (Util.exists(options.disableQueryContextCache)) { Errors.checkArgumentValid(Util.isBoolean(options.disableQueryContextCache), - ErrorCodes.ERR_CONN_CREATE_INVALID_DIABLED_QUERY_CONTEXT_CACHE); + ErrorCodes.ERR_CONN_CREATE_INVALID_DISABLED_QUERY_CONTEXT_CACHE); disableQueryContextCache = options.disableQueryContextCache; } - // var queryContextCacheSize = 5; - // if(Util.exists(options.queryContextCacheSize)){ - // Errors.checkArgumentValid(Util.isNumber(options.queryContextCacheSize), - // ErrorCodes.ERR_CONN_CREATE_INVALID_QUERY_CONTEXT_CACHE_SIZE); - // queryContextCacheSize = options.queryContextCacheSize; - // } - /** * Returns an object that contains information about the proxy hostname, port, * etc. for when http requests are made. @@ -868,7 +861,6 @@ const PARAM_RETRY_SF_MAX_LOGIN_RETRIES = 'sfRetryMaxLoginRetries'; const PARAM_RETRY_SF_MAX_NUM_RETRIES = 'sfRetryMaxNumRetries'; const PARAM_RETRY_SF_STARTING_SLEEP_TIME = 'sfRetryStartingSleepTime'; const PARAM_RETRY_SF_MAX_SLEEP_TIME = 'sfRetryMaxSleepTime'; -const PARAM_QUERY_CONTEXT_CAHCE_SIZE = 'queryContextCacheSize'; /** * Creates the list of known parameters. If a parameter is marked as external, @@ -955,11 +947,6 @@ function createParameters() defaultValue: 16, validate: isNonNegativeNumber }, - { - name: PARAM_QUERY_CONTEXT_CAHCE_SIZE, - defaultValue: 5, - validate: isNonNegativeNumber - }, ]; } @@ -1028,15 +1015,6 @@ ConnectionConfig.prototype.getRetrySfMaxSleepTime = function () return this._getParameterValue(PARAM_RETRY_SF_MAX_SLEEP_TIME); }; -ConnectionConfig.prototype.getQueryContextCacheSize = function () -{ - return this._getParameterValue(PARAM_QUERY_CONTEXT_CAHCE_SIZE); -}; - - - - - /** * Returns the value of a given connection config parameter. * diff --git a/lib/errors.js b/lib/errors.js index b2a941c0b..5ad788b54 100644 --- a/lib/errors.js +++ b/lib/errors.js @@ -69,7 +69,7 @@ codes.ERR_CONN_CREATE_INVALID_ARRAY_BINDING_THRESHOLD = 404037; codes.ERR_CONN_CREATE_INVALID_GCS_USE_DOWNSCOPED_CREDENTIAL = 404038; codes.ERR_CONN_CREATE_INVALID_FORCE_STAGE_BIND_ERROR = 404039; codes.ERR_CONN_CREATE_INVALID_BROWSER_TIMEOUT = 404040; -codes.ERR_CONN_CREATE_INVALID_DIABLED_QUERY_CONTEXT_CACHE = 404041 +codes.ERR_CONN_CREATE_INVALID_DISABLED_QUERY_CONTEXT_CACHE = 404041 // 405001 codes.ERR_CONN_CONNECT_INVALID_CALLBACK = 405001; @@ -114,8 +114,8 @@ codes.ERR_CONN_EXEC_STMT_INVALID_REQUEST_ID = 409013; // 410001 codes.ERR_CONN_FETCH_RESULT_MISSING_OPTIONS = 410001; codes.ERR_CONN_FETCH_RESULT_INVALID_OPTIONS = 410002; -codes.ERR_CONN_FETCH_RESULT_MISSING_STATEMENT_ID = 410003; -codes.ERR_CONN_FETCH_RESULT_INVALID_STATEMENT_ID = 410004; +codes.ERR_CONN_FETCH_RESULT_MISSING_QUERY_ID = 410003; +codes.ERR_CONN_FETCH_RESULT_INVALID_QUERY_ID = 410004; codes.ERR_CONN_FETCH_RESULT_INVALID_COMPLETE = 410005; codes.ERR_CONN_FETCH_RESULT_INVALID_STREAM_RESULT = 410006; codes.ERR_CONN_FETCH_RESULT_INVALID_FETCH_AS_STRING = 410007; diff --git a/lib/parameters.js b/lib/parameters.js index 276b81c66..01e3ca30a 100644 --- a/lib/parameters.js +++ b/lib/parameters.js @@ -111,7 +111,7 @@ var parameters = { name: names.QUERY_CONTEXT_CACHE_SIZE, value: 5, - desc: '' + desc: 'Query Context Cache Size' }), ]; diff --git a/lib/queryContextCache.js b/lib/queryContextCache.js index 176edab04..9ac133674 100644 --- a/lib/queryContextCache.js +++ b/lib/queryContextCache.js @@ -248,7 +248,7 @@ QueryContextCache.prototype.getElements = function(){ */ QueryContextCache.prototype.deserializeQueryContext = function(data) { - Logger.getInstance().debug(`deserializeQueryContext() called: data from server: ${data}`); + Logger.getInstance().debug(`deserializeQueryContext() called: data from server: ${JSON.stringify(data)}`); if (!data||JSON.stringify(data)==='{}'||data.entries === null) { // Clear the cache this.clearCache(); @@ -381,7 +381,7 @@ QueryContextCache.prototype.getQueryContextDTO = function () { } return { - entries: arr + entries: arr || [] }; }; diff --git a/lib/services/sf.js b/lib/services/sf.js index 6b712b2e4..ffc62fac7 100644 --- a/lib/services/sf.js +++ b/lib/services/sf.js @@ -93,10 +93,6 @@ function SnowflakeService(connectionConfig, httpClient, config) Errors.assertInternal(Util.isObject(httpClient)); Errors.assertInternal(!Util.exists(config) || Util.isObject(config)); - if(!connectionConfig.disableQueryContextCache){ - this.qcc = new QueryContextCache(connectionConfig.getQueryContextCacheSize()); - } - // if a config object was specified, verify // that it has all the information we need var tokenInfoConfig; @@ -537,25 +533,31 @@ function SnowflakeService(connectionConfig, httpClient, config) }; this.getQueryContextDTO = function () { - if(connectionConfig.disableQueryContextCache){ + if(!this.qcc){ return; } return this.qcc.getQueryContextDTO(); }; this.deserializeQueryContext = function (data){ - if(connectionConfig.disableQueryContextCache){ + if(!this.qcc){ return; } this.qcc.deserializeQueryContext(data); }; this.clearCache = function() { - if(connectionConfig.disableQueryContextCache){ + if(!this.qcc){ return; } this.qcc.clearCache(); } + + this.initializeQueryContextCache = function(size) { + if(!connectionConfig.disableQueryContextCache){ + this.qcc = new QueryContextCache(size); + } + } } @@ -1082,7 +1084,7 @@ StateConnecting.prototype.continue = function () var context = this.context; var err = context.options.err; var json = context.options.json; - + // if no json was specified, treat this as the first connect // and get the necessary information from connectionConfig if (!json) @@ -1155,9 +1157,7 @@ StateConnecting.prototype.continue = function () sessionParameters.SESSION_PARAMETERS.GCS_USE_DOWNSCOPED_CREDENTIAL = this.connectionConfig.getGcsUseDownscopedCredential(); } - - Util.apply(json.data, clientInfo); Util.apply(json.data, sessionParameters); @@ -1180,7 +1180,7 @@ StateConnecting.prototype.continue = function () { Errors.assertInternal(Util.exists(body)); Errors.assertInternal(Util.exists(body.data)); - + // update the parameters Parameters.update(body.data.parameters); @@ -1189,6 +1189,10 @@ StateConnecting.prototype.continue = function () // we're now connected parent.snowflakeService.transitionToConnected(); + + const qccSize = Parameters.getValue('QUERY_CONTEXT_CACHE_SIZE'); + parent.snowflakeService.initializeQueryContextCache(qccSize); + } else { From 543acb6f6214914249e94ac92f362cab84ce2984 Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-jy Date: Thu, 7 Sep 2023 19:21:42 -0700 Subject: [PATCH 06/30] Fixed getQueryContextDTO --- lib/queryContextCache.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/queryContextCache.js b/lib/queryContextCache.js index 9ac133674..2c4269d59 100644 --- a/lib/queryContextCache.js +++ b/lib/queryContextCache.js @@ -381,7 +381,7 @@ QueryContextCache.prototype.getQueryContextDTO = function () { } return { - entries: arr || [] + entries: arr }; }; From dc5ae8ca7d922bcd9d2c4123bef60341b829df5f Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-jy Date: Thu, 7 Sep 2023 19:53:34 -0700 Subject: [PATCH 07/30] fixed mock_http_client.js erro --- test/unit/mock/mock_http_client.js | 1 - 1 file changed, 1 deletion(-) diff --git a/test/unit/mock/mock_http_client.js b/test/unit/mock/mock_http_client.js index 2eeb20b7e..38a1c7c13 100644 --- a/test/unit/mock/mock_http_client.js +++ b/test/unit/mock/mock_http_client.js @@ -447,7 +447,6 @@ function buildRequestOutputMappings(clientInfo) { disableOfflineChunks: false, sqlText: 'select 1 as "c2";', - queryContextDTO: { entries: [] } } }, output: From a0418503375b5ea30dd53de2e823922d64251395 Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-jy Date: Fri, 8 Sep 2023 10:38:59 -0700 Subject: [PATCH 08/30] Removed unnecessary lines, and give some spaces between the codes and comments --- lib/connection/connection_config.js | 3 +- lib/connection/result/result.js | 6 +- lib/connection/statement.js | 9 +++ lib/queryContextCache.js | 81 ++++++++++++--------------- lib/services/sf.js | 9 +-- test/unit/mock/mock_http_client.js | 13 ++--- test/unit/query_context_cache_test.js | 54 +++++++++--------- 7 files changed, 86 insertions(+), 89 deletions(-) diff --git a/lib/connection/connection_config.js b/lib/connection/connection_config.js index 780820acb..c0e386e32 100644 --- a/lib/connection/connection_config.js +++ b/lib/connection/connection_config.js @@ -772,7 +772,6 @@ function ConnectionConfig(options, validateCredentials, qaMode, clientInfo) return disableQueryContextCache; } - // save config options this.username = options.username; this.password = options.password; @@ -946,7 +945,7 @@ function createParameters() name: PARAM_RETRY_SF_MAX_SLEEP_TIME, defaultValue: 16, validate: isNonNegativeNumber - }, + } ]; } diff --git a/lib/connection/result/result.js b/lib/connection/result/result.js index 655404352..50e15f425 100644 --- a/lib/connection/result/result.js +++ b/lib/connection/result/result.js @@ -126,9 +126,9 @@ function Result(options) { this._statement, this._services); - this.getQueryContext = function() { - return this._queryContext; - } + this.getQueryContext = function() { + return this._queryContext; + } /* Disable the ChunkCache until the implementation is complete. * diff --git a/lib/connection/statement.js b/lib/connection/statement.js index a93363dbf..f1e88d8f2 100644 --- a/lib/connection/statement.js +++ b/lib/connection/statement.js @@ -807,6 +807,14 @@ function createOnStatementRequestSuccRow(statement, context) rowMode: context.rowMode }); // save the query id + Logger.getInstance().trace('Here is JSON Response',{ + response: JSON.stringify(body), + statement: JSON.stringify(statement), + services: JSON.stringify(context.services), + connectionConfig: JSON.stringify(context.connectionConfig), + rowMode: JSON.stringify(context.rowMode), + queryContext:JSON.stringify(context.result.getQueryContext()) + }) context.queryId = context.result.getQueryId(); this.services.sf.deserializeQueryContext(context.result.getQueryContext()); } @@ -1339,6 +1347,7 @@ function sendRequestPreExec(statementContext, onResultAvailable) json.queryContextDTO = statementContext.services.sf.getQueryContextDTO(); } + Logger.getInstance().trace("Here is the constructed body",JSON.stringify(json)); // use the snowflake service to issue the request sendSfRequest(statementContext, { diff --git a/lib/queryContextCache.js b/lib/queryContextCache.js index 2c4269d59..d2a5b329a 100644 --- a/lib/queryContextCache.js +++ b/lib/queryContextCache.js @@ -13,48 +13,46 @@ const crypto = require('crypto'); */ /** Query context information. */ -function QueryContextElement(id,timestamp,priority,context) { +function QueryContextElement (id,timestamp,priority,context) { this.id = id; this.timestamp = timestamp; this.priority = priority; this.context = context; - } -QueryContextElement.prototype.getId = function() { +QueryContextElement.prototype.getId = function () { return this.id; }; -QueryContextElement.prototype.getReadTimestamp = function() { +QueryContextElement.prototype.getReadTimestamp = function () { return this.timestamp; }; -QueryContextElement.prototype.getPriority = function() { +QueryContextElement.prototype.getPriority = function () { return this.priority; }; -QueryContextElement.prototype.getContext = function() { +QueryContextElement.prototype.getContext = function () { return this.context; }; -QueryContextElement.prototype.setId = function(id) { +QueryContextElement.prototype.setId = function (id) { this.id = id; }; -QueryContextElement.prototype.setReadTimestamp = function(timestamp) { +QueryContextElement.prototype.setReadTimestamp = function (timestamp) { this.timestamp = timestamp; }; -QueryContextElement.prototype.setPriority = function(priority) { +QueryContextElement.prototype.setPriority = function (priority) { this.priority = priority; }; -QueryContextElement.prototype.setContext = function(context) { +QueryContextElement.prototype.setContext = function (context) { this.context = context; }; - -QueryContextElement.prototype.equals = function(obj) { +QueryContextElement.prototype.equals = function (obj) { if (!(obj instanceof QueryContextElement)) { return false; } @@ -65,9 +63,8 @@ QueryContextElement.prototype.equals = function(obj) { && this.context.equals(obj.context)); }; -QueryContextElement.prototype.hashCode=function() { +QueryContextElement.prototype.hashCode = function () { let hash = 31; - hash = hash * 31 + parseInt(this.id); hash += (hash * 31) + parseInt(this.timestamp); hash += (hash * 31) + parseInt(this.priority); @@ -86,7 +83,7 @@ QueryContextElement.prototype.hashCode=function() { * @param capacity Maximum capacity of the cache. */ -function QueryContextCache(capacity) { +function QueryContextCache (capacity) { this.capacity = capacity; this.idMap = new Map(); // Map for id and QCC this.treeSet = new Set(); // Order data as per priority @@ -94,17 +91,16 @@ function QueryContextCache(capacity) { this.newPriorityMap = new Map(); // Intermediate map for priority and QCC for current round of merging } - /** * Add an element in the cache. * * @param qce element to add */ -QueryContextCache.prototype.sortTreeSet = function(){ +QueryContextCache.prototype.sortTreeSet = function () { this.treeSet = new Set(Array.from(this.treeSet).sort((a,b)=>a.getPriority()-b.getPriority())); }; -QueryContextCache.prototype.addQCE= function(qce) { +QueryContextCache.prototype.addQCE= function (qce) { this.idMap.set(qce.id,qce); this.newPriorityMap.set(qce.priority,qce); this.treeSet.add(qce); @@ -116,8 +112,7 @@ QueryContextCache.prototype.addQCE= function(qce) { * * @param qce element to remove. */ -QueryContextCache.prototype.removeQCE = function(qce) { - +QueryContextCache.prototype.removeQCE = function (qce) { this.idMap.delete(qce.id); this.priorityMap.delete(qce.priority); this.treeSet.delete(qce); @@ -130,7 +125,7 @@ QueryContextCache.prototype.removeQCE = function(qce) { * @param oldQCE an element exist in the cache * @param newQCE a new element just received. */ -QueryContextCache.prototype.replaceQCE = function(oldQCE, newQCE) { +QueryContextCache.prototype.replaceQCE = function (oldQCE, newQCE) { // Remove old element from the cache this.removeQCE(oldQCE); // Add new element in the cache @@ -149,9 +144,9 @@ QueryContextCache.prototype.replaceQCE = function(oldQCE, newQCE) { * * */ -QueryContextCache.prototype.merge = function(id, timestamp, priority, context) { - +QueryContextCache.prototype.merge = function (id, timestamp, priority, context) { if (this.idMap.has(id)) { + // ID found in the cache const qce = this.idMap.get(id); if (timestamp > qce.getReadTimestamp()) { @@ -173,52 +168,50 @@ QueryContextCache.prototype.merge = function(id, timestamp, priority, context) { // Same read timestamp but change in priority const newQCE = new QueryContextElement(id, timestamp, priority, context); this.replaceQCE(qce, newQCE); - } } // id found else { + // new id if (this.priorityMap.has(priority)) { // Same priority with different id const qce = this.priorityMap.get(priority); + // Replace with new data const newQCE = new QueryContextElement(id, timestamp, priority, context); - this.replaceQCE(qce, newQCE); - } else { + // new priority // Add new element in the cache const newQCE = new QueryContextElement(id, timestamp, priority, context); this.addQCE(newQCE); - } } }; /** Sync the newPriorityMap with the priorityMap at the end of current round of merge */ -QueryContextCache.prototype.syncPriorityMap = function() { +QueryContextCache.prototype.syncPriorityMap = function () { Logger.getInstance().debug( `syncPriorityMap called priorityMap size = ${this.priorityMap.size}, newPrioirtyMap size = ${this.newPriorityMap.size}` ); this.newPriorityMap.forEach((value,key)=>{ this.priorityMap.set(key,value); }); + // clear the newPriorityMap for next round of QCC merge(a round consists of multiple entries) this.newPriorityMap.clear(); }; - - - /** * After the merge, loop through priority list and make sure cache is at most capacity. Remove all * other elements from the list based on priority. */ -QueryContextCache.prototype.checkCacheCapacity = function() { +QueryContextCache.prototype.checkCacheCapacity = function () { Logger.getInstance().debug( `checkCacheCapacity() called. treeSet size ${this.treeSet.size} cache capacity ${this.capacity}` ); + // remove elements based on priority while (this.treeSet.size > this.capacity) { const qce = Array.from(this.treeSet).pop(); @@ -230,7 +223,7 @@ QueryContextCache.prototype.checkCacheCapacity = function() { }; /** Clear the cache. */ -QueryContextCache.prototype.clearCache = function() { +QueryContextCache.prototype.clearCache = function () { Logger.getInstance().debug('clearCache() called'); this.idMap.clear(); this.priorityMap.clear(); @@ -238,18 +231,17 @@ QueryContextCache.prototype.clearCache = function() { Logger.getInstance().debug(`clearCache() returns. Number of entries in cache now ${this.treeSet.size}`,); }; - -QueryContextCache.prototype.getElements = function(){ +QueryContextCache.prototype.getElements = function () { return this.treeSet; }; /** - * @param data: the QueryContext Object serialized as a JSON format string - */ -QueryContextCache.prototype.deserializeQueryContext = function(data) { - + * @param data: the QueryContext Object serialized as a JSON format string + */ +QueryContextCache.prototype.deserializeQueryContext = function (data) { Logger.getInstance().debug(`deserializeQueryContext() called: data from server: ${JSON.stringify(data)}`); if (!data||JSON.stringify(data)==='{}'||data.entries === null) { + // Clear the cache this.clearCache(); Logger.getInstance().debug('deserializeQueryContext() returns'); @@ -303,6 +295,7 @@ QueryContextCache.prototype.deserializeQueryContext = function(data) { } } catch (e) { Logger.getInstance().debug(`deserializeQueryContextJson: Exception = ${e.getMessage}`, ); + // Not rethrowing. clear the cache as incomplete merge can lead to unexpected behavior. this.clearCache(); } @@ -314,7 +307,7 @@ QueryContextCache.prototype.deserializeQueryContext = function(data) { this.logCacheEntries(); }; // Synchronized -QueryContextCache.prototype.deserializeQueryContextElement = function(node) { +QueryContextCache.prototype.deserializeQueryContextElement = function (node) { const entry = new QueryContextElement(); const idNode = node.id; if (typeof idNode === 'number') { @@ -332,7 +325,6 @@ QueryContextCache.prototype.deserializeQueryContextElement = function(node) { return null; } - const priorityNode = node.priority; if (typeof priorityNode === 'number') { entry.setPriority(priorityNode); @@ -341,7 +333,6 @@ QueryContextCache.prototype.deserializeQueryContextElement = function(node) { return null; } - const contextNode = node.context; if (typeof contextNode === 'string') { @@ -358,7 +349,7 @@ QueryContextCache.prototype.deserializeQueryContextElement = function(node) { return entry; }; -QueryContextCache.prototype.logCacheEntries = function() { +QueryContextCache.prototype.logCacheEntries = function () { if(Logger.getInstance().getLevel()===3){ this.treeSet.forEach(function(elem) { Logger.getInstance().debug( @@ -373,7 +364,6 @@ QueryContextCache.prototype.getSize = function() { }; QueryContextCache.prototype.getQueryContextDTO = function () { - const arr = []; const querycontexts =Array.from(this.getElements()); for(let i=0; i Date: Tue, 12 Sep 2023 10:46:03 -0700 Subject: [PATCH 09/30] removed some dead codes --- lib/connection/connection_config.js | 7 +++---- lib/queryContextCache.js | 14 +++++++------- lib/services/sf.js | 2 +- test/unit/query_context_cache_test.js | 1 + 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/lib/connection/connection_config.js b/lib/connection/connection_config.js index c0e386e32..94c9016ef 100644 --- a/lib/connection/connection_config.js +++ b/lib/connection/connection_config.js @@ -485,7 +485,7 @@ function ConnectionConfig(options, validateCredentials, qaMode, clientInfo) } } - var disableQueryContextCache = false; + let disableQueryContextCache = false; if (Util.exists(options.disableQueryContextCache)) { Errors.checkArgumentValid(Util.isBoolean(options.disableQueryContextCache), ErrorCodes.ERR_CONN_CREATE_INVALID_DISABLED_QUERY_CONTEXT_CACHE); @@ -764,9 +764,9 @@ function ConnectionConfig(options, validateCredentials, qaMode, clientInfo) return forceStageBindError; }; /** - * Returns the status of the query contextcache. + * Returns the status of the query context cache. * - * @returns {Number} + * @returns {Boolean} */ this.getDisableQueryContextCache = function () { return disableQueryContextCache; @@ -783,7 +783,6 @@ function ConnectionConfig(options, validateCredentials, qaMode, clientInfo) this.masterToken = options.masterToken; this.masterTokenExpirationTime = options.masterTokenExpirationTime; this.sessionTokenExpirationTime = options.sessionTokenExpirationTime; - this.disableQueryContextCache = disableQueryContextCache; // create the parameters array var parameters = createParameters(); diff --git a/lib/queryContextCache.js b/lib/queryContextCache.js index d2a5b329a..ccb6bb7a3 100644 --- a/lib/queryContextCache.js +++ b/lib/queryContextCache.js @@ -57,9 +57,9 @@ QueryContextElement.prototype.equals = function (obj) { return false; } - return (this.id == obj.id - && this.timestamp == obj.timestamp - && this.priority == obj.priority + return (this.id === obj.id + && this.timestamp === obj.timestamp + && this.priority === obj.priority && this.context.equals(obj.context)); }; @@ -338,7 +338,7 @@ QueryContextCache.prototype.deserializeQueryContextElement = function (node) { entry.setContext(contextNode); - } else if (contextNode === null||contextNode === undefined) { + } else if (contextNode === null || contextNode === undefined) { entry.setContext(null); Logger.getInstance().debug('deserializeQueryContextElement `context` field is empty'); @@ -350,7 +350,7 @@ QueryContextCache.prototype.deserializeQueryContextElement = function (node) { }; QueryContextCache.prototype.logCacheEntries = function () { - if(Logger.getInstance().getLevel()===3){ + if(Logger.getInstance().getLevel() === 3){ this.treeSet.forEach(function(elem) { Logger.getInstance().debug( `Cache Entry: id: ${elem.getId()} timestamp: ${elem.getReadTimestamp()} priority: ${elem.getPriority()}`, @@ -366,7 +366,7 @@ QueryContextCache.prototype.getSize = function() { QueryContextCache.prototype.getQueryContextDTO = function () { const arr = []; const querycontexts =Array.from(this.getElements()); - for(let i=0; i Date: Tue, 12 Sep 2023 10:47:12 -0700 Subject: [PATCH 10/30] removed a dead code --- lib/connection/connection_config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/connection/connection_config.js b/lib/connection/connection_config.js index 94c9016ef..f644de980 100644 --- a/lib/connection/connection_config.js +++ b/lib/connection/connection_config.js @@ -764,7 +764,7 @@ function ConnectionConfig(options, validateCredentials, qaMode, clientInfo) return forceStageBindError; }; /** - * Returns the status of the query context cache. + * Returns whether the Query Context Cache is enabled or not by the configuration * * @returns {Boolean} */ From d2a1ca336d22cdfac9dbb7d57cb7de6eadee0161 Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-jy Date: Tue, 12 Sep 2023 18:38:45 -0700 Subject: [PATCH 11/30] added connction testing for disableQueryContextCache --- .../unit/connection/connection_config_test.js | 29 ++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/test/unit/connection/connection_config_test.js b/test/unit/connection/connection_config_test.js index 5a15d52a1..b8629023c 100644 --- a/test/unit/connection/connection_config_test.js +++ b/test/unit/connection/connection_config_test.js @@ -514,6 +514,17 @@ describe('ConnectionConfig: basic', function () }, errorCode: ErrorCodes.ERR_CONN_CREATE_INVALID_GCS_USE_DOWNSCOPED_CREDENTIAL }, + { + name: 'invalid disableQueryContextCache', + options: + { + username: 'username', + password: 'password', + account: 'account', + disableQueryContextCache: 1234 + }, + errorCode: ErrorCodes.ERR_CONN_CREATE_INVALID_DISABLED_QUERY_CONTEXT_CACHE + }, ]; var createNegativeITCallback = function (testCase) @@ -847,7 +858,23 @@ describe('ConnectionConfig: basic', function () accessUrl: 'https://account.snowflakecomputing.com', account: 'account' } - } + }, + { + name: 'disableQueryContextCache', + input: + { + username: 'username', + password: 'password', + account: 'account', + disableQueryContextCache: true + }, + options: + { + accessUrl: 'https://account.snowflakecomputing.com', + username: 'username', + password: 'password' + } + }, ]; var createItCallback = function (testCase) From bebcf0576a6c0ba9123e6925f8e388367eae70f9 Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-jy Date: Wed, 13 Sep 2023 19:34:22 -0700 Subject: [PATCH 12/30] Fixed typos, formats, comments --- lib/connection/connection_config.js | 26 +++--- lib/queryContextCache.js | 66 +++++++-------- test/unit/query_context_cache_test.js | 116 +++++++++++++------------- 3 files changed, 96 insertions(+), 112 deletions(-) diff --git a/lib/connection/connection_config.js b/lib/connection/connection_config.js index f644de980..3b77cab54 100644 --- a/lib/connection/connection_config.js +++ b/lib/connection/connection_config.js @@ -49,7 +49,7 @@ const DEFAULT_PARAMS = 'arrayBindingThreshold', 'gcsUseDownscopedCredential', 'forceStageBindError', - 'disableQueryContextCache' + 'disableQueryContextCache', ]; function consolidateHostAndAccount(options) @@ -469,6 +469,15 @@ function ConnectionConfig(options, validateCredentials, qaMode, clientInfo) forceStageBindError = options.forceStageBindError; } + let disableQueryContextCache = false; + if (Util.exists(options.disableQueryContextCache)) { + + Errors.checkArgumentValid(Util.isBoolean(options.disableQueryContextCache), + ErrorCodes.ERR_CONN_CREATE_INVALID_DISABLED_QUERY_CONTEXT_CACHE); + + disableQueryContextCache = options.disableQueryContextCache; + } + if (validateDefaultParameters) { for (const [key] of Object.entries(options)) @@ -485,13 +494,6 @@ function ConnectionConfig(options, validateCredentials, qaMode, clientInfo) } } - let disableQueryContextCache = false; - if (Util.exists(options.disableQueryContextCache)) { - Errors.checkArgumentValid(Util.isBoolean(options.disableQueryContextCache), - ErrorCodes.ERR_CONN_CREATE_INVALID_DISABLED_QUERY_CONTEXT_CACHE); - disableQueryContextCache = options.disableQueryContextCache; - } - /** * Returns an object that contains information about the proxy hostname, port, * etc. for when http requests are made. @@ -755,14 +757,6 @@ function ConnectionConfig(options, validateCredentials, qaMode, clientInfo) return forceStageBindError; }; - /** - * Returns the force stage bind error - * - * @returns {Number} - */ - this.getForceStageBindError = function () { - return forceStageBindError; - }; /** * Returns whether the Query Context Cache is enabled or not by the configuration * diff --git a/lib/queryContextCache.js b/lib/queryContextCache.js index ccb6bb7a3..e845ff232 100644 --- a/lib/queryContextCache.js +++ b/lib/queryContextCache.js @@ -3,16 +3,14 @@ */ const Logger = require('./logger'); const crypto = require('crypto'); + /** - * Constructor. - * - * @param id database id - * @param timestamp Server time when this entry read - * @param priority Priority of this entry w.r.t other ids - * @param context Opaque query context, used by query processor in the server. - */ - -/** Query context information. */ + * + * @param {String} id + * @param {Number} timestamp + * @param {Number} priority + * @param {String} context + */ function QueryContextElement (id,timestamp,priority,context) { this.id = id; this.timestamp = timestamp; @@ -78,10 +76,9 @@ QueryContextElement.prototype.hashCode = function () { */ /** - * Constructor. - * - * @param capacity Maximum capacity of the cache. - */ + * + * @param {Number} capacity Maximum capacity of the cache. + */ function QueryContextCache (capacity) { this.capacity = capacity; @@ -91,11 +88,6 @@ function QueryContextCache (capacity) { this.newPriorityMap = new Map(); // Intermediate map for priority and QCC for current round of merging } -/** - * Add an element in the cache. - * - * @param qce element to add - */ QueryContextCache.prototype.sortTreeSet = function () { this.treeSet = new Set(Array.from(this.treeSet).sort((a,b)=>a.getPriority()-b.getPriority())); }; @@ -110,7 +102,7 @@ QueryContextCache.prototype.addQCE= function (qce) { /** * Remove an element from the cache. * - * @param qce element to remove. + * @param {Object} qce element to remove. */ QueryContextCache.prototype.removeQCE = function (qce) { this.idMap.delete(qce.id); @@ -121,11 +113,12 @@ QueryContextCache.prototype.removeQCE = function (qce) { /** * Replace the cache element with a new response element. Remove old element exist in the cache * and add a new element received. - * - * @param oldQCE an element exist in the cache - * @param newQCE a new element just received. + *{ + * @param {Object} oldQCE an element exist in the cache + * @param {Object} newQCE a new element just received. */ QueryContextCache.prototype.replaceQCE = function (oldQCE, newQCE) { + // Remove old element from the cache this.removeQCE(oldQCE); // Add new element in the cache @@ -136,12 +129,10 @@ QueryContextCache.prototype.replaceQCE = function (oldQCE, newQCE) { * Merge a new element comes from the server with the existing cache. Merge is based on read time * stamp for the same id and based on priority for two different ids. * - * @param id Database id. - * @param timestamp Last time read metadata from FDB. - * @param priority 0 to N number, where 0 is the highest priority. Eviction policy is based on - * priority. - * @param context Opaque query context. - * + * @param {Number} id + * @param {Number} timestamp + * @param {Number} priority + * @param {String} context * */ QueryContextCache.prototype.merge = function (id, timestamp, priority, context) { @@ -162,8 +153,8 @@ QueryContextCache.prototype.merge = function (id, timestamp, priority, context) new QueryContextElement(id, timestamp, priority, context); this.replaceQCE(qce, newQCE); - } // new priority - } // new data is recent + } + } else if (timestamp === qce.getReadTimestamp() && qce.getPriority() !== priority) { // Same read timestamp but change in priority const newQCE = new QueryContextElement(id, timestamp, priority, context); @@ -194,7 +185,7 @@ QueryContextCache.prototype.merge = function (id, timestamp, priority, context) /** Sync the newPriorityMap with the priorityMap at the end of current round of merge */ QueryContextCache.prototype.syncPriorityMap = function () { Logger.getInstance().debug( - `syncPriorityMap called priorityMap size = ${this.priorityMap.size}, newPrioirtyMap size = ${this.newPriorityMap.size}` + `syncPriorityMap called priorityMap size = ${this.priorityMap.size}, newPriorityMap size = ${this.newPriorityMap.size}` ); this.newPriorityMap.forEach((value,key)=>{ this.priorityMap.set(key,value); @@ -242,7 +233,6 @@ QueryContextCache.prototype.deserializeQueryContext = function (data) { Logger.getInstance().debug(`deserializeQueryContext() called: data from server: ${JSON.stringify(data)}`); if (!data||JSON.stringify(data)==='{}'||data.entries === null) { - // Clear the cache this.clearCache(); Logger.getInstance().debug('deserializeQueryContext() returns'); this.logCacheEntries(); @@ -250,8 +240,7 @@ QueryContextCache.prototype.deserializeQueryContext = function (data) { } try { - // Deserialize the entries. The first entry with priority is the main entry. On JDBC side, - // we save all entries into one list to simplify the logic. An example JSON is: + // Deserialize the entries. The first entry with priority is the main entry. An example JSON is: // { // "entries": [ // { @@ -321,7 +310,7 @@ QueryContextCache.prototype.deserializeQueryContextElement = function (node) { if (typeof timestampNode === 'number') { entry.setReadTimestamp(timestampNode); } else { - Logger.getInstance().warn('deserializeQueryContextElement: `timestamp` field is not Long type'); + Logger.getInstance().warn('deserializeQueryContextElement: `timestamp` field is not Number type'); return null; } @@ -329,7 +318,7 @@ QueryContextCache.prototype.deserializeQueryContextElement = function (node) { if (typeof priorityNode === 'number') { entry.setPriority(priorityNode); } else { - Logger.getInstance().warn('deserializeQueryContextElement: `priority` field is not Long type'); + Logger.getInstance().warn('deserializeQueryContextElement: `priority` field is not Number type'); return null; } @@ -350,6 +339,8 @@ QueryContextCache.prototype.deserializeQueryContextElement = function (node) { }; QueryContextCache.prototype.logCacheEntries = function () { + + //Debug Level only if(Logger.getInstance().getLevel() === 3){ this.treeSet.forEach(function(elem) { Logger.getInstance().debug( @@ -367,7 +358,8 @@ QueryContextCache.prototype.getQueryContextDTO = function () { const arr = []; const querycontexts =Array.from(this.getElements()); for (let i = 0; i < this.treeSet.size; i++) { - arr.push({id:querycontexts[i].id,timestamp:querycontexts[i].timestamp,priority:querycontexts[i].priority,context:{base64Data:querycontexts[i].context}||null}); + arr.push({id:querycontexts[i].id, timestamp:querycontexts[i].timestamp, + priority:querycontexts[i].priority, context:{base64Data:querycontexts[i].context}||null}); } return { diff --git a/test/unit/query_context_cache_test.js b/test/unit/query_context_cache_test.js index c6cf2ab1d..f31e0473b 100644 --- a/test/unit/query_context_cache_test.js +++ b/test/unit/query_context_cache_test.js @@ -1,65 +1,68 @@ const QueryContextCache = require('../../lib/queryContextCache.js'); const assert = require('assert'); +const BASE_ID = 0; +const BASE_PRIORITY = 0; +const BASE_READ_TIMESTAMP = 1668727958; +const CONTEXT = 'Some query Context'; +const MAX_CAPACITY = 5; + + function TestingQCC () { this.qcc = null; - this.BASE_READ_TIMESTAMP = 1668727958; - this.CONTEXT = 'Some query Context'; - this.BASE_ID = 0; - this.BASE_PRIORITY = 0; - this.MAX_CAPACITY = 5; + this.expectedIDs; this.expectedReadTimestamp; this.expectedPriority; this.initCache = function () { - this.qcc = new QueryContextCache(this.MAX_CAPACITY); + this.qcc = new QueryContextCache(MAX_CAPACITY); }; this.initCacheWithData = function () { - this.initCacheWithDataWithContext(this.CONTEXT); + this.initCacheWithDataWithContext(CONTEXT); }; this.initCacheWithDataWithContext = function (Context) { - this.qcc = new QueryContextCache(this.MAX_CAPACITY); + this.qcc = new QueryContextCache(MAX_CAPACITY); this.expectedIDs = []; this.expectedReadTimestamp = []; this.expectedPriority = []; - for (let i = 0; i < this.MAX_CAPACITY; i++) { - this.expectedIDs[i] = this.BASE_ID + i; - this.expectedReadTimestamp[i] = this.BASE_READ_TIMESTAMP + i; - this.expectedPriority[i] = this.BASE_PRIORITY + i; + for (let i = 0; i < MAX_CAPACITY; i++) { + this.expectedIDs[i] = BASE_ID + i; + this.expectedReadTimestamp[i] = BASE_READ_TIMESTAMP + i; + this.expectedPriority[i] = BASE_PRIORITY + i; this.qcc.merge(this.expectedIDs[i], this.expectedReadTimestamp[i], this.expectedPriority[i], Context); } this.qcc.syncPriorityMap(); }; this.initCacheWithDataInRandomOrder = function () { - this.qcc = new QueryContextCache(this.MAX_CAPACITY); + this.qcc = new QueryContextCache(MAX_CAPACITY); this.expectedIDs = []; this.expectedReadTimestamp = []; this.expectedPriority = []; - for (let i = 0; i < this.MAX_CAPACITY; i++) { - this.expectedIDs[i] = this.BASE_ID + i; - this.expectedReadTimestamp[i] = this.BASE_READ_TIMESTAMP + i; - this.expectedPriority[i] = this.BASE_PRIORITY + i; + for (let i = 0; i < MAX_CAPACITY; i++) { + this.expectedIDs[i] = BASE_ID + i; + this.expectedReadTimestamp[i] = BASE_READ_TIMESTAMP + i; + this.expectedPriority[i] = BASE_PRIORITY + i; } - this.qcc.merge(this.expectedIDs[3], this.expectedReadTimestamp[3], this.expectedPriority[3], this.CONTEXT); - this.qcc.merge(this.expectedIDs[2], this.expectedReadTimestamp[2], this.expectedPriority[2], this.CONTEXT); - this.qcc.merge(this.expectedIDs[4], this.expectedReadTimestamp[4], this.expectedPriority[4], this.CONTEXT); - this.qcc.merge(this.expectedIDs[0], this.expectedReadTimestamp[0], this.expectedPriority[0], this.CONTEXT); - this.qcc.merge(this.expectedIDs[1], this.expectedReadTimestamp[1], this.expectedPriority[1], this.CONTEXT); + this.qcc.merge(this.expectedIDs[3], this.expectedReadTimestamp[3], this.expectedPriority[3], CONTEXT); + this.qcc.merge(this.expectedIDs[2], this.expectedReadTimestamp[2], this.expectedPriority[2], CONTEXT); + this.qcc.merge(this.expectedIDs[4], this.expectedReadTimestamp[4], this.expectedPriority[4], CONTEXT); + this.qcc.merge(this.expectedIDs[0], this.expectedReadTimestamp[0], this.expectedPriority[0], CONTEXT); + this.qcc.merge(this.expectedIDs[1], this.expectedReadTimestamp[1], this.expectedPriority[1], CONTEXT); this.qcc.syncPriorityMap(); }; this.assertCacheData = function () { - this.assertCacheDataWithContext(this.CONTEXT); + this.assertCacheDataWithContext(CONTEXT); }; this.assertCacheDataWithContext = function (Context) { const size = this.qcc.getSize(); - assert.strictEqual(size,this.MAX_CAPACITY); + assert.strictEqual(size,MAX_CAPACITY); const elements = Array.from(this.qcc.getElements()); for (let i = 0; i < size; i++) { assert.strictEqual(this.expectedIDs[i], elements[i].getId()); @@ -70,35 +73,35 @@ function TestingQCC () { }; } -describe('QueryContextCacheTest', function () { +describe('Query Context Cache Test', function () { const testingQcc = new TestingQCC(); /** Test for empty cache */ - it('testIsEmpty',function () { + it('test - the cache is empty',function () { testingQcc.initCache(); assert.strictEqual(testingQcc.qcc.getSize(), 0); }); - it('testWithSomeData',function () { + it('test - some elements in the cache',function () { testingQcc.initCacheWithData(); // Compare elements testingQcc.assertCacheData(); }); - it('testWithSomeDataInRandomOrder',function () { + it('test - query contexts are randomly added in the cache',function () { testingQcc.initCacheWithDataInRandomOrder(); // Compare elements testingQcc.assertCacheData(); }); - it('testMoreThanCapacity',function () { + it('test - the number of contexts is over the size of capacity',function () { testingQcc.initCacheWithData(); // Add one more element at the end - const i = testingQcc.MAX_CAPACITY; - testingQcc.qcc.merge(testingQcc.BASE_ID + i, testingQcc.BASE_READ_TIMESTAMP + i, testingQcc.BASE_PRIORITY + i, testingQcc.CONTEXT); + const i = MAX_CAPACITY; + testingQcc.qcc.merge(BASE_ID + i, BASE_READ_TIMESTAMP + i, BASE_PRIORITY + i, CONTEXT); testingQcc.qcc.syncPriorityMap(); testingQcc.qcc.checkCacheCapacity(); @@ -106,14 +109,14 @@ describe('QueryContextCacheTest', function () { testingQcc.assertCacheData(); }); - it('testUpdateTimestamp',function () { + it('test updating timestamp',function () { testingQcc.initCacheWithData(); // Add one more element with new TS with existing id const updatedID = 1; - testingQcc.expectedReadTimestamp[updatedID] = testingQcc.BASE_READ_TIMESTAMP + updatedID + 10; + testingQcc.expectedReadTimestamp[updatedID] = BASE_READ_TIMESTAMP + updatedID + 10; testingQcc.qcc.merge( - testingQcc.BASE_ID + updatedID, testingQcc.expectedReadTimestamp[updatedID], testingQcc.BASE_PRIORITY + updatedID, testingQcc.CONTEXT); + BASE_ID + updatedID, testingQcc.expectedReadTimestamp[updatedID], BASE_PRIORITY + updatedID, CONTEXT); testingQcc.qcc.syncPriorityMap(); testingQcc.qcc.checkCacheCapacity(); @@ -121,52 +124,52 @@ describe('QueryContextCacheTest', function () { testingQcc.assertCacheData(); }); - it('testUpdatePriority', function () { + it('test updating priority', function () { testingQcc.initCacheWithData(); // Add one more element with new priority with existing id const updatedID = 3; - const updatedPriority = testingQcc.BASE_PRIORITY + updatedID + 7; + const updatedPriority = BASE_PRIORITY + updatedID + 7; testingQcc.expectedPriority[updatedID] = updatedPriority; testingQcc.qcc.merge( - testingQcc.BASE_ID + updatedID, testingQcc.BASE_READ_TIMESTAMP + updatedID, testingQcc.expectedPriority[updatedID], testingQcc.CONTEXT); + BASE_ID + updatedID, BASE_READ_TIMESTAMP + updatedID, testingQcc.expectedPriority[updatedID], CONTEXT); testingQcc.qcc.syncPriorityMap(); testingQcc.qcc.checkCacheCapacity(); - for (let i = updatedID; i < testingQcc.MAX_CAPACITY - 1; i++) { + for (let i = updatedID; i < MAX_CAPACITY - 1; i++) { testingQcc.expectedIDs[i] = testingQcc.expectedIDs[i + 1]; testingQcc.expectedReadTimestamp[i] = testingQcc.expectedReadTimestamp[i + 1]; testingQcc.expectedPriority[i] = testingQcc.expectedPriority[i + 1]; } - testingQcc.expectedIDs[testingQcc.MAX_CAPACITY - 1] = testingQcc.BASE_ID + updatedID; - testingQcc.expectedReadTimestamp[testingQcc.MAX_CAPACITY - 1] = testingQcc.BASE_READ_TIMESTAMP + updatedID; - testingQcc.expectedPriority[testingQcc.MAX_CAPACITY - 1] = updatedPriority; + testingQcc.expectedIDs[MAX_CAPACITY - 1] = BASE_ID + updatedID; + testingQcc.expectedReadTimestamp[MAX_CAPACITY - 1] = BASE_READ_TIMESTAMP + updatedID; + testingQcc.expectedPriority[MAX_CAPACITY - 1] = updatedPriority; testingQcc.assertCacheData(); }); - it('testAddSamePriority',function () { + it('test - the same priority is added',function () { testingQcc.initCacheWithData(); // Add one more element with same priority - const i = testingQcc.MAX_CAPACITY; - const UpdatedPriority = testingQcc.BASE_PRIORITY + 1; - testingQcc.qcc.merge(testingQcc.BASE_ID + i, testingQcc.BASE_READ_TIMESTAMP + i, UpdatedPriority, testingQcc.CONTEXT); + const i = MAX_CAPACITY; + const updatedPriority = BASE_PRIORITY + 1; + testingQcc.qcc.merge(BASE_ID + i, BASE_READ_TIMESTAMP + i, updatedPriority, CONTEXT); testingQcc.qcc.syncPriorityMap(); testingQcc.qcc.checkCacheCapacity(); - testingQcc.expectedIDs[1] = testingQcc.BASE_ID + i; - testingQcc.expectedReadTimestamp[1] = testingQcc.BASE_READ_TIMESTAMP + i; + testingQcc.expectedIDs[1] = BASE_ID + i; + testingQcc.expectedReadTimestamp[1] = BASE_READ_TIMESTAMP + i; // Compare elements testingQcc.assertCacheData(); }); - it('testAddSameIDButStaleTimestamp', function () { + it('test - the new context has the same id but different timestamp ', function () { testingQcc.initCacheWithData(); // Add one more element with same priority const i = 2; - testingQcc.qcc.merge(testingQcc.BASE_ID + i, testingQcc.BASE_READ_TIMESTAMP + i - 10, testingQcc.BASE_PRIORITY + i, testingQcc.CONTEXT); + testingQcc.qcc.merge(BASE_ID + i, BASE_READ_TIMESTAMP + i - 10, BASE_PRIORITY + i, CONTEXT); testingQcc.qcc.syncPriorityMap(); testingQcc.qcc.checkCacheCapacity(); @@ -174,23 +177,22 @@ describe('QueryContextCacheTest', function () { testingQcc.assertCacheData(); }); - it('testEmptyCacheWithNullData', function () { + it('test empty cache with null data', function () { testingQcc.initCacheWithData(); testingQcc.qcc.deserializeQueryContext(null); assert.strictEqual(testingQcc.qcc.getSize(),0,'Empty cache'); }); - it('testEmptyCacheWithEmptyResponseData',function () { + it('test empty cache with empty response data',function () { testingQcc.initCacheWithData(); testingQcc.qcc.deserializeQueryContext({}); assert.strictEqual(testingQcc.qcc.getSize(),0,'Empty cache'); }); - it('testSerializeRequestAndDeserializeResponseData', function () { + it('test serialized request and deserialize response data', function () { testingQcc.initCacheWithData(); testingQcc.assertCacheData(); - // const queryContextDTO = testingQcc.qcc.getQueryContextDTO(); const response = testingQcc.qcc.getSerializeQueryContext(); // Clear testingQcc.qcc @@ -200,7 +202,7 @@ describe('QueryContextCacheTest', function () { testingQcc.assertCacheData(); }); - it('testSerializeRequestAndDeserializeResponseDataWithNulltestingCONTEXT', function () { + it('test serialized request and deserialize response data when context is null', function () { // Init testingQcc.qcc testingQcc.initCacheWithDataWithContext(null); @@ -214,8 +216,4 @@ describe('QueryContextCacheTest', function () { testingQcc.qcc.deserializeQueryContext(response); testingQcc.assertCacheDataWithContext(null); }); -}); - - - - \ No newline at end of file +}); \ No newline at end of file From f17d72e2cfff510cb608630c4e4888df112e3688 Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-jy Date: Sat, 16 Sep 2023 16:44:45 -0700 Subject: [PATCH 13/30] passed whole QCE objects in the merge function and fixed some testing codes --- lib/connection/statement.js | 11 +- lib/logger/browser.js | 4 + lib/queryContextCache.js | 162 +++++--------------------- test/unit/query_context_cache_test.js | 51 ++++---- 4 files changed, 62 insertions(+), 166 deletions(-) diff --git a/lib/connection/statement.js b/lib/connection/statement.js index f1e88d8f2..92aeee51a 100644 --- a/lib/connection/statement.js +++ b/lib/connection/statement.js @@ -806,15 +806,7 @@ function createOnStatementRequestSuccRow(statement, context) connectionConfig: context.connectionConfig, rowMode: context.rowMode }); - // save the query id - Logger.getInstance().trace('Here is JSON Response',{ - response: JSON.stringify(body), - statement: JSON.stringify(statement), - services: JSON.stringify(context.services), - connectionConfig: JSON.stringify(context.connectionConfig), - rowMode: JSON.stringify(context.rowMode), - queryContext:JSON.stringify(context.result.getQueryContext()) - }) + context.queryId = context.result.getQueryId(); this.services.sf.deserializeQueryContext(context.result.getQueryContext()); } @@ -1347,7 +1339,6 @@ function sendRequestPreExec(statementContext, onResultAvailable) json.queryContextDTO = statementContext.services.sf.getQueryContextDTO(); } - Logger.getInstance().trace("Here is the constructed body",JSON.stringify(json)); // use the snowflake service to issue the request sendSfRequest(statementContext, { diff --git a/lib/logger/browser.js b/lib/logger/browser.js index 5be73508b..49e8ced2e 100644 --- a/lib/logger/browser.js +++ b/lib/logger/browser.js @@ -68,6 +68,10 @@ function Logger(options) return common.getLevelNumber(); }; + this.getLevelTag = function () { + return common.getLevelTag(); + } + /** * Logs a given message at the error level. * diff --git a/lib/queryContextCache.js b/lib/queryContextCache.js index e845ff232..fe5824ee7 100644 --- a/lib/queryContextCache.js +++ b/lib/queryContextCache.js @@ -2,7 +2,6 @@ * Copyright (c) 2012-2022 Snowflake Computing Inc. All rights reserved. */ const Logger = require('./logger'); -const crypto = require('crypto'); /** * @@ -18,59 +17,6 @@ function QueryContextElement (id,timestamp,priority,context) { this.context = context; } -QueryContextElement.prototype.getId = function () { - return this.id; -}; - -QueryContextElement.prototype.getReadTimestamp = function () { - return this.timestamp; -}; - -QueryContextElement.prototype.getPriority = function () { - return this.priority; -}; - -QueryContextElement.prototype.getContext = function () { - return this.context; -}; - -QueryContextElement.prototype.setId = function (id) { - this.id = id; -}; - -QueryContextElement.prototype.setReadTimestamp = function (timestamp) { - this.timestamp = timestamp; -}; - -QueryContextElement.prototype.setPriority = function (priority) { - this.priority = priority; -}; - -QueryContextElement.prototype.setContext = function (context) { - this.context = context; -}; - -QueryContextElement.prototype.equals = function (obj) { - if (!(obj instanceof QueryContextElement)) { - return false; - } - - return (this.id === obj.id - && this.timestamp === obj.timestamp - && this.priority === obj.priority - && this.context.equals(obj.context)); -}; - -QueryContextElement.prototype.hashCode = function () { - let hash = 31; - hash = hash * 31 + parseInt(this.id); - hash += (hash * 31) + parseInt(this.timestamp); - hash += (hash * 31) + parseInt(this.priority); - hash += (hash * 31) + parseInt(crypto.createHash('md5').update(Buffer.from(this.context,'utf-8').toString()).digest(),2); - - return hash; -}; - /** * Most Recently Used and Priority based cache. A separate cache for each connection in the driver. */ @@ -85,16 +31,15 @@ function QueryContextCache (capacity) { this.idMap = new Map(); // Map for id and QCC this.treeSet = new Set(); // Order data as per priority this.priorityMap = new Map(); // Map for priority and QCC - this.newPriorityMap = new Map(); // Intermediate map for priority and QCC for current round of merging } QueryContextCache.prototype.sortTreeSet = function () { - this.treeSet = new Set(Array.from(this.treeSet).sort((a,b)=>a.getPriority()-b.getPriority())); + this.treeSet = new Set(Array.from(this.treeSet).sort((a,b)=>a.priority-b.priority)); }; QueryContextCache.prototype.addQCE= function (qce) { this.idMap.set(qce.id,qce); - this.newPriorityMap.set(qce.priority,qce); + this.priorityMap.set(qce.priority,qce); this.treeSet.add(qce); this.sortTreeSet(); }; @@ -135,66 +80,48 @@ QueryContextCache.prototype.replaceQCE = function (oldQCE, newQCE) { * @param {String} context * */ -QueryContextCache.prototype.merge = function (id, timestamp, priority, context) { - if (this.idMap.has(id)) { +QueryContextCache.prototype.merge = function (newQCE) { + if (this.idMap.has(newQCE.id)) { // ID found in the cache - const qce = this.idMap.get(id); - if (timestamp > qce.getReadTimestamp()) { - if (qce.getPriority() === priority) { + const qce = this.idMap.get(newQCE.id); + if (newQCE.timestamp > qce.timestamp) { + if (qce.priority === newQCE.priority) { // Same priority, overwrite new data at same place - qce.setReadTimestamp(timestamp); - qce.setContext(context); + qce.timestamp = newQCE.timestamp; + qce.context = newQCE.context; } else { // Change in priority - const newQCE = - new QueryContextElement(id, timestamp, priority, context); - this.replaceQCE(qce, newQCE); } } - else if (timestamp === qce.getReadTimestamp() && qce.getPriority() !== priority) { + else if (newQCE.timestamp === qce.timestamp && qce.priority !== newQCE.priority) { + // Same read timestamp but change in priority - const newQCE = new QueryContextElement(id, timestamp, priority, context); this.replaceQCE(qce, newQCE); } } // id found else { // new id - if (this.priorityMap.has(priority)) { + if (this.priorityMap.has(newQCE.priority)) { // Same priority with different id - const qce = this.priorityMap.get(priority); + const qce = this.priorityMap.get(newQCE.priority); // Replace with new data - const newQCE = new QueryContextElement(id, timestamp, priority, context); this.replaceQCE(qce, newQCE); } else { // new priority // Add new element in the cache - const newQCE = new QueryContextElement(id, timestamp, priority, context); - this.addQCE(newQCE); + this.addQCE(newQCE,newQCE); } } }; -/** Sync the newPriorityMap with the priorityMap at the end of current round of merge */ -QueryContextCache.prototype.syncPriorityMap = function () { - Logger.getInstance().debug( - `syncPriorityMap called priorityMap size = ${this.priorityMap.size}, newPriorityMap size = ${this.newPriorityMap.size}` - ); - this.newPriorityMap.forEach((value,key)=>{ - this.priorityMap.set(key,value); - }); - - // clear the newPriorityMap for next round of QCC merge(a round consists of multiple entries) - this.newPriorityMap.clear(); -}; - /** * After the merge, loop through priority list and make sure cache is at most capacity. Remove all * other elements from the list based on priority. @@ -230,15 +157,15 @@ QueryContextCache.prototype.getElements = function () { * @param data: the QueryContext Object serialized as a JSON format string */ QueryContextCache.prototype.deserializeQueryContext = function (data) { - Logger.getInstance().debug(`deserializeQueryContext() called: data from server: ${JSON.stringify(data)}`); - if (!data||JSON.stringify(data)==='{}'||data.entries === null) { + const stringifyData = JSON.stringify(data); + Logger.getInstance().debug(`deserializeQueryContext() called: data from server: ${stringifyData}`); + if (!data || stringifyData ==='{}' || data.entries === null) { this.clearCache(); Logger.getInstance().debug('deserializeQueryContext() returns'); this.logCacheEntries(); return; } - try { // Deserialize the entries. The first entry with priority is the main entry. An example JSON is: // { @@ -268,7 +195,7 @@ QueryContextCache.prototype.deserializeQueryContext = function (data) { for (const entryNode of entries) { const entry = this.deserializeQueryContextElement(entryNode); if (entry != null) { - this.merge(entry.getId(), entry.getReadTimestamp(), entry.getPriority(), entry.getContext()); + this.merge(entry); } else { Logger.getInstance().warn( 'deserializeQueryContextJson: deserializeQueryContextElement meets mismatch field type. Clear the QueryContextCache.'); @@ -276,11 +203,7 @@ QueryContextCache.prototype.deserializeQueryContext = function (data) { return; } } - // after merging all entries, sync the internal priority map to priority map. Because of - // priority swicth from GS side, - // there could be priority key conflict if we directly operating on the priorityMap during - // a round of merge. - this.syncPriorityMap(); + } } catch (e) { Logger.getInstance().debug(`deserializeQueryContextJson: Exception = ${e.getMessage}`, ); @@ -297,54 +220,30 @@ QueryContextCache.prototype.deserializeQueryContext = function (data) { }; // Synchronized QueryContextCache.prototype.deserializeQueryContextElement = function (node) { - const entry = new QueryContextElement(); - const idNode = node.id; - if (typeof idNode === 'number') { - entry.setId(idNode); - } else { - Logger.getInstance().warn('deserializeQueryContextElement: `id` field is not Number type'); - return null; - } - - const timestampNode = node.timestamp; - if (typeof timestampNode === 'number') { - entry.setReadTimestamp(timestampNode); - } else { - Logger.getInstance().warn('deserializeQueryContextElement: `timestamp` field is not Number type'); - return null; - } + const {id, timestamp, priority, context} = node; + const entry = new QueryContextElement (id, timestamp, priority, null); - const priorityNode = node.priority; - if (typeof priorityNode === 'number') { - entry.setPriority(priorityNode); - } else { - Logger.getInstance().warn('deserializeQueryContextElement: `priority` field is not Number type'); - return null; + if(typeof context === 'string'){ + entry.context = context; } - - const contextNode = node.context; - if (typeof contextNode === 'string') { - - entry.setContext(contextNode); - - } else if (contextNode === null || contextNode === undefined) { - - entry.setContext(null); + else if (context === null || context === undefined) { + entry.context = null; Logger.getInstance().debug('deserializeQueryContextElement `context` field is empty'); - } else { + } + else { Logger.getInstance().warn('deserializeQueryContextElement: `context` field is not String type'); return null; } + return entry; }; QueryContextCache.prototype.logCacheEntries = function () { - //Debug Level only - if(Logger.getInstance().getLevel() === 3){ + if(Logger.getInstance().getLevelTag() === 'DEBUG'){ this.treeSet.forEach(function(elem) { Logger.getInstance().debug( - `Cache Entry: id: ${elem.getId()} timestamp: ${elem.getReadTimestamp()} priority: ${elem.getPriority()}`, + `Cache Entry: id: ${elem.id} timestamp: ${elem.timestamp} priority: ${elem.priority}`, ); }); } @@ -361,7 +260,6 @@ QueryContextCache.prototype.getQueryContextDTO = function () { arr.push({id:querycontexts[i].id, timestamp:querycontexts[i].timestamp, priority:querycontexts[i].priority, context:{base64Data:querycontexts[i].context}||null}); } - return { entries: arr }; diff --git a/test/unit/query_context_cache_test.js b/test/unit/query_context_cache_test.js index f31e0473b..6f9e06c88 100644 --- a/test/unit/query_context_cache_test.js +++ b/test/unit/query_context_cache_test.js @@ -7,6 +7,14 @@ const BASE_READ_TIMESTAMP = 1668727958; const CONTEXT = 'Some query Context'; const MAX_CAPACITY = 5; +function QueryContextElement (id,timestamp,priority,context) { + this.id = id; + this.timestamp = timestamp; + this.priority = priority; + this.context = context; +} + + function TestingQCC () { this.qcc = null; @@ -32,9 +40,8 @@ function TestingQCC () { this.expectedIDs[i] = BASE_ID + i; this.expectedReadTimestamp[i] = BASE_READ_TIMESTAMP + i; this.expectedPriority[i] = BASE_PRIORITY + i; - this.qcc.merge(this.expectedIDs[i], this.expectedReadTimestamp[i], this.expectedPriority[i], Context); + this.qcc.merge(new QueryContextElement(this.expectedIDs[i], this.expectedReadTimestamp[i], this.expectedPriority[i], Context)); } - this.qcc.syncPriorityMap(); }; this.initCacheWithDataInRandomOrder = function () { @@ -48,12 +55,11 @@ function TestingQCC () { this.expectedPriority[i] = BASE_PRIORITY + i; } - this.qcc.merge(this.expectedIDs[3], this.expectedReadTimestamp[3], this.expectedPriority[3], CONTEXT); - this.qcc.merge(this.expectedIDs[2], this.expectedReadTimestamp[2], this.expectedPriority[2], CONTEXT); - this.qcc.merge(this.expectedIDs[4], this.expectedReadTimestamp[4], this.expectedPriority[4], CONTEXT); - this.qcc.merge(this.expectedIDs[0], this.expectedReadTimestamp[0], this.expectedPriority[0], CONTEXT); - this.qcc.merge(this.expectedIDs[1], this.expectedReadTimestamp[1], this.expectedPriority[1], CONTEXT); - this.qcc.syncPriorityMap(); + this.qcc.merge(new QueryContextElement(this.expectedIDs[3], this.expectedReadTimestamp[3], this.expectedPriority[3], CONTEXT)); + this.qcc.merge(new QueryContextElement(this.expectedIDs[2], this.expectedReadTimestamp[2], this.expectedPriority[2], CONTEXT)); + this.qcc.merge(new QueryContextElement(this.expectedIDs[4], this.expectedReadTimestamp[4], this.expectedPriority[4], CONTEXT)); + this.qcc.merge(new QueryContextElement(this.expectedIDs[0], this.expectedReadTimestamp[0], this.expectedPriority[0], CONTEXT)); + this.qcc.merge(new QueryContextElement(this.expectedIDs[1], this.expectedReadTimestamp[1], this.expectedPriority[1], CONTEXT)); }; this.assertCacheData = function () { @@ -65,10 +71,10 @@ function TestingQCC () { assert.strictEqual(size,MAX_CAPACITY); const elements = Array.from(this.qcc.getElements()); for (let i = 0; i < size; i++) { - assert.strictEqual(this.expectedIDs[i], elements[i].getId()); - assert.strictEqual(this.expectedReadTimestamp[i], elements[i].getReadTimestamp()); - assert.strictEqual(this.expectedPriority[i], elements[i].getPriority()); - assert.strictEqual(Context, elements[i].getContext()); + assert.strictEqual(this.expectedIDs[i], elements[i].id); + assert.strictEqual(this.expectedReadTimestamp[i], elements[i].timestamp); + assert.strictEqual(this.expectedPriority[i], elements[i].priority); + assert.strictEqual(Context, elements[i].context); } }; } @@ -101,8 +107,8 @@ describe('Query Context Cache Test', function () { // Add one more element at the end const i = MAX_CAPACITY; - testingQcc.qcc.merge(BASE_ID + i, BASE_READ_TIMESTAMP + i, BASE_PRIORITY + i, CONTEXT); - testingQcc.qcc.syncPriorityMap(); + const extraQCE = new QueryContextElement(BASE_ID + i, BASE_READ_TIMESTAMP + i, BASE_PRIORITY + i, CONTEXT) + testingQcc.qcc.merge(extraQCE); testingQcc.qcc.checkCacheCapacity(); // Compare elements @@ -115,9 +121,8 @@ describe('Query Context Cache Test', function () { // Add one more element with new TS with existing id const updatedID = 1; testingQcc.expectedReadTimestamp[updatedID] = BASE_READ_TIMESTAMP + updatedID + 10; - testingQcc.qcc.merge( - BASE_ID + updatedID, testingQcc.expectedReadTimestamp[updatedID], BASE_PRIORITY + updatedID, CONTEXT); - testingQcc.qcc.syncPriorityMap(); + const updatedQCE = new QueryContextElement(BASE_ID + updatedID, testingQcc.expectedReadTimestamp[updatedID], BASE_PRIORITY + updatedID, CONTEXT); + testingQcc.qcc.merge(updatedQCE); testingQcc.qcc.checkCacheCapacity(); // Compare elements @@ -131,9 +136,8 @@ describe('Query Context Cache Test', function () { const updatedID = 3; const updatedPriority = BASE_PRIORITY + updatedID + 7; testingQcc.expectedPriority[updatedID] = updatedPriority; - testingQcc.qcc.merge( - BASE_ID + updatedID, BASE_READ_TIMESTAMP + updatedID, testingQcc.expectedPriority[updatedID], CONTEXT); - testingQcc.qcc.syncPriorityMap(); + const updatedQCE = new QueryContextElement(BASE_ID + updatedID, BASE_READ_TIMESTAMP + updatedID, testingQcc.expectedPriority[updatedID], CONTEXT) + testingQcc.qcc.merge(updatedQCE); testingQcc.qcc.checkCacheCapacity(); for (let i = updatedID; i < MAX_CAPACITY - 1; i++) { @@ -154,8 +158,7 @@ describe('Query Context Cache Test', function () { // Add one more element with same priority const i = MAX_CAPACITY; const updatedPriority = BASE_PRIORITY + 1; - testingQcc.qcc.merge(BASE_ID + i, BASE_READ_TIMESTAMP + i, updatedPriority, CONTEXT); - testingQcc.qcc.syncPriorityMap(); + testingQcc.qcc.merge(new QueryContextElement(BASE_ID + i, BASE_READ_TIMESTAMP + i, updatedPriority, CONTEXT)); testingQcc.qcc.checkCacheCapacity(); testingQcc.expectedIDs[1] = BASE_ID + i; testingQcc.expectedReadTimestamp[1] = BASE_READ_TIMESTAMP + i; @@ -169,8 +172,8 @@ describe('Query Context Cache Test', function () { // Add one more element with same priority const i = 2; - testingQcc.qcc.merge(BASE_ID + i, BASE_READ_TIMESTAMP + i - 10, BASE_PRIORITY + i, CONTEXT); - testingQcc.qcc.syncPriorityMap(); + const samePriorityQCE = new QueryContextElement(BASE_ID + i, BASE_READ_TIMESTAMP + i - 10, BASE_PRIORITY + i, CONTEXT) + testingQcc.qcc.merge(samePriorityQCE); testingQcc.qcc.checkCacheCapacity(); // Compare elements From dd9da025a5679ba96c9c03e009d9281aebad86ff Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-jy Date: Mon, 18 Sep 2023 18:03:05 -0700 Subject: [PATCH 14/30] Added integration test --- lib/connection/statement.js | 17 ++++-- lib/services/sf.js | 8 +++ test/integration/testHTAP.js | 102 +++++++++++++++++++++++++++++++++++ 3 files changed, 123 insertions(+), 4 deletions(-) create mode 100644 test/integration/testHTAP.js diff --git a/lib/connection/statement.js b/lib/connection/statement.js index 92aeee51a..f43e616bd 100644 --- a/lib/connection/statement.js +++ b/lib/connection/statement.js @@ -576,6 +576,15 @@ function BaseStatement( sendCancelStatement(context, statement, callback); }; + //Integration Testing purpose. + this.getQueryContextCacheSize = function () { + return services.sf.getQueryContextCacheSize(); + } + + this.getQueryContextDTOSize = function () { + return services.sf.getQueryContextDTO().entries.length; + } + /** * Issues a request to get the statement result again. * @@ -1335,10 +1344,6 @@ function sendRequestPreExec(statementContext, onResultAvailable) json.isInternal = statementContext.internal; } - if(!statementContext.disableQueryContextCache){ - json.queryContextDTO = statementContext.services.sf.getQueryContextDTO(); - } - // use the snowflake service to issue the request sendSfRequest(statementContext, { @@ -1394,6 +1399,10 @@ this.sendRequest = function (statementContext, onResultAvailable) json.isInternal = statementContext.internal; } + if(!statementContext.disableQueryContextCache){ + json.queryContextDTO = statementContext.services.sf.getQueryContextDTO(); + } + var options = { method: 'POST', diff --git a/lib/services/sf.js b/lib/services/sf.js index d8a288337..e0f714842 100644 --- a/lib/services/sf.js +++ b/lib/services/sf.js @@ -557,6 +557,14 @@ function SnowflakeService(connectionConfig, httpClient, config) this.qcc = new QueryContextCache(size); } } + + // testing purpose + this.getQueryContextCacheSize = function () { + if(!this.qcc){ + return; + } + return this.qcc.getSize(); + } } Util.inherits(SnowflakeService, EventEmitter); diff --git a/test/integration/testHTAP.js b/test/integration/testHTAP.js new file mode 100644 index 000000000..92cc0414d --- /dev/null +++ b/test/integration/testHTAP.js @@ -0,0 +1,102 @@ +const assert = require('assert'); +const async = require('async'); +const connOption = require('./connectionOptions').valid; +const testUtil = require('./testUtil'); + +describe('Query Context Cache test', function () { + let connection; + beforeEach(() => { + connection = testUtil.createConnection(connOption); + }); + const querySet = [ + { + sqlTexts:[ + 'create or replace database db1', + 'create or replace hybrid table t1 (a int primary key, b int)', + 'insert into t1 values (1, 2), (2, 3), (3, 4)' + ], + QccSize:2, + }, + { + sqlTexts:[ + 'create or replace database db2', + 'create or replace hybrid table t2 (a int primary key, b int)', + 'insert into t2 values (1, 2), (2, 3), (3, 4)' + ], + QccSize:3, + }, + { + sqlTexts:[ + 'create or replace database db3', + 'create or replace hybrid table t3 (a int primary key, b int)', + 'insert into t3 values (1, 2), (2, 3), (3, 4)' + ], + QccSize:4, + }, + { + sqlTexts:[ + 'select * from db1.public.t1 x, db2.public.t2 y, db3.public.t3 z where x.a = y.a and y.a = z.a;', + 'select * from db1.public.t1 x, db2.public.t2 y where x.a = y.a;', + 'select * from db2.public.t2 y, db3.public.t3 z where y.a = z.a;' + ], + QccSize:4, + }, + ]; + + function createTests () { + const functionArray = []; + for(let i = 0; i < querySet.length; i++) { + const testingFunction = function(callback) { + const {sqlTexts,QccSize} = querySet[i]; + connection.execute({ + sqlText: sqlTexts[0], + complete: function (err) { + if (err) { + callback(err); + } + } + }); + connection.execute({ + sqlText: sqlTexts[1], + complete: function (err) { + if (err) { + callback(err); + } + } + }); + connection.execute({ + sqlText: sqlTexts[2], + complete: function (err, stmt) { + assert.ok(!err,'There should be no error!'); + assert.strictEqual(stmt.getQueryContextCacheSize(), QccSize); + assert.strictEqual(stmt.getQueryContextDTOSize(),QccSize); + callback(); + + } + }); + }; + functionArray.push(testingFunction); + } + return functionArray; + } + + it('test Query Context Cache', function (done) { + async.series( + [ + function (callback) { + connection.connect(function (err, conn) { + assert.ok(!err, 'there should be no error'); + assert.strictEqual(conn, connection, + 'the connect() callback should be invoked with the statement'); + + callback(); + }); + }, + ...createTests() + ], + function () { + done(); + }); + }); + +}); From b827f5c4591033384e0f2e226d3c66de0f27b709 Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-jy Date: Mon, 18 Sep 2023 18:11:03 -0700 Subject: [PATCH 15/30] edited some variable names in the QCC testing --- test/integration/testHTAP.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/integration/testHTAP.js b/test/integration/testHTAP.js index 92cc0414d..8161aa392 100644 --- a/test/integration/testHTAP.js +++ b/test/integration/testHTAP.js @@ -43,8 +43,8 @@ describe('Query Context Cache test', function () { }, ]; - function createTests () { - const functionArray = []; + function createQueryTest () { + const testingSet = []; for(let i = 0; i < querySet.length; i++) { const testingFunction = function(callback) { const {sqlTexts,QccSize} = querySet[i]; @@ -71,16 +71,16 @@ describe('Query Context Cache test', function () { assert.strictEqual(stmt.getQueryContextCacheSize(), QccSize); assert.strictEqual(stmt.getQueryContextDTOSize(),QccSize); callback(); - } }); }; - functionArray.push(testingFunction); + testingSet.push(testingFunction); } - return functionArray; + return testingSet; } it('test Query Context Cache', function (done) { + let queryTests = createQueryTest(); async.series( [ function (callback) { @@ -92,7 +92,7 @@ describe('Query Context Cache test', function () { callback(); }); }, - ...createTests() + ...queryTests ], function () { done(); From 0268c02d2c4edbbd8a09c88f95716f76cb86649b Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-jy Date: Mon, 18 Sep 2023 19:04:57 -0700 Subject: [PATCH 16/30] Remove queryContextDTO from requests --- test/unit/mock/mock_http_client.js | 6 ------ 1 file changed, 6 deletions(-) diff --git a/test/unit/mock/mock_http_client.js b/test/unit/mock/mock_http_client.js index acaf6d784..efb6fbdbf 100644 --- a/test/unit/mock/mock_http_client.js +++ b/test/unit/mock/mock_http_client.js @@ -342,7 +342,6 @@ function buildRequestOutputMappings(clientInfo) { disableOfflineChunks: false, sqlText: 'select 1 as "c1";', - queryContextDTO: { entries: [] } } }, output: @@ -554,7 +553,6 @@ function buildRequestOutputMappings(clientInfo) "1": {type: 'TEXT', value: 'false'}, "2": {type: 'TEXT', value: '1967-06-23'} }, - queryContextDTO: { entries: [] } } }, output: @@ -670,7 +668,6 @@ function buildRequestOutputMappings(clientInfo) { disableOfflineChunks: false, sqlText: 'select;', - queryContextDTO: { entries: [] } } }, output: @@ -981,7 +978,6 @@ function buildRequestOutputMappings(clientInfo) { disableOfflineChunks: false, sqlText: 'select count(*) from table(generator(timelimit=>10));', - queryContextDTO: { entries: [] } } }, output: @@ -1057,7 +1053,6 @@ function buildRequestOutputMappings(clientInfo) { disableOfflineChunks: false, sqlText: 'select \'too many concurrent queries\';', - queryContextDTO: { entries: [] } } }, output: @@ -1198,7 +1193,6 @@ function buildRequestOutputMappings(clientInfo) { disableOfflineChunks: false, sqlText: 'select * from faketable', - queryContextDTO: { entries: [] } } }, output: From 3a619fe967f2210df96cf4fd241ce555a19b03f2 Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-jy Date: Tue, 19 Sep 2023 12:04:42 -0700 Subject: [PATCH 17/30] fixed integration tests and removed comments --- lib/queryContextCache.js | 5 +-- test/integration/testHTAP.js | 63 +++++++++++++++++------------------- 2 files changed, 31 insertions(+), 37 deletions(-) diff --git a/lib/queryContextCache.js b/lib/queryContextCache.js index fe5824ee7..2886f7d8f 100644 --- a/lib/queryContextCache.js +++ b/lib/queryContextCache.js @@ -212,12 +212,9 @@ QueryContextCache.prototype.deserializeQueryContext = function (data) { this.clearCache(); } - // After merging all entries, truncate to capacity this.checkCacheCapacity(); - - // Log existing cache entries this.logCacheEntries(); -}; // Synchronized +}; QueryContextCache.prototype.deserializeQueryContextElement = function (node) { const {id, timestamp, priority, context} = node; diff --git a/test/integration/testHTAP.js b/test/integration/testHTAP.js index 8161aa392..e897804f4 100644 --- a/test/integration/testHTAP.js +++ b/test/integration/testHTAP.js @@ -45,42 +45,41 @@ describe('Query Context Cache test', function () { function createQueryTest () { const testingSet = []; + let testingfunction; for(let i = 0; i < querySet.length; i++) { - const testingFunction = function(callback) { - const {sqlTexts,QccSize} = querySet[i]; - connection.execute({ - sqlText: sqlTexts[0], - complete: function (err) { - if (err) { - callback(err); - } - } - }); - connection.execute({ - sqlText: sqlTexts[1], - complete: function (err) { - if (err) { - callback(err); - } - } - }); - connection.execute({ - sqlText: sqlTexts[2], - complete: function (err, stmt) { - assert.ok(!err,'There should be no error!'); - assert.strictEqual(stmt.getQueryContextCacheSize(), QccSize); - assert.strictEqual(stmt.getQueryContextDTOSize(),QccSize); - callback(); - } - }); - }; - testingSet.push(testingFunction); + const {sqlTexts,QccSize} = querySet[i]; + for(let k = 0; k < sqlTexts.length; k++){ + if(k!==sqlTexts.length-1){ + testingfunction = function(callback) { + connection.execute({ + sqlText: sqlTexts[k], + complete: function () { + callback(); + } + }); + }; + } + else{ + testingfunction = function(callback) { + connection.execute({ + sqlText: sqlTexts[2], + complete: function (err, stmt) { + assert.ok(!err,'There should be no error!'); + assert.strictEqual(stmt.getQueryContextCacheSize(), QccSize); + assert.strictEqual(stmt.getQueryContextDTOSize(), QccSize); + callback(); + } + }); + }; + } + testingSet.push(testingfunction); + } } return testingSet; } - + it('test Query Context Cache', function (done) { - let queryTests = createQueryTest(); + const queryTests = createQueryTest(); async.series( [ function (callback) { @@ -88,7 +87,6 @@ describe('Query Context Cache test', function () { assert.ok(!err, 'there should be no error'); assert.strictEqual(conn, connection, 'the connect() callback should be invoked with the statement'); - callback(); }); }, @@ -98,5 +96,4 @@ describe('Query Context Cache test', function () { done(); }); }); - }); From d02f273a7fe6af637fa51ea59a433056db4aecb0 Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-jy Date: Wed, 20 Sep 2023 10:26:58 -0700 Subject: [PATCH 18/30] Removed getLogLevelTag funcrtion --- lib/logger/browser.js | 4 ---- lib/queryContextCache.js | 2 -- 2 files changed, 6 deletions(-) diff --git a/lib/logger/browser.js b/lib/logger/browser.js index 49e8ced2e..5be73508b 100644 --- a/lib/logger/browser.js +++ b/lib/logger/browser.js @@ -68,10 +68,6 @@ function Logger(options) return common.getLevelNumber(); }; - this.getLevelTag = function () { - return common.getLevelTag(); - } - /** * Logs a given message at the error level. * diff --git a/lib/queryContextCache.js b/lib/queryContextCache.js index 2886f7d8f..609bf6266 100644 --- a/lib/queryContextCache.js +++ b/lib/queryContextCache.js @@ -237,13 +237,11 @@ QueryContextCache.prototype.deserializeQueryContextElement = function (node) { QueryContextCache.prototype.logCacheEntries = function () { - if(Logger.getInstance().getLevelTag() === 'DEBUG'){ this.treeSet.forEach(function(elem) { Logger.getInstance().debug( `Cache Entry: id: ${elem.id} timestamp: ${elem.timestamp} priority: ${elem.priority}`, ); }); - } }; QueryContextCache.prototype.getSize = function() { From 85540ff9998ccb7897b264ce239aac6bd59e8c8f Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-jy Date: Wed, 20 Sep 2023 19:10:03 -0700 Subject: [PATCH 19/30] Fixed the network error and reverted back queryContexDTO to the original location --- lib/connection/result/result.js | 2 +- lib/connection/statement.js | 8 ++++---- test/unit/mock/mock_http_client.js | 8 +++++++- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/lib/connection/result/result.js b/lib/connection/result/result.js index 50e15f425..9d55a0e6c 100644 --- a/lib/connection/result/result.js +++ b/lib/connection/result/result.js @@ -74,7 +74,7 @@ function Result(options) { // convert the parameters array to a map parametersMap = {}; - parametersArray = data.parameters; + parametersArray = data.parameters || []; for (index = 0, length = parametersArray.length; index < length; index++) { parameter = parametersArray[index]; parametersMap[parameter.name] = parameter.value; diff --git a/lib/connection/statement.js b/lib/connection/statement.js index f43e616bd..dd65640e3 100644 --- a/lib/connection/statement.js +++ b/lib/connection/statement.js @@ -1344,6 +1344,10 @@ function sendRequestPreExec(statementContext, onResultAvailable) json.isInternal = statementContext.internal; } + if(!statementContext.disableQueryContextCache){ + json.queryContextDTO = statementContext.services.sf.getQueryContextDTO(); + } + // use the snowflake service to issue the request sendSfRequest(statementContext, { @@ -1399,10 +1403,6 @@ this.sendRequest = function (statementContext, onResultAvailable) json.isInternal = statementContext.internal; } - if(!statementContext.disableQueryContextCache){ - json.queryContextDTO = statementContext.services.sf.getQueryContextDTO(); - } - var options = { method: 'POST', diff --git a/test/unit/mock/mock_http_client.js b/test/unit/mock/mock_http_client.js index efb6fbdbf..47de2823a 100644 --- a/test/unit/mock/mock_http_client.js +++ b/test/unit/mock/mock_http_client.js @@ -342,6 +342,7 @@ function buildRequestOutputMappings(clientInfo) { disableOfflineChunks: false, sqlText: 'select 1 as "c1";', + queryContextDTO: { entries: [] }, } }, output: @@ -551,8 +552,9 @@ function buildRequestOutputMappings(clientInfo) bindings: { "1": {type: 'TEXT', value: 'false'}, - "2": {type: 'TEXT', value: '1967-06-23'} + "2": {type: 'TEXT', value: '1967-06-23'}, }, + queryContextDTO: { entries: [] }, } }, output: @@ -668,6 +670,7 @@ function buildRequestOutputMappings(clientInfo) { disableOfflineChunks: false, sqlText: 'select;', + queryContextDTO: { entries: [] }, } }, output: @@ -978,6 +981,7 @@ function buildRequestOutputMappings(clientInfo) { disableOfflineChunks: false, sqlText: 'select count(*) from table(generator(timelimit=>10));', + queryContextDTO: { entries: [] }, } }, output: @@ -1053,6 +1057,7 @@ function buildRequestOutputMappings(clientInfo) { disableOfflineChunks: false, sqlText: 'select \'too many concurrent queries\';', + queryContextDTO: { entries: [] }, } }, output: @@ -1193,6 +1198,7 @@ function buildRequestOutputMappings(clientInfo) { disableOfflineChunks: false, sqlText: 'select * from faketable', + queryContextDTO: { entries: [] }, } }, output: From 42af59cdb8b24bdb2a2531704c12eab29aa28daf Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-jy Date: Wed, 20 Sep 2023 19:35:55 -0700 Subject: [PATCH 20/30] Removed unnecessary spaces between two classes --- test/unit/query_context_cache_test.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/unit/query_context_cache_test.js b/test/unit/query_context_cache_test.js index 6f9e06c88..5d9cea884 100644 --- a/test/unit/query_context_cache_test.js +++ b/test/unit/query_context_cache_test.js @@ -14,8 +14,6 @@ function QueryContextElement (id,timestamp,priority,context) { this.context = context; } - - function TestingQCC () { this.qcc = null; From a9820a6553404f00330b14ade464f19efd374adc Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-jy Date: Wed, 20 Sep 2023 23:22:51 -0700 Subject: [PATCH 21/30] increased the testing timeout --- test/integration/testHTAP.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test/integration/testHTAP.js b/test/integration/testHTAP.js index e897804f4..7450f21b5 100644 --- a/test/integration/testHTAP.js +++ b/test/integration/testHTAP.js @@ -4,10 +4,16 @@ const connOption = require('./connectionOptions').valid; const testUtil = require('./testUtil'); describe('Query Context Cache test', function () { + this.timeout(1000000); let connection; - beforeEach(() => { + before(() => { connection = testUtil.createConnection(connOption); }); + + after(async () => + { + testUtil.destroyConnectionAsync(connection); + }); const querySet = [ { sqlTexts:[ From e84eef9fcb7fa4bf75ee757ebcf8ef33f81c410d Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-jy Date: Thu, 21 Sep 2023 09:03:39 -0700 Subject: [PATCH 22/30] Added logging options for testing --- test/integration/testHTAP.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/integration/testHTAP.js b/test/integration/testHTAP.js index 7450f21b5..2b5365826 100644 --- a/test/integration/testHTAP.js +++ b/test/integration/testHTAP.js @@ -2,17 +2,22 @@ const assert = require('assert'); const async = require('async'); const connOption = require('./connectionOptions').valid; const testUtil = require('./testUtil'); +const { configureLogger } = require('../configureLogger'); + describe('Query Context Cache test', function () { this.timeout(1000000); let connection; before(() => { connection = testUtil.createConnection(connOption); + configureLogger('TRACE'); }); after(async () => { testUtil.destroyConnectionAsync(connection); + configureLogger('ERROR'); + }); const querySet = [ { From de7fb4f501d0ad123e5ae7ff052623eac386df24 Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-jy Date: Thu, 21 Sep 2023 09:30:58 -0700 Subject: [PATCH 23/30] reduced the number of testing --- test/integration/testHTAP.js | 48 ++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/test/integration/testHTAP.js b/test/integration/testHTAP.js index 2b5365826..c53fe91c9 100644 --- a/test/integration/testHTAP.js +++ b/test/integration/testHTAP.js @@ -28,30 +28,30 @@ describe('Query Context Cache test', function () { ], QccSize:2, }, - { - sqlTexts:[ - 'create or replace database db2', - 'create or replace hybrid table t2 (a int primary key, b int)', - 'insert into t2 values (1, 2), (2, 3), (3, 4)' - ], - QccSize:3, - }, - { - sqlTexts:[ - 'create or replace database db3', - 'create or replace hybrid table t3 (a int primary key, b int)', - 'insert into t3 values (1, 2), (2, 3), (3, 4)' - ], - QccSize:4, - }, - { - sqlTexts:[ - 'select * from db1.public.t1 x, db2.public.t2 y, db3.public.t3 z where x.a = y.a and y.a = z.a;', - 'select * from db1.public.t1 x, db2.public.t2 y where x.a = y.a;', - 'select * from db2.public.t2 y, db3.public.t3 z where y.a = z.a;' - ], - QccSize:4, - }, + // { + // sqlTexts:[ + // 'create or replace database db2', + // 'create or replace hybrid table t2 (a int primary key, b int)', + // 'insert into t2 values (1, 2), (2, 3), (3, 4)' + // ], + // QccSize:3, + // }, + // { + // sqlTexts:[ + // 'create or replace database db3', + // 'create or replace hybrid table t3 (a int primary key, b int)', + // 'insert into t3 values (1, 2), (2, 3), (3, 4)' + // ], + // QccSize:4, + // }, + // { + // sqlTexts:[ + // 'select * from db1.public.t1 x, db2.public.t2 y, db3.public.t3 z where x.a = y.a and y.a = z.a;', + // 'select * from db1.public.t1 x, db2.public.t2 y where x.a = y.a;', + // 'select * from db2.public.t2 y, db3.public.t3 z where y.a = z.a;' + // ], + // QccSize:4, + // }, ]; function createQueryTest () { From e28fea056cef8d095a2e424cb9f8ec046eacedd4 Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-jy Date: Thu, 21 Sep 2023 10:05:36 -0700 Subject: [PATCH 24/30] Removed creating DB for testing --- test/integration/testHTAP.js | 53 ++++++++++++++++++------------------ 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/test/integration/testHTAP.js b/test/integration/testHTAP.js index c53fe91c9..14c7fa2a1 100644 --- a/test/integration/testHTAP.js +++ b/test/integration/testHTAP.js @@ -6,6 +6,7 @@ const { configureLogger } = require('../configureLogger'); describe('Query Context Cache test', function () { + const tableName = ['t1','t2','t3'] this.timeout(1000000); let connection; before(() => { @@ -13,42 +14,41 @@ describe('Query Context Cache test', function () { configureLogger('TRACE'); }); - after(async () => - { - testUtil.destroyConnectionAsync(connection); + after(async () => { configureLogger('ERROR'); - + await testUtil.dropTablesIgnoringErrorsAsync(connection, tableName); + await testUtil.destroyConnectionAsync(connection); }); const querySet = [ { sqlTexts:[ - 'create or replace database db1', + // 'create or replace database db1', 'create or replace hybrid table t1 (a int primary key, b int)', 'insert into t1 values (1, 2), (2, 3), (3, 4)' ], QccSize:2, }, + { + sqlTexts:[ + // 'create or replace database db2', + 'create or replace hybrid table t2 (a int primary key, b int)', + 'insert into t2 values (1, 2), (2, 3), (3, 4)' + ], + QccSize:2, + }, + { + sqlTexts:[ + // 'create or replace database db3', + 'create or replace hybrid table t3 (a int primary key, b int)', + 'insert into t3 values (1, 2), (2, 3), (3, 4)' + ], + QccSize:2, + }, // { // sqlTexts:[ - // 'create or replace database db2', - // 'create or replace hybrid table t2 (a int primary key, b int)', - // 'insert into t2 values (1, 2), (2, 3), (3, 4)' - // ], - // QccSize:3, - // }, - // { - // sqlTexts:[ - // 'create or replace database db3', - // 'create or replace hybrid table t3 (a int primary key, b int)', - // 'insert into t3 values (1, 2), (2, 3), (3, 4)' - // ], - // QccSize:4, - // }, - // { - // sqlTexts:[ - // 'select * from db1.public.t1 x, db2.public.t2 y, db3.public.t3 z where x.a = y.a and y.a = z.a;', - // 'select * from db1.public.t1 x, db2.public.t2 y where x.a = y.a;', - // 'select * from db2.public.t2 y, db3.public.t3 z where y.a = z.a;' + // 'select * from public.t1 x, public.t2 y, public.t3 z where x.a = y.a and y.a = z.a;', + // 'select * from public.t1 x, public.t2 y where x.a = y.a;', + // 'select * from public.t2 y, public.t3 z where y.a = z.a;' // ], // QccSize:4, // }, @@ -64,7 +64,8 @@ describe('Query Context Cache test', function () { testingfunction = function(callback) { connection.execute({ sqlText: sqlTexts[k], - complete: function () { + complete: function (err) { + assert.ok(!err,'There should be no error!'); callback(); } }); @@ -73,7 +74,7 @@ describe('Query Context Cache test', function () { else{ testingfunction = function(callback) { connection.execute({ - sqlText: sqlTexts[2], + sqlText: sqlTexts[k], complete: function (err, stmt) { assert.ok(!err,'There should be no error!'); assert.strictEqual(stmt.getQueryContextCacheSize(), QccSize); From 5492ad7ef0d88d8cb67de3ad9e6f6921de4c461f Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-jy Date: Thu, 21 Sep 2023 10:52:24 -0700 Subject: [PATCH 25/30] added details logging --- test/integration/testHTAP.js | 51 +++++++++++++++++++----------------- 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/test/integration/testHTAP.js b/test/integration/testHTAP.js index 14c7fa2a1..c83e10031 100644 --- a/test/integration/testHTAP.js +++ b/test/integration/testHTAP.js @@ -3,26 +3,31 @@ const async = require('async'); const connOption = require('./connectionOptions').valid; const testUtil = require('./testUtil'); const { configureLogger } = require('../configureLogger'); +const Logger = require('../../lib/logger'); describe('Query Context Cache test', function () { - const tableName = ['t1','t2','t3'] this.timeout(1000000); let connection; - before(() => { + before((done) => { connection = testUtil.createConnection(connOption); + async.series([ + function (callback) + { + testUtil.connect(connection, callback); + }], + done) configureLogger('TRACE'); }); after(async () => { configureLogger('ERROR'); - await testUtil.dropTablesIgnoringErrorsAsync(connection, tableName); await testUtil.destroyConnectionAsync(connection); }); const querySet = [ { sqlTexts:[ - // 'create or replace database db1', + 'create or replace database db1', 'create or replace hybrid table t1 (a int primary key, b int)', 'insert into t1 values (1, 2), (2, 3), (3, 4)' ], @@ -30,28 +35,28 @@ describe('Query Context Cache test', function () { }, { sqlTexts:[ - // 'create or replace database db2', + 'create or replace database db2', 'create or replace hybrid table t2 (a int primary key, b int)', 'insert into t2 values (1, 2), (2, 3), (3, 4)' ], - QccSize:2, + QccSize:3, }, { sqlTexts:[ - // 'create or replace database db3', + 'create or replace database db3', 'create or replace hybrid table t3 (a int primary key, b int)', 'insert into t3 values (1, 2), (2, 3), (3, 4)' ], - QccSize:2, + QccSize:4, + }, + { + sqlTexts:[ + 'select * from db1.public.t1 x, db2.public.t2 y, db3.public.t3 z where x.a = y.a and y.a = z.a;', + 'select * from db1.public.t1 x, db2.public.t2 y where x.a = y.a;', + 'select * from db2.public.t2 y, db3.public.t3 z where y.a = z.a;' + ], + QccSize:4, }, - // { - // sqlTexts:[ - // 'select * from public.t1 x, public.t2 y, public.t3 z where x.a = y.a and y.a = z.a;', - // 'select * from public.t1 x, public.t2 y where x.a = y.a;', - // 'select * from public.t2 y, public.t3 z where y.a = z.a;' - // ], - // QccSize:4, - // }, ]; function createQueryTest () { @@ -65,6 +70,9 @@ describe('Query Context Cache test', function () { connection.execute({ sqlText: sqlTexts[k], complete: function (err) { + if(err){ + Logger.getInstance().trace("The error occurs for the testHTAP", err.message); + } assert.ok(!err,'There should be no error!'); callback(); } @@ -76,6 +84,9 @@ describe('Query Context Cache test', function () { connection.execute({ sqlText: sqlTexts[k], complete: function (err, stmt) { + if(err){ + Logger.getInstance().trace("The error occurs for the testHTAP", err.message); + } assert.ok(!err,'There should be no error!'); assert.strictEqual(stmt.getQueryContextCacheSize(), QccSize); assert.strictEqual(stmt.getQueryContextDTOSize(), QccSize); @@ -94,14 +105,6 @@ describe('Query Context Cache test', function () { const queryTests = createQueryTest(); async.series( [ - function (callback) { - connection.connect(function (err, conn) { - assert.ok(!err, 'there should be no error'); - assert.strictEqual(conn, connection, - 'the connect() callback should be invoked with the statement'); - callback(); - }); - }, ...queryTests ], function () { From 7cc15af2abf2fc0a921ea11ebaa41dee0644c063 Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-jy Date: Thu, 21 Sep 2023 12:53:29 -0700 Subject: [PATCH 26/30] fixed testing issues --- test/integration/testHTAP.js | 185 ++++++++++++++++------------------- 1 file changed, 87 insertions(+), 98 deletions(-) diff --git a/test/integration/testHTAP.js b/test/integration/testHTAP.js index c83e10031..61b2f120d 100644 --- a/test/integration/testHTAP.js +++ b/test/integration/testHTAP.js @@ -2,113 +2,102 @@ const assert = require('assert'); const async = require('async'); const connOption = require('./connectionOptions').valid; const testUtil = require('./testUtil'); -const { configureLogger } = require('../configureLogger'); -const Logger = require('../../lib/logger'); - -describe('Query Context Cache test', function () { - this.timeout(1000000); - let connection; - before((done) => { - connection = testUtil.createConnection(connOption); - async.series([ - function (callback) - { +if(process.env.CLOUD_PROVIDER === 'AWS'){ + describe('Query Context Cache test', function () { + this.timeout(1000000); + let connection; + before((done) => { + connection = testUtil.createConnection(connOption); + async.series([ + function (callback) { testUtil.connect(connection, callback); }], - done) - configureLogger('TRACE'); - }); + done); + }); - after(async () => { - configureLogger('ERROR'); - await testUtil.destroyConnectionAsync(connection); - }); - const querySet = [ - { - sqlTexts:[ - 'create or replace database db1', - 'create or replace hybrid table t1 (a int primary key, b int)', - 'insert into t1 values (1, 2), (2, 3), (3, 4)' - ], - QccSize:2, - }, - { - sqlTexts:[ - 'create or replace database db2', - 'create or replace hybrid table t2 (a int primary key, b int)', - 'insert into t2 values (1, 2), (2, 3), (3, 4)' - ], - QccSize:3, - }, - { - sqlTexts:[ - 'create or replace database db3', - 'create or replace hybrid table t3 (a int primary key, b int)', - 'insert into t3 values (1, 2), (2, 3), (3, 4)' - ], - QccSize:4, - }, - { - sqlTexts:[ - 'select * from db1.public.t1 x, db2.public.t2 y, db3.public.t3 z where x.a = y.a and y.a = z.a;', - 'select * from db1.public.t1 x, db2.public.t2 y where x.a = y.a;', - 'select * from db2.public.t2 y, db3.public.t3 z where y.a = z.a;' - ], - QccSize:4, - }, - ]; + after(async () => { + await testUtil.destroyConnectionAsync(connection); + }); + const querySet = [ + { + sqlTexts:[ + 'create or replace database db1', + 'create or replace hybrid table t1 (a int primary key, b int)', + 'insert into t1 values (1, 2), (2, 3), (3, 4)' + ], + QccSize:2, + }, + { + sqlTexts:[ + 'create or replace database db2', + 'create or replace hybrid table t2 (a int primary key, b int)', + 'insert into t2 values (1, 2), (2, 3), (3, 4)' + ], + QccSize:3, + }, + { + sqlTexts:[ + 'create or replace database db3', + 'create or replace hybrid table t3 (a int primary key, b int)', + 'insert into t3 values (1, 2), (2, 3), (3, 4)' + ], + QccSize:4, + }, + { + sqlTexts:[ + 'select * from db1.public.t1 x, db2.public.t2 y, db3.public.t3 z where x.a = y.a and y.a = z.a;', + 'select * from db1.public.t1 x, db2.public.t2 y where x.a = y.a;', + 'select * from db2.public.t2 y, db3.public.t3 z where y.a = z.a;' + ], + QccSize:4, + }, + ]; - function createQueryTest () { - const testingSet = []; - let testingfunction; - for(let i = 0; i < querySet.length; i++) { - const {sqlTexts,QccSize} = querySet[i]; - for(let k = 0; k < sqlTexts.length; k++){ - if(k!==sqlTexts.length-1){ - testingfunction = function(callback) { - connection.execute({ - sqlText: sqlTexts[k], - complete: function (err) { - if(err){ - Logger.getInstance().trace("The error occurs for the testHTAP", err.message); + function createQueryTest () { + const testingSet = []; + let testingfunction; + for(let i = 0; i < querySet.length; i++) { + const {sqlTexts,QccSize} = querySet[i]; + for(let k = 0; k < sqlTexts.length; k++){ + if(k!==sqlTexts.length-1){ + testingfunction = function(callback) { + connection.execute({ + sqlText: sqlTexts[k], + complete: function (err) { + assert.ok(!err,'There should be no error!'); + callback(); } - assert.ok(!err,'There should be no error!'); - callback(); - } - }); - }; - } - else{ - testingfunction = function(callback) { - connection.execute({ - sqlText: sqlTexts[k], - complete: function (err, stmt) { - if(err){ - Logger.getInstance().trace("The error occurs for the testHTAP", err.message); + }); + }; + } else{ + testingfunction = function(callback) { + connection.execute({ + sqlText: sqlTexts[k], + complete: function (err, stmt) { + assert.ok(!err,'There should be no error!'); + assert.strictEqual(stmt.getQueryContextCacheSize(), QccSize); + assert.strictEqual(stmt.getQueryContextDTOSize(), QccSize); + callback(); } - assert.ok(!err,'There should be no error!'); - assert.strictEqual(stmt.getQueryContextCacheSize(), QccSize); - assert.strictEqual(stmt.getQueryContextDTOSize(), QccSize); - callback(); - } - }); - }; + }); + }; + } + testingSet.push(testingfunction); } - testingSet.push(testingfunction); } + return testingSet; } - return testingSet; - } - it('test Query Context Cache', function (done) { - const queryTests = createQueryTest(); - async.series( - [ - ...queryTests - ], - function () { - done(); - }); + it('test Query Context Cache', function (done) { + const queryTests = createQueryTest(); + async.series( + [ + ...queryTests + ], + function () { + done(); + }); + }); }); -}); +} \ No newline at end of file From 519e81e1c7d9b3d973881f2bf7063cb1bba4c1cf Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-jy Date: Thu, 21 Sep 2023 16:15:28 -0700 Subject: [PATCH 27/30] fixed format and added comments for the qcc test --- test/integration/testHTAP.js | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/test/integration/testHTAP.js b/test/integration/testHTAP.js index 61b2f120d..aebb3fe7a 100644 --- a/test/integration/testHTAP.js +++ b/test/integration/testHTAP.js @@ -3,10 +3,12 @@ const async = require('async'); const connOption = require('./connectionOptions').valid; const testUtil = require('./testUtil'); -if(process.env.CLOUD_PROVIDER === 'AWS'){ +// Only the AWS servers support the hybrid table in the GitHub action. +if (process.env.CLOUD_PROVIDER === 'AWS') { describe('Query Context Cache test', function () { this.timeout(1000000); let connection; + before((done) => { connection = testUtil.createConnection(connOption); async.series([ @@ -19,6 +21,7 @@ if(process.env.CLOUD_PROVIDER === 'AWS'){ after(async () => { await testUtil.destroyConnectionAsync(connection); }); + const querySet = [ { sqlTexts:[ @@ -90,14 +93,9 @@ if(process.env.CLOUD_PROVIDER === 'AWS'){ } it('test Query Context Cache', function (done) { - const queryTests = createQueryTest(); - async.series( - [ - ...queryTests - ], - function () { - done(); - }); + async.series(createQueryTest(), function () { + done(); + }); }); }); } \ No newline at end of file From 6f0d356ef73ddcabff43715f8c5c78e6e33aa80c Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-jy Date: Thu, 21 Sep 2023 19:12:45 -0700 Subject: [PATCH 28/30] fixed a format and added DTO on the both places --- lib/connection/result/result.js | 2 +- lib/connection/statement.js | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/connection/result/result.js b/lib/connection/result/result.js index 9d55a0e6c..3534932e0 100644 --- a/lib/connection/result/result.js +++ b/lib/connection/result/result.js @@ -126,7 +126,7 @@ function Result(options) { this._statement, this._services); - this.getQueryContext = function() { + this.getQueryContext = function () { return this._queryContext; } diff --git a/lib/connection/statement.js b/lib/connection/statement.js index dd65640e3..a8ec5b5cf 100644 --- a/lib/connection/statement.js +++ b/lib/connection/statement.js @@ -1403,6 +1403,10 @@ this.sendRequest = function (statementContext, onResultAvailable) json.isInternal = statementContext.internal; } + if(!statementContext.disableQueryContextCache){ + json.queryContextDTO = statementContext.services.sf.getQueryContextDTO(); + } + var options = { method: 'POST', From 9903f0c48ae8b3890ea1e834db0b5c9832fbc712 Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-jy Date: Fri, 22 Sep 2023 00:30:10 -0700 Subject: [PATCH 29/30] added the copy right and reformat the code --- lib/connection/connection_config.js | 1 - lib/queryContextCache.js | 3 ++- test/integration/testHTAP.js | 17 +++++++---------- test/unit/query_context_cache_test.js | 11 +++++++---- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/lib/connection/connection_config.js b/lib/connection/connection_config.js index 3b77cab54..1a3954b39 100644 --- a/lib/connection/connection_config.js +++ b/lib/connection/connection_config.js @@ -471,7 +471,6 @@ function ConnectionConfig(options, validateCredentials, qaMode, clientInfo) let disableQueryContextCache = false; if (Util.exists(options.disableQueryContextCache)) { - Errors.checkArgumentValid(Util.isBoolean(options.disableQueryContextCache), ErrorCodes.ERR_CONN_CREATE_INVALID_DISABLED_QUERY_CONTEXT_CACHE); diff --git a/lib/queryContextCache.js b/lib/queryContextCache.js index 609bf6266..c68df3234 100644 --- a/lib/queryContextCache.js +++ b/lib/queryContextCache.js @@ -1,6 +1,7 @@ /* - * Copyright (c) 2012-2022 Snowflake Computing Inc. All rights reserved. + * Copyright (c) 2023 Snowflake Computing Inc. All rights reserved. */ + const Logger = require('./logger'); /** diff --git a/test/integration/testHTAP.js b/test/integration/testHTAP.js index aebb3fe7a..9a1f593c2 100644 --- a/test/integration/testHTAP.js +++ b/test/integration/testHTAP.js @@ -1,3 +1,7 @@ +/* + * Copyright (c) 2023 Snowflake Computing Inc. All rights reserved. + */ + const assert = require('assert'); const async = require('async'); const connOption = require('./connectionOptions').valid; @@ -6,16 +10,11 @@ const testUtil = require('./testUtil'); // Only the AWS servers support the hybrid table in the GitHub action. if (process.env.CLOUD_PROVIDER === 'AWS') { describe('Query Context Cache test', function () { - this.timeout(1000000); let connection; - before((done) => { + before(async () => { connection = testUtil.createConnection(connOption); - async.series([ - function (callback) { - testUtil.connect(connection, callback); - }], - done); + await testUtil.connectAsync(connection); }); after(async () => { @@ -93,9 +92,7 @@ if (process.env.CLOUD_PROVIDER === 'AWS') { } it('test Query Context Cache', function (done) { - async.series(createQueryTest(), function () { - done(); - }); + async.series(createQueryTest(), done); }); }); } \ No newline at end of file diff --git a/test/unit/query_context_cache_test.js b/test/unit/query_context_cache_test.js index 5d9cea884..139af8f95 100644 --- a/test/unit/query_context_cache_test.js +++ b/test/unit/query_context_cache_test.js @@ -1,3 +1,7 @@ +/* + * Copyright (c) 2023 Snowflake Computing Inc. All rights reserved. + */ + const QueryContextCache = require('../../lib/queryContextCache.js'); const assert = require('assert'); @@ -80,7 +84,6 @@ function TestingQCC () { describe('Query Context Cache Test', function () { const testingQcc = new TestingQCC(); - /** Test for empty cache */ it('test - the cache is empty',function () { testingQcc.initCache(); assert.strictEqual(testingQcc.qcc.getSize(), 0); @@ -105,7 +108,7 @@ describe('Query Context Cache Test', function () { // Add one more element at the end const i = MAX_CAPACITY; - const extraQCE = new QueryContextElement(BASE_ID + i, BASE_READ_TIMESTAMP + i, BASE_PRIORITY + i, CONTEXT) + const extraQCE = new QueryContextElement(BASE_ID + i, BASE_READ_TIMESTAMP + i, BASE_PRIORITY + i, CONTEXT); testingQcc.qcc.merge(extraQCE); testingQcc.qcc.checkCacheCapacity(); @@ -134,7 +137,7 @@ describe('Query Context Cache Test', function () { const updatedID = 3; const updatedPriority = BASE_PRIORITY + updatedID + 7; testingQcc.expectedPriority[updatedID] = updatedPriority; - const updatedQCE = new QueryContextElement(BASE_ID + updatedID, BASE_READ_TIMESTAMP + updatedID, testingQcc.expectedPriority[updatedID], CONTEXT) + const updatedQCE = new QueryContextElement(BASE_ID + updatedID, BASE_READ_TIMESTAMP + updatedID, testingQcc.expectedPriority[updatedID], CONTEXT); testingQcc.qcc.merge(updatedQCE); testingQcc.qcc.checkCacheCapacity(); @@ -170,7 +173,7 @@ describe('Query Context Cache Test', function () { // Add one more element with same priority const i = 2; - const samePriorityQCE = new QueryContextElement(BASE_ID + i, BASE_READ_TIMESTAMP + i - 10, BASE_PRIORITY + i, CONTEXT) + const samePriorityQCE = new QueryContextElement(BASE_ID + i, BASE_READ_TIMESTAMP + i - 10, BASE_PRIORITY + i, CONTEXT); testingQcc.qcc.merge(samePriorityQCE); testingQcc.qcc.checkCacheCapacity(); From f3c2d406f21f612b6bde61c18d56556ed80ffc85 Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-jy Date: Fri, 22 Sep 2023 01:25:51 -0700 Subject: [PATCH 30/30] removed comments and spaces and ran lint fix --- lib/queryContextCache.js | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/lib/queryContextCache.js b/lib/queryContextCache.js index c68df3234..430e3a0e6 100644 --- a/lib/queryContextCache.js +++ b/lib/queryContextCache.js @@ -23,10 +23,8 @@ function QueryContextElement (id,timestamp,priority,context) { */ /** - * * @param {Number} capacity Maximum capacity of the cache. */ - function QueryContextCache (capacity) { this.capacity = capacity; this.idMap = new Map(); // Map for id and QCC @@ -97,14 +95,12 @@ QueryContextCache.prototype.merge = function (newQCE) { // Change in priority this.replaceQCE(qce, newQCE); } - } - else if (newQCE.timestamp === qce.timestamp && qce.priority !== newQCE.priority) { + } else if (newQCE.timestamp === qce.timestamp && qce.priority !== newQCE.priority) { // Same read timestamp but change in priority this.replaceQCE(qce, newQCE); } - } // id found - else { + } else { // new id if (this.priorityMap.has(newQCE.priority)) { @@ -223,12 +219,10 @@ QueryContextCache.prototype.deserializeQueryContextElement = function (node) { if(typeof context === 'string'){ entry.context = context; - } - else if (context === null || context === undefined) { + } else if (context === null || context === undefined) { entry.context = null; Logger.getInstance().debug('deserializeQueryContextElement `context` field is empty'); - } - else { + } else { Logger.getInstance().warn('deserializeQueryContextElement: `context` field is not String type'); return null; } @@ -238,11 +232,11 @@ QueryContextCache.prototype.deserializeQueryContextElement = function (node) { QueryContextCache.prototype.logCacheEntries = function () { - this.treeSet.forEach(function(elem) { - Logger.getInstance().debug( - `Cache Entry: id: ${elem.id} timestamp: ${elem.timestamp} priority: ${elem.priority}`, - ); - }); + this.treeSet.forEach(function(elem) { + Logger.getInstance().debug( + `Cache Entry: id: ${elem.id} timestamp: ${elem.timestamp} priority: ${elem.priority}`, + ); + }); }; QueryContextCache.prototype.getSize = function() {