From 6de55e6208f8cd0888561609ce0d23adb7221aab Mon Sep 17 00:00:00 2001 From: John Yun <140559986+sfc-gh-ext-simba-jy@users.noreply.github.com> Date: Fri, 22 Sep 2023 17:47:43 +0900 Subject: [PATCH] SNOW-871416: [HTAP] Query Context Caching for Node (#631) * Query Context Cache implementation --- lib/connection/connection_config.js | 20 +- lib/connection/result/result.js | 7 +- lib/connection/statement.js | 20 +- lib/constants/error_messages.js | 1 + lib/errors.js | 1 + lib/parameters.js | 7 + lib/queryContextCache.js | 271 ++++++++++++++++++ lib/services/sf.js | 47 ++- test/integration/testHTAP.js | 98 +++++++ .../unit/connection/connection_config_test.js | 29 +- test/unit/mock/mock_http_client.js | 20 +- test/unit/query_context_cache_test.js | 223 ++++++++++++++ 12 files changed, 729 insertions(+), 15 deletions(-) create mode 100644 lib/queryContextCache.js create mode 100644 test/integration/testHTAP.js create mode 100644 test/unit/query_context_cache_test.js diff --git a/lib/connection/connection_config.js b/lib/connection/connection_config.js index ede1870c2..1a3954b39 100644 --- a/lib/connection/connection_config.js +++ b/lib/connection/connection_config.js @@ -48,7 +48,8 @@ const DEFAULT_PARAMS = 'validateDefaultParameters', 'arrayBindingThreshold', 'gcsUseDownscopedCredential', - 'forceStageBindError' + 'forceStageBindError', + 'disableQueryContextCache', ]; function consolidateHostAndAccount(options) @@ -468,6 +469,14 @@ function ConnectionConfig(options, validateCredentials, qaMode, clientInfo) forceStageBindError = options.forceStageBindError; } + let disableQueryContextCache = false; + if (Util.exists(options.disableQueryContextCache)) { + Errors.checkArgumentValid(Util.isBoolean(options.disableQueryContextCache), + ErrorCodes.ERR_CONN_CREATE_INVALID_DISABLED_QUERY_CONTEXT_CACHE); + + disableQueryContextCache = options.disableQueryContextCache; + } + if (validateDefaultParameters) { for (const [key] of Object.entries(options)) @@ -747,6 +756,15 @@ function ConnectionConfig(options, validateCredentials, qaMode, clientInfo) return forceStageBindError; }; + /** + * Returns whether the Query Context Cache is enabled or not by the configuration + * + * @returns {Boolean} + */ + this.getDisableQueryContextCache = function () { + return disableQueryContextCache; + } + // save config options this.username = options.username; this.password = options.password; diff --git a/lib/connection/result/result.js b/lib/connection/result/result.js index f71f57b97..3534932e0 100644 --- a/lib/connection/result/result.js +++ b/lib/connection/result/result.js @@ -53,6 +53,7 @@ function Result(options) { this._returnedRows = data.returned; this._totalRows = data.total; this._statementTypeId = data.statementTypeId; + this._queryContext = data.queryContext; // if no chunk headers were specified, but a query-result-master-key (qrmk) // was specified, build the chunk headers from the qrmk @@ -73,7 +74,7 @@ function Result(options) { // convert the parameters array to a map parametersMap = {}; - parametersArray = data.parameters; + parametersArray = data.parameters || []; for (index = 0, length = parametersArray.length; index < length; index++) { parameter = parametersArray[index]; parametersMap[parameter.name] = parameter.value; @@ -125,6 +126,10 @@ function Result(options) { this._statement, this._services); + this.getQueryContext = function () { + return this._queryContext; + } + /* Disable the ChunkCache until the implementation is complete. * * // create a chunk cache and save a reference to it in case we need to diff --git a/lib/connection/statement.js b/lib/connection/statement.js index 6adbca25e..a8ec5b5cf 100644 --- a/lib/connection/statement.js +++ b/lib/connection/statement.js @@ -576,6 +576,15 @@ function BaseStatement( sendCancelStatement(context, statement, callback); }; + //Integration Testing purpose. + this.getQueryContextCacheSize = function () { + return services.sf.getQueryContextCacheSize(); + } + + this.getQueryContextDTOSize = function () { + return services.sf.getQueryContextDTO().entries.length; + } + /** * Issues a request to get the statement result again. * @@ -806,8 +815,9 @@ function createOnStatementRequestSuccRow(statement, context) connectionConfig: context.connectionConfig, rowMode: context.rowMode }); - // save the query id + context.queryId = context.result.getQueryId(); + this.services.sf.deserializeQueryContext(context.result.getQueryContext()); } } else @@ -1334,6 +1344,10 @@ function sendRequestPreExec(statementContext, onResultAvailable) json.isInternal = statementContext.internal; } + if(!statementContext.disableQueryContextCache){ + json.queryContextDTO = statementContext.services.sf.getQueryContextDTO(); + } + // use the snowflake service to issue the request sendSfRequest(statementContext, { @@ -1389,6 +1403,10 @@ this.sendRequest = function (statementContext, onResultAvailable) json.isInternal = statementContext.internal; } + if(!statementContext.disableQueryContextCache){ + json.queryContextDTO = statementContext.services.sf.getQueryContextDTO(); + } + var options = { method: 'POST', diff --git a/lib/constants/error_messages.js b/lib/constants/error_messages.js index 0b2887103..248c3975d 100644 --- a/lib/constants/error_messages.js +++ b/lib/constants/error_messages.js @@ -64,6 +64,7 @@ exports[404037] = 'Invalid arrayBindingThreshold. The specified value must be a exports[404038] = 'Invalid gcsUseDownscopedCredential. The specified value must be a boolean.'; exports[404039] = 'Invalid forceStageBindError. The specified value must be a number.'; exports[404040] = 'Invalid browser timeout value. The specified value must be a positive number.'; +exports[404041] = 'Invalid disablQueryContextCache. The specified value must be a boolean.'; // 405001 exports[405001] = 'Invalid callback. The specified value must be a function.'; diff --git a/lib/errors.js b/lib/errors.js index 5361aedb0..5ad788b54 100644 --- a/lib/errors.js +++ b/lib/errors.js @@ -69,6 +69,7 @@ codes.ERR_CONN_CREATE_INVALID_ARRAY_BINDING_THRESHOLD = 404037; codes.ERR_CONN_CREATE_INVALID_GCS_USE_DOWNSCOPED_CREDENTIAL = 404038; codes.ERR_CONN_CREATE_INVALID_FORCE_STAGE_BIND_ERROR = 404039; codes.ERR_CONN_CREATE_INVALID_BROWSER_TIMEOUT = 404040; +codes.ERR_CONN_CREATE_INVALID_DISABLED_QUERY_CONTEXT_CACHE = 404041 // 405001 codes.ERR_CONN_CONNECT_INVALID_CALLBACK = 405001; diff --git a/lib/parameters.js b/lib/parameters.js index af0dd46f1..01e3ca30a 100644 --- a/lib/parameters.js +++ b/lib/parameters.js @@ -60,6 +60,7 @@ names.CLIENT_SESSION_KEEP_ALIVE_HEARTBEAT_FREQUENCY = 'CLIENT_SESSION_KEEP_ALIVE names.JS_TREAT_INTEGER_AS_BIGINT = 'JS_TREAT_INTEGER_AS_BIGINT'; names.CLIENT_STAGE_ARRAY_BINDING_THRESHOLD = 'CLIENT_STAGE_ARRAY_BINDING_THRESHOLD'; names.MULTI_STATEMENT_COUNT = 'MULTI_STATEMENT_COUNT'; +names.QUERY_CONTEXT_CACHE_SIZE = 'QUERY_CONTEXT_CACHE_SIZE'; var parameters = [ @@ -106,6 +107,12 @@ var parameters = value: 1, desc: 'When 1, multi statement is disable, when 0, multi statement is unlimited' }), + new Parameter( + { + name: names.QUERY_CONTEXT_CACHE_SIZE, + value: 5, + desc: 'Query Context Cache Size' + }), ]; // put all the parameters in a map so they're easy to retrieve and update diff --git a/lib/queryContextCache.js b/lib/queryContextCache.js new file mode 100644 index 000000000..430e3a0e6 --- /dev/null +++ b/lib/queryContextCache.js @@ -0,0 +1,271 @@ +/* + * Copyright (c) 2023 Snowflake Computing Inc. All rights reserved. + */ + +const Logger = require('./logger'); + +/** + * + * @param {String} id + * @param {Number} timestamp + * @param {Number} priority + * @param {String} context + */ +function QueryContextElement (id,timestamp,priority,context) { + this.id = id; + this.timestamp = timestamp; + this.priority = priority; + this.context = context; +} + +/** + * Most Recently Used and Priority based cache. A separate cache for each connection in the driver. + */ + +/** + * @param {Number} capacity Maximum capacity of the cache. + */ +function QueryContextCache (capacity) { + this.capacity = capacity; + this.idMap = new Map(); // Map for id and QCC + this.treeSet = new Set(); // Order data as per priority + this.priorityMap = new Map(); // Map for priority and QCC +} + +QueryContextCache.prototype.sortTreeSet = function () { + this.treeSet = new Set(Array.from(this.treeSet).sort((a,b)=>a.priority-b.priority)); +}; + +QueryContextCache.prototype.addQCE= function (qce) { + this.idMap.set(qce.id,qce); + this.priorityMap.set(qce.priority,qce); + this.treeSet.add(qce); + this.sortTreeSet(); +}; + +/** + * Remove an element from the cache. + * + * @param {Object} qce element to remove. + */ +QueryContextCache.prototype.removeQCE = function (qce) { + this.idMap.delete(qce.id); + this.priorityMap.delete(qce.priority); + this.treeSet.delete(qce); +}; + +/** + * Replace the cache element with a new response element. Remove old element exist in the cache + * and add a new element received. + *{ + * @param {Object} oldQCE an element exist in the cache + * @param {Object} newQCE a new element just received. + */ +QueryContextCache.prototype.replaceQCE = function (oldQCE, newQCE) { + + // Remove old element from the cache + this.removeQCE(oldQCE); + // Add new element in the cache + this.addQCE(newQCE); +}; + +/** + * Merge a new element comes from the server with the existing cache. Merge is based on read time + * stamp for the same id and based on priority for two different ids. + * + * @param {Number} id + * @param {Number} timestamp + * @param {Number} priority + * @param {String} context + * + */ +QueryContextCache.prototype.merge = function (newQCE) { + if (this.idMap.has(newQCE.id)) { + + // ID found in the cache + const qce = this.idMap.get(newQCE.id); + if (newQCE.timestamp > qce.timestamp) { + if (qce.priority === newQCE.priority) { + + // Same priority, overwrite new data at same place + qce.timestamp = newQCE.timestamp; + qce.context = newQCE.context; + } else { + + // Change in priority + this.replaceQCE(qce, newQCE); + } + } else if (newQCE.timestamp === qce.timestamp && qce.priority !== newQCE.priority) { + + // Same read timestamp but change in priority + this.replaceQCE(qce, newQCE); + } + } else { + + // new id + if (this.priorityMap.has(newQCE.priority)) { + + // Same priority with different id + const qce = this.priorityMap.get(newQCE.priority); + + // Replace with new data + this.replaceQCE(qce, newQCE); + } else { + + // new priority + // Add new element in the cache + this.addQCE(newQCE,newQCE); + } + } +}; + +/** + * After the merge, loop through priority list and make sure cache is at most capacity. Remove all + * other elements from the list based on priority. + */ +QueryContextCache.prototype.checkCacheCapacity = function () { + Logger.getInstance().debug( + `checkCacheCapacity() called. treeSet size ${this.treeSet.size} cache capacity ${this.capacity}` ); + + // remove elements based on priority + while (this.treeSet.size > this.capacity) { + const qce = Array.from(this.treeSet).pop(); + this.removeQCE(qce); + } + Logger.getInstance().debug( + `checkCacheCapacity() returns. treeSet size ${this.treeSet.size} cache capacity ${this.capacity}`, + ); +}; + +/** Clear the cache. */ +QueryContextCache.prototype.clearCache = function () { + Logger.getInstance().debug('clearCache() called'); + this.idMap.clear(); + this.priorityMap.clear(); + this.treeSet.clear(); + Logger.getInstance().debug(`clearCache() returns. Number of entries in cache now ${this.treeSet.size}`,); +}; + +QueryContextCache.prototype.getElements = function () { + return this.treeSet; +}; + +/** + * @param data: the QueryContext Object serialized as a JSON format string + */ +QueryContextCache.prototype.deserializeQueryContext = function (data) { + const stringifyData = JSON.stringify(data); + Logger.getInstance().debug(`deserializeQueryContext() called: data from server: ${stringifyData}`); + if (!data || stringifyData ==='{}' || data.entries === null) { + + this.clearCache(); + Logger.getInstance().debug('deserializeQueryContext() returns'); + this.logCacheEntries(); + return; + } + try { + // Deserialize the entries. The first entry with priority is the main entry. An example JSON is: + // { + // "entries": [ + // { + // "id": 0, + // "readtimestamp": 123456789, + // "priority": 0, + // "context": "base64 encoded context" + // }, + // { + // "id": 1, + // "readtimestamp": 123456789, + // "priority": 1, + // "context": "base64 encoded context" + // }, + // { + // "id": 2, + // "readtimestamp": 123456789, + // "priority": 2, + // "context": "base64 encoded context" + // } + // ] + + const entries = data.entries; + if (entries !== null && Array.isArray(entries)) { + for (const entryNode of entries) { + const entry = this.deserializeQueryContextElement(entryNode); + if (entry != null) { + this.merge(entry); + } else { + Logger.getInstance().warn( + 'deserializeQueryContextJson: deserializeQueryContextElement meets mismatch field type. Clear the QueryContextCache.'); + this.clearCache(); + return; + } + } + + } + } catch (e) { + Logger.getInstance().debug(`deserializeQueryContextJson: Exception = ${e.getMessage}`, ); + + // Not rethrowing. clear the cache as incomplete merge can lead to unexpected behavior. + this.clearCache(); + } + + this.checkCacheCapacity(); + this.logCacheEntries(); +}; + +QueryContextCache.prototype.deserializeQueryContextElement = function (node) { + const {id, timestamp, priority, context} = node; + const entry = new QueryContextElement (id, timestamp, priority, null); + + if(typeof context === 'string'){ + entry.context = context; + } else if (context === null || context === undefined) { + entry.context = null; + Logger.getInstance().debug('deserializeQueryContextElement `context` field is empty'); + } else { + Logger.getInstance().warn('deserializeQueryContextElement: `context` field is not String type'); + return null; + } + + return entry; +}; + +QueryContextCache.prototype.logCacheEntries = function () { + + this.treeSet.forEach(function(elem) { + Logger.getInstance().debug( + `Cache Entry: id: ${elem.id} timestamp: ${elem.timestamp} priority: ${elem.priority}`, + ); + }); +}; + +QueryContextCache.prototype.getSize = function() { + return this.treeSet.size; +}; + +QueryContextCache.prototype.getQueryContextDTO = function () { + const arr = []; + const querycontexts =Array.from(this.getElements()); + for (let i = 0; i < this.treeSet.size; i++) { + arr.push({id:querycontexts[i].id, timestamp:querycontexts[i].timestamp, + priority:querycontexts[i].priority, context:{base64Data:querycontexts[i].context}||null}); + } + return { + entries: arr + }; +}; + +QueryContextCache.prototype.getSerializeQueryContext = function () { + const arr = []; + const querycontexts =Array.from(this.getElements()); + for (let i = 0; i < this.treeSet.size; i++) { + arr.push({id:querycontexts[i].id,timestamp:querycontexts[i].timestamp,priority:querycontexts[i].priority,context:querycontexts[i].context||null}); + } + + return { + entries: arr + }; +}; + +module.exports = QueryContextCache; + \ No newline at end of file diff --git a/lib/services/sf.js b/lib/services/sf.js index 034d35ae2..e0f714842 100644 --- a/lib/services/sf.js +++ b/lib/services/sf.js @@ -54,7 +54,7 @@ const Url = require('url'); const QueryString = require('querystring'); const Parameters = require('../parameters'); const GSErrors = require('../constants/gs_errors') - +const QueryContextCache = require('../queryContextCache'); const Logger = require('../logger'); function isRetryableNetworkError(err) @@ -265,6 +265,7 @@ function SnowflakeService(connectionConfig, httpClient, config) */ this.destroy = function (options) { + this.clearCache(); new OperationDestroy(options).validate().execute(); }; @@ -529,6 +530,41 @@ function SnowflakeService(connectionConfig, httpClient, config) { return new OperationRequest(options).validate().executeAsync(); }; + + this.getQueryContextDTO = function () { + if(!this.qcc){ + return; + } + return this.qcc.getQueryContextDTO(); + }; + + this.deserializeQueryContext = function (data) { + if(!this.qcc){ + return; + } + this.qcc.deserializeQueryContext(data); + }; + + this.clearCache = function () { + if(!this.qcc){ + return; + } + this.qcc.clearCache(); + } + + this.initializeQueryContextCache = function (size) { + if(!connectionConfig.getDisableQueryContextCache()){ + this.qcc = new QueryContextCache(size); + } + } + + // testing purpose + this.getQueryContextCacheSize = function () { + if(!this.qcc){ + return; + } + return this.qcc.getSize(); + } } Util.inherits(SnowflakeService, EventEmitter); @@ -1053,7 +1089,7 @@ StateConnecting.prototype.continue = function () var context = this.context; var err = context.options.err; var json = context.options.json; - + // if no json was specified, treat this as the first connect // and get the necessary information from connectionConfig if (!json) @@ -1126,7 +1162,7 @@ StateConnecting.prototype.continue = function () sessionParameters.SESSION_PARAMETERS.GCS_USE_DOWNSCOPED_CREDENTIAL = this.connectionConfig.getGcsUseDownscopedCredential(); } - + Util.apply(json.data, clientInfo); Util.apply(json.data, sessionParameters); @@ -1149,7 +1185,7 @@ StateConnecting.prototype.continue = function () { Errors.assertInternal(Util.exists(body)); Errors.assertInternal(Util.exists(body.data)); - + // update the parameters Parameters.update(body.data.parameters); @@ -1158,6 +1194,9 @@ StateConnecting.prototype.continue = function () // we're now connected parent.snowflakeService.transitionToConnected(); + + const qccSize = Parameters.getValue('QUERY_CONTEXT_CACHE_SIZE'); + parent.snowflakeService.initializeQueryContextCache(qccSize); } else { diff --git a/test/integration/testHTAP.js b/test/integration/testHTAP.js new file mode 100644 index 000000000..9a1f593c2 --- /dev/null +++ b/test/integration/testHTAP.js @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2023 Snowflake Computing Inc. All rights reserved. + */ + +const assert = require('assert'); +const async = require('async'); +const connOption = require('./connectionOptions').valid; +const testUtil = require('./testUtil'); + +// Only the AWS servers support the hybrid table in the GitHub action. +if (process.env.CLOUD_PROVIDER === 'AWS') { + describe('Query Context Cache test', function () { + let connection; + + before(async () => { + connection = testUtil.createConnection(connOption); + await testUtil.connectAsync(connection); + }); + + after(async () => { + await testUtil.destroyConnectionAsync(connection); + }); + + const querySet = [ + { + sqlTexts:[ + 'create or replace database db1', + 'create or replace hybrid table t1 (a int primary key, b int)', + 'insert into t1 values (1, 2), (2, 3), (3, 4)' + ], + QccSize:2, + }, + { + sqlTexts:[ + 'create or replace database db2', + 'create or replace hybrid table t2 (a int primary key, b int)', + 'insert into t2 values (1, 2), (2, 3), (3, 4)' + ], + QccSize:3, + }, + { + sqlTexts:[ + 'create or replace database db3', + 'create or replace hybrid table t3 (a int primary key, b int)', + 'insert into t3 values (1, 2), (2, 3), (3, 4)' + ], + QccSize:4, + }, + { + sqlTexts:[ + 'select * from db1.public.t1 x, db2.public.t2 y, db3.public.t3 z where x.a = y.a and y.a = z.a;', + 'select * from db1.public.t1 x, db2.public.t2 y where x.a = y.a;', + 'select * from db2.public.t2 y, db3.public.t3 z where y.a = z.a;' + ], + QccSize:4, + }, + ]; + + function createQueryTest () { + const testingSet = []; + let testingfunction; + for(let i = 0; i < querySet.length; i++) { + const {sqlTexts,QccSize} = querySet[i]; + for(let k = 0; k < sqlTexts.length; k++){ + if(k!==sqlTexts.length-1){ + testingfunction = function(callback) { + connection.execute({ + sqlText: sqlTexts[k], + complete: function (err) { + assert.ok(!err,'There should be no error!'); + callback(); + } + }); + }; + } else{ + testingfunction = function(callback) { + connection.execute({ + sqlText: sqlTexts[k], + complete: function (err, stmt) { + assert.ok(!err,'There should be no error!'); + assert.strictEqual(stmt.getQueryContextCacheSize(), QccSize); + assert.strictEqual(stmt.getQueryContextDTOSize(), QccSize); + callback(); + } + }); + }; + } + testingSet.push(testingfunction); + } + } + return testingSet; + } + + it('test Query Context Cache', function (done) { + async.series(createQueryTest(), done); + }); + }); +} \ No newline at end of file diff --git a/test/unit/connection/connection_config_test.js b/test/unit/connection/connection_config_test.js index 5a15d52a1..b8629023c 100644 --- a/test/unit/connection/connection_config_test.js +++ b/test/unit/connection/connection_config_test.js @@ -514,6 +514,17 @@ describe('ConnectionConfig: basic', function () }, errorCode: ErrorCodes.ERR_CONN_CREATE_INVALID_GCS_USE_DOWNSCOPED_CREDENTIAL }, + { + name: 'invalid disableQueryContextCache', + options: + { + username: 'username', + password: 'password', + account: 'account', + disableQueryContextCache: 1234 + }, + errorCode: ErrorCodes.ERR_CONN_CREATE_INVALID_DISABLED_QUERY_CONTEXT_CACHE + }, ]; var createNegativeITCallback = function (testCase) @@ -847,7 +858,23 @@ describe('ConnectionConfig: basic', function () accessUrl: 'https://account.snowflakecomputing.com', account: 'account' } - } + }, + { + name: 'disableQueryContextCache', + input: + { + username: 'username', + password: 'password', + account: 'account', + disableQueryContextCache: true + }, + options: + { + accessUrl: 'https://account.snowflakecomputing.com', + username: 'username', + password: 'password' + } + }, ]; var createItCallback = function (testCase) diff --git a/test/unit/mock/mock_http_client.js b/test/unit/mock/mock_http_client.js index cfefe231a..47de2823a 100644 --- a/test/unit/mock/mock_http_client.js +++ b/test/unit/mock/mock_http_client.js @@ -341,7 +341,8 @@ function buildRequestOutputMappings(clientInfo) json: { disableOfflineChunks: false, - sqlText: 'select 1 as "c1";' + sqlText: 'select 1 as "c1";', + queryContextDTO: { entries: [] }, } }, output: @@ -551,8 +552,9 @@ function buildRequestOutputMappings(clientInfo) bindings: { "1": {type: 'TEXT', value: 'false'}, - "2": {type: 'TEXT', value: '1967-06-23'} - } + "2": {type: 'TEXT', value: '1967-06-23'}, + }, + queryContextDTO: { entries: [] }, } }, output: @@ -667,7 +669,8 @@ function buildRequestOutputMappings(clientInfo) json: { disableOfflineChunks: false, - sqlText: 'select;' + sqlText: 'select;', + queryContextDTO: { entries: [] }, } }, output: @@ -977,7 +980,8 @@ function buildRequestOutputMappings(clientInfo) json: { disableOfflineChunks: false, - sqlText: 'select count(*) from table(generator(timelimit=>10));' + sqlText: 'select count(*) from table(generator(timelimit=>10));', + queryContextDTO: { entries: [] }, } }, output: @@ -1052,7 +1056,8 @@ function buildRequestOutputMappings(clientInfo) json: { disableOfflineChunks: false, - sqlText: 'select \'too many concurrent queries\';' + sqlText: 'select \'too many concurrent queries\';', + queryContextDTO: { entries: [] }, } }, output: @@ -1192,7 +1197,8 @@ function buildRequestOutputMappings(clientInfo) json: { disableOfflineChunks: false, - sqlText: 'select * from faketable' + sqlText: 'select * from faketable', + queryContextDTO: { entries: [] }, } }, output: diff --git a/test/unit/query_context_cache_test.js b/test/unit/query_context_cache_test.js new file mode 100644 index 000000000..139af8f95 --- /dev/null +++ b/test/unit/query_context_cache_test.js @@ -0,0 +1,223 @@ +/* + * Copyright (c) 2023 Snowflake Computing Inc. All rights reserved. + */ + +const QueryContextCache = require('../../lib/queryContextCache.js'); +const assert = require('assert'); + +const BASE_ID = 0; +const BASE_PRIORITY = 0; +const BASE_READ_TIMESTAMP = 1668727958; +const CONTEXT = 'Some query Context'; +const MAX_CAPACITY = 5; + +function QueryContextElement (id,timestamp,priority,context) { + this.id = id; + this.timestamp = timestamp; + this.priority = priority; + this.context = context; +} + +function TestingQCC () { + this.qcc = null; + + this.expectedIDs; + this.expectedReadTimestamp; + this.expectedPriority; + + this.initCache = function () { + this.qcc = new QueryContextCache(MAX_CAPACITY); + }; + + this.initCacheWithData = function () { + this.initCacheWithDataWithContext(CONTEXT); + }; + + this.initCacheWithDataWithContext = function (Context) { + this.qcc = new QueryContextCache(MAX_CAPACITY); + this.expectedIDs = []; + this.expectedReadTimestamp = []; + this.expectedPriority = []; + for (let i = 0; i < MAX_CAPACITY; i++) { + this.expectedIDs[i] = BASE_ID + i; + this.expectedReadTimestamp[i] = BASE_READ_TIMESTAMP + i; + this.expectedPriority[i] = BASE_PRIORITY + i; + this.qcc.merge(new QueryContextElement(this.expectedIDs[i], this.expectedReadTimestamp[i], this.expectedPriority[i], Context)); + } + }; + + this.initCacheWithDataInRandomOrder = function () { + this.qcc = new QueryContextCache(MAX_CAPACITY); + this.expectedIDs = []; + this.expectedReadTimestamp = []; + this.expectedPriority = []; + for (let i = 0; i < MAX_CAPACITY; i++) { + this.expectedIDs[i] = BASE_ID + i; + this.expectedReadTimestamp[i] = BASE_READ_TIMESTAMP + i; + this.expectedPriority[i] = BASE_PRIORITY + i; + } + + this.qcc.merge(new QueryContextElement(this.expectedIDs[3], this.expectedReadTimestamp[3], this.expectedPriority[3], CONTEXT)); + this.qcc.merge(new QueryContextElement(this.expectedIDs[2], this.expectedReadTimestamp[2], this.expectedPriority[2], CONTEXT)); + this.qcc.merge(new QueryContextElement(this.expectedIDs[4], this.expectedReadTimestamp[4], this.expectedPriority[4], CONTEXT)); + this.qcc.merge(new QueryContextElement(this.expectedIDs[0], this.expectedReadTimestamp[0], this.expectedPriority[0], CONTEXT)); + this.qcc.merge(new QueryContextElement(this.expectedIDs[1], this.expectedReadTimestamp[1], this.expectedPriority[1], CONTEXT)); + }; + + this.assertCacheData = function () { + this.assertCacheDataWithContext(CONTEXT); + }; + + this.assertCacheDataWithContext = function (Context) { + const size = this.qcc.getSize(); + assert.strictEqual(size,MAX_CAPACITY); + const elements = Array.from(this.qcc.getElements()); + for (let i = 0; i < size; i++) { + assert.strictEqual(this.expectedIDs[i], elements[i].id); + assert.strictEqual(this.expectedReadTimestamp[i], elements[i].timestamp); + assert.strictEqual(this.expectedPriority[i], elements[i].priority); + assert.strictEqual(Context, elements[i].context); + } + }; +} + +describe('Query Context Cache Test', function () { + const testingQcc = new TestingQCC(); + + it('test - the cache is empty',function () { + testingQcc.initCache(); + assert.strictEqual(testingQcc.qcc.getSize(), 0); + }); + + it('test - some elements in the cache',function () { + testingQcc.initCacheWithData(); + + // Compare elements + testingQcc.assertCacheData(); + }); + + it('test - query contexts are randomly added in the cache',function () { + testingQcc.initCacheWithDataInRandomOrder(); + + // Compare elements + testingQcc.assertCacheData(); + }); + + it('test - the number of contexts is over the size of capacity',function () { + testingQcc.initCacheWithData(); + + // Add one more element at the end + const i = MAX_CAPACITY; + const extraQCE = new QueryContextElement(BASE_ID + i, BASE_READ_TIMESTAMP + i, BASE_PRIORITY + i, CONTEXT); + testingQcc.qcc.merge(extraQCE); + testingQcc.qcc.checkCacheCapacity(); + + // Compare elements + testingQcc.assertCacheData(); + }); + + it('test updating timestamp',function () { + testingQcc.initCacheWithData(); + + // Add one more element with new TS with existing id + const updatedID = 1; + testingQcc.expectedReadTimestamp[updatedID] = BASE_READ_TIMESTAMP + updatedID + 10; + const updatedQCE = new QueryContextElement(BASE_ID + updatedID, testingQcc.expectedReadTimestamp[updatedID], BASE_PRIORITY + updatedID, CONTEXT); + testingQcc.qcc.merge(updatedQCE); + testingQcc.qcc.checkCacheCapacity(); + + // Compare elements + testingQcc.assertCacheData(); + }); + + it('test updating priority', function () { + testingQcc.initCacheWithData(); + + // Add one more element with new priority with existing id + const updatedID = 3; + const updatedPriority = BASE_PRIORITY + updatedID + 7; + testingQcc.expectedPriority[updatedID] = updatedPriority; + const updatedQCE = new QueryContextElement(BASE_ID + updatedID, BASE_READ_TIMESTAMP + updatedID, testingQcc.expectedPriority[updatedID], CONTEXT); + testingQcc.qcc.merge(updatedQCE); + testingQcc.qcc.checkCacheCapacity(); + + for (let i = updatedID; i < MAX_CAPACITY - 1; i++) { + testingQcc.expectedIDs[i] = testingQcc.expectedIDs[i + 1]; + testingQcc.expectedReadTimestamp[i] = testingQcc.expectedReadTimestamp[i + 1]; + testingQcc.expectedPriority[i] = testingQcc.expectedPriority[i + 1]; + } + + testingQcc.expectedIDs[MAX_CAPACITY - 1] = BASE_ID + updatedID; + testingQcc.expectedReadTimestamp[MAX_CAPACITY - 1] = BASE_READ_TIMESTAMP + updatedID; + testingQcc.expectedPriority[MAX_CAPACITY - 1] = updatedPriority; + testingQcc.assertCacheData(); + }); + + it('test - the same priority is added',function () { + testingQcc.initCacheWithData(); + + // Add one more element with same priority + const i = MAX_CAPACITY; + const updatedPriority = BASE_PRIORITY + 1; + testingQcc.qcc.merge(new QueryContextElement(BASE_ID + i, BASE_READ_TIMESTAMP + i, updatedPriority, CONTEXT)); + testingQcc.qcc.checkCacheCapacity(); + testingQcc.expectedIDs[1] = BASE_ID + i; + testingQcc.expectedReadTimestamp[1] = BASE_READ_TIMESTAMP + i; + + // Compare elements + testingQcc.assertCacheData(); + }); + + it('test - the new context has the same id but different timestamp ', function () { + testingQcc.initCacheWithData(); + + // Add one more element with same priority + const i = 2; + const samePriorityQCE = new QueryContextElement(BASE_ID + i, BASE_READ_TIMESTAMP + i - 10, BASE_PRIORITY + i, CONTEXT); + testingQcc.qcc.merge(samePriorityQCE); + testingQcc.qcc.checkCacheCapacity(); + + // Compare elements + testingQcc.assertCacheData(); + }); + + it('test empty cache with null data', function () { + testingQcc.initCacheWithData(); + testingQcc.qcc.deserializeQueryContext(null); + assert.strictEqual(testingQcc.qcc.getSize(),0,'Empty cache'); + }); + + it('test empty cache with empty response data',function () { + testingQcc.initCacheWithData(); + testingQcc.qcc.deserializeQueryContext({}); + assert.strictEqual(testingQcc.qcc.getSize(),0,'Empty cache'); + }); + + it('test serialized request and deserialize response data', function () { + testingQcc.initCacheWithData(); + testingQcc.assertCacheData(); + + const response = testingQcc.qcc.getSerializeQueryContext(); + + // Clear testingQcc.qcc + testingQcc.qcc.clearCache(); + assert.strictEqual(testingQcc.qcc.getSize(),0,'Empty cache'); + testingQcc.qcc.deserializeQueryContext(response); + testingQcc.assertCacheData(); + }); + + it('test serialized request and deserialize response data when context is null', function () { + + // Init testingQcc.qcc + testingQcc.initCacheWithDataWithContext(null); + testingQcc.assertCacheDataWithContext(null); + const response = testingQcc.qcc.getSerializeQueryContext(); + + //Clear testingQcc.qcc + testingQcc.qcc.clearCache(); + assert.strictEqual(testingQcc.qcc.getSize(), 0,'Empty cache'); + + testingQcc.qcc.deserializeQueryContext(response); + testingQcc.assertCacheDataWithContext(null); + }); +}); \ No newline at end of file