From f0abbf9e239317e69538e63c70023ddf7bec8e03 Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Thu, 19 Oct 2023 17:20:13 -0700 Subject: [PATCH 01/15] SNOW-502598: Add async query execution --- lib/connection/connection.js | 185 ++++++++++++++++++++++++++++++++ lib/connection/statement.js | 47 +++++++- lib/constants/error_messages.js | 10 +- lib/constants/query_status.js | 22 ++++ lib/errors.js | 6 ++ lib/http/base.js | 38 +++++++ lib/services/sf.js | 41 ++++--- 7 files changed, 326 insertions(+), 23 deletions(-) create mode 100644 lib/constants/query_status.js diff --git a/lib/connection/connection.js b/lib/connection/connection.js index 7e1b31a02..88af4eb77 100644 --- a/lib/connection/connection.js +++ b/lib/connection/connection.js @@ -5,6 +5,7 @@ const { v4: uuidv4 } = require('uuid'); const Url = require('url'); const QueryString = require('querystring'); const GSErrors = require('../constants/gs_errors') +const QueryStatus = require('../constants/query_status').code; var Util = require('../util'); var Errors = require('../errors'); @@ -37,6 +38,15 @@ function Connection(context) // generate an id for the connection var id = uuidv4(); + // async max retry and retry pattern from python connector + const asyncNoDataMaxRetry = 24; + const asyncRetryPattern = [1, 1, 2, 3, 4, 8, 10]; + const asyncRetryInMilliseconds = 500; + + // Custom regex based on uuid validate + // Unable to directly use uuid validate because the queryId returned from the server doesn't match the regex + const queryIdRegex = new RegExp(/^(?:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}|00000000-0000-0000-0000-000000000000)$/i); + //Make session tokens available for testing this.getTokens = function () { @@ -411,6 +421,181 @@ function Connection(context) return this; }; + /** + * Gets the response containing the status of the query based on queryId. + * + * @param {String} queryId + * + * @returns {Object} the query response + */ + async function getQueryResponse(queryId) { + // Check if queryId exists and is valid uuid + Errors.checkArgumentExists(Util.exists(queryId), + ErrorCodes.ERR_CONN_FETCH_RESULT_MISSING_QUERY_ID); + Errors.checkArgumentValid(queryIdRegex.test(queryId), + ErrorCodes.ERR_GET_RESPONSE_QUERY_INVALID_UUID, queryId); + + // Form the request options + const options = + { + method: 'GET', + url: Url.format( + { + pathname: `/monitoring/queries/${queryId}` + }), + }; + + // Get the response containing the query status + const response = await services.sf.requestAsync(options); + + return response['body']; + } + + /** + * Extracts the status of the query from the query response. + * + * @param {Object} queryResponse + * + * @returns {String} the query status. + */ + function extractQueryStatus(queryResponse) { + const queries = queryResponse['data']['queries']; + let status = QueryStatus.NO_DATA; // default status + if (queries.length > 0) { + status = queries[0]['status']; + } + + return status; + } + + /** + * Gets the status of the query based on queryId. + * + * @param {String} queryId + * + * @returns {String} the query status. + */ + this.getQueryStatus = async function (queryId) { + return extractQueryStatus(await getQueryResponse(queryId)); + }; + + /** + * Gets the status of the query based on queryId and throws if there's an error. + * + * @param {String} queryId + * + * @returns {String} the query status. + */ + this.getQueryStatusThrowIfError = async function (queryId) { + const response = await getQueryResponse(queryId); + const queries = response['data']['queries']; + const status = extractQueryStatus(response); + let message, code, sqlState = null; + + if (this.isAnError(status)) { + message = response['message'] || ''; + code = response['code'] || -1; + + if (response['data']) { + message += queries.length > 0 ? queries[0]['errorMessage'] : ''; + sqlState = response['data']['sqlState']; + } + + throw Errors.createOperationFailedError( + code, response, message, sqlState); + } + + return status; + }; + + /** + * Gets the results from a previously ran query based on queryId + * + * @param {Object} options + * + * @returns {Object} + */ + this.getResultsFromQueryId = async function (options) { + const queryId = options.queryId; + let status, noDataCounter = 0, retryPatternPos = 0; + + // Wait until query has finished executing + for (;;) { + // Check if query is still running and trigger exception if it failed + status = await this.getQueryStatusThrowIfError(queryId); + if (!this.isStillRunning(status)) { + break; + } + + // Timeout based on python connector + await new Promise((resolve) => { + setTimeout(() => resolve(), asyncRetryInMilliseconds * asyncRetryPattern[retryPatternPos]); + }); + + // If no data, increment the no data counter + if (QueryStatus[status] == QueryStatus.NO_DATA) { + noDataCounter++; + // Check if retry for no data is exceeded + if (noDataCounter > asyncNoDataMaxRetry) { + throw Errors.createClientError( + ErrorCodes.ERR_GET_RESULTS_QUERY_ID_NO_DATA, true, queryId); + } + } + + if (retryPatternPos < asyncRetryPattern.length - 1) { + retryPatternPos++; + } + } + + if (QueryStatus[status] != QueryStatus.SUCCESS) { + throw Errors.createClientError( + ErrorCodes.ERR_GET_RESULTS_QUERY_ID_NOT_SUCCESS_STATUS, true, queryId, status); + } + + return this.fetchResult(options); + }; + + /** + * Checks whether the given status is currently running. + * + * @param {String} status + * + * @returns {Boolean} + */ + this.isStillRunning = function (status) { + const stillRunning = + [ + QueryStatus.RUNNING, + QueryStatus.RESUMING_WAREHOUSE, + QueryStatus.QUEUED, + QueryStatus.QUEUED_REPARING_WAREHOUSE, + QueryStatus.NO_DATA, + ].includes(QueryStatus[status]); + + return stillRunning; + }; + + /** + * Checks whether the given status means that there has been an error. + * + * @param {String} status + * + * @returns {Boolean} + */ + this.isAnError = function (status) { + const isAnError = + [ + QueryStatus.ABORTING, + QueryStatus.FAILED_WITH_ERROR, + QueryStatus.ABORTED, + QueryStatus.FAILED_WITH_INCIDENT, + QueryStatus.DISCONNECTED, + QueryStatus.BLOCKED, + ].includes(QueryStatus[status]); + + return isAnError; + }; + /** * Returns a serialized version of this connection. * diff --git a/lib/connection/statement.js b/lib/connection/statement.js index 130d6bc30..a96c7f7d1 100644 --- a/lib/connection/statement.js +++ b/lib/connection/statement.js @@ -33,6 +33,11 @@ var statementTypes = FILE_POST_EXEC: 'FILE_POST_EXEC' }; +const queryCodes = { + QUERY_IN_PROGRESS: '333333', // GS code: the query is in progress + QUERY_IN_PROGRESS_ASYNC: '333334' // GS code: the query is detached +}; + exports.createContext = function ( options, services, connectionConfig) { @@ -187,6 +192,12 @@ exports.createStatementPostExec = function ( JSON.stringify(fetchAsString[invalidValueIndex])); } + // check for invalid asyncExec + if (Util.exists(statementOptions.asyncExec)) { + Errors.checkArgumentValid(Util.isBoolean(statementOptions.asyncExec), + ErrorCodes.ERR_CONN_CREATE_INVALID_ASYNC_EXEC); + } + const rowMode = statementOptions.rowMode; if (Util.exists(rowMode)) { RowMode.checkRowModeValid(rowMode); @@ -206,6 +217,7 @@ exports.createStatementPostExec = function ( statementContext.multiResultIds = statementOptions.multiResultIds; statementContext.multiCurId = statementOptions.multiCurId; statementContext.rowMode = statementOptions.rowMode; + statementContext.asyncExec = statementOptions.asyncExec; // set the statement type statementContext.type = (statementContext.type == statementTypes.ROW_PRE_EXEC) ? statementTypes.ROW_POST_EXEC : statementTypes.FILE_POST_EXEC; @@ -364,6 +376,12 @@ function createContextPreExec( RowMode.checkRowModeValid(rowMode); } + // if an asyncExec flag is specified, make sure it's boolean + if (Util.exists(statementOptions.asyncExec)) { + Errors.checkArgumentValid(Util.isBoolean(statementOptions.asyncExec), + ErrorCodes.ERR_CONN_CREATE_INVALID_ASYNC_EXEC); + } + // create a statement context var statementContext = createStatementContext(); @@ -374,6 +392,7 @@ function createContextPreExec( statementContext.multiResultIds = statementOptions.multiResultIds; statementContext.multiCurId = statementOptions.multiCurId; statementContext.rowMode = statementOptions.rowMode; + statementContext.asyncExec = statementOptions.asyncExec; // if a binds array is specified, add it to the statement context if (Util.exists(statementOptions.binds)) @@ -682,10 +701,13 @@ function invokeStatementComplete(statement, context) streamResult = context.connectionConfig.getStreamResult(); } - // if the result will be streamed later, + // if the result will be streamed later or in asyncExec mode, // invoke the complete callback right away if (streamResult) { context.complete(Errors.externalize(context.resultError), statement); + } else if (context.asyncExec) { + // return the result object with the query ID inside. + context.complete(null, statement, context.result); } else { process.nextTick(function () { // aggregate all the rows into an array and pass this @@ -779,6 +801,12 @@ function createOnStatementRequestSuccRow(statement, context) // if we don't already have a result if (!context.result) { + if (body.code === queryCodes.QUERY_IN_PROGRESS_ASYNC) { + context.result = { + queryId: body.data.queryId + }; + return; + } if (body.data.resultIds != undefined && body.data.resultIds.length > 0) { //multi statements @@ -1330,6 +1358,11 @@ function sendRequestPreExec(statementContext, onResultAvailable) json.queryContextDTO = statementContext.services.sf.getQueryContextDTO(); } + // include the asyncExec flag if a value was specified + if (Util.exists(statementContext.asyncExec)) { + json.asyncExec = statementContext.asyncExec; + } + // use the snowflake service to issue the request sendSfRequest(statementContext, { @@ -1645,9 +1678,15 @@ function buildResultRequestCallback( statementContext.queryId = body.data.queryId; // if the result is not ready yet, extract the result url from the response - // and issue a GET request to try to fetch the result again - if (body && (body.code === '333333' || body.code === '333334')) - { + // and issue a GET request to try to fetch the result again unless asyncExec is enabled. + if (body && (body.code === queryCodes.QUERY_IN_PROGRESS + || body.code === queryCodes.QUERY_IN_PROGRESS_ASYNC)) { + + if (statementContext.asyncExec) { + await onResultAvailable.call(null, err, body); + return; + } + // extract the result url from the response and try to get the result // again sendSfRequest(statementContext, diff --git a/lib/constants/error_messages.js b/lib/constants/error_messages.js index ad6b407c2..d8990e88d 100644 --- a/lib/constants/error_messages.js +++ b/lib/constants/error_messages.js @@ -68,6 +68,7 @@ exports[404040] = 'Invalid browser timeout value. The specified value must be a exports[404041] = 'Invalid disablQueryContextCache. The specified value must be a boolean.'; exports[404042] = 'Invalid includeRetryReason. The specified value must be a boolean.' exports[404043] = 'Invalid clientConfigFile value. The specified value must be a string.'; +exports[404044] = 'Invalid asyncExec. The specified value must be a boolean.' // 405001 exports[405001] = 'Invalid callback. The specified value must be a function.'; @@ -113,8 +114,8 @@ exports[409013] = 'Invalid requestId. The specified value must be a string.'; // 410001 exports[410001] = 'Fetch-result options must be specified.'; exports[410002] = 'Invalid options. The specified value must be an object.'; -exports[410003] = 'A statement id must be specified.'; -exports[410004] = 'Invalid statement id. The specified value must be a string.'; +exports[410003] = 'A query id/statement id must be specified.'; +exports[410004] = 'Invalid query id/statement id. The specified value must be a string.'; exports[410005] = 'Invalid complete callback. The specified value must be a function.'; exports[410006] = 'Invalid streamResult flag. The specified value must be a boolean.'; exports[410007] = 'Invalid fetchAsString value. The specified value must be an Array.'; @@ -151,3 +152,8 @@ exports[450004] = 'Invalid each() callback. The specified value must be a functi exports[450005] = 'An end() callback must be specified.'; exports[450006] = 'Invalid end() callback. The specified value must be a function.'; exports[450007] = 'Operation failed because the statement is still in progress.'; + +// 460001 +exports[460001] = 'Invalid queryId: %s'; +exports[460002] = 'Cannot retrieve data. No information returned from server for query %s'; +exports[460003] = 'Status of query %s is %s, results are unavailable'; diff --git a/lib/constants/query_status.js b/lib/constants/query_status.js new file mode 100644 index 000000000..2449aeff0 --- /dev/null +++ b/lib/constants/query_status.js @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2023 Snowflake Computing Inc. All rights reserved. + */ + +const code = {}; + +code.RUNNING = 'RUNNING'; +code.ABORTING = 'ABORTING'; +code.SUCCESS = 'SUCCESS'; +code.FAILED_WITH_ERROR = 'FAILED_WITH_ERROR'; +code.ABORTED = 'ABORTED'; +code.QUEUED = 'QUEUED'; +code.FAILED_WITH_INCIDENT = 'FAILED_WITH_INCIDENT'; +code.DISCONNECTED = 'DISCONNECTED'; +code.RESUMING_WAREHOUSE = 'RESUMING_WAREHOUSE'; +// purposeful typo.Is present in QueryDTO.java +code.QUEUED_REPARING_WAREHOUSE = 'QUEUED_REPARING_WAREHOUSE'; +code.RESTARTED = 'RESTARTED'; +code.BLOCKED = 'BLOCKED'; +code.NO_DATA = 'NO_DATA'; + +exports.code = code; diff --git a/lib/errors.js b/lib/errors.js index c04fe7b5a..b4f15f3f5 100644 --- a/lib/errors.js +++ b/lib/errors.js @@ -73,6 +73,7 @@ codes.ERR_CONN_CREATE_INVALID_BROWSER_TIMEOUT = 404040; codes.ERR_CONN_CREATE_INVALID_DISABLED_QUERY_CONTEXT_CACHE = 404041 codes.ERR_CONN_CREATE_INVALID_INCLUDE_RETRY_REASON =404042 codes.ERR_CONN_CREATE_INVALID_CLIENT_CONFIG_FILE = 404043; +codes.ERR_CONN_CREATE_INVALID_ASYNC_EXEC = 404044 // 405001 codes.ERR_CONN_CONNECT_INVALID_CALLBACK = 405001; @@ -158,6 +159,11 @@ codes.ERR_STMT_FETCH_ROWS_MISSING_END = 450005; codes.ERR_STMT_FETCH_ROWS_INVALID_END = 450006; codes.ERR_STMT_FETCH_ROWS_FETCHING_RESULT = 450007; +// 460001 +codes.ERR_GET_RESPONSE_QUERY_INVALID_UUID = 460001; +codes.ERR_GET_RESULTS_QUERY_ID_NO_DATA = 460002; +codes.ERR_GET_RESULTS_QUERY_ID_NOT_SUCCESS_STATUS = 460003; + exports.codes = codes; /** diff --git a/lib/http/base.js b/lib/http/base.js index 2d92f9650..2f83ebdf9 100644 --- a/lib/http/base.js +++ b/lib/http/base.js @@ -88,6 +88,44 @@ HttpClient.prototype.requestAsync = async function (options) return axios.request(requestOptions); }; +/** + * Issues an async HTTP request. + * + * @param {Object} options + * + * @returns {Object} an object representing the request that was issued. + */ +HttpClient.prototype.requestAsync = async function (options) { + // validate input + Errors.assertInternal(Util.isObject(options)); + Errors.assertInternal(Util.isString(options.method)); + Errors.assertInternal(Util.isString(options.url)); + Errors.assertInternal(!Util.exists(options.headers) || + Util.isObject(options.headers)); + Errors.assertInternal(!Util.exists(options.json) || + Util.isObject(options.json)); + + // normalize the headers + const headers = normalizeHeaders(options.headers); + + const timeout = options.timeout || + this._connectionConfig.getTimeout() || + DEFAULT_REQUEST_TIMEOUT; + + const requestOptions = + { + method: options.method, + url: options.url, + headers: headers, + data: options.json, + timeout: timeout + }; + + const response = await axios.request(requestOptions); + + return normalizeResponse(response); +} + /** * @abstract * Returns the module to use when making HTTP requests. Subclasses must override diff --git a/lib/services/sf.js b/lib/services/sf.js index c183bad70..82a2d43c3 100644 --- a/lib/services/sf.js +++ b/lib/services/sf.js @@ -258,6 +258,15 @@ function SnowflakeService(connectionConfig, httpClient, config) new OperationRequest(options).validate().execute(); }; + /** + * Issues a generic async request to Snowflake. + * + * @param {Object} options + */ + this.requestAsync = async function (options) { + return await new OperationRequest(options).validate().executeAsync(); + }; + /** * Terminates the current connection to Snowflake. * @@ -407,9 +416,8 @@ function SnowflakeService(connectionConfig, httpClient, config) /** * @inheritDoc */ - OperationRequest.prototype.executeAsync = function () - { - return currentState.requestAsync(this.options); + OperationRequest.prototype.executeAsync = async function () { + return await currentState.requestAsync(this.options); }; /** @@ -737,24 +745,24 @@ function StateAbstract(options) } /** - * Sends out the post request. + * Sends out the request. * * @returns {Object} the request that was issued. */ - Request.prototype.sendAsync = function () - { + Request.prototype.sendAsync = async function () { // pre-process the request options this.preprocessOptions(this.requestOptions); - var url = this.requestOptions.absoluteUrl; - var header = this.requestOptions.headers; - var body = this.requestOptions.json; + const options = + { + method: this.requestOptions.method, + headers: this.requestOptions.headers, + url: this.requestOptions.absoluteUrl, + json: this.requestOptions.json + }; - // issue the http request - return axios - .post(url, body, { - headers: header - }); + // issue the async http request + return await httpClient.requestAsync(options); }; /** @@ -1339,10 +1347,9 @@ StateConnected.prototype.connect = function (options) }); }; -StateConnected.prototype.requestAsync = function (options) -{ +StateConnected.prototype.requestAsync = async function (options) { // create a session token request from the options and send out the request - return this.createSessionTokenRequest(options).sendAsync(); + return await this.createSessionTokenRequest(options).sendAsync(); }; /** From 903a8d8a5110e2cafec886dd1a82a5f5eb92de4b Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Thu, 19 Oct 2023 17:23:56 -0700 Subject: [PATCH 02/15] SNOW-502598: Add tests --- test/integration/testExecuteAsync.js | 245 +++++++++++++++++++++ test/unit/mock/mock_http_client.js | 63 ++++++ test/unit/snowflake_test.js | 311 +++++++++++++++++++++++++++ 3 files changed, 619 insertions(+) create mode 100644 test/integration/testExecuteAsync.js diff --git a/test/integration/testExecuteAsync.js b/test/integration/testExecuteAsync.js new file mode 100644 index 000000000..4c58da97b --- /dev/null +++ b/test/integration/testExecuteAsync.js @@ -0,0 +1,245 @@ +/* + * Copyright (c) 2015-2019 Snowflake Computing Inc. All rights reserved. + */ + +const assert = require('assert'); +const async = require('async'); +const testUtil = require('./testUtil'); +const ErrorCodes = require('../../lib/errors').codes; +const QueryStatus = require('../../lib/constants/query_status').code; + +describe('ExecuteAsync test', function () { + let connection; + + before(function (done) { + connection = testUtil.createConnection(); + testUtil.connect(connection, done); + }); + + after(function (done) { + testUtil.destroyConnection(connection, done); + }); + + it('testAsyncQueryWithPromise', function (done) { + const expectedSeconds = 3; + const sqlText = `CALL SYSTEM$WAIT(${expectedSeconds}, 'SECONDS')`; + let queryId; + + async.series( + [ + // Execute query in async mode + function (callback) { + connection.execute({ + sqlText: sqlText, + asyncExec: true, + complete: async function (err, stmt) { + assert.ok(!err); + queryId = stmt.getQueryId(); + const status = await connection.getQueryStatus(queryId); + assert.ok(connection.isStillRunning(status)); + callback(); + } + }); + }, + // Get results using query id + async function () { + const statement = await connection.getResultsFromQueryId({ queryId: queryId }); + + await new Promise((resolve, reject) => { + statement.streamRows() + .on('error', (err) => reject(err)) + .on('data', (row) => assert.strictEqual(row['SYSTEM$WAIT'], `waited ${expectedSeconds} seconds`)) + .on('end', async () => { + const status = await connection.getQueryStatus(queryId); + assert.strictEqual(QueryStatus[status], QueryStatus.SUCCESS); + resolve(); + }); + }); + } + ], + done + ); + }); + + it('testAsyncQueryWithCallback', function (done) { + const expectedSeconds = 3; + const sqlText = `CALL SYSTEM$WAIT(${expectedSeconds}, 'SECONDS')`; + let queryId; + + async.series( + [ + // Execute query in async mode + function (callback) { + connection.execute({ + sqlText: sqlText, + asyncExec: true, + complete: async function (err, stmt) { + assert.ok(!err); + queryId = stmt.getQueryId(); + const status = await connection.getQueryStatus(queryId); + assert.ok(connection.isStillRunning(status)); + callback(); + } + }); + }, + // Get results using query id + function (callback) { + connection.getResultsFromQueryId({ + queryId: queryId, + complete: async function (err, _stmt, rows) { + assert.ok(!err); + const status = await connection.getQueryStatus(queryId); + assert.strictEqual(QueryStatus[status], QueryStatus.SUCCESS); + assert.strictEqual(rows[0]['SYSTEM$WAIT'], `waited ${expectedSeconds} seconds`); + callback(); + } + }); + } + ], + done + ); + }); + + it('testFailedQueryThrowsError', function (done) { + const sqlText = 'select * from fakeTable'; + const timeoutInMs = 1000; // 1 second + let queryId; + + async.series( + [ + // Execute query in async mode + function (callback) { + connection.execute({ + sqlText: sqlText, + asyncExec: true, + complete: async function (err, stmt) { + assert.ok(!err); + queryId = stmt.getQueryId(); + callback(); + } + }); + }, + async function () { + // Wait for query to finish executing + while (connection.isStillRunning(await connection.getQueryStatus(queryId))) { + await new Promise((resolve) => { + setTimeout(() => resolve(), timeoutInMs); + }); + } + + // Check query status failed + const status = await connection.getQueryStatus(queryId); + assert.strictEqual(QueryStatus[status], QueryStatus.FAILED_WITH_ERROR); + assert.ok(connection.isAnError(status)); + + // Check getting the query status throws an error + try { + await connection.getQueryStatusThrowIfError(queryId); + assert.fail(); + } catch (err) { + assert.ok(err); + } + + // Check getting the results throws an error + try { + await connection.getResultsFromQueryId({ queryId: queryId }); + assert.fail(); + } catch (err) { + assert.ok(err); + } + } + ], + done + ); + }); + + it('testMixedSyncAndAsyncQueries', function (done) { + const expectedSeconds = '3'; + const sqlTextForAsync = `CALL SYSTEM$WAIT(${expectedSeconds}, 'SECONDS')`; + const sqlTextForSync = 'select 1'; + let queryId; + + async.series( + [ + // Execute query in async mode + function (callback) { + connection.execute({ + sqlText: sqlTextForAsync, + asyncExec: true, + complete: async function (err, stmt) { + assert.ok(!err); + queryId = stmt.getQueryId(); + const status = await connection.getQueryStatus(queryId); + assert.ok(connection.isStillRunning(status)); + callback(); + } + }); + }, + // Execute a different query in non-async mode + function (callback) { + testUtil.executeCmd(connection, sqlTextForSync, callback); + }, + // Get results using query id + function (callback) { + connection.getResultsFromQueryId({ + queryId: queryId, + complete: async function (err, stmt, rows) { + const status = await connection.getQueryStatus(queryId); + assert.strictEqual(QueryStatus[status], QueryStatus.SUCCESS); + assert.strictEqual(rows[0]['SYSTEM$WAIT'], `waited ${expectedSeconds} seconds`); + callback(); + } + }); + } + ], + done + ); + }); + + it('testGetStatusOfInvalidQueryId', async function () { + const fakeQueryId = 'fakeQueryId'; + + // Get the query status using an invalid query id + try { + // Should fail from invalid uuid + await connection.getQueryStatus(fakeQueryId); + assert.fail(); + } catch (err) { + assert.strictEqual(err.code, ErrorCodes.ERR_GET_RESPONSE_QUERY_INVALID_UUID); + } + }); + + it('testGetResultsOfInvalidQueryId', async function () { + const fakeQueryId = 'fakeQueryId'; + + // Get the queryresults using an invalid query id + try { + // Should fail from invalid uuid + await connection.getResultsFromQueryId({ queryId: fakeQueryId }); + assert.fail(); + } catch (err) { + assert.strictEqual(err.code, ErrorCodes.ERR_GET_RESPONSE_QUERY_INVALID_UUID); + } + }); + + it('testGetStatusOfUnknownQueryId', async function () { + const unknownQueryId = '12345678-1234-4123-A123-123456789012'; + + // Get the query status using an unknown query id + const status = await connection.getQueryStatus(unknownQueryId); + assert.strictEqual(QueryStatus[status], QueryStatus.NO_DATA); + }); + + it('testGetResultsOfUnknownQueryId', async function () { + const unknownQueryId = '12345678-1234-4123-A123-123456789012'; + + // Get the query results using an unknown query id + try { + // Should fail from exceeding NO_DATA retry count + await connection.getResultsFromQueryId({ queryId: unknownQueryId }); + assert.fail(); + } catch (err) { + assert.strictEqual(err.code, ErrorCodes.ERR_GET_RESULTS_QUERY_ID_NO_DATA); + } + }); +}); diff --git a/test/unit/mock/mock_http_client.js b/test/unit/mock/mock_http_client.js index 05d3c346b..49c56b2b3 100644 --- a/test/unit/mock/mock_http_client.js +++ b/test/unit/mock/mock_http_client.js @@ -77,6 +77,36 @@ MockHttpClient.prototype.request = function (request) }, delay); }; +/** + * Issues an async request. + * + * @param {Object} request the request options. + */ +MockHttpClient.prototype.requestAsync = function (request) { + // build the request-to-output map if this is the first request + if (!this._mapRequestToOutput) { + this._mapRequestToOutput = + buildRequestToOutputMap(buildRequestOutputMappings(this._clientInfo)); + } + + // Closing a connection includes a requestID as a query parameter in the url + // Example: http://fake504.snowflakecomputing.com/session?delete=true&requestId=a40454c6-c3bb-4824-b0f3-bae041d9d6a2 + if (request.url.includes('session?delete=true')) { + // Remove the requestID query parameter for the mock HTTP client + request.url = request.url.substring(0, request.url.indexOf('&requestId=')); + } + + // get the output of the specified request from the map + const requestOutput = this._mapRequestToOutput[serializeRequest(request)]; + + Errors.assertInternal(Util.isObject(requestOutput), + 'no response available for: ' + serializeRequest(request)); + + const response = JSON.parse(JSON.stringify(requestOutput.response)); + + return response; +}; + /** * Builds a map in which the keys are requests (or rather, serialized versions * of the requests) and the values are the outputs of the corresponding request @@ -993,6 +1023,39 @@ function buildRequestOutputMappings(clientInfo) } } }, + { + request: + { + method: 'GET', + url: 'http://fakeaccount.snowflakecomputing.com/monitoring/queries/00000000-0000-0000-0000-000000000000', + headers: + { + 'Accept': 'application/json', + 'Authorization': 'Snowflake Token="SESSION_TOKEN"', + 'Content-Type': 'application/json' + //"CLIENT_APP_VERSION": clientInfo.version, + //"CLIENT_APP_ID": "JavaScript" + } + }, + output: + { + err: null, + response: + { + statusCode: 200, + statusMessage: "OK", + body: + { + code: null, + data: { + queries: [{ status: 'RESTARTED' }] + }, + message: null, + success: true + } + } + } + }, { request: { diff --git a/test/unit/snowflake_test.js b/test/unit/snowflake_test.js index f95e7d071..f83cd6eed 100644 --- a/test/unit/snowflake_test.js +++ b/test/unit/snowflake_test.js @@ -5,6 +5,7 @@ var Util = require('./../../lib/util'); var ErrorCodes = require('./../../lib/errors').codes; var MockTestUtil = require('./mock/mock_test_util'); +const QueryStatus = require('./../../lib/constants/query_status').code; var assert = require('assert'); var async = require('async'); @@ -1481,6 +1482,316 @@ describe('statement.cancel()', function () }); }); +describe('connection.getResultsFromQueryId() asynchronous errors', function () { + const queryId = '00000000-0000-0000-0000-000000000000'; + + it('not success status', function (done) { + const connection = snowflake.createConnection(connectionOptions); + async.series( + [ + function (callback) { + connection.connect(function (err) { + assert.ok(!err, JSON.stringify(err)); + callback(); + }); + }, + async function () { + try { + await connection.getResultsFromQueryId({ queryId: queryId }); + assert.fail(); + } catch (err) { + assert.strictEqual(err.code, ErrorCodes.ERR_GET_RESULTS_QUERY_ID_NOT_SUCCESS_STATUS); + } + }, + function (callback) { + connection.destroy(function (err) { + assert.ok(!err, JSON.stringify(err)); + callback(); + }); + } + ], + done + ); + }); +}); + +describe('connection.getResultsFromQueryId() synchronous errors', function () { + const connection = snowflake.createConnection(connectionOptions); + + const testCases = + [ + { + name: 'missing queryId', + options: {}, + errorCode: ErrorCodes.ERR_CONN_FETCH_RESULT_MISSING_QUERY_ID + }, + { + name: 'undefined queryId', + options: { queryId: undefined }, + errorCode: ErrorCodes.ERR_CONN_FETCH_RESULT_MISSING_QUERY_ID + }, + { + name: 'null queryId', + options: { queryId: null }, + errorCode: ErrorCodes.ERR_CONN_FETCH_RESULT_MISSING_QUERY_ID + }, + { + name: 'non-string queryId', + options: { queryId: 123 }, + errorCode: ErrorCodes.ERR_GET_RESPONSE_QUERY_INVALID_UUID + }, + { + name: 'invalid queryId', + options: { queryId: 'invalidQueryId' }, + errorCode: ErrorCodes.ERR_GET_RESPONSE_QUERY_INVALID_UUID + } + ]; + + testCases.forEach(testCase => { + it(testCase.name, async function () { + try { + await connection.getResultsFromQueryId(testCase.options); + } catch (err) { + assert.strictEqual(err.code, testCase.errorCode); + } + }); + }); +}); + +describe('connection.getQueryStatus() synchronous errors', function () { + const connection = snowflake.createConnection(connectionOptions); + + const testCases = + [ + { + name: 'undefined queryId', + queryId: undefined, + errorCode: ErrorCodes.ERR_CONN_FETCH_RESULT_MISSING_QUERY_ID + }, + { + name: 'null queryId', + queryId: null, + errorCode: ErrorCodes.ERR_CONN_FETCH_RESULT_MISSING_QUERY_ID + }, + { + name: 'non-string queryId', + queryId: 123, + errorCode: ErrorCodes.ERR_GET_RESPONSE_QUERY_INVALID_UUID + }, + { + name: 'invalid queryId', + queryId: 'invalidQueryId', + errorCode: ErrorCodes.ERR_GET_RESPONSE_QUERY_INVALID_UUID + } + ]; + + testCases.forEach(testCase => { + it(testCase.name, async function () { + try { + await connection.getQueryStatus(testCase.queryId); + } catch (err) { + assert.strictEqual(err.code, testCase.errorCode); + } + }); + }); +}); + +describe('connection.getQueryStatusThrowIfError() synchronous errors', function () { + const connection = snowflake.createConnection(connectionOptions); + + const testCases = + [ + { + name: 'undefined queryId', + queryId: undefined, + errorCode: ErrorCodes.ERR_CONN_FETCH_RESULT_MISSING_QUERY_ID + }, + { + name: 'null queryId', + queryId: null, + errorCode: ErrorCodes.ERR_CONN_FETCH_RESULT_MISSING_QUERY_ID + }, + { + name: 'non-string queryId', + queryId: 123, + errorCode: ErrorCodes.ERR_GET_RESPONSE_QUERY_INVALID_UUID + }, + { + name: 'invalid queryId', + queryId: 'invalidQueryId', + errorCode: ErrorCodes.ERR_GET_RESPONSE_QUERY_INVALID_UUID + } + ]; + + testCases.forEach(testCase => { + it(testCase.name, async function () { + try { + await connection.getQueryStatusThrowIfError(testCase.queryId); + } catch (err) { + assert.strictEqual(err.code, testCase.errorCode); + } + }); + }); +}); + +describe('snowflake.isStillRunning()', function () { + const connection = snowflake.createConnection(connectionOptions); + + const testCases = + [ + { + name: 'Running', + status: QueryStatus.RUNNING, + expectedValue: true + }, + { + name: 'Aborting', + status: QueryStatus.ABORTING, + expectedValue: false + }, + { + name: 'Success', + status: QueryStatus.SUCCESS, + expectedValue: false + }, + { + name: 'Failed with error', + status: QueryStatus.FAILED_WITH_ERROR, + expectedValue: false + }, + { + name: 'Aborted', + status: QueryStatus.ABORTED, + expectedValue: false + }, + { + name: 'Queued', + status: QueryStatus.QUEUED, + expectedValue: true + }, + { + name: 'Failed with incident', + status: QueryStatus.FAILED_WITH_INCIDENT, + expectedValue: false + }, + { + name: 'Disconnected', + status: QueryStatus.DISCONNECTED, + expectedValue: false + }, + { + name: 'Resuming warehouse', + status: QueryStatus.RESUMING_WAREHOUSE, + expectedValue: true + }, + { + name: 'Queued repairing warehouse', + status: QueryStatus.QUEUED_REPARING_WAREHOUSE, + expectedValue: true + }, + { + name: 'Restarted', + status: QueryStatus.RESTARTED, + expectedValue: false + }, + { + name: 'Blocked', + status: QueryStatus.BLOCKED, + expectedValue: false + }, + { + name: 'No data', + status: QueryStatus.NO_DATA, + expectedValue: true + }, + ]; + + testCases.forEach(testCase => { + it(testCase.name, function () { + assert.strictEqual(testCase.expectedValue, connection.isStillRunning(testCase.status)); + }); + }); +}); + +describe('snowflake.isAnError()', function () { + const connection = snowflake.createConnection(connectionOptions); + + const testCases = + [ + { + name: 'Running', + status: QueryStatus.RUNNING, + expectedValue: false + }, + { + name: 'Aborting', + status: QueryStatus.ABORTING, + expectedValue: true + }, + { + name: 'Success', + status: QueryStatus.SUCCESS, + expectedValue: false + }, + { + name: 'Failed with error', + status: QueryStatus.FAILED_WITH_ERROR, + expectedValue: true + }, + { + name: 'Aborted', + status: QueryStatus.ABORTED, + expectedValue: true + }, + { + name: 'Queued', + status: QueryStatus.QUEUED, + expectedValue: false + }, + { + name: 'Failed with incident', + status: QueryStatus.FAILED_WITH_INCIDENT, + expectedValue: true + }, + { + name: 'Disconnected', + status: QueryStatus.DISCONNECTED, + expectedValue: true + }, + { + name: 'Resuming warehouse', + status: QueryStatus.RESUMING_WAREHOUSE, + expectedValue: false + }, + { + name: 'Queued repairing warehouse', + status: QueryStatus.QUEUED_REPARING_WAREHOUSE, + expectedValue: false + }, + { + name: 'Restarted', + status: QueryStatus.RESTARTED, + expectedValue: false + }, + { + name: 'Blocked', + status: QueryStatus.BLOCKED, + expectedValue: true + }, + { + name: 'No data', + status: QueryStatus.NO_DATA, + expectedValue: false + }, + ]; + + testCases.forEach(testCase => { + it(testCase.name, function () { + assert.strictEqual(testCase.expectedValue, connection.isAnError(testCase.status)); + }); + }); +}); + describe('connection.destroy()', function () { it('destroy without connecting', function (done) From 922acf6d4042b622b1e0eff7a9466ac407abc160 Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Thu, 19 Oct 2023 17:46:56 -0700 Subject: [PATCH 03/15] SNOW-502598: Add tests --- test/integration/testExecuteAsync.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/integration/testExecuteAsync.js b/test/integration/testExecuteAsync.js index 4c58da97b..2864a2ac8 100644 --- a/test/integration/testExecuteAsync.js +++ b/test/integration/testExecuteAsync.js @@ -230,7 +230,8 @@ describe('ExecuteAsync test', function () { assert.strictEqual(QueryStatus[status], QueryStatus.NO_DATA); }); - it('testGetResultsOfUnknownQueryId', async function () { + // The test retries until it reaches the max retry count and sometimes it fails due to timeout + it.skip('testGetResultsOfUnknownQueryId', async function () { const unknownQueryId = '12345678-1234-4123-A123-123456789012'; // Get the query results using an unknown query id From a2e92cb472ce4b56cbe2eaa8fb4a7d2ad89edb19 Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Mon, 23 Oct 2023 11:58:24 -0700 Subject: [PATCH 04/15] SNOW-502598: Remove redundant requestAsync() function --- lib/connection/connection.js | 2 +- lib/http/base.js | 38 ------------------------------------ 2 files changed, 1 insertion(+), 39 deletions(-) diff --git a/lib/connection/connection.js b/lib/connection/connection.js index 88af4eb77..ca27193cc 100644 --- a/lib/connection/connection.js +++ b/lib/connection/connection.js @@ -448,7 +448,7 @@ function Connection(context) // Get the response containing the query status const response = await services.sf.requestAsync(options); - return response['body']; + return JSON.parse(response['data']); } /** diff --git a/lib/http/base.js b/lib/http/base.js index 2f83ebdf9..2d92f9650 100644 --- a/lib/http/base.js +++ b/lib/http/base.js @@ -88,44 +88,6 @@ HttpClient.prototype.requestAsync = async function (options) return axios.request(requestOptions); }; -/** - * Issues an async HTTP request. - * - * @param {Object} options - * - * @returns {Object} an object representing the request that was issued. - */ -HttpClient.prototype.requestAsync = async function (options) { - // validate input - Errors.assertInternal(Util.isObject(options)); - Errors.assertInternal(Util.isString(options.method)); - Errors.assertInternal(Util.isString(options.url)); - Errors.assertInternal(!Util.exists(options.headers) || - Util.isObject(options.headers)); - Errors.assertInternal(!Util.exists(options.json) || - Util.isObject(options.json)); - - // normalize the headers - const headers = normalizeHeaders(options.headers); - - const timeout = options.timeout || - this._connectionConfig.getTimeout() || - DEFAULT_REQUEST_TIMEOUT; - - const requestOptions = - { - method: options.method, - url: options.url, - headers: headers, - data: options.json, - timeout: timeout - }; - - const response = await axios.request(requestOptions); - - return normalizeResponse(response); -} - /** * @abstract * Returns the module to use when making HTTP requests. Subclasses must override From 1ee54edec835162b16a138695270cc023f210e14 Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Mon, 23 Oct 2023 12:01:36 -0700 Subject: [PATCH 05/15] SNOW-502598: Replace for loop with while loop --- lib/connection/connection.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/connection/connection.js b/lib/connection/connection.js index ca27193cc..78592ad7f 100644 --- a/lib/connection/connection.js +++ b/lib/connection/connection.js @@ -520,10 +520,12 @@ function Connection(context) let status, noDataCounter = 0, retryPatternPos = 0; // Wait until query has finished executing - for (;;) { + let queryStillExecuting = true; + while (queryStillExecuting) { // Check if query is still running and trigger exception if it failed status = await this.getQueryStatusThrowIfError(queryId); - if (!this.isStillRunning(status)) { + queryStillExecuting = this.isStillRunning(status); + if (!queryStillExecuting) { break; } From b79a595c6508bc4a0ffa7a81cf08838b1a0e553f Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Mon, 23 Oct 2023 12:02:42 -0700 Subject: [PATCH 06/15] SNOW-502598: Edit comment for query status retry --- lib/connection/connection.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/connection/connection.js b/lib/connection/connection.js index 78592ad7f..642349cb1 100644 --- a/lib/connection/connection.js +++ b/lib/connection/connection.js @@ -529,7 +529,7 @@ function Connection(context) break; } - // Timeout based on python connector + // Timeout based on query status retry rules await new Promise((resolve) => { setTimeout(() => resolve(), asyncRetryInMilliseconds * asyncRetryPattern[retryPatternPos]); }); From 0a7b7c8bac3b2e932b08d6d53e1a7dfb368604bb Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Mon, 23 Oct 2023 12:04:13 -0700 Subject: [PATCH 07/15] SNOW-502598: Remove test comment --- test/unit/mock/mock_http_client.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/unit/mock/mock_http_client.js b/test/unit/mock/mock_http_client.js index 49c56b2b3..e75b9782b 100644 --- a/test/unit/mock/mock_http_client.js +++ b/test/unit/mock/mock_http_client.js @@ -1033,8 +1033,6 @@ function buildRequestOutputMappings(clientInfo) 'Accept': 'application/json', 'Authorization': 'Snowflake Token="SESSION_TOKEN"', 'Content-Type': 'application/json' - //"CLIENT_APP_VERSION": clientInfo.version, - //"CLIENT_APP_ID": "JavaScript" } }, output: From 8d8bd877aba89f5e4937459e106aa165f4b36c9f Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Mon, 23 Oct 2023 12:54:12 -0700 Subject: [PATCH 08/15] SNOW-502598: Parse data back into JSON --- lib/connection/connection.js | 2 +- lib/http/base.js | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/connection/connection.js b/lib/connection/connection.js index 642349cb1..ef0a57afe 100644 --- a/lib/connection/connection.js +++ b/lib/connection/connection.js @@ -448,7 +448,7 @@ function Connection(context) // Get the response containing the query status const response = await services.sf.requestAsync(options); - return JSON.parse(response['data']); + return response['body']; } /** diff --git a/lib/http/base.js b/lib/http/base.js index 2d92f9650..f3a2968d1 100644 --- a/lib/http/base.js +++ b/lib/http/base.js @@ -85,7 +85,14 @@ HttpClient.prototype.requestAsync = async function (options) { const requestOptions = prepareRequestOptions.call(this, options); - return axios.request(requestOptions); + let response = await axios.request(requestOptions); + + if (Util.isString(response['data']) && + response['headers']['content-type'] === 'application/json') { + response['data'] = JSON.parse(response['data']); + } + + return normalizeResponse(response); }; /** From a479942bb5cf9c64fe750119d64b1d19c53ad982 Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Mon, 23 Oct 2023 13:49:11 -0700 Subject: [PATCH 09/15] SNOW-502598: Revert normalizing response --- lib/connection/connection.js | 2 +- lib/http/base.js | 2 +- test/unit/mock/mock_http_client.js | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/connection/connection.js b/lib/connection/connection.js index ef0a57afe..16d4b5d5d 100644 --- a/lib/connection/connection.js +++ b/lib/connection/connection.js @@ -448,7 +448,7 @@ function Connection(context) // Get the response containing the query status const response = await services.sf.requestAsync(options); - return response['body']; + return response['data']; } /** diff --git a/lib/http/base.js b/lib/http/base.js index f3a2968d1..190e1ca5a 100644 --- a/lib/http/base.js +++ b/lib/http/base.js @@ -92,7 +92,7 @@ HttpClient.prototype.requestAsync = async function (options) response['data'] = JSON.parse(response['data']); } - return normalizeResponse(response); + return response; }; /** diff --git a/test/unit/mock/mock_http_client.js b/test/unit/mock/mock_http_client.js index e75b9782b..f9cba1f0b 100644 --- a/test/unit/mock/mock_http_client.js +++ b/test/unit/mock/mock_http_client.js @@ -1042,7 +1042,7 @@ function buildRequestOutputMappings(clientInfo) { statusCode: 200, statusMessage: "OK", - body: + data: { code: null, data: { From 72751cb01cb99bfbf7a2070793cb0fd6a3c37aa0 Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Fri, 27 Oct 2023 08:04:08 -0700 Subject: [PATCH 10/15] SNOW-502598: Use async for before/after tests --- test/integration/testExecuteAsync.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/integration/testExecuteAsync.js b/test/integration/testExecuteAsync.js index 2864a2ac8..401045252 100644 --- a/test/integration/testExecuteAsync.js +++ b/test/integration/testExecuteAsync.js @@ -11,13 +11,13 @@ const QueryStatus = require('../../lib/constants/query_status').code; describe('ExecuteAsync test', function () { let connection; - before(function (done) { + before(async () => { connection = testUtil.createConnection(); - testUtil.connect(connection, done); + await testUtil.connectAsync(connection); }); - after(function (done) { - testUtil.destroyConnection(connection, done); + after(async () => { + await testUtil.destroyConnectionAsync(connection); }); it('testAsyncQueryWithPromise', function (done) { From 5ecda9b7f62d53ccadf80faa40d7427680f1ba9c Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Fri, 27 Oct 2023 09:37:25 -0700 Subject: [PATCH 11/15] SNOW-502598: Extract running and error statuses --- lib/connection/connection.js | 31 ++++++------------------------- lib/constants/query_status.js | 23 +++++++++++++++++++++++ 2 files changed, 29 insertions(+), 25 deletions(-) diff --git a/lib/connection/connection.js b/lib/connection/connection.js index 16d4b5d5d..96b7ca74f 100644 --- a/lib/connection/connection.js +++ b/lib/connection/connection.js @@ -5,7 +5,7 @@ const { v4: uuidv4 } = require('uuid'); const Url = require('url'); const QueryString = require('querystring'); const GSErrors = require('../constants/gs_errors') -const QueryStatus = require('../constants/query_status').code; +const QueryStatus = require('../constants/query_status'); var Util = require('../util'); var Errors = require('../errors'); @@ -460,7 +460,7 @@ function Connection(context) */ function extractQueryStatus(queryResponse) { const queries = queryResponse['data']['queries']; - let status = QueryStatus.NO_DATA; // default status + let status = QueryStatus.code.NO_DATA; // default status if (queries.length > 0) { status = queries[0]['status']; } @@ -535,7 +535,7 @@ function Connection(context) }); // If no data, increment the no data counter - if (QueryStatus[status] == QueryStatus.NO_DATA) { + if (QueryStatus.code[status] == QueryStatus.code.NO_DATA) { noDataCounter++; // Check if retry for no data is exceeded if (noDataCounter > asyncNoDataMaxRetry) { @@ -549,7 +549,7 @@ function Connection(context) } } - if (QueryStatus[status] != QueryStatus.SUCCESS) { + if (QueryStatus.code[status] != QueryStatus.code.SUCCESS) { throw Errors.createClientError( ErrorCodes.ERR_GET_RESULTS_QUERY_ID_NOT_SUCCESS_STATUS, true, queryId, status); } @@ -565,16 +565,7 @@ function Connection(context) * @returns {Boolean} */ this.isStillRunning = function (status) { - const stillRunning = - [ - QueryStatus.RUNNING, - QueryStatus.RESUMING_WAREHOUSE, - QueryStatus.QUEUED, - QueryStatus.QUEUED_REPARING_WAREHOUSE, - QueryStatus.NO_DATA, - ].includes(QueryStatus[status]); - - return stillRunning; + return QueryStatus.runningStatuses.includes(QueryStatus.code[status]); }; /** @@ -585,17 +576,7 @@ function Connection(context) * @returns {Boolean} */ this.isAnError = function (status) { - const isAnError = - [ - QueryStatus.ABORTING, - QueryStatus.FAILED_WITH_ERROR, - QueryStatus.ABORTED, - QueryStatus.FAILED_WITH_INCIDENT, - QueryStatus.DISCONNECTED, - QueryStatus.BLOCKED, - ].includes(QueryStatus[status]); - - return isAnError; + return QueryStatus.errorStatuses.includes(QueryStatus.code[status]); }; /** diff --git a/lib/constants/query_status.js b/lib/constants/query_status.js index 2449aeff0..6bc4cf2a4 100644 --- a/lib/constants/query_status.js +++ b/lib/constants/query_status.js @@ -19,4 +19,27 @@ code.RESTARTED = 'RESTARTED'; code.BLOCKED = 'BLOCKED'; code.NO_DATA = 'NO_DATA'; +// All running query statuses +const runningStatuses = + [ + code.RUNNING, + code.RESUMING_WAREHOUSE, + code.QUEUED, + code.QUEUED_REPARING_WAREHOUSE, + code.NO_DATA, + ]; + +// All error query statuses +const errorStatuses = + [ + code.ABORTING, + code.FAILED_WITH_ERROR, + code.ABORTED, + code.FAILED_WITH_INCIDENT, + code.DISCONNECTED, + code.BLOCKED, + ]; + exports.code = code; +exports.runningStatuses = runningStatuses; +exports.errorStatuses = errorStatuses; From 3624ae77bb205afbaaeb5f97a363e516e85a6070 Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Fri, 27 Oct 2023 13:39:00 -0700 Subject: [PATCH 12/15] SNOW-502598: Refactor error into the correct type --- lib/connection/statement.js | 4 ++-- lib/constants/error_messages.js | 2 +- lib/errors.js | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/connection/statement.js b/lib/connection/statement.js index a96c7f7d1..09fcbf0bb 100644 --- a/lib/connection/statement.js +++ b/lib/connection/statement.js @@ -195,7 +195,7 @@ exports.createStatementPostExec = function ( // check for invalid asyncExec if (Util.exists(statementOptions.asyncExec)) { Errors.checkArgumentValid(Util.isBoolean(statementOptions.asyncExec), - ErrorCodes.ERR_CONN_CREATE_INVALID_ASYNC_EXEC); + ErrorCodes.ERR_CONN_EXEC_STMT_INVALID_ASYNC_EXEC); } const rowMode = statementOptions.rowMode; @@ -379,7 +379,7 @@ function createContextPreExec( // if an asyncExec flag is specified, make sure it's boolean if (Util.exists(statementOptions.asyncExec)) { Errors.checkArgumentValid(Util.isBoolean(statementOptions.asyncExec), - ErrorCodes.ERR_CONN_CREATE_INVALID_ASYNC_EXEC); + ErrorCodes.ERR_CONN_EXEC_STMT_INVALID_ASYNC_EXEC); } // create a statement context diff --git a/lib/constants/error_messages.js b/lib/constants/error_messages.js index d2ad5786d..9da8d13bd 100644 --- a/lib/constants/error_messages.js +++ b/lib/constants/error_messages.js @@ -69,7 +69,6 @@ exports[404041] = 'Invalid disablQueryContextCache. The specified value must be exports[404042] = 'Invalid includeRetryReason. The specified value must be a boolean.' exports[404043] = 'Invalid clientConfigFile value. The specified value must be a string.'; exports[404044] = 'Invalid loginTimeout value. The specified value must be a number.'; -exports[404045] = 'Invalid asyncExec. The specified value must be a boolean.' // 405001 exports[405001] = 'Invalid callback. The specified value must be a function.'; @@ -111,6 +110,7 @@ exports[409010] = 'Invalid streamResult flag. The specified value must be a bool exports[409011] = 'Invalid fetchAsString value. The specified value must be an Array.'; exports[409012] = 'Invalid fetchAsString type: %s. The supported types are: String, Boolean, Number, Date, Buffer, and JSON.'; exports[409013] = 'Invalid requestId. The specified value must be a string.'; +exports[409014] = 'Invalid asyncExec. The specified value must be a boolean.' // 410001 exports[410001] = 'Fetch-result options must be specified.'; diff --git a/lib/errors.js b/lib/errors.js index 09a2795e0..cdbb61177 100644 --- a/lib/errors.js +++ b/lib/errors.js @@ -74,7 +74,6 @@ codes.ERR_CONN_CREATE_INVALID_DISABLED_QUERY_CONTEXT_CACHE = 404041 codes.ERR_CONN_CREATE_INVALID_INCLUDE_RETRY_REASON =404042 codes.ERR_CONN_CREATE_INVALID_CLIENT_CONFIG_FILE = 404043; codes.ERR_CONN_CREATE_INVALID_LOGIN_TIMEOUT = 404044; -codes.ERR_CONN_CREATE_INVALID_ASYNC_EXEC = 404045 // 405001 codes.ERR_CONN_CONNECT_INVALID_CALLBACK = 405001; @@ -116,6 +115,7 @@ codes.ERR_CONN_EXEC_STMT_INVALID_STREAM_RESULT = 409010; codes.ERR_CONN_EXEC_STMT_INVALID_FETCH_AS_STRING = 409011; codes.ERR_CONN_EXEC_STMT_INVALID_FETCH_AS_STRING_VALUES = 409012; codes.ERR_CONN_EXEC_STMT_INVALID_REQUEST_ID = 409013; +codes.ERR_CONN_EXEC_STMT_INVALID_ASYNC_EXEC = 409014; // 410001 codes.ERR_CONN_FETCH_RESULT_MISSING_OPTIONS = 410001; From 924052c596e10e7d1644efdcb402e3c0e8f1d452 Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Fri, 27 Oct 2023 14:20:01 -0700 Subject: [PATCH 13/15] SNOW-502598: Keep asyncExec option for execute only --- lib/connection/statement.js | 7 ------- 1 file changed, 7 deletions(-) diff --git a/lib/connection/statement.js b/lib/connection/statement.js index 09fcbf0bb..0e531aeee 100644 --- a/lib/connection/statement.js +++ b/lib/connection/statement.js @@ -192,12 +192,6 @@ exports.createStatementPostExec = function ( JSON.stringify(fetchAsString[invalidValueIndex])); } - // check for invalid asyncExec - if (Util.exists(statementOptions.asyncExec)) { - Errors.checkArgumentValid(Util.isBoolean(statementOptions.asyncExec), - ErrorCodes.ERR_CONN_EXEC_STMT_INVALID_ASYNC_EXEC); - } - const rowMode = statementOptions.rowMode; if (Util.exists(rowMode)) { RowMode.checkRowModeValid(rowMode); @@ -217,7 +211,6 @@ exports.createStatementPostExec = function ( statementContext.multiResultIds = statementOptions.multiResultIds; statementContext.multiCurId = statementOptions.multiCurId; statementContext.rowMode = statementOptions.rowMode; - statementContext.asyncExec = statementOptions.asyncExec; // set the statement type statementContext.type = (statementContext.type == statementTypes.ROW_PRE_EXEC) ? statementTypes.ROW_POST_EXEC : statementTypes.FILE_POST_EXEC; From 6c00000ab5ad01bfe3ff7459a6194588e3094db2 Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Fri, 27 Oct 2023 17:31:12 -0700 Subject: [PATCH 14/15] SNOW-502598: Format unused params in tests --- test/integration/testExecuteAsync.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/testExecuteAsync.js b/test/integration/testExecuteAsync.js index 401045252..a81363259 100644 --- a/test/integration/testExecuteAsync.js +++ b/test/integration/testExecuteAsync.js @@ -183,7 +183,7 @@ describe('ExecuteAsync test', function () { function (callback) { connection.getResultsFromQueryId({ queryId: queryId, - complete: async function (err, stmt, rows) { + complete: async function (_err, _stmt, rows) { const status = await connection.getQueryStatus(queryId); assert.strictEqual(QueryStatus[status], QueryStatus.SUCCESS); assert.strictEqual(rows[0]['SYSTEM$WAIT'], `waited ${expectedSeconds} seconds`); From 357359a54fbfa187d9aaed5063df521020aa2a23 Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Tue, 31 Oct 2023 12:39:36 -0700 Subject: [PATCH 15/15] SNOW-502598: Refactor getQueryStatusThrowIfError --- lib/connection/connection.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/connection/connection.js b/lib/connection/connection.js index 96b7ca74f..0e9f515c3 100644 --- a/lib/connection/connection.js +++ b/lib/connection/connection.js @@ -487,17 +487,17 @@ function Connection(context) * @returns {String} the query status. */ this.getQueryStatusThrowIfError = async function (queryId) { - const response = await getQueryResponse(queryId); - const queries = response['data']['queries']; - const status = extractQueryStatus(response); + const status = this.getQueryStatus(queryId); + let message, code, sqlState = null; if (this.isAnError(status)) { + const response = await getQueryResponse(queryId); message = response['message'] || ''; code = response['code'] || -1; if (response['data']) { - message += queries.length > 0 ? queries[0]['errorMessage'] : ''; + message += response['data']['queries'].length > 0 ? response['data']['queries'][0]['errorMessage'] : ''; sqlState = response['data']['sqlState']; }