SNOW-1801434-Add-GUID-to-request-in-NODE.js-driver: Merged master

lib/file_transfer_agent/gcs_util.js

@@ -69,8 +69,15 @@ function GCSUtil(httpclient, filestream) {
         }
       });
 
-      const storage = new Storage({ interceptors_: interceptors });
-
+      //TODO: SNOW-1789759 hardcoded region will be replaced in the future
+      const isRegionalUrlEnabled = (stageInfo.region).toLowerCase() === 'me-central2' || stageInfo.useRegionalUrl;
+      let endPoint = null;
+      if (stageInfo['endPoint']) {
+        endPoint = stageInfo['endPoint'];
+      } else if (isRegionalUrlEnabled) {
+        endPoint = `storage.${stageInfo.region.toLowerCase()}.rep.googleapis.com`;
+      }
+      const storage = endPoint ? new Storage({ interceptors_: interceptors, apiEndpoint: endPoint }) : new Storage({ interceptors_: interceptors });
       client = { gcsToken: gcsToken, gcsClient: storage };
     } else {
       client = null;

lib/file_transfer_agent/s3_util.js

@@ -54,12 +54,19 @@ function S3Util(connectionConfig, s3, filestream) {
   this.createClient = function (stageInfo, useAccelerateEndpoint) {
     const stageCredentials = stageInfo['creds'];
     const securityToken = stageCredentials['AWS_TOKEN'];
+    const isRegionalUrlEnabled = stageInfo.useRegionalUrl || stageInfo.useS3RegionalUrl;
+
     // if GS sends us an endpoint, it's likely for FIPS. Use it.
     let endPoint = null;
     if (stageInfo['endPoint']) {
-      endPoint = 'https://' + stageInfo['endPoint'];
+      endPoint = `https://${stageInfo['endPoint']}`;
+    } else {
+      if (stageInfo.region && isRegionalUrlEnabled) {
+        const domainSuffixForRegionalUrl = (stageInfo.region).toLowerCase().startsWith('cn-') ? 'amazonaws.com.cn' : 'amazonaws.com';
+        endPoint = `https://s3.${stageInfo.region}.${domainSuffixForRegionalUrl}`;
+      }
     }
-
+  
     const config = {
       apiVersion: '2006-03-01',
       region: stageInfo['region'],
@@ -80,7 +87,7 @@ function S3Util(connectionConfig, s3, filestream) {
       }
     }
     if (proxy) {
-      const proxyAgent = getProxyAgent(proxy, new URL(connectionConfig.accessUrl), SNOWFLAKE_S3_DESTINATION);
+      const proxyAgent = getProxyAgent(proxy, new URL(connectionConfig.accessUrl), endPoint || SNOWFLAKE_S3_DESTINATION);
       config.requestHandler = new NodeHttpHandler({
         httpAgent: proxyAgent,
         httpsAgent: proxyAgent

test/integration/testExecute.js

@@ -431,5 +431,70 @@ describe('Execute test - variant', function () {
 
     it(testCase.name, createItCallback(testCase, rowAsserts));
   });
-});
 
+  describe( 'connection.execute() Resubmitting requests using requestId and different connections', function () {
+    const createTable = 'create or replace table test_request_id(colA string)';
+    let  firstConnection;
+    let  secondConnection;
+    before(async () => {
+      firstConnection = testUtil.createConnection();
+      secondConnection = testUtil.createConnection();
+      await testUtil.connectAsync(firstConnection);
+      await testUtil.connectAsync(secondConnection);
+      await testUtil.executeCmdAsync(firstConnection, createTable);
+    });
+
+    beforeEach(async () => {
+      await testUtil.executeCmdAsync(firstConnection, 'truncate table if exists test_request_id');
+    });
+
+    after(async () => {
+      await testUtil.executeCmdAsync(firstConnection, 'drop table if exists test_request_id');
+      await testUtil.destroyConnectionAsync(firstConnection);
+      await testUtil.destroyConnectionAsync(secondConnection);
+    });
+
+    it('Do not INSERT twice when the same request id and connection', async () => {
+      let result;
+      result = await testUtil.executeCmdAsyncWithAdditionalParameters(firstConnection, 'INSERT INTO test_request_id VALUES (\'testValue\');');
+      const requestId = result.rowStatement.getRequestId();
+
+      result = await testUtil.executeCmdAsyncWithAdditionalParameters(firstConnection,
+        'INSERT INTO test_request_id VALUES (\'testValue\');', 
+        { requestId: requestId });
+      assert.strictEqual(result.rowStatement.getRequestId(), requestId);
+
+      result = await testUtil.executeCmdAsyncWithAdditionalParameters(firstConnection, 'SELECT * from test_request_id ;');
+      assert.strictEqual(result.rows.length, 1);
+    });
+
+    it('Execute INSERT for the same request id and different connection', async () => {
+      let result;
+      result = await testUtil.executeCmdAsyncWithAdditionalParameters(firstConnection, 'INSERT INTO test_request_id VALUES (\'testValue\');');
+      const requestId = result.rowStatement.getRequestId();
+
+      result = await testUtil.executeCmdAsyncWithAdditionalParameters(secondConnection, 'INSERT INTO test_request_id VALUES (\'testValue\');', { requestId: requestId });
+      assert.strictEqual(result.rowStatement.getRequestId(), requestId);
+
+      result = await testUtil.executeCmdAsyncWithAdditionalParameters(firstConnection, 'SELECT * from test_request_id ;');
+      assert.strictEqual(result.rows.length, 2);
+    });
+
+    it('Execute SELECT for the same request id and different data', async () => {
+      await testUtil.executeCmdAsyncWithAdditionalParameters(firstConnection, 'INSERT INTO test_request_id VALUES (\'testValue\');');
+      let result = await testUtil.executeCmdAsyncWithAdditionalParameters(firstConnection, 'SELECT * from test_request_id;');
+      assert.strictEqual(result.rows.length, 1);
+      const requestId = result.rowStatement.getRequestId();
+
+      await testUtil.executeCmdAsyncWithAdditionalParameters(firstConnection, 'INSERT INTO test_request_id VALUES (\'testValue\');');
+      result = await testUtil.executeCmdAsyncWithAdditionalParameters(firstConnection, 'SELECT * from test_request_id;', { requestId: requestId });
+      assert.strictEqual(result.rows.length, 1);
+
+      result = await testUtil.executeCmdAsyncWithAdditionalParameters(firstConnection, 'SELECT * from test_request_id ;');
+      assert.strictEqual(result.rows.length, 2);
+    });
+  });
+
+
+
+});

test/integration/testUtil.js

@@ -108,6 +108,18 @@ const executeCmdAsync = function (connection, sqlText, binds = undefined) {
 
 module.exports.executeCmdAsync = executeCmdAsync;
 
+const executeCmdAsyncWithAdditionalParameters = function (connection, sqlText, additionalParameters) {
+  return new Promise((resolve, reject) => {
+    const executeParams = { ...{
+      sqlText: sqlText,
+      complete: (err, rowStatement, rows) =>
+        err ? reject(err) : resolve({ rowStatement: rowStatement, rows: rows })
+    }, ...additionalParameters };
+    connection.execute(executeParams);
+  });
+};
+
+module.exports.executeCmdAsyncWithAdditionalParameters = executeCmdAsyncWithAdditionalParameters;
 /**
  * Drop tables one by one if exist - any connection error is ignored
  * @param connection Connection

test/unit/file_transfer_agent/gcs_test.js

@@ -64,6 +64,82 @@ describe('GCS client', function () {
     GCS = new SnowflakeGCSUtil(httpclient, filestream);
   });
 
+  describe('GCS client endpoint testing', async function () {
+    const testCases = [
+      {
+        name: 'when the useRegionalURL is only enabled',
+        stageInfo: {
+          endPoint: null,
+          useRegionalUrl: true,
+          region: 'mockLocation',
+        },
+        result: 'https://storage.mocklocation.rep.googleapis.com'
+      },
+      {
+        name: 'when the region is me-central2',
+        stageInfo: {
+          endPoint: null,
+          useRegionalUrl: false,
+          region: 'me-central2'
+        },
+        result: 'https://storage.me-central2.rep.googleapis.com'
+      },
+      {
+        name: 'when the region is me-central2 (mixed case)',
+        stageInfo: {
+          endPoint: null,
+          useRegionalUrl: false,
+          region: 'ME-cEntRal2'
+        },
+        result: 'https://storage.me-central2.rep.googleapis.com'
+      },
+      {
+        name: 'when the region is me-central2 (uppercase)',
+        stageInfo: {
+          endPoint: null,
+          useRegionalUrl: false,
+          region: 'ME-CENTRAL2'
+        },
+        result: 'https://storage.me-central2.rep.googleapis.com'
+      },
+      {
+        name: 'when the endPoint is specified',
+        stageInfo: {
+          endPoint: 'https://storage.specialEndPoint.rep.googleapis.com',
+          useRegionalUrl: false,
+          region: 'ME-cEntRal1'
+        },
+        result: 'https://storage.specialEndPoint.rep.googleapis.com'
+      },
+      {
+        name: 'when both the endPoint and the useRegionalUrl are specified',
+        stageInfo: {
+          endPoint: 'https://storage.specialEndPoint.rep.googleapis.com',
+          useRegionalUrl: true,
+          region: 'ME-cEntRal1'
+        },
+        result: 'https://storage.specialEndPoint.rep.googleapis.com'
+      },
+      {
+        name: 'when both the endPoint is specified and the region is me-central2',
+        stageInfo: {
+          endPoint: 'https://storage.specialEndPoint.rep.googleapis.com',
+          useRegionalUrl: true,
+          region: 'ME-CENTRAL2'
+        },
+        result: 'https://storage.specialEndPoint.rep.googleapis.com'
+      },
+    ];
+
+    testCases.forEach(({ name, stageInfo, result }) => {
+      it(name, () => {
+        const client = GCS.createClient({ ...stageInfo, ...meta.stageInfo, creds: { GCS_ACCESS_TOKEN: 'mockToken' } });
+        assert.strictEqual(client.gcsClient.apiEndpoint, result);
+      } );
+
+    });
+  });
+
   it('extract bucket name and path', async function () {
     const GCS = new SnowflakeGCSUtil();
 

test/unit/file_transfer_agent/s3_test.js

@@ -44,7 +44,7 @@ describe('S3 client', function () {
 
   before(function () {
     mock('s3', {
-      S3: function () {
+      S3: function (config) {
         function S3() {
           this.getObject = function () {
             function getObject() {
@@ -57,6 +57,8 @@ describe('S3 client', function () {
 
             return new getObject;
           };
+
+          this.config = config;
           this.putObject = function () {
             function putObject() {
               this.then = function (callback) {
@@ -82,6 +84,59 @@ describe('S3 client', function () {
     AWS = new SnowflakeS3Util(noProxyConnectionConfig, s3, filesystem);
   });
 
+  describe('AWS client endpoint testing', async function () {
+    const originalStageInfo = meta.stageInfo;
+    const testCases = [
+      {
+        name: 'when useS3RegionalURL is only enabled',
+        stageInfo: {
+          ...originalStageInfo,
+          useS3RegionalUrl: true,
+          endPoint: null,
+        },
+        result: null
+      },
+      {
+        name: 'when useS3RegionalURL and is enabled and domain starts with cn',
+        stageInfo: {
+          ...originalStageInfo,
+          useS3RegionalUrl: true,
+          endPoint: null,
+          region: 'cn-mockLocation'
+        },
+        result: 'https://s3.cn-mockLocation.amazonaws.com.cn'
+      },
+      {
+        name: 'when endPoint is enabled',
+        stageInfo: {
+          ...originalStageInfo,
+          endPoint: 's3.endpoint',
+          useS3RegionalUrl: false
+        },
+        result: 'https://s3.endpoint'
+      },
+      {
+        name: 'when both endPoint and useS3PReiongalUrl is valid',
+        stageInfo: {
+          ...originalStageInfo,
+          endPoint: 's3.endpoint',
+          useS3RegionalUrl: true,
+
+        },
+        result: 'https://s3.endpoint'
+      },
+    ];
+
+    testCases.forEach(({ name, stageInfo, result }) => {
+      it(name, () => {
+        const client = AWS.createClient(stageInfo);
+        assert.strictEqual(client.config.endpoint, result);
+      } );
+
+    });
+  });
+
+
   it('extract bucket name and path', async function () {
     let result = extractBucketNameAndPath('sfc-eng-regression/test_sub_dir/');
     assert.strictEqual(result.bucketName, 'sfc-eng-regression');