fix: when err.code is not numeric or doesnot exist take err.response.…

lib/connection/connection_config.js

@@ -52,6 +52,7 @@ const DEFAULT_PARAMS =
   'includeRetryReason',
   'disableQueryContextCache',
   'retryTimeout',
+  'forceGCPUseDownscopedCredential'
 ];
 const Logger = require('../logger');
 
@@ -481,6 +482,15 @@ function ConnectionConfig(options, validateCredentials, qaMode, clientInfo) {
     disableConsoleLogin = options.disableConsoleLogin;
   }
 
+  if (Util.exists(options.forceGCPUseDownscopedCredential)) {
+    Errors.checkArgumentValid(Util.isBoolean(options.forceGCPUseDownscopedCredential),
+      ErrorCodes.ERR_CONN_CREATE_INVALID_FORCE_GCP_USE_DOWNSCOPED_CREDENTIAL);
+
+    process.env.SNOWFLAKE_FORCE_GCP_USE_DOWNSCOPED_CREDENTIAL = options.forceGCPUseDownscopedCredential;
+  } else {
+    process.env.SNOWFLAKE_FORCE_GCP_USE_DOWNSCOPED_CREDENTIAL = false;
+  }
+
   /**
    * Returns an object that contains information about the proxy hostname, port,
    * etc. for when http requests are made.

lib/connection/statement.js

@@ -15,7 +15,7 @@ const Errors = require('../errors');
 const ErrorCodes = Errors.codes;
 const Logger = require('../logger');
 const NativeTypes = require('./result/data_types').NativeTypes;
-const FileTransferAgent = require('.././file_transfer_agent/file_transfer_agent');
+const FileTransferAgent = require('../file_transfer_agent/file_transfer_agent');
 const Bind = require('./bind_uploader');
 const RowMode = require('./../constants/row_mode');
 

lib/constants/error_messages.js

@@ -72,6 +72,7 @@ exports[404044] = 'Invalid retryTimeout value. The specified value must be a num
 exports[404045] = 'Invalid account. The specified value must be a valid subdomain string.';
 exports[404046] = 'Invalid region. The specified value must be a valid subdomain string.';
 exports[404047] = 'Invalid disableConsoleLogin. The specified value must be a boolean';
+exports[404048] = 'Invalid disableGCPTokenUpload. The specified value must be a boolean';
 
 // 405001
 exports[405001] = 'Invalid callback. The specified value must be a function.';

lib/errors.js

@@ -77,6 +77,7 @@ codes.ERR_CONN_CREATE_INVALID_RETRY_TIMEOUT = 404044;
 codes.ERR_CONN_CREATE_INVALID_ACCOUNT_REGEX = 404045;
 codes.ERR_CONN_CREATE_INVALID_REGION_REGEX = 404046;
 codes.ERR_CONN_CREATE_INVALID_DISABLE_CONSOLE_LOGIN = 404047;
+codes.ERR_CONN_CREATE_INVALID_FORCE_GCP_USE_DOWNSCOPED_CREDENTIAL = 404048;
 
 // 405001
 codes.ERR_CONN_CONNECT_INVALID_CALLBACK = 405001;

lib/file_transfer_agent/file_transfer_agent.js

@@ -4,7 +4,6 @@
 
 const binascii = require('binascii');
 const crypto = require('crypto');
-const glob = require('glob');
 const fs = require('fs');
 const os = require('os');
 const mime = require('mime-types');
@@ -13,14 +12,15 @@ const path = require('path');
 const statement = require('../connection/statement');
 const fileCompressionType = require('./file_compression_type');
 const expandTilde = require('expand-tilde');
-const SnowflakeFileUtil = new (require('./file_util').FileUtil)();
 const SnowflakeRemoteStorageUtil = require('./remote_storage_util').RemoteStorageUtil;
+const LocalUtil = require('./local_util').LocalUtil;
 const SnowflakeFileEncryptionMaterial = require('./remote_storage_util').SnowflakeFileEncryptionMaterial;
 const SnowflakeS3Util = require('./s3_util');
-const SnowflakeLocalUtil = new (require('./local_util').LocalUtil)();
-
+const { FileUtil, getMatchingFilePaths } = require('./file_util');
 const resultStatus = require('./file_util').resultStatus;
 
+const SnowflakeFileUtil = new FileUtil();
+const SnowflakeLocalUtil = new LocalUtil();
 const S3_FS = 'S3';
 const AZURE_FS = 'AZURE';
 const GCS_FS = 'GCS';
@@ -379,7 +379,7 @@ function FileTransferAgent(context) {
       meta['errorDetails'] += ` file=${meta['srcFileName']}, real file=${meta['realSrcFilePath']}`;
     } finally {
       // Remove all files inside tmp folder
-      const matchingFileNames = glob.sync(path.join(meta['tmpDir'], meta['srcFileName'] + '*'));
+      const matchingFileNames = getMatchingFilePaths(meta['tmpDir'], meta['srcFileName'] + '*');
       for (const matchingFileName of matchingFileNames) {
         await new Promise((resolve, reject) => {
           fs.unlink(matchingFileName, err => {
@@ -618,7 +618,7 @@ function FileTransferAgent(context) {
           // If file name has a wildcard
           if (fileName.includes('*')) {
             // Get all file names that matches the wildcard
-            const matchingFileNames = glob.sync(path.join(root, fileName));
+            const matchingFileNames = getMatchingFilePaths(root, fileName);
 
             for (const matchingFileName of matchingFileNames) {
               initEncryptionMaterial();

lib/file_transfer_agent/file_util.js

@@ -7,6 +7,8 @@ const fs = require('fs');
 const path = require('path');
 const struct = require('python-struct');
 const zlib = require('zlib');
+const os = require('os');
+const glob = require('glob');
 
 const resultStatus = {
   ERROR: 'ERROR',
@@ -132,5 +134,14 @@ function FileUtil() {
     };
   };
 }
-
 exports.FileUtil = FileUtil;
+
+function getMatchingFilePaths(dir, fileName) {
+  const pathWithWildcard = path.join(dir, fileName);
+  const pathWithWildcardDependsOnPlatform = os.platform() === 'win32'
+    ? pathWithWildcard.replace(/\\/g, '/')
+    : pathWithWildcard;
+  return glob.sync(pathWithWildcardDependsOnPlatform);
+}
+
+exports.getMatchingFilePaths = getMatchingFilePaths;

lib/file_transfer_agent/gcs_util.js

@@ -4,6 +4,7 @@
 
 const EncryptionMetadata = require('./encrypt_util').EncryptionMetadata;
 const FileHeader = require('./file_util').FileHeader;
+const { shouldPerformGCPBucket } = require('../util');
 
 const GCS_METADATA_PREFIX = 'x-goog-meta-';
 const SFC_DIGEST = 'sfc-digest';
@@ -137,7 +138,7 @@ function GCSUtil(httpclient, filestream) {
         let matDescKey;
 
         try {
-          if (accessToken) {
+          if (shouldPerformGCPBucket(accessToken)) {
             const gcsLocation = this.extractBucketNameAndPath(meta['stageInfo']['location']);
 
             const metadata = await meta['client'].gcsClient
@@ -177,7 +178,7 @@ function GCSUtil(httpclient, filestream) {
             encryptionMetadata
           );
         } catch (err) {
-          const errCode = err['code'] ? err['code'] : err.response.status;
+          const errCode = !isNaN(err['code']) && !isNaN(parseInt(err['code'])) ? err['code'] : err.response.status;
 
           if ([403, 408, 429, 500, 503].includes(errCode)) {
             meta['lastError'] = err;
@@ -277,7 +278,7 @@ function GCSUtil(httpclient, filestream) {
     }
 
     try {
-      if (accessToken) {
+      if (shouldPerformGCPBucket(accessToken)) {
         const gcsLocation = this.extractBucketNameAndPath(meta['stageInfo']['location']);
 
         await meta['client'].gcsClient
@@ -350,7 +351,7 @@ function GCSUtil(httpclient, filestream) {
     let size;
 
     try {
-      if (accessToken) {
+      if (shouldPerformGCPBucket(accessToken)) {
         const gcsLocation = this.extractBucketNameAndPath(meta['stageInfo']['location']);
 
         await meta['client'].gcsClient

lib/util.js

@@ -327,7 +327,7 @@ exports.url =
       if (option.includeRetryReason) {
         retryUrl = this.appendParam(retryUrl, 'retryReason', option.retryReason);
       }
-      
+
       return retryUrl;
     }
   };
@@ -393,8 +393,8 @@ exports.nextSleepTime = function (
 /**
  * Return next sleep time calculated by the jitter rule.
  *
- * @param {Number} numofRetries 
- * @param {Number} currentSleepTime 
+ * @param {Number} numofRetries
+ * @param {Number} currentSleepTime
  * @param {Number} totalElapsedTime
  * @param {Number} maxRetryTimeout
  * @returns {JSON} return next sleep Time and totalTime.
@@ -409,20 +409,20 @@ exports.getJitteredSleepTime = function (numofRetries, currentSleepTime, totalEl
 /**
  * Choose one of the number between two numbers.
  *
- * @param {Number} firstNumber 
- * @param {Number} secondNumber 
+ * @param {Number} firstNumber
+ * @param {Number} secondNumber
  * @returns {Number} return a random number between two numbers.
  */
-function chooseRandom(firstNumber, secondNumber) { 
+function chooseRandom(firstNumber, secondNumber) {
   return Math.random() * (firstNumber - secondNumber) + secondNumber;
 }
 
 exports.chooseRandom = chooseRandom;
 
 /**
  * return the next sleep Time.
- * @param {Number} numofRetries 
- * @param {Number} currentSleepTime 
+ * @param {Number} numofRetries
+ * @param {Number} currentSleepTime
  * @returns {Number} return jitter.
  */
 function getNextSleepTime(numofRetries,  currentSleepTime) {
@@ -434,7 +434,7 @@ exports.getNextSleepTime = getNextSleepTime;
 
 /**
  * return the jitter value.
- * @param {Number} currentSleepTime 
+ * @param {Number} currentSleepTime
  * @returns {Number} return jitter.
  */
 function getJitter(currentSleepTime) {
@@ -694,7 +694,10 @@ exports.isFileNotWritableByGroupOrOthers = async function (configFilePath, fsPro
   return (stats.mode & (1 << 4)) === 0 && (stats.mode & (1 << 1)) === 0;
 };
 
-
 exports.shouldRetryOktaAuth = function ({ maxRetryTimeout, maxRetryCount, numRetries, startTime, remainingTimeout }) {
   return  (maxRetryTimeout === 0 || Date.now() < startTime + remainingTimeout) && numRetries <= maxRetryCount;
-};
+};
+
+exports.shouldPerformGCPBucket = function (accessToken) {
+  return !!accessToken && process.env.SNOWFLAKE_DISABLE_GCP_TOKEN_UPLOAD !== 'true';
+};

test/integration/testUtil.js

@@ -5,6 +5,7 @@ const snowflake = require('./../../lib/snowflake');
 const connOptions = require('./connectionOptions');
 const assert = require('assert');
 const fs = require('fs');
+const fsPromises = require('fs').promises;
 const crypto = require('crypto');
 const Logger = require('../../lib/logger');
 const path = require('path');
@@ -290,6 +291,18 @@ module.exports.createTempFile = function (mainDir, fileName, data = '') {
   fs.writeFileSync(fullpath, data);
   return fullpath;
 };
+/**
+ *  Async version of method to create temp file
+ * @param mainDir string Main directory for created file
+ * @param fileName string Created file name
+ * @param data string Input for created file
+ * @return string
+ */
+module.exports.createTempFileAsync = async function (mainDir, fileName, data = '') {
+  const fullpath = path.join(mainDir, fileName);
+  await fsPromises.writeFile(fullpath, data);
+  return fullpath;
+};
 
 /**
  * @param option object

test/unit/connection/connection_config_test.js

@@ -700,6 +700,16 @@ describe('ConnectionConfig: basic', function () {
         },
         errorCode: ErrorCodes.ERR_CONN_CREATE_INVALID_DISABLE_CONSOLE_LOGIN
       },
+      {
+        name: 'invalid disableGCPTokenUpload',
+        options: {
+          account: 'account',
+          username: 'username',
+          password: 'password',
+          forceGCPUseDownscopedCredential: 'invalud'
+        },
+        errorCode: ErrorCodes.ERR_CONN_CREATE_INVALID_FORCE_GCP_USE_DOWNSCOPED_CREDENTIAL
+      },
     ];
 
   const createNegativeITCallback = function (testCase) {
@@ -1242,7 +1252,7 @@ describe('ConnectionConfig: basic', function () {
             password: 'password',
             account: 'account'
           }
-      }
+      },
     ];
 
   const createItCallback = function (testCase) {

test/unit/file_transfer_agent/file_util_test.js

@@ -0,0 +1,52 @@
+/*
+ * Copyright (c) 2024 Snowflake Computing Inc. All rights reserved.
+ */
+
+const assert = require('assert');
+const testUtil = require('../../integration/testUtil');
+const os = require('os');
+const fsPromises = require('fs').promises;
+const crypto = require('crypto');
+const getMatchingFilePaths = require('../../../lib/file_transfer_agent/file_util').getMatchingFilePaths;
+
+
+describe('matching files by wildcard', function () {
+  const randomName = crypto.randomUUID();
+  const excpetedNomberOfMatchedFiles = 3;
+
+  async function createFiles(options) {
+    for (let i = 0; i < excpetedNomberOfMatchedFiles; i++) {
+      await testUtil.createTempFileAsync(os.tmpdir(), testUtil.createRandomFileName(options));
+    }
+  }
+
+  after(async function () {
+    const matchedFiles = getMatchingFilePaths(os.tmpdir(), `${randomName}matched` + '*');
+    const notmatchedFiles = getMatchingFilePaths(os.tmpdir(), `${randomName}notmatched` + '*');
+    const promises = [];
+
+    for (const filePath of matchedFiles) {
+      promises.push(fsPromises.rm(filePath));
+    }
+    for (const filePath of notmatchedFiles) {
+      promises.push(fsPromises.rm(filePath));
+    }
+    await Promise.all(promises);
+  });
+
+  it('match paths with prefix', async function () {
+    await createFiles({ prefix: `${randomName}matched` });
+    await createFiles({ prefix: `${randomName}notmatched` });
+    const matched = getMatchingFilePaths(os.tmpdir(), `${randomName}matched` + '*');
+    assert.strictEqual(matched.length, excpetedNomberOfMatchedFiles);
+  });
+
+  it('match paths with prefix and extension', async function () {
+    await createFiles({ prefix: `${randomName}matched`, extension: '.gz' });
+    await createFiles({ prefix: `${randomName}matched`, extension: '.txt' });
+    await createFiles({ prefix: `${randomName}notmatched` });
+    const matched = getMatchingFilePaths(os.tmpdir(), `${randomName}matched` + '*.gz');
+    assert.strictEqual(matched.length, excpetedNomberOfMatchedFiles);
+  });
+
+});