SNOW-811103: Add sample with test of json parsers

.npmignore

@@ -13,6 +13,7 @@ snowflake-sdk*.tgz
 coverage
 system_test/
 scripts/
+samples/
 ci/
 .github/
 .eslintrc.js

samples/README.md

@@ -0,0 +1,51 @@
+********************************************************************************
+NodeJS Driver - Samples
+********************************************************************************
+
+Install
+======================================================================
+
+In directory samples Run `npm i`.
+
+Test
+======================================================================
+
+Prepare for tests
+----------------------------------------------------------------------
+
+Specify env variables:
+
+```
+export SNOWFLAKE_TEST_USER=<your_user>
+export SNOWFLAKE_TEST_PASSWORD=<your_password>
+export SNOWFLAKE_TEST_ACCOUNT=<your_account>
+export SNOWFLAKE_TEST_WAREHOUSE=<your_warehouse>
+export SNOWFLAKE_TEST_DATABASE=<your_database>
+export SNOWFLAKE_TEST_SCHEMA=<your_schema>
+export SNOWFLAKE_TEST_PROTOCOL=<your_snowflake_protocol>
+export SNOWFLAKE_TEST_HOST=<your_snowflake_host>
+export SNOWFLAKE_TEST_PORT=<your_snowflake_port>
+```
+
+Run test to compare json parser
+----------------------------------------------------------------------
+
+By default, the test creates a table with 300000 rows of sample variant data (json format)
+and measures the time and number of blocks while retrieving the results.
+```
+npm run jsonParserComparison
+```
+Test can be started with parameters:
+ - number of rows in table
+ - number of selected rows
+ - only for choosen parser: Function, vm, better-eval, JSON
+
+Example:
+```
+npm run jsonParserComparison 300000 300000 Function
+```
+
+ or 
+ ```
+npm run jsonParserComparison 300000 300000 JSON
+```

samples/helpers.js

@@ -0,0 +1,65 @@
+const snowflake = require('snowflake-sdk');
+exports.executeQuery = async function (connection, query, binds) {
+  await new Promise((resolve, reject) => {
+    connection.execute({
+      streamResult: true,
+      sqlText: query,
+      binds: binds,
+      complete: function (err, stmt) {
+        const stream = stmt.streamRows();
+        stream.on('readable', function (row) {
+          while ((row = this.read()) !== null) {
+            console.log(row);
+          }
+        });
+        stream.on('end', function () {
+          resolve();
+        });
+        stream.on('error', function (err) {
+          console.log(err);
+          reject();
+        });
+      }
+    });
+  });
+};
+
+exports.connectUsingEnv = async () => {
+  const snowflakeTestProtocol = process.env.SNOWFLAKE_TEST_PROTOCOL;
+  const snowflakeTestHost = process.env.SNOWFLAKE_TEST_HOST;
+  const snowflakeTestPort = process.env.SNOWFLAKE_TEST_PORT;
+  const snowflakeTestAccount = process.env.SNOWFLAKE_TEST_ACCOUNT;
+  const snowflakeTestUser = process.env.SNOWFLAKE_TEST_USER;
+  const snowflakeTestDatabase = process.env.SNOWFLAKE_TEST_DATABASE;
+  const snowflakeTestWarehouse = process.env.SNOWFLAKE_TEST_WAREHOUSE;
+  const snowflakeTestSchema = process.env.SNOWFLAKE_TEST_SCHEMA;
+  const snowflakeTestPassword = process.env.SNOWFLAKE_TEST_PASSWORD;
+  const snowflakeTestRole = process.env.SNOWFLAKE_TEST_ROLE;
+
+  const connection = snowflake.createConnection({
+    account: snowflakeTestAccount,
+    username: snowflakeTestUser,
+    password: snowflakeTestPassword,
+    role: snowflakeTestRole,
+    database: snowflakeTestDatabase,
+    schema: snowflakeTestSchema,
+    warehouse: snowflakeTestWarehouse,
+    host: snowflakeTestHost,
+    port: snowflakeTestPort,
+    protocol: snowflakeTestProtocol
+  });
+
+  return  new Promise((resolve, reject) => {
+    connection.connect(
+      function (err, conn) {
+        if (err) {
+          console.error('Unable to connect: ' + err.message);
+          reject(new Error(err.message));
+        } else {
+          console.log('Successfully connected to Snowflake');
+          resolve(conn);
+        }
+      }
+    );
+  });
+};

samples/jsonParserComparison.js

@@ -0,0 +1,165 @@
+const snowflake = require('snowflake-sdk');
+const helpers = require('./helpers');
+const blocked = require('blocked-at');
+
+async function run() {
+  console.log('Started with arguments: ');
+  console.log(`Inserted rows amount: ${process.argv[2]} - default 300000`);
+  console.log(`Selected rows amount: ${process.argv[3]} - default 300000`);
+  console.log(`Selected json parse : ${process.argv[4]} - default all of Function, eval, better-eval, JSON`);
+
+  const connection = await helpers.connectUsingEnv();
+
+  const rowCount = process.argv[2] || 300000;
+  const selectLimit = process.argv[3] || 300000;
+  const testVariantTempName = 'testJsonTempTable000';
+
+  const createTempTableWithJsonData = `CREATE OR REPLACE TABLE ${testVariantTempName} (value string)
+    AS select parse_json('{
+        "_id": "6501c357397b66ce47719212",
+        "index": 0,
+        "guid": "e7e0e5d8-82b4-47f7-a2ab-68588c93d81e",
+        "isActive": false,
+        "balance": "$2,611.69",
+        "picture": "http://placehold.it/32x32",
+        "age": 21,
+        "eyeColor": "blue",
+        "name": "Joanna Atkinson",
+        "gender": "female",
+        "company": "AQUAZURE",
+        "email": "[email protected]",
+        "phone": "+1 (925) 582-3869",
+        "address": "395 Karweg Place, Garnet, Mississippi, 9481",
+        "registered": "2017-05-18T11:16:33 -02:00",
+        "latitude": 21.372656,
+        "longitude": -24.488326,
+        "tags": [
+          "aliquip",
+          "aliqua",
+          "magna",
+          "pariatur",
+          "cillum",
+          "esse",
+          "nisi"
+        ],
+        "friends": [
+          {
+            "id": 0,
+            "name": "Davis Blake"
+          },
+          {
+            "id": 1,
+            "name": "Raymond Jefferson"
+          },
+          {
+            "id": 2,
+            "name": "Hoffman Roberts"
+          }
+        ],
+        "greeting": "Hello, Joanna Atkinson! You have 3 unread messages.",
+        "favoriteFruit": "apple"
+      }') 
+      from table(generator(rowcount=>${rowCount}))`;
+  const createTableWithVariant = (tableName) => `create or replace table ${tableName}(colA variant)`;
+
+  const dropTableWithVariant = (tableName) =>`drop table if exists ${tableName}`;
+  const dropTempTable = `drop table if exists ${testVariantTempName}`;
+
+  const insertVariant = (tableName)=> `insert into ${tableName}
+                         select parse_json(value)
+                         from ${testVariantTempName}`;
+  const selectCountVariant = (tableName) => `select count(colA) from ${(tableName)}`;
+
+  const testCases = [];
+  if (!process.argv[4] || process.argv[4].toString().includes('Function')) {
+    testCases.push({parser: 'Function', jsonColumnVariantParser: (rawColumnValue) => new Function(`return (${rawColumnValue})`)});
+  }
+  if (!process.argv[4] || process.argv[4].toString().includes('betterEval')) {
+    testCases.push({parser: 'betterEval', jsonColumnVariantParser: (rawColumnValue) => require('better-eval').call('(' + rawColumnValue + ')')});
+  }
+  if (!process.argv[4] || process.argv[4].toString().includes('vm')) {
+    testCases.push({parser: 'vm', jsonColumnVariantParser: rawColumnValue => require('vm').runInNewContext('(' + rawColumnValue + ')')});
+  }
+  // eval contain vulnerability so we decide to resign using it
+  // if (!process.argv[4] || process.argv[4].toString().contains('eval')) {
+  //   testCases.push({parser: 'eval', jsonColumnVariantParser: rawColumnValue => eval('(' + rawColumnValue + ')')})
+  // };
+  if (!process.argv[4] || process.argv[4].toString().includes('JSON')) {
+    testCases.push({parser: 'JSON', jsonColumnVariantParser: rawColumnValue => JSON.parse(rawColumnValue)});
+  }
+
+  const execute = ({parser, jsonColumnVariantParser}) => {
+    console.log(`\nTesting for parser:  ${parser}`);
+    const testVariantTableName = `testVariantTable000${parser}`;
+
+    return new Promise(async (resolve, reject) => {
+      snowflake.configure({
+        jsonColumnVariantParser: jsonColumnVariantParser
+      });
+
+      await helpers.executeQuery(connection, createTempTableWithJsonData);
+      await helpers.executeQuery(connection, createTableWithVariant(testVariantTableName));
+      await helpers.executeQuery(connection, insertVariant(testVariantTableName));
+      await helpers.executeQuery(connection, selectCountVariant(testVariantTableName));
+
+      const queryTimeLabel = parser + 'SelectTime';
+      let avgBlock = 0, minBlock = 999999999999999, maxBlock = 0;
+      let blockCount = 0;
+      blocked((time) => {
+        blockCount++;
+        avgBlock += time;
+        minBlock = minBlock > time ? time : minBlock;
+        maxBlock = maxBlock < time ? time : maxBlock;
+      });
+
+      console.time(queryTimeLabel);
+      let count = 0;
+      const streamResult = true;
+      connection.execute({
+        streamResult: streamResult,
+        sqlText: `select *
+                  from IDENTIFIER(?) LIMIT ${selectLimit}`,
+        binds: [testVariantTableName],
+        complete: function (err, stmt) {
+          const stream = stmt.streamRows();
+          stream.on('readable', function () {
+            while ((stream.read()) !== null) {
+              count++;
+              if (count % 10000 === 0) {
+                console.log(`Parsed rows: ${count}`);
+              }
+            }
+          });
+          stream.on('end', function () {
+            console.log('parser: ' + parser);
+            console.log('streamResult: ' + streamResult);
+            console.log('row count: ' + count);
+            console.timeEnd(queryTimeLabel);
+            console.log('average block time: ' + avgBlock / blockCount);
+            console.log('minimum block time: ' + minBlock);
+            console.log('maximum block time: ' + maxBlock);
+            console.log('block call count: ' + blockCount);
+            resolve();
+          });
+          stream.on('error', function (err) {
+            console.log(err);
+            reject(err);
+          });
+        }
+      });
+    })
+      .finally(async () => {
+        await helpers.executeQuery(connection, dropTableWithVariant(testVariantTableName));
+        await helpers.executeQuery(connection, dropTempTable);
+      });
+  };
+
+  testCases.reduce( (promise, nextParser) => {
+    return promise.then(() => {
+      return execute(nextParser);
+    });
+  }, Promise.resolve());
+}
+
+run();
+

samples/package.json

@@ -0,0 +1,18 @@
+{
+  "name": "sample",
+  "version": "0.0.1",
+  "description": "Snowflake Node.js driver samples",
+  "main": "jsonParserComparison.js",
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "better-eval": "^1.3.0",
+    "blocked-at": "^1.2.0",
+    "snowflake-sdk": "^1.8.0",
+    "vm": "^0.1.0"
+  },
+  "scripts": {
+    "jsonParserComparison": "node jsonParserComparison.js",
+    "promiseExample": "node promiseExample.js"
+  }
+}