From 378aaeac1b945659446c203061d7524d83d8fb2f Mon Sep 17 00:00:00 2001 From: Dominik Przybysz Date: Mon, 11 Mar 2024 08:31:38 +0100 Subject: [PATCH 1/5] SNOW-1215393: Test chunk fetching causing out of memory --- ci/container/test_component.sh | 6 +++--- lib/connection/result/chunk.js | 15 ++++++++++++++- lib/connection/result/result_stream.js | 1 + lib/connection/result/row_stream.js | 1 + package.json | 2 +- test/integration/testLargeResultSet.js | 20 +++++++++++++++----- 6 files changed, 35 insertions(+), 10 deletions(-) diff --git a/ci/container/test_component.sh b/ci/container/test_component.sh index 28c62832e..6814b2584 100755 --- a/ci/container/test_component.sh +++ b/ci/container/test_component.sh @@ -7,7 +7,7 @@ THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" export WORKSPACE=${WORKSPACE:-/mnt/workspace} export SOURCE_ROOT=${SOURCE_ROOT:-/mnt/host} export DRIVER_NAME=nodejs -export TIMEOUT=180000 +export TIMEOUT=360000 export SF_OCSP_TEST_OCSP_RESPONDER_TIMEOUT=1000 [[ -z "$GIT_BRANCH" ]] && echo "Set GIT_BRANCH to test" && exit 1 @@ -70,11 +70,11 @@ python3 $THIS_DIR/hang_webserver.py 12345 > hang_webserver.out 2>&1 & if [[ "$SHOULD_GENERATE_COVERAGE_REPORT" == "1" ]]; then MOCHA_CMD=( - "npx" "nyc" "--reporter=lcov" "--reporter=text" "mocha" "--exit" "--timeout" "$TIMEOUT" "--recursive" "--full-trace" + "npx" "nyc" "--reporter=lcov" "--reporter=text" "mocha" "--exit" "--timeout" "$TIMEOUT" "--recursive" "--full-trace" "--max-old-space-size=150" ) else MOCHA_CMD=( - "mocha" "--timeout" "$TIMEOUT" "--recursive" "--full-trace" + "mocha" "--timeout" "$TIMEOUT" "--recursive" "--full-trace" "--max-old-space-size=150" ) fi diff --git a/lib/connection/result/chunk.js b/lib/connection/result/chunk.js index c96b77f9e..d8a424ebe 100644 --- a/lib/connection/result/chunk.js +++ b/lib/connection/result/chunk.js @@ -160,6 +160,8 @@ Chunk.prototype.clearRows = function (force) { // clear out all row and rowset related fields this._rowsetAsString = this._rowset = this._rows = undefined; } + this._isProcessed = true; + console.log(`Cleared chunk ${this._id}`); }; /** @@ -199,6 +201,16 @@ Chunk.prototype.isLoading = function () { * @param callback */ Chunk.prototype.load = function (callback) { + // if (this.isLoaded()) { + // console.log(`Ordered fetch of chunk ${this._id} when is already loaded - skipping`); + // return; + // } + if (this._isProcessed) { + console.log(`Ordered fetch of chunk ${this._id} when is already processed`); + } + if (this.isLoaded()) { + console.log(`Ordered fetch of chunk ${this._id} when is already loaded`); + } // we've started loading this._isLoading = true; @@ -237,6 +249,7 @@ Chunk.prototype.load = function (callback) { // if the request succeeded, save the // body as a string version of the rowset if (!err) { + console.log(`Fetched chunk ${self._id}`); self._rowsetAsString = body; } @@ -257,7 +270,7 @@ Chunk.prototype.load = function (callback) { * @private */ function buildId(startIndex, endIndex) { - return Util.format('s=%d, e=%d', startIndex, endIndex); + return Util.format('s=%d, e=%d size=%d', startIndex, endIndex, endIndex - startIndex + 1); } /** diff --git a/lib/connection/result/result_stream.js b/lib/connection/result/result_stream.js index 82fb675c4..b1ccae53a 100644 --- a/lib/connection/result/result_stream.js +++ b/lib/connection/result/result_stream.js @@ -84,6 +84,7 @@ function ResultStream(options) { for (index = currChunk; index < chunks.length && index <= (currChunk + prefetchSize); index++) { chunk = chunks[index]; if (!chunk.isLoading()) { + console.log(`Fetching chunk ${chunk._id}`); chunk.load(); } } diff --git a/lib/connection/result/row_stream.js b/lib/connection/result/row_stream.js index 0c72627d5..dee6b9da7 100644 --- a/lib/connection/result/row_stream.js +++ b/lib/connection/result/row_stream.js @@ -196,6 +196,7 @@ function RowStream(statement, context, options) { rowBuffer = chunk.getRows().slice( Math.max(chunkStart, start) - chunkStart, Math.min(chunkEnd, end) + 1 - chunkStart); + console.log(`Row buffer length is ${rowBuffer.length}` ); // reset the row index rowIndex = 0; diff --git a/package.json b/package.json index 430b58813..82c636d2b 100644 --- a/package.json +++ b/package.json @@ -61,7 +61,7 @@ "lint:check:all:errorsOnly": "npm run lint:check:all -- --quiet", "lint:fix": "eslint --fix", "test": "mocha -timeout 180000 --recursive --full-trace test/unit/**/*.js test/unit/*.js", - "test:integration": "mocha -timeout 180000 --recursive --full-trace test/integration/**/*.js test/integration/*.js", + "test:integration": "mocha -timeout 360000 --recursive --full-trace test/integration/**/*.js test/integration/*.js", "test:single": "mocha -timeout 180000 --full-trace", "test:system": "mocha -timeout 180000 --recursive --full-trace system_test/*.js", "test:unit": "mocha -timeout 180000 --recursive --full-trace test/unit/**/*.js test/unit/*.js", diff --git a/test/integration/testLargeResultSet.js b/test/integration/testLargeResultSet.js index 460fb5c04..d12dbfd78 100644 --- a/test/integration/testLargeResultSet.js +++ b/test/integration/testLargeResultSet.js @@ -5,31 +5,41 @@ const assert = require('assert'); const async = require('async'); const testUtil = require('./testUtil'); const { randomizeName } = require('./testUtil'); +const { writeHeapSnapshot } = require('node:v8'); describe('Large result Set Tests', function () { - const sourceRowCount = 10000; - + const sourceRowCount = 1000000; let connection; const selectAllFromOrders = `select randstr(1000,random()) from table(generator(rowcount=>${sourceRowCount}))`; - + // const selectAllFromOrders = `select * from fake_sample_data.public.customer limit 400000`; + // const selectAllFromOrders = `select * from fake_sample_data.public.customer`; before(async () => { - connection = testUtil.createConnection(); + // configureLogger('TRACE'); + connection = testUtil.createConnection({ + resultPrefetch: 2, + }); await testUtil.connectAsync(connection); + await testUtil.executeCmdAsync(connection, "alter session set CLIENT_RESULT_CHUNK_SIZE=48"); }); after(async () => { await testUtil.destroyConnectionAsync(connection); }); - it('testSimpleLarge', function (done) { + it.only('testSimpleLarge', function (done) { connection.execute({ sqlText: selectAllFromOrders, + streamResult: true, complete: function (err, stmt) { testUtil.checkError(err); const stream = stmt.streamRows(); let rowCount = 0; stream.on('data', function () { rowCount++; + if (rowCount % 5000 === 0) { + console.log(`Row count: ${rowCount}`); + // console.log(writeHeapSnapshot()); + } }); stream.on('error', function (err) { testUtil.checkError(err); From ab61f6dcf2a2eb497c75f7a3f899f4a14c11810c Mon Sep 17 00:00:00 2001 From: Przemyslaw Motacki Date: Wed, 19 Jun 2024 09:15:37 +0200 Subject: [PATCH 2/5] Test OOM chunks --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 82c636d2b..d8a00e400 100644 --- a/package.json +++ b/package.json @@ -62,7 +62,7 @@ "lint:fix": "eslint --fix", "test": "mocha -timeout 180000 --recursive --full-trace test/unit/**/*.js test/unit/*.js", "test:integration": "mocha -timeout 360000 --recursive --full-trace test/integration/**/*.js test/integration/*.js", - "test:single": "mocha -timeout 180000 --full-trace", + "test:single": "mocha -timeout 180001 --full-trace", "test:system": "mocha -timeout 180000 --recursive --full-trace system_test/*.js", "test:unit": "mocha -timeout 180000 --recursive --full-trace test/unit/**/*.js test/unit/*.js", "test:unit:coverage": "nyc npm run test:unit", From 66c649d22b413d60e515cfa0cc291aaccd52d76c Mon Sep 17 00:00:00 2001 From: Przemyslaw Motacki Date: Wed, 19 Jun 2024 09:17:02 +0200 Subject: [PATCH 3/5] Test OOM chunks --- .github/workflows/build-test.yml | 95 +------------------------------- 1 file changed, 3 insertions(+), 92 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 26b813283..02a20c72c 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -48,8 +48,8 @@ jobs: strategy: fail-fast: false matrix: - cloud: [ 'AWS', 'AZURE', 'GCP' ] - nodeVersion: [ '14.x', '16.x', '18.x', '20.x'] + cloud: [ 'AWS' ] + nodeVersion: [ '20.x'] steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v3 @@ -77,93 +77,4 @@ jobs: with: # without the token code cov may fail because of Github limits https://github.com/codecov/codecov-action/issues/557 token: ${{ secrets.CODE_COV_UPLOAD_TOKEN }} - fail_ci_if_error: true - - test-windows: - needs: build - name: Tests on Windows - runs-on: windows-latest - strategy: - fail-fast: false - matrix: - cloud: [ 'AWS', 'AZURE', 'GCP' ] - nodeVersion: [ '14.x', '16.x', '18.x', '20.x'] - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v3 - with: - node-version: ${{ matrix.nodeVersion }} - - uses: actions/setup-python@v4 - with: - python-version: '3.7' - architecture: 'x64' - - name: Download Build Artifacts - uses: actions/download-artifact@v3 - with: - name: artifacts - path: artifacts - - name: Tests - shell: cmd - env: - PARAMETERS_SECRET: ${{ secrets.PARAMETERS_SECRET }} - CLOUD_PROVIDER: ${{ matrix.cloud }} - run: ci\\test_windows.bat - - test-linux: - needs: build - name: Tests on Linux - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - image: [ 'nodejs-centos7-node14', 'nodejs-centos7-fips'] - cloud: [ 'AWS', 'AZURE', 'GCP' ] - steps: - - uses: actions/checkout@v4 - - name: Download Build Artifacts - uses: actions/download-artifact@v3 - with: - name: artifacts - path: artifacts - - name: Tests - shell: bash - env: - PARAMETERS_SECRET: ${{ secrets.PARAMETERS_SECRET }} - CLOUD_PROVIDER: ${{ matrix.cloud }} - TARGET_DOCKER_TEST_IMAGE: ${{ matrix.image }} - run: ./ci/test.sh - - test-ubuntu: - needs: build - name: Tests on Ubuntu - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - cloud: [ 'AWS', 'AZURE', 'GCP' ] - nodeVersion: ['18.x', '20.x'] - steps: - - uses: actions/checkout@v4 - - name: Download Build Artifacts - uses: actions/download-artifact@v3 - with: - name: artifacts - path: artifacts - - name: Use Node.js - uses: actions/setup-node@v3 - with: - node-version: ${{ matrix.node-version }} - - name: Tests - shell: bash - env: - PARAMETERS_SECRET: ${{ secrets.PARAMETERS_SECRET }} - CLOUD_PROVIDER: ${{ matrix.cloud }} - run: ./ci/test_ubuntu.sh - - name: Upload coverage reports to Codecov - uses: codecov/codecov-action@v3 - with: - # without the token code cov may fail because of Github limits https://github.com/codecov/codecov-action/issues/557 - token: ${{ secrets.CODE_COV_UPLOAD_TOKEN }} - fail_ci_if_error: true - - + fail_ci_if_error: true \ No newline at end of file From e9f39e16184a4f2d1f1deeb88655905af39a371e Mon Sep 17 00:00:00 2001 From: Przemyslaw Motacki Date: Wed, 19 Jun 2024 09:38:14 +0200 Subject: [PATCH 4/5] Test OOM chunks --- .github/workflows/lint.yml | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 87a48ab4b..70991015a 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,28 +1,3 @@ name: Lint -on: - push: - branches: - - master - tags: - - v* - pull_request: - branches: - - master - - prep-** -jobs: - lint: - name: Run lint - runs-on: ubuntu-latest - steps: - - name: Check out code - uses: actions/checkout@v1 - - name: Set up Node.js - uses: actions/setup-node@v1 - with: - node-version: '18.x' - - name: Install dependencies - run: npm i - - name: Check formatting - run: npm run lint:check:all From 72bcd81a6cd3d219064cd35316b7f1ab44335022 Mon Sep 17 00:00:00 2001 From: Przemyslaw Motacki Date: Wed, 19 Jun 2024 09:43:03 +0200 Subject: [PATCH 5/5] Test OOM chunks --- .github/workflows/lint.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 70991015a..e62e45131 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,3 +1,27 @@ name: Lint +on: + push: + branches: + - master + tags: + - v* + pull_request: + branches: + - master + - prep-** + +jobs: + lint: + name: Run lint + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v1 + - name: Set up Node.js + uses: actions/setup-node@v1 + with: + node-version: '18.x' + - name: Install dependencies + run: npm i