From 5b0b97c011e90a328eeef9f53d8e19306b747731 Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Tue, 23 May 2023 10:24:28 -0700 Subject: [PATCH 01/22] Set centos7 node version to 14.17 --- ci/image/Dockerfile.nodejs-centos7-fips-test | 2 +- ci/image/Dockerfile.nodejs-centos7-node12-test | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/image/Dockerfile.nodejs-centos7-fips-test b/ci/image/Dockerfile.nodejs-centos7-fips-test index 36c9f402c..f0d030959 100644 --- a/ci/image/Dockerfile.nodejs-centos7-fips-test +++ b/ci/image/Dockerfile.nodejs-centos7-fips-test @@ -35,7 +35,7 @@ SHELL [ "/usr/bin/scl", "enable", "devtoolset-8"] # node-fips environment variables ENV NODE_HOME $HOME/node -ENV NODEJS_VERSION 14.0.0 +ENV NODEJS_VERSION 14.17.0 ENV FIPSDIR $HOME/install-openssl-fips ENV OPENSSL_VERSION 2.0.16 diff --git a/ci/image/Dockerfile.nodejs-centos7-node12-test b/ci/image/Dockerfile.nodejs-centos7-node12-test index b6a65ec1d..9a4b128dd 100644 --- a/ci/image/Dockerfile.nodejs-centos7-node12-test +++ b/ci/image/Dockerfile.nodejs-centos7-node12-test @@ -7,7 +7,7 @@ RUN yum -y install centos-release-scl # nvm environment variables ENV NVM_DIR /usr/local/nvm -ENV NODE_VERSION 12.0.0 +ENV NODE_VERSION 14.17.0 # node RUN mkdir -p $NVM_DIR From 685badbaac53347d51f7332895f8d726bc0b51c8 Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Tue, 23 May 2023 12:53:41 -0700 Subject: [PATCH 02/22] Re-add fips test --- .github/workflows/build-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index fe4fbc649..f6a393184 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -103,7 +103,7 @@ jobs: strategy: max-parallel: 1 matrix: - image: [ 'nodejs-centos6-default', 'nodejs-centos7-node12' ] + image: [ 'nodejs-centos6-default', 'nodejs-centos7-node12', 'nodejs-centos7-fips' ] cloud: [ 'AWS' ] steps: - uses: actions/checkout@v1 @@ -190,7 +190,7 @@ jobs: strategy: max-parallel: 1 matrix: - image: [ 'nodejs-centos6-default', 'nodejs-centos7-node12' ] + image: [ 'nodejs-centos6-default', 'nodejs-centos7-node12', 'nodejs-centos7-fips' ] #REMOVE AZURE for now #cloud: [ 'AZURE', 'GCP' ] cloud: [ 'GCP' ] From 2fc11086b93055e6003ff1f4d5a8db7be30abce3 Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Fri, 23 Jun 2023 09:07:13 -0700 Subject: [PATCH 03/22] issue 207 - Update docker image to node18/openssl3 and enable fips --- ci/image/Dockerfile.nodejs-centos7-fips-test | 38 +++++++++++++++----- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/ci/image/Dockerfile.nodejs-centos7-fips-test b/ci/image/Dockerfile.nodejs-centos7-fips-test index f0d030959..81efb31ad 100644 --- a/ci/image/Dockerfile.nodejs-centos7-fips-test +++ b/ci/image/Dockerfile.nodejs-centos7-fips-test @@ -35,29 +35,49 @@ SHELL [ "/usr/bin/scl", "enable", "devtoolset-8"] # node-fips environment variables ENV NODE_HOME $HOME/node -ENV NODEJS_VERSION 14.17.0 -ENV FIPSDIR $HOME/install-openssl-fips -ENV OPENSSL_VERSION 2.0.16 +ENV NODEJS_VERSION 18.0.0 +ENV OPENSSL_VERSION 3.0.8 +ENV PKG_CONFIG_PATH "/usr/local/lib64/pkgconfig" +ENV LD_LIBRARY_PATH "${LD_LIBRARY_PATH}:/usr/local/lib64" +ENV OPENSSL_CONF /usr/local/ssl/openssl.cnf +ENV FIPSCONF /usr/local/ssl/fipsmodule.cnf +ENV OPENSSL_MODULES=/usr/local/lib64/ossl-modules # Install OpenSSL RUN cd $HOME -RUN curl https://www.openssl.org/source/openssl-fips-$OPENSSL_VERSION.tar.gz -o $HOME/openssl-fips-$OPENSSL_VERSION.tar.gz +RUN curl https://www.openssl.org/source/openssl-$OPENSSL_VERSION.tar.gz -o $HOME/openssl-fips-$OPENSSL_VERSION.tar.gz RUN tar -xvf $HOME/openssl-fips-$OPENSSL_VERSION.tar.gz -RUN mv openssl-fips-$OPENSSL_VERSION $HOME/openssl-fips +RUN mv openssl-$OPENSSL_VERSION $HOME/openssl-fips RUN cd $HOME/openssl-fips - + +# Install OpenSSL dependencies +RUN yum -y install perl-IPC-Cmd +RUN yum -y install perl-Digest-SHA +RUN yum -y install openssl-devel + # You must run ONLY these commands when building the FIPS version of OpenSSL -RUN cd $HOME/openssl-fips && ./config && make && make install - +RUN cd $HOME/openssl-fips && ./config enable-fips && make && make install + +# Enable FIPS by editing the openssl.cnf file +RUN sed -i "s/# .include fipsmodule.cnf/.include ${FIPSCONF//\//\\/}/g" $OPENSSL_CONF +RUN sed -i 's/# fips = fips_sect/fips = fips_sect/g' $OPENSSL_CONF +RUN sed -i 's/# activate = 1/activate = 1/g' $OPENSSL_CONF +RUN sed -i '55ialg_section = algorithm_sect' $OPENSSL_CONF +RUN sed -i '75idefault_properties = fips=yes' $OPENSSL_CONF +RUN sed -i '75i[algorithm_sect]' $OPENSSL_CONF + # Download and build NodeJS RUN git clone --branch v$NODEJS_VERSION https://github.com/nodejs/node.git $NODE_HOME RUN gcc --version RUN g++ --version -RUN cd $NODE_HOME && ./configure --openssl-fips=$FIPSDIR && make -j2 &> /dev/null && make install +RUN cd $NODE_HOME && ./configure --shared-openssl --shared-openssl-libpath=/usr/local/lib64 --shared-openssl-includes=/usr/local/include/openssl --openssl-is-fips +RUN make -j2 &> /dev/null && make install # Should be $NODEJS_VERSION RUN node --version # Should be $OPENSSL_VERSION RUN node -p "process.versions.openssl" +# Should be 1 (fips is enabled by default) +RUN node --enable-fips -p 'crypto.getFips()' # workspace RUN mkdir -p /home/user From 503bf46f27828fca9f20a01b16321aceeb03f6b8 Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Fri, 21 Jul 2023 14:58:25 -0700 Subject: [PATCH 04/22] issue 207 - Fix spaces and remove "--enable-fips" flag --- ci/image/Dockerfile.nodejs-centos7-fips-test | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/ci/image/Dockerfile.nodejs-centos7-fips-test b/ci/image/Dockerfile.nodejs-centos7-fips-test index 81efb31ad..a551e1d33 100644 --- a/ci/image/Dockerfile.nodejs-centos7-fips-test +++ b/ci/image/Dockerfile.nodejs-centos7-fips-test @@ -43,7 +43,7 @@ ENV OPENSSL_CONF /usr/local/ssl/openssl.cnf ENV FIPSCONF /usr/local/ssl/fipsmodule.cnf ENV OPENSSL_MODULES=/usr/local/lib64/ossl-modules -# Install OpenSSL +# Install OpenSSL RUN cd $HOME RUN curl https://www.openssl.org/source/openssl-$OPENSSL_VERSION.tar.gz -o $HOME/openssl-fips-$OPENSSL_VERSION.tar.gz RUN tar -xvf $HOME/openssl-fips-$OPENSSL_VERSION.tar.gz @@ -70,14 +70,13 @@ RUN sed -i '75i[algorithm_sect]' $OPENSSL_CONF RUN git clone --branch v$NODEJS_VERSION https://github.com/nodejs/node.git $NODE_HOME RUN gcc --version RUN g++ --version -RUN cd $NODE_HOME && ./configure --shared-openssl --shared-openssl-libpath=/usr/local/lib64 --shared-openssl-includes=/usr/local/include/openssl --openssl-is-fips -RUN make -j2 &> /dev/null && make install +RUN cd $NODE_HOME && ./configure --shared-openssl --shared-openssl-libpath=/usr/local/lib64 --shared-openssl-includes=/usr/local/include/openssl --openssl-is-fips && make -j2 &> /dev/null && make install # Should be $NODEJS_VERSION RUN node --version # Should be $OPENSSL_VERSION RUN node -p "process.versions.openssl" -# Should be 1 (fips is enabled by default) -RUN node --enable-fips -p 'crypto.getFips()' +# Should be 1 (FIPS is enabled by default) +RUN node -p 'crypto.getFips()' # workspace RUN mkdir -p /home/user From 587592dd24054ba85e8509b9bd858fe34b48abde Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Thu, 3 Aug 2023 14:35:46 -0700 Subject: [PATCH 05/22] issue 217 - Use node 18.17.0 (LTS) for the FIPS docker image --- ci/image/Dockerfile.nodejs-centos7-fips-test | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/image/Dockerfile.nodejs-centos7-fips-test b/ci/image/Dockerfile.nodejs-centos7-fips-test index a551e1d33..dce338c21 100644 --- a/ci/image/Dockerfile.nodejs-centos7-fips-test +++ b/ci/image/Dockerfile.nodejs-centos7-fips-test @@ -35,7 +35,7 @@ SHELL [ "/usr/bin/scl", "enable", "devtoolset-8"] # node-fips environment variables ENV NODE_HOME $HOME/node -ENV NODEJS_VERSION 18.0.0 +ENV NODEJS_VERSION 18.17.0 ENV OPENSSL_VERSION 3.0.8 ENV PKG_CONFIG_PATH "/usr/local/lib64/pkgconfig" ENV LD_LIBRARY_PATH "${LD_LIBRARY_PATH}:/usr/local/lib64" @@ -59,6 +59,7 @@ RUN yum -y install openssl-devel RUN cd $HOME/openssl-fips && ./config enable-fips && make && make install # Enable FIPS by editing the openssl.cnf file +RUN sed -i "s/openssl_conf = openssl_init/nodejs_conf = openssl_init/g" $OPENSSL_CONF RUN sed -i "s/# .include fipsmodule.cnf/.include ${FIPSCONF//\//\\/}/g" $OPENSSL_CONF RUN sed -i 's/# fips = fips_sect/fips = fips_sect/g' $OPENSSL_CONF RUN sed -i 's/# activate = 1/activate = 1/g' $OPENSSL_CONF From 84884f2b710a9d92c9078f12d2bc8e4479a3d4c4 Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Thu, 10 Aug 2023 09:46:15 -0700 Subject: [PATCH 06/22] SNOW-748680: Add FIPS image to the node matrix for Linux builds --- .github/workflows/build-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index f53b3d6e6..121a4e694 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -109,7 +109,7 @@ jobs: strategy: fail-fast: false matrix: - image: [ 'nodejs-centos7-node14' ] + image: [ 'nodejs-centos7-node14', 'nodejs-centos7-fips' ] cloud: [ 'AWS' ] steps: - uses: actions/checkout@v1 @@ -192,7 +192,7 @@ jobs: strategy: fail-fast: false matrix: - image: [ 'nodejs-centos7-node14'] + image: [ 'nodejs-centos7-node14', 'nodejs-centos7-fips' ] cloud: [ 'AZURE', 'GCP' ] steps: - uses: actions/checkout@v1 From 4c7d48ed5fc0dff3ce33058aedf19981ac0acdcc Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Thu, 10 Aug 2023 12:36:05 -0700 Subject: [PATCH 07/22] SNOW-748680: Use legacy provider for openssl --- .github/workflows/build-test.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 121a4e694..006abd805 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -123,7 +123,9 @@ jobs: PARAMETERS_SECRET: ${{ secrets.PARAMETERS_SECRET }} CLOUD_PROVIDER: ${{ matrix.cloud }} TARGET_DOCKER_TEST_IMAGE: ${{ matrix.image }} - run: ./ci/test.sh + run: | + export NODE_OPTIONS=--openssl-legacy-provider + ./ci/test.sh test-mac-optional: needs: build @@ -206,7 +208,9 @@ jobs: PARAMETERS_SECRET: ${{ secrets.PARAMETERS_SECRET }} CLOUD_PROVIDER: ${{ matrix.cloud }} TARGET_DOCKER_TEST_IMAGE: ${{ matrix.image }} - run: ./ci/test.sh + run: | + export NODE_OPTIONS=--openssl-legacy-provider + ./ci/test.sh upload_code_coverage: # it's separate job because codecov action can fail https://github.com/codecov/codecov-action/issues/598 (and we want to know about it), but we don't want to fail required mac build needs: test-mac-required From 45867322692f2f1e34f4da400eb980204dc795cc Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Thu, 10 Aug 2023 13:02:33 -0700 Subject: [PATCH 08/22] SNOW-748680: Revert using legacy provider for openssl --- .github/workflows/build-test.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 006abd805..121a4e694 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -123,9 +123,7 @@ jobs: PARAMETERS_SECRET: ${{ secrets.PARAMETERS_SECRET }} CLOUD_PROVIDER: ${{ matrix.cloud }} TARGET_DOCKER_TEST_IMAGE: ${{ matrix.image }} - run: | - export NODE_OPTIONS=--openssl-legacy-provider - ./ci/test.sh + run: ./ci/test.sh test-mac-optional: needs: build @@ -208,9 +206,7 @@ jobs: PARAMETERS_SECRET: ${{ secrets.PARAMETERS_SECRET }} CLOUD_PROVIDER: ${{ matrix.cloud }} TARGET_DOCKER_TEST_IMAGE: ${{ matrix.image }} - run: | - export NODE_OPTIONS=--openssl-legacy-provider - ./ci/test.sh + run: ./ci/test.sh upload_code_coverage: # it's separate job because codecov action can fail https://github.com/codecov/codecov-action/issues/598 (and we want to know about it), but we don't want to fail required mac build needs: test-mac-required From 00025eb7ec97fd732818d6294a6c73c09cb88206 Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Fri, 11 Aug 2023 10:12:33 -0700 Subject: [PATCH 09/22] SNOW-748680: Update "aws-sdk" to 2.14 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 436c8a45e..1907c9569 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,7 @@ "asn1.js-rfc2560": "^5.0.0", "asn1.js-rfc5280": "^3.0.0", "async": "^3.2.3", - "aws-sdk": "^2.878.0", + "aws-sdk": "^2.1434.0", "axios": "^0.27.2", "big-integer": "^1.6.43", "bignumber.js": "^2.4.0", From f2b9aa096156ec9e019c57c11f44cdb55daa5d24 Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Fri, 11 Aug 2023 10:46:05 -0700 Subject: [PATCH 10/22] SNOW-748680: Revert update "aws-sdk" to 2.14 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 1907c9569..436c8a45e 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,7 @@ "asn1.js-rfc2560": "^5.0.0", "asn1.js-rfc5280": "^3.0.0", "async": "^3.2.3", - "aws-sdk": "^2.1434.0", + "aws-sdk": "^2.878.0", "axios": "^0.27.2", "big-integer": "^1.6.43", "bignumber.js": "^2.4.0", From 32444caad166fe1b00f84baf2680ec89399ced5a Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Mon, 21 Aug 2023 12:32:34 -0700 Subject: [PATCH 11/22] SNOW-748680: Add FIPS image to the node matrix for Linux builds --- .github/workflows/build-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index eeb8542a2..08a074571 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -114,7 +114,7 @@ jobs: strategy: fail-fast: false matrix: - image: [ 'nodejs-centos7-node14'] + image: [ 'nodejs-centos7-node14', 'nodejs-centos7-fips'] cloud: [ 'AWS', 'AZURE', 'GCP' ] steps: - uses: actions/checkout@v1 From 6fd4e26d43de7aca2be9f1cd5132c8fca18d3e0f Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Mon, 21 Aug 2023 13:36:03 -0700 Subject: [PATCH 12/22] SNOW-748680: Run "npm audit fix" for FIPS build --- .github/workflows/build-test.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 08a074571..cb4b70528 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -128,4 +128,6 @@ jobs: PARAMETERS_SECRET: ${{ secrets.PARAMETERS_SECRET }} CLOUD_PROVIDER: ${{ matrix.cloud }} TARGET_DOCKER_TEST_IMAGE: ${{ matrix.image }} - run: ./ci/test.sh + run: | + npm audit fix --force + ./ci/test.sh From 7dbf1b1467de57144a8fc8237338af82a1d175b9 Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Mon, 21 Aug 2023 13:44:20 -0700 Subject: [PATCH 13/22] SNOW-748680: Revert running "npm audit fix" for FIPS build --- .github/workflows/build-test.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index cb4b70528..08a074571 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -128,6 +128,4 @@ jobs: PARAMETERS_SECRET: ${{ secrets.PARAMETERS_SECRET }} CLOUD_PROVIDER: ${{ matrix.cloud }} TARGET_DOCKER_TEST_IMAGE: ${{ matrix.image }} - run: | - npm audit fix --force - ./ci/test.sh + run: ./ci/test.sh From 03c8ec1e1ba7857a8f92fb705757a8a97533c876 Mon Sep 17 00:00:00 2001 From: Dominik Przybysz Date: Fri, 4 Aug 2023 14:35:11 +0200 Subject: [PATCH 14/22] SNOW-749232: Upgrade to aws-sdk v3 --- .../file_transfer_agent.js | 1 - lib/file_transfer_agent/s3_util.js | 305 +++++++----------- package.json | 3 +- 3 files changed, 126 insertions(+), 183 deletions(-) diff --git a/lib/file_transfer_agent/file_transfer_agent.js b/lib/file_transfer_agent/file_transfer_agent.js index a265b8f6d..25854f011 100644 --- a/lib/file_transfer_agent/file_transfer_agent.js +++ b/lib/file_transfer_agent/file_transfer_agent.js @@ -574,7 +574,6 @@ function file_transfer_agent(context) var s3location = SnowflakeS3Util.extractBucketNameAndPath(stageInfo['location']); await client.getBucketAccelerateConfiguration({ Bucket: s3location.bucketName }) - .promise() .then(function (data) { useAccelerateEndpoint = data['Status'] == 'Enabled'; diff --git a/lib/file_transfer_agent/s3_util.js b/lib/file_transfer_agent/s3_util.js index 853d6370b..e41e292b9 100644 --- a/lib/file_transfer_agent/s3_util.js +++ b/lib/file_transfer_agent/s3_util.js @@ -4,16 +4,15 @@ const EncryptionMetadata = require('./encrypt_util').EncryptionMetadata; const FileHeader = require('./file_util').FileHeader; - const expandTilde = require('expand-tilde'); -const AMZ_IV = "x-amz-iv"; -const AMZ_KEY = "x-amz-key"; -const AMZ_MATDESC = "x-amz-matdesc"; +const AMZ_IV = 'x-amz-iv'; +const AMZ_KEY = 'x-amz-key'; +const AMZ_MATDESC = 'x-amz-matdesc'; const SFC_DIGEST = 'sfc-digest'; const EXPIRED_TOKEN = 'ExpiredToken'; -const NO_SUCH_KEY= 'NoSuchKey'; +const NO_SUCH_KEY = 'NoSuchKey'; const ERRORNO_WSAECONNABORTED = 10053; // network connection was aborted @@ -22,147 +21,125 @@ const resultStatus = require('./file_util').resultStatus; const HTTP_HEADER_VALUE_OCTET_STREAM = 'application/octet-stream'; // S3 Location: S3 bucket name + path -function S3Location(bucketName, s3path) -{ +function S3Location (bucketName, s3path) { return { - "bucketName": bucketName, // S3 bucket name - "s3path": s3path // S3 path name - } + 'bucketName': bucketName, // S3 bucket name + 's3path': s3path // S3 path name + }; } /** * Creates an S3 utility object. - * + * * @param {module} s3 * @param {module} filestream - * + * * @returns {Object} * @constructor */ -function s3_util(s3, filestream) -{ - const AWS = typeof s3 !== "undefined" ? s3 : require('aws-sdk'); - const fs = typeof filestream !== "undefined" ? filestream : require('fs'); +function s3_util (s3, filestream) { + const AWS = typeof s3 !== 'undefined' ? s3 : require('@aws-sdk/client-s3'); + const fs = typeof filestream !== 'undefined' ? filestream : require('fs'); // magic number, given from error message. this.DATA_SIZE_THRESHOLD = 67108864; /** - * Create an AWS S3 client using an AWS token. - * - * @param {Object} stageInfo - * - * @returns {String} - */ - this.createClient = function (stageInfo, useAccelerateEndpoint) - { - var stageCredentials = stageInfo['creds']; - var securityToken = stageCredentials['AWS_TOKEN']; + * Create an AWS S3 client using an AWS token. + * + * @param {Object} stageInfo + * + * @returns {AWS.S3} + */ + this.createClient = function (stageInfo, useAccelerateEndpoint) { + const stageCredentials = stageInfo['creds']; + const securityToken = stageCredentials['AWS_TOKEN']; // if GS sends us an endpoint, it's likely for FIPS. Use it. - var endPoint = null; - if (stageInfo['endPoint']) - { + let endPoint = null; + if (stageInfo['endPoint']) { endPoint = 'https://' + stageInfo['endPoint']; } - var config = { + const config = { apiVersion: '2006-03-01', - signatureVersion: 'v4', region: stageInfo['region'], - accessKeyId: stageCredentials['AWS_KEY_ID'], - secretAccessKey: stageCredentials['AWS_SECRET_KEY'], - sessionToken: securityToken, + credentials: { + accessKeyId: stageCredentials['AWS_KEY_ID'], + secretAccessKey: stageCredentials['AWS_SECRET_KEY'], + sessionToken: securityToken, + }, endpoint: endPoint, - useAccelerateEndpoint: useAccelerateEndpoint + useAccelerateEndpoint: useAccelerateEndpoint, }; - var s3 = new AWS.S3(config); - - return s3; - } + return new AWS.S3(config); + }; /** - * Extract the bucket name and path from the metadata's stage location. - * - * @param {String} stageLocation - * - * @returns {Object} - */ - this.extractBucketNameAndPath = function (stageLocation) - { + * Extract the bucket name and path from the metadata's stage location. + * + * @param {String} stageLocation + * + * @returns {Object} + */ + this.extractBucketNameAndPath = function (stageLocation) { // expand '~' and '~user' expressions - if (process.platform !== "win32") - { + if (process.platform !== 'win32') { stageLocation = expandTilde(stageLocation); } - var bucketName = stageLocation; - var s3path; + let bucketName = stageLocation; + let s3path; // split stage location as bucket name and path - if (stageLocation.includes('/')) - { + if (stageLocation.includes('/')) { bucketName = stageLocation.substring(0, stageLocation.indexOf('/')); s3path = stageLocation.substring(stageLocation.indexOf('/') + 1, stageLocation.length); - if (s3path && !s3path.endsWith('/')) - { + if (s3path && !s3path.endsWith('/')) { s3path += '/'; } } return S3Location(bucketName, s3path); - } + }; /** - * Create file header based on file being uploaded or not. - * - * @param {Object} meta - * @param {String} filename - * - * @returns {Object} - */ - this.getFileHeader = async function (meta, filename) - { - var stageInfo = meta['stageInfo']; - var client = this.createClient(stageInfo); - var s3location = this.extractBucketNameAndPath(stageInfo['location']); - - var params = { + * Get file header based on file being uploaded or not. + * + * @param {Object} meta + * @param {String} filename + * + * @returns {Object} + */ + this.getFileHeader = async function (meta, filename) { + const stageInfo = meta['stageInfo']; + const client = this.createClient(stageInfo); + const s3location = this.extractBucketNameAndPath(stageInfo['location']); + + const params = { Bucket: s3location.bucketName, Key: s3location.s3path + filename }; - var akey; + let akey; - try - { + try { await client.getObject(params) - .promise() - .then(function (data) - { + .then(function (data) { akey = data; - }) - } - catch (err) - { - if (err['code'] == EXPIRED_TOKEN) - { + }); + } catch (err) { + if (err['Code'] == EXPIRED_TOKEN) { meta['resultStatus'] = resultStatus.RENEW_TOKEN; return null; - } - else if (err['code'] == NO_SUCH_KEY) - { + } else if (err['Code'] == NO_SUCH_KEY) { meta['resultStatus'] = resultStatus.NOT_FOUND_FILE; return FileHeader(null, null, null); - } - else if (err['code'] == '400') - { + } else if (err['Code'] == '400') { meta['resultStatus'] = resultStatus.RENEW_TOKEN; return null; - } - else - { + } else { meta['resultStatus'] = resultStatus.ERROR; return null; } @@ -170,9 +147,8 @@ function s3_util(s3, filestream) meta['resultStatus'] = resultStatus.UPLOADED; - var encryptionMetadata; - if (akey && akey.Metadata[AMZ_KEY]) - { + let encryptionMetadata; + if (akey && akey.Metadata[AMZ_KEY]) { encryptionMetadata = EncryptionMetadata( akey.Metadata[AMZ_KEY], akey.Metadata[AMZ_IV], @@ -185,54 +161,45 @@ function s3_util(s3, filestream) akey.ContentLength, encryptionMetadata ); - } - - /** - * Create the file metadata then upload the file. - * - * @param {String} dataFile - * @param {Object} meta - * @param {Object} encryptionMetadata - * @param {Number} maxConcurrency - * - * @returns {null} - */ - this.uploadFile = async function (dataFile, meta, encryptionMetadata, maxConcurrency) - { - var fileStream = fs.readFileSync(dataFile); - await this.uploadFileStream(fileStream, meta, encryptionMetadata, maxConcurrency); - } - + }; + /** - * Create the file metadata then upload the file stream. - * - * @param {String} fileStream - * @param {Object} meta - * @param {Object} encryptionMetadata - * @param {Number} maxConcurrency - * - * @returns {null} - */ - this.uploadFileStream = async function (fileStream, meta, encryptionMetadata, maxConcurrency) - { - var s3Metadata = { + * Create the file metadata then upload the file. + * + * @param {String} dataFile + * @param {Object} meta + * @param {Object} encryptionMetadata + */ + this.uploadFile = async function (dataFile, meta, encryptionMetadata) { + const fileStream = fs.readFileSync(dataFile); + await this.uploadFileStream(fileStream, meta, encryptionMetadata); + }; + + /** + * Create the file metadata then upload the file stream. + * + * @param {String} fileStream + * @param {Object} meta + * @param {Object} encryptionMetadata + */ + this.uploadFileStream = async function (fileStream, meta, encryptionMetadata) { + const s3Metadata = { HTTP_HEADER_CONTENT_TYPE: HTTP_HEADER_VALUE_OCTET_STREAM, SFC_DIGEST: meta['SHA256_DIGEST'] }; - if (encryptionMetadata) - { + if (encryptionMetadata) { s3Metadata[AMZ_IV] = encryptionMetadata.iv; s3Metadata[AMZ_KEY] = encryptionMetadata.key; s3Metadata[AMZ_MATDESC] = encryptionMetadata.matDesc; } - var stageInfo = meta['stageInfo']; - var client = this.createClient(stageInfo); + const stageInfo = meta['stageInfo']; + const client = this.createClient(stageInfo); - var s3location = this.extractBucketNameAndPath(meta['stageInfo']['location']); + const s3location = this.extractBucketNameAndPath(meta['stageInfo']['location']); - var params = { + const params = { Bucket: s3location.bucketName, Body: fileStream, Key: s3location.s3path + meta['dstFileName'], @@ -240,26 +207,16 @@ function s3_util(s3, filestream) }; // call S3 to upload file to specified bucket - try - { - await client.upload(params) - .promise(); - } - catch (err) - { - if (err['code'] == EXPIRED_TOKEN) - { + try { + await client.putObject(params); + } catch (err) { + if (err['Code'] == EXPIRED_TOKEN) { meta['resultStatus'] = resultStatus.RENEW_TOKEN; - } - else - { + } else { meta['lastError'] = err; - if (err['code'] == ERRORNO_WSAECONNABORTED) - { + if (err['Code'] == ERRORNO_WSAECONNABORTED) { meta['resultStatus'] = resultStatus.NEED_RETRY_WITH_LOWER_CONCURRENCY; - } - else - { + } else { meta['resultStatus'] = resultStatus.NEED_RETRY; } } @@ -268,7 +225,7 @@ function s3_util(s3, filestream) meta['dstFileSize'] = meta['uploadSize']; meta['resultStatus'] = resultStatus.UPLOADED; - } + }; /** * Download the file. @@ -276,61 +233,47 @@ function s3_util(s3, filestream) * @param {String} dataFile * @param {Object} meta * @param {Object} encryptionMetadata - * @param {Number} maxConcurrency - * - * @returns {null} */ - this.nativeDownloadFile = async function (meta, fullDstPath, maxConcurrency) - { - var stageInfo = meta['stageInfo']; - var client = this.createClient(stageInfo); + this.nativeDownloadFile = async function (meta, fullDstPath) { + const stageInfo = meta['stageInfo']; + const client = this.createClient(stageInfo); - var s3location = this.extractBucketNameAndPath(meta['stageInfo']['location']); + const s3location = this.extractBucketNameAndPath(meta['stageInfo']['location']); - var params = { + const params = { Bucket: s3location.bucketName, Key: s3location.s3path + meta['dstFileName'], }; // call S3 to download file to specified bucket - try - { + try { await client.getObject(params) - .promise() - .then((data) => - { - return new Promise((resolve, reject) => - { - fs.writeFile(fullDstPath, data.Body, 'binary', (err) => - { - if (err) reject(err); + .then(data => data.Body.transformToByteArray()) + .then((data) => { + return new Promise((resolve, reject) => { + fs.writeFile(fullDstPath, data, 'binary', (err) => { + if (err) { + reject(err); + } resolve(); }); }); }); - } - catch (err) - { - if (err['code'] == EXPIRED_TOKEN) - { + } catch (err) { + if (err['Code'] == EXPIRED_TOKEN) { meta['resultStatus'] = resultStatus.RENEW_TOKEN; - } - else - { + } else { meta['lastError'] = err; - if (err['code'] == ERRORNO_WSAECONNABORTED) - { + if (err['Code'] == ERRORNO_WSAECONNABORTED) { meta['resultStatus'] = resultStatus.NEED_RETRY_WITH_LOWER_CONCURRENCY; - } - else - { + } else { meta['resultStatus'] = resultStatus.NEED_RETRY; } } return; } meta['resultStatus'] = resultStatus.DOWNLOADED; - } + }; } module.exports = s3_util; diff --git a/package.json b/package.json index 3a798fbfb..ca97385cc 100644 --- a/package.json +++ b/package.json @@ -3,13 +3,13 @@ "version": "1.8.0", "description": "Node.js driver for Snowflake", "dependencies": { + "@aws-sdk/client-s3": "^3.388.0", "@azure/storage-blob": "^12.11.0", "@google-cloud/storage": "^6.9.3", "@techteamer/ocsp": "1.0.0", "agent-base": "^6.0.2", "asn1.js-rfc2560": "^5.0.0", "asn1.js-rfc5280": "^3.0.0", - "aws-sdk": "^2.878.0", "axios": "^1.5.0", "big-integer": "^1.6.43", "bignumber.js": "^2.4.0", @@ -37,6 +37,7 @@ "winston": "^3.1.0" }, "devDependencies": { + "@aws-sdk/types": "^3.387.0", "async": "^3.2.3", "eslint": "^8.41.0", "mocha": "^10.1.0", From c32e6779cecab9eba2e8b03ae0fd21016a6ff99b Mon Sep 17 00:00:00 2001 From: Dominik Przybysz Date: Fri, 11 Aug 2023 15:05:43 +0200 Subject: [PATCH 15/22] SNOW-749232: Fix s3 unit tests --- test/unit/file_transfer_agent/s3_test.js | 368 +++++++++-------------- 1 file changed, 147 insertions(+), 221 deletions(-) diff --git a/test/unit/file_transfer_agent/s3_test.js b/test/unit/file_transfer_agent/s3_test.js index 2031f85fd..5b0110b9e 100644 --- a/test/unit/file_transfer_agent/s3_test.js +++ b/test/unit/file_transfer_agent/s3_test.js @@ -2,27 +2,26 @@ * Copyright (c) 2021 Snowflake Computing Inc. All rights reserved. */ -var assert = require('assert'); -var mock = require('mock-require'); -var SnowflakeS3Util = require('./../../../lib/file_transfer_agent/s3_util'); -var resultStatus = require('./../../../lib/file_transfer_agent/file_util').resultStatus; - -describe('S3 client', function () -{ - var mockDataFile = 'mockDataFile'; - var mockLocation = 'mockLocation'; - var mockTable = 'mockTable'; - var mockPath = 'mockPath'; - var mockDigest = 'mockDigest'; - var mockKey = 'mockKey'; - var mockIv = 'mockIv'; - var mockMatDesc = 'mockMatDesc'; - - var AWS; - var s3; - var filesystem; - var dataFile = mockDataFile; - var meta = { +const assert = require('assert'); +const mock = require('mock-require'); +const SnowflakeS3Util = require('./../../../lib/file_transfer_agent/s3_util'); +const resultStatus = require('./../../../lib/file_transfer_agent/file_util').resultStatus; + +describe('S3 client', function () { + const mockDataFile = 'mockDataFile'; + const mockLocation = 'mockLocation'; + const mockTable = 'mockTable'; + const mockPath = 'mockPath'; + const mockDigest = 'mockDigest'; + const mockKey = 'mockKey'; + const mockIv = 'mockIv'; + const mockMatDesc = 'mockMatDesc'; + + let AWS; + let s3; + let filesystem; + const dataFile = mockDataFile; + const meta = { stageInfo: { location: mockLocation, path: mockTable + '/' + mockPath + '/', @@ -30,54 +29,43 @@ describe('S3 client', function () }, SHA256_DIGEST: mockDigest, }; - var encryptionMetadata = { + const encryptionMetadata = { key: mockKey, iv: mockIv, matDesc: mockMatDesc }; - before(function () - { + before(function () { mock('s3', { - S3: function (params) - { - function S3() - { - this.getObject = function (params) - { - function getObject() - { - this.promise = function () - { - function promise() - { - this.then = function (callback) - { - callback({ - Metadata: '' - }); - } - } - return new promise; - } + S3: function (params) { + function S3 () { + this.getObject = function (params) { + function getObject () { + this.then = function (callback) { + callback({ + Metadata: '' + }); + }; } + return new getObject; - } - this.upload = function (params) - { - function upload() - { - this.promise = function () { }; + }; + this.putObject = function (params) { + function putObject() { + this.then = function (callback) { + callback(); + }; } - return new upload; - } + + return new putObject; + }; } + return new S3; } }); mock('filesystem', { - readFileSync: async function (data) - { + readFileSync: async function (data) { return data; } }); @@ -87,8 +75,7 @@ describe('S3 client', function () AWS = new SnowflakeS3Util(s3, filesystem); }); - it('extract bucket name and path', async function () - { + it('extract bucket name and path', async function () { var result = AWS.extractBucketNameAndPath('sfc-eng-regression/test_sub_dir/'); assert.strictEqual(result.bucketName, 'sfc-eng-regression'); assert.strictEqual(result.s3path, 'test_sub_dir/'); @@ -110,279 +97,218 @@ describe('S3 client', function () assert.strictEqual(result.s3path, '//'); }); - it('get file header - success', async function () - { + it('get file header - success', async function () { await AWS.getFileHeader(meta, dataFile); assert.strictEqual(meta['resultStatus'], resultStatus.UPLOADED); }); - it('get file header - fail expired token', async function () - { + it('get file header - fail expired token', async function () { mock('s3', { - S3: function (params) - { - function S3() - { - this.getObject = function (params) - { - function getObject() - { - this.promise = function () - { - function promise() - { - this.then = function (callback) - { - let err = new Error(); - err.code = 'ExpiredToken'; - throw err; - } - } - return new promise; - } + S3: function (params) { + function S3 () { + this.getObject = function (params) { + function getObject () { + this.then = function (callback) { + const err = new Error(); + err.Code = 'ExpiredToken'; + throw err; + }; } + return new getObject; - } + }; } + return new S3; } }); s3 = require('s3'); - var AWS = new SnowflakeS3Util(s3); + const AWS = new SnowflakeS3Util(s3); await AWS.getFileHeader(meta, dataFile); assert.strictEqual(meta['resultStatus'], resultStatus.RENEW_TOKEN); }); - it('get file header - fail no such key', async function () - { + it('get file header - fail no such key', async function () { mock('s3', { - S3: function (params) - { - function S3() - { - this.getObject = function (params) - { - function getObject() - { - this.promise = function () - { - function promise() - { - this.then = function (callback) - { - let err = new Error(); - err.code = 'NoSuchKey'; - throw err; - } - } - return new promise; - } + S3: function (params) { + function S3 () { + this.getObject = function (params) { + function getObject () { + this.then = function (callback) { + const err = new Error(); + err.Code = 'NoSuchKey'; + throw err; + }; } + return new getObject; - } + }; } + return new S3; } }); s3 = require('s3'); - var AWS = new SnowflakeS3Util(s3); + const AWS = new SnowflakeS3Util(s3); await AWS.getFileHeader(meta, dataFile); assert.strictEqual(meta['resultStatus'], resultStatus.NOT_FOUND_FILE); }); - it('get file header - fail HTTP 400', async function () - { + it('get file header - fail HTTP 400', async function () { mock('s3', { - S3: function (params) - { - function S3() - { - this.getObject = function (params) - { - function getObject() - { - this.promise = function () - { - function promise() - { - this.then = function (callback) - { - let err = new Error(); - err.code = '400'; - throw err; - } - } - return new promise; - } + S3: function (params) { + function S3 () { + this.getObject = function (params) { + function getObject () { + this.then = function (callback) { + const err = new Error(); + err.Code = '400'; + throw err; + }; } + return new getObject; - } + }; } + return new S3; } }); s3 = require('s3'); - var AWS = new SnowflakeS3Util(s3); + const AWS = new SnowflakeS3Util(s3); await AWS.getFileHeader(meta, dataFile); assert.strictEqual(meta['resultStatus'], resultStatus.RENEW_TOKEN); }); - it('get file header - fail unknown', async function () - { + it('get file header - fail unknown', async function () { mock('s3', { - S3: function (params) - { - function S3() - { - this.getObject = function (params) - { - function getObject() - { - this.promise = function () - { - function promise() - { - this.then = function (callback) - { - let err = new Error(); - err.code = 'unknown'; - throw err; - } - } - return new promise; - } + S3: function (params) { + function S3 () { + this.getObject = function (params) { + function getObject () { + this.then = function (callback) { + const err = new Error(); + err.Code = 'unknown'; + throw err; + }; } + return new getObject; - } + }; } + return new S3; } }); s3 = require('s3'); - var AWS = new SnowflakeS3Util(s3); + const AWS = new SnowflakeS3Util(s3); await AWS.getFileHeader(meta, dataFile); assert.strictEqual(meta['resultStatus'], resultStatus.ERROR); }); - it('upload - success', async function () - { + it('upload - success', async function () { await AWS.uploadFile(dataFile, meta, encryptionMetadata); assert.strictEqual(meta['resultStatus'], resultStatus.UPLOADED); }); - it('upload - fail expired token', async function () - { + it('upload - fail expired token', async function () { mock('s3', { - S3: function (params) - { - function S3() - { - this.upload = function (params) - { - function upload() - { - this.promise = function () - { - let err = new Error(); - err.code = 'ExpiredToken'; + S3: function (params) { + function S3 () { + this.putObject = function (params) { + function putObject () { + this.then = function () { + const err = new Error(); + err.Code = 'ExpiredToken'; throw err; - } + }; } - return new upload; - } + + return new putObject; + }; } + return new S3; } }); mock('filesystem', { - readFileSync: async function (data) - { + readFileSync: async function (data) { return data; } }); s3 = require('s3'); filesystem = require('filesystem'); - var AWS = new SnowflakeS3Util(s3, filesystem); + const AWS = new SnowflakeS3Util(s3, filesystem); await AWS.uploadFile(dataFile, meta, encryptionMetadata); assert.strictEqual(meta['resultStatus'], resultStatus.RENEW_TOKEN); }); - it('upload - fail wsaeconnaborted', async function () - { + it('upload - fail wsaeconnaborted', async function () { mock('s3', { - S3: function (params) - { - function S3() - { - this.upload = function (params) - { - function upload() - { - this.promise = function () - { - let err = new Error(); - err.code = '10053'; + S3: function (params) { + function S3 () { + this.putObject = function (params) { + function putObject () { + this.then = function () { + const err = new Error(); + err.Code = '10053'; throw err; - } + }; } - return new upload; - } + + return new putObject; + }; } + return new S3; } }); mock('filesystem', { - readFileSync: async function (data) - { + readFileSync: async function (data) { return data; } }); s3 = require('s3'); filesystem = require('filesystem'); - var AWS = new SnowflakeS3Util(s3, filesystem); + const AWS = new SnowflakeS3Util(s3, filesystem); await AWS.uploadFile(dataFile, meta, encryptionMetadata); assert.strictEqual(meta['resultStatus'], resultStatus.NEED_RETRY_WITH_LOWER_CONCURRENCY); }); - it('upload - fail HTTP 400', async function () - { + it('upload - fail HTTP 400', async function () { mock('s3', { - S3: function (params) - { - function S3() - { - this.upload = function (params) - { - function upload() - { - this.promise = function () - { - let err = new Error(); - err.code = '400'; + S3: function (params) { + function S3 () { + this.putObject = function (params) { + function putObject () { + this.then = () => { + const err = new Error(); + err.Code = '400'; throw err; - } + }; } - return new upload; - } + + return new putObject; + }; } + return new S3; } }); mock('filesystem', { - readFileSync: async function (data) - { + readFileSync: async function (data) { return data; } }); s3 = require('s3'); filesystem = require('filesystem'); - var AWS = new SnowflakeS3Util(s3, filesystem); + const AWS = new SnowflakeS3Util(s3, filesystem); await AWS.uploadFile(dataFile, meta, encryptionMetadata); assert.strictEqual(meta['resultStatus'], resultStatus.NEED_RETRY); From 73e03b724890195762c7ff012ca2808048cb1804 Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Tue, 23 May 2023 10:24:28 -0700 Subject: [PATCH 16/22] Set centos7 node version to 14.17 --- ci/image/Dockerfile.nodejs-centos7-fips-test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/image/Dockerfile.nodejs-centos7-fips-test b/ci/image/Dockerfile.nodejs-centos7-fips-test index 36c9f402c..f0d030959 100644 --- a/ci/image/Dockerfile.nodejs-centos7-fips-test +++ b/ci/image/Dockerfile.nodejs-centos7-fips-test @@ -35,7 +35,7 @@ SHELL [ "/usr/bin/scl", "enable", "devtoolset-8"] # node-fips environment variables ENV NODE_HOME $HOME/node -ENV NODEJS_VERSION 14.0.0 +ENV NODEJS_VERSION 14.17.0 ENV FIPSDIR $HOME/install-openssl-fips ENV OPENSSL_VERSION 2.0.16 From d286ad27be8f563321786ee977d2f377f4926e0e Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Fri, 23 Jun 2023 09:07:13 -0700 Subject: [PATCH 17/22] issue 207 - Update docker image to node18/openssl3 and enable fips --- ci/image/Dockerfile.nodejs-centos7-fips-test | 38 +++++++++++++++----- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/ci/image/Dockerfile.nodejs-centos7-fips-test b/ci/image/Dockerfile.nodejs-centos7-fips-test index f0d030959..81efb31ad 100644 --- a/ci/image/Dockerfile.nodejs-centos7-fips-test +++ b/ci/image/Dockerfile.nodejs-centos7-fips-test @@ -35,29 +35,49 @@ SHELL [ "/usr/bin/scl", "enable", "devtoolset-8"] # node-fips environment variables ENV NODE_HOME $HOME/node -ENV NODEJS_VERSION 14.17.0 -ENV FIPSDIR $HOME/install-openssl-fips -ENV OPENSSL_VERSION 2.0.16 +ENV NODEJS_VERSION 18.0.0 +ENV OPENSSL_VERSION 3.0.8 +ENV PKG_CONFIG_PATH "/usr/local/lib64/pkgconfig" +ENV LD_LIBRARY_PATH "${LD_LIBRARY_PATH}:/usr/local/lib64" +ENV OPENSSL_CONF /usr/local/ssl/openssl.cnf +ENV FIPSCONF /usr/local/ssl/fipsmodule.cnf +ENV OPENSSL_MODULES=/usr/local/lib64/ossl-modules # Install OpenSSL RUN cd $HOME -RUN curl https://www.openssl.org/source/openssl-fips-$OPENSSL_VERSION.tar.gz -o $HOME/openssl-fips-$OPENSSL_VERSION.tar.gz +RUN curl https://www.openssl.org/source/openssl-$OPENSSL_VERSION.tar.gz -o $HOME/openssl-fips-$OPENSSL_VERSION.tar.gz RUN tar -xvf $HOME/openssl-fips-$OPENSSL_VERSION.tar.gz -RUN mv openssl-fips-$OPENSSL_VERSION $HOME/openssl-fips +RUN mv openssl-$OPENSSL_VERSION $HOME/openssl-fips RUN cd $HOME/openssl-fips - + +# Install OpenSSL dependencies +RUN yum -y install perl-IPC-Cmd +RUN yum -y install perl-Digest-SHA +RUN yum -y install openssl-devel + # You must run ONLY these commands when building the FIPS version of OpenSSL -RUN cd $HOME/openssl-fips && ./config && make && make install - +RUN cd $HOME/openssl-fips && ./config enable-fips && make && make install + +# Enable FIPS by editing the openssl.cnf file +RUN sed -i "s/# .include fipsmodule.cnf/.include ${FIPSCONF//\//\\/}/g" $OPENSSL_CONF +RUN sed -i 's/# fips = fips_sect/fips = fips_sect/g' $OPENSSL_CONF +RUN sed -i 's/# activate = 1/activate = 1/g' $OPENSSL_CONF +RUN sed -i '55ialg_section = algorithm_sect' $OPENSSL_CONF +RUN sed -i '75idefault_properties = fips=yes' $OPENSSL_CONF +RUN sed -i '75i[algorithm_sect]' $OPENSSL_CONF + # Download and build NodeJS RUN git clone --branch v$NODEJS_VERSION https://github.com/nodejs/node.git $NODE_HOME RUN gcc --version RUN g++ --version -RUN cd $NODE_HOME && ./configure --openssl-fips=$FIPSDIR && make -j2 &> /dev/null && make install +RUN cd $NODE_HOME && ./configure --shared-openssl --shared-openssl-libpath=/usr/local/lib64 --shared-openssl-includes=/usr/local/include/openssl --openssl-is-fips +RUN make -j2 &> /dev/null && make install # Should be $NODEJS_VERSION RUN node --version # Should be $OPENSSL_VERSION RUN node -p "process.versions.openssl" +# Should be 1 (fips is enabled by default) +RUN node --enable-fips -p 'crypto.getFips()' # workspace RUN mkdir -p /home/user From 7f557b10ef6ed00a1e8eddb3e311ae1670012962 Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Fri, 21 Jul 2023 14:58:25 -0700 Subject: [PATCH 18/22] issue 207 - Fix spaces and remove "--enable-fips" flag --- ci/image/Dockerfile.nodejs-centos7-fips-test | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/ci/image/Dockerfile.nodejs-centos7-fips-test b/ci/image/Dockerfile.nodejs-centos7-fips-test index 81efb31ad..a551e1d33 100644 --- a/ci/image/Dockerfile.nodejs-centos7-fips-test +++ b/ci/image/Dockerfile.nodejs-centos7-fips-test @@ -43,7 +43,7 @@ ENV OPENSSL_CONF /usr/local/ssl/openssl.cnf ENV FIPSCONF /usr/local/ssl/fipsmodule.cnf ENV OPENSSL_MODULES=/usr/local/lib64/ossl-modules -# Install OpenSSL +# Install OpenSSL RUN cd $HOME RUN curl https://www.openssl.org/source/openssl-$OPENSSL_VERSION.tar.gz -o $HOME/openssl-fips-$OPENSSL_VERSION.tar.gz RUN tar -xvf $HOME/openssl-fips-$OPENSSL_VERSION.tar.gz @@ -70,14 +70,13 @@ RUN sed -i '75i[algorithm_sect]' $OPENSSL_CONF RUN git clone --branch v$NODEJS_VERSION https://github.com/nodejs/node.git $NODE_HOME RUN gcc --version RUN g++ --version -RUN cd $NODE_HOME && ./configure --shared-openssl --shared-openssl-libpath=/usr/local/lib64 --shared-openssl-includes=/usr/local/include/openssl --openssl-is-fips -RUN make -j2 &> /dev/null && make install +RUN cd $NODE_HOME && ./configure --shared-openssl --shared-openssl-libpath=/usr/local/lib64 --shared-openssl-includes=/usr/local/include/openssl --openssl-is-fips && make -j2 &> /dev/null && make install # Should be $NODEJS_VERSION RUN node --version # Should be $OPENSSL_VERSION RUN node -p "process.versions.openssl" -# Should be 1 (fips is enabled by default) -RUN node --enable-fips -p 'crypto.getFips()' +# Should be 1 (FIPS is enabled by default) +RUN node -p 'crypto.getFips()' # workspace RUN mkdir -p /home/user From aff9f8ff5dadf9e09d3b2322e8ab6c3b5743d1d7 Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Thu, 3 Aug 2023 14:35:46 -0700 Subject: [PATCH 19/22] issue 217 - Use node 18.17.0 (LTS) for the FIPS docker image --- ci/image/Dockerfile.nodejs-centos7-fips-test | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/image/Dockerfile.nodejs-centos7-fips-test b/ci/image/Dockerfile.nodejs-centos7-fips-test index a551e1d33..dce338c21 100644 --- a/ci/image/Dockerfile.nodejs-centos7-fips-test +++ b/ci/image/Dockerfile.nodejs-centos7-fips-test @@ -35,7 +35,7 @@ SHELL [ "/usr/bin/scl", "enable", "devtoolset-8"] # node-fips environment variables ENV NODE_HOME $HOME/node -ENV NODEJS_VERSION 18.0.0 +ENV NODEJS_VERSION 18.17.0 ENV OPENSSL_VERSION 3.0.8 ENV PKG_CONFIG_PATH "/usr/local/lib64/pkgconfig" ENV LD_LIBRARY_PATH "${LD_LIBRARY_PATH}:/usr/local/lib64" @@ -59,6 +59,7 @@ RUN yum -y install openssl-devel RUN cd $HOME/openssl-fips && ./config enable-fips && make && make install # Enable FIPS by editing the openssl.cnf file +RUN sed -i "s/openssl_conf = openssl_init/nodejs_conf = openssl_init/g" $OPENSSL_CONF RUN sed -i "s/# .include fipsmodule.cnf/.include ${FIPSCONF//\//\\/}/g" $OPENSSL_CONF RUN sed -i 's/# fips = fips_sect/fips = fips_sect/g' $OPENSSL_CONF RUN sed -i 's/# activate = 1/activate = 1/g' $OPENSSL_CONF From 4b59571cbea94f7cb81daac9a005ab45186a6dc1 Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Thu, 10 Aug 2023 09:46:15 -0700 Subject: [PATCH 20/22] SNOW-748680: Add FIPS image to the node matrix for Linux builds --- .github/workflows/build-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index eeb8542a2..08a074571 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -114,7 +114,7 @@ jobs: strategy: fail-fast: false matrix: - image: [ 'nodejs-centos7-node14'] + image: [ 'nodejs-centos7-node14', 'nodejs-centos7-fips'] cloud: [ 'AWS', 'AZURE', 'GCP' ] steps: - uses: actions/checkout@v1 From 5722e21f97664202622538b200cf6908df165237 Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Mon, 21 Aug 2023 13:36:03 -0700 Subject: [PATCH 21/22] SNOW-748680: Run "npm audit fix" for FIPS build --- .github/workflows/build-test.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 08a074571..cb4b70528 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -128,4 +128,6 @@ jobs: PARAMETERS_SECRET: ${{ secrets.PARAMETERS_SECRET }} CLOUD_PROVIDER: ${{ matrix.cloud }} TARGET_DOCKER_TEST_IMAGE: ${{ matrix.image }} - run: ./ci/test.sh + run: | + npm audit fix --force + ./ci/test.sh From 55f43ae85c693af2a90bb9f80e32c9a13643562d Mon Sep 17 00:00:00 2001 From: sfc-gh-ext-simba-lf Date: Mon, 21 Aug 2023 13:44:20 -0700 Subject: [PATCH 22/22] SNOW-748680: Revert running "npm audit fix" for FIPS build --- .github/workflows/build-test.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index cb4b70528..08a074571 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -128,6 +128,4 @@ jobs: PARAMETERS_SECRET: ${{ secrets.PARAMETERS_SECRET }} CLOUD_PROVIDER: ${{ matrix.cloud }} TARGET_DOCKER_TEST_IMAGE: ${{ matrix.image }} - run: | - npm audit fix --force - ./ci/test.sh + run: ./ci/test.sh