From 72b6d8fc8bed67b74d015a6a04eab483db825a0c Mon Sep 17 00:00:00 2001 From: James Fantin-Hardesty <24646452+jfantinhardesty@users.noreply.github.com> Date: Thu, 28 Sep 2023 16:18:26 -0600 Subject: [PATCH] tests: Run unit tests with localstack and azurite (#5) * use localstack and azurite on linux only --- .github/workflows/unit-test.yml | 165 +++++++++++-------- component/azstorage/azauth_test.go | 33 +--- component/azstorage/block_blob_test.go | 204 ++++++++++++++---------- component/azstorage/datalake_test.go | 4 +- component/azstorage/utils_test.go | 2 + component/file_cache/file_cache_test.go | 2 +- component/s3storage/client.go | 10 +- component/s3storage/client_test.go | 13 +- component/s3storage/config.go | 2 + component/s3storage/connection.go | 1 + component/s3storage/s3storage_test.go | 7 +- component/s3storage/s3wrappers.go | 3 + setup/baseConfig.yaml | 3 +- 13 files changed, 252 insertions(+), 197 deletions(-) diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index 55301a49f..2883bfb6e 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -13,24 +13,24 @@ on: default: true jobs: - test: - strategy: - matrix: - go: ['1.20'] - job_name: ['linux', 'windows'] - - include: - - job_name: linux - os: ubuntu-latest - containerName: 'test-cnt-ubn' - - - job_name: windows - os: windows-latest - containerName: 'test-cnt-win' - cgo: '0' - - name: Build and Test on - runs-on: ${{ matrix.os }} + linux-unit-test: + name: Build and Test on Linux + runs-on: ubuntu-latest + env: + go: '1.20' + cgo: '' + containerName: 'test-cnt-ubn' + + AWS_ENDPOINT: http://localhost:4566 + AWS_REGION: us-east-1 + AWS_ACCESS_KEY_ID: test + AWS_SECRET_ACCESS_KEY: test + AWS_BUCKET_NAME: test + # Using default test credentials for Azurite + BLOB_ENDPOINT: http://127.0.0.1:10000/devstoreaccount1 + BLOB_ACCOUNT: devstoreaccount1 + BLOB_KEY: Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw== + steps: - name: Checkout code uses: actions/checkout@v4 @@ -38,29 +38,24 @@ jobs: - name: Install Go uses: actions/setup-go@v4 with: - go-version: ${{ matrix.go }} + go-version: ${{ env.go }} check-latest: true - run: go version + - name: Install Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + - name: Set CGO shell: bash run: | - if [[ "${{ matrix.cgo }}" != "" ]]; then echo 'CGO_ENABLED=${{ matrix.cgo }}' >> $GITHUB_ENV ; fi + if [[ "${{ env.cgo }}" != "" ]]; then echo 'CGO_ENABLED=${{ env.cgo }}' >> $GITHUB_ENV ; fi - name: Install libfuse on Linux shell: bash run: | sudo apt-get install fuse3 libfuse-dev rpm pkg-config - if: matrix.os == 'ubuntu-latest' - - - name: Go module cache - uses: actions/cache@v3 - with: - path: | - ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go- - name: Build run: | @@ -72,26 +67,78 @@ jobs: echo $cnfFile touch $cnfFile echo "{" > $cnfFile - echo "\"block-acct\"": "\"${{ secrets.AZTEST_BLOCK_ACC_NAME }}\"", >> $cnfFile - echo "\"adls-acct\"": "\"${{ secrets.AZTEST_ADLS_ACC_NAME }}\"", >> $cnfFile - echo "\"block-cont\"": "\"${{ matrix.containerName }}\"", >> $cnfFile - echo "\"adls-cont\"": "\"${{ matrix.containerName }}\"", >> $cnfFile - echo "\"block-key\"": "\"${{ secrets.AZTEST_BLOCK_KEY }}\"", >> $cnfFile - echo "\"adls-key\"": "\"${{ secrets.AZTEST_ADLS_KEY }}\"", >> $cnfFile - echo "\"block-sas\"": "\"${{ secrets.AZTEST_BLOCK_SAS }}\"", >> $cnfFile - echo "\"block-cont-sas-ubn-18\"": "\"${{ secrets.AZTEST_BLOCK_CONT_SAS_UBN_18 }}\"", >> $cnfFile - echo "\"block-cont-sas-ubn-20\"": "\"${{ secrets.AZTEST_BLOCK_CONT_SAS_UBN_20 }}\"", >> $cnfFile - echo "\"adls-sas\"": "\"${{ secrets.AZTEST_ADLS_SAS }}\"", >> $cnfFile - echo "\"msi-appid\"": "\"${{ secrets.AZTEST_APP_ID }}\"", >> $cnfFile - echo "\"msi-resid\"": "\"${{ secrets.AZTEST_RES_ID }}\"", >> $cnfFile - echo "\"msi-objid\"": "\"${{ secrets.AZTEST_OBJ_ID }}\"", >> $cnfFile - echo "\"spn-client\"": "\"${{ secrets.AZTEST_CLIENT }}\"", >> $cnfFile - echo "\"spn-tenant\"": "\"${{ secrets.AZTEST_TENANT }}\"", >> $cnfFile - echo "\"spn-secret\"": "\"${{ secrets.AZTEST_SECRET }}\"", >> $cnfFile + echo "\"block-acct\"": "\"${{ env.BLOB_ACCOUNT }}\"", >> $cnfFile + echo "\"block-key\"": "\"${{ env.BLOB_KEY }}\"", >> $cnfFile + echo "\"endpoint\"": "\"${{ env.BLOB_ENDPOINT }}\"", >> $cnfFile echo "\"skip-msi\"": "true", >> $cnfFile echo "\"proxy-address\"": "\"\"" >> $cnfFile echo "}" >> $cnfFile - if: matrix.os == 'ubuntu-latest' + + - name: Create S3 Configuration File on Linux + run: |- + cnfFile=$HOME/s3test.json + echo $cnfFile + touch $cnfFile + echo "{" > $cnfFile + echo "\"bucket-name\"": "\"${{ env.AWS_BUCKET_NAME }}\"", >> $cnfFile + echo "\"access-key\"": "\"${{ env.AWS_ACCESS_KEY_ID }}\"", >> $cnfFile + echo "\"secret-key\"": "\"${{ env.AWS_SECRET_ACCESS_KEY }}\"", >> $cnfFile + echo "\"endpoint\"": "\"${{ env.AWS_ENDPOINT }}\"", >> $cnfFile + echo "\"region\"": "\"${{ env.AWS_REGION }}\"", >> $cnfFile + echo "\"use-path-style\"": true\ >> $cnfFile + echo "}" >> $cnfFile + + - name: Start LocalStack + run: | + pip install localstack awscli-local[ver1] # install LocalStack cli and awslocal + docker pull localstack/localstack # Make sure to pull the latest version of the image + localstack start -d # Start LocalStack in the background + + echo "Waiting for LocalStack startup..." # Wait 20 seconds for the LocalStack container + localstack wait -t 20 # to become ready before timing out + echo "Startup complete" + + - name: Install Azurite + run: npm install -g azurite + + - name: Start Azurite + run: azurite --silent & + + - name: Run some Tests against LocalStack + run: | + awslocal s3 mb s3://test + awslocal s3 ls + + - name: Run unit tests + run: go test -v -timeout=1h ./... --tags=unittest,azurite + + test: + name: Build and Test on Windows + runs-on: windows-latest + env: + go: '1.20' + cgo: '0' + containerName: 'test-cnt-win' + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install Go + uses: actions/setup-go@v4 + with: + go-version: ${{ env.go }} + check-latest: true + - run: go version + + - name: Set CGO + shell: bash + run: | + if [[ "${{ env.cgo }}" != "" ]]; then echo 'CGO_ENABLED=${{ env.cgo }}' >> $GITHUB_ENV ; fi + + - name: Build + run: | + ./build.sh - name: Create Azure Configuration File on Windows run: |- @@ -101,8 +148,8 @@ jobs: { "block-acct": "${{ secrets.AZTEST_BLOCK_ACC_NAME }}", "adls-acct": "${{ secrets.AZTEST_ADLS_ACC_NAME }}", - "block-cont": "${{ matrix.containerName }}", - "adls-cont": "${{ matrix.containerName }}", + "block-cont": "${{ env.containerName }}", + "adls-cont": "${{ env.containerName }}", "block-key": "${{ secrets.AZTEST_BLOCK_KEY }}", "adls-key": "${{ secrets.AZTEST_ADLS_KEY }}", "block-sas": "${{ secrets.AZTEST_BLOCK_SAS }}", @@ -121,21 +168,6 @@ jobs: "@ $content | Out-File -FilePath $cnfFile -Encoding Ascii shell: powershell - if: matrix.os == 'windows-latest' - - - name: Create S3 Configuration File on Linux - run: |- - cnfFile=$HOME/s3test.json - echo $cnfFile - touch $cnfFile - echo "{" > $cnfFile - echo "\"bucket-name\"": "\"${{ secrets.S3TEST_BUCKET_NAME }}\"", >> $cnfFile - echo "\"access-key\"": "\"${{ secrets.S3TEST_ACCESS_KEY }}\"", >> $cnfFile - echo "\"secret-key\"": "\"${{ secrets.S3TEST_SECRET_KEY }}\"", >> $cnfFile - echo "\"endpoint\"": "\"${{ secrets.S3TEST_ENDPOINT }}\"", >> $cnfFile - echo "\"region\"": "\"${{ secrets.S3TEST_REGION }}\"" >> $cnfFile - echo "}" >> $cnfFile - if: matrix.os == 'ubuntu-latest' - name: Create S3 Configuration File on Windows run: |- @@ -152,11 +184,10 @@ jobs: "@ $content | Out-File -FilePath $cnfFile -Encoding Ascii shell: powershell - if: matrix.os == 'windows-latest' - name: Run unit tests - run: go test -v -timeout=2h ./... --tags=unittest - + run: go test -v -timeout=1h ./... --tags=unittest + lint: name: Lint runs-on: ubuntu-latest diff --git a/component/azstorage/azauth_test.go b/component/azstorage/azauth_test.go index 7ea7f2cf1..155ad7ddd 100644 --- a/component/azstorage/azauth_test.go +++ b/component/azstorage/azauth_test.go @@ -1,5 +1,5 @@ -//go:build !authtest -// +build !authtest +//go:build !authtest && !azurite +// +build !authtest,!azurite /* Licensed under the MIT License . @@ -42,35 +42,6 @@ import ( "github.com/stretchr/testify/suite" ) -type storageTestConfiguration struct { - // Get the mount path from command line argument - BlockAccount string `json:"block-acct"` - AdlsAccount string `json:"adls-acct"` - BlockContainer string `json:"block-cont"` - AdlsContainer string `json:"adls-cont"` - // AdlsDirectory string `json:"adls-dir"` - BlockContainerHuge string `json:"block-cont-huge"` - AdlsContainerHuge string `json:"adls-cont-huge"` - BlockKey string `json:"block-key"` - AdlsKey string `json:"adls-key"` - BlockSas string `json:"block-sas"` - BlockContSasUbn18 string `json:"block-cont-sas-ubn-18"` - BlockContSasUbn20 string `json:"block-cont-sas-ubn-20"` - AdlsSas string `json:"adls-sas"` - // AdlsDirSasUbn18 string `json:"adls-dir-sas-ubn-18"` - // AdlsDirSasUbn20 string `json:"adls-dir-sas-ubn-20"` - MsiAppId string `json:"msi-appid"` - MsiResId string `json:"msi-resid"` - MsiObjId string `json:"msi-objid"` - SpnClientId string `json:"spn-client"` - SpnTenantId string `json:"spn-tenant"` - SpnClientSecret string `json:"spn-secret"` - SkipMsi bool `json:"skip-msi"` - ProxyAddress string `json:"proxy-address"` -} - -var storageTestConfigurationParameters storageTestConfiguration - type authTestSuite struct { suite.Suite } diff --git a/component/azstorage/block_blob_test.go b/component/azstorage/block_blob_test.go index ea7d16d98..f6ba4648d 100644 --- a/component/azstorage/block_blob_test.go +++ b/component/azstorage/block_blob_test.go @@ -58,6 +58,36 @@ import ( "github.com/stretchr/testify/suite" ) +type storageTestConfiguration struct { + // Get the mount path from command line argument + BlockAccount string `json:"block-acct"` + AdlsAccount string `json:"adls-acct"` + BlockContainer string `json:"block-cont"` + AdlsContainer string `json:"adls-cont"` + // AdlsDirectory string `json:"adls-dir"` + BlockContainerHuge string `json:"block-cont-huge"` + AdlsContainerHuge string `json:"adls-cont-huge"` + BlockKey string `json:"block-key"` + AdlsKey string `json:"adls-key"` + BlockSas string `json:"block-sas"` + BlockContSasUbn18 string `json:"block-cont-sas-ubn-18"` + BlockContSasUbn20 string `json:"block-cont-sas-ubn-20"` + AdlsSas string `json:"adls-sas"` + // AdlsDirSasUbn18 string `json:"adls-dir-sas-ubn-18"` + // AdlsDirSasUbn20 string `json:"adls-dir-sas-ubn-20"` + MsiAppId string `json:"msi-appid"` + MsiResId string `json:"msi-resid"` + MsiObjId string `json:"msi-objid"` + SpnClientId string `json:"spn-client"` + SpnTenantId string `json:"spn-tenant"` + SpnClientSecret string `json:"spn-secret"` + SkipMsi bool `json:"skip-msi"` + ProxyAddress string `json:"proxy-address"` + Endpoint string `json:"endpoint"` +} + +var storageTestConfigurationParameters storageTestConfiguration + var ctx = context.Background() const MB = 1024 * 1024 @@ -169,7 +199,9 @@ type blockBlobTestSuite struct { func newTestAzStorage(configuration string) (*AzStorage, error) { _ = config.ReadConfigFromReader(strings.NewReader(configuration)) az := NewazstorageComponent() - err := az.Configure(true) + + // Set to false to allow testing with Azurite + err := az.Configure(false) return az.(*AzStorage), err } @@ -215,8 +247,8 @@ func (s *blockBlobTestSuite) setupTestHelper(configuration string, container str } s.container = container if configuration == "" { - configuration = fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: https://%s.blob.core.windows.net/\n type: block\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true", - storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockKey, s.container) + configuration = fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: %s\n type: block\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true", + storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.Endpoint, storageTestConfigurationParameters.BlockKey, s.container) } s.config = configuration @@ -247,8 +279,8 @@ func (s *blockBlobTestSuite) cleanupTest() { func (s *blockBlobTestSuite) TestInvalidBlockSize() { defer s.cleanupTest() - configuration := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: https://%s.blob.core.windows.net/\n type: block\n block-size-mb: 5000\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true", - storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockKey, s.container) + configuration := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: %s\n type: block\n block-size-mb: 5000\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true", + storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.Endpoint, storageTestConfigurationParameters.BlockKey, s.container) _, err := newTestAzStorage(configuration) s.assert.NotNil(err) } @@ -266,7 +298,7 @@ func (s *blockBlobTestSuite) TestDefault() { s.assert.Empty(s.az.stConfig.authConfig.ClientSecret) s.assert.Empty(s.az.stConfig.authConfig.TenantID) s.assert.Empty(s.az.stConfig.authConfig.ClientID) - s.assert.EqualValues("https://"+s.az.stConfig.authConfig.AccountName+".blob.core.windows.net/", s.az.stConfig.authConfig.Endpoint) + //s.assert.EqualValues("https://"+s.az.stConfig.authConfig.AccountName+".blob.core.windows.net/", s.az.stConfig.authConfig.Endpoint) s.assert.Equal(EAuthType.KEY(), s.az.stConfig.authConfig.AuthMode) s.assert.Equal(s.container, s.az.stConfig.container) s.assert.Empty(s.az.stConfig.prefixPath) @@ -306,25 +338,25 @@ func (s *blockBlobTestSuite) TestModifyEndpoint() { defer s.cleanupTest() // Setup s.tearDownTestHelper(false) // Don't delete the generated container. - config := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: https://%s.dfs.core.windows.net/\n type: block\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true", - storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockKey, s.container) + config := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: %s\n type: block\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true", + storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.Endpoint, storageTestConfigurationParameters.BlockKey, s.container) s.setupTestHelper(config, s.container, true) err := s.az.storage.TestPipeline() s.assert.Nil(err) } -func (s *blockBlobTestSuite) TestNoEndpoint() { - defer s.cleanupTest() - // Setup - s.tearDownTestHelper(false) // Don't delete the generated container. - config := fmt.Sprintf("azstorage:\n account-name: %s\n type: block\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true", - storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockKey, s.container) - s.setupTestHelper(config, s.container, true) +// func (s *blockBlobTestSuite) TestNoEndpoint() { +// defer s.cleanupTest() +// // Setup +// s.tearDownTestHelper(false) // Don't delete the generated container. +// config := fmt.Sprintf("azstorage:\n account-name: %s\n type: block\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true", +// storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockKey, s.container) +// s.setupTestHelper(config, s.container, true) - err := s.az.storage.TestPipeline() - s.assert.Nil(err) -} +// err := s.az.storage.TestPipeline() +// s.assert.Nil(err) +// } func (s *blockBlobTestSuite) TestListContainers() { defer s.cleanupTest() @@ -734,8 +766,8 @@ func (s *blockBlobTestSuite) TestReadDirWindowsNameConvert() { if runtime.GOOS != "windows" { return } - config := fmt.Sprintf("restricted-characters-windows: true\nazstorage:\n account-name: %s\n endpoint: https://%s.dfs.core.windows.net/\n type: block\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true", - storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockKey, s.container) + config := fmt.Sprintf("restricted-characters-windows: true\nazstorage:\n account-name: %s\n endpoint: %s\n type: block\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true", + storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.Endpoint, storageTestConfigurationParameters.BlockKey, s.container) s.setupTestHelper(config, s.container, true) defer s.cleanupTest() // Setup @@ -780,8 +812,8 @@ func (s *blockBlobTestSuite) TestReadDirListBlocked() { s.tearDownTestHelper(false) // Don't delete the generated container. listBlockedTime := 10 - config := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: https://%s.blob.core.windows.net/\n type: block\n account-key: %s\n mode: key\n container: %s\n block-list-on-mount-sec: %d\n fail-unsupported-op: true\n", - storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockKey, s.container, listBlockedTime) + config := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: %s\n type: block\n account-key: %s\n mode: key\n container: %s\n block-list-on-mount-sec: %d\n fail-unsupported-op: true\n", + storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.Endpoint, storageTestConfigurationParameters.BlockKey, s.container, listBlockedTime) s.setupTestHelper(config, s.container, true) name := generateDirectoryName() @@ -973,8 +1005,8 @@ func (s *blockBlobTestSuite) TestCreateFileWindowsNameConvert() { if runtime.GOOS != "windows" { return } - config := fmt.Sprintf("restricted-characters-windows: true\nazstorage:\n account-name: %s\n endpoint: https://%s.dfs.core.windows.net/\n type: block\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true", - storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockKey, s.container) + config := fmt.Sprintf("restricted-characters-windows: true\nazstorage:\n account-name: %s\n endpoint: %s\n type: block\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true", + storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.Endpoint, storageTestConfigurationParameters.BlockKey, s.container) s.setupTestHelper(config, s.container, true) defer s.cleanupTest() // Setup @@ -1077,8 +1109,8 @@ func (s *blockBlobTestSuite) TestDeleteFileWindowsNameConvert() { if runtime.GOOS != "windows" { return } - config := fmt.Sprintf("restricted-characters-windows: true\nazstorage:\n account-name: %s\n endpoint: https://%s.dfs.core.windows.net/\n type: block\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true", - storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockKey, s.container) + config := fmt.Sprintf("restricted-characters-windows: true\nazstorage:\n account-name: %s\n endpoint: %s\n type: block\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true", + storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.Endpoint, storageTestConfigurationParameters.BlockKey, s.container) s.setupTestHelper(config, s.container, true) defer s.cleanupTest() // Setup @@ -1136,8 +1168,8 @@ func (s *blockBlobTestSuite) TestRenameFileWindowsNameConvert() { if runtime.GOOS != "windows" { return } - config := fmt.Sprintf("restricted-characters-windows: true\nazstorage:\n account-name: %s\n endpoint: https://%s.dfs.core.windows.net/\n type: block\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true", - storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockKey, s.container) + config := fmt.Sprintf("restricted-characters-windows: true\nazstorage:\n account-name: %s\n endpoint: %s\n type: block\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true", + storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.Endpoint, storageTestConfigurationParameters.BlockKey, s.container) s.setupTestHelper(config, s.container, true) defer s.cleanupTest() // Setup @@ -1331,8 +1363,8 @@ func (s *blockBlobTestSuite) TestWriteFileWindowsNameConvert() { if runtime.GOOS != "windows" { return } - config := fmt.Sprintf("restricted-characters-windows: true\nazstorage:\n account-name: %s\n endpoint: https://%s.dfs.core.windows.net/\n type: block\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true", - storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockKey, s.container) + config := fmt.Sprintf("restricted-characters-windows: true\nazstorage:\n account-name: %s\n endpoint: %s\n type: block\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true", + storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.Endpoint, storageTestConfigurationParameters.BlockKey, s.container) s.setupTestHelper(config, s.container, true) defer s.cleanupTest() // Setup @@ -1382,8 +1414,8 @@ func (s *blockBlobTestSuite) TestTruncateSmallFileSmallerWindowsNameConvert() { if runtime.GOOS != "windows" { return } - config := fmt.Sprintf("restricted-characters-windows: true\nazstorage:\n account-name: %s\n endpoint: https://%s.dfs.core.windows.net/\n type: block\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true", - storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockKey, s.container) + config := fmt.Sprintf("restricted-characters-windows: true\nazstorage:\n account-name: %s\n endpoint: %s\n type: block\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true", + storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.Endpoint, storageTestConfigurationParameters.BlockKey, s.container) s.setupTestHelper(config, s.container, true) defer s.cleanupTest() // Setup @@ -1896,8 +1928,8 @@ func (s *blockBlobTestSuite) TestCopyFromFileWindowsNameConvert() { if runtime.GOOS != "windows" { return } - config := fmt.Sprintf("restricted-characters-windows: true\nazstorage:\n account-name: %s\n endpoint: https://%s.dfs.core.windows.net/\n type: block\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true", - storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockKey, s.container) + config := fmt.Sprintf("restricted-characters-windows: true\nazstorage:\n account-name: %s\n endpoint: %s\n type: block\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true", + storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.Endpoint, storageTestConfigurationParameters.BlockKey, s.container) s.setupTestHelper(config, s.container, true) defer s.cleanupTest() // Setup @@ -1973,8 +2005,8 @@ func (s *blockBlobTestSuite) TestReadLinkError() { func (s *blockBlobTestSuite) TestGetAttrDir() { defer s.cleanupTest() - vdConfig := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: https://%s.blob.core.windows.net/\n type: block\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true\n virtual-directory: true", - storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockKey, s.container) + vdConfig := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: %s\n type: block\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true\n virtual-directory: true", + storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.Endpoint, storageTestConfigurationParameters.BlockKey, s.container) configs := []string{"", vdConfig} for _, c := range configs { // This is a little janky but required since testify suite does not support running setup or clean up for subtests. @@ -2002,8 +2034,8 @@ func (s *blockBlobTestSuite) TestGetAttrDir() { func (s *blockBlobTestSuite) TestGetAttrVirtualDir() { defer s.cleanupTest() - vdConfig := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: https://%s.blob.core.windows.net/\n type: block\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true\n virtual-directory: true", - storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockKey, s.container) + vdConfig := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: %s\n type: block\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true\n virtual-directory: true", + storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.Endpoint, storageTestConfigurationParameters.BlockKey, s.container) // This is a little janky but required since testify suite does not support running setup or clean up for subtests. s.tearDownTestHelper(false) s.setupTestHelper(vdConfig, s.container, true) @@ -2028,8 +2060,8 @@ func (s *blockBlobTestSuite) TestGetAttrVirtualDir() { func (s *blockBlobTestSuite) TestGetAttrVirtualDirSubDir() { defer s.cleanupTest() - vdConfig := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: https://%s.blob.core.windows.net/\n type: block\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true\n virtual-directory: true", - storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockKey, s.container) + vdConfig := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: %s\n type: block\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true\n virtual-directory: true", + storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.Endpoint, storageTestConfigurationParameters.BlockKey, s.container) // This is a little janky but required since testify suite does not support running setup or clean up for subtests. s.tearDownTestHelper(false) s.setupTestHelper(vdConfig, s.container, true) @@ -2062,8 +2094,8 @@ func (s *blockBlobTestSuite) TestGetAttrVirtualDirSubDir() { func (s *blockBlobTestSuite) TestGetAttrFile() { defer s.cleanupTest() - vdConfig := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: https://%s.blob.core.windows.net/\n type: block\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true\n virtual-directory: true", - storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockKey, s.container) + vdConfig := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: %s\n type: block\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true\n virtual-directory: true", + storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.Endpoint, storageTestConfigurationParameters.BlockKey, s.container) configs := []string{"", vdConfig} for _, c := range configs { // This is a little janky but required since testify suite does not support running setup or clean up for subtests. @@ -2089,8 +2121,8 @@ func (s *blockBlobTestSuite) TestGetAttrFile() { func (s *blockBlobTestSuite) TestGetAttrLink() { defer s.cleanupTest() - vdConfig := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: https://%s.blob.core.windows.net/\n type: block\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true\n virtual-directory: true", - storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockKey, s.container) + vdConfig := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: %s\n type: block\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true\n virtual-directory: true", + storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.Endpoint, storageTestConfigurationParameters.BlockKey, s.container) configs := []string{"", vdConfig} for _, c := range configs { // This is a little janky but required since testify suite does not support running setup or clean up for subtests. @@ -2120,8 +2152,8 @@ func (s *blockBlobTestSuite) TestGetAttrLink() { func (s *blockBlobTestSuite) TestGetAttrFileSize() { defer s.cleanupTest() - vdConfig := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: https://%s.blob.core.windows.net/\n type: block\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true\n virtual-directory: true", - storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockKey, s.container) + vdConfig := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: %s\n type: block\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true\n virtual-directory: true", + storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.Endpoint, storageTestConfigurationParameters.BlockKey, s.container) configs := []string{"", vdConfig} for _, c := range configs { // This is a little janky but required since testify suite does not support running setup or clean up for subtests. @@ -2151,8 +2183,8 @@ func (s *blockBlobTestSuite) TestGetAttrFileSize() { func (s *blockBlobTestSuite) TestGetAttrFileTime() { defer s.cleanupTest() - vdConfig := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: https://%s.blob.core.windows.net/\n type: block\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true\n virtual-directory: true", - storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockKey, s.container) + vdConfig := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: %s\n type: block\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true\n virtual-directory: true", + storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.Endpoint, storageTestConfigurationParameters.BlockKey, s.container) configs := []string{"", vdConfig} for _, c := range configs { // This is a little janky but required since testify suite does not support running setup or clean up for subtests. @@ -2189,8 +2221,8 @@ func (s *blockBlobTestSuite) TestGetAttrFileTime() { func (s *blockBlobTestSuite) TestGetAttrError() { defer s.cleanupTest() - vdConfig := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: https://%s.blob.core.windows.net/\n type: block\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true\n virtual-directory: true", - storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockKey, s.container) + vdConfig := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: %s\n type: block\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true\n virtual-directory: true", + storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.Endpoint, storageTestConfigurationParameters.BlockKey, s.container) configs := []string{"", vdConfig} for _, c := range configs { // This is a little janky but required since testify suite does not support running setup or clean up for subtests. @@ -2228,8 +2260,8 @@ func (s *blockBlobTestSuite) TestChmodIgnore() { // Setup s.tearDownTestHelper(false) // Don't delete the generated container. - config := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: https://%s.blob.core.windows.net/\n type: block\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: false\n", - storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockKey, s.container) + config := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: %s\n type: block\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: false\n", + storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.Endpoint, storageTestConfigurationParameters.BlockKey, s.container) s.setupTestHelper(config, s.container, true) name := generateFileName() s.az.CreateFile(internal.CreateFileOptions{Name: name}) @@ -2254,8 +2286,8 @@ func (s *blockBlobTestSuite) TestChownIgnore() { // Setup s.tearDownTestHelper(false) // Don't delete the generated container. - config := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: https://%s.blob.core.windows.net/\n type: block\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: false\n", - storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockKey, s.container) + config := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: %s\n type: block\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: false\n", + storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.Endpoint, storageTestConfigurationParameters.BlockKey, s.container) s.setupTestHelper(config, s.container, true) name := generateFileName() s.az.CreateFile(internal.CreateFileOptions{Name: name}) @@ -2877,8 +2909,8 @@ func (s *blockBlobTestSuite) TestUpdateConfig() { func (s *blockBlobTestSuite) TestMD5SetOnUpload() { defer s.cleanupTest() - vdConfig := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: https://%s.blob.core.windows.net/\n type: block\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true\n virtual-directory: true", - storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockKey, s.container) + vdConfig := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: %s\n type: block\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true\n virtual-directory: true", + storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.Endpoint, storageTestConfigurationParameters.BlockKey, s.container) configs := []string{"", vdConfig} for _, c := range configs { // This is a little janky but required since testify suite does not support running setup or clean up for subtests. @@ -2892,8 +2924,8 @@ func (s *blockBlobTestSuite) TestMD5SetOnUpload() { // Setup s.tearDownTestHelper(false) // Don't delete the generated container. - config := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: https://%s.blob.core.windows.net/\n type: block\n account-key: %s\n mode: key\n container: %s\n update-md5: true\n", - storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockKey, s.container) + config := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: %s\n type: block\n account-key: %s\n mode: key\n container: %s\n update-md5: true\n", + storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.Endpoint, storageTestConfigurationParameters.BlockKey, s.container) s.setupTestHelper(config, s.container, true) name := generateFileName() @@ -2930,8 +2962,8 @@ func (s *blockBlobTestSuite) TestMD5SetOnUpload() { func (s *blockBlobTestSuite) TestMD5NotSetOnUpload() { defer s.cleanupTest() - vdConfig := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: https://%s.blob.core.windows.net/\n type: block\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true\n virtual-directory: true", - storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockKey, s.container) + vdConfig := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: %s\n type: block\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true\n virtual-directory: true", + storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.Endpoint, storageTestConfigurationParameters.BlockKey, s.container) configs := []string{"", vdConfig} for _, c := range configs { // This is a little janky but required since testify suite does not support running setup or clean up for subtests. @@ -2945,8 +2977,8 @@ func (s *blockBlobTestSuite) TestMD5NotSetOnUpload() { // Setup s.tearDownTestHelper(false) // Don't delete the generated container. - config := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: https://%s.blob.core.windows.net/\n type: block\n account-key: %s\n mode: key\n container: %s\n update-md5: false\n", - storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockKey, s.container) + config := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: %s\n type: block\n account-key: %s\n mode: key\n container: %s\n update-md5: false\n", + storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.Endpoint, storageTestConfigurationParameters.BlockKey, s.container) s.setupTestHelper(config, s.container, true) name := generateFileName() @@ -2978,8 +3010,8 @@ func (s *blockBlobTestSuite) TestMD5NotSetOnUpload() { func (s *blockBlobTestSuite) TestMD5AutoSetOnUpload() { defer s.cleanupTest() - vdConfig := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: https://%s.blob.core.windows.net/\n type: block\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true\n virtual-directory: true", - storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockKey, s.container) + vdConfig := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: %s\n type: block\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true\n virtual-directory: true", + storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.Endpoint, storageTestConfigurationParameters.BlockKey, s.container) configs := []string{"", vdConfig} for _, c := range configs { // This is a little janky but required since testify suite does not support running setup or clean up for subtests. @@ -2993,8 +3025,8 @@ func (s *blockBlobTestSuite) TestMD5AutoSetOnUpload() { // Setup s.tearDownTestHelper(false) // Don't delete the generated container. - config := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: https://%s.blob.core.windows.net/\n type: block\n account-key: %s\n mode: key\n container: %s\n update-md5: false\n", - storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockKey, s.container) + config := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: %s\n type: block\n account-key: %s\n mode: key\n container: %s\n update-md5: false\n", + storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.Endpoint, storageTestConfigurationParameters.BlockKey, s.container) s.setupTestHelper(config, s.container, true) name := generateFileName() @@ -3031,8 +3063,8 @@ func (s *blockBlobTestSuite) TestMD5AutoSetOnUpload() { func (s *blockBlobTestSuite) TestInvalidateMD5PostUpload() { defer s.cleanupTest() - vdConfig := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: https://%s.blob.core.windows.net/\n type: block\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true\n virtual-directory: true", - storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockKey, s.container) + vdConfig := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: %s\n type: block\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true\n virtual-directory: true", + storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.Endpoint, storageTestConfigurationParameters.BlockKey, s.container) configs := []string{"", vdConfig} for _, c := range configs { // This is a little janky but required since testify suite does not support running setup or clean up for subtests. @@ -3046,8 +3078,8 @@ func (s *blockBlobTestSuite) TestInvalidateMD5PostUpload() { // Setup s.tearDownTestHelper(false) // Don't delete the generated container. - config := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: https://%s.blob.core.windows.net/\n type: block\n account-key: %s\n mode: key\n container: %s\n update-md5: true\n validate-md5: true\n", - storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockKey, s.container) + config := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: %s\n type: block\n account-key: %s\n mode: key\n container: %s\n update-md5: true\n validate-md5: true\n", + storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.Endpoint, storageTestConfigurationParameters.BlockKey, s.container) s.setupTestHelper(config, s.container, true) name := generateFileName() @@ -3087,8 +3119,8 @@ func (s *blockBlobTestSuite) TestInvalidateMD5PostUpload() { func (s *blockBlobTestSuite) TestValidateAutoMD5OnRead() { defer s.cleanupTest() - vdConfig := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: https://%s.blob.core.windows.net/\n type: block\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true\n virtual-directory: true", - storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockKey, s.container) + vdConfig := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: %s\n type: block\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true\n virtual-directory: true", + storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.Endpoint, storageTestConfigurationParameters.BlockKey, s.container) configs := []string{"", vdConfig} for _, c := range configs { // This is a little janky but required since testify suite does not support running setup or clean up for subtests. @@ -3102,8 +3134,8 @@ func (s *blockBlobTestSuite) TestValidateAutoMD5OnRead() { // Setup s.tearDownTestHelper(false) // Don't delete the generated container. - config := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: https://%s.blob.core.windows.net/\n type: block\n account-key: %s\n mode: key\n container: %s\n update-md5: false\n validate-md5: true\n", - storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockKey, s.container) + config := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: %s\n type: block\n account-key: %s\n mode: key\n container: %s\n update-md5: false\n validate-md5: true\n", + storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.Endpoint, storageTestConfigurationParameters.BlockKey, s.container) s.setupTestHelper(config, s.container, true) name := generateFileName() @@ -3143,8 +3175,8 @@ func (s *blockBlobTestSuite) TestValidateAutoMD5OnRead() { func (s *blockBlobTestSuite) TestValidateManualMD5OnRead() { defer s.cleanupTest() - vdConfig := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: https://%s.blob.core.windows.net/\n type: block\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true\n virtual-directory: true", - storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockKey, s.container) + vdConfig := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: %s\n type: block\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true\n virtual-directory: true", + storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.Endpoint, storageTestConfigurationParameters.BlockKey, s.container) configs := []string{"", vdConfig} for _, c := range configs { // This is a little janky but required since testify suite does not support running setup or clean up for subtests. @@ -3158,8 +3190,8 @@ func (s *blockBlobTestSuite) TestValidateManualMD5OnRead() { // Setup s.tearDownTestHelper(false) // Don't delete the generated container. - config := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: https://%s.blob.core.windows.net/\n type: block\n account-key: %s\n mode: key\n container: %s\n update-md5: true\n validate-md5: true\n", - storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockKey, s.container) + config := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: %s\n type: block\n account-key: %s\n mode: key\n container: %s\n update-md5: true\n validate-md5: true\n", + storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.Endpoint, storageTestConfigurationParameters.BlockKey, s.container) s.setupTestHelper(config, s.container, true) name := generateFileName() @@ -3199,8 +3231,8 @@ func (s *blockBlobTestSuite) TestValidateManualMD5OnRead() { func (s *blockBlobTestSuite) TestInvalidMD5OnRead() { defer s.cleanupTest() - vdConfig := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: https://%s.blob.core.windows.net/\n type: block\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true\n virtual-directory: true", - storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockKey, s.container) + vdConfig := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: %s\n type: block\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true\n virtual-directory: true", + storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.Endpoint, storageTestConfigurationParameters.BlockKey, s.container) configs := []string{"", vdConfig} for _, c := range configs { // This is a little janky but required since testify suite does not support running setup or clean up for subtests. @@ -3214,8 +3246,8 @@ func (s *blockBlobTestSuite) TestInvalidMD5OnRead() { // Setup s.tearDownTestHelper(false) // Don't delete the generated container. - config := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: https://%s.blob.core.windows.net/\n type: block\n account-key: %s\n mode: key\n container: %s\n update-md5: true\n validate-md5: true\n", - storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockKey, s.container) + config := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: %s\n type: block\n account-key: %s\n mode: key\n container: %s\n update-md5: true\n validate-md5: true\n", + storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.Endpoint, storageTestConfigurationParameters.BlockKey, s.container) s.setupTestHelper(config, s.container, true) name := generateFileName() @@ -3259,8 +3291,8 @@ func (s *blockBlobTestSuite) TestInvalidMD5OnRead() { func (s *blockBlobTestSuite) TestInvalidMD5OnReadNoVaildate() { defer s.cleanupTest() - vdConfig := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: https://%s.blob.core.windows.net/\n type: block\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true\n virtual-directory: true", - storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockKey, s.container) + vdConfig := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: %s\n type: block\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true\n virtual-directory: true", + storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.Endpoint, storageTestConfigurationParameters.BlockKey, s.container) configs := []string{"", vdConfig} for _, c := range configs { // This is a little janky but required since testify suite does not support running setup or clean up for subtests. @@ -3274,8 +3306,8 @@ func (s *blockBlobTestSuite) TestInvalidMD5OnReadNoVaildate() { // Setup s.tearDownTestHelper(false) // Don't delete the generated container. - config := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: https://%s.blob.core.windows.net/\n type: block\n account-key: %s\n mode: key\n container: %s\n update-md5: true\n validate-md5: false\n", - storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockKey, s.container) + config := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: %s\n type: block\n account-key: %s\n mode: key\n container: %s\n update-md5: true\n validate-md5: false\n", + storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.Endpoint, storageTestConfigurationParameters.BlockKey, s.container) s.setupTestHelper(config, s.container, true) name := generateFileName() diff --git a/component/azstorage/datalake_test.go b/component/azstorage/datalake_test.go index 212a918ec..8db369e9e 100644 --- a/component/azstorage/datalake_test.go +++ b/component/azstorage/datalake_test.go @@ -1,5 +1,5 @@ -//go:build !authtest -// +build !authtest +//go:build !authtest && !azurite +// +build !authtest,!azurite /* Licensed under the MIT License . diff --git a/component/azstorage/utils_test.go b/component/azstorage/utils_test.go index 4e8b20268..a27719d16 100644 --- a/component/azstorage/utils_test.go +++ b/component/azstorage/utils_test.go @@ -1,3 +1,5 @@ +//go:build !azurite + /* Licensed under the MIT License . diff --git a/component/file_cache/file_cache_test.go b/component/file_cache/file_cache_test.go index be0cf199b..8ab2c6de6 100644 --- a/component/file_cache/file_cache_test.go +++ b/component/file_cache/file_cache_test.go @@ -1147,7 +1147,7 @@ func (suite *fileCacheTestSuite) TestGetAttrCase4() { // Wait file is evicted _, err = os.Stat(common.JoinUnixFilepath(suite.cache_path, file)) - for i := 0; i < 10 && !os.IsNotExist(err); i++ { + for i := 0; i < 20 && !os.IsNotExist(err); i++ { time.Sleep(time.Second) _, err = os.Stat(common.JoinUnixFilepath(suite.cache_path, file)) } diff --git a/component/s3storage/client.go b/component/s3storage/client.go index ea236a466..64bff8332 100644 --- a/component/s3storage/client.go +++ b/component/s3storage/client.go @@ -131,8 +131,16 @@ func (cl *Client) Configure(cfg Config) error { log.Err("Client::Configure : config.LoadDefaultConfig() failed. Here's why: %v", err) return err } + // Create an Amazon S3 service client - cl.awsS3Client = s3.NewFromConfig(defaultConfig) + if cl.Config.usePathStyle { + cl.awsS3Client = s3.NewFromConfig(defaultConfig, func(o *s3.Options) { + o.UsePathStyle = true + }) + } else { + cl.awsS3Client = s3.NewFromConfig(defaultConfig) + } + // ListBuckets here to test connection _, err = cl.ListBuckets() if err != nil { diff --git a/component/s3storage/client_test.go b/component/s3storage/client_test.go index 58c8e4773..b2d2a42fc 100644 --- a/component/s3storage/client_test.go +++ b/component/s3storage/client_test.go @@ -87,6 +87,7 @@ func newTestClient(configuration string) (*Client, error) { disableConcurrentDownload: conf.DisableConcurrentDownload, partSize: conf.PartSizeMb * common.MbToBytes, uploadCutoff: conf.UploadCutoffMb * common.MbToBytes, + usePathStyle: conf.UsePathStyle, } // create a Client client, err := NewConnection(configForS3Client) @@ -135,10 +136,12 @@ func (s *clientTestSuite) setupTestHelper(configuration string, create bool) err storageTestConfigurationParameters.UploadCutoffMb = 5 } if configuration == "" { - configuration = fmt.Sprintf("s3storage:\n bucket-name: %s\n key-id: %s\n secret-key: %s\n endpoint: %s\n region: %s\n part-size-mb: %d\n upload-cutoff-mb: %d\n", + configuration = fmt.Sprintf("s3storage:\n bucket-name: %s\n key-id: %s\n secret-key: %s\n endpoint: %s\n region: %s\n part-size-mb: %d\n"+ + " upload-cutoff-mb: %d\n use-path-style: %t\n", storageTestConfigurationParameters.BucketName, storageTestConfigurationParameters.KeyID, storageTestConfigurationParameters.SecretKey, storageTestConfigurationParameters.Endpoint, storageTestConfigurationParameters.Region, - storageTestConfigurationParameters.PartSizeMb, storageTestConfigurationParameters.UploadCutoffMb) + storageTestConfigurationParameters.PartSizeMb, storageTestConfigurationParameters.UploadCutoffMb, + storageTestConfigurationParameters.UsePathStyle) } s.config = configuration @@ -176,8 +179,8 @@ func (s *clientTestSuite) TestEnvVarCredentials() { // setup os.Setenv("AWS_ACCESS_KEY_ID", storageTestConfigurationParameters.KeyID) os.Setenv("AWS_SECRET_ACCESS_KEY", storageTestConfigurationParameters.SecretKey) - config := fmt.Sprintf("s3storage:\n bucket-name: %s", - storageTestConfigurationParameters.BucketName) + config := fmt.Sprintf("s3storage:\n bucket-name: %s\n endpoint: %s", storageTestConfigurationParameters.BucketName, + storageTestConfigurationParameters.Endpoint) // S3 connection should find credentials from environment variables err := s.setupTestHelper(config, false) s.assert.Nil(err) @@ -201,7 +204,7 @@ func (s *clientTestSuite) TestListBuckets() { // TODO: generalize this test by creating, listing, then destroying a bucket buckets, err := s.client.ListBuckets() s.assert.Nil(err) - s.assert.Equal(buckets, []string{storageTestConfigurationParameters.BucketName}) + s.assert.Contains(buckets, storageTestConfigurationParameters.BucketName) } func (s *clientTestSuite) TestSetPrefixPath() { defer s.cleanupTest() diff --git a/component/s3storage/config.go b/component/s3storage/config.go index c643dc299..87ce6c325 100644 --- a/component/s3storage/config.go +++ b/component/s3storage/config.go @@ -55,6 +55,7 @@ type Options struct { DisableConcurrentDownload bool `config:"disable-concurrent-download" yaml:"disable-concurrent-download,omitempty"` EnableChecksum bool `config:"enable-checksum" yaml:"enable-checksum,omitempty"` ChecksumAlgorithm types.ChecksumAlgorithm `config:"checksum-algorithm" yaml:"checksum-algorithm,omitempty"` + UsePathStyle bool `config:"use-path-style" yaml:"use-path-style,omitempty"` } // ParseAndValidateConfig : Parse and validate config @@ -77,6 +78,7 @@ func ParseAndValidateConfig(s3 *S3Storage, opt Options) error { // Set restricted characters s3.stConfig.restrictedCharsWin = opt.RestrictedCharsWin s3.stConfig.disableConcurrentDownload = opt.DisableConcurrentDownload + s3.stConfig.usePathStyle = opt.UsePathStyle // Part size must be at least 5 MB and smaller than 5GB. Otherwise, set to default. if opt.PartSizeMb < 5 || opt.PartSizeMb > MaxPartSizeMb { diff --git a/component/s3storage/connection.go b/component/s3storage/connection.go index b1d8dd277..ca0bf3304 100644 --- a/component/s3storage/connection.go +++ b/component/s3storage/connection.go @@ -50,6 +50,7 @@ type Config struct { disableConcurrentDownload bool enableChecksum bool checksumAlgorithm types.ChecksumAlgorithm + usePathStyle bool } // TODO: move s3AuthConfig to s3auth.go diff --git a/component/s3storage/s3storage_test.go b/component/s3storage/s3storage_test.go index c19fb1533..c19bbd92e 100644 --- a/component/s3storage/s3storage_test.go +++ b/component/s3storage/s3storage_test.go @@ -94,6 +94,7 @@ type storageTestConfiguration struct { PartSizeMb int64 `json:"part-size-mb"` UploadCutoffMb int64 `json:"upload-cutoff-mb"` DisableConcurrentDownload bool `json:"disable-concurrent-download"` + UsePathStyle bool `json:"use-path-style"` } var storageTestConfigurationParameters storageTestConfiguration @@ -325,10 +326,10 @@ func (s *s3StorageTestSuite) setupTestHelper(configuration string, bucket string func generateConfigYaml(testParams storageTestConfiguration) string { return fmt.Sprintf("s3storage:\n bucket-name: %s\n key-id: %s\n secret-key: %s\n"+ " region: %s\n profile: %s\n endpoint: %s\n subdirectory: %s\n restricted-characters-windows: %t\n"+ - " part-size-mb: %d\n upload-cutoff-mb: %d\n disable-concurrent-download: %t", + " part-size-mb: %d\n upload-cutoff-mb: %d\n disable-concurrent-download: %t\n use-path-style: %t\n", testParams.BucketName, testParams.KeyID, testParams.SecretKey, testParams.Region, testParams.Profile, testParams.Endpoint, testParams.Prefix, testParams.RestrictedCharsWin, testParams.PartSizeMb, - testParams.UploadCutoffMb, testParams.DisableConcurrentDownload) + testParams.UploadCutoffMb, testParams.DisableConcurrentDownload, testParams.UsePathStyle) } func (s *s3StorageTestSuite) tearDownTestHelper(delete bool) { @@ -378,7 +379,7 @@ func (s *s3StorageTestSuite) TestListBuckets() { buckets, err := s.s3Storage.ListBuckets() s.assert.Nil(err) - s.assert.Equal(buckets, []string{storageTestConfigurationParameters.BucketName}) + s.assert.Contains(buckets, storageTestConfigurationParameters.BucketName) } func (s *s3StorageTestSuite) TestCreateDir() { diff --git a/component/s3storage/s3wrappers.go b/component/s3storage/s3wrappers.go index 64a389e5d..4a3f034e3 100644 --- a/component/s3storage/s3wrappers.go +++ b/component/s3storage/s3wrappers.go @@ -165,6 +165,9 @@ func (cl *Client) deleteObject(name string, isSymLink bool) error { // Wrapper for awsS3Client.DeleteObjects. // names is a list of paths to the objects. func (cl *Client) deleteObjects(objects []*internal.ObjAttr) error { + if objects == nil { + return nil + } log.Trace("Client::deleteObjects : deleting %d objects", len(objects)) // build list to send to DeleteObjects keyList := make([]types.ObjectIdentifier, len(objects)) diff --git a/setup/baseConfig.yaml b/setup/baseConfig.yaml index 9f96a46fc..f7e3559bf 100644 --- a/setup/baseConfig.yaml +++ b/setup/baseConfig.yaml @@ -185,7 +185,8 @@ s3storage: concurrency: disable-concurrent-download: true|false enable-checksum: true|false - checksum-algorithm: CRC32|CRC32C|SHA1|SHA256 + checksum-algorithm: CRC32|CRC32C|SHA1|SHA256 + usePathStyle: true|false # Mount all configuration mountall: