-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'main' into fix-spelling-errors
- Loading branch information
Showing
78 changed files
with
5,274 additions
and
907 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,150 @@ | ||
parameters: | ||
- name: fuselib | ||
type: string | ||
- name: tags | ||
type: string | ||
- name: container | ||
type: string | ||
- name: skip_msi | ||
type: string | ||
default: "true" | ||
- name: proxy_address | ||
type: string | ||
default: "" | ||
- name: skip_fmt | ||
type: boolean | ||
default: true | ||
|
||
steps: | ||
- checkout: self | ||
|
||
- task: GoTool@0 | ||
inputs: | ||
version: '1.20.5' | ||
displayName: "Select Go Version" | ||
|
||
# Install fuse and build the code | ||
- script: | | ||
sudo apt-get update --fix-missing | ||
sudo apt-get install ${{ parameters.fuselib }} gcc -y | ||
displayName: 'Install libfuse' | ||
- task: Go@0 | ||
inputs: | ||
command: 'build' | ||
workingDirectory: ./ | ||
arguments: "-tags ${{ parameters.tags }} -o cloudfuse" | ||
displayName: "Build" | ||
|
||
- script: | | ||
cnfFile=$HOME/azuretest.json | ||
echo $cnfFile | ||
touch $cnfFile | ||
echo "{" > $cnfFile | ||
echo "\"block-acct\"": "\"$(AZTEST_BLOCK_ACC_NAME)\"", >> $cnfFile | ||
echo "\"adls-acct\"": "\"$(AZTEST_ADLS_ACC_NAME)\"", >> $cnfFile | ||
echo "\"block-cont\"": "\"${{ parameters.container }}\"", >> $cnfFile | ||
echo "\"adls-cont\"": "\"${{ parameters.container }}\"", >> $cnfFile | ||
echo "\"block-key\"": "\"$(AZTEST_BLOCK_KEY)\"", >> $cnfFile | ||
echo "\"adls-key\"": "\"$(AZTEST_ADLS_KEY)\"", >> $cnfFile | ||
echo "\"block-sas\"": "\"$(AZTEST_BLOCK_SAS)\"", >> $cnfFile | ||
echo "\"block-cont-sas-ubn-18\"": "\"$(AZTEST_BLOCK_CONT_SAS_UBN_18)\"", >> $cnfFile | ||
echo "\"block-cont-sas-ubn-20\"": "\"$(AZTEST_BLOCK_CONT_SAS_UBN_20)\"", >> $cnfFile | ||
echo "\"adls-sas\"": "\"$(adlsSas)\"", >> $cnfFile | ||
echo "\"msi-appid\"": "\"$(AZTEST_APP_ID)\"", >> $cnfFile | ||
echo "\"msi-resid\"": "\"$(AZTEST_RES_ID)\"", >> $cnfFile | ||
echo "\"msi-objid\"": "\"$(AZTEST_OBJ_ID)\"", >> $cnfFile | ||
echo "\"spn-client\"": "\"$(AZTEST_CLIENT)\"", >> $cnfFile | ||
echo "\"spn-tenant\"": "\"$(AZTEST_TENANT)\"", >> $cnfFile | ||
echo "\"spn-secret\"": "\"$(AZTEST_SECRET)\"", >> $cnfFile | ||
echo "\"skip-msi\"": "${{ parameters.skip_msi }}", >> $cnfFile | ||
echo "\"proxy-address\"": "\"${{ parameters.proxy_address }}\"" >> $cnfFile | ||
echo "}" >> $cnfFile | ||
cat $cnfFile | ||
displayName: "Create Configuration File" | ||
continueOnError: false | ||
workingDirectory: ./ | ||
- script: | | ||
cnfFile=$HOME/s3test.json | ||
echo $cnfFile | ||
touch $cnfFile | ||
echo "{" > $cnfFile | ||
echo "\"bucket-name\"": "\"$(S3TEST_BUCKET_NAME)\"", >> $cnfFile | ||
echo "\"access-key\"": "\"$(S3TEST_ACCESS_KEY)\"", >> $cnfFile | ||
echo "\"secret-key\"": "\"$(S3TEST_SECRET_KEY)\"", >> $cnfFile | ||
echo "\"endpoint\"": "\"$(S3TEST_ENDPOINT)\"", >> $cnfFile | ||
echo "\"region\"": "\"$(S3TEST_REGION)\"" >> $cnfFile | ||
echo "}" >> $cnfFile | ||
cat $cnfFile | ||
displayName: "Create S3 Configuration File" | ||
continueOnError: false | ||
workingDirectory: ./ | ||
# Code lint checks (Static-analysis) | ||
# Exit code changed to prevent failing in CI/CD pipeline | ||
# TODO: Remove this once we are passing file coverage checks consistently | ||
- script: | | ||
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin | ||
$(go env GOPATH)/bin/golangci-lint --version | ||
$(go env GOPATH)/bin/golangci-lint run --tests=false --build-tags ${{ parameters.tags }} --skip-dirs test,common/stats_collector,common/stats_monitor --max-issues-per-linter=0 --skip-files component/libfuse/libfuse2_handler_test_wrapper.go,component/libfuse/libfuse_handler_test_wrapper.go > lint.log | ||
result=$(cat lint.log | wc -l) | ||
if [ $result -ne 0 ]; then | ||
echo "-----------------------------------" | ||
echo "Below issues are found in SA" | ||
cat lint.log | ||
echo "-----------------------------------" | ||
exit 0 | ||
else | ||
echo "-----------------------------------" | ||
echo "No issues are found in SA" | ||
echo "-----------------------------------" | ||
fi | ||
displayName: 'Static Analysis (Lint)' | ||
condition: always() | ||
workingDirectory: ./ | ||
# Copyright checks | ||
# - script: | | ||
# result=$(grep -L -r --include \*.go "`date +%Y` Microsoft Corporation" ./ | wc -l) | ||
# if [ $result -ne 0 ]; then | ||
# exit 1 | ||
# else | ||
# echo "Copyright statements are up to date" | ||
# fi | ||
# displayName: 'Copyright check' | ||
# condition: always() | ||
# failOnStderr: true | ||
# workingDirectory: ./ | ||
|
||
# Go code formatting checks | ||
- script: | | ||
gofmt -s -l -d . | tee >&2 | ||
displayName: 'Go Format Check' | ||
condition: eq(${{ parameters.skip_fmt }}, true) | ||
failOnStderr: true | ||
workingDirectory: ./ | ||
# Notices files check | ||
- script: | | ||
./notices_fix.sh | ||
result=$(git diff NOTICE | wc -l) | ||
if [ $result -ne 0 ]; then | ||
echo "Notices needs a fix. Run ./notices_fix.sh and commit NOTICE file." | ||
exit 1 | ||
else | ||
echo "Notices are up to date." | ||
fi | ||
displayName: 'Notice file check' | ||
condition: always() | ||
failOnStderr: true | ||
workingDirectory: ./ | ||
# Running unit tests for fuse3 on ubn-20 | ||
- task: Go@0 | ||
inputs: | ||
command: 'test' | ||
arguments: '-v -timeout=2h ./... --tags=unittest,${{ parameters.tags }} -coverprofile utcover.cov' | ||
workingDirectory: ./ | ||
displayName: 'Unit tests' | ||
continueOnError: false |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,118 @@ | ||
parameters: | ||
- name: conf_template | ||
type: string | ||
- name: config_file | ||
type: string | ||
- name: container | ||
type: string | ||
- name: temp_dir | ||
type: string | ||
- name: mount_dir | ||
type: string | ||
- name: idstring | ||
type: string | ||
- name: adls | ||
type: boolean | ||
- name: account_name | ||
type: string | ||
- name: account_key | ||
type: string | ||
- name: account_type | ||
type: string | ||
- name: account_endpoint | ||
- name: distro_name | ||
type: string | ||
- name: quick_test | ||
type: boolean | ||
default: true | ||
- name: verbose_log | ||
type: boolean | ||
default: false | ||
- name: clone | ||
type: boolean | ||
default: false | ||
- name: stream_direct_test | ||
type: boolean | ||
default: false | ||
|
||
steps: | ||
- script: | | ||
$(WORK_DIR)/blobfuse2 gen-test-config --config-file=$(WORK_DIR)/testdata/config/azure_key.yaml --container-name=${{ parameters.container }} --temp-path=${{ parameters.temp_dir }} --output-file=${{ parameters.config_file }} | ||
displayName: 'Create Config File for RW mount' | ||
env: | ||
NIGHTLY_STO_ACC_NAME: ${{ parameters.account_name }} | ||
NIGHTLY_STO_ACC_KEY: ${{ parameters.account_key }} | ||
ACCOUNT_TYPE: ${{ parameters.account_type }} | ||
ACCOUNT_ENDPOINT: ${{ parameters.account_endpoint }} | ||
VERBOSE_LOG: ${{ parameters.verbose_log }} | ||
continueOnError: false | ||
- script: | ||
cat ${{ parameters.config_file }} | ||
displayName: 'Print config file' | ||
|
||
- template: 'mount.yml' | ||
parameters: | ||
working_dir: $(WORK_DIR) | ||
mount_dir: ${{ parameters.mount_dir }} | ||
temp_dir: ${{ parameters.temp_dir }} | ||
prefix: ${{ parameters.idstring }} | ||
mountStep: | ||
script: | | ||
$(WORK_DIR)/blobfuse2 mount ${{ parameters.mount_dir }} --config-file=${{ parameters.config_file }} --default-working-dir=$(WORK_DIR) --file-cache-timeout=3200 | ||
- script: | | ||
for i in {1,2,3,4,5,6,7,8,9,10,20,30,50,100,200,1024,2048,4096}; do echo $i; done | parallel --will-cite -j 5 'head -c {}M < /dev/urandom > ${{ parameters.mount_dir }}/myfile_{}' | ||
ls -l ${{ parameters.mount_dir }}/myfile_* | ||
displayName: 'Generate data' | ||
- script: | | ||
md5sum ${{ parameters.mount_dir }}/myfile_* > $(WORK_DIR)/md5sum_file_cache.txt | ||
displayName: 'Generate md5Sum with File-Cache' | ||
- script: | | ||
$(WORK_DIR)/blobfuse2 unmount all | ||
displayName: 'Unmount RW mount' | ||
- script: | | ||
$(WORK_DIR)/blobfuse2 gen-test-config --config-file=$(WORK_DIR)/testdata/config/azure_key_bc.yaml --container-name=${{ parameters.container }} --temp-path=${{ parameters.temp_dir }} --output-file=${{ parameters.config_file }} | ||
displayName: 'Create Config File for RO mount' | ||
env: | ||
NIGHTLY_STO_ACC_NAME: ${{ parameters.account_name }} | ||
NIGHTLY_STO_ACC_KEY: ${{ parameters.account_key }} | ||
ACCOUNT_TYPE: ${{ parameters.account_type }} | ||
ACCOUNT_ENDPOINT: ${{ parameters.account_endpoint }} | ||
VERBOSE_LOG: ${{ parameters.verbose_log }} | ||
continueOnError: false | ||
- template: 'mount.yml' | ||
parameters: | ||
working_dir: $(WORK_DIR) | ||
mount_dir: ${{ parameters.mount_dir }} | ||
temp_dir: ${{ parameters.temp_dir }} | ||
prefix: ${{ parameters.idstring }} | ||
ro_mount: true | ||
mountStep: | ||
script: | | ||
$(WORK_DIR)/blobfuse2 mount ${{ parameters.mount_dir }} --config-file=${{ parameters.config_file }} --default-working-dir=$(WORK_DIR) -o ro | ||
- script: | | ||
md5sum ${{ parameters.mount_dir }}/myfile_* > $(WORK_DIR)/md5sum_block_cache.txt | ||
displayName: 'Generate md5Sum with Block-Cache' | ||
- script: | | ||
$(WORK_DIR)/blobfuse2 unmount all | ||
displayName: 'Unmount RO mount' | ||
- script: | | ||
diff $(WORK_DIR)/md5sum_block_cache.txt $(WORK_DIR)/md5sum_file_cache.txt | ||
if [ $? -ne 0 ]; then | ||
exit 1 | ||
fi | ||
displayName: 'Compare md5Sum' | ||
- template: 'cleanup.yml' | ||
parameters: | ||
working_dir: $(WORK_DIR) | ||
mount_dir: ${{ parameters.mount_dir }} | ||
temp_dir: ${{ parameters.temp_dir }} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.