forked from GoogleCloudPlatform/magic-modules
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
nightly vcr cassettes update (GoogleCloudPlatform#9678)
* nightly vcr cassettes update setup * fix comment + delete cassettes to test recording * trigger full test run * update comments * update * changed cassette source * activate service account first * updated cassette folder * new SA for vcr-nightly
- Loading branch information
1 parent
b6a3165
commit 1df558c
Showing
2 changed files
with
243 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,45 @@ | ||
--- | ||
steps: | ||
- name: 'gcr.io/graphite-docker-images/go-plus' | ||
id: gcb-vcr-nightly | ||
entrypoint: '/workspace/.ci/scripts/go-plus/vcr-cassette-update/vcr_cassette_update.sh' | ||
secretEnv: ["GITHUB_TOKEN", "GOOGLE_BILLING_ACCOUNT", "GOOGLE_CUST_ID", "GOOGLE_FIRESTORE_PROJECT", "GOOGLE_IDENTITY_USER", "GOOGLE_MASTER_BILLING_ACCOUNT", "GOOGLE_ORG", "GOOGLE_ORG_2", "GOOGLE_ORG_DOMAIN", "GOOGLE_PROJECT", "GOOGLE_PROJECT_NUMBER", "GOOGLE_SERVICE_ACCOUNT", "SA_KEY", "GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION"] | ||
args: | ||
- $BUILD_ID | ||
|
||
# Long timeout to enable waiting on VCR test | ||
timeout: 20000s | ||
options: | ||
machineType: 'N1_HIGHCPU_32' | ||
|
||
logsBucket: 'gs://cloudbuild-vcr-nightly-logs' | ||
availableSecrets: | ||
secretManager: | ||
- versionName: projects/673497134629/secrets/github-magician-token/versions/latest | ||
env: GITHUB_TOKEN | ||
- versionName: projects/673497134629/secrets/ci-test-billing-account/versions/latest | ||
env: GOOGLE_BILLING_ACCOUNT | ||
- versionName: projects/673497134629/secrets/ci-test-cust-id/versions/latest | ||
env: GOOGLE_CUST_ID | ||
- versionName: projects/673497134629/secrets/ci-test-firestore-project/versions/latest | ||
env: GOOGLE_FIRESTORE_PROJECT | ||
- versionName: projects/673497134629/secrets/ci-test-identity-user/versions/latest | ||
env: GOOGLE_IDENTITY_USER | ||
- versionName: projects/673497134629/secrets/ci-test-master-billing-account/versions/latest | ||
env: GOOGLE_MASTER_BILLING_ACCOUNT | ||
- versionName: projects/673497134629/secrets/ci-test-org/versions/latest | ||
env: GOOGLE_ORG | ||
- versionName: projects/673497134629/secrets/ci-test-org-2/versions/latest | ||
env: GOOGLE_ORG_2 | ||
- versionName: projects/673497134629/secrets/ci-test-org-domain/versions/latest | ||
env: GOOGLE_ORG_DOMAIN | ||
- versionName: projects/673497134629/secrets/ci-test-project/versions/latest | ||
env: GOOGLE_PROJECT | ||
- versionName: projects/673497134629/secrets/ci-test-project-number/versions/latest | ||
env: GOOGLE_PROJECT_NUMBER | ||
- versionName: projects/673497134629/secrets/ci-test-service-account/versions/latest | ||
env: GOOGLE_SERVICE_ACCOUNT | ||
- versionName: projects/673497134629/secrets/ci-test-service-account-key/versions/latest | ||
env: SA_KEY | ||
- versionName: projects/673497134629/secrets/ci-test-public-advertised-prefix-description/versions/latest | ||
env: GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION |
198 changes: 198 additions & 0 deletions
198
.ci/scripts/go-plus/vcr-cassette-update/vcr_cassette_update.sh
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,198 @@ | ||
#!/bin/bash | ||
|
||
set -e | ||
|
||
build_id=$1 | ||
github_username=hashicorp | ||
gh_repo=terraform-provider-google-beta | ||
git_remote=https://github.com/$github_username/$gh_repo | ||
local_path=$GOPATH/src/github.com/hashicorp/$gh_repo | ||
mkdir -p "$(dirname $local_path)" | ||
git clone $git_remote $local_path --depth 1 | ||
pushd $local_path | ||
|
||
# get today's date in YYYY-MM-DD format | ||
today=$(date +%F) | ||
|
||
set +e | ||
|
||
echo $SA_KEY > sa_key.json | ||
gcloud auth activate-service-account $GOOGLE_SERVICE_ACCOUNT --key-file=$local_path/sa_key.json --project=$GOOGLE_PROJECT | ||
|
||
# cassette retrieval | ||
mkdir fixtures | ||
|
||
# pull main cassettes | ||
gsutil -m -q cp gs://ci-vcr-cassettes/beta/fixtures/* fixtures/ | ||
|
||
# main cassettes backup | ||
# incase nightly run goes wrong. this will be used to restore the cassettes | ||
gsutil -m -q cp fixtures/* gs://vcr-nightly/beta/$today/$build_id/main_cassettes_backup/fixtures/ | ||
|
||
mkdir testlog | ||
mkdir testlog/replaying | ||
mkdir testlog/recording | ||
mkdir testlog/recording_build | ||
|
||
export GOOGLE_REGION=us-central1 | ||
export GOOGLE_ZONE=us-central1-a | ||
export VCR_PATH=$local_path/fixtures | ||
export VCR_MODE=REPLAYING | ||
export ACCTEST_PARALLELISM=32 | ||
export GOOGLE_CREDENTIALS=$SA_KEY | ||
export GOOGLE_APPLICATION_CREDENTIALS=$local_path/sa_key.json | ||
export GOOGLE_TEST_DIRECTORY=$(go list ./... | grep -v github.com/hashicorp/terraform-provider-google-beta/scripts) | ||
|
||
echo "checking terraform version" | ||
terraform version | ||
|
||
go build $GOOGLE_TEST_DIRECTORY | ||
if [ $? != 0 ]; then | ||
echo "Skipping tests: Build failure detected" | ||
exit 1 | ||
fi | ||
|
||
echo "running tests in REPLAYING mode now" | ||
TF_LOG=DEBUG TF_LOG_PATH_MASK=$local_path/testlog/replaying/%s.log TF_ACC=1 TF_SCHEMA_PANIC_ON_ERROR=1 go test $GOOGLE_TEST_DIRECTORY -parallel $ACCTEST_PARALLELISM -v -run=TestAcc -timeout 240m -ldflags="-X=github.com/hashicorp/terraform-provider-google-beta/version.ProviderVersion=acc" > replaying_test.log | ||
|
||
# store replaying build log | ||
gsutil -h "Content-Type:text/plain" -q cp replaying_test.log gs://vcr-nightly/beta/$today/$build_id/logs/build-log/ | ||
|
||
# store replaying test logs | ||
gsutil -h "Content-Type:text/plain" -m -q cp testlog/replaying/* gs://vcr-nightly/beta/$today/$build_id/logs/replaying/ | ||
|
||
# handle provider crash | ||
TESTS_PANIC=$(grep "^panic: " replaying_test.log) | ||
if [[ -n $TESTS_PANIC ]]; then | ||
echo "#################################" | ||
echo "The provider crashed while running the VCR tests in REPLAYING mode" | ||
echo "#################################" | ||
exit 0 | ||
fi | ||
|
||
FAILED_TESTS=$(grep "^--- FAIL: TestAcc" replaying_test.log) | ||
PASSED_TESTS=$(grep "^--- PASS: TestAcc" replaying_test.log) | ||
SKIPPED_TESTS=$(grep "^--- SKIP: TestAcc" replaying_test.log) | ||
|
||
if [[ -n $FAILED_TESTS ]]; then | ||
FAILED_TESTS_COUNT=$(echo "$FAILED_TESTS" | wc -l) | ||
else | ||
FAILED_TESTS_COUNT=0 | ||
fi | ||
|
||
if [[ -n $PASSED_TESTS ]]; then | ||
PASSED_TESTS_COUNT=$(echo "$PASSED_TESTS" | wc -l) | ||
else | ||
PASSED_TESTS_COUNT=0 | ||
fi | ||
|
||
if [[ -n $SKIPPED_TESTS ]]; then | ||
SKIPPED_TESTS_COUNT=$(echo "$SKIPPED_TESTS" | wc -l) | ||
else | ||
SKIPPED_TESTS_COUNT=0 | ||
fi | ||
|
||
FAILED_TESTS_PATTERN=$(grep "^--- FAIL: TestAcc" replaying_test.log | awk '{print $3}' | awk -v d="|" '{s=(NR==1?s:s d)$0}END{print s}') | ||
|
||
echo "#################################" | ||
echo "Tests Analytics" | ||
echo "Total tests: $(($FAILED_TESTS_COUNT+$PASSED_TESTS_COUNT+$SKIPPED_TESTS_COUNT))" | ||
echo "Passed tests: $PASSED_TESTS_COUNT" | ||
echo "Skipped tests: $SKIPPED_TESTS_COUNT" | ||
echo "Affected tests: $FAILED_TESTS_COUNT" | ||
echo "Affected tests list: $FAILED_TESTS_PATTERN" | ||
echo "#################################" | ||
echo "" | ||
|
||
if [[ -n $FAILED_TESTS_PATTERN ]]; then | ||
echo "running affected tests in RECORDING mode now" | ||
|
||
# Clear fixtures folder | ||
rm $VCR_PATH/* | ||
|
||
# set RECORDING mode | ||
export VCR_MODE=RECORDING | ||
|
||
FAILED_TESTS=$(grep "^--- FAIL: TestAcc" replaying_test.log | awk '{print $3}') | ||
# test_exit_code=0 | ||
parallel --jobs 16 TF_LOG=DEBUG TF_LOG_PATH_MASK=$local_path/testlog/recording/%s.log TF_ACC=1 TF_SCHEMA_PANIC_ON_ERROR=1 go test {1} -parallel 1 -v -run="{2}$" -timeout 240m -ldflags="-X=github.com/hashicorp/terraform-provider-google-beta/version.ProviderVersion=acc" ">>" testlog/recording_build/{2}_recording_test.log ::: $GOOGLE_TEST_DIRECTORY ::: $FAILED_TESTS | ||
|
||
test_exit_code=$? | ||
|
||
# Concatenate recording build logs to one file | ||
# Note: build logs are different from debug logs | ||
for failed_test in $FAILED_TESTS | ||
do | ||
cat testlog/recording_build/${failed_test}_recording_test.log >> recording_test.log | ||
done | ||
|
||
# store cassettes | ||
gsutil -m -q cp fixtures/* gs://ci-vcr-cassettes/beta/fixtures/ | ||
|
||
# store recording build log | ||
gsutil -h "Content-Type:text/plain" -q cp recording_test.log gs://vcr-nightly/beta/$today/$build_id/logs/build-log/ | ||
|
||
# store recording individual build logs | ||
gsutil -h "Content-Type:text/plain" -m -q cp testlog/recording_build/* gs://vcr-nightly/beta/$today/$build_id/logs/build-log/recording_build/ | ||
|
||
# store recording test logs | ||
gsutil -h "Content-Type:text/plain" -m -q cp testlog/recording/* gs://vcr-nightly/beta/$today/$build_id/logs/recording/ | ||
|
||
# handle provider crash | ||
RECORDING_TESTS_PANIC=$(grep "^panic: " recording_test.log) | ||
|
||
if [[ -n $RECORDING_TESTS_PANIC ]]; then | ||
echo "#################################" | ||
echo "The provider crashed while running the VCR tests in RECORDING mode" | ||
echo "#################################" | ||
exit 0 | ||
fi | ||
|
||
RECORDING_FAILED_TESTS=$(grep "^--- FAIL: TestAcc" recording_test.log | awk '{print $3}') | ||
RECORDING_PASSED_TESTS=$(grep "^--- PASS: TestAcc" recording_test.log | awk '{print $3}') | ||
|
||
RECORDING_PASSED_TESTS_COUNT=0 | ||
RECORDING_FAILED_TESTS_COUNT=0 | ||
|
||
echo "#################################" | ||
echo "RECORDING Tests Report" | ||
if [[ -n $RECORDING_PASSED_TESTS ]]; then | ||
RECORDING_PASSED_TESTS_COUNT=$(echo "$RECORDING_PASSED_TESTS" | wc -l) | ||
echo "Tests passed during RECORDING mode:" | ||
echo $RECORDING_PASSED_TESTS | ||
echo "" | ||
fi | ||
|
||
if [[ -n $RECORDING_FAILED_TESTS ]]; then | ||
RECORDING_FAILED_TESTS_COUNT=$(echo "$RECORDING_FAILED_TESTS" | wc -l) | ||
echo "Tests failed during RECORDING mode:" | ||
echo $RECORDING_FAILED_TESTS | ||
echo "" | ||
if [[ $RECORDING_PASSED_TESTS_COUNT+$RECORDING_FAILED_TESTS_COUNT -lt $FAILED_TESTS_COUNT ]]; then | ||
echo "Several tests got terminated during RECORDING mode" | ||
fi | ||
else | ||
if [[ $RECORDING_PASSED_TESTS_COUNT+$RECORDING_FAILED_TESTS_COUNT -lt $FAILED_TESTS_COUNT ]]; then | ||
echo "Several tests got terminated during RECORDING mode" | ||
elif [[ $test_exit_code -ne 0 ]]; then | ||
# check for any uncaught errors in RECORDING mode | ||
echo "Errors occurred during RECORDING mode." | ||
else | ||
echo "All tests passed!" | ||
fi | ||
fi | ||
echo "#################################" | ||
else | ||
if [[ $test_exit_code -ne 0 ]]; then | ||
# check for any uncaught errors errors in REPLAYING mode | ||
echo "#################################" | ||
echo "Errors occurred during REPLAYING mode." | ||
echo "#################################" | ||
else | ||
echo "#################################" | ||
echo "All tests passed in REPLAYING mode." | ||
echo "#################################" | ||
fi | ||
fi | ||
|
||
set -e |