From 97fbf1b44a720039b2a4fa731be256384c7a55d9 Mon Sep 17 00:00:00 2001 From: Shuya Ma <87669292+shuyama1@users.noreply.github.com> Date: Wed, 8 May 2024 11:36:12 -0700 Subject: [PATCH] Copy TeamCity debug logs to GCS (#10495) --- .../builds/build_configuration_per_package.kt | 3 +- .../builds/build_configuration_sweepers.kt | 2 +- .../build_configuration_vcr_recording.kt | 2 +- .../components/builds/build_parameters.kt | 21 +++++++-- .../components/builds/build_steps.kt | 47 ++++++++++++++++++- .../terraform/.teamcity/settings.kts | 5 +- .../tests/build_configuration_features.kt | 31 ++++++++++++ .../terraform/.teamcity/tests/test_utils.kt | 3 +- 8 files changed, 102 insertions(+), 12 deletions(-) diff --git a/mmv1/third_party/terraform/.teamcity/components/builds/build_configuration_per_package.kt b/mmv1/third_party/terraform/.teamcity/components/builds/build_configuration_per_package.kt index f9be546f7eec..33962a6ea7bb 100644 --- a/mmv1/third_party/terraform/.teamcity/components/builds/build_configuration_per_package.kt +++ b/mmv1/third_party/terraform/.teamcity/components/builds/build_configuration_per_package.kt @@ -71,6 +71,7 @@ class PackageDetails(private val packageName: String, private val displayName: S configureGoEnv() downloadTerraformBinary() runAcceptanceTests() + saveArtifactsToGCS() } features { @@ -88,7 +89,7 @@ class PackageDetails(private val packageName: String, private val displayName: S params { configureGoogleSpecificTestParameters(environmentVariables) acceptanceTestBuildParams(parallelism, testPrefix, testTimeout) - terraformLoggingParameters(providerName) + terraformLoggingParameters(environmentVariables, providerName) terraformCoreBinaryTesting() terraformShouldPanicForSchemaErrors() readOnlySettings() diff --git a/mmv1/third_party/terraform/.teamcity/components/builds/build_configuration_sweepers.kt b/mmv1/third_party/terraform/.teamcity/components/builds/build_configuration_sweepers.kt index 28ff7b99cdd0..2be92fdedd28 100644 --- a/mmv1/third_party/terraform/.teamcity/components/builds/build_configuration_sweepers.kt +++ b/mmv1/third_party/terraform/.teamcity/components/builds/build_configuration_sweepers.kt @@ -93,7 +93,7 @@ class SweeperDetails(private val sweeperName: String, private val parentProjectN configureGoogleSpecificTestParameters(environmentVariables) acceptanceTestBuildParams(parallelism, testPrefix, testTimeout) sweeperParameters(sweeperRegions, sweeperRun) - terraformLoggingParameters(providerName) + terraformLoggingParameters(environmentVariables, providerName) terraformCoreBinaryTesting() terraformShouldPanicForSchemaErrors() readOnlySettings() diff --git a/mmv1/third_party/terraform/.teamcity/components/builds/build_configuration_vcr_recording.kt b/mmv1/third_party/terraform/.teamcity/components/builds/build_configuration_vcr_recording.kt index 134070d909ae..9294250900fa 100644 --- a/mmv1/third_party/terraform/.teamcity/components/builds/build_configuration_vcr_recording.kt +++ b/mmv1/third_party/terraform/.teamcity/components/builds/build_configuration_vcr_recording.kt @@ -68,7 +68,7 @@ class VcrDetails(private val providerName: String, private val buildId: String, configureGoogleSpecificTestParameters(environmentVariables) vcrEnvironmentVariables(environmentVariables, providerName) acceptanceTestBuildParams(parallelism, testPrefix, testTimeout) - terraformLoggingParameters(providerName) + terraformLoggingParameters(environmentVariables, providerName) terraformCoreBinaryTesting() terraformShouldPanicForSchemaErrors() readOnlySettings() diff --git a/mmv1/third_party/terraform/.teamcity/components/builds/build_parameters.kt b/mmv1/third_party/terraform/.teamcity/components/builds/build_parameters.kt index fad3586fdd49..8e9942934f3a 100644 --- a/mmv1/third_party/terraform/.teamcity/components/builds/build_parameters.kt +++ b/mmv1/third_party/terraform/.teamcity/components/builds/build_parameters.kt @@ -68,6 +68,9 @@ class AllContextParameters( // VCR specific val infraProject: String, // GOOGLE_INFRA_PROJECT val vcrBucketName: String, // VCR_BUCKET_NAME + + // GCS specific (for nightly + upstream MM logs) + val credentialsGCS: String, // GOOGLE_CREDENTIALS_GCS ) // AccTestConfiguration is used to easily pass values set via Context Parameters into build configurations. @@ -90,6 +93,9 @@ class AccTestConfiguration( // VCR specific val infraProject: String, val vcrBucketName: String, + + // GCS specific (for nightly + upstream MM logs) + val credentialsGCS: String, ) fun getGaAcceptanceTestConfig(allConfig: AllContextParameters): AccTestConfiguration { @@ -109,7 +115,8 @@ fun getGaAcceptanceTestConfig(allConfig: AllContextParameters): AccTestConfigura allConfig.serviceAccountGa, allConfig.zone, allConfig.infraProject, - allConfig.vcrBucketName + allConfig.vcrBucketName, + allConfig.credentialsGCS ) } @@ -130,7 +137,8 @@ fun getBetaAcceptanceTestConfig(allConfig: AllContextParameters): AccTestConfigu allConfig.serviceAccountBeta, allConfig.zone, allConfig.infraProject, - allConfig.vcrBucketName + allConfig.vcrBucketName, + allConfig.credentialsGCS ) } @@ -151,7 +159,8 @@ fun getVcrAcceptanceTestConfig(allConfig: AllContextParameters): AccTestConfigur allConfig.serviceAccountVcr, allConfig.zone, allConfig.infraProject, - allConfig.vcrBucketName + allConfig.vcrBucketName, + allConfig.credentialsGCS ) } @@ -226,7 +235,7 @@ fun ParametrizedWithType.vcrEnvironmentVariables(config: AccTestConfiguration, p // ParametrizedWithType.terraformLoggingParameters sets environment variables and build parameters that // affect which logs are shown and allows them to be saved -fun ParametrizedWithType.terraformLoggingParameters(providerName: String) { +fun ParametrizedWithType.terraformLoggingParameters(config: AccTestConfiguration, providerName: String) { // Set logging levels to match old projects text("env.TF_LOG", "DEBUG") text("env.TF_LOG_CORE", "WARN") @@ -234,7 +243,9 @@ fun ParametrizedWithType.terraformLoggingParameters(providerName: String) { // Set where logs are sent text("PROVIDER_NAME", providerName) - text("env.TF_LOG_PATH_MASK", "%system.teamcity.build.checkoutDir%/debug-%PROVIDER_NAME%-%env.BUILD_NUMBER%-%s.txt") // .txt extension used to make artifacts open in browser, instead of download + text("env.TF_LOG_PATH_MASK", "%system.teamcity.build.checkoutDir%/debug-%PROVIDER_NAME%-%env.BUILD_NUMBER%-%teamcity.build.id%-%s.txt") // .txt extension used to make artifacts open in browser, instead of download + + hiddenPasswordVariable("env.GOOGLE_CREDENTIALS_GCS", config.credentialsGCS, "The Google credentials for copying debug logs to the GCS bucket") } fun ParametrizedWithType.readOnlySettings() { diff --git a/mmv1/third_party/terraform/.teamcity/components/builds/build_steps.kt b/mmv1/third_party/terraform/.teamcity/components/builds/build_steps.kt index 7286b2b82f7c..68d9aa0a0206 100644 --- a/mmv1/third_party/terraform/.teamcity/components/builds/build_steps.kt +++ b/mmv1/third_party/terraform/.teamcity/components/builds/build_steps.kt @@ -45,11 +45,11 @@ fun BuildSteps.tagBuildToIndicateTriggerMethod() { TRIGGERED_BY_USERNAME=%teamcity.build.triggeredBy.username% if [[ "${'$'}TRIGGERED_BY_USERNAME" = "n/a" ]] ; then - echo "Build was triggered as part of automated testing. We know this because the `triggeredBy.username` value was `n/a`, value: ${'$'}{TRIGGERED_BY_USERNAME}" + echo "Build was triggered as part of automated testing. We know this because the \`triggeredBy.username\` value was \`n/a\`, value: ${'$'}{TRIGGERED_BY_USERNAME}" TAG="cron-trigger" echo "##teamcity[addBuildTag '${'$'}{TAG}']" else - echo "Build was triggered manually. We know this because `triggeredBy.username` has a non- `n/a` value: ${'$'}{TRIGGERED_BY_USERNAME}" + echo "Build was triggered manually. We know this because \`triggeredBy.username\` has a non- \`n/a\` value: ${'$'}{TRIGGERED_BY_USERNAME}" TAG="manual-trigger" echo "##teamcity[addBuildTag '${'$'}{TAG}']" fi @@ -131,4 +131,47 @@ fun BuildSteps.runAcceptanceTests() { """.trimIndent() }) } +} + +fun BuildSteps.saveArtifactsToGCS() { + step(ScriptBuildStep { + name = "Tasks after running nightly tests: push artifacts(debug logs) to GCS" + scriptContent = """ + #!/bin/bash + echo "Post-test step - storge artifacts(debug logs) to GCS" + + # Authenticate gcloud CLI + echo "${'$'}{GOOGLE_CREDENTIALS_GCS}" > google-account.json + chmod 600 google-account.json + gcloud auth activate-service-account --key-file=google-account.json + + # Get current date for nightly tests + CURRENT_DATE=$(date +"%%Y-%%m-%%d") + // "%%" is used to escape "%" see details at https://www.jetbrains.com/help/teamcity/9.0/defining-and-using-build-parameters-in-build-configuration.html#using-build-parameters-in-build-configuration-settings + + # Detect Trigger Method + TRIGGERED_BY_USERNAME=%teamcity.build.triggeredBy.username% + BRANCH_NAME=%teamcity.build.branch% + if [[ "${'$'}TRIGGERED_BY_USERNAME" = "n/a" ]] ; then + echo "Build was triggered as part of automated testing. We know this because the \`triggeredBy.username\` value was \`n/a\`, value: ${'$'}{TRIGGERED_BY_USERNAME}" + FOLDER="nightly/%teamcity.project.id%/${'$'}{CURRENT_DATE}" + else + echo "Build was triggered manually. We know this because \`triggeredBy.username\` has a non- \`n/a\` value: ${'$'}{TRIGGERED_BY_USERNAME}" + FOLDER="manual/%teamcity.project.id%/${'$'}{BRANCH_NAME}" + fi + + # Copy logs to GCS + gsutil -m cp %teamcity.build.checkoutDir%/debug* gs://teamcity-logs/${'$'}{FOLDER}/%env.BUILD_NUMBER%/ + + # Cleanup + rm google-account.json + gcloud auth application-default revoke + gcloud auth revoke --all + + echo "Finished" + """.trimIndent() + // ${'$'} is required to allow creating a script in TeamCity that contains + // parts like ${GIT_HASH_SHORT} without having Kotlin syntax issues. For more info see: + // https://youtrack.jetbrains.com/issue/KT-2425/Provide-a-way-for-escaping-the-dollar-sign-symbol-in-multiline-strings-and-string-templates + }) } \ No newline at end of file diff --git a/mmv1/third_party/terraform/.teamcity/settings.kts b/mmv1/third_party/terraform/.teamcity/settings.kts index 45600e0b5646..a87b2b8c5d19 100644 --- a/mmv1/third_party/terraform/.teamcity/settings.kts +++ b/mmv1/third_party/terraform/.teamcity/settings.kts @@ -59,6 +59,8 @@ val zone = DslContext.getParameter("zone", "") // GOOGLE_ val infraProject = DslContext.getParameter("infraProject", "") // GOOGLE_INFRA_PROJECT val vcrBucketName = DslContext.getParameter("vcrBucketName", "") // VCR_BUCKET_NAME +// Used for copying nightly + upstream MM debug logs to GCS bucket +val credentialsGCS = DslContext.getParameter("credentialsGCS", "") // GOOGLE_CREDENTIALS_GCS var allContextParams = AllContextParameters( credentialsGa, @@ -90,7 +92,8 @@ var allContextParams = AllContextParameters( region, zone, infraProject, - vcrBucketName + vcrBucketName, + credentialsGCS ) // This is the entry point of the code in .teamcity/ diff --git a/mmv1/third_party/terraform/.teamcity/tests/build_configuration_features.kt b/mmv1/third_party/terraform/.teamcity/tests/build_configuration_features.kt index 7e3cc5b6ff58..d606524fef92 100644 --- a/mmv1/third_party/terraform/.teamcity/tests/build_configuration_features.kt +++ b/mmv1/third_party/terraform/.teamcity/tests/build_configuration_features.kt @@ -62,4 +62,35 @@ class BuildConfigurationFeatureTests { } } } + + @Test + fun nonVCRBuildShouldHaveSaveArtifactsToGCS() { + val project = googleCloudRootProject(testContextParameters()) + + // Find GA nightly test project + var gaNightlyTestProject = getSubProject(project, gaProjectName, nightlyTestsProjectName) + + // Find GA MM Upstream project + var gaMMUpstreamProject = getSubProject(project, gaProjectName, mmUpstreamProjectName) + + // Find Beta nightly test project + var betaNightlyTestProject = getSubProject(project, betaProjectName, nightlyTestsProjectName) + + // Find Beta MM Upstream project + var betaMMUpstreamProject = getSubProject(project, betaProjectName, mmUpstreamProjectName) + + (gaNightlyTestProject.buildTypes + gaMMUpstreamProject.buildTypes + betaNightlyTestProject.buildTypes + betaMMUpstreamProject.buildTypes).forEach{bt -> + var found: Boolean = false + for (item in bt.steps.items) { + if (item.name == "Tasks after running nightly tests: push artifacts(debug logs) to GCS") { + found = true + break + } + } + // service sweeper does not contain push artifacts to GCS step + if (bt.name != "Service Sweeper") { + assertTrue("Build configuration `${bt.name}` contains a build step that pushes artifacts to GCS", found) + } + } + } } diff --git a/mmv1/third_party/terraform/.teamcity/tests/test_utils.kt b/mmv1/third_party/terraform/.teamcity/tests/test_utils.kt index 642829cc9355..05f976436ca1 100644 --- a/mmv1/third_party/terraform/.teamcity/tests/test_utils.kt +++ b/mmv1/third_party/terraform/.teamcity/tests/test_utils.kt @@ -50,7 +50,8 @@ fun testContextParameters(): AllContextParameters { "region", "zone", "infraProject", - "vcrBucketName") + "vcrBucketName", + "credentialsGCS") } fun getSubProject(rootProject: Project, parentProjectName: String, subProjectName: String): Project {