Copy TeamCity debug logs to GCS (GoogleCloudPlatform#10495)

mmv1/third_party/terraform/.teamcity/components/builds/build_configuration_per_package.kt

@@ -71,6 +71,7 @@ class PackageDetails(private val packageName: String, private val displayName: S
                 configureGoEnv()
                 downloadTerraformBinary()
                 runAcceptanceTests()
+                saveArtifactsToGCS()
             }
 
             features {
@@ -88,7 +89,7 @@ class PackageDetails(private val packageName: String, private val displayName: S
             params {
                 configureGoogleSpecificTestParameters(environmentVariables)
                 acceptanceTestBuildParams(parallelism, testPrefix, testTimeout)
-                terraformLoggingParameters(providerName)
+                terraformLoggingParameters(environmentVariables, providerName)
                 terraformCoreBinaryTesting()
                 terraformShouldPanicForSchemaErrors()
                 readOnlySettings()

mmv1/third_party/terraform/.teamcity/components/builds/build_configuration_sweepers.kt

@@ -93,7 +93,7 @@ class SweeperDetails(private val sweeperName: String, private val parentProjectN
                 configureGoogleSpecificTestParameters(environmentVariables)
                 acceptanceTestBuildParams(parallelism, testPrefix, testTimeout)
                 sweeperParameters(sweeperRegions, sweeperRun)
-                terraformLoggingParameters(providerName)
+                terraformLoggingParameters(environmentVariables, providerName)
                 terraformCoreBinaryTesting()
                 terraformShouldPanicForSchemaErrors()
                 readOnlySettings()

mmv1/third_party/terraform/.teamcity/components/builds/build_configuration_vcr_recording.kt

@@ -68,7 +68,7 @@ class VcrDetails(private val providerName: String, private val buildId: String,
                 configureGoogleSpecificTestParameters(environmentVariables)
                 vcrEnvironmentVariables(environmentVariables, providerName)
                 acceptanceTestBuildParams(parallelism, testPrefix, testTimeout)
-                terraformLoggingParameters(providerName)
+                terraformLoggingParameters(environmentVariables, providerName)
                 terraformCoreBinaryTesting()
                 terraformShouldPanicForSchemaErrors()
                 readOnlySettings()

mmv1/third_party/terraform/.teamcity/components/builds/build_parameters.kt

@@ -68,6 +68,9 @@ class AllContextParameters(
     // VCR specific
     val infraProject: String,     // GOOGLE_INFRA_PROJECT
     val vcrBucketName: String,    // VCR_BUCKET_NAME
+
+    // GCS specific (for nightly + upstream MM logs)
+    val credentialsGCS: String,   // GOOGLE_CREDENTIALS_GCS
     )
 
 // AccTestConfiguration is used to easily pass values set via Context Parameters into build configurations.
@@ -90,6 +93,9 @@ class AccTestConfiguration(
     // VCR specific
     val infraProject: String,
     val vcrBucketName: String,
+
+    // GCS specific (for nightly + upstream MM logs)
+    val credentialsGCS: String,
     )
 
 fun getGaAcceptanceTestConfig(allConfig: AllContextParameters): AccTestConfiguration {
@@ -109,7 +115,8 @@ fun getGaAcceptanceTestConfig(allConfig: AllContextParameters): AccTestConfigura
         allConfig.serviceAccountGa,
         allConfig.zone,
         allConfig.infraProject,
-        allConfig.vcrBucketName
+        allConfig.vcrBucketName,
+        allConfig.credentialsGCS
     )
 }
 
@@ -130,7 +137,8 @@ fun getBetaAcceptanceTestConfig(allConfig: AllContextParameters): AccTestConfigu
         allConfig.serviceAccountBeta,
         allConfig.zone,
         allConfig.infraProject,
-        allConfig.vcrBucketName
+        allConfig.vcrBucketName,
+        allConfig.credentialsGCS
     )
 }
 
@@ -151,7 +159,8 @@ fun getVcrAcceptanceTestConfig(allConfig: AllContextParameters): AccTestConfigur
         allConfig.serviceAccountVcr,
         allConfig.zone,
         allConfig.infraProject,
-        allConfig.vcrBucketName
+        allConfig.vcrBucketName,
+        allConfig.credentialsGCS
     )
 }
 
@@ -226,15 +235,17 @@ fun ParametrizedWithType.vcrEnvironmentVariables(config: AccTestConfiguration, p
 
 // ParametrizedWithType.terraformLoggingParameters sets environment variables and build parameters that
 // affect which logs are shown and allows them to be saved
-fun ParametrizedWithType.terraformLoggingParameters(providerName: String) {
+fun ParametrizedWithType.terraformLoggingParameters(config: AccTestConfiguration, providerName: String) {
     // Set logging levels to match old projects
     text("env.TF_LOG", "DEBUG")
     text("env.TF_LOG_CORE", "WARN")
     text("env.TF_LOG_SDK_FRAMEWORK", "INFO")
 
     // Set where logs are sent
     text("PROVIDER_NAME", providerName)
-    text("env.TF_LOG_PATH_MASK", "%system.teamcity.build.checkoutDir%/debug-%PROVIDER_NAME%-%env.BUILD_NUMBER%-%s.txt") // .txt extension used to make artifacts open in browser, instead of download
+    text("env.TF_LOG_PATH_MASK", "%system.teamcity.build.checkoutDir%/debug-%PROVIDER_NAME%-%env.BUILD_NUMBER%-%teamcity.build.id%-%s.txt") // .txt extension used to make artifacts open in browser, instead of download
+
+    hiddenPasswordVariable("env.GOOGLE_CREDENTIALS_GCS", config.credentialsGCS, "The Google credentials for copying debug logs to the GCS bucket")
 }
 
 fun ParametrizedWithType.readOnlySettings() {

mmv1/third_party/terraform/.teamcity/components/builds/build_steps.kt

@@ -45,11 +45,11 @@ fun BuildSteps.tagBuildToIndicateTriggerMethod() {
             TRIGGERED_BY_USERNAME=%teamcity.build.triggeredBy.username%
 
             if [[ "${'$'}TRIGGERED_BY_USERNAME" = "n/a" ]] ; then
-                echo "Build was triggered as part of automated testing. We know this because the `triggeredBy.username` value was `n/a`, value: ${'$'}{TRIGGERED_BY_USERNAME}"
+                echo "Build was triggered as part of automated testing. We know this because the \`triggeredBy.username\` value was \`n/a\`, value: ${'$'}{TRIGGERED_BY_USERNAME}"
                 TAG="cron-trigger"
                 echo "##teamcity[addBuildTag '${'$'}{TAG}']"
             else
-                echo "Build was triggered manually. We know this because `triggeredBy.username` has a non- `n/a` value: ${'$'}{TRIGGERED_BY_USERNAME}"
+                echo "Build was triggered manually. We know this because \`triggeredBy.username\` has a non- \`n/a\` value: ${'$'}{TRIGGERED_BY_USERNAME}"
                 TAG="manual-trigger"
                 echo "##teamcity[addBuildTag '${'$'}{TAG}']"
             fi
@@ -131,4 +131,47 @@ fun BuildSteps.runAcceptanceTests() {
             """.trimIndent()
         })
     }
+}
+
+fun BuildSteps.saveArtifactsToGCS() {
+    step(ScriptBuildStep {
+        name = "Tasks after running nightly tests: push artifacts(debug logs) to GCS"
+        scriptContent = """
+            #!/bin/bash
+            echo "Post-test step - storge artifacts(debug logs) to GCS"
+
+            # Authenticate gcloud CLI
+            echo "${'$'}{GOOGLE_CREDENTIALS_GCS}" > google-account.json
+            chmod 600 google-account.json
+            gcloud auth activate-service-account --key-file=google-account.json
+
+            # Get current date for nightly tests
+            CURRENT_DATE=$(date +"%%Y-%%m-%%d") 
+            // "%%" is used to escape "%" see details at https://www.jetbrains.com/help/teamcity/9.0/defining-and-using-build-parameters-in-build-configuration.html#using-build-parameters-in-build-configuration-settings
+
+            # Detect Trigger Method 
+            TRIGGERED_BY_USERNAME=%teamcity.build.triggeredBy.username%
+            BRANCH_NAME=%teamcity.build.branch%
+            if [[ "${'$'}TRIGGERED_BY_USERNAME" = "n/a" ]] ; then
+                echo "Build was triggered as part of automated testing. We know this because the \`triggeredBy.username\` value was \`n/a\`, value: ${'$'}{TRIGGERED_BY_USERNAME}"
+                FOLDER="nightly/%teamcity.project.id%/${'$'}{CURRENT_DATE}"
+            else
+                echo "Build was triggered manually. We know this because \`triggeredBy.username\` has a non- \`n/a\` value: ${'$'}{TRIGGERED_BY_USERNAME}"
+                FOLDER="manual/%teamcity.project.id%/${'$'}{BRANCH_NAME}"
+            fi
+
+            # Copy logs to GCS
+            gsutil -m cp %teamcity.build.checkoutDir%/debug* gs://teamcity-logs/${'$'}{FOLDER}/%env.BUILD_NUMBER%/
+
+            # Cleanup
+            rm google-account.json
+            gcloud auth application-default revoke
+            gcloud auth revoke --all
+
+            echo "Finished"
+        """.trimIndent()
+        // ${'$'} is required to allow creating a script in TeamCity that contains
+        // parts like ${GIT_HASH_SHORT} without having Kotlin syntax issues. For more info see:
+        // https://youtrack.jetbrains.com/issue/KT-2425/Provide-a-way-for-escaping-the-dollar-sign-symbol-in-multiline-strings-and-string-templates
+    })
 }

mmv1/third_party/terraform/.teamcity/settings.kts

@@ -59,6 +59,8 @@ val zone            = DslContext.getParameter("zone", "")             // GOOGLE_
 val infraProject             = DslContext.getParameter("infraProject", "") // GOOGLE_INFRA_PROJECT
 val vcrBucketName            = DslContext.getParameter("vcrBucketName", "") // VCR_BUCKET_NAME
 
+// Used for copying nightly + upstream MM debug logs to GCS bucket
+val credentialsGCS = DslContext.getParameter("credentialsGCS", "") // GOOGLE_CREDENTIALS_GCS
 
 var allContextParams = AllContextParameters(
     credentialsGa,
@@ -90,7 +92,8 @@ var allContextParams = AllContextParameters(
     region,
     zone,
     infraProject,
-    vcrBucketName
+    vcrBucketName,
+    credentialsGCS
 )
 
 // This is the entry point of the code in .teamcity/

mmv1/third_party/terraform/.teamcity/tests/build_configuration_features.kt

@@ -62,4 +62,35 @@ class BuildConfigurationFeatureTests {
             }
         }
     }
+
+    @Test
+    fun nonVCRBuildShouldHaveSaveArtifactsToGCS() {
+        val project = googleCloudRootProject(testContextParameters())
+
+        // Find GA nightly test project
+        var gaNightlyTestProject = getSubProject(project, gaProjectName, nightlyTestsProjectName)
+
+        // Find GA MM Upstream project
+        var gaMMUpstreamProject = getSubProject(project, gaProjectName, mmUpstreamProjectName)
+
+        // Find Beta nightly test project
+        var betaNightlyTestProject = getSubProject(project, betaProjectName, nightlyTestsProjectName)
+
+        // Find Beta MM Upstream project
+        var betaMMUpstreamProject = getSubProject(project, betaProjectName, mmUpstreamProjectName)
+
+        (gaNightlyTestProject.buildTypes + gaMMUpstreamProject.buildTypes + betaNightlyTestProject.buildTypes + betaMMUpstreamProject.buildTypes).forEach{bt ->
+            var found: Boolean = false
+            for (item in bt.steps.items) {
+                if (item.name == "Tasks after running nightly tests: push artifacts(debug logs) to GCS") {
+                    found = true
+                    break
+                }
+            }
+            // service sweeper does not contain push artifacts to GCS step
+            if (bt.name != "Service Sweeper") {
+                assertTrue("Build configuration `${bt.name}` contains a build step that pushes artifacts to GCS", found)
+            }
+        }
+    }
 }

mmv1/third_party/terraform/.teamcity/tests/test_utils.kt

@@ -50,7 +50,8 @@ fun testContextParameters(): AllContextParameters {
         "region",
         "zone",
         "infraProject",
-        "vcrBucketName")
+        "vcrBucketName",
+        "credentialsGCS")
 }
 
 fun getSubProject(rootProject: Project, parentProjectName: String, subProjectName: String): Project {