*: setup AWS kola job & move aws replication to release job

Creates a new job specifically for running the kola tests on AWS. As a
result moves the AWS replication out of the main `Jenkinsfile` into the
`Jenkinsfile.release`.

We'll likely have to move the AWS replication to it's own job down the
line if we want to start signing the `meta.json` / `release.json` files.

Jenkinsfile

@@ -248,18 +248,6 @@ podTemplate(cloud: 'openshift', label: 'coreos-assembler', yaml: pod, defaultCon
                         --grant-user ${FEDORA_AWS_TESTING_USER_ID}
                     """)
                 }
-
-                // Replicate the newly uploaded AMI to other regions. Intentionally
-                // split out from the 'Upload AWS' stage to allow for tests to be added
-                // at a later date before replicating said image.
-                //
-                // TODO: move to separate job (release?) along with code to buildprep,
-                // regenerate the release.json, and buildupload said meta & release.json
-              //stage('Replicate AWS AMI') {
-              //    utils.shwrap("""
-              //    coreos-assembler aws-replicate --build=${newBuildID}
-              //    """)
-              //}
             }
         }
 
@@ -298,6 +286,19 @@ podTemplate(cloud: 'openshift', label: 'coreos-assembler', yaml: pod, defaultCon
             }
         }
 
+
+        if (!params.MINIMAL && s3_stream_dir) {
+            stage('Kola Runs') {
+                container('jnlp') {
+                    utils.shwrap("""
+                        oc start-build fedora-coreos-pipeline-kola-aws \
+                            -e VERSION=${newBuildID} \
+                            -e S3_STREAM_DIR=${s3_stream_dir}
+                    """)
+                }
+            }
+        }
+
         // For now, we auto-release all non-production streams builds. That
         // way, we can e.g. test testing-devel AMIs easily.
         if (official && !(params.STREAM in streams.production)) {

Jenkinsfile.kola.aws

@@ -0,0 +1,69 @@
+def utils, streams
+node {
+    checkout scm
+    utils = load("utils.groovy")
+    streams = load("streams.groovy")
+    pod = readFile(file: "manifests/pod.yaml")
+}
+
+properties([
+    pipelineTriggers([]),
+    parameters([
+      string(name: 'VERSION',
+             description: 'Fedora CoreOS Build ID to test',
+             defaultValue: '',
+             trim: true),
+      string(name: 'S3_STREAM_DIR',
+             description: 'Fedora CoreOS S3 Stream Directory',
+             defaultValue: '',
+             trim: true)
+    ])
+])
+
+currentBuild.description = "[${params.VERSION}] Running"
+
+// substitute the right COSA image into the pod definition before spawning it
+pod = pod.replace("COREOS_ASSEMBLER_IMAGE", "coreos-assembler:master")
+
+podTemplate(cloud: 'openshift', label: 'coreos-assembler', yaml: pod, defaultContainer: 'jnlp') {
+    node('coreos-assembler') { container('coreos-assembler') {
+        def ami, ami_region
+        def no_ami = false
+
+        stage('Fetch Metadata') {
+            utils.shwrap("""
+            coreos-assembler buildprep s3://${params.S3_STREAM_DIR}/builds
+            """)
+
+            def basearch = utils.shwrap_capture("coreos-assembler basearch")
+            def meta_json = "builds/${params.VERSION}/${basearch}/meta.json"
+            def meta = readJSON file: meta_json
+            if (meta.amis.size() > 0) {
+                ami = meta['amis'][0]['hvm']
+                ami_region = meta['amis'][0]['name']
+            } else {
+                no_ami = true
+            }
+        }
+
+        // fail immediately if the build contained no AMIs
+        if (no_ami) {
+          currentBuild.result = 'FAILURE'
+          return
+        }
+
+        stage('AWS Kola Run') {
+          utils.shwrap("""
+          kola run -p aws --aws-ami ${ami} --aws-region ${ami_region} -b fcos -j 10 || :
+          tar -cf - _kola_temp/ | xz -c9 > _kola_temp.tar.xz
+          """)
+          archiveArtifacts "_kola_temp.tar.xz"
+        }
+
+        def report = readJSON file: "_kola_temp/aws-latest/reports/report.json"
+        if (report["result"] != "PASS") {
+          currentBuild.result = 'FAILURE'
+          return
+        }
+    }}
+}

Jenkinsfile.release

@@ -30,6 +30,26 @@ pod = pod.replace("COREOS_ASSEMBLER_IMAGE", "coreos-assembler:master")
 
 podTemplate(cloud: 'openshift', label: 'coreos-assembler', yaml: pod, defaultContainer: 'jnlp') {
     node('coreos-assembler') { container('coreos-assembler') {
+        // Replicate the newly uploaded AMI to other regions. Intentionally
+        // split out from the 'Upload AWS' stage to allow for tests to be added
+        // at a later date before replicating said image.
+        //
+        // We have to re-run the coreos-meta-translator as aws-replicate
+        // only modifies the meta.json
+        stage('Replicate AWS AMI') {
+            s3_stream_dir = "${s3_bucket}/prod/streams/${params.STREAM}"
+            // TODO: Once buildprep supports pulling specific builds
+            // operate on the specific build rather than the most
+            // recent build
+            utils.shwrap("""
+            coreos-assembler buildprep s3://${s3_stream_dir}/builds
+            coreos-assembler aws-replicate --build=${params.VERSION}
+            git clone https://github.com/coreos/fedora-coreos-releng-automation /var/tmp/fcos-releng
+            /var/tmp/fcos-releng/coreos-meta-translator/trans.py --workdir .
+            coreos-assembler buildupload --skip-builds-json s3 --acl=public-read ${s3_stream_dir}/builds
+            """)
+        }
+
         stage('Publish') {
           // Run plume to publish official builds; This will handle modifying
           // object ACLs, modifying AMI image attributes,

manifests/pipeline.yaml

@@ -308,3 +308,22 @@ objects:
         jenkinsPipelineStrategy:
           type: JenkinsPipeline
           jenkinsfilePath: Jenkinsfile.stream.metadata.generator
+
+  ### FEDORA COREOS TEST PIPELINES ###
+
+  - kind: BuildConfig
+    apiVersion: "build.openshift.io/v1"
+    metadata:
+      name: fedora-coreos-pipeline-kola-aws
+      annotations:
+        coreos.com/deploy-default: "false"
+    spec:
+      source:
+        type: Git
+        git:
+          uri: ${PIPELINE_REPO_URL}
+          ref: ${PIPELINE_REPO_REF}
+      strategy:
+        jenkinsPipelineStrategy:
+          type: JenkinsPipeline
+          jenkinsfilePath: Jenkinsfile.kola.aws