Working groovy scripts

aws-samples · Oct 14, 2020 · eb936a1 · eb936a1
1 parent 98d5b62
commit eb936a1
Show file tree

Hide file tree

Showing 6 changed files with 121 additions and 193 deletions.
diff --git a/consumer-java/aws-kinesis-video-producer-sdk-canary-consumer/run-consumer.sh b/consumer-java/aws-kinesis-video-producer-sdk-canary-consumer/run-consumer.sh
@@ -1,8 +1,7 @@
 mvn package
 # Create a temporary filename in /tmp directory
-jar_files=$(mktemp)
+touch tmp_jar
 # Create classpath string of dependencies from the local repository to a file
-mvn -Dmdep.outputFile=$jar_files dependency:build-classpath
-classpath_values=$(cat $jar_files)
+mvn -Dmdep.outputFile=tmp_jar dependency:build-classpath
 # Start the consumer
-java -classpath target/aws-kinesisvideo-producer-sdk-canary-consumer-1.0-SNAPSHOT.jar:$classpath_values -Daws.accessKeyId=${AWS_ACCESS_KEY_ID} -Daws.secretKey=${AWS_SECRET_ACCESS_KEY} com.amazon.kinesis.video.canary.consumer.ProducerSdkCanaryConsumer
+java -classpath target/aws-kinesisvideo-producer-sdk-canary-consumer-1.0-SNAPSHOT.jar:$(cat tmp_jar) -Daws.accessKeyId=${AWS_ACCESS_KEY_ID} -Daws.secretKey=${AWS_SECRET_ACCESS_KEY} com.amazon.kinesis.video.canary.consumer.ProducerSdkCanaryConsumer
diff --git a/...mer/src/main/java/com/amazon/kinesis/video/canary/consumer/ProducerSdkCanaryConsumer.java b/...mer/src/main/java/com/amazon/kinesis/video/canary/consumer/ProducerSdkCanaryConsumer.java
@@ -25,7 +25,6 @@
 
 @Slf4j
 public class ProducerSdkCanaryConsumer {
-    static ContinuousGetMediaWorker getMediaWorker;
     public static void main(final String[] args) throws Exception {
         String streamNamePrefix = System.getenv("CANARY_STREAM_NAME");
         String canaryType = System.getenv("CANARY_TYPE");
@@ -60,7 +59,7 @@ public void process(InputStream inputStream, FragmentMetadataCallback fragmentMe
                 };
             }
         };
-        getMediaWorker = ContinuousGetMediaWorker.create(Regions.fromName(region),
+        ContinuousGetMediaWorker getMediaWorker = ContinuousGetMediaWorker.create(Regions.fromName(region),
                 credentialsProvider, streamName, new StartSelector().withStartSelectorType(StartSelectorType.NOW),
                 amazonKinesisVideo,
                 consumerFactory);

diff --git a/...sdk-canary-consumer/src/main/java/com/amazon/kinesis/video/canary/consumer/Terminate.java b/...sdk-canary-consumer/src/main/java/com/amazon/kinesis/video/canary/consumer/Terminate.java
diff --git a/producer-c/producer-cloudwatch-integ/jobs/canary_seed.groovy b/producer-c/producer-cloudwatch-integ/jobs/canary_seed.groovy
@@ -35,18 +35,19 @@ approveSignatures([
     "staticField java.lang.Long MAX_VALUE"
 ])
 
-pipelineJob("${NAMESPACE}-setup") {
+pipelineJob("${NAMESPACE}-orchestrator") {
 	description('Run producer and consumer')
 	logRotator {
 		daysToKeep(DAYS_TO_KEEP_LOGS)
 		numToKeep(NUMBER_OF_LOGS)
-	}	
-	throttleConcurrentBuilds {
-		maxPerNode(MAX_EXECUTION_PER_NODE)
 	}
+    throttleConcurrentBuilds {
+        maxPerNode(MAX_EXECUTION_PER_NODE)
+    }
+
 	definition {
         cps {
-        	script(readFileFromWorkspace("${GROOVY_SCRIPT_DIR}/setup.groovy"))
+        	script(readFileFromWorkspace("${GROOVY_SCRIPT_DIR}/orchestrator.groovy"))
             sandbox()
         }
     }
@@ -58,9 +59,6 @@ pipelineJob("${NAMESPACE}-runner-0") {
 		daysToKeep(DAYS_TO_KEEP_LOGS)
 		numToKeep(NUMBER_OF_LOGS)
 	}
-	throttleConcurrentBuilds {
-		maxPerNode(MAX_EXECUTION_PER_NODE)
-	}
 	definition {
         cps {
         	script(readFileFromWorkspace("${GROOVY_SCRIPT_DIR}/runner.groovy"))
@@ -75,9 +73,6 @@ pipelineJob("${NAMESPACE}-runner-1") {
         daysToKeep(DAYS_TO_KEEP_LOGS)
         numToKeep(NUMBER_OF_LOGS)
     }
-    throttleConcurrentBuilds {
-        maxPerNode(MAX_EXECUTION_PER_NODE)
-    }
     definition {
         cps {
             script(readFileFromWorkspace("${GROOVY_SCRIPT_DIR}/runner.groovy"))
@@ -86,4 +81,4 @@ pipelineJob("${NAMESPACE}-runner-1") {
     }
 }
 
-queue("${NAMESPACE}-setup")
+queue("${NAMESPACE}-orchestrator")
diff --git a/...oducer-cloudwatch-integ/jobs/setup.groovy → ...cloudwatch-integ/jobs/orchestrator.groovy b/...oducer-cloudwatch-integ/jobs/setup.groovy → ...cloudwatch-integ/jobs/orchestrator.groovy
@@ -5,14 +5,15 @@ WORKSPACE_CONSUMER="consumer-java/aws-kinesis-video-producer-sdk-canary-consumer
 GIT_URL='https://github.com/aws-samples/amazon-kinesis-video-streams-demos.git'
 GIT_HASH='producer'
 RUNNER_JOB_NAME_PREFIX = "producer-runner"
-CANARY_DURATION_IN_SECONDS = 120
+
+// TODO: Set up configurability to run different parameter combinations
+// Run long run canary for 12 hours. New changes in the SDK will be pulled
+// in every 12 hours
+CANARY_DURATION_IN_SECONDS = 12 * 60 * 60
 COLD_STARTUP_DELAY_IN_SECONDS = 60 * 60
 MIN_RETRY_DELAY_IN_SECONDS = 60
 FRAGMENT_SIZE_IN_BYTES = 1048576
-
-JAR_FILES=""
-CLASSPATH_VALUES=""
-
+AWS_DEFAULT_REGION = "us-west-2"
 
 COMMON_PARAMS = [
     string(name: 'AWS_KVS_LOG_LEVEL', value: "2"),
@@ -109,31 +110,22 @@ pipeline {
                         script {
                             echo "New runner: ${NEXT_AVAILABLE_RUNNER}"
                             Jenkins.instance.getItemByFullName(NEXT_AVAILABLE_RUNNER).setDisabled(false)
-                        }
 
-                        // TODO: Use matrix to spawn runners
-                        build(
-                            job: NEXT_AVAILABLE_RUNNER,
-                            parameters: COMMON_PARAMS + [
-                                string(name: 'CANARY_DURATION_IN_SECONDS', value: CANARY_DURATION_IN_SECONDS.toString()),
-                                string(name: 'PRODUCER_NODE_LABEL', value: "producer-uw2"),
-                                string(name: 'CONSUMER_NODE_LABEL', value: "consumer-uw2"),
-                                string(name: 'CANARY_TYPE', value: "realtime"),
-                                string(name: 'RUNNER_LABEL', value: "Periodic"),
-                                string(name: 'FRAGMENT_SIZE_IN_BYTES', value: FRAGMENT_SIZE_IN_BYTES.toString()),
-                            ],
-                            wait: false
-                        )
+                            def gitHash = sh(returnStdout: true, script: 'git rev-parse HEAD')
+                            COMMON_PARAMS << string(name: 'GIT_HASH', value: gitHash)
+                        }
 
+                        // TODO: Use matrix to provide configurability in parameters
                         build(
                             job: NEXT_AVAILABLE_RUNNER,
                             parameters: COMMON_PARAMS + [
                                 string(name: 'CANARY_DURATION_IN_SECONDS', value: CANARY_DURATION_IN_SECONDS.toString()),
                                 string(name: 'PRODUCER_NODE_LABEL', value: "producer-uw2"),
                                 string(name: 'CONSUMER_NODE_LABEL', value: "consumer-uw2"),
-                                string(name: 'CANARY_TYPE', value: "realtime"),
-                                string(name: 'RUNNER_LABEL', value: "Periodic"),
+                                string(name: 'CANARY_TYPE', value: "Realtime"),
+                                string(name: 'RUNNER_LABEL', value: "Longrun"),
                                 string(name: 'FRAGMENT_SIZE_IN_BYTES', value: FRAGMENT_SIZE_IN_BYTES.toString()),
+                                string(name: 'AWS_DEFAULT_REGION', value: AWS_DEFAULT_REGION),
                             ],
                             wait: false
                         )