Skip to content

Commit

Permalink
Working groovy scripts
Browse files Browse the repository at this point in the history
  • Loading branch information
@disa6302
disa6302 committed Oct 7, 2020
1 parent 71bd4ae commit a47e088
Show file tree
Hide file tree
Showing 4 changed files with 91 additions and 102 deletions.
3 changes: 1 addition & 2 deletions ...mer/src/main/java/com/amazon/kinesis/video/canary/consumer/ProducerSdkCanaryConsumer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@

@Slf4j
public class ProducerSdkCanaryConsumer {
static ContinuousGetMediaWorker getMediaWorker;
public static void main(final String[] args) throws Exception {
String streamNamePrefix = System.getenv("CANARY_STREAM_NAME");
String canaryType = System.getenv("CANARY_TYPE");
Expand Down Expand Up @@ -60,7 +59,7 @@ public void process(InputStream inputStream, FragmentMetadataCallback fragmentMe
};
}
};
getMediaWorker = ContinuousGetMediaWorker.create(Regions.fromName(region),
ContinuousGetMediaWorker getMediaWorker = ContinuousGetMediaWorker.create(Regions.fromName(region),
credentialsProvider, streamName, new StartSelector().withStartSelectorType(StartSelectorType.NOW),
amazonKinesisVideo,
consumerFactory);
Expand Down
26 changes: 0 additions & 26 deletions ...sdk-canary-consumer/src/main/java/com/amazon/kinesis/video/canary/consumer/Terminate.java
View file

This file was deleted.

128 changes: 68 additions & 60 deletions producer-c/producer-cloudwatch-integ/jobs/runner.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,16 @@ import jenkins.model.*
import org.jenkinsci.plugins.workflow.steps.FlowInterruptedException

START_TIMESTAMP = new Date().getTime()
RUNNING_NODES_IN_BUILDING = 0
HAS_ERROR = false

GIT_URL='https://github.com/aws-samples/amazon-kinesis-video-streams-demos.git'
GIT_HASH='producer'

RUNNING_NODES=0

JAR_FILES=""
CLASSPATH_VALUES=""

producerBuilt=false
def buildProducer() {
sh """
cd $WORKSPACE/producer-c/producer-cloudwatch-integ &&
Expand All @@ -23,7 +23,6 @@ def buildProducer() {
sudo cmake .. &&
sudo make
"""
producerBuilt=true
}

def buildConsumer() {
Expand All @@ -36,11 +35,6 @@ def buildConsumer() {
export PATH
cd $WORKSPACE/consumer-java/aws-kinesis-video-producer-sdk-canary-consumer
mvn package
# Create a temporary filename in /tmp directory
JAR_FILES=$(mktemp)
# Create classpath string of dependencies from the local repository to a file
mvn -Dmdep.outputFile=$JAR_FILES dependency:build-classpath
CLASSPATH_VALUES=$(cat ${JAR_FILES})
'''
}

Expand All @@ -53,7 +47,6 @@ def runProducer(envs) {
secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'
]
]
RUNNING_NODES_IN_BUILDING++
withEnv(envs) {
withCredentials(credentials) {
try {
Expand Down Expand Up @@ -82,13 +75,26 @@ def runConsumer(envs) {
secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'
]
]

withEnv(envs) {
withCredentials(credentials) {
try {
sh """
java -classpath target/aws-kinesisvideo-producer-sdk-canary-consumer-1.0-SNAPSHOT.jar:$CLASSPATH_VALUES -Daws.accessKeyId=$AWS_ACCESS_KEY_ID -Daws.secretKey=$AWS_SECRET_ACCESS_KEY com.amazon.kinesis.video.canary.consumer.ProducerSdkCanaryConsumer "${envs.CANARY_STREAM_NAME}" "${envs.CANARY_TYPE}" "${envs.FRAGMENT_SIZE_IN_BYTES}" "us-west-2"
"""
sh '''
echo "Building comsumer"
JAVA_HOME='/opt/jdk-13.0.1'
PATH="$JAVA_HOME/bin:$PATH"
export PATH
M2_HOME='/opt/apache-maven-3.6.3'
PATH="$M2_HOME/bin:$PATH"
export PATH
cd $WORKSPACE/consumer-java/aws-kinesis-video-producer-sdk-canary-consumer
# Create a temporary filename in /tmp directory
JAR_FILES=$(mktemp)
# Create classpath string of dependencies from the local repository to a file
mvn -Dmdep.outputFile=$JAR_FILES dependency:build-classpath
CLASSPATH_VALUES=$(cat $JAR_FILES)
java -classpath target/aws-kinesisvideo-producer-sdk-canary-consumer-1.0-SNAPSHOT.jar:${CLASSPATH_VALUES} -Daws.accessKeyId=$AWS_ACCESS_KEY_ID -Daws.secretKey=$AWS_SECRET_ACCESS_KEY com.amazon.kinesis.video.canary.consumer.ProducerSdkCanaryConsumer
'''
} catch (FlowInterruptedException err) {
echo 'Aborted consumer due to cancellation'
throw err
Expand All @@ -105,9 +111,11 @@ def runClient(isProducer, params) {
def envs = [
'AWS_KVS_LOG_LEVEL': params.AWS_KVS_LOG_LEVEL,
'CANARY_STREAM_NAME': "${env.JOB_NAME}",
'CANARY_LABEL': "${params.RUNNER_LABEL}",
'CANARY_TYPE': params.CANARY_TYPE,
'FRAGMENT_SIZE_IN_BYTES' : params.FRAGMENT_SIZE_IN_BYTES,
'CANARY_DURATION_IN_SECONDS': params.CANARY_DURATION_IN_SECONDS
'CANARY_DURATION_IN_SECONDS': params.CANARY_DURATION_IN_SECONDS,
'AWS_DEFAULT_REGION': params.AWS_DEFAULT_REGION,
].collect({k,v -> "${k}=${v}" })

def credentials = [
Expand All @@ -119,17 +127,34 @@ def runClient(isProducer, params) {
]
]

RUNNING_NODES_IN_BUILDING++

def consumerStartUpDelay = 10
def consumerStartUpDelay = 30
echo "NODE_NAME = ${env.NODE_NAME}"
checkout([
scm: [
$class: 'GitSCM',
branches: [[name: GIT_HASH]],
userRemoteConfigs: [[url: GIT_URL]]
]
])
RUNNING_NODES++
echo "Number of running nodes: ${RUNNING_NODES}"
if(isProducer) {
buildProducer()
}
else {
// This is to make sure that the consumer does not make RUNNING_NODES
// zero before producer build starts. Should handle this in a better
// way
buildConsumer()
}
RUNNING_NODES--
echo "Number of running nodes after build: ${RUNNING_NODES}"
waitUntil {
RUNNING_NODES == 0
}


echo "Done waiting in NODE_NAME = ${env.NODE_NAME}"
if(!isProducer) {
waitUntil {
producerBuilt == true
}
producerBuilt = false
sleep consumerStartUpDelay
// Run consumer
runConsumer(envs)
}
Expand All @@ -149,11 +174,12 @@ pipeline {
string(name: 'CONSUMER_NODE_LABEL')
string(name: 'GIT_URL')
string(name: 'GIT_HASH')
string(name: 'CANARY_STREAM_NAME')
string(name: 'CANARY_TYPE')
string(name: 'FRAGMENT_SIZE_IN_BYTES')
string(name: 'RUNNER_LABEL')
string(name: 'CANARY_DURATION_IN_SECONDS')
string(name: 'MIN_RETRY_DELAY_IN_SECONDS')
string(name: 'AWS_DEFAULT_REGION')
}

stages {
Expand All @@ -167,20 +193,12 @@ pipeline {
parallel {
stage('Producer') {
steps {
script{
checkout([
scm: [
$class: 'GitSCM',
branches: [[name: GIT_HASH]],
userRemoteConfigs: [[url: GIT_URL]]]
])
echo "NODE_NAME = ${env.NODE_NAME}"
echo 'Build Producer'
buildProducer()
PRODUCER_NODE=env.NODE_NAME
}
script {
runClient(true, params)
sh """
cd $WORKSPACE
sudo rm -r *
"""
}
}
}
Expand All @@ -189,21 +207,12 @@ pipeline {
label params.CONSUMER_NODE_LABEL
}
steps {
script{
echo env.NODE_NAME
checkout([
scm: [
$class: 'GitSCM',
branches: [[name: GIT_HASH]],
userRemoteConfigs: [[url: GIT_URL]]]
])
echo "NODE_NAME = ${env.NODE_NAME}"
echo 'Build Consumer'
buildConsumer()
CONSUMER_NODE=env.NODE_NAME
}
script {
runClient(false, params)
sh """
cd $WORKSPACE
sudo rm -r *
"""
}
}
}
Expand All @@ -220,25 +229,24 @@ pipeline {
sleep Math.max(0, params.MIN_RETRY_DELAY_IN_SECONDS.toInteger() - currentBuild.duration.intdiv(1000))
}
}

stage('Reschedule') {
steps {
// TODO: Maybe there's a better way to write this instead of duplicating it
build(
job: env.JOB_NAME,
parameters: [
parameters: [
string(name: 'AWS_KVS_LOG_LEVEL', value: "2"),
string(name: 'GIT_URL', value: GIT_URL),
string(name: 'GIT_HASH', value: GIT_HASH),
string(name: 'CANARY_DURATION_IN_SECONDS', value: CANARY_DURATION_IN_SECONDS.toString()),
string(name: 'PRODUCER_NODE_LABEL', value: params.PRODUCER_NODE_LABEL),
string(name: 'CONSUMER_NODE_LABEL', value: params.CONSUMER_NODE_LABEL),
string(name: 'GIT_URL', value: params.GIT_URL),
string(name: 'GIT_HASH', value: params.GIT_HASH),
string(name: 'CANARY_TYPE', value: params.CANARY_TYPE),
string(name: 'FRAGMENT_SIZE_IN_BYTES', value: params.FRAGMENT_SIZE_IN_BYTES),
string(name: 'CANARY_DURATION_IN_SECONDS', value: params.CANARY_DURATION_IN_SECONDS),
string(name: 'MIN_RETRY_DELAY_IN_SECONDS', value: params.MIN_RETRY_DELAY_IN_SECONDS),
string(name: 'FRAGMENT_SIZE_IN_BYTES', value: params.FRAGMENT_SIZE_IN_BYTES.toString()),
string(name: 'PRODUCER_NODE_LABEL', value: "producer-uw2"),
string(name: 'CONSUMER_NODE_LABEL', value: "consumer-uw2"),
string(name: 'RUNNER_LABEL', value: "Periodic"),
string(name: 'AWS_DEFAULT_REGION', value: params.AWS_DEFAULT_REGION),
string(name: 'RUNNER_LABEL', value: params.RUNNER_LABEL),
],
],
wait: false
)
}
Expand Down
36 changes: 22 additions & 14 deletions producer-c/producer-cloudwatch-integ/jobs/setup.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,15 @@ WORKSPACE_CONSUMER="consumer-java/aws-kinesis-video-producer-sdk-canary-consumer
GIT_URL='https://github.com/aws-samples/amazon-kinesis-video-streams-demos.git'
GIT_HASH='producer'
RUNNER_JOB_NAME_PREFIX = "producer-runner"
CANARY_DURATION_IN_SECONDS = 120

// TODO: Set up configurability to run different parameter combinations
// Run long run canary for 12 hours. New changes in the SDK will be pulled
// in every 12 hours
CANARY_DURATION_IN_SECONDS = 12 * 60 * 60
COLD_STARTUP_DELAY_IN_SECONDS = 60 * 60
MIN_RETRY_DELAY_IN_SECONDS = 60
FRAGMENT_SIZE_IN_BYTES = 1048576
AWS_DEFAULT_REGION = "us-west-2"

JAR_FILES=""
CLASSPATH_VALUES=""
Expand Down Expand Up @@ -119,24 +124,27 @@ pipeline {
string(name: 'PRODUCER_NODE_LABEL', value: "producer-uw2"),
string(name: 'CONSUMER_NODE_LABEL', value: "consumer-uw2"),
string(name: 'CANARY_TYPE', value: "realtime"),
string(name: 'RUNNER_LABEL', value: "Periodic"),
string(name: 'RUNNER_LABEL', value: "Longrun"),
string(name: 'FRAGMENT_SIZE_IN_BYTES', value: FRAGMENT_SIZE_IN_BYTES.toString()),
string(name: 'AWS_DEFAULT_REGION', value: AWS_DEFAULT_REGION),
],
wait: false
)

build(
job: NEXT_AVAILABLE_RUNNER,
parameters: COMMON_PARAMS + [
string(name: 'CANARY_DURATION_IN_SECONDS', value: CANARY_DURATION_IN_SECONDS.toString()),
string(name: 'PRODUCER_NODE_LABEL', value: "producer-uw2"),
string(name: 'CONSUMER_NODE_LABEL', value: "consumer-uw2"),
string(name: 'CANARY_TYPE', value: "realtime"),
string(name: 'RUNNER_LABEL', value: "Periodic"),
string(name: 'FRAGMENT_SIZE_IN_BYTES', value: FRAGMENT_SIZE_IN_BYTES.toString()),
],
wait: false
)
// TODO: Will enable this once instance in other regions are set up
// build(
// job: NEXT_AVAILABLE_RUNNER,
// parameters: COMMON_PARAMS + [
// string(name: 'CANARY_DURATION_IN_SECONDS', value: CANARY_DURATION_IN_SECONDS.toString()),
// string(name: 'PRODUCER_NODE_LABEL', value: "producer-uw2"),
// string(name: 'CONSUMER_NODE_LABEL', value: "consumer-uw2"),
// string(name: 'CANARY_TYPE', value: "realtime"),
// string(name: 'RUNNER_LABEL', value: "crossregion"),
// string(name: 'FRAGMENT_SIZE_IN_BYTES', value: FRAGMENT_SIZE_IN_BYTES.toString()),
// string(name: 'AWS_DEFAULT_REGION', value: AWS_DEFAULT_REGION),
// ],
// wait: false
// )
}
}

Expand Down

0 comments on commit a47e088

Please sign in to comment.