Updated sensors build

- Cleanup sensors external assets folder to remove unused sensor code from previous builds - Update SIMULATOR_SENSORS environment variables with actual value from build parameter. - Specify Sensor repo GIT reference in form "RepoName:branch_or_tag"
lgsvl · Apr 17, 2020 · 230c1fa · 230c1fa
1 parent 6abe12b
commit 230c1fa
Show file tree

Hide file tree

Showing 4 changed files with 63 additions and 18 deletions.
diff --git a/Jenkins/Jenkinsfile b/Jenkins/Jenkinsfile
@@ -25,7 +25,7 @@ pipeline {
     booleanParam(name: "BUILD_LINUX", defaultValue: true, description: "")
     booleanParam(name: "BUILD_MACOS", defaultValue: false, description: "")
     string(name: "CLOUD_URL", defaultValue: "${SIMULATOR_STAGING_CLOUD_URL}", description: "")
-    string(name: "BUILD_SIMULATOR_SENSORS", defaultValue: "${SIMULATOR_SENSORS}", description: "List of sensors to build from https://auto-gitlab.lgsvl.net/HDRP/Sensors")
+    string(name: "BUILD_SIMULATOR_SENSORS", defaultValue: "${SIMULATOR_SENSORS_RELEASE}", description: "List of sensors to build")
   }
 
   environment {
@@ -38,6 +38,8 @@ pipeline {
     JENKINS_BUILD_ID = "${BUILD_ID}"
     GIT_BRANCH = "${BRANCH_NAME}"
     CODE_SIGNING_FILE = "/dev/urandom"
+    SIMULATOR_ENVIRONMENTS = "${SIMULATOR_ENVIRONMENTS_RELEASE}"
+    SIMULATOR_SENSORS = "${BUILD_SIMULATOR_SENSORS}"
   }
 
   stages {
@@ -59,6 +61,12 @@ pipeline {
           ]]
         ])
 
+        sh label:"Cleanup Assets/External/Sensors", script:"""
+            cd Assets/External/Sensors
+            find .  -mindepth 1 -maxdepth 1  -type d | xargs rm -rf
+            git clean -xdf .
+        """
+
         script {
 
           def environments = env.SIMULATOR_ENVIRONMENTS.split(',')
@@ -118,12 +126,24 @@ pipeline {
             ])
           }
 
-          def sensors = env.BUILD_SIMULATOR_SENSORS.split(',')
+          def sensors = env.SIMULATOR_SENSORS.split(',')
+          def sim_sensors = []
+
           for (int i=0; i<sensors.size(); i++) {
-            def sensor = sensors[i]
+            def sensor_git_ref = sensors[i].split(":")
+
+            def sensor = sensor_git_ref[0]
+            def git_ref = "refs/heads/master"
+
+            if (sensor_git_ref.size() > 1) {
+                git_ref = sensor_git_ref[1]
+            }
+
+            sim_sensors.add(sensor)
+
             checkout([
               $class: "GitSCM",
-              branches: [[name: "refs/heads/master"]],
+              branches: [[name: "${git_ref}"]],
               browser: [$class: "GitLab", repoUrl: "https://${GITLAB_HOST}/HDRP/Sensors/${sensor}", version: env.GITLAB_VERSION],
               extensions: [
                 [$class: "RelativeTargetDirectory", relativeTargetDir: "Assets/External/Sensors/${sensor}"],
@@ -141,9 +161,12 @@ pipeline {
           env.GIT_COMMIT = sh(returnStdout: true, script: "git rev-parse HEAD").trim()
           env.SIM_ENVIRONMENTS = sh(returnStdout: true, script: "./Jenkins/get-assets.sh Assets/External/Environments ${S3_DOWNLOAD_HOST} environment 1").trim()
           env.SIM_VEHICLES = sh(returnStdout: true, script: "./Jenkins/get-assets.sh Assets/External/Vehicles ${S3_DOWNLOAD_HOST} vehicle 1").trim()
+          env.SIM_SENSORS = sim_sensors.join(",")
         } // script
 
         sh "rm -rf *.zip *.html *.xml"
+
+        sh label:"SIMULATOR env vars", script:"printenv | grep -E 'SIMULATOR|^SIM_' | sort"
       }
     }
 

diff --git a/Jenkins/Jenkinsfile-Release b/Jenkins/Jenkinsfile-Release
@@ -26,13 +26,16 @@ pipeline {
     GIT_TAG = "${GIT_TAG}"
     CODE_SIGNING_FILE = "/dev/urandom"
     FORCE_REBUILD = "${FORCE_REBUILD}"
-  }
-
-  parameters {
-    string(name: "SIMULATOR_SENSORS", defaultValue: "CheckControlSensor", description: "List of sensors to build from https://auto-gitlab.lgsvl.net/HDRP/Sensors")
+    SIMULATOR_ENVIRONMENTS = "${SIMULATOR_ENVIRONMENTS_RELEASE}"
+    SIMULATOR_SENSORS = "${SIMULATOR_SENSORS_RELEASE}"
   }
 
   stages {
+    stage("Build environment") {
+        steps {
+            sh label:"SIMULATOR env vars", script:"printenv | sort | grep SIMULATOR"
+        }
+    }
 
     stage("Git") {
       steps {
@@ -60,9 +63,11 @@ pipeline {
           ]]
         ])
 
-        script {
-          env.GIT_COMMIT = sh(returnStdout: true, script: "git rev-parse HEAD").trim()
-        }
+        sh label:"Cleanup Assets/External/Sensors", script:"""
+            cd Assets/External/Sensors
+            find .  -mindepth 1 -maxdepth 1  -type d | xargs rm -rf
+            git clean -xdf .
+        """
 
         script {
 
@@ -124,11 +129,23 @@ pipeline {
           }
 
           def sensors = env.SIMULATOR_SENSORS.split(',')
+          def sim_sensors = []
+
           for (int i=0; i<sensors.size(); i++) {
-            def sensor = sensors[i]
+            def sensor_git_ref = sensors[i].split(":")
+
+            def sensor = sensor_git_ref[0]
+            def git_ref = "refs/heads/master"
+
+            if (sensor_git_ref.size() > 1) {
+                git_ref = sensor_git_ref[1]
+            }
+
+            sim_sensors.add(sensor)
+
             checkout([
               $class: "GitSCM",
-              branches: [[name: "refs/heads/master"]],
+              branches: [[name: "${git_ref}"]],
               browser: [$class: "GitLab", repoUrl: "https://${GITLAB_HOST}/HDRP/Sensors/${sensor}", version: env.GITLAB_VERSION],
               extensions: [
                 [$class: "RelativeTargetDirectory", relativeTargetDir: "Assets/External/Sensors/${sensor}"],
@@ -144,11 +161,15 @@ pipeline {
 
 
           env.FORCE_REBUILD = "0"
+          env.GIT_COMMIT = sh(returnStdout: true, script: "git rev-parse HEAD").trim()
           env.SIM_ENVIRONMENTS = sh(returnStdout: true, script: "./Jenkins/get-assets.sh Assets/External/Environments ${S3_DOWNLOAD_HOST} environment 1").trim()
           env.SIM_VEHICLES = sh(returnStdout: true, script: "./Jenkins/get-assets.sh Assets/External/Vehicles ${S3_DOWNLOAD_HOST} vehicle 1").trim()
+          env.SIM_SENSORS = sim_sensors.join(",")
         } // script
 
         sh "rm -rf *.zip *.html *.xml"
+
+        sh label:"SIMULATOR env vars", script:"printenv | grep -E 'SIMULATOR|^SIM_' | sort"
       }
     }
 

diff --git a/Jenkins/build-simulator.sh b/Jenkins/build-simulator.sh
@@ -61,8 +61,8 @@ else
   CONTROLLABLES=
 fi
 
-if [ ! -z ${SIMULATOR_SENSORS+x} ]; then
-  SENSORS="-buildBundles -buildSensors ${SIMULATOR_SENSORS}"
+if [ ! -z ${SIM_SENSORS+x} ]; then
+  SENSORS="-buildBundles -buildSensors ${SIM_SENSORS}"
 else
   SENSORS=
 fi
@@ -148,8 +148,9 @@ else
 
 fi
 
-rm -Rf /mnt/AssetBundles/Controllables || true
-mkdir -p /mnt/AssetBundles/Controllables || true
+rm -Rf /mnt/AssetBundles || true
+mkdir -p /mnt/AssetBundles/{Controllables,Sensors} || true
+
 
 /opt/Unity/Editor/Unity ${DEVELOPMENT_BUILD} \
   -batchmode \

diff --git a/Jenkins/docker-compose.yml b/Jenkins/docker-compose.yml
@@ -26,7 +26,7 @@ services:
       - SIM_ENVIRONMENTS
       - SIM_VEHICLES
       - SIMULATOR_CONTROLLABLES
-      - SIMULATOR_SENSORS
+      - SIM_SENSORS
       - CLOUD_URL
       - JENKINS_BUILD_ID
       - GIT_COMMIT