Merge branch 'develop'

.circleci/config.yml

@@ -88,10 +88,6 @@ workflows:
           name: testCentaurAws
           build-type: "centaurAws"
           build-mysql: "5.7"
-      - test:
-          name: testCentaurBcs
-          build-type: "centaurBcs"
-          build-mysql: "5.7"
       - test:
           name: testCentaurDummy
           build-type: "centaurDummy"

.github/workflows/make_publish_prs.yml

@@ -0,0 +1,73 @@
+name: Make publish PRs
+
+on:
+  workflow_dispatch:
+    inputs:
+      old_cromwell_version:
+        description: 'What version of Cromwell to update from'
+        required: true
+      new_cromwell_version:
+        description: 'What version of Cromwell to update to'
+        required: true
+      jira_ticket:
+        description: 'The JIRA ticket to link the update PR to'
+        required: true
+
+jobs:
+  make-firecloud-develop-pr:
+    name: Create firecloud-develop PR
+    runs-on: self-hosted # Faster machines; see https://github.com/broadinstitute/cromwell/settings/actions/runners
+    steps:
+      - name: Clone firecloud-develop
+        uses: actions/checkout@v2
+        with:
+          repository: broadinstitute/firecloud-develop
+          token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} # Has to be set at checkout AND later when pushing to work
+          path: firecloud-develop
+      - name: Update & push to firecloud-develop
+        id: update-and-push
+        env:
+          BROADBOT_GITHUB_TOKEN: ${{ secrets.BROADBOT_GITHUB_TOKEN }}
+        run: |
+          cd firecloud-develop
+          JIRA_TICKET=${{ github.event.inputs.jira_ticket }}
+          OLD_CROMWELL_V=${{ github.event.inputs.old_cromwell_version }}
+          NEW_CROMWELL_V=${{ github.event.inputs.new_cromwell_version }}
+          NEW_BRANCH_NAME="${JIRA_TICKET}_cromwell_${NEW_CROMWELL_V}"
+          TEMP=$(mktemp)
+          git checkout -b "${NEW_BRANCH_NAME}"
+          SED_STRING="s/${OLD_CROMWELL_V}/${NEW_CROMWELL_V}/g"
+          FILES=( "base-configs/cromwell/cromwell.conf.ctmpl" "run-context/fiab/scripts/FiaB_images.env" )
+          for i in "${FILES[@]}"
+          do
+            echo "Updating ${i}"
+            sed "${SED_STRING}" ${i} > ${TEMP}
+            mv ${TEMP} ${i}
+            git add ${i}
+          done
+          echo "Pushing to firecloud-develop branch ${NEW_BRANCH_NAME}"
+          git config --global user.name "broadbot"
+          git config --global user.email "[email protected]"
+          git commit -m "Updating Cromwell version to ${NEW_CROMWELL_V}"
+          git push https://broadbot:[email protected]/broadinstitute/firecloud-develop.git ${NEW_BRANCH_NAME}
+          echo ::set-output name=NEW_BRANCH_NAME::${NEW_BRANCH_NAME}
+      - name: Create firecloud-develop PR
+        uses: actions/github-script@v6
+        with:
+          github-token: ${{ secrets.BROADBOT_GITHUB_TOKEN }}
+          script: |
+            const result = await github.rest.pulls.create({
+              title: '${{ github.event.inputs.jira_ticket }}: Update Cromwell version to ${{ github.event.inputs.new_cromwell_version }}',
+              owner: 'broadinstitute',
+              repo: 'firecloud-develop',
+              head: '${{ steps.update-and-push.outputs.NEW_BRANCH_NAME }}',
+              base: 'dev',
+              body: [
+                'This PR is auto-generated by',
+                '[Cromwell actions/make_publish_prs](https://github.com/broadinstitute/cromwell/blob/develop/.github/workflows/make_publish_prs.yml), using',
+                '[github actions/github-script](https://github.com/actions/github-script).',
+                '',
+                'It updates cromwell from version ${{ github.event.inputs.old_cromwell_version }} to ${{ github.event.inputs.new_cromwell_version }}.'
+              ].join('\n')
+            });
+

.gitignore

@@ -31,8 +31,6 @@ scripts/docker-compose-mysql/compose/mysql/data
 # Vault rendered resources
 artifactory_credentials.properties
 aws_credentials
-bcs_application.conf
-bcs_login.inc.sh
 centaur_secure.inc.conf
 cromwell-centaur-requester-pays-service-account.json
 cromwell-centaur-service-account.json

.travis.yml

@@ -22,9 +22,6 @@ env:
     - >-
       BUILD_TYPE=centaurAws
       BUILD_MYSQL=5.7
-    - >-
-      BUILD_TYPE=centaurBcs
-      BUILD_MYSQL=5.7
     - >-
       BUILD_TYPE=centaurDummy
       BUILD_MYSQL=5.7
@@ -75,15 +72,6 @@ env:
       BUILD_MYSQL=5.7
     - >-
       BUILD_TYPE=checkPublish
-    - >-
-      BUILD_TYPE=conformanceLocal
-      BUILD_MYSQL=5.7
-    - >-
-      BUILD_TYPE=conformancePapiV2beta
-      BUILD_MYSQL=5.7
-    - >-
-      BUILD_TYPE=conformanceTesk
-      BUILD_MYSQL=5.7
     - >-
       BUILD_TYPE=horicromtalDeadlock
     - >-

CHANGELOG.md

@@ -1,5 +1,17 @@
 # Cromwell Change Log
 
+## 81 Release Notes
+
+### Workflow labels in TES tasks
+
+Beginning in Cromwell 81 we will populate the `tags` field of tasks created by the TES backend
+with the labels applied to the workflow at creation time.  No guarantee is made about labels
+added while the workflow is running.
+
+### Alibaba BCS backend and OSS filesystem removed
+
+The BCS backend and OSS filesystem (both of which support Alibaba Cloud) have been removed.
+
 ## 80 Release Notes
 
 ### Direct WES support in Cromwell

backend/src/test/scala/cromwell/backend/BackendSpec.scala

@@ -33,7 +33,8 @@ trait BackendSpec extends ScalaFutures with Matchers with ScaledTimeSpans {
   def buildWorkflowDescriptor(workflowSource: WorkflowSource,
                               inputFileAsJson: Option[String],
                               options: WorkflowOptions = WorkflowOptions(JsObject(Map.empty[String, JsValue])),
-                              runtime: String = ""): BackendWorkflowDescriptor = {
+                              runtime: String = "",
+                              labels: Labels = Labels.empty): BackendWorkflowDescriptor = {
     val wdlNamespace = WdlNamespaceWithWorkflow.load(workflowSource.replaceAll("RUNTIME", runtime),
       Seq.empty[Draft2ImportResolver]).get
     val executable = wdlNamespace.toWomExecutable(inputFileAsJson, NoIoFunctionSet, strictValidation = true) match {
@@ -46,7 +47,7 @@ trait BackendSpec extends ScalaFutures with Matchers with ScaledTimeSpans {
       executable.entryPoint,
       executable.resolvedExecutableInputs.flatMap({case (port, v) => v.select[WomValue] map { port -> _ }}),
       options,
-      Labels.empty,
+      labels,
       HogGroup("foo"),
       List.empty,
       None
@@ -56,9 +57,10 @@ trait BackendSpec extends ScalaFutures with Matchers with ScaledTimeSpans {
   def buildWdlWorkflowDescriptor(workflowSource: WorkflowSource,
                               inputFileAsJson: Option[String] = None,
                               options: WorkflowOptions = WorkflowOptions(JsObject(Map.empty[String, JsValue])),
-                              runtime: String = ""): BackendWorkflowDescriptor = {
+                              runtime: String = "",
+                              labels: Labels = Labels.empty): BackendWorkflowDescriptor = {
 
-    buildWorkflowDescriptor(workflowSource, inputFileAsJson, options, runtime)
+    buildWorkflowDescriptor(workflowSource, inputFileAsJson, options, runtime, labels)
   }
 
   def fqnWdlMapToDeclarationMap(m: Map[String, WomValue]): Map[InputDefinition, WomValue] = {

build.sbt

@@ -113,12 +113,6 @@ lazy val gcsFileSystem = (project in file("filesystems/gcs"))
   .dependsOn(cloudSupport % "test->test")
   .dependsOn(common % "test->test")
 
-lazy val ossFileSystem = (project in file("filesystems/oss"))
-  .withLibrarySettings("cromwell-ossFileSystem", ossFileSystemDependencies)
-  .dependsOn(core)
-  .dependsOn(core % "test->test")
-  .dependsOn(common % "test->test")
-
 lazy val sraFileSystem = (project in file("filesystems/sra"))
   .withLibrarySettings("cromwell-srafilesystem")
   .dependsOn(core)
@@ -248,21 +242,9 @@ lazy val tesBackend = (project in backendRoot / "tes")
   .dependsOn(backend % "test->test")
   .dependsOn(common % "test->test")
 
-lazy val bcsBackend = (project in backendRoot / "bcs")
-  .withLibrarySettings("cromwell-bcs-backend", bcsBackendDependencies)
-  .dependsOn(backend)
-  .dependsOn(ossFileSystem)
-  .dependsOn(gcsFileSystem)
-  .dependsOn(core % "test->test")
-  .dependsOn(backend % "test->test")
-  .dependsOn(ossFileSystem % "test->test")
-  .dependsOn(services % "test->test")
-  .dependsOn(common % "test->test")
-
 lazy val engine = project
   .withLibrarySettings("cromwell-engine", engineDependencies, engineSettings)
   .dependsOn(backend)
-  .dependsOn(ossFileSystem)
   .dependsOn(gcsFileSystem)
   .dependsOn(drsFileSystem)
   .dependsOn(sraFileSystem)
@@ -285,7 +267,6 @@ lazy val engine = project
   // For now, all the engine tests run on the "Local" backend, an implementation of an impl.sfs.config backend.
   .dependsOn(sfsBackend % "test->compile")
   .dependsOn(gcsFileSystem % "test->test")
-  .dependsOn(ossFileSystem % "test->test")
 
 // Executables
 
@@ -390,7 +371,6 @@ lazy val server = project
   .dependsOn(engine)
   .dependsOn(googlePipelinesV2Alpha1)
   .dependsOn(googlePipelinesV2Beta)
-  .dependsOn(bcsBackend)
   .dependsOn(awsBackend)
   .dependsOn(tesBackend)
   .dependsOn(cromwellApiClient)
@@ -412,7 +392,6 @@ lazy val root = (project in file("."))
   .aggregate(awsBackend)
   .aggregate(awsS3FileSystem)
   .aggregate(backend)
-  .aggregate(bcsBackend)
   .aggregate(centaur)
   .aggregate(centaurCwlRunner)
   .aggregate(cloudSupport)
@@ -434,7 +413,6 @@ lazy val root = (project in file("."))
   .aggregate(googlePipelinesV2Beta)
   .aggregate(httpFileSystem)
   .aggregate(languageFactoryCore)
-  .aggregate(ossFileSystem)
   .aggregate(perf)
   .aggregate(server)
   .aggregate(services)

cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/HashType.scala

@@ -13,8 +13,6 @@ object HashType extends Enumeration {
   // GCS crc32c, which is base64-encoded instead of a hex string
   val GcsCrc32c: HashType.Value = Value
   val Md5: HashType.Value = Value
-  // Alibaba OSS etag
-  val OssEtag: HashType.Value = Value
   // AWS S3 etag
   val S3Etag: HashType.Value = Value
   val Sha256: HashType.Value = Value
@@ -33,7 +31,6 @@ object HashType extends Enumeration {
         Base64.getEncoder.encodeToString(byteBuffer.array)
       case Md5 =>
         org.apache.commons.codec.digest.DigestUtils.md5Hex(s)
-      case OssEtag => throw new UnsupportedOperationException("Cannot currently calculate Alibaba OSS etag hash")
       case S3Etag =>
         val chunkSize = 8 * 1024 * 1024
         val numChunks = (s.length.toDouble / chunkSize).ceil.toInt

core/src/main/resources/reference.conf

@@ -342,9 +342,6 @@ filesystems {
   gcs {
     class = "cromwell.filesystems.gcs.GcsPathBuilderFactory"
   }
-  oss {
-    class = "cromwell.filesystems.oss.OssPathBuilderFactory"
-  }
   s3 {
     class = "cromwell.filesystems.s3.S3PathBuilderFactory"
   }
@@ -417,7 +414,6 @@ docker {
     }
     dockerhub.num-threads = 10
     quay.num-threads = 10
-    alibabacloudcr.num-threads = 10
   }
 }
 

cromwell.example.backends/README.md

@@ -11,7 +11,6 @@ repository.
 ### Cloud Providers
 
  - [AWS](AWS.conf): Amazon Web Services ([documentation](https://cromwell.readthedocs.io/en/stable/tutorials/AwsBatch101/))
- - [BCS](BCS.conf) Alibaba Cloud Batch Compute (BCS) backend ([documentation](https://cromwell.readthedocs.io/en/stable/backends/BCS/))
  - [TES](TES.conf) is a backend that submits jobs to a server with protocol defined by GA4GH ([documentation](https://cromwell.readthedocs.io/en/stable/backends/TES/))
  - [PAPIv2](PAPIv2.conf): Google Pipelines API backend (version 2!) ([documentation](https://cromwell.readthedocs.io/en/stable/backends/Google/))
 

cromwell.example.backends/TES.conf

@@ -1,4 +1,4 @@
-# This is an example of how you can use the TES backend provider. 
+# This is an example of how you can use the TES backend provider.
 # *This is not a complete configuration file!* The
 # content here should be copy pasted to the backend -> providers section
 # of cromwell.example.backends/cromwell.examples.conf in the root of the repository.
@@ -30,4 +30,5 @@ backend {
         }
       }
     }
+  }
 }