Merge branch 'master' of github.com:adoptium/temurin-build into gaction

andrew-m-leonard · Apr 26, 2024 · 7fd22ff · 7fd22ff
2 parents 6399253 + fae0d1a
commit 7fd22ff
Show file tree

Hide file tree

Showing 26 changed files with 120 additions and 72 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -409,7 +409,7 @@ jobs:
  path: workspace/target/*
 
  - name: Restore build artifacts
- uses: actions/download-artifact@9c19ed7fe5d278cd354c7dfd5d3b88589c7e2395 # v4.1.6
+ uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
  with:
  name: ${{matrix.version}}-${{matrix.os}}-${{matrix.variant}}
  path: ~/${{matrix.version}}-${{matrix.os}}-${{matrix.variant}}

diff --git a/.licenserc.yaml b/.licenserc.yaml
@@ -46,5 +46,6 @@ header:
  - 'sbin/*.template'
  - '.github/linters/*'
  - 'cyclonedx-lib/getDependencies'
+ - 'cyclonedx-lib/dependency_data/**'
  - 'makejdk-any-platform.1'
  - 'serverTimestamp.properties'
diff --git a/cyclonedx-lib/README.md b/cyclonedx-lib/README.md
@@ -34,13 +34,12 @@ formulation section (mostly because the author of this added such a section
 recently)
 
 1. Check that the version of CycloneDX you are using supports the
- functionality you want.  If not, it will need to be updated. To do this
- you need to do three steps [Sample PR](https://github.com/adoptium/temurin-build/pull/3558/files):
- - Update [getDependencies](https://github.com/adoptium/temurin-build/blob/master/cyclonedx-lib/getDependencies) with the new version and SHA
+ functionality you want. If not, it will need to be updated. To do this
+ you need to do two steps:
+ - Update the [sha and version files](https://github.com/adoptium/temurin-build/blob/master/cyclonedx-lib/dependency_data) for each jar you plan to change.
  - Ensure [build.getDependency](https://ci.adoptium.net/job/build.getDependency/) is run to pick up the new version
- - Update [build.xml](https://github.com/adoptium/temurin-build/blob/master/cyclonedx-lib/build.xml) with the new SHA to allow it to be downloaded successfully during the build (This change can be made along with the getDependencies one if you co--ordinate the update appopriately
-2. If the build and java code does not already have support for the CycloneDX functionality that you need the follow ([Sample PR](https://github.com/adoptium/temurin-build/blob/master/cyclonedx-lib/build.xml))
- - Updates to [cyclonedx-lib/TemurinGenSBOM.java])https://github.com/adoptium/temurin-build/blob/master/sbin/common/sbom.sh) to add a new parameter, a new function to implement it, the call to that function from the `switch` functionality in the `main` function
+2. If the build and java code does not already have support for the CycloneDX functionality that you need, then follow these steps ([Sample PR](https://github.com/adoptium/temurin-build/pull/3538))
+ - Updates to [cyclonedx-lib/TemurinGenSBOM.java](https://github.com/adoptium/temurin-build/blob/master/sbin/common/sbom.sh) to add a new parameter, a new function to implement it, the call to that function from the `switch` functionality in the `main` function
  - Update [cyclonedx-lib/build.xml](https://github.com/adoptium/temurin-build/blob/master/cyclonedx-lib/build.xml) to add tests for the new functionality
  - Add a new function to [sbin/common/sbom.sh](https://github.com/adoptium/temurin-build/blob/master/sbin/common/sbom.sh) to add the fields you need
  - Updates to [sbin/build.sh](https://github.com/adoptium/temurin-build/blob/master/sbin/build.sh) to invoke the new function(s) in sbom.sh

diff --git a/cyclonedx-lib/build.xml b/cyclonedx-lib/build.xml
@@ -20,9 +20,12 @@
 // jscpd:ignore-start
 -->
 
- <!-- All other dependency versions are now listed in the ./getDependencies file -->
+ <!-- All other dependency versions are listed in the relevant ./dependency_data/versions files -->
  <property name="openkeystore-version" value="1.0.0"/>
 
+ <!-- When retrieving sbom dependency jars with default settings, we will look here. -->
+ <property name="sbom_dependency_default_location" value="https://ci.adoptium.net/view/all/job/build.getDependency/lastSuccessfulBuild/artifact/sbom_dependencies"/>
+
  <!-- classpath for running application -->
  <property name="classpath" value="build/jar/temurin-gen-sbom.jar:build/jar/cyclonedx-core-java.jar:build/jar/jackson-core.jar:build/jar/jackson-dataformat-xml.jar:build/jar/jackson-databind.jar:build/jar/jackson-annotations.jar:build/jar/json-schema.jar:build/jar/commons-codec.jar:build/jar/commons-io.jar:build/jar/github-package-url.jar:build/jar/webpki.org-libext-1.00.jar:build/jar/temurin-sign-sbom.jar"/>
 
@@ -41,20 +44,12 @@
 
  <target name="download-cyclonedx" unless="cyclonedx_available">
  <echo message="Downloading cyclonedx-core-java"/>
- <download-file
- destfile="cyclonedx-core-java.jar"
- checksum="ecc371d12808dfe76047f87f8235665d74dd6cf8ec12c41d052715a3fd79e0b5"
- srcurl="https://ci.adoptium.net/view/all/job/build.getDependency/lastSuccessfulBuild/artifact/sbom_dependencies/cyclonedx-core-java.jar"
- />
- </target>
+ <download-file-with-default-options destfile="cyclonedx-core-java.jar"/>
+ </target>
 
  <target name="download-jackson-core" unless="jackson-core_available">
  <echo message="Downloading jackson-core"/>
- <download-file
- checksum="b5d37a77c88277b97e3593c8740925216c06df8e4172bbde058528df04ad3e7a"
- destfile="jackson-core.jar"
- srcurl="https://ci.adoptium.net/view/all/job/build.getDependency/lastSuccessfulBuild/artifact/sbom_dependencies/jackson-core.jar"
- />
+ <download-file-with-default-options destfile="jackson-core.jar"/>
  </target>
 
  <target name="clone-and-build-openkeystore" unless="openkeystore_available">
@@ -91,58 +86,37 @@
 
  <target name="download-jackson-dataformat-xml" unless="jackson-dataformat_available">
  <echo message="Downloading jackson-dataformat-xml"/>
- <download-file
- checksum="edbda6c775a36049cf0088b111ab958cca0dc70cb9326918d6cf153cb3fa426b"
- destfile="jackson-dataformat-xml.jar"
- srcurl="https://ci.adoptium.net/view/all/job/build.getDependency/lastSuccessfulBuild/artifact/sbom_dependencies/jackson-dataformat-xml.jar"/>
+ <download-file-with-default-options destfile="jackson-dataformat-xml.jar"/>
  </target>
 
  <target name="download-jackson-databind" unless="jackson-databind_available">
  <echo message="Downloading jackson-databind"/>
- <download-file
- checksum="501d3abce4d18dcc381058ec593c5b94477906bba6efbac14dae40a642f77424"
- destfile="jackson-databind.jar"
- srcurl="https://ci.adoptium.net/view/all/job/build.getDependency/lastSuccessfulBuild/artifact/sbom_dependencies/jackson-databind.jar"/>
+ <download-file-with-default-options destfile="jackson-databind.jar"/>
  </target>
 
  <target name="download-jackson-annotations" unless="jackson-annotations_available">
  <echo message="Downloading jackson-annotations"/>
- <download-file
- checksum="2c6869d505cf60dc066734b7d50339f975bd3adc635e26a78abb71acb4473c0d"
- destfile="jackson-annotations.jar"
- srcurl="https://ci.adoptium.net/view/all/job/build.getDependency/lastSuccessfulBuild/artifact/sbom_dependencies/jackson-annotations.jar"/>
+ <download-file-with-default-options destfile="jackson-annotations.jar"/>
  </target>
 
  <target name="download-json-schema" unless="json-schema_available">
  <echo message="Downloading json-schema"/>
- <download-file
- checksum="968991e5718520cdd7b224770f790cf2c241cddf64d10a36c21f9f8b4a15e79c"
- destfile="json-schema.jar"
- srcurl="https://ci.adoptium.net/view/all/job/build.getDependency/lastSuccessfulBuild/artifact/sbom_dependencies/json-schema.jar"/>
+ <download-file-with-default-options destfile="json-schema.jar"/>
  </target>
 
  <target name="download-commons-codec" unless="commons-codec_available">
  <echo message="Downloading commons-codec"/>
- <download-file
- checksum="b3e9f6d63a790109bf0d056611fbed1cf69055826defeb9894a71369d246ed63"
- destfile="commons-codec.jar"
- srcurl="https://ci.adoptium.net/view/all/job/build.getDependency/lastSuccessfulBuild/artifact/sbom_dependencies/commons-codec.jar"/>
+ <download-file-with-default-options destfile="commons-codec.jar"/>
  </target>
 
  <target name="download-commons-io" unless="commons-io_available">
  <echo message="Downloading commons-io"/>
- <download-file
- checksum="961b2f6d87dbacc5d54abf45ab7a6e2495f89b75598962d8c723cea9bc210908"
- destfile="commons-io.jar"
- srcurl="https://ci.adoptium.net/view/all/job/build.getDependency/lastSuccessfulBuild/artifact/sbom_dependencies/commons-io.jar"/>
+ <download-file-with-default-options destfile="commons-io.jar"/>
  </target>
 
  <target name="download-github-package-url" unless="github-package-url_available">
  <echo message="Downloading github-package-url"/>
- <download-file
- checksum="8e23280221afd1e6561d433dfb133252cd287167acb0eca5a991667118ff10a2"
- destfile="github-package-url.jar"
- srcurl="https://ci.adoptium.net/view/all/job/build.getDependency/lastSuccessfulBuild/artifact/sbom_dependencies/github-package-url.jar"/>
+ <download-file-with-default-options destfile="github-package-url.jar"/>
  </target>
 
  <target name="build" depends="dep-checks, download-cyclonedx, download-jackson-core, download-jackson-dataformat-xml, download-jackson-databind, download-jackson-annotations, download-json-schema, download-commons-codec, download-commons-io, download-github-package-url, compile, jar">
@@ -505,6 +479,20 @@
 
  </target>
 
+ <macrodef name="download-file-with-default-options" description="Use curl to download a file and retrieve its checksum from default locations.">
+ <attribute name="destfile" description="File name of the downloaded file"/>
+ <sequential>
+ <echo message="Executing macro download-file-with-default-options"/>
+ <echo message="Fetching checksum prior to downloading file."/>
+ <loadfile property="checksum-for-@{destfile}" srcFile="dependency_data/shas/@{destfile}.sha256"/>
+ <echo message="Checksum found: ${checksum-for-@{destfile}}"/>
+ <download-file
+ checksum="${checksum-for-@{destfile}}"
+ destfile="@{destfile}"
+ srcurl="${sbom_dependency_default_location}/@{destfile}"/>
+ </sequential>
+ </macrodef>
+
  <macrodef name="download-file" description="Use curl to download a file">
  <attribute name="srcurl" description="URL of file to download"/>
  <attribute name="destdir" default="build/jar" description="Directory in which to place the downloaded file"/>

diff --git a/cyclonedx-lib/dependency_data/shas/commons-codec.jar.sha256 b/cyclonedx-lib/dependency_data/shas/commons-codec.jar.sha256
@@ -0,0 +1 @@
+b3e9f6d63a790109bf0d056611fbed1cf69055826defeb9894a71369d246ed63
diff --git a/cyclonedx-lib/dependency_data/shas/commons-io.jar.sha256 b/cyclonedx-lib/dependency_data/shas/commons-io.jar.sha256
@@ -0,0 +1 @@
+961b2f6d87dbacc5d54abf45ab7a6e2495f89b75598962d8c723cea9bc210908
diff --git a/cyclonedx-lib/dependency_data/shas/cyclonedx-core-java.jar.sha256 b/cyclonedx-lib/dependency_data/shas/cyclonedx-core-java.jar.sha256
@@ -0,0 +1 @@
+ecc371d12808dfe76047f87f8235665d74dd6cf8ec12c41d052715a3fd79e0b5
diff --git a/cyclonedx-lib/dependency_data/shas/github-package-url.jar.sha256 b/cyclonedx-lib/dependency_data/shas/github-package-url.jar.sha256
@@ -0,0 +1 @@
+8e23280221afd1e6561d433dfb133252cd287167acb0eca5a991667118ff10a2
diff --git a/cyclonedx-lib/dependency_data/shas/jackson-annotations.jar.sha256 b/cyclonedx-lib/dependency_data/shas/jackson-annotations.jar.sha256
@@ -0,0 +1 @@
+2c6869d505cf60dc066734b7d50339f975bd3adc635e26a78abb71acb4473c0d
diff --git a/cyclonedx-lib/dependency_data/shas/jackson-core.jar.sha256 b/cyclonedx-lib/dependency_data/shas/jackson-core.jar.sha256
@@ -0,0 +1 @@
+b5d37a77c88277b97e3593c8740925216c06df8e4172bbde058528df04ad3e7a
diff --git a/cyclonedx-lib/dependency_data/shas/jackson-databind.jar.sha256 b/cyclonedx-lib/dependency_data/shas/jackson-databind.jar.sha256
@@ -0,0 +1 @@
+501d3abce4d18dcc381058ec593c5b94477906bba6efbac14dae40a642f77424
diff --git a/cyclonedx-lib/dependency_data/shas/jackson-dataformat-xml.jar.sha256 b/cyclonedx-lib/dependency_data/shas/jackson-dataformat-xml.jar.sha256
@@ -0,0 +1 @@
+edbda6c775a36049cf0088b111ab958cca0dc70cb9326918d6cf153cb3fa426b
diff --git a/cyclonedx-lib/dependency_data/shas/json-schema.jar.sha256 b/cyclonedx-lib/dependency_data/shas/json-schema.jar.sha256
@@ -0,0 +1 @@
+968991e5718520cdd7b224770f790cf2c241cddf64d10a36c21f9f8b4a15e79c
diff --git a/cyclonedx-lib/dependency_data/versions/commons-codec.jar.version b/cyclonedx-lib/dependency_data/versions/commons-codec.jar.version
@@ -0,0 +1 @@
+1.15
diff --git a/cyclonedx-lib/dependency_data/versions/commons-io.jar.version b/cyclonedx-lib/dependency_data/versions/commons-io.jar.version
@@ -0,0 +1 @@
+2.11.0
diff --git a/cyclonedx-lib/dependency_data/versions/cyclonedx-core-java.jar.version b/cyclonedx-lib/dependency_data/versions/cyclonedx-core-java.jar.version
@@ -0,0 +1 @@
+8.0.3
diff --git a/cyclonedx-lib/dependency_data/versions/github-package-url.jar.version b/cyclonedx-lib/dependency_data/versions/github-package-url.jar.version
@@ -0,0 +1 @@
+1.4.1
diff --git a/cyclonedx-lib/dependency_data/versions/jackson-annotations.jar.version b/cyclonedx-lib/dependency_data/versions/jackson-annotations.jar.version
@@ -0,0 +1 @@
+2.14.2
diff --git a/cyclonedx-lib/dependency_data/versions/jackson-core.jar.version b/cyclonedx-lib/dependency_data/versions/jackson-core.jar.version
@@ -0,0 +1 @@
+2.14.2
diff --git a/cyclonedx-lib/dependency_data/versions/jackson-databind.jar.version b/cyclonedx-lib/dependency_data/versions/jackson-databind.jar.version
@@ -0,0 +1 @@
+2.14.2
diff --git a/cyclonedx-lib/dependency_data/versions/jackson-dataformat-xml.jar.version b/cyclonedx-lib/dependency_data/versions/jackson-dataformat-xml.jar.version
@@ -0,0 +1 @@
+2.14.2
diff --git a/cyclonedx-lib/dependency_data/versions/json-schema.jar.version b/cyclonedx-lib/dependency_data/versions/json-schema.jar.version
@@ -0,0 +1 @@
+1.0.77
diff --git a/cyclonedx-lib/getDependencies b/cyclonedx-lib/getDependencies
@@ -2,16 +2,27 @@
 
 LABEL=params.LABEL ? params.LABEL : 'ci.role.test&&hw.arch.x86&&sw.os.linux'
 
+TEMURIN_BUILD_REPO="https://github.com/adamfarley/temurin-build"
+TEMURIN_BUILD_BRANCH="add_versions_and_shas_to_build_getdependencies"
+
 stage('Queue') {
  node("$LABEL") {
  cleanWs()
  fetchDeps()
  }
 }
 
-def fetchSingleFile(String jarFile, String sha, String mavenURL) {
- sh 'echo "' + sha + ' sbom_dependencies/' + jarFile + '" >> sbom_dep_shas.txt'
+// Fetch a single file from a location under Maven.
+// Args:
+// - jarFile: The name of the file after we've downloaded it.
+// - mavenURL: The file and location of the jarfile to be downloaded (under Maven).
+def fetchSingleFile(String jarFile, String mavenURL) {
+ // Downloads the requested jarfile
  sh 'curl -L -o "sbom_dependencies/' + jarFile + '" "https://search.maven.org/classic/remotecontent?filepath=' + mavenURL + '"'
+
+ // Puts the matching sha for the jarfile into the list of shas to check.
+ def jarSHA = readFile(file : './temurin-build/cyclonedx-lib/dependency_data/shas/' + jarFile + '.sha256').replaceAll("\\s","")
+ sh 'echo "' + jarSHA + ' sbom_dependencies/' + jarFile + '" >> sbom_dependency_shas.txt'
 }
 
 def fetchDeps() {
@@ -21,38 +32,45 @@ def fetchDeps() {
  }
  timeout(time: time_limit, unit: 'HOURS') {
  try {
+ // This allows us to easily specify which of our workspace files are to be archived later.
  sh 'mkdir sbom_dependencies'
-
-
+
  // These versions come from https://github.com/CycloneDX/cyclonedx-core-java/tags
  // Version->spec mappings are in https://github.com/CycloneDX/cyclonedx-core-java#cyclonedx-schema-support
- def cyclonedx_core_java_version = "8.0.3"
- def jackson_core_version = "2.14.2"
- def jackson_annotations_version = "2.14.2"
- def jackson_databind_version = "2.14.2"
- def json_schema_version = "1.0.77"
- def commons_codec_version = "1.15"
- def commons_io_version = "2.11.0"
- def github_package_url_version = "1.4.1"
-
- fetchSingleFile("cyclonedx-core-java.jar", "ecc371d12808dfe76047f87f8235665d74dd6cf8ec12c41d052715a3fd79e0b5", "org/cyclonedx/cyclonedx-core-java/${cyclonedx_core_java_version}/cyclonedx-core-java-${cyclonedx_core_java_version}.jar")
- fetchSingleFile("jackson-core.jar", "b5d37a77c88277b97e3593c8740925216c06df8e4172bbde058528df04ad3e7a", "com/fasterxml/jackson/core/jackson-core/${jackson_core_version}/jackson-core-${jackson_core_version}.jar")
- fetchSingleFile("jackson-dataformat-xml.jar", "edbda6c775a36049cf0088b111ab958cca0dc70cb9326918d6cf153cb3fa426b", "com/fasterxml/jackson/dataformat/jackson-dataformat-xml/${jackson_databind_version}/jackson-dataformat-xml-${jackson_databind_version}.jar")
- fetchSingleFile("jackson-databind.jar", "501d3abce4d18dcc381058ec593c5b94477906bba6efbac14dae40a642f77424", "com/fasterxml/jackson/core/jackson-databind/${jackson_databind_version}/jackson-databind-${jackson_databind_version}.jar")
- fetchSingleFile("jackson-annotations.jar", "2c6869d505cf60dc066734b7d50339f975bd3adc635e26a78abb71acb4473c0d", "com/fasterxml/jackson/core/jackson-annotations/${jackson_annotations_version}/jackson-annotations-${jackson_annotations_version}.jar")
- fetchSingleFile("json-schema.jar", "968991e5718520cdd7b224770f790cf2c241cddf64d10a36c21f9f8b4a15e79c", "com/networknt/json-schema-validator/${json_schema_version}/json-schema-validator-${json_schema_version}.jar")
- fetchSingleFile("commons-codec.jar", "b3e9f6d63a790109bf0d056611fbed1cf69055826defeb9894a71369d246ed63", "commons-codec/commons-codec/${commons_codec_version}/commons-codec-${commons_codec_version}.jar")
- fetchSingleFile("github-package-url.jar", "8e23280221afd1e6561d433dfb133252cd287167acb0eca5a991667118ff10a2", "com/github/package-url/packageurl-java/${github_package_url_version}/packageurl-java-${github_package_url_version}.jar")
- fetchSingleFile("commons-io.jar", "961b2f6d87dbacc5d54abf45ab7a6e2495f89b75598962d8c723cea9bc210908", "commons-io/commons-io/${commons_io_version}/commons-io-${commons_io_version}.jar")
-
- sh 'sha256sum -c sbom_dep_shas.txt'
+ sh 'git clone -b ' + TEMURIN_BUILD_BRANCH + ' ' + TEMURIN_BUILD_REPO + ' temurin-build'
+ def dep_data = WORKSPACE + "/temurin-build/cyclonedx-lib/dependency_data/"
+
+ def cyclonedx_core_java_version = readFile(file : dep_data + 'versions/cyclonedx-core-java.jar.version').replaceAll("\\s","")
+ def jackson_core_version = readFile(file : dep_data + 'versions/jackson-core.jar.version').replaceAll("\\s","")
+ def jackson_annotations_version = readFile(file : dep_data + 'versions/jackson-annotations.jar.version').replaceAll("\\s","")
+ def jackson_databind_version = readFile(file : dep_data + 'versions/jackson-databind.jar.version').replaceAll("\\s","")
+ def json_schema_version = readFile(file : dep_data + 'versions/json-schema.jar.version').replaceAll("\\s","")
+ def commons_codec_version = readFile(file : dep_data + 'versions/commons-codec.jar.version').replaceAll("\\s","")
+ def commons_io_version = readFile(file : dep_data + 'versions/commons-io.jar.version').replaceAll("\\s","")
+ def github_package_url_version = readFile(file : dep_data + 'versions/github-package-url.jar.version').replaceAll("\\s","")
 
+ // Each of these fetches a jar in the format: fetchSingleFile(jar name post-download, current jar location under Maven)
+ fetchSingleFile("cyclonedx-core-java.jar", "org/cyclonedx/cyclonedx-core-java/${cyclonedx_core_java_version}/cyclonedx-core-java-${cyclonedx_core_java_version}.jar")
+ fetchSingleFile("jackson-core.jar", "com/fasterxml/jackson/core/jackson-core/${jackson_core_version}/jackson-core-${jackson_core_version}.jar")
+ fetchSingleFile("jackson-dataformat-xml.jar", "com/fasterxml/jackson/dataformat/jackson-dataformat-xml/${jackson_databind_version}/jackson-dataformat-xml-${jackson_databind_version}.jar")
+ fetchSingleFile("jackson-databind.jar", "com/fasterxml/jackson/core/jackson-databind/${jackson_databind_version}/jackson-databind-${jackson_databind_version}.jar")
+ fetchSingleFile("jackson-annotations.jar", "com/fasterxml/jackson/core/jackson-annotations/${jackson_annotations_version}/jackson-annotations-${jackson_annotations_version}.jar")
+ fetchSingleFile("json-schema.jar", "com/networknt/json-schema-validator/${json_schema_version}/json-schema-validator-${json_schema_version}.jar")
+ fetchSingleFile("commons-codec.jar", "commons-codec/commons-codec/${commons_codec_version}/commons-codec-${commons_codec_version}.jar")
+ fetchSingleFile("github-package-url.jar", "com/github/package-url/packageurl-java/${github_package_url_version}/packageurl-java-${github_package_url_version}.jar")
+ fetchSingleFile("commons-io.jar", "commons-io/commons-io/${commons_io_version}/commons-io-${commons_io_version}.jar")
+
+
+ // Check that every file matches the sha of the file we expected.
+ sh 'sha256sum -c sbom_dependency_shas.txt'
+
+ // Store all retrieved files on Jenkins, for use during sbom creation.
  archiveArtifacts '**/sbom_dependencies/*'
  } finally {
+ // Clean up workspace.
  cleanWs()
  }
  }
 }
 
-
 return this