Enable Mac Aarch64 reproducible comparing tests (#3972)

test/system/reproducibleCompare/build.xml

@@ -65,6 +65,7 @@
  <copy todir="." failonerror="false">
  <fileset dir="temurin-build/tooling/reproducible/" includes="*.sh" />
  </copy>
+ <chmod dir="." perm="755" includes="**/*.sh"/>
  <copy todir="." failonerror="false">
  <fileset dir="${TEST_ROOT}/../jdkbinary/" >
  <include name="*-sbom_*.json"/>

test/system/reproducibleCompare/playlist.xml

@@ -60,4 +60,25 @@
  </versions>
  <platformRequirements>os.win</platformRequirements>
  </test>
+ <test>
+ <testCaseName>Rebuild_Same_JDK_Reproducibility_Test_Mac</testCaseName>
+ <command>unset SPEC;\
+ ls -l $(TEST_ROOT)$(D)system$(D)reproducibleCompare$(D);\
+ $(TEST_ROOT)$(D)system$(D)reproducibleCompare$(D)macos_repro_build_compare.sh $(SBOM_FILE) $(JDK_FILE) $(REPORTDIR); \
+ $(TEST_STATUS)
+ </command>
+ <levels>
+ <level>dev</level>
+ </levels>
+ <groups>
+ <group>system</group>
+ </groups>
+ <vendors>
+ <vendor>eclipse</vendor>
+ </vendors>
+ <versions>
+ <version>21+</version>
+ </versions>
+ <platformRequirements>os.osx,arch.aarch64</platformRequirements>
+ </test>
 </playlist>

test/system/reproducibleCompare/reproducible.mk

@@ -16,6 +16,7 @@ ifndef SBOM_FILE
  SBOM_FILE := $(TEST_ROOT)/../jdkbinary/$(SBOM_FILE)
 endif
 ifndef JDK_FILE
+ JDK_FILE := $(shell find $(TEST_ROOT)/../jdkbinary/ -type f -name '*-jdk_*.tar.gz')
  ifneq (,$(findstring win,$(SPEC)))
  JDK_FILE := $(shell find $(TEST_ROOT)/../jdkbinary/ -type f -name '*-jdk_*.zip')
  endif

tooling/reproducible/macos_repro_build_compare.sh

@@ -46,16 +46,11 @@ REPORT_DIR="$3"
 # The Defaults Below Are Suitable For An Adoptium Mac OS X Build Environment
 # Which Has Been Created Via The Ansible Infrastructure Playbooks
 
-WORK_DIR=~/comp-jdk-build
+WORK_DIR=$(realpath "$(dirname "$0")")/comp-jdk-build
 MAC_COMPILER_BASE=/Applications
 MAC_COMPILER_APP_PREFIX=Xcode
 MAC_SDK_LOCATION=/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk
 
-# These 3 variables dictate which parameters passed to makejdk_any_platform.sh are config arguments, build arguments or should be ignored.
-CONFIG_ARGS=("--disable-warnings-as-errors" "--openjdk-target" "--with-sysroot" "--with-extra-cxxflags='" "--enable-dtrace" "--with-version-opt")
-NOTUSE_ARGS=("--assemble-exploded-image" "--configure-args")
-FINAL_ARG=("--build-variant")
-
 # These variables relate to the pre-requisite ant installation
 ANT_VERSION="1.10.5"
 ANT_CONTRIB_VERSION="1.0b3"
@@ -76,17 +71,6 @@ is_url() {
  fi
 }
 
-# Function to check if a value is in the array
-containsElement () {
- local e
- for e in "${@:2}"; do
- if [ "$e" == "$1" ]; then
- return 0 # Match found
- fi
- done
- return 1 # No match found
-}
-
 Create_WorkDir() {
  # Check if the folder exists & remove if it does
  echo "Checking If Working Directory: $WORK_DIR Exists"
@@ -371,13 +355,13 @@ Check_And_Install_Ant() {
  echo "Ant Doesnt Exist At The Correct Version - Installing"
  # Ant Version Not Found... Check And Create Paths
  echo Downloading ant for SBOM creation:
- curl https://archive.apache.org/dist/ant/binaries/apache-ant-${ANT_VERSION}-bin.zip > $WORK_DIR/apache-ant-${ANT_VERSION}-bin.zip
- (cd $WORK_DIR && unzip -qn ./apache-ant-${ANT_VERSION}-bin.zip)
- rm $WORK_DIR/apache-ant-${ANT_VERSION}-bin.zip
+ curl https://archive.apache.org/dist/ant/binaries/apache-ant-${ANT_VERSION}-bin.zip > "${WORK_DIR}"/apache-ant-${ANT_VERSION}-bin.zip
+ (cd "$WORK_DIR" && unzip -qn ./apache-ant-${ANT_VERSION}-bin.zip)
+ rm "$WORK_DIR"/apache-ant-${ANT_VERSION}-bin.zip
  echo Downloading ant-contrib-${ANT_CONTRIB_VERSION}:
- curl -L https://sourceforge.net/projects/ant-contrib/files/ant-contrib/${ANT_CONTRIB_VERSION}/ant-contrib-${ANT_CONTRIB_VERSION}-bin.zip > $WORK_DIR/ant-contrib-${ANT_CONTRIB_VERSION}-bin.zip
- (unzip -qnj $WORK_DIR/ant-contrib-${ANT_CONTRIB_VERSION}-bin.zip ant-contrib/ant-contrib-${ANT_CONTRIB_VERSION}.jar -d $WORK_DIR/apache-ant-${ANT_VERSION}/lib)
- rm $WORK_DIR/ant-contrib-${ANT_CONTRIB_VERSION}-bin.zip
+ curl -L https://sourceforge.net/projects/ant-contrib/files/ant-contrib/${ANT_CONTRIB_VERSION}/ant-contrib-${ANT_CONTRIB_VERSION}-bin.zip > "$WORK_DIR"/ant-contrib-${ANT_CONTRIB_VERSION}-bin.zip
+ (unzip -qnj "$WORK_DIR"/ant-contrib-${ANT_CONTRIB_VERSION}-bin.zip ant-contrib/ant-contrib-${ANT_CONTRIB_VERSION}.jar -d "$WORK_DIR"/apache-ant-${ANT_VERSION}/lib)
+ rm "$WORK_DIR"/ant-contrib-${ANT_CONTRIB_VERSION}-bin.zip
  else
  echo "Ant Version: $ANT_VERSION Is Already Installed"
  fi
@@ -441,118 +425,24 @@ Prepare_Env_For_Build() {
  echo "Setting Variables"
  export BOOTJDK_HOME=$WORK_DIR/jdk-${bootJDK}/Contents/Home
 
- echo "Parsing Make JDK Any Platform ARGS For Build"
- # Split the string into an array of words
- IFS=' ' read -ra words <<< "$buildArgs"
-
- # Add The Build Time Stamp In Case It Wasnt In The SBOM ARGS
- words+=( " --build-reproducible-date \"$buildStamp\"" )
-
- # Initialize variables
- param=""
- value=""
- params=()
-
- # Loop through the words
- for word in "${words[@]}"; do
- # Check if the word starts with '--'
- if [[ $word == --* ]] || [[ $word == -b* ]]; then
- # If a parameter already exists, store it in the params array
- if [[ -n $param ]]; then
- params+=("$param $value")
- fi
- # Reset variables for the new parameter
- param="$word"
- value=""
- else
- value+=" $word"
- fi
- done
-
- # Add the last parameter to the array
- params+=("$param $value")
-
- # Loop Through The Parameters And Reformat Appropriately
- export fixed_param=""
- export fixed_value=""
- export fixed_params=()
- export new_params=""
- CONFIG_ARRAY=()
- BUILD_ARRAY=()
- FINAL_ARRAY=()
- for element in "${params[@]}"; do
- IFS=' ' read -ra parts <<< "$element"
- prepped_part0=${parts[0]}
- prepped_part1=${parts[1]}
- prepped_part2=${parts[2]}
-
- fixed_param="$prepped_part0"
- fixed_value="$prepped_part1 $prepped_part2"
-
- # Handle Special Parameters ( overrides and = seperated )
- if [ "$fixed_param" == "--jdk-boot-dir" ]; then fixed_value=" $BOOTJDK_HOME " ; fi
- if [[ "$fixed_param" == "--with-sysroot="* ]]; then
- IFS='=' read -r split_param split_value <<< "$fixed_param"
- fixed_param="$split_param"
- fixed_value="=$MAC_SDK_LOCATION " ;
- fi
- if [[ "$fixed_param" == "--openjdk-target="* ]]; then
- IFS='=' read -r split_param split_value <<< "$fixed_param"
- fixed_param="$split_param"
- fixed_value="=$split_value "
- fi
- if [[ "$fixed_param" == "--with-version-opt="* ]]; then
- IFS='=' read -r split_param split_value <<< "$fixed_param"
- fixed_param="$split_param"
- fixed_value="=$split_value"
- fi
- if [ "$fixed_param" == "--tag" ]; then fixed_param="-b" fixed_value=" $fixed_value" ; fi
- if [ "$fixed_param" == "--target-file-name" ]; then
- target_file="$(echo -e "${fixed_value}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')"
- fixed_value=" $fixed_value"
- fi
- if [ "$fixed_param" == "--freetype-dir" ]; then fixed_value=" $fixed_value" ; fi
- if [ "$fixed_param" == "--user-openjdk-build-root-directory" ]; then fixed_value=" $WORK_DIR/temurin-build/workspace/build/openjdkbuild/ " ; fi
- if [ "$fixed_param" == "--build-variant" ]; then fixed_value=" $fixed_value " ; fi
- if [ "$fixed_param" == "--build-reproducible-date" ]; then fixed_value=" $fixed_value" ; fi
-
- if containsElement "$fixed_param" "${CONFIG_ARGS[@]}"; then
- # Check if fixed_param is in CONFIG_ARGS
- STRINGTOADD="$fixed_param$fixed_value"
- CONFIG_ARRAY+=("$STRINGTOADD")
- elif containsElement "$fixed_param" "${NOTUSE_ARGS[@]}"; then
- # Check if fixed_param is in NOTUSE_ARGS
- STRINGTOADD="$fixed_param$fixed_value"
- IGNORED_ARRAY+=("$STRINGTOADD")
- elif containsElement "$fixed_param" "${FINAL_ARG[@]}"; then
- # Check if parameter should be at the end, usually just version & variant
- STRINGTOADD="$fixed_param$fixed_value"
- FINAL_ARRAY+=("$STRINGTOADD")
- else
- # Neither Config Or Ignore, so Build
- STRINGTOADD="$fixed_param$fixed_value"
- BUILD_ARRAY+=("$STRINGTOADD")
- fi
- done
-
- # Construct Final Parameter String
- for element in "${BUILD_ARRAY[@]}"; do
- build_string+="$element"
- done
-
- for element in "${CONFIG_ARRAY[@]}"; do
- config_string+="$element"
- done
-
- for element in "${FINAL_ARRAY[@]}"; do
- final_string+="$element"
- done
-
- final_params="$build_string --configure-args \"$config_string\" $final_string"
+ # set --build-reproducible-date if not yet
+ if [[ "${buildArgs}" != *"--build-reproducible-date"* ]]; then
+ buildArgs="--build-reproducible-date \"${buildStamp}\" ${buildArgs}" 
+ fi
+ # reset --jdk-boot-dir
+ # shellcheck disable=SC2001
+ buildArgs="$(echo "$buildArgs" | sed -e "s|--jdk-boot-dir [^ ]*|--jdk-boot-dir ${BOOTJDK_HOME}|")"
+ # shellcheck disable=SC2001
+ buildArgs="$(echo "$buildArgs" | sed -e "s|--with-sysroot=[^ ]*|--with-sysroot=${MAC_SDK_LOCATION}|")"
+ # shellcheck disable=SC2001
+ buildArgs="$(echo "$buildArgs" | sed -e "s|--user-openjdk-build-root-directory [^ ]*|--user-openjdk-build-root-directory ${WORK_DIR}/temurin-build/workspace/build/openjdkbuild/|")"
+ # remove ingored options
+ buildArgs=${buildArgs/--assemble-exploded-image /}
+ buildArgs=${buildArgs/--enable-sbom-strace /}
 
-echo ""
+ echo ""
  echo "Make JDK Any Platform Argument List = "
- echo "$final_params"
+ echo "$buildArgs"
  echo ""
  echo "Parameters Parsed Successfully"
 }
@@ -562,79 +452,57 @@ Build_JDK() {
 
  # Trigger Build
  cd "$WORK_DIR"
- echo "cd temurin-build && ./makejdk-any-platform.sh $final_params 2>&1 | tee build.$$.log" | sh
+ echo "cd temurin-build && ./makejdk-any-platform.sh $buildArgs 2>&1 | tee build.$$.log" | sh
  # Copy The Built JDK To The Working Directory
- cp $WORK_DIR/temurin-build/workspace/target/"$target_file" $WORK_DIR/reproJDK.tar.gz
+ cp "$WORK_DIR"/temurin-build/workspace/target/OpenJDK*-jdk_*tar.gz "$WORK_DIR"/reproJDK.tar.gz
 }
 
 Compare_JDK() {
  echo "Comparing JDKs"
  echo ""
  mkdir "$WORK_DIR/compare"
- cp $WORK_DIR/src_jdk_dist.tar.gz $WORK_DIR/compare
- cp $WORK_DIR/reproJDK.tar.gz $WORK_DIR/compare
-
-
- # Check The Comparison Scripts Are Present And Copy To The Working Directory
+ cp "$WORK_DIR"/src_jdk_dist.tar.gz "$WORK_DIR"/compare
+ cp "$WORK_DIR"/reproJDK.tar.gz "$WORK_DIR"/compare
+ cp "$(dirname "$0")"/repro_*.sh "$WORK_DIR"/compare/
 
- if [ -f ./repro_common.sh ]; then
- cp ./repro_common.sh $WORK_DIR/compare/repro_common.sh
- else
- wget -O "$WORK_DIR/compare/repro_common.sh" "https://raw.githubusercontent.com/adoptium/temurin-build/master/tooling/reproducible/repro_common.sh"
- fi
- if [ -f ./repro_compare.sh ]; then
- cp ./repro_compare.sh $WORK_DIR/compare/repro_compare.sh
- else
- wget -O "$WORK_DIR/compare/repro_compare.sh" "https://raw.githubusercontent.com/adoptium/temurin-build/master/tooling/reproducible/repro_compare.sh"
- fi
- if [ -f ./repro_process.sh ]; then
- cp ./repro_process.sh $WORK_DIR/compare/repro_process.sh
- else
- wget -O "$WORK_DIR/compare/repro_process.sh" "https://raw.githubusercontent.com/adoptium/temurin-build/master/tooling/reproducible/repro_process.sh"
- fi
-
  # Set Permissions
  chmod +x "$WORK_DIR/compare/"*sh
  cd "$WORK_DIR/compare"
 
  # Unzip And Rename The Source JDK
  echo "Unzip Source"
- tar xvfz src_jdk_dist.tar.gz
+ tar xfz src_jdk_dist.tar.gz
  original_directory_name=$(find . -maxdepth 1 -type d | tail -1)
  mv "$original_directory_name" src_jdk
 
  #Unzip And Rename The Target JDK
  echo "Unzip Target"
- tar xvfz reproJDK.tar.gz
+ tar xfz reproJDK.tar.gz
  original_directory_name=$(find . -maxdepth 1 -type d | grep -v src_jdk | tail -1)
  mv "$original_directory_name" tar_jdk
 
  # Ensure Java Home Is Set
  export JAVA_HOME=$BOOTJDK_HOME
  export PATH=$JAVA_HOME/bin:$PATH
- echo "cd $WORK_DIR/compare && ./repro_compare.sh temurin src_jdk/Contents/Home temurin tar_jdk/Contents/Home Darwin 2>&1" | sh &
- wait
- rc=$?
- set -e
+ rc=0
+ ./repro_compare.sh temurin src_jdk/Contents/Home temurin tar_jdk/Contents/Home Darwin 2>&1 || rc=$?
  cd "$WORK_DIR"
- # Display The Content Of reprotest.diff
- echo ""
- echo "---------------------------------------------"
- echo "Output From JDK Comparison Script"
- echo "---------------------------------------------"
- cat $WORK_DIR/compare/reprotest.diff
- echo ""
- echo "---------------------------------------------"
- echo "Copying Output To $(dirname "$0")"
- cp $WORK_DIR/compare/reprotest.diff $WORK_DIR/reprotest.diff
-
+
+ if [ $rc -eq 0 ]; then
+ echo "Compare identical !"
+ else
+ echo "Differences found..., logged in: reprotest.diff"
+ fi
+
  if [ -n "$REPORT_DIR" ]; then
  echo "Copying Output To $REPORT_DIR"
- cp $WORK_DIR/compare/reprotest.diff "$REPORT_DIR"
- cp $WORK_DIR/reproJDK.tar.gz "$REPORT_DIR"
+ cp "$WORK_DIR"/compare/reprotest.diff "$REPORT_DIR"
+ cp "$WORK_DIR"/reproJDK.tar.gz "$REPORT_DIR"
  fi
-
+ Clean_Up_Everything
+ exit $rc 
 }
+
 #
 Clean_Up_Everything() {
  # Remove Working Directorys
@@ -674,6 +542,3 @@ echo "---------------------------------------------"
 Build_JDK
 echo "---------------------------------------------"
 Compare_JDK
-echo "---------------------------------------------"
-Clean_Up_Everything
-exit $rc