From 14de8ce3179cbb7e0092189279917f294a2c6daa Mon Sep 17 00:00:00 2001 From: Anna Babu Palathingal Date: Fri, 1 Dec 2023 16:58:54 -0500 Subject: [PATCH] Added code to detect micro architecture - zlinux version detection - add tests Signed-off-by: Anna Babu Palathingal --- examples/platformRequirements/playlist.xml | 70 ++++- makeGen.mk | 2 +- scripts/testTKG/test_platformRequirements.py | 259 ++++++++++++------- src/org/openj9/envInfo/MachineInfo.java | 36 ++- src/org/testKitGen/TestInfoParser.java | 64 +++-- 5 files changed, 295 insertions(+), 136 deletions(-) diff --git a/examples/platformRequirements/playlist.xml b/examples/platformRequirements/playlist.xml index 0bf3f54b..5a309164 100644 --- a/examples/platformRequirements/playlist.xml +++ b/examples/platformRequirements/playlist.xml @@ -107,5 +107,73 @@ $(TEST_STATUS) ^os.linux.ubuntu.20+ + + test_not_arch_390 + echo "test on none arch.390"; \ + $(TEST_STATUS) + ^arch.390 + + + + test_arch_x86_skylake + echo "test arch.x86.skylake"; \ + $(TEST_STATUS) + + arch.x86.skylake + + + + + test_arch_390_z15plus + echo "test arch.390.z15+"; \ + $(TEST_STATUS) + + arch.390.z15+ + + + + + test_arch_390_z15 + echo "test arch.390.z15"; \ + $(TEST_STATUS) + + arch.390.z15 + + + + test_arch_390_z14plus + echo "test arch.390.z14+"; \ + $(TEST_STATUS) + + arch.390.z14+ + + + + + test_arch_390_z14 + echo "test arch.390.z14"; \ + $(TEST_STATUS) + + arch.390.z14 + + + + + test_arch_390_z13plus + echo "test arch.390.z13+"; \ + $(TEST_STATUS) + + arch.390.z13+ + + + + + test_arch_390_z13 + echo "test arch.390.z13"; \ + $(TEST_STATUS) + + arch.390.z13 + + - + \ No newline at end of file diff --git a/makeGen.mk b/makeGen.mk index 4db53d59..89ece3ea 100644 --- a/makeGen.mk +++ b/makeGen.mk @@ -42,7 +42,7 @@ autoconfig: perl scripts$(D)configure.pl autogen: autoconfig - ${TEST_JDK_HOME}/bin/java -cp $(Q)$(TKG_JAR)$(P)$(JSON_SIMPLE)$(Q) org.testKitGen.MainRunner --mode=$(MODE) --spec=$(SPEC) --microArch=$(MICROARCH) --osLabel=$(Q)$(OS_LABEL)$(Q) --jdkVersion=$(JDK_VERSION) --impl=$(JDK_IMPL) --vendor=$(Q)$(JDK_VENDOR)$(Q) --buildList=${BUILD_LIST} --iterations=$(TKG_ITERATIONS) --aotIterations=$(AOT_ITERATIONS) --testFlag=$(TEST_FLAG) --testTarget=$(TESTTARGET) --testList=$(TESTLIST) --numOfMachines=$(NUM_MACHINES) --testTime=$(TEST_TIME) --TRSSURL=$(TRSS_URL) $(OPTS) + ${TEST_JDK_HOME}/bin/java -cp $(Q)$(TKG_JAR)$(P)$(JSON_SIMPLE)$(Q) org.testKitGen.MainRunner --mode=$(MODE) --spec=$(SPEC) --microArch=$(Q)$(MICROARCH)$(Q) --osLabel=$(Q)$(OS_LABEL)$(Q) --jdkVersion=$(JDK_VERSION) --impl=$(JDK_IMPL) --vendor=$(Q)$(JDK_VENDOR)$(Q) --buildList=${BUILD_LIST} --iterations=$(TKG_ITERATIONS) --aotIterations=$(AOT_ITERATIONS) --testFlag=$(TEST_FLAG) --testTarget=$(TESTTARGET) --testList=$(TESTLIST) --numOfMachines=$(NUM_MACHINES) --testTime=$(TEST_TIME) --TRSSURL=$(TRSS_URL) $(OPTS) AUTOGEN_FILES = $(wildcard $(CURRENT_DIR)$(D)jvmTest.mk) AUTOGEN_FILES += $(wildcard $(CURRENT_DIR)$(D)machineConfigure.mk) diff --git a/scripts/testTKG/test_platformRequirements.py b/scripts/testTKG/test_platformRequirements.py index 62e0bc65..a41d4c5b 100644 --- a/scripts/testTKG/test_platformRequirements.py +++ b/scripts/testTKG/test_platformRequirements.py @@ -16,105 +16,164 @@ from utils import * def run_test(): - rt = True - printTestheader("platformRequirements") - - buildList = "TKG/examples/platformRequirements" - command = "make _all" - print(f"\t{command}") - result = subprocess.run(f"{EXPORT_BUILDLIST}={buildList}; {CD_TKG}; {MAKE_CLEAN}; {MAKE_COMPILE}; {command}", stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True, check=False) - - stdout = result.stdout.decode() - specStr = re.search(r"set SPEC to (.*)", stdout) - - if specStr is not None: - spec = specStr.group(1) - else: - printError("Could not parse spec from output.") - return False - - passed = set() - skipped = set() - - if 'linux' in spec: - skipped.add('test_not_linux_arch_x86_0') - else: - if 'x86' in spec: - passed.add('test_not_linux_arch_x86_0') - else: - skipped.add('test_not_linux_arch_x86_0') - - if 'x86' in spec: - passed.add('test_arch_x86_0') - skipped.add('test_arch_nonx86_0') - else: - passed.add('test_arch_nonx86_0') - skipped.add('test_arch_x86_0') - - if '64' in spec: - passed.add('test_bits_64_0') - else: - skipped.add('test_bits_64_0') - - if 'osx' in spec: - passed.add('test_os_osx_0') - else: - skipped.add('test_os_osx_0') - - if 'osx_x86-64' in spec: - passed.add('test_osx_x86-64_0') - else: - skipped.add('test_osx_x86-64_0') - - if 'x86' in spec or '390' in spec: - passed.add('test_arch_x86_390_0') - else: - skipped.add('test_arch_x86_390_0') - - if 'osx_x86-64' in spec or ('win_x86' in spec and 'win_x86-64' not in spec) or 'aix_ppc-64' in spec: - passed.add('test_osx_x86-64_win_x86_aix_ppc-64_0') - else: - skipped.add('test_osx_x86-64_win_x86_aix_ppc-64_0') - - os_label = re.search(r"set OS_LABEL to (.*)", stdout) - if os_label is not None: - os_label = os_label.group(1) - label_str = os_label.split(".") - # os_label example: ubuntu.22 - try: - ver = int(label_str[1],10) - if label_str[0] == "ubuntu": - if ver >= 22: - passed.add('test_os_linux_ubuntu22_0') - if ver >= 20: - passed.add('test_os_linux_ubuntu20plus_0') - passed.add('test_os_linux_ubuntu20plus_rhel8plus_0') - if ver < 20: - passed.add('test_not_os_linux_ubuntu20plus_0') - - if label_str[0] == "rhel": - if ver >= 8: - passed.add('test_os_linux_ubuntu20plus_rhel8plus_0') - except ValueError as ve: - print ("warning: os version value failed to convert to an integer") - passed.add('test_not_os_linux_ubuntu20plus_0') - else: - passed.add('test_not_os_linux_ubuntu20plus_0') - - if 'test_os_linux_ubuntu20plus_0' not in passed: - skipped.add('test_os_linux_ubuntu20plus_0') - - if 'test_os_linux_ubuntu22_0' not in passed: - skipped.add('test_os_linux_ubuntu22_0') - - if 'test_os_linux_ubuntu20plus_rhel8plus_0' not in passed: - skipped.add('test_os_linux_ubuntu20plus_rhel8plus_0') - - if 'test_not_os_linux_ubuntu20plus_0' not in passed: - skipped.add('test_not_os_linux_ubuntu20plus_0') - - rt &= checkResult(result, passed, set(), set(), skipped) - return rt + rt = True + printTestheader("platformRequirements") + + buildList = "TKG/examples/platformRequirements" + command = "make _all" + print(f"\t{command}") + result = subprocess.run(f"{EXPORT_BUILDLIST}={buildList}; {CD_TKG}; {MAKE_CLEAN}; {MAKE_COMPILE}; {command}", stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True, check=False) + + stdout = result.stdout.decode() + specStr = re.search(r"set SPEC to (.*)", stdout) + + if specStr is not None: + spec = specStr.group(1) + else: + printError("Could not parse spec from output.") + return False + + passed = set() + skipped = set() + + if 'linux' in spec: + skipped.add('test_not_linux_arch_x86_0') + else: + if 'x86' in spec: + passed.add('test_not_linux_arch_x86_0') + else: + skipped.add('test_not_linux_arch_x86_0') + + if 'x86' in spec: + passed.add('test_arch_x86_0') + skipped.add('test_arch_nonx86_0') + else: + passed.add('test_arch_nonx86_0') + skipped.add('test_arch_x86_0') + + if '64' in spec: + passed.add('test_bits_64_0') + else: + skipped.add('test_bits_64_0') + + if 'osx' in spec: + passed.add('test_os_osx_0') + else: + skipped.add('test_os_osx_0') + + if 'osx_x86-64' in spec: + passed.add('test_osx_x86-64_0') + else: + skipped.add('test_osx_x86-64_0') + + if 'x86' in spec or '390' in spec: + passed.add('test_arch_x86_390_0') + else: + skipped.add('test_arch_x86_390_0') + + if 'osx_x86-64' in spec or ('win_x86' in spec and 'win_x86-64' not in spec) or 'aix_ppc-64' in spec: + passed.add('test_osx_x86-64_win_x86_aix_ppc-64_0') + else: + skipped.add('test_osx_x86-64_win_x86_aix_ppc-64_0') + + os_label = re.search(r"set OS_LABEL to (.*)", stdout) + print("os_label in : {}".format(os_label)) + if os_label is not None: + os_label = os_label.group(1) + label_str = os_label.split(".") + # os_label example: ubuntu.22 + try: + ver = int(label_str[1],10) + if label_str[0] == "ubuntu": + if ver >= 22: + passed.add('test_os_linux_ubuntu22_0') + if ver >= 20: + passed.add('test_os_linux_ubuntu20plus_0') + passed.add('test_os_linux_ubuntu20plus_rhel8plus_0') + if ver < 20: + passed.add('test_not_os_linux_ubuntu20plus_0') + + if label_str[0] == "rhel": + if ver >= 8: + passed.add('test_os_linux_ubuntu20plus_rhel8plus_0') + except ValueError as ve: + print ("warning: os version value failed to convert to an integer") + passed.add('test_not_os_linux_ubuntu20plus_0') + else: + passed.add('test_not_os_linux_ubuntu20plus_0') + + if 'test_os_linux_ubuntu20plus_0' not in passed: + skipped.add('test_os_linux_ubuntu20plus_0') + + if 'test_os_linux_ubuntu22_0' not in passed: + skipped.add('test_os_linux_ubuntu22_0') + + if 'test_os_linux_ubuntu20plus_rhel8plus_0' not in passed: + skipped.add('test_os_linux_ubuntu20plus_rhel8plus_0') + + if 'test_not_os_linux_ubuntu20plus_0' not in passed: + skipped.add('test_not_os_linux_ubuntu20plus_0') + + micro_arch = re.search(r"set MICROARCH to (.*)", stdout) + # micro_arch = "set MICROARCH to z14" + print("micro_arch in platform req: {}".format(micro_arch)) + pattern = r"set MICROARCH to (.*)" + # micro_arch = re.search(pattern, micro_arch) + if micro_arch is not None: + print(micro_arch) + micro_arch = micro_arch.group(1) + label_str = micro_arch + print("Label_str:",label_str) + # micro_arch example: skylake + try: + ver = int(''.join(filter(str.isdigit, micro_arch.split()[-1]))) + print("ver:",ver) + if label_str == "skylake": + passed.add('test_arch_x86_skylake_0') + if ver == 13: + passed.add('test_arch_390_z13_0') + elif ver > 13: + passed.add('test_arch_390_z13plus_0') + if ver == 14: + passed.add('test_arch_390_z14_0') + elif ver > 14: + passed.add('test_arch_390_z14plus_0') + if ver == 15: + passed.add('test_arch_390_z15_0') + elif ver > 15: + passed.add('test_arch_390_z15plus_0') + else: + skipped.add('test_not_arch_390_0') + except ValueError as ve: + print("warning: microarch version value failed to convert to an integer") + skipped.add('test_not_arch_390_0') + else: + passed.add('test_not_arch_390_0') + + if 'test_arch_390_z15_0' not in passed: + skipped.add('test_arch_390_z15_0') + + if 'test_arch_390_z14_0' not in passed: + skipped.add('test_arch_390_z14_0') + + if 'test_arch_390_z13_0' not in passed: + skipped.add('test_arch_390_z13_0') + + if 'test_arch_390_z15plus_0' not in passed: + skipped.add('test_arch_390_z15plus_0') + + if 'test_arch_390_z14plus_0' not in passed: + skipped.add('test_arch_390_z14plus_0') + + if 'test_arch_390_z13plus_0' not in passed: + skipped.add('test_arch_390_z13plus_0') + + if 'test_arch_x86_skylake_0' not in passed: + skipped.add('test_arch_x86_skylake_0') + + + rt &= checkResult(result, passed, set(), set(), skipped) + return rt if __name__ == "__main__": - run_test() \ No newline at end of file + run_test() \ No newline at end of file diff --git a/src/org/openj9/envInfo/MachineInfo.java b/src/org/openj9/envInfo/MachineInfo.java index 94151154..4ee164e6 100644 --- a/src/org/openj9/envInfo/MachineInfo.java +++ b/src/org/openj9/envInfo/MachineInfo.java @@ -38,7 +38,8 @@ public class MachineInfo { public static final String[] LINUX_OS_NAME_CMD = new String[] {"bash", "-c", "grep '^NAME' /etc/os-release | awk -F'=' ' gsub(/\"/,\"\") { print $2}'"}; public static final String[] LINUX_OS_VERSION_CMD = new String[] {"bash", "-c", "grep '^VERSION_ID' /etc/os-release | awk -F'=' ' gsub(/\"/,\"\") { print $2}'"}; - public static final String[] MICRO_ARCH_CMD = new String[] {"bash", "-c", "cat /proc/cpuinfo | grep 'model name' | uniq"}; + public static final String[] MICRO_ARCH_CMD = new String[] {"bash", "-c", "cat /proc/cpuinfo | grep -E 'machine|model name' | uniq"}; + public static final String[] ULIMIT_CMD = new String[] {"bash", "-c", "ulimit -a"}; public static final String[] INSTALLED_MEM_CMD = new String[] {"bash", "-c", "grep MemTotal /proc/meminfo | awk '{print $2}"}; @@ -116,8 +117,8 @@ public String toString() { } private String parseInfo(String output) { - if (output == null) return ""; - Pattern pattern = Pattern.compile("[0-9]+[.][0-9]+([.][0-9]+)?"); + if (output == null) return ""; + Pattern pattern = Pattern.compile("[0-9]+[.][0-9]+([.][0-9]+)?"); Matcher matcher = pattern.matcher(output); if (matcher.find()) { return matcher.group(0); @@ -201,15 +202,36 @@ private void getSysInfo() { putInfo(new Info("sysArch", SYS_ARCH_CMD, ce.execute(SYS_ARCH_CMD), null)); putInfo(new Info("procArch", PROC_ARCH_CMD, ce.execute(PROC_ARCH_CMD), null)); String microArchOutput = ce.execute(MICRO_ARCH_CMD); + String microArch = ""; + // Check for specific machine versions and set the microArch accordingly if (microArchOutput.toLowerCase().contains("skylake")) { - String microArch = "skylake"; + microArch = "skylake"; + } else if (microArchOutput.contains("8562")) { + microArch = "z15"; + } else if (microArchOutput.contains("8561")) { + microArch = "z15"; + } else if (microArchOutput.contains("3907")) { + microArch = "z14"; + } else if (microArchOutput.contains("3906")) { + microArch = "z14"; + } else if (microArchOutput.contains("2965")) { + microArch = "z13"; + } else if (microArchOutput.contains("2964")) { + microArch= "z13"; + } + else { + System.out.println("Unfamiliar microArch detected in TKG. It will not be added in TKG microArch!"); + System.out.println("microArchOutput: " + microArchOutput); + } + + if (!microArch.isEmpty()) { putInfo(new Info("microArch", MICRO_ARCH_CMD, microArch, null)); } putInfo(new Info("sysOS", SYS_OS_CMD, ce.execute(SYS_OS_CMD), null)); putInfo(new Info("ulimit", ULIMIT_CMD, ce.execute(ULIMIT_CMD), null)); putInfo(new Info("docker", CHECK_DOCKER_CMD, ce.execute(CHECK_DOCKER_CMD), null)); } - + private void getOsLabel() { if (System.getProperty("os.name").toLowerCase().contains("linux")) { String osName = ce.execute(LINUX_OS_NAME_CMD).toLowerCase(); @@ -231,7 +253,7 @@ private void getOsLabel() { putInfo(new Info("osInfo", LINUX_OS_CMD, ce.execute(LINUX_OS_CMD), null)); } } - + private void getPrerequisiteInfo() { putInfo(new Info("antVersion", ANT_VERSION_CMD, ce.execute(ANT_VERSION_CMD), "1.9.6")); /* required version is 4.1, but some exception applies, do not verify for now*/ @@ -252,7 +274,7 @@ private void getRuntimeInfo() { putInfo(new Info("vmVendor", new String[] {"ManagementFactory.getRuntimeMXBean().getSpecVendor()"}, ManagementFactory.getRuntimeMXBean().getSpecVendor(), null)); putInfo(new Info("vmVersion", new String[] {"ManagementFactory.getRuntimeMXBean().getVmVersion()"}, ManagementFactory.getRuntimeMXBean().getVmVersion(), null)); } - + private void getPhysicalMemoryInfo() { OperatingSystemMXBean osBean = (OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean(); putInfo(new Info("Total Physical Memory Size", new String[] {"osBean.getTotalPhysicalMemorySize()"}, String.valueOf(osBean.getTotalPhysicalMemorySize()), null)); diff --git a/src/org/testKitGen/TestInfoParser.java b/src/org/testKitGen/TestInfoParser.java index df0b7c67..93eb652d 100644 --- a/src/org/testKitGen/TestInfoParser.java +++ b/src/org/testKitGen/TestInfoParser.java @@ -15,6 +15,7 @@ package org.testKitGen; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -412,7 +413,6 @@ private boolean checkPlatformReq(List platformRequirementsList) { } String spec = arg.getSpec(); String fullSpec = spec; - // Special case 32/31-bit specs which do not have 32 or 31 in the name (i.e. // aix_ppc) if (!spec.contains("-64")) { @@ -442,14 +442,13 @@ private boolean matchPlat(String fullSpec, String pr) { if (!fullSpec.contains(prSplitOnDot[1])) { return false; } - if (prSplitOnDot[0].equals("arch") && (prSplitOnDot.length == 3)) { - String microArch = prSplitOnDot[2]; - if (!microArch.equals(arg.getMicroArch())) { - return false; - } + String requiredMicroArch = prSplitOnDot[2]; + String actualMicroArch = arg.getMicroArch(); + return compareVersion(requiredMicroArch, actualMicroArch); } else if (prSplitOnDot[0].equals("os") && (prSplitOnDot.length == 4)) { String osName = prSplitOnDot[2]; + String osVersion = prSplitOnDot[3]; if (arg.getOsLabel().isEmpty()) { return false; } @@ -457,31 +456,42 @@ private boolean matchPlat(String fullSpec, String pr) { if (!osLabelArg[0].equals(osName)) { return false; } - String osVersion = prSplitOnDot[3]; - if (osVersion.endsWith("+")) { - int verInt = 0; - try { - verInt = Integer.parseInt(osVersion.substring(0, osVersion.length() - 1)); - } catch (NumberFormatException e) { - System.out.println("Error: unrecognized platformRequirement: " + prSplitOnDot + ". Only support integer OS version."); - System.exit(1); - } - int argVerInt = 0; + return compareVersion(osVersion, osLabelArg[1]); + } + return true; + } + + private boolean compareVersion(String requiredLabel, String actualLabel) { + if (requiredLabel.isEmpty() || actualLabel.isEmpty()) { + return false; + } else if (requiredLabel.equals(actualLabel)) { + return true; + } else if (requiredLabel.endsWith("+")) { + Pattern pattern = Pattern.compile("(\\D+)?(\\d+)"); + Matcher requiredLabelMatcher = pattern.matcher(requiredLabel); + Matcher actualLabelMatcher = pattern.matcher(actualLabel); + + if (requiredLabelMatcher.find() && actualLabelMatcher.find()) { + String requiredPrefix = requiredLabelMatcher.group(1) != null ? requiredLabelMatcher.group(1) : ""; + String actualPrefix = actualLabelMatcher.group(1) != null ? actualLabelMatcher.group(1) : ""; + if (requiredPrefix.equals(actualPrefix)) { + int requiredLabelNum = 0; + int actualLabelNum = 0; try { - argVerInt = Integer.parseInt(osLabelArg[1]); + requiredLabelNum = Integer.parseInt(requiredLabelMatcher.group(2)); + actualLabelNum = Integer.parseInt(actualLabelMatcher.group(2)); } catch (NumberFormatException e) { - System.out.println("Error: unrecognized osLabel: " + arg.getOsLabel() + ". Only support integer OS version."); - System.exit(1); - } - if (verInt > argVerInt) { - return false; + System.out.println("Error: unrecognized requiredLabel:" + requiredLabel + " or actualLabel:" + actualLabel); + System.err.println(e.getMessage()); + System.exit(1); } - } else { - if (!osLabelArg[1].equals(osVersion)) { - return false; + if (actualLabelNum >= requiredLabelNum) { + return true; } } + } } - return true; - } + return false; + } + }