diff --git a/build.py b/build.py index 8b0bfd4a22..deb5603649 100755 --- a/build.py +++ b/build.py @@ -37,6 +37,15 @@ import tarfile import time import zipfile +import json + +########################################################### +# mapping for platform string from python builtin str +platform_mapping = { + "darwin": "osx", + "windows": "win", + "linux": "lin" + } ############################################################ # Helpers for printing output @@ -635,6 +644,40 @@ def Package(context): ############################################################ # InstallContext class InstallContext: + + kFilesToExclude = [] + kPluginsToExclude = [] + kPluginsToInclude = [] + kPlatformToInclude = [] + kPlatformToExclude = [] + allLabelsToInclude = [] + + ctestArgs = list() + + # get list of labels to pass to ctest-args + def get_ctest_labels(self): + try: + with open("tests-to-run.json", 'r') as file: + ctest_labels_config = json.load(file) + except json.JSONDecodeError as e: + print(f"Failed to decode JSON: {e}") + except FileNotFoundError as e: + print(f"File not found: {e}") + except Exception as e: + print(f"An error occurred: {e}") + + for key, values in ctest_labels_config.items(): + if key == "plugins_to_include": + self.kPluginsToInclude = values + if key == "plugins_to_exclude": + self.kPluginsToExclude = values + if key == "platforms_to_include": + self.kPlatformToInclude = values + if key == "platforms_to_exclude": + self.kPlatformToExclude = values + if key == "files_to_exclude": + self.kFilesToExclude = values + def __init__(self, args): # Assume the project's top level cmake is in the current source directory self.mayaHydraSrcDir = os.path.normpath( @@ -716,12 +759,33 @@ def __init__(self, args): self.stagesArgs.append(arg) # CTest arguments - self.ctestArgs = list() for argList in args.ctest_args: for arg in argList.split(","): self.ctestArgs.append(arg) - # Redirect output stream to file + # get labels to be passed for ctest + self.get_ctest_labels() + + # add -E args for test file to be excluded by name + if self.kFilesToExclude: + self.ctestArgs.append(f'{"-E"} {"|".join(self.kFilesToExclude)}') + + # add -L args, test with following labels to run + if self.kPluginsToInclude: + self.ctestArgs.append(f'{"-L"} {"|".join(self.kPluginsToInclude)}') + + # Determine the current platform + current_platform = platform.system().lower() + exclPlatform = [] + for a in self.kPlatformToExclude: + if platform_mapping.get(current_platform) in a: + exclPlatform.append(a) + + allExcludeLabels = self.kPluginsToExclude + exclPlatform + # add -LE args, test with following labels to be skipped + if allExcludeLabels: + self.ctestArgs.append(f'{"-LE"} {"|".join(allExcludeLabels)}') + self.redirectOutstreamFile = args.redirect_outstream_file try: diff --git a/cmake/test.cmake b/cmake/test.cmake index c5bf6b674d..52abe999ec 100644 --- a/cmake/test.cmake +++ b/cmake/test.cmake @@ -32,6 +32,54 @@ if(MayaUsd_FOUND) endif() endif() +function(find_labels label_set label_list) + string(REPLACE ":" ";" split_labels ${label_set}) + list(LENGTH split_labels len) + if(len GREATER 0) + list(GET split_labels 1 labels_value) + # we expect comma separated labels + string(REPLACE "," ";" all_labels ${labels_value}) + set(local_label_list "") + foreach(label ${all_labels}) + list(APPEND local_label_list ${label}) + endforeach() + set(${label_list} ${local_label_list} PARENT_SCOPE) + endif() +endfunction() + +function(get_testfile_and_labels all_labels test_filename test_script) + # fetch labels for each test file + string(REPLACE "|" ";" tests_with_tags ${test_script}) + list(GET tests_with_tags 0 filename) + # set the test file to input as no labels were passed + set(${test_filename} ${filename} PARENT_SCOPE) + list(LENGTH tests_with_tags length) + math(EXPR one_less_length "${length} - 1") + if(length GREATER 1) + set(collect_labels "") + foreach(i RANGE 1 ${one_less_length}) + list(GET tests_with_tags ${i} item) + find_labels(${item} label_list) + list(APPEND collect_labels ${label_list}) + endforeach() + set(${all_labels} ${collect_labels} PARENT_SCOPE) + else() + set(${all_labels} "" PARENT_SCOPE) + endif() +endfunction() + +function(apply_labels_to_test test_labels test_file) + set_property(TEST ${test_file} APPEND PROPERTY LABELS "default") + list(LENGTH test_labels list_length) + if(${list_length} GREATER 0) + # if(NOT ${test_labels} STREQUAL "") + foreach(label ${test_labels}) + set_property(TEST ${test_file} APPEND PROPERTY LABELS ${label}) + message(STATUS "Added test label \"${label}\" for ${test_file}") + endforeach() + endif() +endfunction() + function(mayaUsd_get_unittest_target unittest_target unittest_basename) get_filename_component(unittest_name ${unittest_basename} NAME_WE) set(${unittest_target} "${unittest_name}" PARENT_SCOPE) @@ -94,7 +142,7 @@ endif() # separate_argument_list before passing to this func # if you start with a cmake-style list. # -function(mayaUsd_add_test test_name) +function(mayaUsd_add_test test_name) # ----------------- # 1) Arg processing # ----------------- diff --git a/test/lib/mayaUsd/render/mayaToHydra/CMakeLists.txt b/test/lib/mayaUsd/render/mayaToHydra/CMakeLists.txt index f54ee68148..3345b4d701 100644 --- a/test/lib/mayaUsd/render/mayaToHydra/CMakeLists.txt +++ b/test/lib/mayaUsd/render/mayaToHydra/CMakeLists.txt @@ -1,4 +1,10 @@ # Interactive Unit test scripts (launched with maya.exe instead of mayapy.exe) +# Unit test can be disabled with ctest labels. +# Adding labels to the unit test follows this convention: +# testMyTestFile.py|depOnPlugins:lookdev,mtoa,newPlugin|skipOnPlatform:[osx,win,lin] +# See accompanying README.md for currently supported list of label mapping. +# only single platform exclusion is supported currently. +# For specifying exclusion based on plugin dependency, look at: test_to_run.json set(INTERACTIVE_TEST_SCRIPT_FILES testImageDiffing.py testMtohCommand.py @@ -12,17 +18,17 @@ set(INTERACTIVE_TEST_SCRIPT_FILES testStageAddPrim.py testTransforms.py testRefinement.py - testMaterialXOnNative.py + testMaterialXOnNative.py|depOnPlugins:lookdevx testNewSceneWithStage.py # To be reenabled after investigation - #testMayaDisplayModes.py + #testMayaDisplayModes.py|skipOnPlatform:osx testMayaShadingModes.py testMayaDisplayLayers.py testMayaIsolateSelect.py testMayaLights.py testUSDLights.py testUVandUDIM.py - testArnoldLights.py + testArnoldLights.py|depOnPlugins:mtoa testLookThrough.py testObjectTemplate.py testStandardSurface.py @@ -37,7 +43,7 @@ set(INTERACTIVE_TEST_SCRIPT_FILES testGrid.py testUsdTextureToggle.py # To be reenabled after investigation - #testDataProducerSelHighlight.py + testDataProducerSelHighlight.py|skipOnPlatform:osx testPassingNormalsOnMayaNative.py testViewportFilters.py cpp/testColorPreferences.py @@ -88,47 +94,15 @@ set(INTERACTIVE_TEST_SCRIPT_FILES_DISABLE_VP2_RENDER_DELEGATE cpp/testUsdStageFromFile.py ) -# Unit test scripts run with mayapy.exe (no UI, so no image diffing and not possible to set Hydra Storm as the renderer) -set(TEST_SCRIPT_FILES -) - -foreach(script ${TEST_SCRIPT_FILES}) - mayaUsd_get_unittest_target(target ${script}) - mayaUsd_add_test(${target} - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - PYTHON_MODULE ${target} - ENV - "MAYA_PLUG_IN_PATH=${CMAKE_INSTALL_PREFIX}/lib/maya" - "LD_LIBRARY_PATH=${ADDITIONAL_LD_LIBRARY_PATH}" - "IMAGE_DIFF_TOOL=${IMAGE_DIFF_TOOL}" - - # LD_LIBRARY_PATH needs to be set for the idiff executable because its - # RPATH is absolute rather than relative to ORIGIN, meaning the RPATH - # points to the absolute path on the machine where idiff was built. - # This absence of relative paths for RPATH comes from OpenImageIO. - # We introduce a second workaround to avoid Maya using usd's libpng, - # because both use incompatible versions of libpng. This is done by - # setting LD_LIBRARY_PATH to IDIFF_LD_LIBRARY_PATH only when we run - # idiff using Python's subprocess module. - "IDIFF_LD_LIBRARY_PATH=${ADDITIONAL_LD_LIBRARY_PATH}:${PXR_USD_LOCATION}/lib64:${PXR_USD_LOCATION}/lib" - - # Maya uses a very old version of GLEW, so we need support for - # pre-loading a newer version from elsewhere. - "LD_PRELOAD=${ADDITIONAL_LD_PRELOAD}" - ) - - # Assign a CTest label to these tests for easy filtering. - set_property(TEST ${target} APPEND PROPERTY LABELS mayaHydra) -endforeach() - # Use mesh adapter for mesh support instead of MRenderItem. using Interactive (Maya.exe with UI) foreach(script ${INTERACTIVE_TEST_SCRIPT_FILES_MESH_ADAPTER}) - mayaUsd_get_unittest_target(target ${script}) + get_testfile_and_labels(all_labels test_filename ${script}) + mayaUsd_get_unittest_target(target ${test_filename}) set(target "${target}_meshAdapter") mayaUsd_add_test(${target} INTERACTIVE WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - PYTHON_SCRIPT ${script} + PYTHON_SCRIPT ${test_filename} ENV "MAYA_PLUG_IN_PATH=${CMAKE_INSTALL_PREFIX}/lib/maya" "LD_LIBRARY_PATH=${ADDITIONAL_LD_LIBRARY_PATH}" @@ -139,16 +113,18 @@ foreach(script ${INTERACTIVE_TEST_SCRIPT_FILES_MESH_ADAPTER}) ) # Assign a CTest label to these tests for easy filtering. - set_property(TEST ${target} APPEND PROPERTY LABELS mayaHydraMeshAdapter) + apply_labels_to_test("${all_labels}" ${target}) + endforeach() # Disable VP2 render delegate and use interactive Maya with UI. foreach(script ${INTERACTIVE_TEST_SCRIPT_FILES_DISABLE_VP2_RENDER_DELEGATE}) - mayaUsd_get_unittest_target(target ${script}) + get_testfile_and_labels(all_labels test_filename ${script}) + mayaUsd_get_unittest_target(target ${test_filename}) mayaUsd_add_test(${target} INTERACTIVE WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - PYTHON_SCRIPT ${script} + PYTHON_SCRIPT ${test_filename} ENV "MAYA_PLUG_IN_PATH=${CMAKE_INSTALL_PREFIX}/lib/maya" "LD_LIBRARY_PATH=${ADDITIONAL_LD_LIBRARY_PATH}" @@ -159,16 +135,18 @@ foreach(script ${INTERACTIVE_TEST_SCRIPT_FILES_DISABLE_VP2_RENDER_DELEGATE}) ) # Assign a CTest label to these tests for easy filtering. - set_property(TEST ${target} APPEND PROPERTY LABELS MayaHydraInteractive) + apply_labels_to_test("${all_labels}" ${target}) + endforeach() # Use Maya.exe with UI, interactive tests. -foreach(script ${INTERACTIVE_TEST_SCRIPT_FILES}) - mayaUsd_get_unittest_target(target ${script}) +foreach(script ${INTERACTIVE_TEST_SCRIPT_FILES}) + get_testfile_and_labels(all_labels test_filename ${script}) + mayaUsd_get_unittest_target(target ${test_filename}) mayaUsd_add_test(${target} INTERACTIVE WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - PYTHON_SCRIPT ${script} + PYTHON_SCRIPT ${test_filename} ENV "MAYA_PLUG_IN_PATH=${CMAKE_INSTALL_PREFIX}/lib/maya" "LD_LIBRARY_PATH=${ADDITIONAL_LD_LIBRARY_PATH}" @@ -191,7 +169,8 @@ foreach(script ${INTERACTIVE_TEST_SCRIPT_FILES}) ) # Add a ctest label to these tests for easy filtering. - set_property(TEST ${target} APPEND PROPERTY LABELS MayaHydraInteractive) + apply_labels_to_test("${all_labels}" ${target}) + endforeach() # C++ unit tests diff --git a/test/lib/mayaUsd/render/mayaToHydra/README.md b/test/lib/mayaUsd/render/mayaToHydra/README.md index f32dc05913..6779cbe543 100644 --- a/test/lib/mayaUsd/render/mayaToHydra/README.md +++ b/test/lib/mayaUsd/render/mayaToHydra/README.md @@ -96,6 +96,43 @@ To create a reference snapshot, an easy way is to first write your test, and the :warning: **Important note** : While there *is* an `assertSnapshotEqual` method, its use is discouraged, as renders can differ very slightly between renderer architectures. +# Adding labels for tests +ctest allows attaching labels to tests to determine whether a test needs to be run or not. + +For example, you might have some tests that are all related to 'performance', so you label them with 'performance'. Other tests might be about 'security', so you label them 'security'. Later, when you want to run your tests, you can tell CTest to only run the tests with a certain label. Say you only want to run your 'performance' tests. You can tell CTest to do this by using the `-L` option followed by 'performance'. CTest will then only run the tests that have the 'performance' label, and ignore the rest. Similarly, tests can be excluded by passing arguments to `-LE` flag. + +To add label in MayaHydra test suite +1. For any particlar unit test of interest in [CMakeLists.txt](./CMakeLists.txt), add the required labels with the following convention: + `testMyTestFile.py|depOnPlugins:lookdev,mtoa,newPlugin|skipOnPlatform:[osx,win,lin]` +2. Where the label groups follows the unit test name and are separated by `'|'`. You may have multiple label groups. +3. The label groups are of key:values type. They label key can be any meaningful string and has correspondence to the accompanying [json file](../../../../../../maya-hydra/tests-to-run.json) +4. The label values are comma separated lists. The values are added to the test via cmake's `set_tests_properties` + +MayaHydra currently is filtering tests based on dependent plugins and platform on which to run them. + +To add flags to ctest commandline, modify the corresponding [JSON file](../../../../../../maya-hydra/tests-to-run.json) to indicate which test to run or skip. +For ex: if you want to run only tests tagged with "lookdevx" then edit the json file like so: `"plugins_to_include": ["lookdevx]`. If the include list is empty, all tests are run by default. Similarly, edit `"plugin_to_exclude"` in the json file to skip test with certain labels. + +Note that for platform dependent runs, we only support exclusion mode for a single platform. So if you tag the test `skipOnPlatform:osx`, it skips on MacOS. No corresponding changes to json file is required in this case. + +Label mapping +As mentioned earlier in (4), the label key/values in CMakeLists have a correspondence to the values in [JSON file](../../../../../../maya-hydra/tests-to-run.json). For ex: +A line in CMakeLists.txt like so: +`testMyNewTestFile|depOnPlugins:lookdevx,mtoa` would have a corresponding JSON file like so: +`"plugins_to_include": ["lookdevx","mtoa"]` + +`testAnotherTestFile||depOnPlugins:bifrost` would have a corresponding JSON file like so: +`"plugins_to_exclude": ["bifrost"]` + +Similar pattern applies for exclusion/inclusion of tests by filenames except that there is no correpsonding change required in CMakeLists.txt as the pattern matching is by the test name. + +Note that `depOnPlugins` key string is only used as cue while adding values in the JSON. The variables in the JSON file and the label values in the CMakeLists.txt are what get used by ctest for label matching. +`"files_to_exclude": ["testStandardSurface"]` + +The supported values for platform exclusion are: "osx", "win", "lin". + + + # Utilities Utility files are located under [/test/testUtils](../../../../testUtils/). Note that at the time of writing this (2023/08/16), many of the utils files and their contents were inherited from the USD plugin, and are not all used. diff --git a/tests-to-run.json b/tests-to-run.json new file mode 100644 index 0000000000..dcaf04dbb2 --- /dev/null +++ b/tests-to-run.json @@ -0,0 +1,7 @@ +{ + "files_to_include": [], + "files_to_exclude": [], + "plugins_to_include": [], + "plugins_to_exclude": [], + "platforms_to_exclude": ["osx"] +} \ No newline at end of file