Skip to content

Commit

Permalink
HYDRA-1033 : Support for controlled unit test disabling (#134)
Browse files Browse the repository at this point in the history
* First pass at adding support for include/exclude of running unit test via ctest labels

* Re-enable test with OSX skip tags

* disable mayaDisplayModes test

* Update README

* Move set_property to a function. Add "default' labels on all tests.

* added support for exclusion by filename. addressed comments

* update json file

* Update README.md
  • Loading branch information
roopavr-adsk authored Jun 6, 2024
1 parent 3e52cb4 commit 9b698ab
Show file tree
Hide file tree
Showing 5 changed files with 185 additions and 50 deletions.
68 changes: 66 additions & 2 deletions build.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,15 @@
import tarfile
import time
import zipfile
import json

###########################################################
# mapping for platform string from python builtin str
platform_mapping = {
"darwin": "osx",
"windows": "win",
"linux": "lin"
}

############################################################
# Helpers for printing output
Expand Down Expand Up @@ -635,6 +644,40 @@ def Package(context):
############################################################
# InstallContext
class InstallContext:

kFilesToExclude = []
kPluginsToExclude = []
kPluginsToInclude = []
kPlatformToInclude = []
kPlatformToExclude = []
allLabelsToInclude = []

ctestArgs = list()

# get list of labels to pass to ctest-args
def get_ctest_labels(self):
try:
with open("tests-to-run.json", 'r') as file:
ctest_labels_config = json.load(file)
except json.JSONDecodeError as e:
print(f"Failed to decode JSON: {e}")
except FileNotFoundError as e:
print(f"File not found: {e}")
except Exception as e:
print(f"An error occurred: {e}")

for key, values in ctest_labels_config.items():
if key == "plugins_to_include":
self.kPluginsToInclude = values
if key == "plugins_to_exclude":
self.kPluginsToExclude = values
if key == "platforms_to_include":
self.kPlatformToInclude = values
if key == "platforms_to_exclude":
self.kPlatformToExclude = values
if key == "files_to_exclude":
self.kFilesToExclude = values

def __init__(self, args):
# Assume the project's top level cmake is in the current source directory
self.mayaHydraSrcDir = os.path.normpath(
Expand Down Expand Up @@ -716,12 +759,33 @@ def __init__(self, args):
self.stagesArgs.append(arg)

# CTest arguments
self.ctestArgs = list()
for argList in args.ctest_args:
for arg in argList.split(","):
self.ctestArgs.append(arg)

# Redirect output stream to file
# get labels to be passed for ctest
self.get_ctest_labels()

# add -E args for test file to be excluded by name
if self.kFilesToExclude:
self.ctestArgs.append(f'{"-E"} {"|".join(self.kFilesToExclude)}')

# add -L args, test with following labels to run
if self.kPluginsToInclude:
self.ctestArgs.append(f'{"-L"} {"|".join(self.kPluginsToInclude)}')

# Determine the current platform
current_platform = platform.system().lower()
exclPlatform = []
for a in self.kPlatformToExclude:
if platform_mapping.get(current_platform) in a:
exclPlatform.append(a)

allExcludeLabels = self.kPluginsToExclude + exclPlatform
# add -LE args, test with following labels to be skipped
if allExcludeLabels:
self.ctestArgs.append(f'{"-LE"} {"|".join(allExcludeLabels)}')

self.redirectOutstreamFile = args.redirect_outstream_file

try:
Expand Down
50 changes: 49 additions & 1 deletion cmake/test.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,54 @@ if(MayaUsd_FOUND)
endif()
endif()

function(find_labels label_set label_list)
string(REPLACE ":" ";" split_labels ${label_set})
list(LENGTH split_labels len)
if(len GREATER 0)
list(GET split_labels 1 labels_value)
# we expect comma separated labels
string(REPLACE "," ";" all_labels ${labels_value})
set(local_label_list "")
foreach(label ${all_labels})
list(APPEND local_label_list ${label})
endforeach()
set(${label_list} ${local_label_list} PARENT_SCOPE)
endif()
endfunction()

function(get_testfile_and_labels all_labels test_filename test_script)
# fetch labels for each test file
string(REPLACE "|" ";" tests_with_tags ${test_script})
list(GET tests_with_tags 0 filename)
# set the test file to input as no labels were passed
set(${test_filename} ${filename} PARENT_SCOPE)
list(LENGTH tests_with_tags length)
math(EXPR one_less_length "${length} - 1")
if(length GREATER 1)
set(collect_labels "")
foreach(i RANGE 1 ${one_less_length})
list(GET tests_with_tags ${i} item)
find_labels(${item} label_list)
list(APPEND collect_labels ${label_list})
endforeach()
set(${all_labels} ${collect_labels} PARENT_SCOPE)
else()
set(${all_labels} "" PARENT_SCOPE)
endif()
endfunction()

function(apply_labels_to_test test_labels test_file)
set_property(TEST ${test_file} APPEND PROPERTY LABELS "default")
list(LENGTH test_labels list_length)
if(${list_length} GREATER 0)
# if(NOT ${test_labels} STREQUAL "")
foreach(label ${test_labels})
set_property(TEST ${test_file} APPEND PROPERTY LABELS ${label})
message(STATUS "Added test label \"${label}\" for ${test_file}")
endforeach()
endif()
endfunction()

function(mayaUsd_get_unittest_target unittest_target unittest_basename)
get_filename_component(unittest_name ${unittest_basename} NAME_WE)
set(${unittest_target} "${unittest_name}" PARENT_SCOPE)
Expand Down Expand Up @@ -94,7 +142,7 @@ endif()
# separate_argument_list before passing to this func
# if you start with a cmake-style list.
#
function(mayaUsd_add_test test_name)
function(mayaUsd_add_test test_name)
# -----------------
# 1) Arg processing
# -----------------
Expand Down
73 changes: 26 additions & 47 deletions test/lib/mayaUsd/render/mayaToHydra/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,4 +1,10 @@
# Interactive Unit test scripts (launched with maya.exe instead of mayapy.exe)
# Unit test can be disabled with ctest labels.
# Adding labels to the unit test follows this convention:
# testMyTestFile.py|depOnPlugins:lookdev,mtoa,newPlugin|skipOnPlatform:[osx,win,lin]
# See accompanying README.md for currently supported list of label mapping.
# only single platform exclusion is supported currently.
# For specifying exclusion based on plugin dependency, look at: test_to_run.json
set(INTERACTIVE_TEST_SCRIPT_FILES
testImageDiffing.py
testMtohCommand.py
Expand All @@ -12,17 +18,17 @@ set(INTERACTIVE_TEST_SCRIPT_FILES
testStageAddPrim.py
testTransforms.py
testRefinement.py
testMaterialXOnNative.py
testMaterialXOnNative.py|depOnPlugins:lookdevx
testNewSceneWithStage.py
# To be reenabled after investigation
#testMayaDisplayModes.py
#testMayaDisplayModes.py|skipOnPlatform:osx
testMayaShadingModes.py
testMayaDisplayLayers.py
testMayaIsolateSelect.py
testMayaLights.py
testUSDLights.py
testUVandUDIM.py
testArnoldLights.py
testArnoldLights.py|depOnPlugins:mtoa
testLookThrough.py
testObjectTemplate.py
testStandardSurface.py
Expand All @@ -37,7 +43,7 @@ set(INTERACTIVE_TEST_SCRIPT_FILES
testGrid.py
testUsdTextureToggle.py
# To be reenabled after investigation
#testDataProducerSelHighlight.py
testDataProducerSelHighlight.py|skipOnPlatform:osx
testPassingNormalsOnMayaNative.py
testViewportFilters.py
cpp/testColorPreferences.py
Expand Down Expand Up @@ -88,47 +94,15 @@ set(INTERACTIVE_TEST_SCRIPT_FILES_DISABLE_VP2_RENDER_DELEGATE
cpp/testUsdStageFromFile.py
)

# Unit test scripts run with mayapy.exe (no UI, so no image diffing and not possible to set Hydra Storm as the renderer)
set(TEST_SCRIPT_FILES
)

foreach(script ${TEST_SCRIPT_FILES})
mayaUsd_get_unittest_target(target ${script})
mayaUsd_add_test(${target}
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
PYTHON_MODULE ${target}
ENV
"MAYA_PLUG_IN_PATH=${CMAKE_INSTALL_PREFIX}/lib/maya"
"LD_LIBRARY_PATH=${ADDITIONAL_LD_LIBRARY_PATH}"
"IMAGE_DIFF_TOOL=${IMAGE_DIFF_TOOL}"

# LD_LIBRARY_PATH needs to be set for the idiff executable because its
# RPATH is absolute rather than relative to ORIGIN, meaning the RPATH
# points to the absolute path on the machine where idiff was built.
# This absence of relative paths for RPATH comes from OpenImageIO.
# We introduce a second workaround to avoid Maya using usd's libpng,
# because both use incompatible versions of libpng. This is done by
# setting LD_LIBRARY_PATH to IDIFF_LD_LIBRARY_PATH only when we run
# idiff using Python's subprocess module.
"IDIFF_LD_LIBRARY_PATH=${ADDITIONAL_LD_LIBRARY_PATH}:${PXR_USD_LOCATION}/lib64:${PXR_USD_LOCATION}/lib"

# Maya uses a very old version of GLEW, so we need support for
# pre-loading a newer version from elsewhere.
"LD_PRELOAD=${ADDITIONAL_LD_PRELOAD}"
)

# Assign a CTest label to these tests for easy filtering.
set_property(TEST ${target} APPEND PROPERTY LABELS mayaHydra)
endforeach()

# Use mesh adapter for mesh support instead of MRenderItem. using Interactive (Maya.exe with UI)
foreach(script ${INTERACTIVE_TEST_SCRIPT_FILES_MESH_ADAPTER})
mayaUsd_get_unittest_target(target ${script})
get_testfile_and_labels(all_labels test_filename ${script})
mayaUsd_get_unittest_target(target ${test_filename})
set(target "${target}_meshAdapter")
mayaUsd_add_test(${target}
INTERACTIVE
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
PYTHON_SCRIPT ${script}
PYTHON_SCRIPT ${test_filename}
ENV
"MAYA_PLUG_IN_PATH=${CMAKE_INSTALL_PREFIX}/lib/maya"
"LD_LIBRARY_PATH=${ADDITIONAL_LD_LIBRARY_PATH}"
Expand All @@ -139,16 +113,18 @@ foreach(script ${INTERACTIVE_TEST_SCRIPT_FILES_MESH_ADAPTER})
)

# Assign a CTest label to these tests for easy filtering.
set_property(TEST ${target} APPEND PROPERTY LABELS mayaHydraMeshAdapter)
apply_labels_to_test("${all_labels}" ${target})

endforeach()

# Disable VP2 render delegate and use interactive Maya with UI.
foreach(script ${INTERACTIVE_TEST_SCRIPT_FILES_DISABLE_VP2_RENDER_DELEGATE})
mayaUsd_get_unittest_target(target ${script})
get_testfile_and_labels(all_labels test_filename ${script})
mayaUsd_get_unittest_target(target ${test_filename})
mayaUsd_add_test(${target}
INTERACTIVE
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
PYTHON_SCRIPT ${script}
PYTHON_SCRIPT ${test_filename}
ENV
"MAYA_PLUG_IN_PATH=${CMAKE_INSTALL_PREFIX}/lib/maya"
"LD_LIBRARY_PATH=${ADDITIONAL_LD_LIBRARY_PATH}"
Expand All @@ -159,16 +135,18 @@ foreach(script ${INTERACTIVE_TEST_SCRIPT_FILES_DISABLE_VP2_RENDER_DELEGATE})
)

# Assign a CTest label to these tests for easy filtering.
set_property(TEST ${target} APPEND PROPERTY LABELS MayaHydraInteractive)
apply_labels_to_test("${all_labels}" ${target})

endforeach()

# Use Maya.exe with UI, interactive tests.
foreach(script ${INTERACTIVE_TEST_SCRIPT_FILES})
mayaUsd_get_unittest_target(target ${script})
foreach(script ${INTERACTIVE_TEST_SCRIPT_FILES})
get_testfile_and_labels(all_labels test_filename ${script})
mayaUsd_get_unittest_target(target ${test_filename})
mayaUsd_add_test(${target}
INTERACTIVE
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
PYTHON_SCRIPT ${script}
PYTHON_SCRIPT ${test_filename}
ENV
"MAYA_PLUG_IN_PATH=${CMAKE_INSTALL_PREFIX}/lib/maya"
"LD_LIBRARY_PATH=${ADDITIONAL_LD_LIBRARY_PATH}"
Expand All @@ -191,7 +169,8 @@ foreach(script ${INTERACTIVE_TEST_SCRIPT_FILES})
)

# Add a ctest label to these tests for easy filtering.
set_property(TEST ${target} APPEND PROPERTY LABELS MayaHydraInteractive)
apply_labels_to_test("${all_labels}" ${target})

endforeach()

# C++ unit tests
Expand Down
37 changes: 37 additions & 0 deletions test/lib/mayaUsd/render/mayaToHydra/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,43 @@ To create a reference snapshot, an easy way is to first write your test, and the

:warning: **Important note** : While there *is* an `assertSnapshotEqual` method, its use is discouraged, as renders can differ very slightly between renderer architectures.

# Adding labels for tests
ctest allows attaching labels to tests to determine whether a test needs to be run or not.

For example, you might have some tests that are all related to 'performance', so you label them with 'performance'. Other tests might be about 'security', so you label them 'security'. Later, when you want to run your tests, you can tell CTest to only run the tests with a certain label. Say you only want to run your 'performance' tests. You can tell CTest to do this by using the `-L` option followed by 'performance'. CTest will then only run the tests that have the 'performance' label, and ignore the rest. Similarly, tests can be excluded by passing arguments to `-LE` flag.

To add label in MayaHydra test suite
1. For any particlar unit test of interest in [CMakeLists.txt](./CMakeLists.txt), add the required labels with the following convention:
`testMyTestFile.py|depOnPlugins:lookdev,mtoa,newPlugin|skipOnPlatform:[osx,win,lin]`
2. Where the label groups follows the unit test name and are separated by `'|'`. You may have multiple label groups.
3. The label groups are of key:values type. They label key can be any meaningful string and has correspondence to the accompanying [json file](../../../../../../maya-hydra/tests-to-run.json)
4. The label values are comma separated lists. The values are added to the test via cmake's `set_tests_properties`

MayaHydra currently is filtering tests based on dependent plugins and platform on which to run them.

To add flags to ctest commandline, modify the corresponding [JSON file](../../../../../../maya-hydra/tests-to-run.json) to indicate which test to run or skip.
For ex: if you want to run only tests tagged with "lookdevx" then edit the json file like so: `"plugins_to_include": ["lookdevx]`. If the include list is empty, all tests are run by default. Similarly, edit `"plugin_to_exclude"` in the json file to skip test with certain labels.
Note that for platform dependent runs, we only support exclusion mode for a single platform. So if you tag the test `skipOnPlatform:osx`, it skips on MacOS. No corresponding changes to json file is required in this case.
Label mapping
As mentioned earlier in (4), the label key/values in CMakeLists have a correspondence to the values in [JSON file](../../../../../../maya-hydra/tests-to-run.json). For ex:
A line in CMakeLists.txt like so:
`testMyNewTestFile|depOnPlugins:lookdevx,mtoa` would have a corresponding JSON file like so:
`"plugins_to_include": ["lookdevx","mtoa"]`
`testAnotherTestFile||depOnPlugins:bifrost` would have a corresponding JSON file like so:
`"plugins_to_exclude": ["bifrost"]`
Similar pattern applies for exclusion/inclusion of tests by filenames except that there is no correpsonding change required in CMakeLists.txt as the pattern matching is by the test name.
Note that `depOnPlugins` key string is only used as cue while adding values in the JSON. The variables in the JSON file and the label values in the CMakeLists.txt are what get used by ctest for label matching.
`"files_to_exclude": ["testStandardSurface"]`
The supported values for platform exclusion are: "osx", "win", "lin".
# Utilities
Utility files are located under [/test/testUtils](../../../../testUtils/). Note that at the time of writing this (2023/08/16), many of the utils files and their contents were inherited from the USD plugin, and are not all used.
7 changes: 7 additions & 0 deletions tests-to-run.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
{
"files_to_include": [],
"files_to_exclude": [],
"plugins_to_include": [],
"plugins_to_exclude": [],
"platforms_to_exclude": ["osx"]
}

0 comments on commit 9b698ab

Please sign in to comment.