HYDRA-861 & HYDRA-932 : Fix MtoA & LookdevX loading in tests + Relate…

…d test improvements (#108)

* HYDRA-861 : Fix mtoa loading

* HYDRA-861 : Make USD stage path relative in testArnoldLights scene

* HYDRA-932 : Remove Arnold/mtoa/ai references in testMayaDisplayModes scene

* HYDRA-861 : Fix LookdevX loading

* HYDRA-861 : Rework plugin loading test infrastructure and fail if plugin can't load

* HYDRA-861 : Use relative paths in all test scene files

* HYDRA-861 : Kill ADPClientService.exe on test suite end

* HYDRA-861 : Add comment for killing ADPClientService

* HYDRA-861 : Update point instance picking test and remove specific checkForPlugin functions

* HYDRA-861 : Add back skipping testSceneBrowser on OSX

* HYDRA-861 : Document force flag usage in taskkill

* HYDRA-861 : Add best practices section in test README

* HYDRA-861 : Document MAYA_MODULE_PATH usage for MtoA

* HYDRA-861 : Re-add existing line about MAYA_MODULE_PATH

* HYDRA-861 : Add note on what happens before each test

* HYDRA-861 : Add other best practices

* HYDRA-861 : Rename _extraPluginsToLoad to _requiredPlugins

* HYDRA-861 : Adjust testRefinement and testUsdNativeInstancePicking + remove a bunch of unused imports

cmake/test.cmake

@@ -224,6 +224,7 @@ finally:
 
     set(ALL_PATH_VARS
         PYTHONPATH
+        MAYA_MODULE_PATH
         MAYA_PLUG_IN_PATH
         MAYA_SCRIPT_PATH
         XBMLANGPATH
@@ -299,18 +300,14 @@ finally:
 
     # mtoa
     if(DEFINED MTOA_LOCATION)
-        list(APPEND MAYAUSD_VARNAME_PATH
-             "${MTOA_LOCATION}/bin")
-        list(APPEND MAYAUSD_VARNAME_MAYA_SCRIPT_PATH
-             "${MTOA_LOCATION}/scripts/mtoa/mel")   
-        list(APPEND MAYAUSD_VARNAME_MAYA_PXR_PLUGINPATH_NAME
-             "${MTOA_LOCATION}/usd")
-        list(APPEND MAYAUSD_VARNAME_MAYA_RENDER_DESC_PATH
-             "${MTOA_LOCATION}/")
-        list(APPEND MAYAUSD_MATERIALX_SEARCH_PATH
-             "${MTOA_LOCATION}/materialx/arnold")
-             list(APPEND MAYAUSD_MATERIALX_SEARCH_PATH
-                  "${MTOA_LOCATION}/materialx/targets")
+        # It seems like we need to use MAYA_MODULE_PATH for MtoA to work properly.
+        # Even if we emulate the .mod file by manually setting the same env vars
+        # to the same values, MtoA itself will appear to load successfully when 
+        # calling loadPlugin, but some of its extensions will fail to initialize,
+        # leading to incorrect behavior and test failures. In those cases, it seems
+        # like having a locally installed MtoA fixed it, but we can't rely on that.
+        list(APPEND MAYAUSD_VARNAME_MAYA_MODULE_PATH
+             "${MTOA_LOCATION}")
     endif()
 
     # lookdevx
@@ -325,6 +322,8 @@ finally:
              "${LOOKDEVX_LOCATION}/scripts")
         list(APPEND MAYAUSD_VARNAME_PYTHONPATH
              "${LOOKDEVX_LOCATION}/python")
+        list(APPEND MAYAUSD_VARNAME_MAYA_PLUG_IN_PATH
+             "${LOOKDEVX_LOCATION}/plug-ins")
     endif()
 
     if(IS_WINDOWS AND DEFINED ENV{PYTHONHOME})

test/lib/mayaUsd/render/mayaToHydra/README.md

@@ -26,7 +26,7 @@ For example, to run testVisibility on RelWithDebInfo :
 
 ### Running tests with MayaUSD
 
-Some tests require the MayaUSD plugin to be loaded in order to be run. If MayaUSD cannot be loaded, those tests will be skipped. In order for the test framework to find and load MayaUSD, the `-DMAYAUSD_LOCATION` CMake flag must be set to your MayaUSD installation directory when building MayaHydra. For convenience, you can use the `--mayausd-location` parameter when using the [build.py](../../../../../build.py) script, which will set the CMake flag for you with the given argument. The specified path should point to the directory where your MayaUSD `.mod` file resides. Note that the test framework will not find MayaUSD using the MAYA_MODULE_PATH environment variable.
+The tests require MayaUSD in order to be run. In order for the test framework to find and load MayaUSD, the `-DMAYAUSD_LOCATION` CMake flag must be set to your MayaUSD installation directory when building MayaHydra. For convenience, you can use the `--mayausd-location` parameter when using the [build.py](../../../../../build.py) script, which will set the CMake flag for you with the given argument. The specified path should point to the directory where your MayaUSD `.mod` file resides. Note that the test framework will not find MayaUSD using the MAYA_MODULE_PATH environment variable.
 
 For example, if your MayaUSD `.mod` file resides in `<some-path>/install/RelWithDebInfo`, you could call build.py as such :
 
@@ -54,29 +54,34 @@ Looking to add a C++ test suite? See the corresponding [README.md](./cpp/README.
 To add a new Python-only test suite : 
 
 1. Create a new test[...].py file in the current folder.
-2. Create a test class that derives from unittest.TestCase (doesn't need to directly inherit from it).
-3. Add `test_[...]` methods for each test case in your test suite.
-4. Add the following snippet at the bottom of your file :
+2. Create a test class that derives from `mtohUtils.MayaHydraBaseTestCase`.
+3. Add the line `_file = __file__` in your class definition.
+4. If needed, set `_requiredPlugins` to list any plugins that need to be loaded for your test. MayaUSD does not need to be specified, it will be loaded automatically.
+5. Add `test_[...]` methods for each test case in your test suite.
+6. Add the following snippet at the bottom of your file :
     ```python
     if __name__ == '__main__':
         fixturesUtils.runTests(globals())
     ```
-5. Add the name of your test[...].py file to the list of `TEST_SCRIPT_FILES` in the [current folder's CMakeLists.txt](./CMakeLists.txt)
+7. Add the name of your test[...].py file to the list of `TEST_SCRIPT_FILES` in the [current folder's CMakeLists.txt](./CMakeLists.txt)
 
 Some important notes :
-- By default, the mayaHydra plugin will not be loaded. An easy way to load it is to have your test class derive from `MayaHydraBaseTestCase`, which will automatically load mayaHydra when running tests. Don't forget to add the line `_file = __file__` in your class.
-- After loading mayaHydra, the renderer will still be on Viewport 2.0. If you want to use another renderer, your test will have to switch to it. If you want to use HdStorm, an easy way to do it is to have your test class derive from `MayaHydraBaseTestCase` and call `self.setHdStormRenderer()`
+- Before each test, a new file will be opened and the renderer will be switched to Hydra. If you need to switch between renderers, you can use the `self.setHdStormRenderer()` and `self.setViewport2Renderer()` methods.
+
+# Best practices
+- Don't skip tests unless necessary. If a test requires a certain plugin to be loaded, don't skip the test if the plugin fails to load, as this will falsely be reported as a pass. For such cases, prefer setting the `_requiredPlugins` variable in your test class.
+- Use relative paths in test data to make sure the tests will work anywhere.
+- Prefer storing USD data in text-form .usda instead of binary, for readability and ease of modification should a test need to be tweaked.
 
 # Image comparison
 
 Image comparison is done using [idiff from OpenImageIO](https://openimageio.readthedocs.io/en/latest/idiff.html).
 
 Some utilities exist to facilitate image comparison tests. Here is the simplest way to get going : 
-1. Make your test class derive from `MtohTestCase`.
-2. Don't forget to add the line `_file = __file__` in your class.
-3. Create a folder called [TestName]Test (e.g. for testVisibility, create a folder VisibilityTest)
-4. Put the images you want to use for comparison in that folder
-5. Call `self.assertImagesClose`, `self.assertImagesEqual` and/or `self.assertSnapshotClose` with the file names of the images to compare with.
+1. Make sure your test class derives from `mtohUtils.MayaHydraBaseTestCase` (and has the line `_file = __file__`).
+2. Create a folder called [TestName]Test (e.g. for testVisibility, create a folder VisibilityTest)
+3. Put the images you want to use for comparison in that folder
+4. Call `self.assertImagesClose`, `self.assertImagesEqual` and/or `self.assertSnapshotClose` with the file names of the images to compare with.
 
 For the `assert[...]Close` methods, you will need to pass in a fail threshold for individual pixels and a fail percentage for the whole image. Other parameters are also available for you to specify. You can find more information on these parameters on the [idiff documentation page](https://openimageio.readthedocs.io/en/latest/idiff.html).
 
@@ -88,10 +93,4 @@ To create a reference snapshot, an easy way is to first write your test, and the
 
 # Utilities
 
-Utility files are located under [/test/testUtils](../../../../testUtils/). Note that at the time of writing this (2023/08/16), many of the utils files and their contents were inherited from the USD plugin, and are not all used. Here are some of the main utilities you might find useful :
-
-- mtohutils.py :
-    - `checkForMayaUsdPlugin()` will try to load the MayaUsd plugin and return a boolean indicating if the plugin was loaded successfully. By decorating a test case with the following line, you can run the test only if the MayaUsd plugin is found and loaded : 
-    ```@unittest.skipUnless(mtohUtils.checkForMayaUsdPlugin(), "Requires Maya USD Plugin.")```
-    - `MayaHydraBaseTestCase` is a base class you can use for your test classes that will provide you with some useful functionality, such as automatically loading the mayaHydra plugin. It also provides a method to use the HdStorm renderer (`setHdStormRenderer`), and another to create a simple scene with a cube (`makeCubeScene`).
-    - `MtohTestCase` is a class you can use as a base for your test classes that will provide you with image comparison functionality (see [Image comparison](#Image-comparison) section). Note that this class already derives from MayaHydraBaseTestCase, so it will also automatically load the mayaHydra plugin.
+Utility files are located under [/test/testUtils](../../../../testUtils/). Note that at the time of writing this (2023/08/16), many of the utils files and their contents were inherited from the USD plugin, and are not all used.

test/lib/mayaUsd/render/mayaToHydra/cpp/testMayaUsdUfeItems.py

@@ -16,7 +16,6 @@
 
 import fixturesUtils
 import mtohUtils
-import unittest
 
 from testUtils import PluginLoaded
 
@@ -37,7 +36,6 @@ def setupUsdStage(self):
         UsdLux.RectLight.Define(usdStage, "/USDRectLight")
         cmds.refresh()
 
-    @unittest.skipUnless(mtohUtils.checkForMayaUsdPlugin(), "Requires Maya USD Plugin.")
     def test_SkipMayaUsdUfeLights(self):
         self.setupUsdStage()
         with PluginLoaded('mayaHydraCppTests'):

test/lib/mayaUsd/render/mayaToHydra/cpp/testPathInterface.py

@@ -23,8 +23,6 @@
 
 from testUtils import PluginLoaded
 
-import unittest
-
 class TestPathInterface(mtohUtils.MayaHydraBaseTestCase):
     # MayaHydraBaseTestCase.setUpClass requirement.
     _file = __file__
@@ -66,7 +64,6 @@ def setupScene(self):
 
         cmds.refresh()
 
-    @unittest.skipUnless(mtohUtils.checkForMayaUsdPlugin(), "Requires Maya USD Plugin.")
     def test_PathInterface(self):
         self.setupScene()
         with PluginLoaded('mayaHydraCppTests'):

test/lib/mayaUsd/render/mayaToHydra/cpp/testPicking.py

@@ -16,11 +16,10 @@
 import fixturesUtils
 import mayaUtils
 import mtohUtils
-import unittest
 
 from testUtils import PluginLoaded
 
-class TestPicking(mtohUtils.MtohTestCase):
+class TestPicking(mtohUtils.MayaHydraBaseTestCase):
     # MayaHydraBaseTestCase.setUpClass requirement.
     _file = __file__
 
@@ -89,31 +88,27 @@ def test_PickMayaLight(self):
         with PluginLoaded('mayaHydraCppTests'):
             cmds.mayaHydraCppTest(directionalLightObjectName, "simpleLight", f="TestPicking.pickObject")
 
-    @unittest.skipUnless(mtohUtils.checkForMayaUsdPlugin(), "Requires Maya USD Plugin.")
     def test_PickUsdMesh(self):
         import mayaUsd_createStageWithNewLayer
         stagePath = mayaUsd_createStageWithNewLayer.createStageWithNewLayer()
         cubeObjectName = self.createUsdCubeFromMaya(stagePath)
         with PluginLoaded('mayaHydraCppTests'):
             cmds.mayaHydraCppTest(cubeObjectName, "mesh", f="TestPicking.pickObject")
 
-    @unittest.skipUnless(mtohUtils.checkForMayaUsdPlugin(), "Requires Maya USD Plugin.")
     def test_PickUsdImplicitSurface(self):
         import mayaUsd_createStageWithNewLayer
         stagePath = mayaUsd_createStageWithNewLayer.createStageWithNewLayer()
         cubeObjectName = self.createUsdCube(stagePath)
         with PluginLoaded('mayaHydraCppTests'):
             cmds.mayaHydraCppTest(cubeObjectName, "mesh", f="TestPicking.pickObject")
 
-    @unittest.skipUnless(mtohUtils.checkForMayaUsdPlugin(), "Requires Maya USD Plugin.")
     def test_PickUsdLight(self):
         import mayaUsd_createStageWithNewLayer
         stagePath = mayaUsd_createStageWithNewLayer.createStageWithNewLayer()
         rectLightObjectName = self.createUsdRectLight(stagePath)
         with PluginLoaded('mayaHydraCppTests'):
             cmds.mayaHydraCppTest(rectLightObjectName, "rectLight", f="TestPicking.pickObject")
 
-    @unittest.skipUnless(mtohUtils.checkForMayaUsdPlugin(), "Requires Maya USD Plugin.")
     def test_MarqueeSelection(self):
         import mayaUsd_createStageWithNewLayer
         stagePath = mayaUsd_createStageWithNewLayer.createStageWithNewLayer()

test/lib/mayaUsd/render/mayaToHydra/cpp/testPointInstancePicking.py

@@ -15,13 +15,12 @@
 import maya.cmds as cmds
 import fixturesUtils
 import mtohUtils
-import unittest
 import usdUtils
 
 import testUtils
 from testUtils import PluginLoaded
 
-class TestPointInstancePicking(mtohUtils.MtohTestCase):
+class TestPointInstancePicking(mtohUtils.MayaHydraBaseTestCase):
     # MayaHydraBaseTestCase.setUpClass requirement.
     _file = __file__
 
@@ -45,7 +44,6 @@ def setUp(self):
         cmds.setAttr('persp.rotate', -33.4, 63.0, 0, type='float3')
         cmds.refresh()
 
-    @unittest.skipUnless(mtohUtils.checkForMayaUsdPlugin(), "Requires Maya USD Plugin.")
     def test_PickPointInstancer(self):
         with PluginLoaded('mayaHydraCppTests'):
 
@@ -63,7 +61,6 @@ def test_PickPointInstancer(self):
                     self.PICK_PATH + marker,
                     f="TestPointInstancePicking.pickPointInstance")
 
-    @unittest.skipUnless(mtohUtils.checkForMayaUsdPlugin(), "Requires Maya USD Plugin.")
     def test_PickInstances(self):
         with PluginLoaded('mayaHydraCppTests'):
 
@@ -87,7 +84,6 @@ def test_PickInstances(self):
                     self.PICK_PATH + marker,
                     f="TestPointInstancePicking.pickPointInstance")
 
-    @unittest.skipUnless(mtohUtils.checkForMayaUsdPlugin(), "Requires Maya USD Plugin.")
     def test_PickPrototypes(self):
         with PluginLoaded('mayaHydraCppTests'):
 

test/lib/mayaUsd/render/mayaToHydra/cpp/testPrimInstancing.py

@@ -16,7 +16,6 @@
 
 import fixturesUtils
 import mtohUtils
-import unittest
 
 import testUtils
 from testUtils import PluginLoaded
@@ -32,7 +31,6 @@ def loadUsdScene(self):
         self.setHdStormRenderer()
         cmds.refresh()
 
-    @unittest.skipUnless(mtohUtils.checkForMayaUsdPlugin(), "Requires Maya USD Plugin.")
     def test_UsdPrimInstancing(self):
         self.loadUsdScene()
         with PluginLoaded('mayaHydraCppTests'):

test/lib/mayaUsd/render/mayaToHydra/cpp/testSceneCorrectness.py

@@ -16,7 +16,6 @@
 
 import fixturesUtils
 import mtohUtils
-import unittest
 
 import testUtils
 from testUtils import PluginLoaded
@@ -31,7 +30,6 @@ def loadUsdScene(self):
         usdUtils.createStageFromFile(usdScenePath)
         self.setHdStormRenderer()
 
-    @unittest.skipUnless(mtohUtils.checkForMayaUsdPlugin(), "Requires Maya USD Plugin.")
     def test_HydraFromUsdSceneCorrectness(self):
         self.loadUsdScene()
         cmds.select(all=True)

test/lib/mayaUsd/render/mayaToHydra/cpp/testUsdNativeInstancePicking.py

@@ -15,13 +15,12 @@
 import maya.cmds as cmds
 import fixturesUtils
 import mtohUtils
-import unittest
 import usdUtils
 
 import testUtils
 from testUtils import PluginLoaded
 
-class TestUsdNativeInstancePicking(mtohUtils.MtohTestCase):
+class TestUsdNativeInstancePicking(mtohUtils.MayaHydraBaseTestCase):
     # MayaHydraBaseTestCase.setUpClass requirement.
     _file = __file__
 
@@ -36,7 +35,6 @@ def setUp(self):
         self.loadUsdScene()
         cmds.refresh()
 
-    @unittest.skipUnless(mtohUtils.checkForMayaUsdPlugin(), "Requires Maya USD Plugin.")
     def test_NativeInstances(self):
         with PluginLoaded('mayaHydraCppTests'):
             instances = ["/cubes_1", "/cubes_2"]

test/lib/mayaUsd/render/mayaToHydra/cpp/testUsdStageLayerMuting.py

@@ -16,7 +16,6 @@
 
 import fixturesUtils
 import mtohUtils
-import unittest
 
 from testUtils import PluginLoaded
 
@@ -57,7 +56,6 @@ def setupUsdStage(self):
 
         cmds.refresh()
 
-    @unittest.skipUnless(mtohUtils.checkForMayaUsdPlugin(), "Requires Maya USD Plugin.")
     def test_UsdStageLayerMuting(self):
         self.setupUsdStage()
         with PluginLoaded('mayaHydraCppTests'):

test/lib/mayaUsd/render/mayaToHydra/cpp/testWireframeSelectionHighlightSceneIndex.py

@@ -4,13 +4,10 @@
 import mayaUtils
 from testUtils import PluginLoaded
 
-import unittest
-
 class TestWireframeSelectionHighlightSceneIndex(mtohUtils.MayaHydraBaseTestCase):
     # MayaHydraBaseTestCase.setUpClass requirement.
     _file = __file__
 
-    @unittest.skipUnless(mtohUtils.checkForMayaUsdPlugin(), "Requires Maya USD Plugin.")
     def test_wireframeSelectionHighlightSceneIndex(self):
         self.setHdStormRenderer()
         with PluginLoaded('mayaHydraCppTests'):

test/lib/mayaUsd/render/mayaToHydra/testArnoldLights.py

@@ -18,12 +18,12 @@
 import fixturesUtils
 import mtohUtils
 import mayaUtils
-import unittest
 import platform
 
-class TestArnoldLights(mtohUtils.MtohTestCase): #Subclassing mtohUtils.MtohTestCase to be able to call self.assertSnapshotClose
+class TestArnoldLights(mtohUtils.MayaHydraBaseTestCase): #Subclassing mtohUtils.MayaHydraBaseTestCase to be able to call self.assertSnapshotClose
     # MayaHydraBaseTestCase.setUpClass requirement.
     _file = __file__
+    _requiredPlugins = ['mtoa']
 
     @property
     def imageDiffFailThreshold(self):
@@ -74,7 +74,6 @@ def verifyLightingModes(self, shadowOn):
         #self.assertSnapshotClose("noLight" + imageSuffix + ".png", self.imageDiffFailThreshold, self.imageDiffFailPercent)
 
     #Test arnold lights (e.g., aiSkyDomeLight,etc.) with a maya native sphere and usd sphere.
-    @unittest.skipUnless(mtohUtils.checkForMayaUsdPlugin() and mtohUtils.checkForMtoAPlugin(), "Requires Maya USD and MtoA Plugins.")
     def test_ArnoldLights(self):
         cmds.file(new=True, force=True)
 

test/lib/mayaUsd/render/mayaToHydra/testCurveTools.py

@@ -18,7 +18,7 @@
 
 import platform
 
-class TestCurveTools(mtohUtils.MtohTestCase):
+class TestCurveTools(mtohUtils.MayaHydraBaseTestCase):
     # MayaHydraBaseTestCase.setUpClass requirement.
     _file = __file__
 

test/lib/mayaUsd/render/mayaToHydra/testFlowViewportAPI.py

@@ -31,7 +31,7 @@ def setRotateY(matrixAsAList, angle):
     matrixAsAList[2+4*0] = sin(angle)
     return matrixAsAList
 
-class TestFlowViewportAPI(mtohUtils.MtohTestCase): #Subclassing mtohUtils.MtohTestCase to be able to call self.assertSnapshotClose
+class TestFlowViewportAPI(mtohUtils.MayaHydraBaseTestCase): #Subclassing mtohUtils.MayaHydraBaseTestCase to be able to call self.assertSnapshotClose
     # MayaHydraBaseTestCase.setUpClass requirement.
     _file = __file__
 

test/lib/mayaUsd/render/mayaToHydra/testFootPrintNode.py

@@ -26,7 +26,7 @@
 HD_STORM = "HdStormRendererPlugin"
 HD_STORM_OVERRIDE = "mayaHydraRenderOverride_" + HD_STORM
 
-class TestFootPrintNode(mtohUtils.MtohTestCase): #Subclassing mtohUtils.MtohTestCase to be able to call self.assertSnapshotClose
+class TestFootPrintNode(mtohUtils.MayaHydraBaseTestCase): #Subclassing mtohUtils.MayaHydraBaseTestCase to be able to call self.assertSnapshotClose
     # MayaHydraBaseTestCase.setUpClass requirement.
     _file = __file__
 

test/lib/mayaUsd/render/mayaToHydra/testImageDiffing.py

@@ -12,17 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-import os.path
-import sys
-import unittest
-
-import maya.cmds as cmds
-import maya.mel
-
 import fixturesUtils
 import mtohUtils
 
-class TestImageDiffing(mtohUtils.MtohTestCase):
+class TestImageDiffing(mtohUtils.MayaHydraBaseTestCase):
     """Test the image diffing setup to make sure it works and returns the expected results."""
 
     _file = __file__

test/lib/mayaUsd/render/mayaToHydra/testLookThrough.py

@@ -20,7 +20,7 @@
 import mayaUtils
 import platform
 
-class TestLookThrough(mtohUtils.MtohTestCase): #Subclassing mtohUtils.MtohTestCase to be able to call self.assertSnapshotClose
+class TestLookThrough(mtohUtils.MayaHydraBaseTestCase): #Subclassing mtohUtils.MayaHydraBaseTestCase to be able to call self.assertSnapshotClose
     # MayaHydraBaseTestCase.setUpClass requirement.
     _file = __file__