HYDRA-1294 - fix doubleSided and CullStyle for MRenderItem (#210)

* fix doubleSided and CullStyle for MRenderItem

* polish comment

* rebaseline two images as the doubleSided attribute is respected now

* handle for Maya w and w/o API

cmake/modules/FindMaya.cmake

@@ -24,6 +24,7 @@
 # MAYA_HAS_DISPLAY_LAYER_API Presence of MFnDisplayLayer
 # MAYA_HAS_NEW_DISPLAY_LAYER_MESSAGING_API Presence of MDisplayLayerMemberChangedFunction
 # MAYA_HAS_RENDER_ITEM_HIDE_ON_PLAYBACK_API Presence of MRenderItem has HideOnPlayback API
+# MAYA_HAS_RENDER_ITEM_CULL_MODE_API Presence of MRenderItem has CullMode API
 # MAYA_HAS_VIEW_SELECTED_OBJECT_API Presence of M3dView::viewSelectedObject
 # MAYA_LINUX_BUILT_WITH_CXX11_ABI Maya Linux was built with new cxx11 ABI.
 # MAYA_MACOSX_BUILT_WITH_UB2 Maya OSX was built with Universal Binary 2.
@@ -404,6 +405,15 @@ if(MAYA_INCLUDE_DIRS AND EXISTS "${MAYA_INCLUDE_DIR}/maya/MHWGeometry.h")
     endif()
 endif()
 
+set(MAYA_HAS_RENDER_ITEM_CULL_MODE_API FALSE CACHE INTERNAL "hasRenderItemCullModeFunction")
+if(MAYA_INCLUDE_DIRS AND EXISTS "${MAYA_INCLUDE_DIR}/maya/MHWGeometry.h")
+    file(STRINGS ${MAYA_INCLUDE_DIR}/maya/MHWGeometry.h MAYA_HAS_API REGEX "cullMode")
+    if(MAYA_HAS_API)
+        set(MAYA_HAS_RENDER_ITEM_CULL_MODE_API TRUE CACHE INTERNAL "hasRenderItemCullModeFunction")
+        message(STATUS "MRenderItem has CullMode API")
+    endif()
+endif()
+
 set(MAYA_HAS_VIEW_SELECTED_OBJECT_API FALSE CACHE INTERNAL "hasViewSelectedObject")
 if(MAYA_INCLUDE_DIRS AND EXISTS "${MAYA_INCLUDE_DIR}/maya/M3dView.h")
     file(STRINGS ${MAYA_INCLUDE_DIR}/maya/M3dView.h MAYA_HAS_API REGEX "numViewSelectedObjects")

lib/mayaHydra/hydraExtensions/adapters/CMakeLists.txt

@@ -41,6 +41,13 @@ set(HEADERS
     tokens.h
 )
 
+if (MAYA_HAS_RENDER_ITEM_CULL_MODE_API)
+    target_compile_definitions(${TARGET_NAME}
+        PRIVATE
+            MAYA_HAS_RENDER_ITEM_CULL_MODE_API=1
+    )
+endif()
+
 # -----------------------------------------------------------------------------
 # promoted headers
 # -----------------------------------------------------------------------------

lib/mayaHydra/hydraExtensions/adapters/renderItemAdapter.cpp

@@ -64,6 +64,9 @@ MayaHydraRenderItemAdapter::MayaHydraRenderItemAdapter(
     , _primitive(ri.primitive())
     , _name(ri.name())
     , _fastId(fastId)
+#ifdef MAYA_HAS_RENDER_ITEM_CULL_MODE_API
+    , _cullMode(ri.cullMode())
+#endif
 {
     _InsertRprim(this);
 }
@@ -145,6 +148,18 @@ void MayaHydraRenderItemAdapter::UpdateFromDelta(const UpdateFromDeltaData& data
 
     HdDirtyBits dirtyBits = 0;
 
+#ifdef MAYA_HAS_RENDER_ITEM_CULL_MODE_API
+    MRenderItem::CullMode cullMode = data._ri.cullMode();
+    if (cullMode != _cullMode) {
+        //  MRenderItem uses CullNone to denote doubleSided
+        if (IsDoubleSided(_cullMode) || IsDoubleSided(cullMode)) {
+            dirtyBits |= HdChangeTracker::DirtyDoubleSided;
+        }
+        dirtyBits |= HdChangeTracker::DirtyCullStyle;
+        _cullMode = cullMode;
+    }
+#endif
+
     if (data._wireframeColor != _wireframeColor) {
         _wireframeColor = data._wireframeColor;
         dirtyBits |= HdChangeTracker::DirtyPrimvar; // displayColor primVar
@@ -545,10 +560,19 @@ void MayaHydraRenderItemAdapter::SetPlaybackChanged()
 
 HdCullStyle MayaHydraRenderItemAdapter::GetCullStyle() const
 {
-    // HdCullStyleNothing means no culling, HdCullStyledontCare means : let the renderer choose
-    // between back or front faces culling. We don't want culling, since we want to see the
-    // backfaces being unlit with MayaHydraSceneDelegate::GetDoubleSided returning false.
-    return _isArnoldSkyDomeLightTriangleShape ? HdCullStyleFront : HdCullStyleNothing;
+    if (_isArnoldSkyDomeLightTriangleShape) {
+        return HdCullStyleFront;
+    }
+#ifdef MAYA_HAS_RENDER_ITEM_CULL_MODE_API
+    switch (_cullMode) {
+    case MRenderItem::CullNone: return HdCullStyleNothing;
+    case MRenderItem::CullFront: return HdCullStyleFront;
+    case MRenderItem::CullBack: return HdCullStyleBack;
+    default: return HdCullStyleNothing;
+    }
+#else
+    return HdCullStyleNothing;
+#endif
 }
 
 ///////////////////////////////////////////////////////////////////////

lib/mayaHydra/hydraExtensions/adapters/renderItemAdapter.h

@@ -46,6 +46,11 @@ std::string kRenderItemTypeName = "renderItem";
 static constexpr const char* kPointSize = "pointSize";
 
 static const SdfPath kInvalidMaterial = SdfPath("InvalidMaterial");
+
+#ifdef MAYA_HAS_RENDER_ITEM_CULL_MODE_API
+// Extract doubleSided attribute from CullMode as MRenderItem uses CullNone to denote doubleSided.
+static bool IsDoubleSided(MRenderItem::CullMode cullMode) { return cullMode == MRenderItem::CullNone; }
+#endif
 } // namespace
 
 using MayaHydraRenderItemAdapterPtr = std::shared_ptr<class MayaHydraRenderItemAdapter>;
@@ -81,7 +86,13 @@ class MayaHydraRenderItemAdapter : public MayaHydraAdapter
     virtual bool IsSupported() const override;
 
     MAYAHYDRALIB_API
-    bool GetDoubleSided() const override { return false; };
+    bool GetDoubleSided() const override { 
+#ifdef MAYA_HAS_RENDER_ITEM_CULL_MODE_API
+        return IsDoubleSided(_cullMode);
+#else
+        return false;
+#endif
+    };
 
     MAYAHYDRALIB_API
     GfBBox3d GetBoundingBox()const override { return _bounds; }
@@ -209,6 +220,9 @@ class MayaHydraRenderItemAdapter : public MayaHydraAdapter
     bool                        _isHideOnPlayback = false;
     bool                        _isArnoldSkyDomeLightTriangleShape = false;
     GfBBox3d                    _bounds;//Bounding box
+#ifdef MAYA_HAS_RENDER_ITEM_CULL_MODE_API
+    MRenderItem::CullMode       _cullMode = MRenderItem::CullNone;
+#endif
 };
 
 PXR_NAMESPACE_CLOSE_SCOPE

test/lib/mayaUsd/render/mayaToHydra/CMakeLists.txt

@@ -164,6 +164,8 @@ foreach(script ${INTERACTIVE_TEST_SCRIPT_FILES_DISABLE_VP2_RENDER_DELEGATE})
             "LD_PRELOAD=${ADDITIONAL_LD_PRELOAD}"
 
             "MAYAUSD_DISABLE_VP2_RENDER_DELEGATE=1"
+
+            "MAYA_HAS_RENDER_ITEM_CULL_MODE_API=${MAYA_HAS_RENDER_ITEM_CULL_MODE_API}"
     )
 
     # Assign a CTest label to these tests for easy filtering.
@@ -201,6 +203,8 @@ foreach(script ${INTERACTIVE_TEST_SCRIPT_FILES})
 
             # Use standard surface as default for now
             "MAYA_DEFAULT_SURFACE_SHADER=standardSurface"
+
+            "MAYA_HAS_RENDER_ITEM_CULL_MODE_API=${MAYA_HAS_RENDER_ITEM_CULL_MODE_API}"
     )
 
     # Add a ctest label to these tests for easy filtering.

test/lib/mayaUsd/render/mayaToHydra/testPolygonPrimitives.py

@@ -17,6 +17,7 @@
 import mtohUtils
 
 import platform
+import os
 
 class TestPolygonPrimitives(mtohUtils.MayaHydraBaseTestCase):
     # MayaHydraBaseTestCase.setUpClass requirement.
@@ -25,10 +26,10 @@ class TestPolygonPrimitives(mtohUtils.MayaHydraBaseTestCase):
     IMAGE_DIFF_FAIL_THRESHOLD = 0.05
     IMAGE_DIFF_FAIL_PERCENT = 1.5
 
-    def compareSnapshot(self, referenceFilename, cameraDistance=15):
+    def compareSnapshot(self, referenceFilename, cameraDistance=15, imageVersion=None):
         self.setBasicCam(cameraDistance)
         cmds.refresh()
-        self.assertSnapshotClose(referenceFilename, self.IMAGE_DIFF_FAIL_THRESHOLD, self.IMAGE_DIFF_FAIL_PERCENT)
+        self.assertSnapshotClose(referenceFilename, self.IMAGE_DIFF_FAIL_THRESHOLD, self.IMAGE_DIFF_FAIL_PERCENT, imageVersion)
 
     def setupScene(self, polygonCreationCallable):
         self.setHdStormRenderer()
@@ -315,7 +316,10 @@ def test_PolygonSphericalHarmonics(self):
         self.compareSnapshot("sphericalHarmonics_modified.png")
 
         self.setupSuperShapeHelix(polyCreatorNodeName)
-        self.compareSnapshot("sphericalHarmonics_helix.png")
+        imageVersion = None
+        if(os.getenv('MAYA_HAS_RENDER_ITEM_CULL_MODE_API', 'NOT-FOUND') in ('1', 'TRUE')):
+            imageVersion = "RenderItemHasCullModeAPI"
+        self.compareSnapshot("sphericalHarmonics_helix.png", 15, imageVersion)
 
     def test_PolygonUltra(self):
         polyCreatorNodeName = self.setupScene(self.getSuperShapeCreationCallable("UltraShape"))
@@ -346,7 +350,10 @@ def test_PolygonUltra(self):
         self.compareSnapshot("ultra_modified.png")
 
         self.setupSuperShapeHelix(polyCreatorNodeName)
-        self.compareSnapshot("ultra_helix.png")
+        imageVersion = None
+        if(os.getenv('MAYA_HAS_RENDER_ITEM_CULL_MODE_API', 'NOT-FOUND') in ('1', 'TRUE')):
+            imageVersion = "RenderItemHasCullModeAPI"
+        self.compareSnapshot("ultra_helix.png", 15 , imageVersion)
 
 if __name__ == '__main__':
     fixturesUtils.runTests(globals())