HYDRA-843 : Add picking test + Link C++ tests with Qt for UI/viewport interaction

cmake/compiler_config.cmake

@@ -102,6 +102,53 @@ set(MSVC_DEFINITIONS
     # Needed to prevent Python from adding a define for snprintf
     # since it was added in Visual Studio 2015.
     HAVE_SNPRINTF
+
+    # In debug builds, Boost's wrap_python.hpp will #undef _DEBUG when BOOST_DEBUG_PYTHON is not defined,
+    # and will then include pyconfig.h :
+    # https://github.com/boostorg/python/blob/boost-1.81.0/include/boost/python/detail/wrap_python.hpp#L23-L57
+    # pyconfig.h will then autolink the Python lib; however, since _DEBUG was #undef'd, it will try to
+    # link against the release version of the Python lib, causing a linker error :
+    # https://github.com/python/cpython/blob/v3.11.4/PC/pyconfig.h#L269-L286
+    $<$<STREQUAL:${CMAKE_BUILD_TYPE},Debug>:BOOST_DEBUG_PYTHON>
+
+    # When using the qmath.h Qt header, we can run into macro redefinition issues.
+    # Specifically, qmath.h wants to define the math macros listed here :
+    # https://learn.microsoft.com/en-us/cpp/c-runtime-library/math-constants?view=msvc-170
+    # To do so, it will first try to include cmath. cmath includes cstdlib, which includes math.h,
+    # which includes corecrt_math_defines.h, which defines the math macros.
+    # However, corecrt_math_defines.h is only included by math.h if _USE_MATH_DEFINES is defined.
+    # If _USE_MATH_DEFINES is not defined, Qt will temporarily define it itself, include cmath 
+    # so that corecrt_math_defines.h ends up defining the math macros, and undefine it afterwards. See here :
+    # https://github.com/qt/qtbase/blob/85c69f023fe281b7f16e1a93e61be4432f7fef9b/src/corelib/kernel/qmath.h#L18-L28
+    # If the math macros are still not defined after that, Qt will resort to defining the macros itself : 
+    # https://github.com/qt/qtbase/blob/85c69f023fe281b7f16e1a93e61be4432f7fef9b/src/corelib/kernel/qmath.h#L188-L238
+    # As such, it is guaranteed that the math macros are defined after including qmath.h.
+    # So what's the issue? Well, 2 things combined. First, it turns out that while cmath, cstdlib and 
+    # corecrt_math_defines.h have header guards, math.h does not. Second, corecrt_math_defines.h does
+    # not check that the math macros are not defined before defining them itself.
+    # This can lead to the following scenario : 
+    # 1. _USE_MATH_DEFINES is not defined.
+    # 2. We hit an #include <cmath> or <cstdlib>.
+    # 3. The cmath or cstdlib header guards get defined.
+    # 4. math.h gets included, but since _USE_MATH_DEFINES is not defined, corecrt_math_defines.h
+    #    is not included, and the math macros do not get defined.
+    # 5. We hit an #include <qmath.h>.
+    # 6. Qt temporarily defines _USE_MATH_DEFINES and includes cmath.
+    # 7. We hit the cmath or cstdlib header guard and stop the inclusion, not reaching math.h 
+    #    and corecrt_math_defines.h. The math macros are still not defined.
+    # 8. Qt sees that the math macros are still not defined, and defines them itself.
+    # 9. _USE_MATH_DEFINES gets defined.
+    # 10. We hit an #include <math.h>.
+    # 11. Since there is no header guard for math.h, it is re-included.
+    # 12. As _USE_MATH_DEFINES is now defined, corecrt_math_defines.h does get included this time.
+    # 13. corecrt_math_defines.h defines the macros without checking if they have already been defined :
+    #     macro redefinition -> warning -> warning treated-as-error -> error -> compilation fails.
+    # In our case, step 9 (_USE_MATH_DEFINES getting defined late) happens in MTypes.h, and step 10 
+    # (math.h re-inclusion) in some other Maya headers, for example MFloatVector.h.
+    # By defining _USE_MATH_DEFINES from the get-go, we ensure that the math macros get defined by
+    # corecrt_math_defines.h the first time around. Afterwards, even if math.h does not have a
+    # header guard, the one in corecrt_math_defines.h avoids redefining the macros.
+    _USE_MATH_DEFINES
 )
 
 #------------------------------------------------------------------------------
@@ -165,13 +212,6 @@ function(mayaHydra_compile_config TARGET)
         target_compile_definitions(${TARGET} 
             PRIVATE
                 ${MSVC_DEFINITIONS}
-                # In debug builds, Boost's wrap_python.hpp will #undef _DEBUG when BOOST_DEBUG_PYTHON is not defined,
-                # and will then include pyconfig.h :
-                # https://github.com/boostorg/python/blob/boost-1.81.0/include/boost/python/detail/wrap_python.hpp#L23-L57
-                # pyconfig.h will then autolink the Python lib; however, since _DEBUG was #undef'd, it will try to
-                # link against the release version of the Python lib, causing a linker error :
-                # https://github.com/python/cpython/blob/v3.11.4/PC/pyconfig.h#L269-L286
-                $<$<STREQUAL:${CMAKE_BUILD_TYPE},Debug>:BOOST_DEBUG_PYTHON>
         )
     endif()
 

test/lib/mayaUsd/render/mayaToHydra/CMakeLists.txt

@@ -45,6 +45,7 @@ set(INTERACTIVE_TEST_SCRIPT_FILES
     cpp/testFlowViewportAPIFilterPrims.py
     cpp/testSceneCorrectness.py
     cpp/testPrimInstancing.py
+    cpp/testPicking.py
 )
 
 #Add this test only if the MayaUsd_FOUND (so also MAYAUSDAPI_LIBRARY) has been found during compile time.

test/lib/mayaUsd/render/mayaToHydra/cpp/CMakeLists.txt

@@ -3,6 +3,11 @@
 # -----------------------------------------------------------------------------
 find_package(GTest REQUIRED)
 
+if (NOT Qt6_FOUND)
+    message(WARNING "No Qt6 package found. Cannot build ${TARGET_NAME}.")
+    return()
+endif()
+
 set(TARGET_NAME mayaHydraCppTests)
 add_library(${TARGET_NAME} SHARED)
 
@@ -29,11 +34,16 @@ target_sources(${TARGET_NAME}
         testFlowViewportAPIFilterPrims.cpp
         testSceneCorrectness.cpp
         testPrimInstancing.cpp
+        testPicking.cpp
 )
 
 # -----------------------------------------------------------------------------
 # compiler configuration
 # -----------------------------------------------------------------------------
+# QT_NO_KEYWORDS prevents Qt from defining the foreach, signals, slots and emit macros.
+# this avoids overlap between Qt macros and boost, and enforces using Q_ macros.
+set_target_properties(Qt6::Core PROPERTIES INTERFACE_COMPILE_DEFINITIONS QT_NO_KEYWORDS)
+
 mayaHydra_compile_config(${TARGET_NAME})
 
 target_compile_definitions(${TARGET_NAME}
@@ -59,6 +69,8 @@ target_link_libraries(${TARGET_NAME}
         mayaHydraLib
         ${GTEST_LIBRARIES}
         flowViewport
+        Qt6::Core
+        Qt6::Widgets
 )
 
 # -----------------------------------------------------------------------------

test/lib/mayaUsd/render/mayaToHydra/cpp/testPicking.cpp

@@ -0,0 +1,196 @@
+// Copyright 2024 Autodesk
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#include "testUtils.h"
+
+#include <pxr/imaging/hd/selectionSchema.h>
+#include <pxr/imaging/hd/selectionsSchema.h>
+#include <pxr/imaging/hd/xformSchema.h>
+
+#include <maya/M3dView.h>
+#include <maya/MPoint.h>
+
+#include <gtest/gtest.h>
+
+PXR_NAMESPACE_USING_DIRECTIVE
+
+using namespace MayaHydra;
+
+namespace {
+
+FindPrimPredicate findPickPrimPredicate(const std::string& objectName, const TfToken& primType)
+{
+    return [objectName,
+            primType](const HdSceneIndexBaseRefPtr& sceneIndex, const SdfPath& primPath) -> bool {
+        return primPath.GetAsString().find(objectName) != std::string::npos
+            && sceneIndex->GetPrim(primPath).primType == primType;
+    };
+}
+
+void getPrimMouseCoords(const HdSceneIndexPrim& prim, M3dView& view, QPoint& outMouseCoords)
+{
+    HdDataSourceBaseHandle xformDataSource = HdContainerDataSource::Get(prim.dataSource, HdXformSchema::GetDefaultLocator());
+    ASSERT_NE(xformDataSource, nullptr);
+    HdContainerDataSourceHandle xformContainerDataSource = HdContainerDataSource::Cast(xformDataSource);
+    ASSERT_NE(xformContainerDataSource, nullptr);
+    HdXformSchema xformSchema(xformContainerDataSource);
+    ASSERT_NE(xformSchema.GetMatrix(), nullptr);
+    GfMatrix4d xformMatrix = xformSchema.GetMatrix()->GetTypedValue(0);
+    GfVec3d translation = xformMatrix.ExtractTranslation();
+
+    MPoint worldPosition(translation[0], translation[1], translation[2], 1.0);
+    short   viewportX = 0, viewportY = 0;
+    MStatus worldToViewStatus;
+    // First assert checks that the point was not clipped, second assert checks the general MStatus
+    ASSERT_TRUE(view.worldToView(worldPosition, viewportX, viewportY, &worldToViewStatus));
+    ASSERT_TRUE(worldToViewStatus);
+
+    // Qt and M3dView use opposite Y-coordinates
+    outMouseCoords = QPoint(viewportX, view.portHeight() - viewportY);
+}
+
+void ensureSelected(const SceneIndexInspector& inspector, const FindPrimPredicate& primPredicate)
+{
+    // 2024-03-01 : Due to the extra "Lighted" hierarchy, it is possible for an object to be split
+    // into two prims, only one of which will be selected. We will tolerate this in the test, but 
+    // we'll make sure there are at most two prims for that object. We'll also allow a prim not 
+    // to have any selections, but at least one prim must be selected.
+    PrimEntriesVector primEntries = inspector.FindPrims(primPredicate);
+    ASSERT_GE(primEntries.size(), 1u);
+    ASSERT_LE(primEntries.size(), 2u);
+
+    size_t nbSelectedPrims = 0;
+    for (const auto& primEntry : primEntries) {
+        HdSelectionsSchema selectionsSchema = HdSelectionsSchema::GetFromParent(primEntry.prim.dataSource);
+        if (selectionsSchema.GetNumElements() > 0u) {
+            ASSERT_EQ(selectionsSchema.GetNumElements(), 1u);
+            HdSelectionSchema selectionSchema = selectionsSchema.GetElement(0);
+            EXPECT_TRUE(selectionSchema.GetFullySelected());
+            nbSelectedPrims++;
+        }
+    }
+
+    ASSERT_GT(nbSelectedPrims, 0u);
+}
+
+void ensureUnselected(const SceneIndexInspector& inspector, const FindPrimPredicate& primPredicate)
+{
+    PrimEntriesVector primEntries = inspector.FindPrims(primPredicate);
+    for (const auto& primEntry : primEntries) {
+        HdSelectionsSchema selectionsSchema
+            = HdSelectionsSchema::GetFromParent(primEntry.prim.dataSource);
+        ASSERT_EQ(selectionsSchema.IsDefined(), false);
+    }
+}
+
+} // namespace
+
+TEST(TestPicking, pickObject)
+{
+    const SceneIndicesVector& sceneIndices = GetTerminalSceneIndices();
+    ASSERT_GT(sceneIndices.size(), 0u);
+    SceneIndexInspector inspector(sceneIndices.front());
+
+    auto [argc, argv] = getTestingArgs();
+    ASSERT_EQ(argc, 2);
+    const std::string objectName(argv[0]);
+    const TfToken primType(argv[1]);
+
+    ensureUnselected(inspector, PrimNamePredicate(objectName));
+
+    PrimEntriesVector prims = inspector.FindPrims(findPickPrimPredicate(objectName, primType));
+    ASSERT_EQ(prims.size(), 1u);
+
+    M3dView active3dView = M3dView::active3dView();
+
+    QPoint primMouseCoords;
+    getPrimMouseCoords(prims.front().prim, active3dView, primMouseCoords);
+
+    mousePress(Qt::MouseButton::LeftButton, active3dView.widget(), primMouseCoords);
+    mouseRelease(Qt::MouseButton::LeftButton, active3dView.widget(), primMouseCoords);
+
+    active3dView.refresh();
+
+    ensureSelected(inspector, PrimNamePredicate(objectName));
+}
+
+TEST(TestPicking, marqueeSelect)
+{
+    const SceneIndicesVector& sceneIndices = GetTerminalSceneIndices();
+    ASSERT_GT(sceneIndices.size(), 0u);
+    SceneIndexInspector inspector(sceneIndices.front());
+
+    auto [argc, argv] = getTestingArgs();
+    ASSERT_TRUE(argc % 2 == 0); // Each object is identified by both its name and a type
+    ASSERT_TRUE(argc >= 4); // We need at least two objects to do the marquee selection
+    std::vector<std::pair<std::string, TfToken>> objectsToSelect;
+    for (int iArg = 0; iArg < argc; iArg += 2) {
+        objectsToSelect.push_back(std::make_pair(std::string(argv[iArg]), TfToken(argv[iArg + 1])));
+    }
+
+    for (const auto& object : objectsToSelect) {
+        ensureUnselected(inspector, PrimNamePredicate(object.first));
+    }
+
+    M3dView active3dView = M3dView::active3dView();
+
+    // We get the first prim's mouse coordinates and initialize the selection rectangle
+    // with them; we then iterate on the other prims and expand the selection rectangle
+    // to fit them all.
+
+    // Get the first prim's mouse coords
+    PrimEntriesVector initialPrimEntries = inspector.FindPrims(
+        findPickPrimPredicate(objectsToSelect.front().first, objectsToSelect.front().second));
+    ASSERT_EQ(initialPrimEntries.size(), 1u);
+    QPoint initialMouseCoords;
+    getPrimMouseCoords(initialPrimEntries.front().prim, active3dView, initialMouseCoords);
+
+    // Initialize the selection rectangle
+    QPoint topLeftMouseCoords = initialMouseCoords;
+    QPoint bottomRightMouseCoords = initialMouseCoords;
+
+    // Expand the selection rectangle to fit all prims
+    for (size_t iObject = 1; iObject < objectsToSelect.size(); iObject++) {
+        PrimEntriesVector objectPrims = inspector.FindPrims(
+            findPickPrimPredicate(objectsToSelect[iObject].first, objectsToSelect[iObject].second));
+        ASSERT_EQ(objectPrims.size(), 1u);
+        QPoint objectMouseCoords;
+        getPrimMouseCoords(objectPrims.front().prim, active3dView, objectMouseCoords);
+
+        if (objectMouseCoords.x() > bottomRightMouseCoords.x()) {
+            bottomRightMouseCoords.setX(objectMouseCoords.x());
+        }
+        if (objectMouseCoords.x() < topLeftMouseCoords.x()) {
+            topLeftMouseCoords.setX(objectMouseCoords.x());
+        }
+        if (objectMouseCoords.y() > topLeftMouseCoords.y()) {
+            topLeftMouseCoords.setY(objectMouseCoords.y());
+        }
+        if (objectMouseCoords.y() < bottomRightMouseCoords.y()) {
+            bottomRightMouseCoords.setY(objectMouseCoords.y());
+        }
+    }
+
+    // Perform the marquee selection
+    mousePress(Qt::MouseButton::LeftButton, active3dView.widget(), topLeftMouseCoords);
+    mouseMoveTo(active3dView.widget(), bottomRightMouseCoords);
+    mouseRelease(Qt::MouseButton::LeftButton, active3dView.widget(), bottomRightMouseCoords);
+
+    active3dView.refresh();
+
+    for (const auto& object : objectsToSelect) {
+        ensureSelected(inspector, PrimNamePredicate(object.first));
+    }
+}