depthmap/normalmap rendering

alicevision · Oct 3, 2024 · 44613ef · 44613ef
1 parent 1d5b5c5
commit 44613ef
Show file tree

Hide file tree

Showing 6 changed files with 351 additions and 3 deletions.
diff --git a/src/aliceVision/mesh/MeshIntersection.cpp b/src/aliceVision/mesh/MeshIntersection.cpp
@@ -28,7 +28,7 @@ bool MeshIntersection::initialize(const std::string & pathToModel)
     return true;
 }
 
-bool MeshIntersection::peek(Vec3 & output, const camera::IntrinsicBase & intrinsic, const Vec2 & imageCoords)
+bool MeshIntersection::peekPoint(Vec3 & output, const camera::IntrinsicBase & intrinsic, const Vec2 & imageCoords)
 {
     const Vec3 posCamera = _pose.center();
     const Vec3 wdir = intrinsic.backproject(imageCoords, true, _pose, 1.0);
@@ -58,5 +58,35 @@ bool MeshIntersection::peek(Vec3 & output, const camera::IntrinsicBase & intrins
     return true;
 }
 
+bool MeshIntersection::peekNormal(Vec3 & output, const camera::IntrinsicBase & intrinsic, const Vec2 & imageCoords)
+{
+    const Vec3 posCamera = _pose.center();
+    const Vec3 wdir = intrinsic.backproject(imageCoords, true, _pose, 1.0);
+    const Vec3 dir = (wdir - posCamera).normalized();
+
+    //Create geogram ray from alicevision ray
+    GEO::Ray ray;
+    ray.origin.x = posCamera.x();
+    ray.origin.y = -posCamera.y();
+    ray.origin.z = -posCamera.z();
+    ray.direction.x = dir.x();
+    ray.direction.y = -dir.y();
+    ray.direction.z = -dir.z();
+
+    GEO::MeshFacetsAABB::Intersection intersection;
+    if (!_aabb.ray_nearest_intersection(ray, intersection)) 
+    {
+        return false;
+    }
+
+    const GEO::vec3 n = GEO::normalize(intersection.N);
+
+    output.x() = n.x;
+    output.y() = -n.y;
+    output.z() = -n.z;
+
+    return true;
+}
+
 }
 }
diff --git a/src/aliceVision/mesh/MeshIntersection.hpp b/src/aliceVision/mesh/MeshIntersection.hpp
@@ -37,7 +37,16 @@ class MeshIntersection
      * @param imageCoords the camera observation we want to use to estimate its 'depth'
      * @return true if the ray intersect the mesh.
     */
-    bool peek(Vec3 & output, const camera::IntrinsicBase & intrinsic, const Vec2 & imageCoords);
+    bool peekPoint(Vec3 & output, const camera::IntrinsicBase & intrinsic, const Vec2 & imageCoords);
+
+    /**
+     * @brief peek a point and get its normal on the mesh given a input camera observation
+     * @param output the output measured normal
+     * @param intrinsic the camera intrinsics to use for ray computation
+     * @param imageCoords the camera observation we want to use to estimate its 'depth'
+     * @return true if the ray intersect the mesh.
+    */
+    bool peekNormal(Vec3 & output, const camera::IntrinsicBase & intrinsic, const Vec2 & imageCoords);
 
 private:
     GEO::Mesh _mesh;

diff --git a/src/software/utils/CMakeLists.txt b/src/software/utils/CMakeLists.txt
@@ -399,6 +399,32 @@ if(ALICEVISION_BUILD_MVS)
               Geogram::geogram
               ${Boost_LIBRARIES}
     )
+
+    # Rendering of depthmaps
+    alicevision_add_software(aliceVision_depthMapRendering
+        SOURCE main_depthMapRendering.cpp
+        FOLDER ${FOLDER_SOFTWARE_UTILS}
+        LINKS aliceVision_system
+            aliceVision_cmdline
+            aliceVision_feature
+            aliceVision_mesh
+            aliceVision_sfm
+            aliceVision_sfmData
+            aliceVision_sfmDataIO
+    )
+
+    # Rendering of normalmaps
+    alicevision_add_software(aliceVision_normalMapRendering
+        SOURCE main_normalMapRendering.cpp
+        FOLDER ${FOLDER_SOFTWARE_UTILS}
+        LINKS aliceVision_system
+            aliceVision_cmdline
+            aliceVision_feature
+            aliceVision_mesh
+            aliceVision_sfm
+            aliceVision_sfmData
+            aliceVision_sfmDataIO
+    )
 
     # Mesh Remove Unseen Faces
     alicevision_add_software(aliceVision_meshRemoveUnseenFaces

diff --git a/src/software/utils/main_depthMapRendering.cpp b/src/software/utils/main_depthMapRendering.cpp
@@ -0,0 +1,151 @@
+// This file is part of the AliceVision project.
+// Copyright (c) 2024 AliceVision contributors.
+// This Source Code Form is subject to the terms of the Mozilla Public License,
+// v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+#include <aliceVision/sfmData/SfMData.hpp>
+#include <aliceVision/sfmDataIO/sfmDataIO.hpp>
+#include <aliceVision/system/Logger.hpp>
+#include <aliceVision/cmdline/cmdline.hpp>
+#include <aliceVision/system/main.hpp>
+#include <aliceVision/image/Image.hpp>
+#include <aliceVision/mesh/MeshIntersection.hpp>
+
+#include <filesystem>
+
+
+
+// These constants define the current software version.
+// They must be updated when the command line is changed.
+#define ALICEVISION_SOFTWARE_VERSION_MAJOR 1
+#define ALICEVISION_SOFTWARE_VERSION_MINOR 0
+
+using namespace aliceVision;
+
+namespace po = boost::program_options;
+
+int aliceVision_main(int argc, char** argv)
+{
+    // command-line parameters
+    std::string sfmDataFilename;
+    std::string meshFilename;
+    std::string outputDirectory;
+
+    // clang-format off
+    po::options_description requiredParams("Required parameters");
+    requiredParams.add_options()
+        ("input,i", po::value<std::string>(&sfmDataFilename)->required(),
+         "SfMData file.")
+        ("mesh,i", po::value<std::string>(&meshFilename)->required(),
+         "mesh file.")
+        ("output,o", po::value<std::string>(&outputDirectory)->required(),
+         "Output directory for depthmaps.");
+    // clang-format on
+
+    CmdLine cmdline("AliceVision depthmapRendering");
+    cmdline.add(requiredParams);
+    if (!cmdline.execute(argc, argv))
+    {
+        return EXIT_FAILURE;
+    }
+
+     // set maxThreads
+    HardwareContext hwc = cmdline.getHardwareContext();
+    omp_set_num_threads(hwc.getMaxThreads());
+
+    std::filesystem::path pathOutputDirectory(outputDirectory);
+
+    // Load input scene
+    sfmData::SfMData sfmData;
+    if (!sfmDataIO::load(sfmData, sfmDataFilename, sfmDataIO::ESfMData::ALL))
+    {
+        ALICEVISION_LOG_ERROR("The input SfMData file '" << sfmDataFilename << "' cannot be read");
+        return EXIT_FAILURE;
+    }
+
+    //Load mesh in the mesh intersection object
+    ALICEVISION_LOG_INFO("Loading mesh");
+    mesh::MeshIntersection mi;
+    if (!mi.initialize(meshFilename))
+    {
+        return EXIT_FAILURE;
+    }
+
+    for (const auto & [index, view] : sfmData.getViews())
+    {
+        if (!sfmData.isPoseAndIntrinsicDefined(index))
+        {
+            continue;
+        }
+
+
+        ALICEVISION_LOG_INFO("Generating depthmap for view " << index);
+
+        //Retrieve metadatas for copying in the depthmap
+        oiio::ParamValueList metadata = image::readImageMetadata(view->getImageInfo()->getImagePath());
+
+        const auto & intrinsic = sfmData.getIntrinsicSharedPtr(*view);
+        std::shared_ptr<camera::Pinhole> pinHole = std::dynamic_pointer_cast<camera::Pinhole>(intrinsic);
+        if (!pinHole)
+        {
+            ALICEVISION_LOG_INFO("This view is not a pinhole camera. Ignoring.");
+            continue;
+        }
+
+        const auto pose = sfmData.getPose(*view).getTransform();
+
+        Vec3 center = pose.center();
+
+        mi.setPose(pose);
+
+        int w = view->getImageInfo()->getWidth();
+        int h = view->getImageInfo()->getHeight();
+
+        image::Image<float> image(w, h, 0.0f);
+        image::Image<unsigned char> mask(w, h, 0);
+
+        #pragma omp parallel for
+        for (int i = 0; i < h; i++)
+        {
+            for (int j = 0; j < w; j++)
+            {
+                Vec2 pt;
+                pt.x() = j;
+                pt.y() = i;
+
+                //Find the 3d point 
+                //Which is the intersection of the ray and the mesh
+                Vec3 pt3d;
+                if (!mi.peekPoint(pt3d, *intrinsic, pt))
+                {
+                    continue;
+                }
+
+                //Assume depth map contains length to camera center
+                double length = (pt3d - center).norm();
+                image(i, j) = length;
+                mask(i, j) = 255;
+            }
+        }
+
+
+        //Store metadata used for 3D Viewer
+        Eigen::Matrix<double, 3, 3, Eigen::RowMajor> iCamArr = pose.rotation().transpose() * pinHole->K().inverse();
+        metadata.push_back(oiio::ParamValue("AliceVision:CArr", oiio::TypeDesc(oiio::TypeDesc::DOUBLE, oiio::TypeDesc::VEC3), 1, &center[0]));
+        metadata.push_back(oiio::ParamValue("AliceVision:iCamArr", oiio::TypeDesc(oiio::TypeDesc::DOUBLE, oiio::TypeDesc::MATRIX33), 1, iCamArr.data()));
+
+
+        //Store depthmap
+        auto path = (pathOutputDirectory / (std::to_string(index) + "_depthMap.exr"));
+        ALICEVISION_LOG_INFO("Ouput depthmap to " << path);
+        image::writeImage(path.string(), image, image::ImageWriteOptions(), metadata);
+
+        //Store mask
+        path = (pathOutputDirectory / (std::to_string(index) + "_mask.exr"));
+        ALICEVISION_LOG_INFO("Ouput mask to " << path);
+        image::writeImage(path.string(), mask, image::ImageWriteOptions(), metadata);
+    }
+
+    return EXIT_SUCCESS;
+}
diff --git a/src/software/utils/main_normalMapRendering.cpp b/src/software/utils/main_normalMapRendering.cpp
@@ -0,0 +1,132 @@
+// This file is part of the AliceVision project.
+// Copyright (c) 2024 AliceVision contributors.
+// This Source Code Form is subject to the terms of the Mozilla Public License,
+// v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+#include <aliceVision/sfmData/SfMData.hpp>
+#include <aliceVision/sfmDataIO/sfmDataIO.hpp>
+#include <aliceVision/system/Logger.hpp>
+#include <aliceVision/cmdline/cmdline.hpp>
+#include <aliceVision/system/main.hpp>
+#include <aliceVision/image/Image.hpp>
+#include <aliceVision/mesh/MeshIntersection.hpp>
+
+#include <filesystem>
+
+
+
+// These constants define the current software version.
+// They must be updated when the command line is changed.
+#define ALICEVISION_SOFTWARE_VERSION_MAJOR 1
+#define ALICEVISION_SOFTWARE_VERSION_MINOR 0
+
+using namespace aliceVision;
+
+namespace po = boost::program_options;
+
+int aliceVision_main(int argc, char** argv)
+{
+    // command-line parameters
+    std::string sfmDataFilename;
+    std::string meshFilename;
+    std::string outputDirectory;
+
+    // clang-format off
+    po::options_description requiredParams("Required parameters");
+    requiredParams.add_options()
+        ("input,i", po::value<std::string>(&sfmDataFilename)->required(),
+         "SfMData file.")
+        ("mesh,i", po::value<std::string>(&meshFilename)->required(),
+         "mesh file.")
+        ("output,o", po::value<std::string>(&outputDirectory)->required(),
+         "Output directory for depthmaps.");
+    // clang-format on
+
+    CmdLine cmdline("AliceVision normalmapRendering");
+    cmdline.add(requiredParams);
+    if (!cmdline.execute(argc, argv))
+    {
+        return EXIT_FAILURE;
+    }
+
+     // set maxThreads
+    HardwareContext hwc = cmdline.getHardwareContext();
+    omp_set_num_threads(hwc.getMaxThreads());
+
+    std::filesystem::path pathOutputDirectory(outputDirectory);
+
+    // Load input scene
+    sfmData::SfMData sfmData;
+    if (!sfmDataIO::load(sfmData, sfmDataFilename, sfmDataIO::ESfMData::ALL))
+    {
+        ALICEVISION_LOG_ERROR("The input SfMData file '" << sfmDataFilename << "' cannot be read");
+        return EXIT_FAILURE;
+    }
+
+    //Load mesh in the mesh intersection object
+    ALICEVISION_LOG_INFO("Loading mesh");
+    mesh::MeshIntersection mi;
+    if (!mi.initialize(meshFilename))
+    {
+        return EXIT_FAILURE;
+    }
+
+    for (const auto & [index, view] : sfmData.getViews())
+    {
+        if (!sfmData.isPoseAndIntrinsicDefined(index))
+        {
+            continue;
+        }
+
+
+        ALICEVISION_LOG_INFO("Generating depthmap for view " << index);
+
+        //Retrieve metadatas for copying in the depthmap
+        oiio::ParamValueList metadata = image::readImageMetadata(view->getImageInfo()->getImagePath());
+
+        const auto & intrinsic = sfmData.getIntrinsicSharedPtr(*view);
+        const auto pose = sfmData.getPose(*view);
+
+        Vec3 center = pose.getTransform().center();
+
+        mi.setPose(pose.getTransform());
+
+        int w = view->getImageInfo()->getWidth();
+        int h = view->getImageInfo()->getHeight();
+        image::Image<image::RGBfColor> image(w, h, image::RGBfColor(0.0f,0.0f,0.0f));
+
+        #pragma omp parallel for
+        for (int i = 0; i < h; i++)
+        {
+            for (int j = 0; j < w; j++)
+            {
+                Vec2 pt;
+                pt.x() = j;
+                pt.y() = i;
+
+                //Find the 3d point 
+                //Which is the intersection of the ray and the mesh
+                //And get its normal
+                Vec3 normal;
+                if (!mi.peekNormal(normal, *intrinsic, pt))
+                {
+                    continue;
+                }
+
+                Vec3 cnormal = pose.getTransform().rotation() * normal;
+
+                auto & rgb = image(i, j);
+                rgb.r() = cnormal.x();
+                rgb.g() = cnormal.y();
+                rgb.b() = cnormal.z();
+            }
+        }
+
+        auto path = (pathOutputDirectory / (std::to_string(index) + "_normalMap.exr"));
+        ALICEVISION_LOG_INFO("Ouput to " << path);
+        image::writeImage(path.string(), image, image::ImageWriteOptions(), metadata);
+    }
+
+    return EXIT_SUCCESS;
+}
diff --git a/src/software/utils/main_sfmTransform.cpp b/src/software/utils/main_sfmTransform.cpp
@@ -409,7 +409,7 @@ bool parseLineUp(const std::string & lineUpFilename, const std::string & tracksF
         const Vec2 & imageCoords = trackitem.coords;
 
         Vec3 pt3d;
-        if (!meshIntersection.peek(pt3d, *intrinsic, imageCoords))
+        if (!meshIntersection.peekPoint(pt3d, *intrinsic, imageCoords))
         {
             continue;
         }