v0.8.2.0 Release

v0.8.2.0 Release

CHANGELOG

@@ -1,16 +1,15 @@
-Changelog (v0.8.1.0)
+Changelog (v0.8.2.0)
 --------------------
- - NEW! Eye Contact feature: an AI algorithm to help users keep their gaze engaged in video communication. The feature jointly estimates a user’s gaze direction and redirects it to frontal in video sequences.
- - NEW! Face Expression Estimation (Beta) feature estimates facial expression coefficients From the video or the provided facial landmarks. ExpressionApp is added to demonstrate the new Face Expressions feature.
- - NEW! Default face model for the Face 3D mesh and tracking feature, face_model2.nvf, now ships with the SDK. The old SFM based face_model0.nvf is no longer required.
- - 3D Body Pose Estimation:
-    - NEW! Added the support for Multi Person Tracking. This feature is supported by the Windows SDK only.
-    - FocalLength is now a NvAR_Parameter_Input. Users can now change FocalLength at every NvAR_Run() without having to call NvAR_Load().
-    - The reference pose returned by the feature has been updated
- - Facial landmark estimation
-    - NEW! There are now 2 modalities for facial landmark tracking: {0,1} -> {performance, quality}. Make sure to choose the preferred mode for your application. The default for face mesh fitting and expression estimation are 1, and the others are 0.
-    - Head Pose output from the NvAR_Feature_LandmarkDetection feature is now in the OpenGL convention. Changed from X-back(towards the camera), Y-right, Z-down to X-right, Y-up, Z-back(towards the camera).
-    - The sample apps now show the headpose in the OpenGL convention. The color coding of the axes is  Red - X , Green - Y, Blue - Z
- - NvCVImage_Transfer() now sets alpha to 255 or 1.0f when doing RGB -> RGBA. NvCVImage_CompositeRect() has a premultiplied alpha mode added
- - Migrated to TensorRT 8.4.2.2
- - Migrated to CUDA 11.6u1
+ - Face Expression Estimation
+    - 6DOF head pose now available.
+        - Enable by setting NvAR_Parameter_Config(PoseMode) to 1. Default is 0. 
+        - Get the pose translation using NvAR_Parameter_Output(PoseTranslation). 
+        - Camera intrinsics can be set using NvAR_Parameter_Input(CameraIntrinsicParams).
+    - Expression estimation model updated. The new model is more accurate, and enables subtle expressions such as asymmetric brows, inner/outer brow separation, asymmetric smile and frown, jaw left/right, squint shapes, crossing eye gaze, cheek puff, and more. 
+        - Enable cheek puff shapes by setting NvAR_Parameter_Config(EnableCheekPuff) to 1.
+    - New face model for visualization recommended for FaceExpressions feature. The model has updated blendshapes, and face area partitioning. 
+        - Enable using --render_model=face_model3.nvf in ExpressionApp.
+ - Eye Contact
+    - Performance improvements
+        - CUDA graphs functionality to enable CUDA optimization now available for Eye contact feature. 
+        - Use the parameter NvAR_Parameter_Config(UseCudaGraph) to enable/disable CUDA graphs. Default is OFF

nvar/include/nvAR_defs.h

@@ -117,20 +117,19 @@ typedef const char* NvAR_FeatureID;
 #define NvAR_Feature_BodyPoseEstimation   "BodyPoseEstimation"     // 
 #define NvAR_Feature_GazeRedirection      "GazeRedirection"        // 
 #define NvAR_Feature_FaceExpressions      "FaceExpressions"        // 
-#define NvAR_Feature_LivePortrait         "LivePortrait"           // 
-#define NvAR_Feature_FrameSelection       "FrameSelection"         // !FrameSelection!
 
 #define NvAR_Parameter_Input(Name) "NvAR_Parameter_Input_" #Name
 #define NvAR_Parameter_Output(Name) "NvAR_Parameter_Output_" #Name
 #define NvAR_Parameter_Config(Name) "NvAR_Parameter_Config_" #Name
 #define NvAR_Parameter_InOut(Name) "NvAR_Parameter_InOut_" #Name
 
-#define NVAR_TEMPORAL_FILTER_FACE_BOX                 (1 << 0)  // 0x001
-#define NVAR_TEMPORAL_FILTER_FACIAL_LANDMARKS         (1 << 1)  // 0x002
-#define NVAR_TEMPORAL_FILTER_FACE_ROTATIONAL_POSE     (1 << 2)  // 0x004
-#define NVAR_TEMPORAL_FILTER_FACIAL_EXPRESSIONS       (1 << 4)  // 0x010
-#define NVAR_TEMPORAL_FILTER_FACIAL_GAZE              (1 << 5)  // 0x020
-#define NVAR_TEMPORAL_FILTER_ENHANCE_EXPRESSIONS      (1 << 8)  // 0x100
+#define NVAR_TEMPORAL_FILTER_FACE_BOX                 (1U << 0)  // 0x001
+#define NVAR_TEMPORAL_FILTER_FACIAL_LANDMARKS         (1U << 1)  // 0x002
+#define NVAR_TEMPORAL_FILTER_FACE_ROTATIONAL_POSE     (1U << 2)  // 0x004
+#define NVAR_TEMPORAL_FILTER_FACIAL_EXPRESSIONS       (1U << 4)  // 0x010
+#define NVAR_TEMPORAL_FILTER_FACIAL_GAZE              (1U << 5)  // 0x020
+#define NVAR_TEMPORAL_FILTER_ENHANCE_EXPRESSIONS      (1U << 8)  // 0x100
+
 
 /*
 Parameters supported by each NvAR_FeatureID
@@ -178,6 +177,7 @@ NvAR_Parameter_Config(CUDAStream) - OPTIONAL                            //
 NvAR_Parameter_Config(Temporal) - OPTIONAL                              // 
 NvAR_Parameter_Config(GazeMode) - OPTIONAL                              // 
 NvAR_Parameter_Config(ModelName) - OPTIONAL                             // 
+NvAR_Parameter_Config(Mode) - OPTIONAL                                  // 
 NvAR_Parameter_Config(GPU) - OPTIONAL                                   // 
 NvAR_Parameter_Config(VertexCount) - QUERY                              // 
 NvAR_Parameter_Config(TriangleCount) - QUERY                            // 
@@ -286,45 +286,20 @@ NvAR_Parameter_Config(ExpressionCount) - QUERY                          //
 Input:                                                                  // 
 NvAR_Parameter_Input(Image)                                             // 
 NvAR_Parameter_Input(Landmarks) - OPTIONAL                              // 
+NvAR_Parameter_Input(PoseMode) - OPTIONAL                               // 
+NvAR_Parameter_Input(CameraIntrinsicParams) - OPTIONAL                  // 
                                                                         // 
 Output:                                                                 // 
 NvAR_Parameter_Output(ExpressionCoefficients)                           // 
 NvAR_Parameter_Output(Landmarks) - OPTIONAL                             // 
 NvAR_Parameter_Output(LandmarksConfidence) - OPTIONAL                   // 
 NvAR_Parameter_Output(Pose) - OPTIONAL                                  // 
+NvAR_Parameter_Output(PoseTranslation) - OPTIONAL                       // 
 NvAR_Parameter_Output(BoundingBoxes) - OPTIONAL                         // 
 NvAR_Parameter_Output(BoundingBoxesConfidence) - OPTIONAL               // 
 
-*******NvAR_Feature_LivePortrait*******                                 // 
-Config:                                                                 // 
-NvAR_Parameter_Config(FeatureDescription)                               // 
-NvAR_Parameter_Config(CUDAStream)                                       // 
-NvAR_Parameter_Config(ModelDir)                                         // 
-NvAR_Parameter_Config(Temporal)                                         // 
-NvAR_Parameter_Config(Mode)                                             // 
-                                                                        // 
-Input:                                                                  // 
-NvAR_Parameter_Input(SourceImage)                                       // 
-NvAR_Parameter_Input(DriveImage)                                        // 
-NvAR_Parameter_Input(BackgroundImage)                                   // 
 
-Output:                                                                 // 
-NvAR_Parameter_Output(GeneratedImage)                                   // 
 
-*******NvAR_Feature_FrameSelection*******                               // !FrameSelection!
-Config:                                                                 // !FrameSelection!
-NvAR_Parameter_Config(FeatureDescription)                               // !FrameSelection!
-NvAR_Parameter_Config(CUDAStream)                                       // !FrameSelection!
-NvAR_Parameter_Config(ModelDir)                                         // !FrameSelection!
-NvAR_Parameter_Config(Mode)                                             // !FrameSelection!
-                                                                        // !FrameSelection!
-Input:                                                                  // !FrameSelection!
-NvAR_Parameter_Input(Image)                                             // !FrameSelection!
-                                                                        // !FrameSelection!
-Output:                                                                 // !FrameSelection!
-NvAR_Parameter_Output(FrameSelected)                                    // !FrameSelection!
 
 */
-
-
 #endif  // NvAR_DEFS_H

nvar/include/nvCVImage.h

@@ -538,6 +538,7 @@ NvCV_Status NvCV_API NvCVImage_TransferToYUV(
 
 //! Composite one source image over another using the given matte.
 //! This accommodates all RGB and RGBA formats, with u8 and f32 components.
+//! If the bg has alpha, then the dst alpha is updated for use in subsequent composition.
 //! \param[in]  fg      the foreground source image.
 //! \param[in]  bg      the background source image.
 //! \param[in]  mat     the matte  Yu8   (or Au8)   image, indicating where the src should come through.
@@ -547,7 +548,6 @@ NvCV_Status NvCV_API NvCVImage_TransferToYUV(
 //! \return NVCV_ERR_PIXELFORMAT if the pixel format is not accommodated.
 //! \return NVCV_ERR_MISMATCH    if either the fg & bg & dst formats do not match, or if fg & bg & dst & mat are not
 //!                              in the same address space (CPU or GPU).
-//! \bug  Though RGBA destinations are accommodated, the A channel is not updated at all.
 #if RTX_CAMERA_IMAGE == 0
 NvCV_Status NvCV_API NvCVImage_Composite(const NvCVImage *fg, const NvCVImage *bg, const NvCVImage *mat, NvCVImage *dst,
             struct CUstream_st *stream);
@@ -556,8 +556,11 @@ NvCV_Status NvCV_API NvCVImage_Composite(const NvCVImage *fg, const NvCVImage *b
 #endif // RTX_CAMERA_IMAGE == 1
 
 
-//! Composite one source image over another using the given matte.
+//! Composite one source image rectangular region over another using the given matte.
 //! This accommodates all RGB and RGBA formats, with u8 and f32 components.
+//! If the bg has alpha, then the dst alpha is updated for use in subsequent composition.
+//! If the background is not opaque, it is recommended that all images be premultiplied by alpha,
+//! and mode 1 composition be used, to yield the most meaningful composite matte.
 //! \param[in]      fg      the foreground source image.
 //! \param[in]      fgOrg   the upper-left corner of the fg image to be composited (NULL implies (0,0)).
 //! \param[in]      bg      the background source image.
@@ -570,14 +573,11 @@ NvCV_Status NvCV_API NvCVImage_Composite(const NvCVImage *fg, const NvCVImage *b
 //! \param[in]      dstOrg  the upper-left corner of the dst image to be updated (NULL implies (0,0)).
 //! \param[in]      stream  the CUDA stream on which the composition is to be performed.
 //! \note   If a smaller region of a matte is desired, a window can be created using
-//!         NvCVImage_InitView() for chunky or NvCVImage_Init() for planar pixels.
+//!         NvCVImage_InitView() for chunky pixels, as illustrated below in NvCVImage_CompositeRectA().
 //! \return NVCV_SUCCESS         if the operation was successful.
 //! \return NVCV_ERR_PIXELFORMAT if the pixel format is not accommodated.
 //! \return NVCV_ERR_MISMATCH    if either the fg & bg & dst formats do not match, or if fg & bg & dst & mat are not
 //!                              in the same address space (CPU or GPU).
-//! \bug  Though RGBA destinations are accommodated, the A channel is not updated at all.
-//! \todo Accommodate premultiplied alpha, either as a flag in NvCVImage or as a different mode.
-//! \todo If the destination has an A channel, update it as per Adobe and Pixar.
 NvCV_Status NvCV_API NvCVImage_CompositeRect(
     const NvCVImage *fg,  const NvCVPoint2i *fgOrg,
     const NvCVImage *bg,  const NvCVPoint2i *bgOrg,
@@ -586,6 +586,41 @@ NvCV_Status NvCV_API NvCVImage_CompositeRect(
     struct CUstream_st *stream);
 
 
+//! Composite one RGBA or BGRA source image rectangular region over another RGB, BGR, RGBA or BGRA region.
+//! This accommodates all RGB and RGBA formats, with u8 and f32 components.
+//! If the bg has alpha, then the dst alpha is updated for use in subsequent composition.
+//! If the background is not opaque, it is recommended that all images be premultiplied by alpha,
+//! and mode 1 composition be used, to yield the most meaningful composite matte.
+//! \param[in]      fg      the foreground RGBA or BGRA source image.
+//! \param[in]      fgRect  a sub-rectangle of the fg image (NULL implies the whole image).
+//! \param[in]      bg      the background source image.
+//! \param[in]      bgOrg   the upper-left corner of the bg image to be composited (NULL implies (0,0)).
+//! \param[in]      mode    the composition mode: 0 (straight alpha over) or 1 (premultiplied alpha over).
+//! \param[out]     dst     the destination image. This can be the same as fg or bg.
+//! \param[in]      dstOrg  the upper-left corner of the dst image to be updated (NULL implies (0,0)).
+//! \param[in]      stream  the CUDA stream on which the composition is to be performed.
+//! \return NVCV_SUCCESS         if the operation was successful.
+//! \return NVCV_ERR_PIXELFORMAT if the pixel format is not accommodated.
+//! \return NVCV_ERR_MISMATCH    if either the fg & bg & dst formats do not match, or if fg & bg & dst & mat are not
+//!                              in the same address space (CPU or GPU).
+//! \bug  fgRect will only work for chunky images, not planar.
+#ifdef __cplusplus
+inline NvCV_Status NvCVImage_CompositeRectA(
+    const NvCVImage *fg, const NvCVRect2i  *fgRect,
+    const NvCVImage *bg, const NvCVPoint2i *bgOrg,
+    unsigned mode,
+    NvCVImage       *dst, const NvCVPoint2i *dstOrg,
+    struct CUstream_st *stream
+) {
+  if (fgRect) {
+    NvCVImage fgView(const_cast<NvCVImage*>(fg), fgRect->x, fgRect->y, fgRect->width, fgRect->height);
+    return NvCVImage_CompositeRect(&fgView, nullptr, bg, bgOrg, &fgView, mode, dst, dstOrg, stream);
+  }
+  return NvCVImage_CompositeRect(fg, nullptr, bg, bgOrg, fg, mode, dst, dstOrg, stream);
+}
+#endif // __cplusplus
+
+
 //! Composite a source image over a constant color field using the given matte.
 //! \param[in]      src     the source image.
 //! \param[in]      mat     the matte  image, indicating where the src should come through.
@@ -598,7 +633,6 @@ NvCV_Status NvCV_API NvCVImage_CompositeRect(
 //! \return NVCV_ERR_MISMATCH    if fg & mat & dst & bgColor are not in the same address space (CPU or GPU).
 //! \note   The bgColor must remain valid until complete; this is an important consideration especially if
 //!         the buffers are on the GPU and NvCVImage_CompositeOverConstant() runs asynchronously.
-//! \bug    Though RGBA destinations are accommodated, the A channel is not updated at all.
 NvCV_Status NvCV_API NvCVImage_CompositeOverConstant(
 #if RTX_CAMERA_IMAGE == 0
     const NvCVImage *src, const NvCVImage *mat, const void *bgColor, NvCVImage *dst, struct CUstream_st *stream

nvar/include/nvCVStatus.h

@@ -80,6 +80,10 @@ typedef enum NvCV_Status {
   NVCV_ERR_TOOSMALL              = -33,  //!< A supplied parameter or buffer is not large enough.
   NVCV_ERR_TOOBIG                = -34,  //!< A supplied parameter is too big.
   NVCV_ERR_WRONGSIZE             = -35,  //!< A supplied parameter is not the expected size.
+  NVCV_ERR_OBJECTNOTFOUND        = -36,  //!< The specified object was not found.
+  NVCV_ERR_SINGULAR              = -37,  //!< A mathematical singularity has been encountered.
+  NVCV_ERR_NOTHINGRENDERED       = -38,  //!< Nothing was rendered in the specified region.
+  NVCV_ERR_CONVERGENCE           = -39,  //!< An iteration did not converge satisfactorily.
 
   NVCV_ERR_OPENGL                = -98,  //!< An OpenGL error has occurred.
   NVCV_ERR_DIRECT3D              = -99,  //!< A Direct3D error has occurred.