From b42b40e287d3af9ba7e48583caaf0f81753a26a6 Mon Sep 17 00:00:00 2001 From: Sajna Nazeer K Date: Mon, 8 Mar 2021 10:50:05 +0530 Subject: [PATCH] RDKC-9030, RDKC-9366 : deliveryDetection in SmTN,2TN with same timestamp Reason for change: Added delivery detection prebuilt binaries Test Procedure: Build with no issues. Risks:LOW Signed-off-by: Sajna_NazeerK@comcast.com RDKC-9030 : Implement delivery detection in SmartThumbnail Reason for change: Added delivery detection thread. Added memory usage reduction. Test Procedure: Build with no issues. Risks:LOW Signed-off-by: Sajna_NazeerK@comcast.com RDKC-9030 : Implement delivery detection in SmartThumbnail Reason for change: Added delivery detection prebuilt binaries. Edited IMAGE METADATA json string Test Procedure: Build with no issues. Risks:LOW Signed-off-by: Sajna_NazeerK@comcast.com RDKC-9030 : deliveryDetection in SmTN Reason for change: Added new union bounding box for detection. Added new logs. Test Procedure: Build with no issues. Risks:LOW Signed-off-by: Sajna_NazeerK@comcast.com RDKC-9368, RDKC-9030 : Add delivery detection test support in TestHarness Reason for change: Added methods to read frame from xvision and send delivery information to xvision. Integrated mediapipe changes to fix negative coordinates in person cropping. Test Procedure: Build with no issues. Risks:LOW Signed-off-by: Sajna_NazeerK@comcast.com RDKC-9713, RDKC-9714 : Add debug symbols, Reduce Delivery Threshold to 0.3 Reason for change: Added debug symbols for crash analysis. Reduce the delivery threshold to 0.3 Test Procedure: Build with no issues. Risks:LOW Signed-off-by: Sajna_NazeerK@comcast.com RDKC-9713, RDKC-9727 : Add crash symbols, configurable detection constants Reason for change: Added debug sym files for smartthumbnail, make constants for delivery detection configurable. Test Procedure: Build with no issues. Risks:LOW Signed-off-by: Sajna_NazeerK@comcast.com RDKC-9713, RDKC-9727 : Add crash symbols, configurable detection constants Reason for change: Added debug sym files for smartthumbnail, make constants for delivery detection configurable. Test Procedure: Build with no issues. Risks:LOW Signed-off-by: Sajna_NazeerK@comcast.com RDKC-9707, RDKC-9746 : upload delivery notification anytime,mediapipe crash Reason for change: Added debug symbols for crash analysis. Reduce the delivery threshold to 0.3. safeguard crop region - to fix crash from mediapipe [change in file: simple_image_cropping_calculator.cc] Test Procedure: Build with no issues. Risks:LOW Signed-off-by: Sajna_NazeerK@comcast.com RDKC-9707, RDKC-9746 : upload delivery notification anytime,mediapipe crash Reason for change: Added debug symbols for crash analysis. Reduce the delivery threshold to 0.3. safeguard crop region - to fix crash from mediapipe [change in file: simple_image_cropping_calculator.cc] Test Procedure: Build with no issues. Risks:LOW Signed-off-by: Sajna_NazeerK@comcast.com RDKC-9752 : Crash during smartthumbnail upload with delivery metadata Reason for change: Avoid creating smartthumbnail already one delivery detection is in progress. Test Procedure: Build with no issues. Risks:LOW Signed-off-by: Sajna_NazeerK@comcast.com RDKC-9758 : detection fails even if the score exceeds threshold Reason for change: Corrected the default value macros Test Procedure: Build with no issues. Risks:LOW Signed-off-by: Sajna_NazeerK@comcast.com RDKC-9030, RDKC-9699 : Integrate mediapipe into RDKC Reason for change: Added libdeliverydetector.so in LDFLAGS Test Procedure: Build with no issues. Risks:LOW Signed-off-by: Sajna_NazeerK@comcast.com RDKC-9701 : RFC feature control for delivery detection Reason for change: Added RFC feature checking. Test Procedure: Build with no issues. Risks:LOW Signed-off-by: Sajna_NazeerK@comcast.com RDKC-9992 : Smart thumbnails pending processing...discard Reason for change: Corrected the chance for a deadlock that can happen due to thread switch Test Procedure: Build with no issues. Risks:LOW Signed-off-by: Sajna_NazeerK@comcast.com RDKC-10066 : RFC check from service script Reason for change: Added check for RFC feature change. Test Procedure: Build with no issues. Risks:LOW Signed-off-by: Sajna_NazeerK@comcast.com RDKC-9833 : Detection stops at thumbnail generation Reason for change: Implemented as the new design. Detection stops at thumbnail generation Test Procedure: Build with no issues. Risks:LOW Signed-off-by: Sajna_NazeerK@comcast.com Signed-off-by: Sajna Nazeer K RDKC-9758, RDKC-9858, RDKC-9888: DetectionStats,last frame,motion issue Reason for change: Added new parameters in Detection Stats log. Changed mpipe_onThunbmailEvet api to mpipe_onLastFrameEvent. Added a fix for do not uploading motion events issue. Test Procedure: Build with no issues. Risks:LOW Signed-off-by: Sajna_NazeerK@comcast.com RDKC-9888 : maximum of mean augmentation score in Detection Stats Reason for change: Changed mean max augmentation score calculation. Test Procedure: Build with no issues. Risks:LOW Signed-off-by: Sajna_NazeerK@comcast.com RDKC-10085 : Wrong value for the time upload thread waited Reason for change: corrected the time calculation Test Procedure: Build with no issues. Risks:LOW Signed-off-by: Sajna_NazeerK@comcast.com RDKC-10093: continuous smartthumbnail process crash Reason for change: Corrected the wrong memset Test Procedure: Build with no issues. Risks:LOW Signed-off-by: Sajna_NazeerK@comcast.com RDKC-10086 : Received motion ROI is continuously logged Reason for change: Changed the logging Test Procedure: Build with no issues. Risks:LOW Signed-off-by: Sajna_NazeerK@comcast.com RDKC-10127: Delivery detection to run exactly on 1fps Reason for change: Adjusted sleep time in delivery detection thread to 1fps Test Procedure: Build with no issues. Risks:LOW Signed-off-by: Sajna_NazeerK@comcast.com RDKC-10159 : smart thumbnail cvr lite memory leak Reason for change: rtMessage_Release() to relaese memory associated with rtMessage in xcam2 instead of free(). Test Procedure: Build with no issues. Risks:LOW Signed-off-by: Sajna_NazeerK@comcast.com RDKC-10175, RDKC-9700 : Revised Logic for Smart Thumbnail Cropping Reason for change: cropping smartthumbnail from resized frame, if union bounding box is larger than the thumbnail size Also, added 400x300 cropping from buffer 1. Test Procedure: Build with no issues. Risks:LOW Signed-off-by: Sajna_NazeerK@comcast.com RDKC-10167 : Excessive logs in smart thumbnail logs with delivery detection Reason for change: modified mediapipe loglines to add timestamp. Test Procedure: Build with no issues. Risks:LOW Signed-off-by: Sajna_NazeerK@comcast.com RDKC-10376 : Separate comcast proprietary code from thumbnail Reason for change: Separating comcast edited mediapipe code to separate repo. Test Procedure: Build with no issues. Risks: LOW Signed-off-by: Sajna_NazeerK@comcast.com Change-Id: Ia078c4973d850b0222248e031800594fe88142cd --- rdk_build.sh | 26 + smart_thumbnail/Makefile | 16 + smart_thumbnail/include/mpipe_port.h | 46 ++ smart_thumbnail/include/smart_thumbnail.h | 98 ++- smart_thumbnail/main.cpp | 23 +- smart_thumbnail/mpipe_port.cpp | 216 +++++++ smart_thumbnail/smart_thumbnail.cpp | 592 +++++++++++++++++- smart_thumbnail_cvr_lite/Makefile | 16 + smart_thumbnail_cvr_lite/include/mpipe_port.h | 46 ++ .../include/smart_thumbnail.h | 102 ++- smart_thumbnail_cvr_lite/main.cpp | 23 +- smart_thumbnail_cvr_lite/mpipe_port.cpp | 147 +++++ smart_thumbnail_cvr_lite/smart_thumbnail.cpp | 473 +++++++++++++- 13 files changed, 1779 insertions(+), 45 deletions(-) create mode 100644 smart_thumbnail/include/mpipe_port.h create mode 100644 smart_thumbnail/mpipe_port.cpp create mode 100644 smart_thumbnail_cvr_lite/include/mpipe_port.h create mode 100644 smart_thumbnail_cvr_lite/mpipe_port.cpp diff --git a/rdk_build.sh b/rdk_build.sh index 0e22ce3..6b31c79 100755 --- a/rdk_build.sh +++ b/rdk_build.sh @@ -61,6 +61,11 @@ else . ${RDK_PROJECT_ROOT_PATH}/build/components/sdk/setenv2 fi +if [ "$XCAM_MODEL" == "XHB1" ]; then + echo "Enabling delivery detection by default for DBC" + export ENABLE_DELIVERY_DETECTION=true +fi + export PLATFORM_SDK=${RDK_TOOLCHAIN_PATH} export FSROOT=$RDK_FSROOT_PATH @@ -133,9 +138,28 @@ function install() make install + if [ $ENABLE_DELIVERY_DETECTION == 'true' ]; then + cp -r ./motion_notification/pre_built/mediapipe ${RDK_SDROOT}/etc + fi + echo "thumbnail Installation is done" } +# This function enables delivery detection +function setDeliveryDetection() +{ + echo "setDeliveryDetection - Enable delivery detection" + export ENABLE_DELIVERY_DETECTION=true +} + +# This function enables TESTHARNESS +function setTH() +{ + echo "setTH - Enable delivery detection and TestHarness" + export ENABLE_DELIVERY_DETECTION=true + export ENABLE_TEST_HARNESS=true +} + # This function disables XSTREAMER flag for Hydra function setHydra() { @@ -149,6 +173,8 @@ HIT=false for i in "$@"; do case $i in enableHydra) HIT=true; setHydra ;; + enableDeliveryDetection) HIT=true; setDeliveryDetection ;; + enableTH) HIT=true; setTH ;; configure) HIT=true; configure ;; clean) HIT=true; clean ;; build) HIT=true; build ;; diff --git a/smart_thumbnail/Makefile b/smart_thumbnail/Makefile index e43f257..6a1a420 100644 --- a/smart_thumbnail/Makefile +++ b/smart_thumbnail/Makefile @@ -46,6 +46,14 @@ ifeq ($(XCAM_MODEL), SCHC2) USE_DEWARP = yes endif +ifeq ($(ENABLE_DELIVERY_DETECTION), true) +USE_MISC = yes +endif + +ifeq ($(ENABLE_TEST_HARNESS), true) +CFLAGS += -DENABLE_TEST_HARNESS +endif + ENABLE_DIRECT_FRAME_READ := true ifneq ($(ENABLE_XSTREAMER), true) @@ -84,6 +92,14 @@ LIBS += -L$(RDK_PROJECT_ROOT_PATH)/opensource/lib -lopencv_core -lopencv_img LIBS += -Wl,-rpath-link=$(PROJ_ROOT)/vendor/img/fs/shadow_root/usr/lib CFLAGS += -DUSE_MFRLIB +ifeq ($(ENABLE_DELIVERY_DETECTION), true) +CFLAGS += -D_OBJ_DETECTION_ +SRC += mpipe_port.cpp +CFLAGS += -I$(BUILD_ROOT_DIR)/opensource/include +LIBS += -L$(BUILD_ROOT_DIR)/opensource/lib -lbase64 +LIBS += -ldeliverydetector +endif + SRC += smart_thumbnail.cpp main.cpp OBJ = $(SRC:.cpp=.o) diff --git a/smart_thumbnail/include/mpipe_port.h b/smart_thumbnail/include/mpipe_port.h new file mode 100644 index 0000000..6473c2c --- /dev/null +++ b/smart_thumbnail/include/mpipe_port.h @@ -0,0 +1,46 @@ +#ifndef MPIPE_PORT_H__ +#define MPIPE_PORT_H__ + +#include +#include + +typedef struct DetectionResult_ { + /* The bounding boxes of each (n) person detected in the thumbnail */ + std::vector> personBBoxes; + /* The detection score of each (n) person detected in the thumbnail */ + std::vector personScores; + float deliveryScore; + int maxAugScore; +} DetectionResult; + +void *__mpipe_thread_main__(); + +/* open the device in the given path. The 'width and height' is the resolution that + * mpipe desires. If the camera does not support, the camera's resolution is updated + * to widht and height */ +int mpipe_port_initialize(const std::string &input_video_path, int &width, int &height); +int mpipe_port_initialize(int &width, int &height); + +/* get next frame, in COLOR_RGB format */ +cv::Mat mpipe_port_getNextFrame(); +void mpipe_port_term(); + + +/* ---------------------------------------------------------------------- + * The following are mpipe APIs available for porting application to call. + * porting application should not implement these APIs. + * ----------------------------------------------------------------------*/ +/* header only. Do not implement */ +/* notify mpipe of last frame for person detection, in cv::Mat, RGB + * the lastFrame can be empty or non-empty. If non-empty, it will be + * process for person dection before continue onto delivery detection. + */ +void mpipe_port_onLastFrameEvent(const cv::Mat &thumbnail, int width, int height); +/* notify mpipe of motion deteced event */ +/* header only. Do not implement */ +void mpipe_port_onMotionEvent(bool motionDetected); +typedef void (* OnDetectionChanged_t) (const DetectionResult &detection_result); +/* mpipe will notify application of detection results */ +/* header only. Do not implement */ +void mpipe_port_setOnDetectionChanged(OnDetectionChanged_t cb); +#endif diff --git a/smart_thumbnail/include/smart_thumbnail.h b/smart_thumbnail/include/smart_thumbnail.h index 485e837..98841c2 100644 --- a/smart_thumbnail/include/smart_thumbnail.h +++ b/smart_thumbnail/include/smart_thumbnail.h @@ -30,6 +30,14 @@ #include #include #include +#ifdef _OBJ_DETECTION_ +#include +#include +#include "RFCCommon.h" +#endif +#ifdef ENABLE_TEST_HARNESS +#include "dev_config.h" +#endif #ifdef LEGACY_CFG_MGR #include "dev_config.h" @@ -65,6 +73,11 @@ extern "C" { #include "opencv2/opencv.hpp" +#ifdef _OBJ_DETECTION_ +#include "mpipe_port.h" +#include "base64.h" +#endif + #define CACHE_SMARTTHUMBNAIL "/tmp/cache_smart_thumbnail.txt" #define FW_NAME_MAX_LENGTH 512 @@ -77,6 +90,7 @@ extern "C" { #define STN_HRES_BUFFER_ID 2 #elif defined(XHB1) || defined (XHC3) #define STN_HRES_BUFFER_ID 2 +#define STN_MRES_BUFFER_ID 1 #else #define STN_HRES_BUFFER_ID 0 #endif @@ -90,6 +104,12 @@ extern "C" { //default width and height of smart thumbnail #define STN_DEFAULT_WIDTH 640 #define STN_DEFAULT_HEIGHT 480 +#ifdef XHB1 +#define STN_HRES_CROP_WIDTH 640 +#define STN_HRES_CROP_HEIGHT 480 +#define STN_MRES_CROP_WIDTH 400 +#define STN_MRES_CROP_HEIGHT 300 +#endif #define STN_TIMESTAMP_TAG "timestamp" #define STN_UPLOAD_TIME_INTERVAL 30 @@ -117,6 +137,31 @@ extern "C" { #define BLOB_BB_MAX_LEN 256 #define INVALID_BBOX_ORD (-1) +#ifdef ENABLE_TEST_HARNESS +#define TEST_HARNESS_ON_FILE_ENABLED "Test_Harness_on_File_Enabled" +#endif + +#ifdef _OBJ_DETECTION_ +#define DEFAULT_INPUT_DEV "/dev/video0" +#define DEFAULT_GRAPH_PATH "/etc/mediapipe/graphs/rdk/delivery_detection/g_delivery_detection_cpu.pbtxt" +#define DEFAULT_FRAME_READ_DELAY "1000" +#define DEFAULT_MAX_FRAMES_CACHED_FOR_DELIVERY_DETECTION "5" +#define DEFAULT_DELIVERY_DETECTION_MODEL_MIN_SCORE_THRESHOLD "0.3" +#define DEFAULT_DELIVERY_DETECTION_MIN_SCORE_THRESHOLD "1" +#define DETECTION_CONFIG_FILE "/opt/usr_config/detection_attr.conf" +#define DEFAULT_FRAME_COUNT_TO_PROCESS "5" + +typedef struct detection_config_ { + std::string input_video_path; + std::string delivery_detection_graph_path; + std::string frame_read_delay; + std::string max_num_frames_cached_for_delivery_detection; + std::string delivery_detection_model_min_score_threshold; + std::string delivery_detection_min_score_threshold; + std::string frame_count_to_process; +}DetectionConfig; +#endif + typedef enum { STH_ERROR = -1, STH_SUCCESS, @@ -146,6 +191,11 @@ typedef struct { #ifdef _HAS_DING_ uint64_t dingtstamp; #endif +#ifdef _OBJ_DETECTION_ + json_t *detectionResult; + uint64_t motionTime; + bool deliveryDetected; +#endif }STHPayload; typedef struct { @@ -170,11 +220,12 @@ class SmartThumbnail public: static SmartThumbnail* getInstance(); //Initialize the buffers and starts msg monitoring, upload thread. - STH_STATUS init(char* mac,bool isCVREnabled,int stnondelayType,int stnondelayTime); + STH_STATUS init(char* mac,bool isCVREnabled,int stnondelayType,int stnondelayTime,bool isDetectionEnabled); //get upload status bool getUploadStatus(); //set upload status STH_STATUS setUploadStatus(bool status); + //Upload smart thumbnail data void uploadPayload(); //notify start or end of smart thumbnail process @@ -183,6 +234,17 @@ class SmartThumbnail STH_STATUS destroy(); //Thread routine to receive data STH_STATUS receiveRtmessage(); +#ifdef _OBJ_DETECTION_ +#ifdef ENABLE_TEST_HARNESS + void notifyXvision(const DetectionResult &result); + void waitForNextDetectionFrame(); +#endif + STH_STATUS setDetectionStatus(bool status); + bool getDetectionStatus(); + void onCompletedDeliveryDetection(const DetectionResult &result); + friend cv::Mat mpipe_port_getNextFrame(); + friend unsigned char * mpipe_port_getNextFrame(int *h, int *w); +#endif static void sigHandler(int signum); @@ -195,6 +257,10 @@ class SmartThumbnail STH_STATUS delSTN(char* uploadFname); void printSTNList(); STH_STATUS createPayload(char* uploadFname); +#ifdef _OBJ_DETECTION_ + STH_STATUS updateUploadPayload(char * fname, DetectionResult result); + json_t* createJSONFromDetectionResult(DetectionResult result); +#endif STH_STATUS getTnUploadConf(); STH_STATUS getEventConf(); //sets the camera firmware version. @@ -211,7 +277,7 @@ class SmartThumbnail STH_STATUS resizeAspect(cv::Mat im, int w, int h, cv::Mat& im_resized); cv::Point2f getActualCentroid(cv::Rect boundRect); cv::Point2f alignCentroid(cv::Point2f orgCenter, cv::Mat origFrame, cv::Size cropSize); - cv::Size getCropSize(cv::Rect boundRect,double w,double h); + cv::Size getCropSize(cv::Rect boundRect,double w,double h, double *rescaleSize); cv::Rect getRelativeBoundingBox(cv::Rect boundRect, cv::Size cropSize, cv::Point2f allignedCenter); void stringifyEventDateTime(char* strEvtDateTime , size_t evtdatetimeSize, time_t evtDateTime); void resetObjFrameData(); @@ -266,6 +332,30 @@ class SmartThumbnail bool isPayloadAvailable; std::mutex stnMutex; std::mutex uploadMutex; +#ifdef _OBJ_DETECTION_ + std::mutex detectionMutex; + cv::Mat nextFrameForDetection; + unsigned char* mpipe_hres_yuvData; + cv::Rect currentBbox; + bool detectionCompleted; + json_t *detectionResult; + struct timeval detectionStartTime, detectionEndTime, uploadTriggeredTime, clipStartTime; + int mpipeProcessedframes; + double motionTriggeredTime; + char currDetectionSTNFname[CONFIG_STRING_MAX]; + bool detectionInProgress; + bool detectionEnabled; +#ifdef ENABLE_TEST_HARNESS + bool testHarnessOnFileFeed; + std::vector yuvPlanes, yuvChannels; + cv::Mat fileFrameYUV, planeUV, curr_frame; + uint64_t currTstamp, detectionTstamp; + std::condition_variable detectionCv; + std::mutex hres_data_lock; + int THFileNum, THFrameNum; + int FileNum = 0, FrameNum = 0; +#endif +#endif std::condition_variable cv; rtConnection connectionRecv; rtConnection connectionSend; @@ -295,6 +385,7 @@ class SmartThumbnail int sTnHeight; int sTnWidth; + uint16_t buf_id; char uploadFname[CONFIG_STRING_MAX]; char currSTNFname[CONFIG_STRING_MAX]; char modelName[CONFIG_STRING_MAX]; @@ -319,6 +410,9 @@ struct SmarttnMetadata_thumb char const *strFramePTS; int32_t event_type; double motionScore; +#ifdef _OBJ_DETECTION_ + BoundingBox deliveryUnionBox; +#endif BoundingBox unionBox; BoundingBox objectBoxs [UPPER_LIMIT_BLOB_BB]; char const *s_curr_time; diff --git a/smart_thumbnail/main.cpp b/smart_thumbnail/main.cpp index 150b228..43106d5 100644 --- a/smart_thumbnail/main.cpp +++ b/smart_thumbnail/main.cpp @@ -25,9 +25,15 @@ #include "breakpadwrap.h" #endif +#ifdef _OBJ_DETECTION_ +SmartThumbnail *smTnInstance = NULL; +#endif + int main(int argc, char** argv) { +#ifndef _OBJ_DETECTION_ SmartThumbnail *smTnInstance = NULL; +#endif STH_STATUS status = STH_SUCCESS; time_t remainingTime = 0; char *param = NULL; @@ -35,6 +41,7 @@ int main(int argc, char** argv) int cvrEnabled = 0; int stnondelayType = 0; int stnondelayTime = 60; + int isDetectionEnabled = 0; struct timespec currTime; struct timespec startTime; @@ -121,6 +128,20 @@ int main(int argc, char** argv) break; } } + + if(strcmp(argv[itr],"--detectionEnabled")==0) + { + itr++; + + if (itr < argc) + { + isDetectionEnabled = atoi(argv[itr]); + } + else + { + break; + } + } itr++; } @@ -135,7 +156,7 @@ int main(int argc, char** argv) } //initialize smart thumbnail - status = smTnInstance-> init(param,cvrEnabled,stnondelayType,stnondelayTime); + status = smTnInstance-> init(param,cvrEnabled,stnondelayType,stnondelayTime,isDetectionEnabled); if (STH_ERROR == status) { RDK_LOG( RDK_LOG_ERROR,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): Error creating Smart thumbnail instance.\n", __FILE__, __LINE__); return STH_ERROR; diff --git a/smart_thumbnail/mpipe_port.cpp b/smart_thumbnail/mpipe_port.cpp new file mode 100644 index 0000000..bb13f01 --- /dev/null +++ b/smart_thumbnail/mpipe_port.cpp @@ -0,0 +1,216 @@ +#if 0 +#include + +#include +#include +#include +#include +#include +#endif +//#include "mpipe_port.h" +#include "smart_thumbnail.h" +#define CROPPING_WIDTH 848 +#define CROPPING_HEIGHT 480 + +extern SmartThumbnail *smTnInstance; + +int mpipe_port_initialize(const std::string &input_video_path, int &width, int &height) { + std::cout << "rdkc PORT Initialize the camera" << std::endl; + return 0; +} + +int mpipe_port_initialize(int &width, int &height) { + mpipe_port_initialize(std::string(""), width, height); + return 0; +} +#if 1 +cv::Mat mpipe_port_getNextFrame() { +#if 1 +// cv::Mat next_frame; +// cv::Mat next_frame_rgb; + +// if(enableDetectionOnHighResolution) { //High resolution frame + cv::Mat cv_frame, croppedObj; + smTnInstance->mpipeProcessedframes++; + +#ifdef ENABLE_TEST_HARNESS + smTnInstance -> waitForNextDetectionFrame(); + cv_frame = cv::Mat(smTnInstance ->curr_frame.rows, smTnInstance ->curr_frame.cols, CV_8UC3); + cv::cvtColor(smTnInstance ->curr_frame, cv_frame, cv::COLOR_BGR2RGB); + smTnInstance -> detectionTstamp = smTnInstance -> currTstamp; +#else + + smTnInstance -> hres_y_height = smTnInstance -> frameInfo->height; + smTnInstance -> hres_y_width = smTnInstance -> frameInfo->width; + smTnInstance -> hres_y_size = smTnInstance -> frameInfo->width * smTnInstance -> frameInfo->height; + smTnInstance -> hres_uv_size = smTnInstance -> frameInfo->width * smTnInstance -> frameInfo->height/2; + int alloc_size = smTnInstance -> hres_y_size + smTnInstance -> hres_uv_size; + + if(smTnInstance -> mpipe_hres_yuvData == NULL){ + smTnInstance -> mpipe_hres_yuvData = (unsigned char *) malloc(alloc_size * sizeof(unsigned char)); + }else if(sizeof(smTnInstance -> mpipe_hres_yuvData) != (alloc_size* sizeof(unsigned char))) { + smTnInstance -> mpipe_hres_yuvData = (unsigned char *) realloc(smTnInstance -> mpipe_hres_yuvData, alloc_size * sizeof(unsigned char)); + } + if( NULL == smTnInstance -> mpipe_hres_yuvData){ + RDK_LOG(RDK_LOG_INFO,"LOG.RDK.SMARTTHUMBNAIL","%s(%d) malloc failed for hres_yuvData.\n", __FUNCTION__ , __LINE__); + return cv_frame; + } + +// if(!smTnInstance -> nextFrameForDetection.empty()) smTnInstance -> nextFrameForDetection.release(); + memset( smTnInstance -> mpipe_hres_yuvData, 0, (smTnInstance -> hres_y_size + smTnInstance -> hres_uv_size) * sizeof(unsigned char) ); + memcpy( smTnInstance -> mpipe_hres_yuvData, smTnInstance -> frameInfo->y_addr, smTnInstance -> hres_y_size); + memcpy( smTnInstance -> mpipe_hres_yuvData + smTnInstance -> hres_y_size, smTnInstance -> frameInfo->uv_addr, smTnInstance -> hres_uv_size); + + cv::Mat next_frame = cv::Mat(smTnInstance -> frameInfo -> height + (smTnInstance -> frameInfo -> height)/2, smTnInstance -> frameInfo -> width, CV_8UC1, smTnInstance -> mpipe_hres_yuvData); + + cv_frame = cv::Mat(next_frame.rows, next_frame.cols, CV_8UC3); + cv::cvtColor(next_frame, cv_frame, cv::COLOR_YUV2RGB_NV12/*cv::COLOR_BGR2RGB*/); + next_frame.release(); +#endif + + double scaleFactor = 1; + cv::Size cropSize = smTnInstance -> getCropSize(smTnInstance -> currentBbox, smTnInstance -> sTnWidth, smTnInstance -> sTnHeight, &scaleFactor); + //Resize the union blob with scaleFactor + smTnInstance -> currentBbox.width = smTnInstance -> currentBbox.width/scaleFactor; + smTnInstance -> currentBbox.height = smTnInstance -> currentBbox.height/scaleFactor; + smTnInstance -> currentBbox.x = smTnInstance -> currentBbox.x/scaleFactor; + smTnInstance -> currentBbox.y = smTnInstance -> currentBbox.y/scaleFactor; + cv::Size rescaleSize = cv::Size(cv_frame.cols/scaleFactor, cv_frame.rows/scaleFactor); + //resize the frame to fit the union blob in the thumbnail + cv::resize(cv_frame, cv_frame, rescaleSize); + cv::Point2f orgCenter = smTnInstance -> getActualCentroid(smTnInstance -> currentBbox); + cv::Point2f allignedCenter = smTnInstance -> alignCentroid(orgCenter, cv_frame, cropSize); + getRectSubPix(cv_frame, cropSize, allignedCenter, croppedObj); + + RDK_LOG(RDK_LOG_INFO,"LOG.RDK.SMARTTHUMBNAIL","%s(%d)The motion blob in the frame is : x = %d, y = %d, w = %d, h = %d\n", __FILE__, __LINE__, smTnInstance -> currentBbox.x, smTnInstance -> currentBbox.y, smTnInstance -> currentBbox.width, smTnInstance -> currentBbox.height); + RDK_LOG(RDK_LOG_INFO,"LOG.RDK.SMARTTHUMBNAIL","%s(%d)Cropped Frame center [x = %f, y = %f].\n", __FILE__, __LINE__, allignedCenter.x, allignedCenter.y); + RDK_LOG(RDK_LOG_INFO,"LOG.RDK.SMARTTHUMBNAIL","%s(%d)Cropped Frame coordinates [x = %f, y = %f].\n", __FILE__, __LINE__, allignedCenter.x - (cropSize.width / 2), allignedCenter.y - (cropSize.height / 2)); + +/* uint8_t *camera_buf = (uint8_t *) malloc(next_frame.rows * next_frame.cols * 3); + memcpy(camera_buf, cv_frame.data, next_frame.rows * next_frame.cols* 3); + cv::Mat cv_frame_dst = cv::Mat(next_frame.rows, next_frame.cols, CV_8UC3, camera_buf); +*/ + //uint8_t *camera_buf = (uint8_t *) malloc(848 * 480 * 3); + uint8_t *camera_buf = new uint8_t [cropSize.height * cropSize.width * 3]; + memcpy(camera_buf, croppedObj.data, cropSize.height*cropSize.width * 3); + cv::Mat cv_frame_dst = cv::Mat(croppedObj.rows, croppedObj.cols, CV_8UC3, camera_buf); +// cv::Mat oFrame; +// cv::cvtColor(cv_frame_dst, oFrame, cv::COLOR_RGB2BGR); +// imwrite("/tmp/frame.jpg", cv_frame_dst); +#ifdef ENABLE_TEST_HARNESS + RDK_LOG(RDK_LOG_INFO,"LOG.RDK.SMARTTHUMBNAIL","%s(%d) Sending %dth frame for detection in %dth file in THlist.\n", __FUNCTION__ , __LINE__, smTnInstance -> FrameNum, smTnInstance -> FileNum); +#endif + + cv_frame.release(); + croppedObj.release(); + +#if 0 + if(hres_yuvData) { + free(hres_yuvData); + hres_yuvData = NULL; + } +#endif +#if 0 + } else{ //Low resolution frame + //TODO: Add ReadYUVFrame code here to read frame from camera low resolution buffer + nextFrame_rgb = cv::Mat(frameInfo -> height, frameInfo -> width, CV_8UC4); + nextFrame = cv::Mat(frameInfo -> height + (frameInfo -> height)/2, frameInfo -> width, CV_8UC1, frameInfo->frame_ptr).clone(); + cv::cvtColor(next_frame, smTnInstance -> nextFrameForDetection, cv::COLOR_BGR2RGB); +#endif + + return (cv_frame_dst); +#else + //return smTnInstance -> nextFrameForDetection; + cv::Mat cv_frame = cv::imread("/opt/sajna/person2.jpg", cv::IMREAD_COLOR); + cv::Mat cv_frame_rgb; + cv::cvtColor(cv_frame, cv_frame_rgb, cv::COLOR_BGR2RGB); + + return cv_frame_rgb; +#endif +} +#else +unsigned char * mpipe_port_getNextFrame(int *h, int *w) { + unsigned char * frame_buf = NULL; + smTnInstance -> hres_y_height = smTnInstance -> frameInfo->height; + smTnInstance -> hres_y_width = smTnInstance -> frameInfo->width; + smTnInstance -> hres_y_size = smTnInstance -> frameInfo->width * smTnInstance -> frameInfo->height; + smTnInstance -> hres_uv_size = smTnInstance -> frameInfo->width * smTnInstance -> frameInfo->height/2; + int alloc_size = smTnInstance -> hres_y_size + smTnInstance -> hres_uv_size; + + frame_buf = (unsigned char *) malloc(alloc_size * sizeof(unsigned char)); + if( NULL == frame_buf){ + RDK_LOG(RDK_LOG_INFO,"LOG.RDK.SMARTTHUMBNAIL","%s(%d) malloc failed for hres_yuvData.\n", __FUNCTION__ , __LINE__); + return NULL; + } + + memset( frame_buf, 0, (smTnInstance -> hres_y_size + smTnInstance -> hres_uv_size) * sizeof(unsigned char) ); + memcpy( frame_buf, smTnInstance -> frameInfo->y_addr, smTnInstance -> hres_y_size); + memcpy( frame_buf + smTnInstance -> hres_y_size, smTnInstance -> frameInfo->uv_addr, smTnInstance -> hres_uv_size); + + *w = smTnInstance -> frameInfo->width; + *h = smTnInstance -> frameInfo->height; + return frame_buf; +} +#endif + +void loadDetectionConfig(DetectionConfig *config, char *configFile) +{ + FileUtils m_settings; + + config->input_video_path = DEFAULT_INPUT_DEV; + config->delivery_detection_graph_path = DEFAULT_GRAPH_PATH; + config->frame_read_delay = DEFAULT_FRAME_READ_DELAY; + config->max_num_frames_cached_for_delivery_detection = DEFAULT_MAX_FRAMES_CACHED_FOR_DELIVERY_DETECTION; + config->delivery_detection_model_min_score_threshold = DEFAULT_DELIVERY_DETECTION_MODEL_MIN_SCORE_THRESHOLD; + config->delivery_detection_min_score_threshold = DEFAULT_DELIVERY_DETECTION_MIN_SCORE_THRESHOLD; + config->frame_count_to_process = DEFAULT_FRAME_COUNT_TO_PROCESS; + + if(!m_settings.loadFromFile(std::string(configFile))) { + RDK_LOG(RDK_LOG_WARN,"LOG.RDK.THUMBNAILUPLOAD","%s(%d): Loading detection config file failed, loading default settings\n", __FILE__, __LINE__); + } else + { + m_settings.get("input_video_path", config->input_video_path); + m_settings.get("delivery_detection_graph_path", config->delivery_detection_graph_path); + m_settings.get("frame_read_delay", config->frame_read_delay); + m_settings.get("max_num_frames_cached_for_delivery_detection", config->max_num_frames_cached_for_delivery_detection); + m_settings.get("delivery_detection_model_min_score_threshold", config->delivery_detection_model_min_score_threshold); + m_settings.get("delivery_detection_min_score_threshold", config->delivery_detection_min_score_threshold); + m_settings.get("frame_count_to_process", config->frame_count_to_process); + } +} + +void mpipe_port_term() { +} + +void *__mpipe_thread_main__() { + std::string argv0 = __FUNCTION__; + + DetectionConfig config; + + loadDetectionConfig(&config, DETECTION_CONFIG_FILE); + std::string argv1 = "--input_video_path=\"" + config.input_video_path + "\""; + std::string argv2 = "--delivery_detection_graph_path=" + config.delivery_detection_graph_path; +// std::string argv2 = "--graph_path=/etc/mediapipe/graphs/rdk/delivery_detection/g_person_detection_rdkc_live_cpu.pbtxt"; + std::string argv3 = "--frame_read_delay=" + config.frame_read_delay; + std::string argv4 = "--max_num_frames_cached_for_delivery_detection=" + config.max_num_frames_cached_for_delivery_detection; + std::string argv5 = "--delivery_detection_model_min_score_threshold=" + config.delivery_detection_model_min_score_threshold; + std::string argv6 = "--delivery_detection_min_score_threshold=" + config.delivery_detection_min_score_threshold; + std::string argv7 = "--frame_count_to_process=" + config.frame_count_to_process; + + + std::vector vargv0(argv0.begin(), argv0.end()); + std::vector vargv1(argv1.begin(), argv1.end()); + std::vector vargv2(argv2.begin(), argv2.end()); + std::vector vargv3(argv3.begin(), argv3.end()); + std::vector vargv4(argv4.begin(), argv4.end()); + std::vector vargv5(argv5.begin(), argv5.end()); + std::vector vargv6(argv6.begin(), argv6.end()); + std::vector vargv7(argv7.begin(), argv7.end()); + + char *argv[] = { vargv0.data(), vargv1.data(), vargv2.data(), vargv3.data(), vargv4.data(), vargv5.data(), vargv6.data(), vargv7.data()}; + + extern int __mpipe_main__(int, char **); + __mpipe_main__(sizeof(argv)/sizeof(argv[0]), argv); + return NULL; +} + diff --git a/smart_thumbnail/smart_thumbnail.cpp b/smart_thumbnail/smart_thumbnail.cpp index eb3929b..e394c0d 100644 --- a/smart_thumbnail/smart_thumbnail.cpp +++ b/smart_thumbnail/smart_thumbnail.cpp @@ -26,6 +26,115 @@ volatile bool SmartThumbnail::termFlag = false; volatile bool SmartThumbnail::tnUploadConfRefreshed = false; volatile bool SmartThumbnail::eventConfRefreshed = false; int SmartThumbnail::waitingInterval = 1 ; +#ifdef _OBJ_DETECTION_ +#ifdef ENABLE_TEST_HARNESS +void SmartThumbnail::waitForNextDetectionFrame() +{ + std::unique_lock lock(hres_data_lock); + detectionCv.wait(lock, [this] {return ((currTstamp >= (detectionTstamp + 100)) || (detectionTstamp == 0) || termFlag);}); + lock.unlock(); +} + +void SmartThumbnail::notifyXvision(const DetectionResult &result) +{ + rtMessage m; + rtMessage_Create(&m); + rtMessage_SetInt32(m, "FileNum", THFileNum); + rtMessage_SetInt32(m, "FrameNum", THFrameNum); + rtMessage_SetDouble(m, "deliveryConfidence", result.deliveryScore); + + for(int i = 0; i < result.personBBoxes.size(); i++) { + rtMessage personInfo; + rtMessage_Create(&personInfo); + rtMessage_SetInt32(personInfo, "boundingBoxXOrd", result.personBBoxes[i][0]); + rtMessage_SetInt32(personInfo, "boundingBoxYOrd", result.personBBoxes[i][1]); + rtMessage_SetInt32(personInfo, "boundingBoxWidth", result.personBBoxes[i][2]); + rtMessage_SetInt32(personInfo, "boundingBoxHeight", result.personBBoxes[i][3]); + rtMessage_SetDouble(personInfo, "confidence", result.personScores[i]*100); + rtMessage_AddMessage(m, "Persons", personInfo); + rtMessage_Release(personInfo); + } + + rtError err = rtConnection_SendMessage(connectionSend, m, "RDKC.TESTHARNESS.DELIVERYDATA"); + rtLog_Debug("SendRequest:%s", rtStrError(err)); + + if (err != RT_OK) + { + RDK_LOG( RDK_LOG_ERROR,"LOG.RDK.VIDEOANALYTICS","%s(%d) Error sending msg via rtmessage\n", __FILE__,__LINE__); + } + rtMessage_Release(m); + +} +#endif +extern SmartThumbnail *smTnInstance; + +/** @description: Checks if the feature is enabled via RFC + * @param[in] rfc_feature_fname: RFC feature filename + * @param[in] plane1: RFC parameter name + * @return: bool + */ +static bool check_enabled_rfc_feature(char* rfc_feature_fname, char* feature_name) +{ + /* set cvr audio through RFC files */ + char value[MAX_SIZE] = {0}; + + if((NULL == rfc_feature_fname) || + (NULL == feature_name)) { + return false; + } + + /* Check if RFC configuration file exists */ + if( RDKC_SUCCESS == IsRFCFileAvailable(rfc_feature_fname)) { + /* Get the value from RFC file */ + if( RDKC_SUCCESS == GetValueFromRFCFile(rfc_feature_fname, feature_name, value) ) { + if( strcmp(value, RDKC_TRUE) == 0) { + RDK_LOG( RDK_LOG_INFO,"LOG.RDK.XCV","%s(%d): %s is enabled via RFC.\n",__FILE__, __LINE__, feature_name); + return true; + } else { + RDK_LOG( RDK_LOG_INFO,"LOG.RDK.XCV","%s(%d): %s is disabled via RFC.\n",__FILE__, __LINE__, feature_name); + return false; + } + } + /* If RFC file is not present, disable the featur */ + } else { + RDK_LOG( RDK_LOG_INFO,"LOG.RDK.XCV","%s(%d): rfc feature file %s is not present.\n",__FILE__, __LINE__, rfc_feature_fname); + return false; + } +} + +void callback_func(const DetectionResult &result) +{ + RDK_LOG( RDK_LOG_DEBUG,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): Got data result callback\n", __FUNCTION__, __LINE__); + RDK_LOG( RDK_LOG_DEBUG,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): No of persons detected : %d\n", __FUNCTION__, __LINE__, result.personScores.size()); + mpipe_port_onMotionEvent(false); + smTnInstance->onCompletedDeliveryDetection(result); +} + +void SmartThumbnail::onCompletedDeliveryDetection(const DetectionResult &result) +{ + + gettimeofday(&(smartThInst->detectionEndTime), NULL); + double time_taken = 0, time_waited = 0; + + time_taken = (smartThInst->detectionEndTime.tv_sec - smartThInst->detectionStartTime.tv_sec) * 1e6; + time_taken = (time_taken + (smartThInst->detectionEndTime.tv_usec - smartThInst->detectionStartTime.tv_usec)) * 1e-6; + if((smartThInst->uploadTriggeredTime.tv_sec != 0) || (smartThInst->uploadTriggeredTime.tv_usec != 0)) { + time_waited = (smartThInst->detectionEndTime.tv_sec - smartThInst->uploadTriggeredTime.tv_sec) * 1e6; + time_waited = (time_waited + (smartThInst->detectionEndTime.tv_usec - smartThInst->uploadTriggeredTime.tv_usec)) * 1e-6; + } + memset(&(smartThInst -> uploadTriggeredTime), 0, sizeof(smartThInst -> uploadTriggeredTime)); + RDK_LOG( RDK_LOG_INFO,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): Detection Stats:%0.2f,%d,%d,%0.2lf,%d,%0.2lf,%0.2lf\n", __FUNCTION__, + __LINE__, result.deliveryScore, result.maxAugScore, result.personScores.size(), + motionTriggeredTime, mpipeProcessedframes, time_taken, time_waited); +#ifdef ENABLE_TEST_HARNESS + smartThInst->notifyXvision(result); +#endif + smartThInst->updateUploadPayload(smartThInst->currDetectionSTNFname, result); + memset(smartThInst->currDetectionSTNFname, 0, sizeof(smartThInst->currDetectionSTNFname)); + smartThInst->setDetectionStatus(true); + smTnInstance->detectionInProgress = false; +} +#endif /** @description: Constructor * @param[in] void @@ -62,7 +171,15 @@ SmartThumbnail::SmartThumbnail(): sTnHeight(STN_DEFAULT_HEIGHT), sTnWidth(STN_DEFAULT_WIDTH), stnUploadTime(0), +#ifndef _OBJ_DETECTION_ eventquietTimeStart(0) +#else + eventquietTimeStart(0), + motion_time(0), + detectionCompleted(false), + mpipe_hres_yuvData(NULL), + detectionInProgress(false) +#endif { memset(uploadFname, 0, sizeof(uploadFname)); memset(smtTnUploadURL, 0, sizeof(smtTnUploadURL)); @@ -72,6 +189,16 @@ SmartThumbnail::SmartThumbnail(): memset(macAddress, 0, sizeof(macAddress)); memset(firmwareName, 0, sizeof(firmwareName)); memset(motionLog, 0, sizeof(motionLog)); + memset(&currSTN, 0, sizeof(currSTN)); +#ifdef XHB1 + /* Adding the below two lines to hardcode the thumbnail size to 400x300 */ + sTnWidth = 400; + sTnHeight = 300; +#endif + +#ifdef _OBJ_DETECTION_ + memset(&uploadTriggeredTime, 0, sizeof(uploadTriggeredTime)); +#endif #ifdef _HAS_XSTREAM_ #ifndef _DIRECT_FRAME_READ_ frameHandler = {NULL, -1}; @@ -119,17 +246,21 @@ SmartThumbnail *SmartThumbnail::getInstance() * @param[in] void * @return: STH_SUCCESS on success, STH_ERROR otherwise */ -STH_STATUS SmartThumbnail::init(char* mac,bool isCvrEnabled,int stnondelayType,int stnondelayTime) +STH_STATUS SmartThumbnail::init(char* mac,bool isCvrEnabled,int stnondelayType,int stnondelayTime,bool isDetectionEnabled) { -#ifdef LEGACY_CFG_MGR - char usrVal[CONFIG_STRING_MAX]; - char *configParam = NULL; - +#if defined(LEGACY_CFG_MGR) || defined(ENABLE_TEST_HARNESS) //initializing config Manager if (STH_SUCCESS != config_init()) { RDK_LOG(RDK_LOG_ERROR,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): Error loading config manager.\n", __FILE__, __LINE__); return STH_ERROR; } +#endif +#ifdef ENABLE_TEST_HARNESS + testHarnessOnFileFeed = (strcmp(rdkc_envGet(TEST_HARNESS_ON_FILE_ENABLED), "true") == 0) ? true : false; +#endif +#ifdef LEGACY_CFG_MGR + char usrVal[CONFIG_STRING_MAX]; + char *configParam = NULL; // get upload url memset(usrVal, 0, sizeof(usrVal)); @@ -190,10 +321,20 @@ STH_STATUS SmartThumbnail::init(char* mac,bool isCvrEnabled,int stnondelayType,i } //initialize +#ifdef XHB1 + buf_id = STN_MRES_BUFFER_ID; + /* Commenting the below check for hardcoding the thumbnail resolution to 400x300 */ +/* if((sTnWidth == STN_HRES_CROP_WIDTH) && (sTnHeight == STN_HRES_CROP_HEIGHT)) { + buf_id = STN_HRES_BUFFER_ID; + }*/ +#else + buf_id = STN_HRES_BUFFER_ID; +#endif + #ifdef _DIRECT_FRAME_READ_ - if(STH_SUCCESS != smartThInst->consumer->RAWInit((u16)STN_HRES_BUFFER_ID)){ + if(STH_SUCCESS != smartThInst->consumer->RAWInit((u16)buf_id)){ #else - frameHandler = smartThInst->consumer->RAWInit(STN_HRES_BUFFER_ID, FORMAT_YUV, 1); + frameHandler = smartThInst->consumer->RAWInit(buf_id, FORMAT_YUV, 1); if(frameHandler.sockfd < 0) { #endif RDK_LOG( RDK_LOG_ERROR,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): Failed to initialize resources for reading raw frames.\n", __FUNCTION__, __LINE__); @@ -243,6 +384,25 @@ STH_STATUS SmartThumbnail::init(char* mac,bool isCvrEnabled,int stnondelayType,i stnOnDelayTime = stnondelayTime; rtMsgInit(); +#ifdef _OBJ_DETECTION_ + if( RDKC_SUCCESS != RFCConfigInit() ) + { + RDK_LOG( RDK_LOG_ERROR,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): RFC config Init fails\n", __FILE__, __LINE__); + return STH_ERROR; + } + //detectionEnabled = check_enabled_rfc_feature(RFCFILE, DELIVERY_DETECTION_RFC); + detectionEnabled = isDetectionEnabled; + + if(detectionEnabled) { + //Initialize delivery detection thread + RDK_LOG( RDK_LOG_INFO,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): Creating delivery detection thread.\n", __FUNCTION__, __LINE__); + std::thread deliveryDetectionThread(__mpipe_thread_main__); + deliveryDetectionThread.detach(); + + mpipe_port_setOnDetectionChanged(callback_func); + } +#endif + //Initialize upload routine RDK_LOG( RDK_LOG_INFO,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): Creating smart thumbnail upload thread.\n", __FUNCTION__, __LINE__); std::thread uploadPayloadThread(uploadSTN); @@ -292,6 +452,12 @@ STH_STATUS SmartThumbnail::getTnUploadConf() sTnHeight = atoi(stnCfg -> height); sTnWidth = atoi(stnCfg -> width); +#ifdef XHB1 + /* Adding the below two lines to hardcode the thumbnail size to 400x300 */ + sTnWidth = 400; + sTnHeight = 300; +#endif + if (stnCfg) { free(stnCfg); stnCfg = NULL; @@ -383,8 +549,12 @@ STH_STATUS SmartThumbnail::saveSTN() // matrix to store color image lHresRGBMat = cv::Mat(hres_y_height, hres_y_width, CV_8UC4); +#ifdef ENABLE_TEST_HARNESS + lHresRGBMat = ofData.maxBboxObjYUVFrame.clone(); +#else // convert the frame to BGR format cv::cvtColor(ofData.maxBboxObjYUVFrame,lHresRGBMat, cv::COLOR_YUV2BGR_NV12); +#endif unionBox.x = ofData.boundingBoxXOrd; unionBox.y = ofData.boundingBoxYOrd; @@ -401,13 +571,26 @@ STH_STATUS SmartThumbnail::saveSTN() // extracted the below logic from server scala code RDK_LOG( RDK_LOG_INFO,"LOG.RDK.SMARTTHUMBNAIL","%s(%d):unionBox.x %d unionBox.y %d unionBox.height %d unionBox.width %d hres_y_width=%d ,hres_y_height=%d\n", __FILE__, __LINE__, unionBox.x, unionBox.y, unionBox.height, unionBox.width,hres_y_width,hres_y_height); + + /* ScaleFactor for downsizing the frame before cropping the thumbnail. If the + * union blob is bigger than the thumbnail size, downsize the frame to fit the + * union blob in thumbnail */ + double scaleFactor = 1; + cv::Size cropSize = getCropSize(unionBox, sTnWidth, sTnHeight, &scaleFactor); + cv::Size rescaleSize = cv::Size(lHresRGBMat.cols/scaleFactor, lHresRGBMat.rows/scaleFactor); + //resize the frame to fit the union blob in the thumbnail + cv::resize(lHresRGBMat, lHresRGBMat, rescaleSize); + //Resize the union blob also accordingly + unionBox.width = unionBox.width/scaleFactor; + unionBox.height = unionBox.height/scaleFactor; + unionBox.x = unionBox.x/scaleFactor; + unionBox.y = unionBox.y/scaleFactor; cv::Point2f orgCenter = getActualCentroid(unionBox); - cv::Size cropSize = getCropSize(unionBox, sTnWidth, sTnHeight); cv::Point2f allignedCenter = alignCentroid(orgCenter, lHresRGBMat, cropSize); if(debugBlob) { cropSize = {lHresRGBMat.cols, lHresRGBMat.rows}; getRectSubPix(lHresRGBMat, cropSize, allignedCenter, croppedObj); - cropSize = getCropSize(unionBox, sTnWidth, sTnHeight); + cropSize = getCropSize(unionBox, sTnWidth, sTnHeight, &scaleFactor); } else { getRectSubPix(lHresRGBMat, cropSize, allignedCenter, croppedObj); } @@ -445,14 +628,25 @@ STH_STATUS SmartThumbnail::saveSTN() #endif //RDK_LOG( RDK_LOG_INFO,"LOG.RDK.SMARTTHUMBNAIL","%s(%d):Resized the cropped object frame.\n", __FILE__, __LINE__); //cv::resize(croppedObj, resizedRGBMat,cv::Size(sTnWidth, sTnHeight)); - strcpy(currSTN.fname, currSTNFname); - snprintf(fPath, sizeof(fPath), "%s/%s", STN_PATH, currSTNFname); + snprintf(fPath, sizeof(fPath), "%s/%s", STN_PATH, currSTN.fname); RDK_LOG( RDK_LOG_TRACE1,"LOG.RDK.SMARTTHUMBNAIL","%s(%d):smart thumbnail full path:%s .\n", __FILE__, __LINE__, fPath); #ifdef USE_FILE_UPLOAD //Write smart thumbnail to file. //imwrite(fPath, resizedRGBMat); imwrite(fPath, croppedObj); + +#if 0 //def _OBJ_DETECTION_ +// if(detectionEnabled) { + cv::Mat cv_frame_rgb; + cv::cvtColor(croppedObj, cv_frame_rgb, cv::COLOR_BGR2RGB); + uint8_t *camera_buf = (uint8_t *) malloc(croppedObj.cols * croppedObj.rows * 3); + memcpy(camera_buf, cv_frame_rgb.data, croppedObj.cols * croppedObj.rows * 3); + cv::Mat cv_frame_dst = cv::Mat(croppedObj.rows, croppedObj.cols, CV_8UC3, camera_buf); + mpipe_port_onThunbmailEvent(cv_frame_dst, croppedObj.cols, croppedObj.rows); + cv_frame_rgb.release(); + // } +#endif #endif ret = STH_SUCCESS; } @@ -600,6 +794,9 @@ STH_STATUS SmartThumbnail::delSTN(char* uploadFname) else { RDK_LOG(RDK_LOG_INFO,"LOG.RDK.SMARTTHUMBNAIL","%s(%d) Caching %s file for reference\n", __FUNCTION__ , __LINE__, uploadFname); } +#ifdef _OBJ_DETECTION_ + // json_decref((*it).detectionResult); +#endif STNList.erase(it); break; @@ -623,6 +820,11 @@ STH_STATUS SmartThumbnail::createPayload(char* uploadFname) if(!strcmp((*it).fname, uploadFname)){ memset(&payload, 0 , sizeof(STHPayload)); memcpy(&payload, &(*it), sizeof(STHPayload)); +#ifdef _OBJ_DETECTION_ + if((*it).detectionResult != NULL) { + payload.detectionResult = json_copy((*it).detectionResult); + } +#endif ret = STH_SUCCESS; RDK_LOG(RDK_LOG_TRACE1,"LOG.RDK.SMARTTHUMBNAIL","%s(%d) payload.fname:%s payload.tstamp :%llu \n", __FUNCTION__ , __LINE__, payload.fname,payload.tstamp); break; @@ -634,6 +836,124 @@ STH_STATUS SmartThumbnail::createPayload(char* uploadFname) return ret; } +#ifdef _OBJ_DETECTION_ +json_t* SmartThumbnail::createJSONFromDetectionResult(DetectionResult result) +{ + json_t* resultJson = json_object(); + json_t* tags_array = json_array(); + std::ostringstream statStringStream; + + if(result.deliveryScore != 0) { + RDK_LOG(RDK_LOG_DEBUG,"LOG.RDK.SMARTTHUMBNAIL","%s(%d) Delivery detected with confidence: %f\n", __FUNCTION__ , __LINE__, result.deliveryScore); + json_t* object = json_object(); + json_object_set_new(object, "name", json_string("Delivery")); + json_object_set_new(object, "confidence", json_real(result.deliveryScore)); + json_array_append_new(tags_array, object); + } + int i = 0; + statStringStream << result.personScores.size(); + for(std::vector::iterator itr = result.personScores.begin(); itr < result.personScores.end(); itr++) { + statStringStream << "," < lock(stnMutex); + detectionResult = createJSONFromDetectionResult(result); + if(!strcmp(currSTN.fname, fname)) { + if(result.deliveryScore != 0) { + currSTN.deliveryDetected = true; + } + //json_decref(currSTN.detectionResult); + currSTN.detectionResult = json_copy(detectionResult); + } + if(!strcmp(payload.fname, fname)) { + if(result.deliveryScore != 0) { + payload.deliveryDetected = true; + } +// json_decref(payload.detectionResult); + payload.detectionResult = json_copy(detectionResult); + } + for (std::vector::iterator it = STNList.begin(); it != STNList.end(); ++it) { + if(!strcmp((*it).fname, fname)){ +// json_decref((*it).detectionResult); + (*it).detectionResult = json_copy(detectionResult); + if(result.deliveryScore != 0) { + (*it).deliveryDetected = true; + } + } + } + + //payload.detectionResult = createJSONFromDetectionResult(result); +// json_decref(detectionResult); + ret = STH_SUCCESS; + RDK_LOG(RDK_LOG_DEBUG,"LOG.RDK.SMARTTHUMBNAIL","%s(%d) payload updated with json string\n", __FUNCTION__ , __LINE__); + + // lock.unlock(); + return ret; + +} +/** @description: get upload status. + * @param[in] : void + * @return: true if payload is ready, false otherwise + */ +bool SmartThumbnail::getDetectionStatus() +{ + bool status = false; + + { + std::unique_lock lock(detectionMutex); + cv.wait(lock, [this] {return (detectionCompleted || termFlag);}); + + if(!termFlag) { + RDK_LOG( RDK_LOG_TRACE1,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): Wait over due detectionCompleted flag!!\n",__FUNCTION__,__LINE__); + status = detectionCompleted; + detectionCompleted = false; + } else { + RDK_LOG( RDK_LOG_TRACE1,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): Wait over due term flag!!\n",__FUNCTION__,__LINE__); + status = false; + } + + lock.unlock(); + } + + return status; +} + +/** @description: set upload status. + * @param[in] : status + * @return: STH_SUCCESS on success, STH_ERROR otherwise + */ +STH_STATUS SmartThumbnail::setDetectionStatus(bool status) +{ + { + std::unique_lock lock(detectionMutex); + detectionCompleted = status; + lock.unlock(); + } + + cv.notify_one(); + return STH_SUCCESS; +} + +#endif /** @description: get upload status. * @param[in] : void * @return: true if payload is ready, false otherwise @@ -765,9 +1085,10 @@ cv::Rect SmartThumbnail::getRelativeBoundingBox(cv::Rect boundRect, cv::Size cro * @param[in] bounding rectangle * @param[in] minimum width of the crop window * @param[in] minimum height of the crop window + * @param[out] the rescale factor of the frame if the union blob is greater than thumbnail size * @return: size of the crop window */ -cv::Size SmartThumbnail::getCropSize(cv::Rect boundRect,double w,double h) { +cv::Size SmartThumbnail::getCropSize(cv::Rect boundRect,double w,double h, double *resizeScale) { int newWidth = 0; int newHeight = 0; int adjustFactor = 1; @@ -776,6 +1097,7 @@ cv::Size SmartThumbnail::getCropSize(cv::Rect boundRect,double w,double h) { newWidth = boundRect.width; newHeight = boundRect.height; + *resizeScale = 1; #if 0 if (boundRect.width > (boundRect.height *(w/h))) { @@ -797,6 +1119,15 @@ cv::Size SmartThumbnail::getCropSize(cv::Rect boundRect,double w,double h) { #endif sz.height = h; sz.width = w; + /* As per RDKC-10175, to crop the thumbnail from the frame, where the union blob size is + * greater than the thumbnail size, the frame resized so that the union blob fits in the + * thumbnail. */ + + if((boundRect.width > w) || (boundRect.height > h)) { + // Calculate the resizing scale for the frame + *resizeScale = STN_MAX(boundRect.width/w, boundRect.height/h); + } + return sz; } @@ -857,6 +1188,12 @@ STH_STATUS SmartThumbnail::destroy() consumer = NULL; } +#ifdef _OBJ_DETECTION_ + if(smTnInstance -> mpipe_hres_yuvData){ + free(smTnInstance -> mpipe_hres_yuvData); + smTnInstance -> mpipe_hres_yuvData = NULL; + } +#endif frameInfo = NULL; #ifndef _DIRECT_FRAME_READ_ frameHandler.curl_handle = NULL; @@ -986,10 +1323,16 @@ void SmartThumbnail::onMsgCvr(rtMessageHeader const* hdr, uint8_t const* buff, u //take action on clip gen status if(clipGenStatus == CVR_CLIP_GEN_START) { + memset(&(smartThInst->currSTN), 0, sizeof(smartThInst->currSTN)); +#ifdef _OBJ_DETECTION_ +// json_object_set_new(smartThInst->currSTN.detectionResult, "tags", json_array()); + gettimeofday(&(smartThInst -> clipStartTime), NULL); +#endif memset(smartThInst->currSTNFname, 0, sizeof(smartThInst->currSTNFname)); strcpy(smartThInst->currSTNFname, cvrClipFname); strcat(smartThInst->currSTNFname, ".jpg"); RDK_LOG(RDK_LOG_DEBUG,"LOG.RDK.SMARTTHUMBNAIL","(%s):%d STN fname to be generated: %s\n", __FUNCTION__, __LINE__, smartThInst->currSTNFname); + strcpy(smartThInst->currSTN.fname, smartThInst->currSTNFname); smartThInst -> cvrClipGenStarted = true; smartThInst->logMotionEvent = true; smartThInst->logROIMotionEvent = true; @@ -1008,9 +1351,14 @@ void SmartThumbnail::onMsgCvr(rtMessageHeader const* hdr, uint8_t const* buff, u if((smartThInst->cvrClipGenStarted) && (smartThInst->isPayloadAvailable)) { struct timespec currTime; - bool ignoreMotion = false; memset (&currTime, 0, sizeof(struct timespec)); clock_gettime(CLOCK_REALTIME, &currTime); + bool ignoreMotion = false; +#ifdef _OBJ_DETECTION_ + if(smartThInst -> detectionEnabled) { + smartThInst->currSTN.motionTime = currTime.tv_sec; + } else { +#endif if((currTime.tv_sec - smartThInst->stnUploadTime) < smartThInst->event_quiet_time) { ignoreMotion = true; @@ -1021,16 +1369,26 @@ void SmartThumbnail::onMsgCvr(rtMessageHeader const* hdr, uint8_t const* buff, u RDK_LOG( RDK_LOG_INFO,"LOG.RDK.CVR","%s(%d): %s\n", __FILE__, __LINE__, smartThInst->motionLog); #endif } +#ifdef _OBJ_DETECTION_ + } +#endif + //save smart thumbnail from memory to file if( false == ignoreMotion ) { - RDK_LOG( RDK_LOG_INFO,"LOG.RDK.CVR","%s(%d): %s\n", __FILE__, __LINE__, smartThInst->motionLog); smartThInst->saveSTN(); smartThInst->addSTN(); smartThInst->checkSTN(); smartThInst->cvrClipGenStarted = false; + smartThInst->motion_time = currTime.tv_sec; } } +#ifdef _OBJ_DETECTION_ + if(smartThInst->detectionInProgress) { + cv::Mat emptyMat; + mpipe_port_onLastFrameEvent(emptyMat, 0, 0); + } +#endif if(!smartThInst->isPayloadAvailable) { RDK_LOG(RDK_LOG_TRACE1,"LOG.RDK.SMARTTHUMBNAIL","(%s):%d \t3. Smart Thumbnail is not available during current period.\n", __FUNCTION__, __LINE__); @@ -1086,6 +1444,22 @@ void SmartThumbnail::onMsgCvrUpload(rtMessageHeader const* hdr, uint8_t const* b if(STH_SUCCESS == createPayloadStatus) { RDK_LOG(RDK_LOG_INFO,"LOG.RDK.SMARTTHUMBNAIL","(%s):%d payload created for CVR clip :%s.\n", __FUNCTION__, __LINE__, cvrClipFname); smartThInst->setUploadStatus(true); +#ifdef _OBJ_DETECTION_ +#if 0 + RDK_LOG(RDK_LOG_ERROR,"LOG.RDK.SMARTTHUMBNAIL","(%s):%d: Starting %s to delivery detector thread\n", __FUNCTION__, __LINE__, stnFname); + cv::Mat cv_frame = cv::imread(stnFname, cv::IMREAD_COLOR); + cv::Mat cv_frame_rgb; + cv::cvtColor(cv_frame, cv_frame_rgb, cv::COLOR_BGR2RGB); + uint8_t *camera_buf = (uint8_t *) malloc(cv_frame.cols * cv_frame.rows * 3); + memcpy(camera_buf, cv_frame_rgb.data, cv_frame.cols * cv_frame.rows * 3); + cv::Mat cv_frame_dst = cv::Mat(cv_frame.rows, cv_frame.cols, CV_8UC3, camera_buf); + mpipe_port_onThunbmailEvent(cv_frame_dst, cv_frame.cols, cv_frame.rows); + cv_frame.release(); + cv_frame_rgb.release(); + //mpipe_port_onMotionEvent(false); +#endif + gettimeofday(&(smartThInst -> uploadTriggeredTime), NULL); +#endif } else if (STH_NO_PAYLOAD == createPayloadStatus) { RDK_LOG(RDK_LOG_INFO,"LOG.RDK.SMARTTHUMBNAIL","(%s):%d No payload available for CVR clip:%s.\n", __FUNCTION__, __LINE__, cvrClipFname); } else { @@ -1178,6 +1552,10 @@ void SmartThumbnail::onMsgCaptureFrame(rtMessageHeader const* hdr, uint8_t const rtMessage_FromBytes(&m, buff, n); rtMessage_GetInt32(m, "processID", &processPID); rtMessage_GetString(m, "timestamp", &strFramePTS); +#ifdef ENABLE_TEST_HARNESS + rtMessage_GetInt32(m, "fileNum", &(smartThInst -> FileNum)); + rtMessage_GetInt32(m, "frameNum", &(smartThInst -> FrameNum)); +#endif RDK_LOG(RDK_LOG_TRACE1,"LOG.RDK.SMARTTHUMBNAIL","%s(%d) strFramePTS:%s \n", __FUNCTION__ , __LINE__, strFramePTS); RDK_LOG(RDK_LOG_TRACE1,"LOG.RDK.SMARTTHUMBNAIL","%s(%d) capture invoked by process %d \n", __FUNCTION__ , __LINE__, processPID); @@ -1188,23 +1566,63 @@ void SmartThumbnail::onMsgCaptureFrame(rtMessageHeader const* hdr, uint8_t const rtMessage_Release(m); -/* - RDK_LOG(RDK_LOG_TRACE1,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): Last Motion time stamp: %d\n", __FUNCTION__ , __LINE__, smartThInst->motion_time); +#ifdef ENABLE_TEST_HARNESS + std::string frame_filename; + if(smartThInst->testHarnessOnFileFeed)/*strcmp(rdkc_envGet(TEST_HARNESS_ON_FILE_ENABLED), "true") == 0)*/ { + //Read the frame from file + frame_filename = "/tmp/THFrame_" + std::to_string(lResFramePTS) + ".jpg"; + RDK_LOG(RDK_LOG_INFO,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): Reading frame file : %s\n", __FUNCTION__ , __LINE__, frame_filename.c_str()); + } +#endif +#if 0 +def _OBJ_DETECTION_ + RDK_LOG(RDK_LOG_DEBUG,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): Last Motion time stamp: %d\n", __FUNCTION__ , __LINE__, smartThInst->motion_time); // ignore frames and metadata for event_quiet_time if( (currTime.tv_sec < (smartThInst->motion_time + smartThInst->event_quiet_time)) && (0 != smartThInst->motion_time) ) { - RDK_LOG(RDK_LOG_TRACE1,"LOG.RDK.SMARTTHUMBNAIL","%s(%d):Within event quiet time, Ignoring event, time passed: %lu\n", __FUNCTION__ , __LINE__, (currTime.tv_sec-smartThInst->motion_time)); + RDK_LOG(RDK_LOG_DEBUG,"LOG.RDK.SMARTTHUMBNAIL","%s(%d):Within event quiet time(%d), Ignoring event, time passed: %lu\n", __FUNCTION__ , __LINE__, smartThInst->event_quiet_time, (currTime.tv_sec-smartThInst->motion_time)); +#ifdef ENABLE_TEST_HARNESS + if(smartThInst->testHarnessOnFileFeed)/*strcmp(rdkc_envGet(TEST_HARNESS_ON_FILE_ENABLED), "true") == 0)*/ { + unlink(frame_filename.c_str()); + } + smartThInst -> currTstamp = lResFramePTS; + smartThInst->detectionCv.notify_one(); +#endif return; } - RDK_LOG(RDK_LOG_TRACE1,"LOG.RDK.SMARTTHUMBNAIL","%s(%d):Not within event quiet time, Capturing frame.\n", __FUNCTION__ , __LINE__); -*/ + RDK_LOG(RDK_LOG_DEBUG,"LOG.RDK.SMARTTHUMBNAIL","%s(%d):Not within event quiet time, Capturing frame.\n", __FUNCTION__ , __LINE__); +#endif +#ifdef ENABLE_TEST_HARNESS + std::unique_lock lock(smartThInst->hres_data_lock); +if(smartThInst->testHarnessOnFileFeed)/*strcmp(rdkc_envGet(TEST_HARNESS_ON_FILE_ENABLED), "true") == 0)*/ { +/* cv::Mat*/ smartThInst->curr_frame = cv::imread(frame_filename, cv::IMREAD_COLOR); +#if 0 + cv::cvtColor(curr_frame, smartThInst->fileFrameYUV, cv::COLOR_BGR2YUV); + split(smartThInst->fileFrameYUV, smartThInst->yuvPlanes); + smartThInst->yuvChannels.clear(); + + smartThInst->yuvChannels.push_back((smartThInst->yuvPlanes)[1]); + smartThInst->yuvChannels.push_back((smartThInst->yuvPlanes)[2]); + + merge(smartThInst->yuvChannels, smartThInst->planeUV); + + smartThInst->frameInfo->y_addr = (smartThInst->yuvPlanes)[0].data; + smartThInst->frameInfo->width = (smartThInst->yuvPlanes)[0].cols; + smartThInst->frameInfo->height = (smartThInst->yuvPlanes)[0].rows; + smartThInst->frameInfo->uv_addr = smartThInst->planeUV.data; + smartThInst->frameInfo->mono_pts = 0; +#endif + unlink(frame_filename.c_str()); + smartThInst -> isHresFrameReady = true; +} else { +#endif //read the 720*1280 YUV data. #ifdef _HAS_XSTREAM_ #ifdef _DIRECT_FRAME_READ_ - ret = smartThInst->consumer->ReadRAWFrame((u16)STN_HRES_BUFFER_ID, (u16)FORMAT_YUV, smartThInst->frameInfo); + ret = smartThInst->consumer->ReadRAWFrame((u16)smartThInst->buf_id, (u16)FORMAT_YUV, smartThInst->frameInfo); #else - ret = smartThInst->consumer->ReadRAWFrame(STN_HRES_BUFFER_ID, FORMAT_YUV, smartThInst->frameInfo); + ret = smartThInst->consumer->ReadRAWFrame(smartThInst->buf_id, FORMAT_YUV, smartThInst->frameInfo); #endif #else @@ -1213,7 +1631,7 @@ void SmartThumbnail::onMsgCaptureFrame(rtMessageHeader const* hdr, uint8_t const } memset(smartThInst -> hres_frame_info, 0, sizeof(RDKC_PLUGIN_YUVInfo)); - ret = smartThInst -> recorder -> ReadYUVData(STN_HRES_BUFFER_ID, smartThInst -> hres_frame_info); + ret = smartThInst -> recorder -> ReadYUVData(smartThInst->buf_id, smartThInst -> hres_frame_info); #endif if( STH_SUCCESS != ret) { @@ -1254,7 +1672,13 @@ void SmartThumbnail::onMsgCaptureFrame(rtMessageHeader const* hdr, uint8_t const //Ensure metadata is received after the frame is captured. smartThInst -> isHresFrameReady = true; +#ifdef ENABLE_TEST_HARNESS + } + smartThInst -> currTstamp = lResFramePTS; + lock.unlock(); + smartThInst->detectionCv.notify_one(); +#endif } /** @description : Callback function to generate smart thumbnail based on motion. @@ -1281,8 +1705,8 @@ void SmartThumbnail::onMsgProcessFrame(rtMessageHeader const* hdr, uint8_t const memset (&currTime, 0, sizeof(struct timespec)); clock_gettime(CLOCK_REALTIME, &currTime); - // RDK_LOG(RDK_LOG_INFO,"LOG.RDK.SMARTTHUMBNAIL","%s(%d) time-spent:%lu \n", __FUNCTION__ , __LINE__, currTime.tv_nsec - va_msg_prev_time); - //va_msg_prev_time = currTime.tv_nsec; +// RDK_LOG(RDK_LOG_INFO,"LOG.RDK.SMARTTHUMBNAIL","%s(%d) time-spent:%lu \n", __FUNCTION__ , __LINE__, currTime.tv_nsec - va_msg_prev_time); +// va_msg_prev_time = currTime.tv_nsec; #endif (void) closure; @@ -1308,6 +1732,25 @@ void SmartThumbnail::onMsgProcessFrame(rtMessageHeader const* hdr, uint8_t const RDK_LOG( RDK_LOG_TRACE1,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): curr timestamp (uint64):%llu\n", __FILE__, __LINE__,curr_time); iss.clear(); +#ifdef _OBJ_DETECTION_ + RDK_LOG(RDK_LOG_DEBUG,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): Last Motion time stamp: %d\n", __FUNCTION__ , __LINE__, smartThInst->motion_time); + if(smartThInst->detectionEnabled && sm.event_type == 4) { + smartThInst->currentBbox.x = sm.deliveryUnionBox.boundingBoxXOrd; + smartThInst->currentBbox.y = sm.deliveryUnionBox.boundingBoxYOrd; + smartThInst->currentBbox.width = sm.deliveryUnionBox.boundingBoxWidth; + smartThInst->currentBbox.height = sm.deliveryUnionBox.boundingBoxHeight; + } +#if 0 + // ignore frames and metadata for event_quiet_time + if( (currTime.tv_sec < (smartThInst->motion_time + smartThInst->event_quiet_time)) && (0 != smartThInst->motion_time) ) { + RDK_LOG(RDK_LOG_DEBUG,"LOG.RDK.SMARTTHUMBNAIL","%s(%d):Within event quiet time(%d), Ignoring event, time passed: %lu\n", __FUNCTION__ , __LINE__, smartThInst->event_quiet_time, (currTime.tv_sec-smartThInst->motion_time)); + rtMessage_Release(m); + return; + } + RDK_LOG(RDK_LOG_DEBUG,"LOG.RDK.SMARTTHUMBNAIL","%s(%d):Not within event quiet time, Capturing frame.\n", __FUNCTION__ , __LINE__); +#endif +#endif + lBboxArea = sm.unionBox.boundingBoxWidth * sm.unionBox.boundingBoxHeight; RDK_LOG(RDK_LOG_TRACE1,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): Current Area : %d\n", __FILE__ , __LINE__, lBboxArea); @@ -1336,6 +1779,13 @@ void SmartThumbnail::onMsgProcessFrame(rtMessageHeader const* hdr, uint8_t const (sm.event_type == 4) && (lBboxArea > smartThInst->maxBboxArea) && (isInsideROI == 1)) { + + +#ifdef _OBJ_DETECTION_ + if(smartThInst->motion_time == 0) { + smartThInst->motion_time = currTime.tv_sec - smartThInst->event_quiet_time; + } +#endif RDK_LOG(RDK_LOG_TRACE1,"LOG.RDK.SMARTTHUMBNAIL","%s(%d) Processing metadata .\n", __FUNCTION__ , __LINE__); @@ -1347,8 +1797,30 @@ void SmartThumbnail::onMsgProcessFrame(rtMessageHeader const* hdr, uint8_t const //std::unique_lock lock(stnMutex); if (!(smartThInst -> isPayloadAvailable)) { RDK_LOG(RDK_LOG_TRACE1,"LOG.RDK.SMARTTHUMBNAIL","%s(%d) Payload will be available for this interval.\n", __FUNCTION__ , __LINE__); +#ifndef _OBJ_DETECTION_ //To indicate payload will there to upload smartThInst -> isPayloadAvailable = true; +#else + if(!smartThInst -> detectionEnabled) { + //To indicate payload will there to upload + smartThInst -> isPayloadAvailable = true; + } else if(!smartThInst -> detectionInProgress) { + smartThInst->setDetectionStatus(false); + gettimeofday(&(smartThInst -> detectionStartTime), NULL); + smartThInst -> motionTriggeredTime = smartThInst->detectionStartTime.tv_sec - smartThInst->clipStartTime.tv_sec; + smartThInst -> detectionInProgress = true; + strcpy(smartThInst->currDetectionSTNFname, smartThInst->currSTNFname); + smartThInst->mpipeProcessedframes = 0; + mpipe_port_onMotionEvent(true); + smartThInst -> isPayloadAvailable = true; + //To indicate payload will there to upload + } +#ifdef ENABLE_TEST_HARNESS + smartThInst -> detectionTstamp = 0; + smartThInst -> THFileNum = smartThInst -> FileNum; + smartThInst -> THFrameNum = smartThInst -> FrameNum; +#endif +#endif } smartThInst -> updateObjFrameData(sm.unionBox.boundingBoxXOrd, sm.unionBox.boundingBoxYOrd, sm.unionBox.boundingBoxWidth, sm.unionBox.boundingBoxHeight, curr_time); memset(smartThInst->objectBoxs, INVALID_BBOX_ORD, sizeof(smartThInst->objectBoxs)); @@ -1424,6 +1896,7 @@ void SmartThumbnail::resetObjFrameData() if(!smartThInst -> ofData.maxBboxObjYUVFrame.empty()) { //RDK_LOG( RDK_LOG_INFO,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): Releasing maxBboxObjYUVFrame\n", __FILE__, __LINE__); + RDK_LOG( RDK_LOG_DEBUG,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): Releasing maxBboxObjYUVFrame\n", __FILE__, __LINE__); smartThInst -> ofData.maxBboxObjYUVFrame.release(); } } @@ -1452,6 +1925,11 @@ void SmartThumbnail::updateObjFrameData(int32_t boundingBoxXOrd,int32_t bounding snprintf(tmpFname, sizeof(tmpFname), "%llu.jpg", ofData.currTime); #endif +#ifdef ENABLE_TEST_HARNESS + if(testHarnessOnFileFeed) { + ofData.maxBboxObjYUVFrame = curr_frame.clone(); + } else { +#endif #ifdef _HAS_XSTREAM_ smartThInst -> hres_y_height = smartThInst -> frameInfo->height; smartThInst -> hres_y_width = smartThInst -> frameInfo->width; @@ -1488,6 +1966,10 @@ void SmartThumbnail::updateObjFrameData(int32_t boundingBoxXOrd,int32_t bounding ofData.maxBboxObjYUVFrame = cv::Mat(smartThInst -> hres_frame_info -> height + (smartThInst -> hres_frame_info -> height)/2, smartThInst -> hres_frame_info -> width, CV_8UC1, hres_yuvData).clone(); #endif +#ifdef ENABLE_TEST_HARNESS + } +#endif + if(hres_yuvData) { free(hres_yuvData); hres_yuvData = NULL; @@ -1604,6 +2086,9 @@ void SmartThumbnail::uploadPayload() int retry = 0; int remainingTime = 0; time_t stnTS = (time_t)(smartThInst->payload.tstamp); +#ifdef _OBJ_DETECTION_ + char encodedBuff[512]; +#endif struct timespec currTime; struct timespec startTime; @@ -1634,6 +2119,7 @@ void SmartThumbnail::uploadPayload() //snprintf(sTnTStamp, sizeof(sTnTStamp), "%llu", payload.tstamp); stringifyEventDateTime(sTnTStamp, sizeof(sTnTStamp), smartThInst->payload.tstamp); RDK_LOG( RDK_LOG_TRACE1,"LOG.RDK.SMARTTHUMBNAIL","%s(%d):STN Time is %s\n", __FILE__, __LINE__, sTnTStamp); + RDK_LOG( RDK_LOG_INFO,"LOG.RDK.SMARTTHUMBNAIL","%s(%d):STN name is %s\n", __FILE__, __LINE__, smartThInst->payload.fname); //check for empty payload and break if so /*if (payload.objFrame.empty()) { @@ -1659,6 +2145,33 @@ void SmartThumbnail::uploadPayload() continue;*/ } +#ifdef _OBJ_DETECTION_ + if(smartThInst -> detectionEnabled) { + RDK_LOG( RDK_LOG_DEBUG,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): Waiting for detection to finish\n", __FUNCTION__, __LINE__); + struct timeval start, end; + gettimeofday(&start, NULL); + if((!strcmp(currDetectionSTNFname, payload.fname)) && (smartThInst->getDetectionStatus())) { + RDK_LOG( RDK_LOG_DEBUG,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): Detection completed\n", __FUNCTION__, __LINE__); + } + + if((!payload.deliveryDetected) && ((payload.motionTime - smartThInst->stnUploadTime) < smartThInst->event_quiet_time)) + { + lock.unlock(); + delSTN(sTnUploadFpath); + RDK_LOG( RDK_LOG_INFO,"LOG.RDK.CVR","%s(%d): Skipping Motion events! curr motion time %ld prev motion upload time %ld\n", __FILE__, __LINE__, payload.motionTime, smartThInst->stnUploadTime); + return; + } + + RDK_LOG( RDK_LOG_INFO,"LOG.RDK.CVR","%s(%d): %s\n", __FILE__, __LINE__, smartThInst->motionLog); + + gettimeofday(&end, NULL); + double time_taken; + time_taken = (end.tv_sec - start.tv_sec) * 1e6; + time_taken = (time_taken + (end.tv_usec - start.tv_usec)) * 1e-6; + RDK_LOG( RDK_LOG_INFO,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): Upload waited %lf seconds for delivery detection!!!\n",__FUNCTION__,__LINE__, time_taken); + } +#endif + fileLen = fileStat.st_size; RDK_LOG(RDK_LOG_INFO, "LOG.RDK.SMARTTHUMBNAIL", "%s(%d):Length of the smart thumbnail file to be uploaded:%d\n", __FILE__, __LINE__,fileLen); @@ -1754,6 +2267,19 @@ void SmartThumbnail::uploadPayload() memset(packHead, 0, sizeof(packHead)); snprintf(packHead, sizeof(packHead), "CVR"); smartThInst->httpClient->addHeader("X-VIDEO-RECORDING", packHead); +#ifdef _OBJ_DETECTION_ + if(smartThInst -> detectionEnabled) { + //char *jsonStr = json_dumps(detectionResult, 0); + char *jsonStr = json_dumps(payload.detectionResult, 0); + RDK_LOG( RDK_LOG_INFO,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): X-IMAGE-METADATA: %s\n", __FUNCTION__, __LINE__, jsonStr); + //Apply base64 encoding on the metadata + memset(encodedBuff, 0, sizeof(encodedBuff)); + b64_encode((uint8_t*)jsonStr, strlen(jsonStr), (uint8_t*)encodedBuff); + RDK_LOG( RDK_LOG_DEBUG,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): encoded X-IMAGE-METADATA: %s\n", __FUNCTION__, __LINE__, (char*)encodedBuff); + smartThInst->httpClient->addHeader("X-IMAGE-METADATA", encodedBuff); + free(jsonStr); + } +#endif #ifdef USE_FILE_UPLOAD memset(packHead, 0, sizeof(packHead)); @@ -1796,6 +2322,11 @@ void SmartThumbnail::uploadPayload() stnUploadTime = currTime.tv_sec; #endif eventquietTimeStart = currTime.tv_sec; +#ifdef ENABLE_TEST_HARNESS + stnUploadTime = currTstamp; +#else + stnUploadTime = currTime.tv_sec; +#endif RDK_LOG( RDK_LOG_INFO,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): Smart Thumbnail uploaded successfully with header X-EVENT-TIME: %s X-BoundingBox: %d %d %d %d X-VIDEO-RECORDING:CVR X-BoundingBoxes: %s\n",__FUNCTION__,__LINE__,sTnTStamp, relativeBBox.x, relativeBBox.y, relativeBBox.width, relativeBBox.height,objectBoxsBuf); RDK_LOG( RDK_LOG_INFO,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): StnTimestamp,CurrentTimestamp,Latency:%ld,%ld,%ld\n",__FUNCTION__,__LINE__, stnTS, currTime.tv_sec, (currTime.tv_sec-stnTS)); }else { @@ -2055,6 +2586,13 @@ void SmarttnMetadata_thumb::from_rtMessage(SmarttnMetadata_thumb *smInfo, const rtMessage_GetInt32(m, "boundingBoxWidth", &(smInfo->unionBox.boundingBoxWidth)); rtMessage_GetInt32(m, "boundingBoxHeight", &(smInfo->unionBox.boundingBoxHeight)); +#ifdef _OBJ_DETECTION_ + rtMessage_GetInt32(m, "d_boundingBoxXOrd", &(smInfo->deliveryUnionBox.boundingBoxXOrd)); + rtMessage_GetInt32(m, "d_boundingBoxYOrd", &(smInfo->deliveryUnionBox.boundingBoxYOrd)); + rtMessage_GetInt32(m, "d_boundingBoxWidth", &(smInfo->deliveryUnionBox.boundingBoxWidth)); + rtMessage_GetInt32(m, "d_boundingBoxHeight", &(smInfo->deliveryUnionBox.boundingBoxHeight)); +#endif + RDK_LOG( RDK_LOG_DEBUG,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): strFramePTS:%s \n", __FUNCTION__ , __LINE__, smInfo->strFramePTS); RDK_LOG( RDK_LOG_DEBUG,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): event Type: %d\n",__FILE__, __LINE__, smInfo->event_type); RDK_LOG( RDK_LOG_DEBUG,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): motionScore: %f\n",__FILE__, __LINE__, smInfo->motionScore); @@ -2062,6 +2600,12 @@ void SmarttnMetadata_thumb::from_rtMessage(SmarttnMetadata_thumb *smInfo, const RDK_LOG( RDK_LOG_DEBUG,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): boundingBoxYOrd:%d\n", __FILE__, __LINE__, smInfo->unionBox.boundingBoxYOrd); RDK_LOG( RDK_LOG_DEBUG,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): boundingBoxWidth:%d\n", __FILE__, __LINE__, smInfo->unionBox.boundingBoxWidth); RDK_LOG( RDK_LOG_DEBUG,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): boundingBoxHeight:%d\n", __FILE__, __LINE__, smInfo->unionBox.boundingBoxHeight); +#ifdef _OBJ_DETECTION_ + RDK_LOG( RDK_LOG_DEBUG,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): del_boundingBoxXOrd:%d\n", __FILE__, __LINE__, smInfo->deliveryUnionBox.boundingBoxXOrd); + RDK_LOG( RDK_LOG_DEBUG,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): del_boundingBoxYOrd:%d\n", __FILE__, __LINE__, smInfo->deliveryUnionBox.boundingBoxYOrd); + RDK_LOG( RDK_LOG_DEBUG,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): del_boundingBoxWidth:%d\n", __FILE__, __LINE__, smInfo->deliveryUnionBox.boundingBoxWidth); + RDK_LOG( RDK_LOG_DEBUG,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): del_boundingBoxHeight:%d\n", __FILE__, __LINE__, smInfo->deliveryUnionBox.boundingBoxHeight); +#endif RDK_LOG( RDK_LOG_DEBUG,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): curr timestamp:%s\n", __FILE__, __LINE__, smInfo->s_curr_time); rtMessage_GetArrayLength(m, "objectBoxs", &len); diff --git a/smart_thumbnail_cvr_lite/Makefile b/smart_thumbnail_cvr_lite/Makefile index f734d07..797e75d 100644 --- a/smart_thumbnail_cvr_lite/Makefile +++ b/smart_thumbnail_cvr_lite/Makefile @@ -47,6 +47,10 @@ ifeq ($(XCAM_MODEL), SCHC2) USE_DEWARP = yes endif +ifeq ($(ENABLE_DELIVERY_DETECTION), true) +USE_MISC = yes +endif + ENABLE_DIRECT_FRAME_READ := true ifneq ($(ENABLE_XSTREAMER), true) @@ -85,6 +89,18 @@ LIBS += -L$(RDK_PROJECT_ROOT_PATH)/opensource/lib -lopencv_core -lopencv_img LIBS += -Wl,-rpath-link=$(PROJ_ROOT)/vendor/img/fs/shadow_root/usr/lib CFLAGS += -DUSE_MFRLIB +ifeq ($(ENABLE_DELIVERY_DETECTION), true) +CFLAGS += -D_OBJ_DETECTION_ +SRC += mpipe_port.cpp +CFLAGS += -I$(BUILD_ROOT_DIR)/opensource/include +LIBS += -L$(BUILD_ROOT_DIR)/opensource/lib -lbase64 +LIBS += -ldeliverydetector +endif + +ifeq ($(ENABLE_TEST_HARNESS), true) +CFLAGS += -DENABLE_TEST_HARNESS +endif + SRC += smart_thumbnail.cpp main.cpp OBJ = $(SRC:.cpp=.o) diff --git a/smart_thumbnail_cvr_lite/include/mpipe_port.h b/smart_thumbnail_cvr_lite/include/mpipe_port.h new file mode 100644 index 0000000..c4acc2e --- /dev/null +++ b/smart_thumbnail_cvr_lite/include/mpipe_port.h @@ -0,0 +1,46 @@ +#ifndef MPIPE_PORT_H__ +#define MPIPE_PORT_H__ + +#include +#include + +typedef struct DetectionResult_ { + /* The bounding boxes of each (n) person detected in the thumbnail */ + std::vector> personBBoxes; + /* The detection score of each (n) person detected in the thumbnail */ + std::vector personScores; + float deliveryScore; + int maxAugScore; +} DetectionResult; + +void *__mpipe_thread_main__(); + +/* open the device in the given path. The 'width and height' is the resolution that + * mpipe desires. If the camera does not support, the camera's resolution is updated + * to widht and height */ +int mpipe_port_initialize(const std::string &input_video_path, int &width, int &height); +int mpipe_port_initialize(int &width, int &height); + +/* get next frame, in COLOR_RGB format */ +cv::Mat mpipe_port_getNextFrame(); +void mpipe_port_term(); + + +/* ---------------------------------------------------------------------- + * The following are mpipe APIs available for porting application to call. + * porting application should not implement these APIs. + * ----------------------------------------------------------------------*/ +/* header only. Do not implement */ +/* notify mpipe of last frame for person detection, in cv::Mat, RGB + * the lastFrame can be empty or non-empty. If non-empty, it will be + * process for person dection before continue onto delivery detection. + */ +void mpipe_port_onLastFrameEvent(const cv::Mat &thumbnail, int width, int height); +/* notify mpipe of motion deteced event */ +/* header only. Do not implement */ +void mpipe_port_onMotionEvent(bool motionDetected); +typedef void (* OnDetectionChanged_t) (const DetectionResult &detection_result); +/* mpipe will notify application of detection results */ +/* header only. Do not implement */ +void mpipe_port_setOnDetectionChanged(OnDetectionChanged_t cb); +#endif; diff --git a/smart_thumbnail_cvr_lite/include/smart_thumbnail.h b/smart_thumbnail_cvr_lite/include/smart_thumbnail.h index 4dd40ef..77bc359 100644 --- a/smart_thumbnail_cvr_lite/include/smart_thumbnail.h +++ b/smart_thumbnail_cvr_lite/include/smart_thumbnail.h @@ -28,7 +28,14 @@ #include #include #include - +#ifdef _OBJ_DETECTION_ +#include +#include +#include "RFCCommon.h" +#ifdef ENABLE_TEST_HARNESS +#include "dev_config.h" +#endif +#endif #ifdef LEGACY_CFG_MGR #include "dev_config.h" #else @@ -64,6 +71,10 @@ extern "C" { #include "RFCCommon.h" #include "opencv2/opencv.hpp" +#ifdef _OBJ_DETECTION_ +#include "mpipe_port.h" +#include "base64.h" +#endif #define FW_NAME_MAX_LENGTH 512 #define CONFIG_STRING_MAX (256) @@ -75,6 +86,7 @@ extern "C" { #define STN_HRES_BUFFER_ID 2 #elif defined(XHB1) || defined (XHC3) #define STN_HRES_BUFFER_ID 2 +#define STN_MRES_BUFFER_ID 1 #else #define STN_HRES_BUFFER_ID 0 #endif @@ -88,9 +100,19 @@ extern "C" { //default width and height of smart thumbnail #define STN_DEFAULT_WIDTH 640 #define STN_DEFAULT_HEIGHT 480 +#ifdef XHB1 +#define STN_HRES_CROP_WIDTH 640 +#define STN_HRES_CROP_HEIGHT 480 +#define STN_MRES_CROP_WIDTH 400 +#define STN_MRES_CROP_HEIGHT 300 +#endif #define STN_TIMESTAMP_TAG "timestamp" +#ifdef ENABLE_TEST_HARNESS +#define STN_UPLOAD_TIME_INTERVAL 12 +#else #define STN_UPLOAD_TIME_INTERVAL 4 +#endif #define STN_PATH "/tmp" #define STN_UPLOAD_SEND_LEN 2048 @@ -113,6 +135,31 @@ extern "C" { #define BLOB_BB_MAX_LEN 256 #define INVALID_BBOX_ORD (-1) +#if defined(_OBJ_DETECTION_) && defined(ENABLE_TEST_HARNESS) +#define TEST_HARNESS_ON_FILE_ENABLED "Test_Harness_on_File_Enabled" +#endif + +#ifdef _OBJ_DETECTION_ +#define DEFAULT_INPUT_DEV "/dev/video0" +#define DEFAULT_GRAPH_PATH "/etc/mediapipe/graphs/rdk/delivery_detection/g_delivery_detection_cpu.pbtxt" +#define DEFAULT_FRAME_READ_DELAY "1000" +#define DEFAULT_MAX_FRAMES_CACHED_FOR_DELIVERY_DETECTION "5" +#define DEFAULT_DELIVERY_DETECTION_MODEL_MIN_SCORE_THRESHOLD "0.3" +#define DEFAULT_DELIVERY_DETECTION_MIN_SCORE_THRESHOLD "1" +#define DETECTION_CONFIG_FILE "/opt/usr_config/detection_attr.conf" +#define DEFAULT_FRAME_COUNT_TO_PROCESS "5" + +typedef struct detection_config_ { + std::string input_video_path; + std::string delivery_detection_graph_path; + std::string frame_read_delay; + std::string max_num_frames_cached_for_delivery_detection; + std::string delivery_detection_model_min_score_threshold; + std::string delivery_detection_min_score_threshold; + std::string frame_count_to_process; +}DetectionConfig; +#endif + typedef enum { STH_ERROR = -1, STH_SUCCESS, @@ -124,6 +171,11 @@ typedef struct { #ifdef _HAS_DING_ uint64_t dingtstamp; #endif +#ifdef _OBJ_DETECTION_ + json_t *detectionResult; + uint64_t motionTime; + bool deliveryDetected; +#endif }STHPayload; typedef struct { @@ -148,7 +200,7 @@ class SmartThumbnail public: static SmartThumbnail* getInstance(); //Initialize the buffers and starts msg monitoring, upload thread. - STH_STATUS init(char* mac,bool isCVREnabled); + STH_STATUS init(char* mac,bool isCVREnabled, bool isDetectionEnabled); //Pushes the data to the upload queue at the end of interval. STH_STATUS createPayload(); //Upload smart thumbnail data @@ -162,6 +214,14 @@ class SmartThumbnail STH_STATUS notify(const char* status); //call Smart thumbnail destructor and deallocates dynamic allocated memory. STH_STATUS destroy(); +#ifdef _OBJ_DETECTION_ +#ifdef ENABLE_TEST_HARNESS + void notifyXvision(const DetectionResult &result); +#endif + void onCompletedDeliveryDetection(const DetectionResult &result); + friend cv::Mat mpipe_port_getNextFrame(); +#endif + private: SmartThumbnail(); ~SmartThumbnail(); @@ -195,10 +255,19 @@ class SmartThumbnail static void receiveRtmessage(); //Callback function for dynamic logging. static void dynLogOnMessage(rtMessageHeader const* hdr, uint8_t const* buff, uint32_t n, void* closure); +#ifdef _OBJ_DETECTION_ + STH_STATUS updateUploadPayload(DetectionResult result); + json_t* createJSONFromDetectionResult(DetectionResult result); + bool getDeliveryDetectionStatus(); + STH_STATUS setDeliveryDetectionCompleted(bool status); +#ifdef ENABLE_TEST_HARNESS + void waitForNextDetectionFrame(); +#endif +#endif cv::Point2f getActualCentroid(cv::Rect boundRect); cv::Point2f alignCentroid(cv::Point2f orgCenter, cv::Mat origFrame, cv::Size cropSize); - cv::Size getCropSize(cv::Rect boundRect,double w,double h); + cv::Size getCropSize(cv::Rect boundRect,double w,double h, double *rescaleSize); cv::Rect getRelativeBoundingBox(cv::Rect boundRect, cv::Size cropSize, cv::Point2f allignedCenter); static SmartThumbnail* smartThInst; @@ -226,9 +295,30 @@ class SmartThumbnail RDKC_PLUGIN_YUVInfo* hres_frame_info; #endif +#ifdef _OBJ_DETECTION_ + unsigned char* mpipe_hres_yuvData; + cv::Rect currentBbox; + std::mutex deliveryDetectionMutex; + bool detectionCompleted; + std::condition_variable detection_cv; + struct timeval detectionStartTime, detectionEndTime, uploadTriggeredTime; + bool detectionInProgress; + bool detectionEnabled; + int mpipeProcessedframes; +#ifdef ENABLE_TEST_HARNESS + bool testHarnessOnFileFeed; + std::vector yuvPlanes, yuvChannels; + cv::Mat fileFrameYUV, planeUV, curr_frame; + uint64_t currTstamp, detectionTstamp; + std::condition_variable detectionCv; + std::mutex hres_data_lock; + int THFileNum, THFrameNum; + int FileNum = 0, FrameNum = 0; +#endif + +#endif bool logMotionEvent; bool logROIMotionEvent; - std::thread rtMessageReceive; std::thread uploadThread; bool rtmessageSTHThreadExit; @@ -262,6 +352,7 @@ class SmartThumbnail char firmwareName[FW_NAME_MAX_LENGTH]; int sTnHeight; int sTnWidth; + uint16_t buf_id; char uploadFname[256]; cv::Rect relativeBBox; cv::Rect smartThumbCoord; @@ -280,6 +371,9 @@ struct SmarttnMetadata_thumb char const *strFramePTS; int32_t event_type; double motionScore; +#ifdef _OBJ_DETECTION_ + BoundingBox deliveryUnionBox; +#endif BoundingBox unionBox; BoundingBox objectBoxs [UPPER_LIMIT_BLOB_BB]; char const *s_curr_time; diff --git a/smart_thumbnail_cvr_lite/main.cpp b/smart_thumbnail_cvr_lite/main.cpp index fccb0ef..8b889df 100644 --- a/smart_thumbnail_cvr_lite/main.cpp +++ b/smart_thumbnail_cvr_lite/main.cpp @@ -24,7 +24,9 @@ #ifdef BREAKPAD #include "breakpadwrap.h" #endif - +#ifdef _OBJ_DETECTION_ +SmartThumbnail *smTnInstance = NULL; +#endif #if 0 /** @description: Checks if the feature is enabled via RFC * @param[in] rfc_feature_fname: RFC feature filename @@ -63,7 +65,9 @@ bool checkEnabledRFCFeature(char* rfcFeatureFname, char* featureName) int main(int argc, char** argv) { +#ifndef _OBJ_DETECTION_ SmartThumbnail *smTnInstance = NULL; +#endif STH_STATUS status = STH_SUCCESS; time_t remainingTime = 0; char *param = NULL; @@ -76,6 +80,7 @@ int main(int argc, char** argv) memset(&currTime, 0, sizeof(currTime)); memset(&prevTime, 0, sizeof(prevTime)); int itr =0; + int isDetectionEnabled = 0; /* Registering callback function for Breakpadwrap Function */ #ifdef BREAKPAD @@ -124,6 +129,20 @@ int main(int argc, char** argv) break; } } + + if(strcmp(argv[itr],"--detectionEnabled")==0) + { + itr++; + + if (itr < argc) + { + isDetectionEnabled = atoi(argv[itr]); + } + else + { + break; + } + } itr++; } @@ -140,7 +159,7 @@ int main(int argc, char** argv) } //initialize smart thumbnail - status = smTnInstance-> init(param,cvrEnabled); + status = smTnInstance-> init(param,cvrEnabled, isDetectionEnabled); if (STH_ERROR == status) { RDK_LOG( RDK_LOG_ERROR,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): Error creating Smart thumbnail instance.\n", __FILE__, __LINE__); return STH_ERROR; diff --git a/smart_thumbnail_cvr_lite/mpipe_port.cpp b/smart_thumbnail_cvr_lite/mpipe_port.cpp new file mode 100644 index 0000000..299a803 --- /dev/null +++ b/smart_thumbnail_cvr_lite/mpipe_port.cpp @@ -0,0 +1,147 @@ +#include "smart_thumbnail.h" + +#define CROPPING_WIDTH 848 +#define CROPPING_HEIGHT 480 +extern SmartThumbnail *smTnInstance; + +int mpipe_port_initialize(const std::string &input_video_path, int &width, int &height) { + std::cout << "rdkc PORT Initialize the camera" << std::endl; + return 0; +} + +int mpipe_port_initialize(int &width, int &height) { + mpipe_port_initialize(std::string(""), width, height); + return 0; +} +cv::Mat mpipe_port_getNextFrame() { + + RDK_LOG(RDK_LOG_INFO,"LOG.RDK.SMARTTHUMBNAIL","%s(%d) In mpipe_port_getNextFrame.\n", __FUNCTION__ , __LINE__); + cv::Mat cv_frame, croppedObj; + smTnInstance->mpipeProcessedframes++; + +#ifdef ENABLE_TEST_HARNESS + smTnInstance -> waitForNextDetectionFrame(); + cv_frame = cv::Mat(smTnInstance ->curr_frame.rows, smTnInstance ->curr_frame.cols, CV_8UC3); + cv::cvtColor(smTnInstance ->curr_frame, cv_frame, cv::COLOR_BGR2RGB); + smTnInstance -> detectionTstamp = smTnInstance -> currTstamp; +#else + + smTnInstance -> hres_y_height = smTnInstance -> frameInfo->height; + smTnInstance -> hres_y_width = smTnInstance -> frameInfo->width; + smTnInstance -> hres_y_size = smTnInstance -> frameInfo->width * smTnInstance -> frameInfo->height; + smTnInstance -> hres_uv_size = smTnInstance -> frameInfo->width * smTnInstance -> frameInfo->height/2; + int alloc_size = smTnInstance -> hres_y_size + smTnInstance -> hres_uv_size; + + if(smTnInstance -> mpipe_hres_yuvData == NULL){ + smTnInstance -> mpipe_hres_yuvData = (unsigned char *) malloc(alloc_size * sizeof(unsigned char)); + }else if(sizeof(smTnInstance -> mpipe_hres_yuvData) != (alloc_size* sizeof(unsigned char))) { + smTnInstance -> mpipe_hres_yuvData = (unsigned char *) realloc(smTnInstance -> mpipe_hres_yuvData, alloc_size * sizeof(unsigned char)); + } + if( NULL == smTnInstance -> mpipe_hres_yuvData){ + RDK_LOG(RDK_LOG_INFO,"LOG.RDK.SMARTTHUMBNAIL","%s(%d) malloc failed for hres_yuvData.\n", __FUNCTION__ , __LINE__); + return cv_frame; + } + + memset( smTnInstance -> mpipe_hres_yuvData, 0, (smTnInstance -> hres_y_size + smTnInstance -> hres_uv_size) * sizeof(unsigned char) ); + memcpy( smTnInstance -> mpipe_hres_yuvData, smTnInstance -> frameInfo->y_addr, smTnInstance -> hres_y_size); + memcpy( smTnInstance -> mpipe_hres_yuvData + smTnInstance -> hres_y_size, smTnInstance -> frameInfo->uv_addr, smTnInstance -> hres_uv_size); + + cv::Mat next_frame = cv::Mat(smTnInstance -> frameInfo -> height + (smTnInstance -> frameInfo -> height)/2, smTnInstance -> frameInfo -> width, CV_8UC1, smTnInstance -> mpipe_hres_yuvData); + + cv_frame = cv::Mat(next_frame.rows, next_frame.cols, CV_8UC3); + cv::cvtColor(next_frame, cv_frame, cv::COLOR_YUV2RGB_NV12); + next_frame.release(); +#endif + + double scaleFactor = 1; + cv::Size cropSize = smTnInstance -> getCropSize(smTnInstance -> currentBbox, smTnInstance -> sTnWidth, smTnInstance -> sTnHeight, &scaleFactor); + //Resize the union blob with scaleFactor + smTnInstance -> currentBbox.width = smTnInstance -> currentBbox.width/scaleFactor; + smTnInstance -> currentBbox.height = smTnInstance -> currentBbox.height/scaleFactor; + smTnInstance -> currentBbox.x = smTnInstance -> currentBbox.x/scaleFactor; + smTnInstance -> currentBbox.y = smTnInstance -> currentBbox.y/scaleFactor; + cv::Size rescaleSize = cv::Size(cv_frame.cols/scaleFactor, cv_frame.rows/scaleFactor); + //resize the frame to fit the union blob in the thumbnail + cv::resize(cv_frame, cv_frame, rescaleSize); + cv::Point2f orgCenter = smTnInstance -> getActualCentroid(smTnInstance -> currentBbox); + cv::Point2f allignedCenter = smTnInstance -> alignCentroid(orgCenter, cv_frame, cropSize); + getRectSubPix(cv_frame, cropSize, allignedCenter, croppedObj); + +#ifdef ENABLE_TEST_HARNESS + RDK_LOG(RDK_LOG_INFO,"LOG.RDK.SMARTTHUMBNAIL","%s(%d) Sending %dth frame for detection in %dth file in THlist.\n", __FILE__, __LINE__, smTnInstance -> FrameNum, smTnInstance -> FileNum); +#endif + RDK_LOG(RDK_LOG_INFO,"LOG.RDK.SMARTTHUMBNAIL","%s(%d)The motion blob in the frame is : x = %d, y = %d, w = %d, h = %d\n", __FILE__, __LINE__, smTnInstance -> currentBbox.x, smTnInstance -> currentBbox.y, smTnInstance -> currentBbox.width, smTnInstance -> currentBbox.height); + RDK_LOG(RDK_LOG_INFO,"LOG.RDK.SMARTTHUMBNAIL","%s(%d)Cropped Frame center [x = %f, y = %f].\n", __FILE__, __LINE__, allignedCenter.x, allignedCenter.y); + RDK_LOG(RDK_LOG_INFO,"LOG.RDK.SMARTTHUMBNAIL","%s(%d)Cropped Frame coordinates [x = %f, y = %f, w = %d, h = %d].\n", __FILE__, __LINE__, allignedCenter.x - (cropSize.width / 2), allignedCenter.y - (cropSize.height / 2), cropSize.width, cropSize.height); + + uint8_t *camera_buf = new uint8_t [cropSize.height * cropSize.width * 3]; + + memcpy(camera_buf, croppedObj.data, cropSize.height*cropSize.width * 3); + cv::Mat cv_frame_dst = cv::Mat(croppedObj.rows, croppedObj.cols, CV_8UC3, camera_buf); + + cv_frame.release(); + croppedObj.release(); + + + return (cv_frame_dst); +} + +void loadDetectionConfig(DetectionConfig *config, char *configFile) +{ + FileUtils m_settings; + + config->input_video_path = DEFAULT_INPUT_DEV; + config->delivery_detection_graph_path = DEFAULT_GRAPH_PATH; + config->frame_read_delay = DEFAULT_FRAME_READ_DELAY; + config->max_num_frames_cached_for_delivery_detection = DEFAULT_MAX_FRAMES_CACHED_FOR_DELIVERY_DETECTION; + config->delivery_detection_model_min_score_threshold = DEFAULT_DELIVERY_DETECTION_MODEL_MIN_SCORE_THRESHOLD; + config->delivery_detection_min_score_threshold = DEFAULT_DELIVERY_DETECTION_MIN_SCORE_THRESHOLD; + config->frame_count_to_process = DEFAULT_FRAME_COUNT_TO_PROCESS; + + if(!m_settings.loadFromFile(std::string(configFile))) { + RDK_LOG(RDK_LOG_WARN,"LOG.RDK.THUMBNAILUPLOAD","%s(%d): Loading detection config file failed, loading default settings\n", __FILE__, __LINE__); + } else + { + m_settings.get("input_video_path", config->input_video_path); + m_settings.get("delivery_detection_graph_path", config->delivery_detection_graph_path); + m_settings.get("frame_read_delay", config->frame_read_delay); + m_settings.get("max_num_frames_cached_for_delivery_detection", config->max_num_frames_cached_for_delivery_detection); + m_settings.get("delivery_detection_model_min_score_threshold", config->delivery_detection_model_min_score_threshold); + m_settings.get("delivery_detection_min_score_threshold", config->delivery_detection_min_score_threshold); + m_settings.get("frame_count_to_process", config->frame_count_to_process); + } +} + +void mpipe_port_term() { +} + +void *__mpipe_thread_main__() { + std::string argv0 = __FUNCTION__; + + DetectionConfig config; + + loadDetectionConfig(&config, DETECTION_CONFIG_FILE); + std::string argv1 = "--input_video_path=\"" + config.input_video_path + "\""; + std::string argv2 = "--delivery_detection_graph_path=" + config.delivery_detection_graph_path; + std::string argv3 = "--frame_read_delay=" + config.frame_read_delay; + std::string argv4 = "--max_num_frames_cached_for_delivery_detection=" + config.max_num_frames_cached_for_delivery_detection; + std::string argv5 = "--delivery_detection_model_min_score_threshold=" + config.delivery_detection_model_min_score_threshold; + std::string argv6 = "--delivery_detection_min_score_threshold=" + config.delivery_detection_min_score_threshold; + std::string argv7 = "--frame_count_to_process=" + config.frame_count_to_process; + + std::vector vargv0(argv0.begin(), argv0.end()); + std::vector vargv1(argv1.begin(), argv1.end()); + std::vector vargv2(argv2.begin(), argv2.end()); + std::vector vargv3(argv3.begin(), argv3.end()); + std::vector vargv4(argv4.begin(), argv4.end()); + std::vector vargv5(argv5.begin(), argv5.end()); + std::vector vargv6(argv6.begin(), argv6.end()); + std::vector vargv7(argv7.begin(), argv7.end()); + + char *argv[] = { vargv0.data(), vargv1.data(), vargv2.data(), vargv3.data(), vargv4.data(), vargv5.data(), vargv6.data(), vargv7.data()}; + + extern int __mpipe_main__(int, char **); + __mpipe_main__(sizeof(argv)/sizeof(argv[0]), argv); + return NULL; +} + diff --git a/smart_thumbnail_cvr_lite/smart_thumbnail.cpp b/smart_thumbnail_cvr_lite/smart_thumbnail.cpp index a9b47c2..8aab0a5 100644 --- a/smart_thumbnail_cvr_lite/smart_thumbnail.cpp +++ b/smart_thumbnail_cvr_lite/smart_thumbnail.cpp @@ -62,6 +62,123 @@ char SmartThumbnail::firmwareName[FW_NAME_MAX_LENGTH] = {0}; int32_t SmartThumbnail::event_quiet_time = STN_DEFAULT_EVT_QUIET_TIME; #endif +#ifdef _OBJ_DETECTION_ +#ifdef ENABLE_TEST_HARNESS + +void SmartThumbnail::waitForNextDetectionFrame() +{ + std::unique_lock lock(hres_data_lock); + detectionCv.wait(lock, [this] {return ((currTstamp >= (detectionTstamp + 100))|| (detectionTstamp == 0));}); + lock.unlock(); +} + +void SmartThumbnail::notifyXvision(const DetectionResult &result) +{ + rtMessage m; + rtMessage_Create(&m); + rtMessage_SetInt32(m, "FileNum", THFileNum); + rtMessage_SetInt32(m, "FrameNum", THFrameNum); + rtMessage_SetDouble(m, "deliveryConfidence", result.deliveryScore); + + for(int i = 0; i < result.personBBoxes.size(); i++) { + rtMessage personInfo; + rtMessage_Create(&personInfo); + rtMessage_SetInt32(personInfo, "boundingBoxXOrd", result.personBBoxes[i][0]); + rtMessage_SetInt32(personInfo, "boundingBoxYOrd", result.personBBoxes[i][1]); + rtMessage_SetInt32(personInfo, "boundingBoxWidth", result.personBBoxes[i][2]); + rtMessage_SetInt32(personInfo, "boundingBoxHeight", result.personBBoxes[i][3]); + rtMessage_SetDouble(personInfo, "confidence", result.personScores[i]*100); + rtMessage_AddMessage(m, "Persons", personInfo); + rtMessage_Release(personInfo); + } + + rtError err = rtConnection_SendMessage(connectionSend, m, "RDKC.TESTHARNESS.DELIVERYDATA"); + rtLog_Debug("SendRequest:%s", rtStrError(err)); + + if (err != RT_OK) + { + RDK_LOG( RDK_LOG_ERROR,"LOG.RDK.VIDEOANALYTICS","%s(%d) Error sending msg via rtmessage\n", __FILE__,__LINE__); + } + rtMessage_Release(m); + +} +#endif + +extern SmartThumbnail *smTnInstance; + +void callback_func(const DetectionResult &result) +{ + RDK_LOG( RDK_LOG_DEBUG,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): Got data result callback\n", __FUNCTION__, __LINE__); + RDK_LOG( RDK_LOG_DEBUG,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): No of persons detected : %d\n", __FUNCTION__, __LINE__, result.personScores.size()); + mpipe_port_onMotionEvent(false); + smTnInstance->onCompletedDeliveryDetection(result); +} + +void SmartThumbnail::onCompletedDeliveryDetection(const DetectionResult &result) +{ + gettimeofday(&(smartThInst->detectionEndTime), NULL); + + double time_taken = 0, time_waited = 0; + + time_taken = (smartThInst->detectionEndTime.tv_sec - smartThInst->detectionStartTime.tv_sec) * 1e6; + time_taken = (time_taken + (smartThInst->detectionEndTime.tv_usec - smartThInst->detectionStartTime.tv_usec)) * 1e-6; + if((smartThInst->uploadTriggeredTime.tv_sec != 0) || (smartThInst->uploadTriggeredTime.tv_usec != 0)) { + time_waited = (smartThInst->detectionEndTime.tv_sec - smartThInst->uploadTriggeredTime.tv_sec) * 1e6; + time_waited = (time_waited + (smartThInst->detectionEndTime.tv_usec - smartThInst->uploadTriggeredTime.tv_usec)) * 1e-6; + } + memset(&(smartThInst -> uploadTriggeredTime), 0, sizeof(smartThInst -> uploadTriggeredTime)); + RDK_LOG( RDK_LOG_INFO,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): Detection Stats:%0.2f,%d,%d,0,%d,%0.2lf,%0.2lf\n", __FUNCTION__, __LINE__, result.deliveryScore, result.maxAugScore, result.personScores.size(), mpipeProcessedframes, time_taken, time_waited); + +#ifdef ENABLE_TEST_HARNESS + smartThInst->notifyXvision(result); +#endif + smartThInst->updateUploadPayload(result); + + smartThInst->setDeliveryDetectionCompleted(true); + +#ifdef _HAS_DING_ + smartThInst->setUploadStatus(true); +#endif + smTnInstance->detectionInProgress = false; + +} + +/** @description: Checks if the feature is enabled via RFC + * @param[in] rfc_feature_fname: RFC feature filename + * @param[in] plane1: RFC parameter name + * @return: bool + */ +static bool check_enabled_rfc_feature(char* rfc_feature_fname, char* feature_name) +{ + /* set cvr audio through RFC files */ + char value[MAX_SIZE] = {0}; + + if((NULL == rfc_feature_fname) || + (NULL == feature_name)) { + return false; + } + + /* Check if RFC configuration file exists */ + if( RDKC_SUCCESS == IsRFCFileAvailable(rfc_feature_fname)) { + /* Get the value from RFC file */ + if( RDKC_SUCCESS == GetValueFromRFCFile(rfc_feature_fname, feature_name, value) ) { + if( strcmp(value, RDKC_TRUE) == 0) { + RDK_LOG( RDK_LOG_INFO,"LOG.RDK.XCV","%s(%d): %s is enabled via RFC.\n",__FILE__, __LINE__, feature_name); + return true; + } else { + RDK_LOG( RDK_LOG_INFO,"LOG.RDK.XCV","%s(%d): %s is disabled via RFC.\n",__FILE__, __LINE__, feature_name); + return false; + } + } + /* If RFC file is not present, disable the featur */ + } else { + RDK_LOG( RDK_LOG_INFO,"LOG.RDK.XCV","%s(%d): rfc feature file %s is not present.\n",__FILE__, __LINE__, rfc_feature_fname); + return false; + } +} + +#endif + /** @description: Constructor * @param[in] void * @return: void @@ -95,6 +212,11 @@ SmartThumbnail::SmartThumbnail():dnsCacheTimeout(STN_DEFAULT_DNS_CACHE_TIMEOUT), cvrEnabled(false), prev_time(0), maxBboxArea(0), +#ifdef _OBJ_DETECTION_ + detectionCompleted(false), + mpipe_hres_yuvData(NULL), + detectionInProgress(false), +#endif event_quiet_time(STN_DEFAULT_EVT_QUIET_TIME), logMotionEvent(true), logROIMotionEvent(true), @@ -106,6 +228,15 @@ SmartThumbnail::SmartThumbnail():dnsCacheTimeout(STN_DEFAULT_DNS_CACHE_TIMEOUT), memset(modelName, 0, sizeof(modelName)); memset(macAddress, 0, sizeof(macAddress)); memset(firmwareName, 0, sizeof(firmwareName)); +#ifdef XHB1 + /* Adding the below two lines to hardcode the thumbnail size to 400x300 */ + sTnWidth = 400; + sTnHeight = 300; +#endif + +#ifdef _OBJ_DETECTION_ + memset(&uploadTriggeredTime, 0, sizeof(uploadTriggeredTime)); +#endif #ifdef _HAS_XSTREAM_ #ifndef _DIRECT_FRAME_READ_ frameHandler = {NULL, -1}; @@ -145,11 +276,21 @@ SmartThumbnail *SmartThumbnail::getInstance() * @param[in] void * @return: STH_SUCCESS on success, STH_ERROR otherwise */ -STH_STATUS SmartThumbnail::init(char* mac,bool isCvrEnabled) +STH_STATUS SmartThumbnail::init(char* mac,bool isCvrEnabled, bool isDetectionEnabled) { int ret= STH_SUCCESS; char usrVal[CONFIG_STRING_MAX]; +#ifdef ENABLE_TEST_HARNESS + //initializing config Manager + if (STH_SUCCESS != config_init()) { + RDK_LOG(RDK_LOG_ERROR,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): Error loading config manager.\n", __FILE__, __LINE__); + return STH_ERROR; + } + + testHarnessOnFileFeed = (strcmp(rdkc_envGet(TEST_HARNESS_ON_FILE_ENABLED), "true") == 0) ? true : false; +#endif + if(STH_SUCCESS == getTnUploadConf()) { RDK_LOG( RDK_LOG_INFO,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): getTnUploadConf success!!", __FUNCTION__, __LINE__); } @@ -185,11 +326,21 @@ STH_STATUS SmartThumbnail::init(char* mac,bool isCvrEnabled) } //Initialize +#ifdef XHB1 + buf_id = STN_MRES_BUFFER_ID; + /* Commenting the below check to hardcode thumbnail resolution to 400x300 */ +/* if((sTnWidth == STN_HRES_CROP_WIDTH) && (sTnHeight ==STN_HRES_CROP_HEIGHT)) { + buf_id = STN_HRES_BUFFER_ID; + }*/ +#else + buf_id = STN_HRES_BUFFER_ID; +#endif + #ifdef _DIRECT_FRAME_READ_ - ret = consumer->RAWInit((u16)STN_HRES_BUFFER_ID); + ret = consumer->RAWInit((u16)buf_id); if( ret < 0){ #else - frameHandler = consumer->RAWInit(STN_HRES_BUFFER_ID, FORMAT_YUV, 0); + frameHandler = consumer->RAWInit(buf_id, FORMAT_YUV, 0); if (frameHandler.sockfd < 0){ #endif RDK_LOG(RDK_LOG_ERROR,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): StreamInit failed.\n", __FUNCTION__, __LINE__); @@ -236,6 +387,25 @@ STH_STATUS SmartThumbnail::init(char* mac,bool isCvrEnabled) RDK_LOG(RDK_LOG_ERROR,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): Failed to open the URL\n", __FUNCTION__, __LINE__); } +#ifdef _OBJ_DETECTION_ + if( RDKC_SUCCESS != RFCConfigInit() ) + { + RDK_LOG( RDK_LOG_ERROR,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): RFC config Init fails\n", __FILE__, __LINE__); + return STH_ERROR; + } + //detectionEnabled = check_enabled_rfc_feature(RFCFILE, DELIVERY_DETECTION_RFC); + detectionEnabled = isDetectionEnabled; + + if(detectionEnabled) { + //Initialize delivery detection thread + RDK_LOG( RDK_LOG_INFO,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): Creating delivery detection thread.\n", __FUNCTION__, __LINE__); + std::thread deliveryDetectionThread(__mpipe_thread_main__); + deliveryDetectionThread.detach(); + + mpipe_port_setOnDetectionChanged(callback_func); + } +#endif + //Initializing thread to listen to incoming messages RDK_LOG(RDK_LOG_INFO,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): Creating rtMessage receive thread.\n", __FUNCTION__, __LINE__); rtMsgInit(); @@ -285,6 +455,12 @@ STH_STATUS SmartThumbnail::getTnUploadConf() sTnHeight = atoi(stnCfg -> height); sTnWidth = atoi(stnCfg -> width); +#ifdef XHB1 + /* Adding the below two lines to hardcode the thumbnail size to 400x300 */ + sTnWidth = 400; + sTnHeight = 300; +#endif + if (stnCfg) { free(stnCfg); stnCfg = NULL; @@ -420,7 +596,7 @@ STH_STATUS SmartThumbnail::createPayload() { //Acquire lock std::unique_lock lock(smartThInst -> QMutex); - if (smartThInst -> isPayloadAvailable) + if (smartThInst -> isPayloadAvailable ) { memset(&payload,0,sizeof(payload)); RDK_LOG(RDK_LOG_INFO,"LOG.RDK.SMARTTHUMBNAIL","%s(%d):Payload is available.\n", __FILE__, __LINE__); @@ -428,8 +604,12 @@ STH_STATUS SmartThumbnail::createPayload() // matrix to store color image lHresRGBMat = cv::Mat(hres_y_height, hres_y_width, CV_8UC4); +#ifdef ENABLE_TEST_HARNESS + lHresRGBMat = ofData.maxBboxObjYUVFrame.clone(); +#else // convert the frame to BGR format cv::cvtColor(smartThInst -> ofData.maxBboxObjYUVFrame,lHresRGBMat, cv::COLOR_YUV2BGR_NV12); +#endif unionBox.x = smartThInst -> ofData.boundingBoxXOrd; unionBox.y = smartThInst -> ofData.boundingBoxYOrd; @@ -438,8 +618,21 @@ STH_STATUS SmartThumbnail::createPayload() // extracted the below logic from server scala code RDK_LOG( RDK_LOG_INFO,"LOG.RDK.SMARTTHUMBNAIL","%s(%d):unionBox.x %d unionBox.y %d unionBox.height %d unionBox.width %d\n", __FILE__, __LINE__, unionBox.x, unionBox.y, unionBox.height, unionBox.width); + /* ScaleFactor for downsizing the frame before cropping the thumbnail. If the + * union blob is bigger than the thumbnail size, downsize the frame to fit the + * union blob in thumbnail */ + + double scaleFactor = 1; + cv::Size cropSize = getCropSize(unionBox, sTnWidth, sTnHeight, &scaleFactor); + cv::Size rescaleSize = cv::Size(lHresRGBMat.cols/scaleFactor, lHresRGBMat.rows/scaleFactor); + //resize the frame to fit the union blob in the thumbnail + cv::resize(lHresRGBMat, lHresRGBMat, rescaleSize); + //Resize the union blob also accordingly + unionBox.width = unionBox.width/scaleFactor; + unionBox.height = unionBox.height/scaleFactor; + unionBox.x = unionBox.x/scaleFactor; + unionBox.y = unionBox.y/scaleFactor; cv::Point2f orgCenter = getActualCentroid(unionBox); - cv::Size cropSize = getCropSize(unionBox, sTnWidth, sTnHeight); cv::Point2f allignedCenter = alignCentroid(orgCenter, lHresRGBMat, cropSize); getRectSubPix(lHresRGBMat, cropSize, allignedCenter, croppedObj); relativeBBox = getRelativeBoundingBox(unionBox, cropSize, allignedCenter); @@ -461,6 +654,25 @@ STH_STATUS SmartThumbnail::createPayload() //Write smart thumbnail to file. imwrite(uploadFname,croppedObj); #endif +#ifdef _OBJ_DETECTION_ +#if 0 +// if(detectionEnabled) { + cv::Mat cv_frame_rgb; + cv::cvtColor(croppedObj, cv_frame_rgb, cv::COLOR_BGR2RGB); + uint8_t *camera_buf = (uint8_t *) malloc(croppedObj.cols * croppedObj.rows * 3); + memcpy(camera_buf, cv_frame_rgb.data, croppedObj.cols * croppedObj.rows * 3); + cv::Mat cv_frame_dst = cv::Mat(croppedObj.rows, croppedObj.cols, CV_8UC3, camera_buf); + mpipe_port_onThunbmailEvent(cv_frame_dst, croppedObj.cols, croppedObj.rows); + cv_frame_rgb.release(); + //mpipe_port_onThunbmailEvent(croppedObj, sTnWidth, sTnHeight); + //RDK_LOG( RDK_LOG_INFO,"LOG.RDK.SMARTTHUMBNAIL","%s(%d):notifying mpipe_port_onThunbmailEvent .\n", __FILE__, __LINE__); + // } +#endif + if(smartThInst->detectionInProgress) { + cv::Mat emptyMat; + mpipe_port_onLastFrameEvent(emptyMat, 0, 0); + } +#endif //reset payload flag smartThInst -> isPayloadAvailable = false; @@ -497,6 +709,57 @@ STH_STATUS SmartThumbnail::createPayload() return ret; } +#ifdef _OBJ_DETECTION_ +json_t* SmartThumbnail::createJSONFromDetectionResult(DetectionResult result) +{ + json_t* resultJson = json_object(); + json_t* tags_array = json_array(); + std::ostringstream statStringStream; + + if(result.deliveryScore != 0) { + RDK_LOG(RDK_LOG_DEBUG,"LOG.RDK.SMARTTHUMBNAIL","%s(%d) Delivery detected with confidence: %f\n", __FUNCTION__ , __LINE__, result.deliveryScore); + json_t* object = json_object(); + json_object_set_new(object, "name", json_string("Delivery")); + json_object_set_new(object, "confidence", json_real(result.deliveryScore)); + json_array_append_new(tags_array, object); + } + int i = 0; + statStringStream << result.personScores.size(); + for(std::vector::iterator itr = result.personScores.begin(); itr < result.personScores.end(); itr++) { + statStringStream << "," < lock(smartThInst -> QMutex); + payload.detectionResult = createJSONFromDetectionResult(result); + //json_decref(root); + ret = STH_SUCCESS; + RDK_LOG(RDK_LOG_DEBUG,"LOG.RDK.SMARTTHUMBNAIL","%s(%d) payload updated with json string\n", __FUNCTION__ , __LINE__); + + lock.unlock(); + return ret; + +} +#endif + /** @description: find the actual centroid of the bounding rectangle * @param[in] bounding rectangle * @return: central point {x,y} @@ -550,9 +813,10 @@ cv::Point2f SmartThumbnail::alignCentroid(cv::Point2f orgCenter, cv::Mat origFra * @param[in] bounding rectangle * @param[in] minimum width of the crop window * @param[in] minimum height of the crop window + * @param[out] the rescale factor of the frame if the union blob is greater than thumbnail size * @return: size of the crop window */ -cv::Size SmartThumbnail::getCropSize(cv::Rect boundRect,double w,double h) { +cv::Size SmartThumbnail::getCropSize(cv::Rect boundRect,double w,double h, double *resizeScale) { int newWidth = 0; int newHeight = 0; int adjustFactor = 1; @@ -560,6 +824,7 @@ cv::Size SmartThumbnail::getCropSize(cv::Rect boundRect,double w,double h) { cv::Size sz; newWidth = boundRect.width; newHeight = boundRect.height; + *resizeScale = 1; #if 0 if (boundRect.width > (boundRect.height *(w/h))) { @@ -574,10 +839,21 @@ cv::Size SmartThumbnail::getCropSize(cv::Rect boundRect,double w,double h) { newHeight = (int)(boundRect.width * (h/w)); } #endif - +#if 0 // always ensure the minimum size of the crop size window is {w, h} sz.height = STN_MAX(h,(newHeight* adjustFactor)); sz.width = STN_MAX(w,(newWidth* adjustFactor)); +#endif + sz.height = h; + sz.width = w; + /* As per RDKC-10175, to crop the thumbnail from the frame, where the union blob size is + * greater than the thumbnail size, the frame resized so that the union blob fits in the + * thumbnail. */ + + if((boundRect.width > w) || (boundRect.height > h)) { + // Calculate the resizing scale for the frame + *resizeScale = STN_MAX(boundRect.width/w, boundRect.height/h); + } return sz; } @@ -674,6 +950,13 @@ STH_STATUS SmartThumbnail::destroy() consumer = NULL; } +#ifdef _OBJ_DETECTION_ + if(smTnInstance -> mpipe_hres_yuvData){ + free(smTnInstance -> mpipe_hres_yuvData); + smTnInstance -> mpipe_hres_yuvData = NULL; + } +#endif + frameInfo = NULL; #ifndef _DIRECT_FRAME_READ_ frameHandler.curl_handle = NULL; @@ -735,7 +1018,9 @@ void SmartThumbnail::onDingNotification(rtMessageHeader const* hdr, uint8_t cons smartThInst ->m_dingTime = currTime.tv_sec; smartThInst -> m_dingNotif = true; smartThInst->m_ding->signalDing(true,smartThInst->m_dingTime); +#ifndef _OBJ_DETECTION_ smartThInst->setUploadStatus(true); +#endif } } rtMessage_Release(req); @@ -777,6 +1062,37 @@ bool SmartThumbnail::waitFor(int quiteInterVal) return isTimedOut; } +#endif +#ifdef _OBJ_DETECTION_ +STH_STATUS SmartThumbnail::setDeliveryDetectionCompleted(bool status) +{ + { + std::unique_lock lock(deliveryDetectionMutex); + detectionCompleted = status; + lock.unlock(); + } + + detection_cv.notify_one(); + return STH_SUCCESS; +} + +bool SmartThumbnail::getDeliveryDetectionStatus() +{ + bool status = false; + + { + std::unique_lock lock(deliveryDetectionMutex); + detection_cv.wait(lock, [this] {return (detectionCompleted);}); + + RDK_LOG( RDK_LOG_TRACE1,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): Wait over due detectionCompleted flag!!\n",__FUNCTION__,__LINE__); + status = detectionCompleted; + detectionCompleted = false; + + lock.unlock(); + } + + return status; +} #endif /** @description: Callback function for dynamic logging * @param[in] hdr : pointer to rtMessage Header @@ -847,6 +1163,10 @@ void SmartThumbnail::onMsgCaptureFrame(rtMessageHeader const* hdr, uint8_t const rtMessage_FromBytes(&m, buff, n); rtMessage_GetInt32(m, "processID", &processPID); rtMessage_GetString(m, "timestamp", &strFramePTS); +#ifdef ENABLE_TEST_HARNESS + rtMessage_GetInt32(m, "fileNum", &(smartThInst -> FileNum)); + rtMessage_GetInt32(m, "frameNum", &(smartThInst -> FrameNum)); +#endif RDK_LOG(RDK_LOG_DEBUG,"LOG.RDK.SMARTTHUMBNAIL","%s(%d) strFramePTS:%s \n", __FUNCTION__ , __LINE__, strFramePTS); RDK_LOG(RDK_LOG_DEBUG,"LOG.RDK.SMARTTHUMBNAIL","%s(%d) capture invoked by process %d \n", __FUNCTION__ , __LINE__, processPID); @@ -857,25 +1177,69 @@ void SmartThumbnail::onMsgCaptureFrame(rtMessageHeader const* hdr, uint8_t const rtMessage_Release(m); +#ifdef ENABLE_TEST_HARNESS + std::string frame_filename; + if(smartThInst->testHarnessOnFileFeed)/*strcmp(rdkc_envGet(TEST_HARNESS_ON_FILE_ENABLED), "true") == 0)*/ { + //Read the frame from file + frame_filename = "/tmp/THFrame_" + std::to_string(lResFramePTS) + ".jpg"; + RDK_LOG(RDK_LOG_INFO,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): Reading frame file : %s\n", __FUNCTION__ , __LINE__, frame_filename.c_str()); + } +#endif RDK_LOG(RDK_LOG_DEBUG,"LOG.RDK.SMARTTHUMBNAIL","%s(%d):prev_time :%d\n", __FUNCTION__ , __LINE__, smartThInst -> prev_time); - +#ifdef _OBJ_DETECTION_ + if(!smartThInst -> detectionEnabled) { +#endif // ignore frames and metadata for event_quiet_time if( (currTime.tv_sec < (smartThInst -> prev_time + smartThInst -> event_quiet_time)) && (0 != smartThInst -> prev_time) ) { RDK_LOG(RDK_LOG_DEBUG,"LOG.RDK.SMARTTHUMBNAIL","%s(%d):Within event quiet time, Ignoring event, time passed: %lu\n", __FUNCTION__ , __LINE__, (currTime.tv_sec- smartThInst -> prev_time)); +//#ifdef ENABLE_TEST_HARNESS +// if(smartThInst->testHarnessOnFileFeed)/*strcmp(rdkc_envGet(TEST_HARNESS_ON_FILE_ENABLED), "true") == 0)*/ { +// unlink(frame_filename.c_str()); +// } +//#endif return; } +#ifdef _OBJ_DETECTION_ + } +#endif RDK_LOG(RDK_LOG_DEBUG,"LOG.RDK.SMARTTHUMBNAIL","%s(%d):Not within event quiet time, Capturing frame.\n", __FUNCTION__ , __LINE__); +#ifdef ENABLE_TEST_HARNESS + std::unique_lock lock(smartThInst->hres_data_lock); +if(smartThInst->testHarnessOnFileFeed) { + smartThInst->curr_frame = cv::imread(frame_filename, cv::IMREAD_COLOR); +#if 0 + + cv::cvtColor(curr_frame, smartThInst->fileFrameYUV, cv::COLOR_BGR2YUV); + split(smartThInst->fileFrameYUV, smartThInst->yuvPlanes); + + smartThInst->yuvChannels.clear(); + + smartThInst->yuvChannels.push_back((smartThInst->yuvPlanes)[1]); + smartThInst->yuvChannels.push_back((smartThInst->yuvPlanes)[2]); + + merge(smartThInst->yuvChannels, smartThInst->planeUV); + + smartThInst->frameInfo->y_addr = (smartThInst->yuvPlanes)[0].data; + smartThInst->frameInfo->width = (smartThInst->yuvPlanes)[0].cols; + smartThInst->frameInfo->height = (smartThInst->yuvPlanes)[0].rows; + smartThInst->frameInfo->uv_addr = smartThInst->planeUV.data; + smartThInst->frameInfo->mono_pts = 0; +#endif + unlink(frame_filename.c_str()); + smartThInst -> isHresFrameReady = true; +} else { +#endif //read the 720*1280 YUV data. std::unique_lock lock(smartThInst -> QMutex); #ifdef _HAS_XSTREAM_ if ((NULL != smartThInst) && (NULL != smartThInst->consumer)){ #ifdef _DIRECT_FRAME_READ_ - ret = smartThInst -> consumer -> ReadRAWFrame((u16)STN_HRES_BUFFER_ID, (u16)FORMAT_YUV, smartThInst -> frameInfo); + ret = smartThInst -> consumer -> ReadRAWFrame((u16)smartThInst->buf_id, (u16)FORMAT_YUV, smartThInst -> frameInfo); #else - ret = smartThInst -> consumer -> ReadRAWFrame(STN_HRES_BUFFER_ID, FORMAT_YUV, smartThInst -> frameInfo); + ret = smartThInst -> consumer -> ReadRAWFrame(smartThInst->buf_id, FORMAT_YUV, smartThInst -> frameInfo); #endif } #else @@ -884,7 +1248,7 @@ void SmartThumbnail::onMsgCaptureFrame(rtMessageHeader const* hdr, uint8_t const } memset(smartThInst -> hres_frame_info, 0, sizeof(RDKC_PLUGIN_YUVInfo)); - ret = smartThInst -> recorder -> ReadYUVData(STN_HRES_BUFFER_ID, smartThInst -> hres_frame_info); + ret = smartThInst -> recorder -> ReadYUVData(smartThInst->buf_id, smartThInst -> hres_frame_info); #endif if( STH_SUCCESS != ret) { @@ -901,6 +1265,13 @@ void SmartThumbnail::onMsgCaptureFrame(rtMessageHeader const* hdr, uint8_t const lock.unlock(); RDK_LOG( RDK_LOG_DEBUG,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): hResframePTS:%llu\n", __FUNCTION__, __LINE__, hResFramePTS); RDK_LOG( RDK_LOG_DEBUG,"LOG.RDK.SMARTTHUMBNAIL","%s(%d):Time gap (hResFramePTS - lResframePTS):%llu\n", __FUNCTION__, __LINE__, (hResFramePTS - lResFramePTS)); +#ifdef ENABLE_TEST_HARNESS + } + + smartThInst -> currTstamp = lResFramePTS; + lock.unlock(); + smartThInst->detectionCv.notify_one(); +#endif } /** @description : Callback function to generate smart thumbnail based on motion. @@ -977,6 +1348,15 @@ void SmartThumbnail::onMsgProcessFrame(rtMessageHeader const* hdr, uint8_t const smartThInst -> ignoreMotion = true; } +#ifdef _OBJ_DETECTION_ + if(smartThInst->detectionEnabled && sm.event_type == 4) { + smartThInst->currentBbox.x = sm.deliveryUnionBox.boundingBoxXOrd; + smartThInst->currentBbox.y = sm.deliveryUnionBox.boundingBoxYOrd; + smartThInst->currentBbox.width = sm.deliveryUnionBox.boundingBoxWidth; + smartThInst->currentBbox.height = sm.deliveryUnionBox.boundingBoxHeight; + } +#endif + // if motion is detected update the metadata. if ((smartThInst -> isHresFrameReady) && //(motionScore != 0.0) && @@ -994,8 +1374,26 @@ void SmartThumbnail::onMsgProcessFrame(rtMessageHeader const* hdr, uint8_t const std::unique_lock lock(smartThInst -> QMutex); if (!smartThInst -> isPayloadAvailable) { RDK_LOG(RDK_LOG_DEBUG,"LOG.RDK.SMARTTHUMBNAIL","%s(%d) Payload will be available for this interval.\n", __FUNCTION__ , __LINE__); +#ifndef _OBJ_DETECTION_ //To indicate payload will there to upload smartThInst -> isPayloadAvailable = true; +#else + if(!smartThInst -> detectionEnabled) { + //To indicate payload will there to upload + smartThInst -> isPayloadAvailable = true; + } else if(!smartThInst -> detectionInProgress) { + smartThInst -> isPayloadAvailable = true; + smartThInst->mpipeProcessedframes = 0; + mpipe_port_onMotionEvent(true); + gettimeofday(&(smartThInst -> detectionStartTime), NULL); +#ifdef ENABLE_TEST_HARNESS + smartThInst -> detectionTstamp = 0; + smartThInst -> THFileNum = smartThInst -> FileNum; + smartThInst -> THFrameNum = smartThInst -> FrameNum; +#endif + smartThInst -> detectionInProgress = true; + } +#endif } updateObjFrameData(sm.unionBox.boundingBoxXOrd, sm.unionBox.boundingBoxYOrd, sm.unionBox.boundingBoxWidth, sm.unionBox.boundingBoxHeight, curr_time); @@ -1093,6 +1491,11 @@ void SmartThumbnail::updateObjFrameData(int32_t boundingBoxXOrd,int32_t bounding smartThInst -> ofData.boundingBoxHeight = boundingBoxHeight; smartThInst -> ofData.currTime = currTime; +#ifdef ENABLE_TEST_HARNESS + if(smartThInst -> testHarnessOnFileFeed) { + smartThInst -> ofData.maxBboxObjYUVFrame = smartThInst -> curr_frame.clone(); + } else { +#endif #ifdef _HAS_XSTREAM_ smartThInst -> hres_y_height = smartThInst -> frameInfo->height; smartThInst -> hres_y_width = smartThInst -> frameInfo-> width; @@ -1121,6 +1524,10 @@ void SmartThumbnail::updateObjFrameData(int32_t boundingBoxXOrd,int32_t bounding smartThInst -> ofData.maxBboxObjYUVFrame = cv::Mat(smartThInst -> hres_frame_info -> height + (smartThInst -> hres_frame_info -> height)/2, smartThInst -> hres_frame_info -> width, CV_8UC1, hres_yuvData).clone(); #endif +#ifdef ENABLE_TEST_HARNESS + } +#endif + if(hres_yuvData) { free(hres_yuvData); hres_yuvData = NULL; @@ -1194,6 +1601,10 @@ int SmartThumbnail::uploadPayload(time_t timeLeft) int dataLen = 0; //std::vector dataPtr; //dataPtr.reserve(STN_DATA_MAX_SIZE); +#endif +#ifdef _OBJ_DETECTION_ + char encodedBuff[512]; + gettimeofday(&(smartThInst -> uploadTriggeredTime), NULL); #endif char packHead[STN_UPLOAD_SEND_LEN+1]; int retry = 0; @@ -1329,6 +1740,31 @@ int SmartThumbnail::uploadPayload(time_t timeLeft) strcpy(objectBoxsBuf, packHead); smartThInst->httpClient->addHeader("X-BoundingBoxes", packHead); +#ifdef _OBJ_DETECTION_ + if(smartThInst -> detectionEnabled) { + struct timeval start, end; + gettimeofday(&start, NULL); + smartThInst->getDeliveryDetectionStatus(); + + gettimeofday(&end, NULL); + double time_taken; + time_taken = (end.tv_sec - start.tv_sec) * 1e6; + time_taken = (time_taken + (end.tv_usec - start.tv_usec)) * 1e-6; + RDK_LOG( RDK_LOG_INFO,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): Upload waited %lf seconds for delivery detection!!!\n",__FUNCTION__,__LINE__, time_taken); + char *jsonStr = json_dumps(payload.detectionResult, 0); + RDK_LOG( RDK_LOG_INFO,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): X-IMAGE-METADATA: %s\n", __FUNCTION__, __LINE__, jsonStr); + if(jsonStr == NULL) { + jsonStr = "{\"tags\": []}"; + } + memset(encodedBuff, 0, sizeof(encodedBuff)); + b64_encode((uint8_t*)jsonStr, strlen(jsonStr), (uint8_t*)encodedBuff); + RDK_LOG( RDK_LOG_DEBUG,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): encoded X-IMAGE-METADATA: %s\n", __FUNCTION__, __LINE__, (char*)encodedBuff); + smartThInst->httpClient->addHeader("X-IMAGE-METADATA", encodedBuff); + free(jsonStr); + json_decref(payload.detectionResult); + } +#endif + memset(packHead, 0, sizeof(packHead)); if(smartThInst->cvrEnabled) { @@ -1581,6 +2017,13 @@ void SmarttnMetadata_thumb::from_rtMessage(SmarttnMetadata_thumb *smInfo, const rtMessage_GetInt32(m, "boundingBoxWidth", &(smInfo->unionBox.boundingBoxWidth)); rtMessage_GetInt32(m, "boundingBoxHeight", &(smInfo->unionBox.boundingBoxHeight)); +#ifdef _OBJ_DETECTION_ + rtMessage_GetInt32(m, "d_boundingBoxXOrd", &(smInfo->deliveryUnionBox.boundingBoxXOrd)); + rtMessage_GetInt32(m, "d_boundingBoxYOrd", &(smInfo->deliveryUnionBox.boundingBoxYOrd)); + rtMessage_GetInt32(m, "d_boundingBoxWidth", &(smInfo->deliveryUnionBox.boundingBoxWidth)); + rtMessage_GetInt32(m, "d_boundingBoxHeight", &(smInfo->deliveryUnionBox.boundingBoxHeight)); +#endif + RDK_LOG( RDK_LOG_DEBUG,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): strFramePTS:%s \n", __FUNCTION__ , __LINE__, smInfo->strFramePTS); RDK_LOG( RDK_LOG_DEBUG,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): event Type: %d\n",__FILE__, __LINE__, smInfo->event_type); RDK_LOG( RDK_LOG_DEBUG,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): motionScore: %f\n",__FILE__, __LINE__, smInfo->motionScore); @@ -1588,6 +2031,12 @@ void SmarttnMetadata_thumb::from_rtMessage(SmarttnMetadata_thumb *smInfo, const RDK_LOG( RDK_LOG_DEBUG,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): boundingBoxYOrd:%d\n", __FILE__, __LINE__, smInfo->unionBox.boundingBoxYOrd); RDK_LOG( RDK_LOG_DEBUG,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): boundingBoxWidth:%d\n", __FILE__, __LINE__, smInfo->unionBox.boundingBoxWidth); RDK_LOG( RDK_LOG_DEBUG,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): boundingBoxHeight:%d\n", __FILE__, __LINE__, smInfo->unionBox.boundingBoxHeight); +#ifdef _OBJ_DETECTION_ + RDK_LOG( RDK_LOG_DEBUG,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): del_boundingBoxXOrd:%d\n", __FILE__, __LINE__, smInfo->deliveryUnionBox.boundingBoxXOrd); + RDK_LOG( RDK_LOG_DEBUG,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): del_boundingBoxYOrd:%d\n", __FILE__, __LINE__, smInfo->deliveryUnionBox.boundingBoxYOrd); + RDK_LOG( RDK_LOG_DEBUG,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): del_boundingBoxWidth:%d\n", __FILE__, __LINE__, smInfo->deliveryUnionBox.boundingBoxWidth); + RDK_LOG( RDK_LOG_DEBUG,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): del_boundingBoxHeight:%d\n", __FILE__, __LINE__, smInfo->deliveryUnionBox.boundingBoxHeight); +#endif RDK_LOG( RDK_LOG_DEBUG,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): curr timestamp:%s\n", __FILE__, __LINE__, smInfo->s_curr_time); rtMessage_GetArrayLength(m, "objectBoxs", &len); @@ -1606,10 +2055,10 @@ void SmarttnMetadata_thumb::from_rtMessage(SmarttnMetadata_thumb *smInfo, const RDK_LOG( RDK_LOG_DEBUG,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): objectBoxs[%d].boundingBoxYOrd:%d\n", __FILE__, __LINE__,i,smInfo->objectBoxs[i].boundingBoxYOrd); RDK_LOG( RDK_LOG_DEBUG,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): objectBoxs[%d].boundingBoxWidth:%d\n", __FILE__, __LINE__,i,smInfo->objectBoxs[i].boundingBoxWidth); RDK_LOG( RDK_LOG_DEBUG,"LOG.RDK.SMARTTHUMBNAIL","%s(%d): objectBoxs[%d].boundingBoxHeight:%d\n", __FILE__, __LINE__,i,smInfo->objectBoxs[i].boundingBoxHeight); + rtMessage_Release(bbox); } - } /* @description: SmarttnMetadata_thumb constructor * @parametar: void