Motion Vector Extraction from H.264 Encoded Bitstream without frame reconstruction(CABAC Only)

codec/api/wels/codec_api.h

@@ -45,6 +45,7 @@ typedef unsigned char bool;
 
 #include "codec_app_def.h"
 #include "codec_def.h"
+#include <stdint.h>
 
 #if defined(_WIN32) || defined(__cdecl)
 #define EXTAPI __cdecl
@@ -451,6 +452,43 @@ class ISVCDecoder {
  int& iHeight,
  int& iColorFormat) = 0;
 
+ /**
+ * @brief parse bitstream and get motion vectors - skip frame reconstruction
+ * @param pSrc the h264 stream to be decoded
+ * @param iSrcLen the length of h264 stream
+ * @param ppDst buffer pointer of decoded data (YUV)
+ * @param pDstInfo information provided to API(width, height, etc.)
+ * @param MotionVectorSize size of the total motion vector for the given P frame.
+ * @param MotionVectorData Motion vector data.(ex: MotionX, MotionY, Xoffset, Yoffset)
+ * @return 0 - success; otherwise -failed;
+ */
+ virtual DECODING_STATE EXTAPI ParseBitstreamGetMotionVectors (const unsigned char* kpSrc,
+ const int kiSrcLen,
+ unsigned char** ppDst,
+ SParserBsInfo* pDstInfo,
+ SBufferInfo* ppDecodeInfo,
+ int32_t* motionVectorSize,
+ int16_t** motionVectorData) = 0;
+
+ /**
+ @brief ParseBitstreamGetMotionVectors is used to parse the encoded bitstream and get the motion vectors for the given P frame.
+ * If bParseOnly mode is enabled then only motionVectorData and motionVectorSize is updated and ppDst is returned with NULL Value.
+ * If bParseOnly mode is disabled then along MotionVectorData and motionVectorSize the deocoded YUV buffer is updated in ppDst. 
+ * @param pSrc the h264 stream to be decoded
+ * @param iSrcLen the length of h264 stream
+ * @param ppDst buffer pointer of decoded data (YUV)
+ * @param pDstInfo information provided to API(width, height, etc.)
+ * @param MotionVectorSize size of the total motion vector for the given P frame.
+ * @param MotionVectorData Motion vector data.(ex: MotionX, MotionY, Xoffset, Yoffset)
+ * @return 0 - success; otherwise -failed;
+ */
+ virtual DECODING_STATE EXTAPI DecodeFrameGetMotionVectorsNoDelay (const unsigned char* pSrc,
+ const int iSrcLen,
+ unsigned char** ppDst,
+ SBufferInfo* pDstInfo,
+ int32_t* motionVectorSize,
+ int16_t** motionVectorData) = 0;
+
  /**
  * @brief Set option for decoder, detail option type, please refer to enumurate DECODER_OPTION.
  * @param pOption option for decoder such as OutDataFormat, Eos Flag, EC method, ...
@@ -510,6 +548,21 @@ DECODING_STATE (*DecodeFrameNoDelay) (ISVCDecoder*, const unsigned char* pSrc,
  unsigned char** ppDst,
  SBufferInfo* pDstInfo);
 
+DECODING_STATE (*ParseBitstreamGetMotionVectors) (const unsigned char* kpSrc,
+ const int kiSrcLen,
+ unsigned char** ppDst,
+ SParserBsInfo* pDstInfo,
+ SBufferInfo* ppDecodeInfo,
+ int32_t* motionVectorSize,
+ int16_t** motionVectorData);
+
+DECODING_STATE (*DecodeFrameGetMotionVectorsNoDelay) (const unsigned char* pSrc,
+ const int iSrcLen,
+ unsigned char** ppDst,
+ SBufferInfo* pDstInfo,
+ int32_t* motionVectorSize,
+ int16_t** motionVectorData);
+
 DECODING_STATE (*DecodeFrame2) (ISVCDecoder*, const unsigned char* pSrc,
  const int iSrcLen,
  unsigned char** ppDst,

codec/decoder/core/inc/decoder_context.h

@@ -513,6 +513,8 @@ typedef struct TagWelsDecoderContext {
  CMemoryAlign* pMemAlign;
  void* pThreadCtx;
  void* pLastThreadCtx;
+ int32_t mMotionVectorSize = 0; 
+ int16_t* mMotionVectorData ; 
  WELS_MUTEX* pCsDecoder;
  int16_t lastReadyHeightOffset[LIST_A][MAX_REF_PIC_COUNT]; //last ready reference MB offset
  PPictInfo pPictInfoList;

codec/decoder/core/inc/parse_mb_syn_cabac.h

@@ -84,6 +84,7 @@ void UpdateP8x8DirectCabac (PDqLayer pCurDqLayer, int32_t iPartIdx);
 void UpdateP16x16DirectCabac (PDqLayer pCurDqLayer);
 void UpdateP8x8RefCacheIdxCabac (int8_t pRefIndex[LIST_A][30], const int16_t& iPartIdx, const int32_t& listIdx,
  const int8_t& iRef);
+void UpdateMotionVector (PWelsDecoderContext pCtx , int16_t pMotionX, int16_t pMotionY, int16_t xOffset, int16_t yOffset);
 }
 //#pragma pack()
 #endif

codec/decoder/core/src/decoder_core.cpp

@@ -1587,6 +1587,9 @@ int32_t InitialDqLayersContext (PWelsDecoderContext pCtx, const int32_t kiMaxWid
  pCtx->sMb.pMbRefConcealedFlag[i] = (bool*) pMa->WelsMallocz (pCtx->sMb.iMbWidth * pCtx->sMb.iMbHeight * sizeof (
  bool),
  "pCtx->pMbRefConcealedFlag[]");
+
+ pCtx->mMotionVectorData = (int16_t*) pMa->WelsMallocz (pCtx->iImgWidthInPixel * pCtx->iImgHeightInPixel * 8 ,
+ "pCtx->pMbRefConcealedFlag[]");
 
  // check memory block valid due above allocated..
  WELS_VERIFY_RETURN_IF (ERR_INFO_OUT_OF_MEMORY,
@@ -1645,6 +1648,13 @@ void UninitialDqLayersContext (PWelsDecoderContext pCtx) {
  continue;
  }
 
+ if (pCtx->mMotionVectorData)
+ {
+ pCtx->mMotionVectorData -= pCtx->mMotionVectorSize;
+ pMa->WelsFree (pCtx->mMotionVectorData, "pCtx->mMotionVectorData");
+ pCtx->mMotionVectorData = nullptr;
+ }
+
  if (pCtx->sMb.pMbType[i]) {
  pMa->WelsFree (pCtx->sMb.pMbType[i], "pCtx->sMb.pMbType[]");
 

codec/decoder/core/src/parse_mb_syn_cabac.cpp

@@ -517,6 +517,15 @@ int32_t ParseIntraPredModeChromaCabac (PWelsDecoderContext pCtx, uint8_t uiNeigh
  return ERR_NONE;
 }
 
+void UpdateMotionVector (PWelsDecoderContext pCtx , int16_t pMotionX, int16_t pMotionY, int16_t xOffset, int16_t yOffset) {
+ pCtx->mMotionVectorSize += 4;
+ pCtx->mMotionVectorData[0] = pMotionX;
+ pCtx->mMotionVectorData[1] = pMotionY;
+ pCtx->mMotionVectorData[2] = xOffset;
+ pCtx->mMotionVectorData[3] = yOffset;
+ pCtx->mMotionVectorData += 4;
+}
+
 int32_t ParseInterPMotionInfoCabac (PWelsDecoderContext pCtx, PWelsNeighAvail pNeighAvail, uint8_t* pNonZeroCount,
  int16_t pMotionVector[LIST_A][30][MV_A], int16_t pMvdCache[LIST_A][30][MV_A], int8_t pRefIndex[LIST_A][30]) {
  PSlice pSlice = &pCtx->pCurDqLayer->sLayerInfo.sSliceInLayer;
@@ -535,6 +544,10 @@ int32_t ParseInterPMotionInfoCabac (PWelsDecoderContext pCtx, PWelsNeighAvail pN
  pRefCount[0] = pSliceHeader->uiRefCount[0];
  pRefCount[1] = pSliceHeader->uiRefCount[1];
 
+ int16_t iMBOffsetX = pCurDqLayer->iMbX << 4;
+ int16_t iMBOffsetY = pCurDqLayer->iMbY << 4;
+ int16_t iBlk8X, iBlk8Y, iBlk4X, iBlk4Y;
+
  bool bIsPending = GetThreadCount (pCtx) > 1;
 
  switch (pCurDqLayer->pDec->pMbType[iMbXy]) {
@@ -559,6 +572,7 @@ int32_t ParseInterPMotionInfoCabac (PWelsDecoderContext pCtx, PWelsNeighAvail pN
  pMv[0] += pMvd[0];
  pMv[1] += pMvd[1];
  WELS_CHECK_SE_BOTH_WARNING (pMv[1], iMinVmv, iMaxVmv, "vertical mv");
+ UpdateMotionVector(pCtx, pMv[0], pMv[1], iMBOffsetX, iMBOffsetY);
  UpdateP16x16MotionInfo (pCurDqLayer, LIST_0, iRef[0], pMv);
  UpdateP16x16MvdCabac (pCurDqLayer, pMvd, LIST_0);
  }
@@ -589,6 +603,15 @@ int32_t ParseInterPMotionInfoCabac (PWelsDecoderContext pCtx, PWelsNeighAvail pN
  pMv[0] += pMvd[0];
  pMv[1] += pMvd[1];
  WELS_CHECK_SE_BOTH_WARNING (pMv[1], iMinVmv, iMaxVmv, "vertical mv");
+ if(!i)
+ {
+ UpdateMotionVector(pCtx, pMv[0], pMv[1], iMBOffsetX, iMBOffsetY);
+ }
+ else
+ {
+ // comes after first 16x8 block - so add 8 to Y offset
+ UpdateMotionVector(pCtx, pMv[0], pMv[1], iMBOffsetX, iMBOffsetY + 8);
+ }
  UpdateP16x8MotionInfo (pCurDqLayer, pMotionVector, pRefIndex, LIST_0, iPartIdx, iRef[i], pMv);
  UpdateP16x8MvdCabac (pCurDqLayer, pMvdCache, iPartIdx, pMvd, LIST_0);
  }
@@ -620,6 +643,14 @@ int32_t ParseInterPMotionInfoCabac (PWelsDecoderContext pCtx, PWelsNeighAvail pN
  pMv[0] += pMvd[0];
  pMv[1] += pMvd[1];
  WELS_CHECK_SE_BOTH_WARNING (pMv[1], iMinVmv, iMaxVmv, "vertical mv");
+ if(i)
+ {
+ UpdateMotionVector(pCtx, pMv[0], pMv[1], iMBOffsetX + 8, iMBOffsetY);
+ }
+ else
+ {
+ UpdateMotionVector(pCtx, pMv[0], pMv[1], iMBOffsetX, iMBOffsetY);
+ }
  UpdateP8x16MotionInfo (pCurDqLayer, pMotionVector, pRefIndex, LIST_0, iPartIdx, iRef[i], pMv);
  UpdateP8x16MvdCabac (pCurDqLayer, pMvdCache, iPartIdx, pMvd, LIST_0);
  }
@@ -661,6 +692,11 @@ int32_t ParseInterPMotionInfoCabac (PWelsDecoderContext pCtx, PWelsNeighAvail pN
  }
  //mv
  for (i = 0; i < 4; i++) {
+ int32_t iXOffset, iYOffset;
+ iBlk8X = (i & 1) << 3;
+ iBlk8Y = (i >> 1) << 3;
+ iXOffset = iMBOffsetX + iBlk8X;
+ iYOffset = iMBOffsetY + iBlk8Y;
  int8_t iPartCount = pSubPartCount[i];
  uiSubMbType = pCurDqLayer->pSubMbType[iMbXy][i];
  int16_t iPartIdx, iBlockW = pPartW[i];
@@ -680,6 +716,7 @@ int32_t ParseInterPMotionInfoCabac (PWelsDecoderContext pCtx, PWelsNeighAvail pN
  pMv[1] += pMvd[1];
  WELS_CHECK_SE_BOTH_WARNING (pMv[1], iMinVmv, iMaxVmv, "vertical mv");
  if (SUB_MB_TYPE_8x8 == uiSubMbType) {
+ UpdateMotionVector(pCtx, pMv[0], pMv[1], iXOffset, iYOffset);
  ST32 ((pMv + 2), LD32 (pMv));
  ST32 ((pMvd + 2), LD32 (pMvd));
  ST64 (pCurDqLayer->pDec->pMv[0][iMbXy][iScan4Idx], LD64 (pMv));
@@ -691,13 +728,29 @@ int32_t ParseInterPMotionInfoCabac (PWelsDecoderContext pCtx, PWelsNeighAvail pN
  ST64 (pMvdCache[0][iCacheIdx ], LD64 (pMvd));
  ST64 (pMvdCache[0][iCacheIdx + 6], LD64 (pMvd));
  } else if (SUB_MB_TYPE_8x4 == uiSubMbType) {
+ if(j)
+ {
+ UpdateMotionVector(pCtx, pMv[0], pMv[1], iXOffset, iYOffset + 4);
+ }
+ else
+ {
+ UpdateMotionVector(pCtx, pMv[0], pMv[1], iXOffset, iYOffset);
+ }
  ST32 ((pMv + 2), LD32 (pMv));
  ST32 ((pMvd + 2), LD32 (pMvd));
  ST64 (pCurDqLayer->pDec->pMv[0][iMbXy][iScan4Idx ], LD64 (pMv));
  ST64 (pCurDqLayer->pMvd[0][iMbXy][iScan4Idx ], LD64 (pMvd));
  ST64 (pMotionVector[0][iCacheIdx ], LD64 (pMv));
  ST64 (pMvdCache[0][iCacheIdx ], LD64 (pMvd));
  } else if (SUB_MB_TYPE_4x8 == uiSubMbType) {
+ if(j)
+ {
+ UpdateMotionVector(pCtx, pMv[0], pMv[1], iXOffset + 4, iYOffset);
+ }
+ else
+ {
+ UpdateMotionVector(pCtx, pMv[0], pMv[1], iXOffset, iYOffset);
+ }
  ST32 (pCurDqLayer->pDec->pMv[0][iMbXy][iScan4Idx ], LD32 (pMv));
  ST32 (pCurDqLayer->pDec->pMv[0][iMbXy][iScan4Idx + 4], LD32 (pMv));
  ST32 (pCurDqLayer->pMvd[0][iMbXy][iScan4Idx ], LD32 (pMvd));
@@ -707,6 +760,9 @@ int32_t ParseInterPMotionInfoCabac (PWelsDecoderContext pCtx, PWelsNeighAvail pN
  ST32 (pMvdCache[0][iCacheIdx ], LD32 (pMvd));
  ST32 (pMvdCache[0][iCacheIdx + 6], LD32 (pMvd));
  } else { //SUB_MB_TYPE_4x4
+ iBlk4X = (j & 1) << 2;
+ iBlk4Y = (j >> 1) << 2;
+ UpdateMotionVector(pCtx, pMv[0], pMv[1], iXOffset + iBlk4X, iYOffset + iBlk4Y);
  ST32 (pCurDqLayer->pDec->pMv[0][iMbXy][iScan4Idx ], LD32 (pMv));
  ST32 (pCurDqLayer->pMvd[0][iMbXy][iScan4Idx ], LD32 (pMvd));
  ST32 (pMotionVector[0][iCacheIdx ], LD32 (pMv));

codec/decoder/plus/inc/welsDecoderExt.h

@@ -91,6 +91,21 @@ class CWelsDecoder : public ISVCDecoder {
  unsigned char** ppDst,
  SBufferInfo* pDstInfo);
 
+ virtual DECODING_STATE EXTAPI ParseBitstreamGetMotionVectors (const unsigned char* kpSrc,
+ const int kiSrcLen,
+ unsigned char** ppDst,
+ SParserBsInfo* pDstInfo,
+ SBufferInfo* ppDecodeInfo,
+ int32_t* motionVectorSize,
+ int16_t** motionVectorData);
+
+ virtual DECODING_STATE EXTAPI DecodeFrameGetMotionVectorsNoDelay (const unsigned char* pSrc,
+ const int iSrcLen,
+ unsigned char** ppDst,
+ SBufferInfo* pDstInfo,
+ int32_t* motionVectorSize,
+ int16_t** motionVectorData);
+
  virtual DECODING_STATE EXTAPI FlushFrame (unsigned char** ppDst,
  SBufferInfo* pDstInfo);