Merge pull request #28 from Tencent/feature/ios_support_h265_test

Feature/ios support h265 test
Tencent · Dec 2, 2020 · 1497b1f · 1497b1f
2 parents dd4b80e + 69171cc
commit 1497b1f
Show file tree

Hide file tree

Showing 17 changed files with 309 additions and 79 deletions.
diff --git a/iOS/QGVAPlayer/QGVAPlayer/Classes/Controllers/Decoders/QGMP4FrameHWDecoder.m b/iOS/QGVAPlayer/QGVAPlayer/Classes/Controllers/Decoders/QGMP4FrameHWDecoder.m
@@ -91,6 +91,8 @@ @interface QGMP4FrameHWDecoder() {
 @property (atomic, strong) dispatch_queue_t decodeQueue; //dispatch decode task
 @property (nonatomic, strong) NSData *ppsData; //Picture Parameter Set
 @property (nonatomic, strong) NSData *spsData; //Sequence Parameter Set
+/** Video Parameter Set */
+@property (nonatomic, strong) NSData *vpsData;
 
 @end
 
@@ -174,7 +176,8 @@ - (void)_decodeFrame:(NSInteger)frameIndex {
     if (_isFinish) {
         return ;
     }
-    if (!_buffers || _buffers.count == 0) {
+
+    if (!_buffers) {
         return ;
     }
 
@@ -191,6 +194,9 @@ - (void)_decodeFrame:(NSInteger)frameIndex {
         return;
     }
 
+    // 获取当前帧pts,pts是在parse mp4 box时得到的
+    uint64_t currentPts = [_mp4Parser.videoSamples[frameIndex] pts];
+
     CVPixelBufferRef outputPixelBuffer = NULL;
     // 4. get NALUnit payload into a CMBlockBuffer,
     CMBlockBufferRef blockBuffer = NULL;
@@ -237,14 +243,19 @@ - (void)_decodeFrame:(NSInteger)frameIndex {
             // imagebuffer会在frame回收时释放
             CVPixelBufferRetain(imageBuffer);
             newFrame.pixelBuffer = imageBuffer;
-            newFrame.frameIndex = frameIndex;
+            newFrame.frameIndex = frameIndex; //dts顺序
             NSTimeInterval decodeTime = [[NSDate date] timeIntervalSinceDate:startDate]*1000;
             newFrame.decodeTime = decodeTime;
             newFrame.defaultFps =(int) strongSelf->_mp4Parser.fps;
+            newFrame.pts = currentPts;
+
+            // 8. insert into buffer
+            [strongSelf->_buffers addObject:newFrame];
 
-            //8. insert into buffer
-            NSInteger index = frameIndex % (strongSelf->_buffers.count);
-            strongSelf->_buffers[index] = newFrame;
+            // 9. sort
+            [strongSelf->_buffers sortUsingComparator:^NSComparisonResult(QGMP4AnimatedImageFrame * _Nonnull obj1, QGMP4AnimatedImageFrame * _Nonnull obj2) {
+                return [@(obj1.pts) compare:@(obj2.pts)];
+            }];
         });
     } else {
         // 7. use VTDecompressionSessionDecodeFrame
@@ -266,9 +277,13 @@ - (void)_decodeFrame:(NSInteger)frameIndex {
         newFrame.decodeTime = decodeTime;
         newFrame.defaultFps = (int)_mp4Parser.fps;
 
-        //8. insert into buffer
-        NSInteger index = frameIndex%_buffers.count;
-        _buffers[index] = newFrame;
+        // 8. insert into buffer
+        [_buffers addObject:newFrame];
+
+        // 9. sort
+        [_buffers sortUsingComparator:^NSComparisonResult(QGMP4AnimatedImageFrame * _Nonnull obj1, QGMP4AnimatedImageFrame * _Nonnull obj2) {
+            return [@(obj1.pts) compare:@(obj2.pts)];
+        }];
     }
 }
 
@@ -307,6 +322,7 @@ - (BOOL)onInputStart {
     }
 
     _isFinish = NO;
+    self.vpsData = nil;
     self.spsData = nil;
     self.ppsData = nil;
     _outputWidth = (int)_mp4Parser.picWidth;
@@ -323,18 +339,54 @@ - (BOOL)initPPSnSPS {
         VAP_Error(kQGVAPModuleCommon, @"sps&pps is already has value.");
         return YES;
     }
+
     self.spsData = _mp4Parser.spsData;
     self.ppsData = _mp4Parser.ppsData;
+    self.vpsData = _mp4Parser.vpsData;
 
     // 2. create  CMFormatDescription
-    if (self.spsData != nil && self.ppsData != nil) {
-        const uint8_t* const parameterSetPointers[2] = { (const uint8_t*)[self.spsData bytes], (const uint8_t*)[self.ppsData bytes] };
-        const size_t parameterSetSizes[2] = { [self.spsData length], [self.ppsData length] };
-        _status = CMVideoFormatDescriptionCreateFromH264ParameterSets(kCFAllocatorDefault, 2, parameterSetPointers, parameterSetSizes, 4, &_mFormatDescription);
-        if (_status != noErr) {
-            VAP_Event(kQGVAPModuleCommon, @"CMVideoFormatDescription. Creation: %@.", (_status == noErr) ? @"successfully." : @"failed.");
-            _constructErr = [NSError errorWithDomain:QGMP4HWDErrorDomain code:QGMP4HWDErrorCode_ErrorCreateVTBDesc userInfo:[self errorUserInfo]];
-            return NO;
+    if (self.spsData != nil && self.ppsData != nil && _mp4Parser.videoCodecID != QGMP4VideoStreamCodecIDUnknown) {
+        if (_mp4Parser.videoCodecID == QGMP4VideoStreamCodecIDH264) {
+            const uint8_t* const parameterSetPointers[2] = { (const uint8_t*)[self.spsData bytes], (const uint8_t*)[self.ppsData bytes] };
+            const size_t parameterSetSizes[2] = { [self.spsData length], [self.ppsData length] };
+
+            _status = CMVideoFormatDescriptionCreateFromH264ParameterSets(kCFAllocatorDefault,
+                                                                          2,
+                                                                          parameterSetPointers,
+                                                                          parameterSetSizes,
+                                                                          4,
+                                                                          &_mFormatDescription);
+            if (_status != noErr) {
+                VAP_Event(kQGVAPModuleCommon, @"CMVideoFormatDescription. Creation: %@.", (_status == noErr) ? @"successfully." : @"failed.");
+                _constructErr = [NSError errorWithDomain:QGMP4HWDErrorDomain code:QGMP4HWDErrorCode_ErrorCreateVTBDesc userInfo:[self errorUserInfo]];
+                return NO;
+            }
+        } else if (_mp4Parser.videoCodecID == QGMP4VideoStreamCodecIDH265) {
+            if (@available(iOS 11.0, *)) {
+                if(VTIsHardwareDecodeSupported(kCMVideoCodecType_HEVC)) {
+                    const uint8_t* const parameterSetPointers[3] = {(const uint8_t*)[self.vpsData bytes], (const uint8_t*)[self.spsData bytes], (const uint8_t*)[self.ppsData bytes]};
+                    const size_t parameterSetSizes[3] = {[self.vpsData length], [self.spsData length], [self.ppsData length]};
+
+                    _status = CMVideoFormatDescriptionCreateFromHEVCParameterSets(kCFAllocatorDefault,
+                                                                                  3,                    // parameter_set_count
+                                                                                  parameterSetPointers, // &parameter_set_pointers
+                                                                                  parameterSetSizes,    // &parameter_set_sizes
+                                                                                  4,                    // nal_unit_header_length
+                                                                                  NULL,
+                                                                                  &_mFormatDescription);
+                    if (_status != noErr) {
+                        VAP_Event(kQGVAPModuleCommon, @"CMVideoFormatDescription. Creation: %@.", (_status == noErr) ? @"successfully." : @"failed.");
+                        _constructErr = [NSError errorWithDomain:QGMP4HWDErrorDomain code:QGMP4HWDErrorCode_ErrorCreateVTBDesc userInfo:[self errorUserInfo]];
+                        return NO;
+                    }
+                } else {
+                    VAP_Event(kQGVAPModuleCommon, @"H.265 decoding is un-supported because of the hardware");
+                    return NO;
+                }
+            } else {
+                VAP_Event(kQGVAPModuleCommon, @"System version is too low to support H.265 decoding");
+                return NO;
+            }
         }
     }
 
@@ -390,9 +442,10 @@ - (void)_onInputEnd  {
         CFRelease(_mDecodeSession);
         _mDecodeSession = NULL;
     }
-    if (self.spsData || self.ppsData) {
+    if (self.spsData || self.ppsData || self.vpsData) {
         self.spsData = nil;
         self.ppsData = nil;
+        self.vpsData = nil;
     }
     if (_mFormatDescription) {
         CFRelease(_mFormatDescription);

diff --git a/iOS/QGVAPlayer/QGVAPlayer/Classes/Controllers/QGAnimatedImageBufferManager.h b/iOS/QGVAPlayer/QGVAPlayer/Classes/Controllers/QGAnimatedImageBufferManager.h
@@ -24,5 +24,6 @@
 - (instancetype)initWithConfig:(QGAnimatedImageDecodeConfig *)config;
 - (QGBaseAnimatedImageFrame *)getBufferedFrame:(NSInteger)frameIndex;
 - (BOOL)isBufferFull;
+- (QGBaseAnimatedImageFrame *)popVideoFrame;
 
 @end
diff --git a/iOS/QGVAPlayer/QGVAPlayer/Classes/Controllers/QGAnimatedImageBufferManager.m b/iOS/QGVAPlayer/QGVAPlayer/Classes/Controllers/QGAnimatedImageBufferManager.m
@@ -35,12 +35,7 @@ - (instancetype)initWithConfig:(QGAnimatedImageDecodeConfig *)config {
 }
 
 - (void)createBuffersWithConfig:(QGAnimatedImageDecodeConfig *)config {
-
-    _buffers = [QGVAPSafeMutableArray new];
-    for (int i = 0; i < config.bufferCount; i++) {
-        NSObject *frame = [NSObject new];
-        [_buffers addObject:frame];
-    }
+    _buffers = [[QGVAPSafeMutableArray alloc] initWithCapacity:config.bufferCount];
 }
 
 /**
@@ -67,6 +62,21 @@ - (QGBaseAnimatedImageFrame *)getBufferedFrame:(NSInteger)frameIndex {
     return frame;
 }
 
+- (QGBaseAnimatedImageFrame *)popVideoFrame {
+    if (!_buffers.count) {
+        return nil;
+    }
+
+    if (![_buffers.firstObject isKindOfClass:[QGBaseAnimatedImageFrame class]]) {
+        return nil;
+    }
+
+    QGBaseAnimatedImageFrame *frame = _buffers.firstObject;
+    [_buffers removeObjectAtIndex:0];
+
+    return frame;
+}
+
 /**
  判断当前缓冲区是否被填满
  

diff --git a/iOS/QGVAPlayer/QGVAPlayer/Classes/Controllers/QGAnimatedImageDecodeConfig.m b/iOS/QGVAPlayer/QGVAPlayer/Classes/Controllers/QGAnimatedImageDecodeConfig.m
@@ -21,7 +21,7 @@ + (instancetype)defaultConfig {
 
     QGAnimatedImageDecodeConfig *config = [QGAnimatedImageDecodeConfig new];
     config.threadCount= 1;
-    config.bufferCount = 1;
+    config.bufferCount = 5;
     return config;
 }
 

diff --git a/iOS/QGVAPlayer/QGVAPlayer/Classes/Controllers/QGAnimatedImageDecodeManager.m b/iOS/QGVAPlayer/QGVAPlayer/Classes/Controllers/QGAnimatedImageDecodeManager.m
@@ -61,13 +61,16 @@ - (instancetype)initWith:(QGBaseDFileInfo *)fileInfo
 - (QGBaseAnimatedImageFrame *)consumeDecodedFrame:(NSInteger)frameIndex {
 
     @synchronized (self) {
-        if (frameIndex==0 && ![_bufferManager isBufferFull]) {
+        // 控制何时命中第一帧，缓存满了才命中
+        if (frameIndex == 0 && _bufferManager.buffers.count < _config.bufferCount) {
             return nil;
         }
         [self checkIfDecodeFinish:frameIndex];
-        QGBaseAnimatedImageFrame *frame = [_bufferManager getBufferedFrame:frameIndex];
-        if (frame && frame.frameIndex == frameIndex) {
-            [self decodeFrame:frame.frameIndex+_bufferManager.buffers.count];
+        QGBaseAnimatedImageFrame *frame = [_bufferManager popVideoFrame];
+        if (frame) {
+            // pts顺序
+            frame.frameIndex = frameIndex;
+            [self decodeFrame:frameIndex+_config.bufferCount];
         }
         return frame;
     }
@@ -133,7 +136,7 @@ - (void)createDecodersByConfig:(QGAnimatedImageDecodeConfig *)config {
 
 - (void)initializeBuffers {
 
-    for (int i = 0; i < _bufferManager.buffers.count; i++) {
+    for (int i = 0; i < _config.bufferCount; i++) {
         [self decodeFrame:i];
     }
 }

diff --git a/iOS/QGVAPlayer/QGVAPlayer/Classes/MP4Parser/QGMP4Box.h b/iOS/QGVAPlayer/QGVAPlayer/Classes/MP4Parser/QGMP4Box.h
@@ -75,9 +75,29 @@ typedef NS_ENUM(NSUInteger, QGMP4BoxType) {
     QGMP4BoxType_wide           =   ATOM_TYPE('w','i','d','e'),//0x77696465,
     QGMP4BoxType_loci           =   ATOM_TYPE('l','o','c','i'),//0x6c6f6369,
     QGMP4BoxType_smhd           =   ATOM_TYPE('s','m','h','d'),//0x736d6864,
-    QGMP4BoxType_vapc           =   ATOM_TYPE('v','a','p','c')//0x76617063,//vap专属，存储json配置信息
+    QGMP4BoxType_vapc           =   ATOM_TYPE('v','a','p','c'),//0x76617063,//vap专属，存储json配置信息
+    QGMP4BoxType_hvc1           =   ATOM_TYPE('h','v','c','1'),
+    QGMP4BoxType_hvcC           =   ATOM_TYPE('h','v','c','C')
 };
 
+typedef NS_ENUM(NSUInteger, QGMP4VideoStreamCodecID) {
+    QGMP4VideoStreamCodecIDUnknown = 0,
+    QGMP4VideoStreamCodecIDH264,
+    QGMP4VideoStreamCodecIDH265
+};
+
+/**
+ * QGCttsEntry
+ */
+@interface QGCttsEntry : NSObject
+
+/** sampleCount */
+@property (nonatomic, assign) uint32_t sampleCount;
+/** compositionOffset */
+@property (nonatomic, assign) uint32_t compositionOffset;
+
+@end
+
 @interface QGMP4BoxFactory : NSObject
 
 + (BOOL)isTypeValueValid:(QGMP4BoxType)type;
@@ -115,6 +135,9 @@ typedef NS_ENUM(NSUInteger, QGMP4BoxType) {
 @interface QGMP4AvccBox : QGMP4Box
 @end
 
+@interface QGMP4HvccBox : QGMP4Box
+@end
+
 @interface QGMP4MvhdBox : QGMP4Box
 @end
 
@@ -145,6 +168,16 @@ The table is compactly coded. Each entry gives the index of the first chunk of a
 
 @end
 
+/**
+ * ctts
+ */
+@interface QGMP4CttsBox : QGMP4Box
+
+/** compositionOffsets */
+@property (nonatomic, strong) NSMutableArray<NSNumber *> *compositionOffsets;
+
+@end
+
 //This box contains a compact version of a table that allows indexing from decoding time to sample number. Other tables give sample sizes and pointers, from the sample number. Each entry in the table gives the number of consecutive samples with the same time delta, and the delta of those samples. By adding the deltas a complete time-to-sample map may be built.
 @interface QGSttsEntry : NSObject
 

diff --git a/iOS/QGVAPlayer/QGVAPlayer/Classes/MP4Parser/QGMP4Box.m b/iOS/QGVAPlayer/QGVAPlayer/Classes/MP4Parser/QGMP4Box.m
@@ -107,6 +107,44 @@ - (NSString *)typeString {
 
 @end
 
+#pragma mark -- hvcc box
+/**
+ * QGMP4HvccBox
+ */
+@implementation QGMP4HvccBox
+@end
+
+/**
+ * QGCttsEntry 通过dts计算pts
+ */
+@implementation QGCttsEntry
+
+@end
+
+/**
+ * QGMP4CttsBox 通过dts计算pts
+ */
+@implementation QGMP4CttsBox
+- (void)boxDidParsed:(QGMp4BoxDataFetcher)datablock {
+    if (!_compositionOffsets) {
+        _compositionOffsets = [NSMutableArray new];
+    }
+
+    NSData *cttsData = datablock(self);
+    const char *bytes = cttsData.bytes;
+    uint32_t entryCount = READ32BIT(&bytes[12]);
+
+    for (int i = 0; i < entryCount; ++i) {
+        uint32_t sampleCount = READ32BIT(&bytes[16+i*8]);
+        uint32_t compositionOffset = READ32BIT(&bytes[16+i*8+4]);
+        for (int j = 0; j < sampleCount; j++) {
+            [_compositionOffsets addObject:@(compositionOffset)];
+        }
+    }
+}
+
+@end
+
 #pragma mark -- mdat box
 @implementation QGMP4MdatBox
 
@@ -331,6 +369,10 @@ + (Class)boxClassForType:(QGMP4BoxType)type {
             return [QGMP4SttsBox class];
         case QGMP4BoxType_stco:
             return [QGMP4StcoBox class];
+        case QGMP4BoxType_hvcC:
+            return [QGMP4HvccBox class];
+        case QGMP4BoxType_ctts:
+            return [QGMP4CttsBox class];
         default:
             return nil;
     }

diff --git a/iOS/QGVAPlayer/QGVAPlayer/Classes/MP4Parser/QGMP4Parser.h b/iOS/QGVAPlayer/QGVAPlayer/Classes/MP4Parser/QGMP4Parser.h
@@ -52,6 +52,10 @@
 @property (nonatomic, strong) QGMP4Box *rootBox;        //mp4文件根box
 @property (nonatomic, strong) QGMP4TrackBox *videoTrackBox;     //视频track
 @property (nonatomic, strong) QGMP4TrackBox *audioTrackBox;     //音频track
+/** vps */
+@property (nonatomic, strong) NSData *vpsData;
+/** 视频流编码器ID类型 */
+@property (nonatomic, assign) QGMP4VideoStreamCodecID videoCodecID;
 
 - (void)parse;
 - (NSData *)readPacketOfSample:(NSInteger)sampleIndex;