在OS X上使用AVFoundation进行H.264视频流传输？

Question

基于这个堆栈溢出问题以及WWDC 2014的Apple 对视频编码和解码的直接访问，我制作了一个小型Xcode项目，演示了如何使用AVFoundation解码和显示H.264流。该项目可在github 此处获得。

我已将一系列1000个NALU转储到文件中，并将它们包含在项目中（nalu_000.bin ... nalu_999.bin）。

下面包括代码的有趣部分，该部分分析NALU并将其流式传输到AVSampleBufferDisplayLayer：

@import AVKit;

typedef enum {
    NALUTypeSliceNoneIDR = 1,
    NALUTypeSliceIDR = 5,
    NALUTypeSPS = 7,
    NALUTypePPS = 8
} NALUType;

@interface ViewController ()

@property (nonatomic, strong, readonly) VideoView * videoView;
@property (nonatomic, strong) NSData * spsData;
@property (nonatomic, strong) NSData * ppsData;
@property (nonatomic) CMVideoFormatDescriptionRef videoFormatDescr;
@property (nonatomic) BOOL videoFormatDescriptionAvailable;

@end

@implementation ViewController

- (VideoView *)videoView {
    return (VideoView *) self.view;
}

- (instancetype)initWithCoder:(NSCoder *)coder {
    self = [super initWithCoder:coder];

    if (self) {
        _videoFormatDescriptionAvailable = NO;
    }

    return self;
}

- (int)getNALUType:(NSData *)NALU {
    uint8_t * bytes = (uint8_t *) NALU.bytes;

    return bytes[0] & 0x1F;
}

- (void)handleSlice:(NSData *)NALU {
    if (self.videoFormatDescriptionAvailable) {
        /* The length of the NALU in big endian */
        const uint32_t NALUlengthInBigEndian = CFSwapInt32HostToBig((uint32_t) NALU.length);

        /* Create the slice */
        NSMutableData * slice = [[NSMutableData alloc] initWithBytes:&NALUlengthInBigEndian length:4];

        /* Append the contents of the NALU */
        [slice appendData:NALU];

        /* Create the video block */
        CMBlockBufferRef videoBlock = NULL;

        OSStatus status;

        status =
            CMBlockBufferCreateWithMemoryBlock
                (
                    NULL,
                    (void *) slice.bytes,
                    slice.length,
                    kCFAllocatorNull,
                    NULL,
                    0,
                    slice.length,
                    0,
                    & videoBlock
                );

        NSLog(@"BlockBufferCreation: %@", (status == kCMBlockBufferNoErr) ? @"successfully." : @"failed.");

        /* Create the CMSampleBuffer */
        CMSampleBufferRef sbRef = NULL;

        const size_t sampleSizeArray[] = { slice.length };

        status =
            CMSampleBufferCreate
                (
                    kCFAllocatorDefault,
                    videoBlock,
                    true,
                    NULL,
                    NULL,
                    _videoFormatDescr,
                    1,
                    0,
                    NULL,
                    1,
                    sampleSizeArray,
                    & sbRef
                );

        NSLog(@"SampleBufferCreate: %@", (status == noErr) ? @"successfully." : @"failed.");

        /* Enqueue the CMSampleBuffer in the AVSampleBufferDisplayLayer */
        CFArrayRef attachments = CMSampleBufferGetSampleAttachmentsArray(sbRef, YES);
        CFMutableDictionaryRef dict = (CFMutableDictionaryRef)CFArrayGetValueAtIndex(attachments, 0);
        CFDictionarySetValue(dict, kCMSampleAttachmentKey_DisplayImmediately, kCFBooleanTrue);

        NSLog(@"Error: %@, Status: %@",
              self.videoView.videoLayer.error,
                (self.videoView.videoLayer.status == AVQueuedSampleBufferRenderingStatusUnknown)
                    ? @"unknown"
                    : (
                        (self.videoView.videoLayer.status == AVQueuedSampleBufferRenderingStatusRendering)
                            ? @"rendering"
                            :@"failed"
                      )
             );

        dispatch_async(dispatch_get_main_queue(),^{
            [self.videoView.videoLayer enqueueSampleBuffer:sbRef];
            [self.videoView.videoLayer setNeedsDisplay];
        });

        NSLog(@" ");
    }
}

- (void)handleSPS:(NSData *)NALU {
    _spsData = [NALU copy];
}

- (void)handlePPS:(NSData *)NALU {
    _ppsData = [NALU copy];
}

- (void)updateFormatDescriptionIfPossible {
    if (_spsData != nil && _ppsData != nil) {
        const uint8_t * const parameterSetPointers[2] = {
            (const uint8_t *) _spsData.bytes,
            (const uint8_t *) _ppsData.bytes
        };

        const size_t parameterSetSizes[2] = {
            _spsData.length,
            _ppsData.length
        };

        OSStatus status =
            CMVideoFormatDescriptionCreateFromH264ParameterSets
                (
                    kCFAllocatorDefault,
                    2,
                    parameterSetPointers,
                    parameterSetSizes,
                    4,
                    & _videoFormatDescr
                );

        _videoFormatDescriptionAvailable = YES;

        NSLog(@"Updated CMVideoFormatDescription. Creation: %@.", (status == noErr) ? @"successfully." : @"failed.");
    }
}

- (void)parseNALU:(NSData *)NALU {
    int type = [self getNALUType: NALU];

    NSLog(@"NALU with Type \"%@\" received.", naluTypesStrings[type]);

    switch (type)
    {
        case NALUTypeSliceNoneIDR:
        case NALUTypeSliceIDR:
            [self handleSlice:NALU];
            break;
        case NALUTypeSPS:
            [self handleSPS:NALU];
            [self updateFormatDescriptionIfPossible];
            break;
        case NALUTypePPS:
            [self handlePPS:NALU];
            [self updateFormatDescriptionIfPossible];
            break;
        default:
            break;
    }
}

- (IBAction)streamVideo:(id)sender {
    NSBundle * mainBundle = [NSBundle mainBundle];

    for (int k = 0; k < 1000; k++) {
        NSString * resource = [NSString stringWithFormat:@"nalu_%03d", k];
        NSString * path = [mainBundle pathForResource:resource ofType:@"bin"];
        NSData * NALU = [NSData dataWithContentsOfFile:path];
        [self parseNALU:NALU];
    }
}

@end

基本上，代码的工作方式如下：

1. 它使用CMVideoFormatDescriptionCreateFromH264ParameterSets从SPS和PPS NALU创建CMVideoFormatDescriptionRef。
2. 它根据AVCC格式重新打包NALU。由于已经删除了NALU起始代码，因此仅在其前面附加了一个4字节的NALU长度标头（采用big-endian格式）。
3. 它将所有VLC NALU帧打包为CMSampleBuffers并将其馈送到AVSampleBufferDisplayLayer。

该代码似乎正确读取了SPS和PPS参数集。不幸的是，将CMSampleBuffers馈送到AVSampleBufferDisplayLayer时出了点问题。对于每一帧，Xcode都会在日志窗口中转储以下消息（程序不会崩溃）：

[16:05:22.533] <<<< VMC >>>> vmc2PostDecodeError: posting DecodeError (-8969) -- PTS was nan = 0/0
[16:05:22.534] vtDecompressionDuctDecodeSingleFrame signalled err=-8969 (err) (VTVideoDecoderDecodeFrame returned error) at /SourceCache/CoreMedia_frameworks/CoreMedia-1562.235/Sources/VideoToolbox/VTDecompressionSession.c line 3241
[16:05:22.535] <<<< VMC >>>> vmc2DequeueAndDecodeFrame: frame failed - err -8969

此外，框架看起来像莫奈的莫奈画作：

我不是H.264格式（或一般来说是视频编码/解码）的专家，并且如果对这一主题有更深入的了解的人可以看一下演示项目并为我指明正确的方向，将不胜感激。

以后，我会将代码留在github上，作为对在OS X / iOS上解码H.264感兴趣的其他人的示例。