使用AVAssetWriter录制无间隙音频

Question

我正在尝试记录音频片段并重新组合它们而不会产生音频差距.

最终的目标是也有视频,但我发现音频本身在结合时会产生差距 ffmpeg -f concat -i list.txt -c copy out.mp4

如果我把音频放在HLS播放列表中,也有间隙,所以我不认为这是ffmpeg独有的.

这个想法是样本连续进入,我的控制器将样本路由到正确的样本AVAssetWriter.如何消除音频中的差距？

import Foundation
import UIKit
import AVFoundation

class StreamController: UIViewController, AVCaptureAudioDataOutputSampleBufferDelegate, AVCaptureVideoDataOutputSampleBufferDelegate {
    var closingAudioInput: AVAssetWriterInput?
    var closingAssetWriter: AVAssetWriter?

    var currentAudioInput: AVAssetWriterInput?
    var currentAssetWriter: AVAssetWriter?

    var nextAudioInput: AVAssetWriterInput?
    var nextAssetWriter: AVAssetWriter?

    var videoHelper: VideoHelper?

    var startTime: NSTimeInterval = 0
    let closeAssetQueue: dispatch_queue_t = dispatch_queue_create("closeAssetQueue", nil);

    override func viewDidLoad() {
        super.viewDidLoad()
        startTime = NSDate().timeIntervalSince1970
        createSegmentWriter()
        videoHelper = VideoHelper()
        videoHelper!.delegate = self
        videoHelper!.startSession()
        NSTimer.scheduledTimerWithTimeInterval(1, target: self, selector: "createSegmentWriter", userInfo: nil, repeats: true)
    }

    func createSegmentWriter() {
        print("Creating segment writer at t=\(NSDate().timeIntervalSince1970 - self.startTime)")
        let outputPath = OutputFileNameHelper.instance.pathForOutput()
        OutputFileNameHelper.instance.incrementSegmentIndex()
        try? NSFileManager.defaultManager().removeItemAtPath(outputPath)
        nextAssetWriter = try! AVAssetWriter(URL: NSURL(fileURLWithPath: outputPath), fileType: AVFileTypeMPEG4)
        nextAssetWriter!.shouldOptimizeForNetworkUse = true

        let audioSettings: [String:AnyObject] = EncodingSettings.AUDIO
        nextAudioInput = AVAssetWriterInput(mediaType: AVMediaTypeAudio, outputSettings: audioSettings)
        nextAudioInput!.expectsMediaDataInRealTime = true
        nextAssetWriter?.addInput(nextAudioInput!)

        nextAssetWriter!.startWriting()
    }

    func closeWriterIfNecessary() {
        if closing && audioFinished {
            closing = false
            audioFinished = false
            let outputFile = closingAssetWriter?.outputURL.pathComponents?.last
            closingAssetWriter?.finishWritingWithCompletionHandler() {
                let delta = NSDate().timeIntervalSince1970 - self.startTime
                print("segment \(outputFile!) finished at t=\(delta)")
            }
            self.closingAudioInput = nil
            self.closingAssetWriter = nil
        }
    }

    var audioFinished = false
    var closing = false

    func captureOutput(captureOutput: AVCaptureOutput!, didOutputSampleBuffer sampleBuffer: CMSampleBufferRef, fromConnection connection: AVCaptureConnection!) {
        if let nextWriter = nextAssetWriter {
            if nextWriter.status.rawValue != 0 {
                if (currentAssetWriter != nil) {
                    closing = true
                }

                var sampleTiming: CMSampleTimingInfo = kCMTimingInfoInvalid
                CMSampleBufferGetSampleTimingInfo(sampleBuffer, 0, &sampleTiming)

                print("Switching asset writers at t=\(NSDate().timeIntervalSince1970 - self.startTime)")
                closingAssetWriter = currentAssetWriter
                closingAudioInput = currentAudioInput

                currentAssetWriter = nextAssetWriter
                currentAudioInput = nextAudioInput

                nextAssetWriter = nil
                nextAudioInput = nil

                currentAssetWriter?.startSessionAtSourceTime(sampleTiming.presentationTimeStamp)
            }
        }

        if let _ = captureOutput as? AVCaptureVideoDataOutput {
        } else if let _ = captureOutput as? AVCaptureAudioDataOutput {
            captureAudioSample(sampleBuffer)
        }

        dispatch_async(closeAssetQueue) {
            self.closeWriterIfNecessary()
        }
    }

    func printTimingInfo(sampleBuffer: CMSampleBufferRef, prefix: String) {
        var sampleTiming: CMSampleTimingInfo = kCMTimingInfoInvalid
        CMSampleBufferGetSampleTimingInfo(sampleBuffer, 0, &sampleTiming)
        let presentationTime = Double(sampleTiming.presentationTimeStamp.value) / Double(sampleTiming.presentationTimeStamp.timescale)
        print("\(prefix):\(presentationTime)")
    }

    func captureAudioSample(sampleBuffer: CMSampleBufferRef) {
        printTimingInfo(sampleBuffer, prefix: "A")
        if (closing && !audioFinished) {
            if closingAudioInput?.readyForMoreMediaData == true {
                closingAudioInput?.appendSampleBuffer(sampleBuffer)
            }
            closingAudioInput?.markAsFinished()
            audioFinished = true
        } else {
            if currentAudioInput?.readyForMoreMediaData == true {
                currentAudioInput?.appendSampleBuffer(sampleBuffer)
            }
        }
    }
}

Answer 1

对于像AAC这样的数据包格式,您在开始时使用静音启动帧(也称为编码器延迟),在结束时使用其余帧(当您的音频长度不是数据包大小的倍数时).在你的情况下,它们是每个文件开头的2112个.启动和剩余帧会破坏连接文件而不转码它们的可能性,所以你不能责怪ffmpeg -c copy不能产生无缝输出.

我不确定这会给你留下什么视频 - 显然音频会同步到视频,即使在启动帧的情况下也是如此.

这一切都取决于你打算如何连接最终的音频(最终视频).如果您自己使用AVFoundation,那么您可以使用检测和计算启动/余数帧

CMGetAttachment(buffer, kCMSampleBufferAttachmentKey_TrimDurationAtStart, NULL) 
CMGetAttachment(audioBuffer, kCMSampleBufferAttachmentKey_TrimDurationAtEnd, NULL) 

作为一个短期解决方案,您可以切换到非"分组化"以获得无间隙,可连接(使用ffmpeg)文件.

例如

AVFormatIDKey: kAudioFormatAppleIMA4,fileType: AVFileTypeAIFC,后缀".aifc"或者 AVFormatIDKey: kAudioFormatLinearPCM,fileType: AVFileTypeWAVE,后缀".WAV"

ps你可以使用无处不在的afinfo工具看到启动和剩余帧和数据包大小.

afinfo chunk.mp4

数据格式:2 ch,44100 Hz,'aac'(0x00000000)0位/通道,0字节/包,1024帧/包,0字节/帧
...
音频39596有效帧+ 2112启动+ 276余数= 41984
. ..