那些遇到过的问题

使 CIContext.render(CIImage, CVPixelBuffer) 与 AVAssetWriter 一起使用

Ian*_*hek 5 core-graphics core-image avfoundation metal

我想使用 Core Image 处理一堆对象并将它们转换为macOSCGImage上的 QuickTime 影片。以下代码演示了所需内容，但输出包含大量空白（黑色）帧：

\n\n
import AppKit\nimport AVFoundation\nimport CoreGraphics\nimport Foundation\nimport CoreVideo\nimport Metal\n\n// Video output url.\nlet url: URL = try! FileManager.default.url(for: .downloadsDirectory, in: .userDomainMask, appropriateFor: nil, create: false).appendingPathComponent("av.mov")\ntry? FileManager.default.removeItem(at: url)\n\n// Video frame size, total frame count, frame rate and frame image.\nlet frameSize: CGSize = CGSize(width: 2000, height: 1000)\nlet frameCount: Int = 100\nlet frameRate: Double = 1 / 30\nlet frameImage: CGImage\n\nframeImage = NSImage(size: frameSize, flipped: false, drawingHandler: {\n    NSColor.red.setFill()\n    $0.fill()\n    return true\n}).cgImage(forProposedRect: nil, context: nil, hints: nil)!\n\nlet pixelBufferAttributes: [CFString: Any]\nlet outputSettings: [String: Any]\n\npixelBufferAttributes = [\n    kCVPixelBufferPixelFormatTypeKey: Int(kCVPixelFormatType_32ARGB),\n    kCVPixelBufferWidthKey: Float(frameSize.width),\n    kCVPixelBufferHeightKey: Float(frameSize.height),\n    kCVPixelBufferMetalCompatibilityKey: true,\n    kCVPixelBufferCGImageCompatibilityKey: true,\n    kCVPixelBufferCGBitmapContextCompatibilityKey: true,\n]\n\noutputSettings = [\n    AVVideoCodecKey: AVVideoCodecType.h264,\n    AVVideoWidthKey: Int(frameSize.width),\n    AVVideoHeightKey: Int(frameSize.height),\n]\n\nlet writer: AVAssetWriter = try! AVAssetWriter(outputURL: url, fileType: .mov)\nlet input: AVAssetWriterInput = AVAssetWriterInput(mediaType: .video, outputSettings: outputSettings)\nlet pixelBufferAdaptor: AVAssetWriterInputPixelBufferAdaptor = AVAssetWriterInputPixelBufferAdaptor(assetWriterInput: input, sourcePixelBufferAttributes: pixelBufferAttributes as [String: Any])\n\ninput.expectsMediaDataInRealTime = true\n\nprecondition(writer.canAdd(input))\nwriter.add(input)\n\nprecondition(writer.startWriting())\nwriter.startSession(atSourceTime: CMTime.zero)\n\nlet colorSpace: CGColorSpace = CGColorSpace(name: CGColorSpace.sRGB) ?? CGColorSpaceCreateDeviceRGB()\nlet context = CIContext(mtlDevice: MTLCreateSystemDefaultDevice()!)\n\nSwift.print("Starting the render\xe2\x80\xa6")\n\n// Preferred scenario: using CoreImage to fill the buffer from the pixel buffer adapter. Shows that\n// CIImage + AVAssetWriterInputPixelBufferAdaptor are not working together.\n\nfor frameNumber in 0 ..< frameCount {\n    var pixelBuffer: CVPixelBuffer?\n    guard let pixelBufferPool: CVPixelBufferPool = pixelBufferAdaptor.pixelBufferPool else { preconditionFailure() }\n    precondition(CVPixelBufferPoolCreatePixelBuffer(nil, pixelBufferPool, &pixelBuffer) == kCVReturnSuccess)\n\n    precondition(CVPixelBufferLockBaseAddress(pixelBuffer!, []) == kCVReturnSuccess)\n    defer { precondition(CVPixelBufferUnlockBaseAddress(pixelBuffer!, []) == kCVReturnSuccess) }\n\n    let ciImage = CIImage(cgImage: frameImage)\n    context.render(ciImage, to: pixelBuffer!)\n\n    //  This fails \xe2\x80\x93 the pixel buffer doesn\'t get filled. AT ALL! Why? How to make it work?\n    let bytes = UnsafeBufferPointer(start: CVPixelBufferGetBaseAddress(pixelBuffer!)!.assumingMemoryBound(to: UInt8.self), count: CVPixelBufferGetDataSize(pixelBuffer!))\n    precondition(bytes.contains(where: { $0 != 0 }))\n\n    while !input.isReadyForMoreMediaData { Thread.sleep(forTimeInterval: 10 / 1000) }\n    precondition(pixelBufferAdaptor.append(pixelBuffer!, withPresentationTime: CMTime(seconds: Double(frameNumber) * frameRate, preferredTimescale: 600)))\n}\n\n\n// Unpreferred scenario: using CoreImage to fill the manually created buffer. Proves that CIImage \n// can fill buffer and working.\n\n// for frameNumber in 0 ..< frameCount {\n//     var pixelBuffer: CVPixelBuffer?\n//     precondition(CVPixelBufferCreate(nil, frameImage.width, frameImage.height, kCVPixelFormatType_32ARGB, pixelBufferAttributes as CFDictionary, &pixelBuffer) == kCVReturnSuccess)\n//\n//     precondition(CVPixelBufferLockBaseAddress(pixelBuffer!, []) == kCVReturnSuccess)\n//     defer { precondition(CVPixelBufferUnlockBaseAddress(pixelBuffer!, []) == kCVReturnSuccess) }\n//\n//     let ciImage = CIImage(cgImage: frameImage)\n//     context.render(ciImage, to: pixelBuffer!)\n//\n//     // \xe2\x9c\x85 This passes.\n//     let bytes = UnsafeBufferPointer(start: CVPixelBufferGetBaseAddress(pixelBuffer!)!.assumingMemoryBound(to: UInt8.self), count: CVPixelBufferGetDataSize(pixelBuffer!))\n//     precondition(bytes.contains(where: { $0 != 0 }))\n//\n//     while !input.isReadyForMoreMediaData { Thread.sleep(forTimeInterval: 10 / 1000) }\n//     precondition(pixelBufferAdaptor.append(pixelBuffer!, withPresentationTime: CMTime(seconds: Double(frameNumber) * frameRate, preferredTimescale: 600)))\n// }\n\n\n// Unpreferred scenario: using CoreGraphics to fill the buffer from the pixel buffer adapter. Shows that\n// buffer from pixel buffer adapter can be filled and working.\n\n// for frameNumber in 0 ..< frameCount {\n//     var pixelBuffer: CVPixelBuffer?\n//     guard let pixelBufferPool: CVPixelBufferPool = pixelBufferAdaptor.pixelBufferPool else { preconditionFailure() }\n//     precondition(CVPixelBufferPoolCreatePixelBuffer(nil, pixelBufferPool, &pixelBuffer) == kCVReturnSuccess)\n//\n//     precondition(CVPixelBufferLockBaseAddress(pixelBuffer!, []) == kCVReturnSuccess)\n//     defer { precondition(CVPixelBufferUnlockBaseAddress(pixelBuffer!, []) == kCVReturnSuccess) }\n//\n//     guard let context: CGContext = CGContext(data: CVPixelBufferGetBaseAddress(pixelBuffer!), width: frameImage.width, height: frameImage.height, bitsPerComponent: 8, bytesPerRow: CVPixelBufferGetBytesPerRow(pixelBuffer!), space: colorSpace, bitmapInfo: CGImageAlphaInfo.premultipliedFirst.rawValue) else { preconditionFailure() }\n//     context.clear(CGRect(origin: .zero, size: frameSize))\n//     context.draw(frameImage, in: CGRect(origin: .zero, size: frameSize))\n//\n//     // \xe2\x9c\x85 This passes.\n//     let bytes = UnsafeBufferPointer(start: CVPixelBufferGetBaseAddress(pixelBuffer!)!.assumingMemoryBound(to: UInt8.self), count: CVPixelBufferGetDataSize(pixelBuffer!))\n//     precondition(bytes.contains(where: { $0 != 0 }))\n//\n//     while !input.isReadyForMoreMediaData { Thread.sleep(forTimeInterval: 10 / 1000) }\n//     precondition(pixelBufferAdaptor.append(pixelBuffer!, withPresentationTime: CMTime(seconds: Double(frameNumber) * frameRate, preferredTimescale: 600)))\n// }\n\nlet semaphore = DispatchSemaphore(value: 0)\n\ninput.markAsFinished()\nwriter.endSession(atSourceTime: CMTime(seconds: Double(frameCount) * frameRate, preferredTimescale: 600))\nwriter.finishWriting(completionHandler: { semaphore.signal() })\n\nsemaphore.wait()\n\nSwift.print("Successfully finished rendering to \\(url.path)")\n
Run Code Online (Sandbox Code Playgroud)\n\n

然而，以下内容适用于CGContext，但我需要 CIContext 才能使用 GPU。问题似乎出在AVAssetWriterInputPixelBufferAdaptor缓冲池提供的像素缓冲区上。渲染CIContext到单独创建的缓冲区并将其附加到适配器是可行的，但效率非常低。渲染到适配器池提供的缓冲区中会导致根本CIContext没有数据写入缓冲区，它实际上包含全零，就好像两个不兼容一样！然而，使用渲染是有效的，就像手动复制数据一样。CGImage

\n\n

主要观察结果是，CIContext.render似乎异步工作，或者缓冲区填充和数据写入视频流之间出现问题。换句话说，当缓冲区被刷新时，缓冲区中没有数据。以下内容有点指向这个方向：

\n\n
    \n
  1. 删除缓冲区锁定会导致几乎所有帧都被写入（除了前几个帧），上述代码实际上会产生正确的输出，但对于实际数据，其行为如所描述的那样。
    2. \n
  2. 使用不同的编解码器（例如 ProRes422）会导致几乎所有帧都被正确写入，只有一些空白 \xe2\x80\x93 上面的代码也会产生正确的输出，但较大和复杂的图像会导致跳帧。
    3. \n
\n\n

这段代码有什么问题，正确的方法是什么？

\n\n

PS 大多数 iOS 示例都使用几乎相同的实现，并且看起来工作得很好。我发现一个提示，它可能在 macOS 上有所不同，但看不到任何关于此的官方文档。

\n

Ian*_*hek 4

与 Apple 开发人员技术支持交谈后发现：

Core Image 推迟渲染，直到客户端请求访问帧缓冲区，即CVPixelBufferLockBaseAddress。

因此，解决方案很简单，CVPixelBufferLockBaseAddress调用后执行CIContext.render如下操作：

for frameNumber in 0 ..< frameCount {
    var pixelBuffer: CVPixelBuffer?
    guard let pixelBufferPool: CVPixelBufferPool = pixelBufferAdaptor.pixelBufferPool else { preconditionFailure() }
    precondition(CVPixelBufferPoolCreatePixelBuffer(nil, pixelBufferPool, &pixelBuffer) == kCVReturnSuccess)

    let ciImage = CIImage(cgImage: frameImage)
    context.render(ciImage, to: pixelBuffer!)

    precondition(CVPixelBufferLockBaseAddress(pixelBuffer!, []) == kCVReturnSuccess)
    defer { precondition(CVPixelBufferUnlockBaseAddress(pixelBuffer!, []) == kCVReturnSuccess) }

    let bytes = UnsafeBufferPointer(start: CVPixelBufferGetBaseAddress(pixelBuffer!)!.assumingMemoryBound(to: UInt8.self), count: CVPixelBufferGetDataSize(pixelBuffer!))
    precondition(bytes.contains(where: { $0 != 0 }))

    while !input.isReadyForMoreMediaData { Thread.sleep(forTimeInterval: 10 / 1000) }
    precondition(pixelBufferAdaptor.append(pixelBuffer!, withPresentationTime: CMTime(seconds: Double(frameNumber) * frameRate, preferredTimescale: 600)))
}
Run Code Online (Sandbox Code Playgroud)

归档时间：

查看次数：

3091 次

最近记录：

5 年，5 月 前
相关归档
在视频文件上应用过滤器 17
如何将UIImage/CGImageRef的alpha通道转换为掩码？ 14
在AVCaptureVideoPreviewLayer中精确裁剪捕获的图像 13
AVAssetWriter未知错误 12
对焦(自动对焦)无法在相机中工作(AVFoundation AVCaptureSession) 9
CoreGraphics中的Float比较(相等) 6
在CIFilter中使用来自CIColor的CIImage:获取空图像 5
仅将CIGaussianBlur应用于图像的一小部分 5
将 UIKit 坐标转换为 Quartz/CoreImage (Swift) 3
是否可以在iPhone上使用Core Graphics保存图像？ 2
难疑归档
如何重定向到其他网页？ 7725
如何从当前的Git工作树中删除本地(未跟踪)文件？ 6561
使用Java创建内存泄漏 3076
使用Git版本控制查看文件的更改历史记录 2920
为什么GCC不优化a*a*a*a*a*a到(a*a*a)*(a*a*a)？ 2083
如何在Bash中的分隔符上拆分字符串？ 1885
如何在Java中将数字舍入到n个小数位 1209
哪些字符在CSS类名/选择器中有效？ 1171
如何在Git中仅提交区分大小写的文件名更改？ 1157
如何在JavaScript中将浮点数转换为整数？ 1043