本文介绍了使CIContext.Render(CIImage,CVPixelBuffer)使用AVAssetWriter的处理方法,对大家解决问题具有一定的参考价值,需要的朋友们下面随着小编来一起学习吧!
问题描述
我要使用Core Image处理一组CGImage
对象,并在MacOS上将它们转换为QuickTime电影。以下代码演示了所需的内容,但output contains a lot of blank (black) frames:
import AppKit
import AVFoundation
import CoreGraphics
import Foundation
import CoreVideo
import Metal
// Video output url.
let url: URL = try! FileManager.default.url(for: .downloadsDirectory, in: .userDomainMask, appropriateFor: nil, create: false).appendingPathComponent("av.mov")
try? FileManager.default.removeItem(at: url)
// Video frame size, total frame count, frame rate and frame image.
let frameSize: CGSize = CGSize(width: 2000, height: 1000)
let frameCount: Int = 100
let frameRate: Double = 1 / 30
let frameImage: CGImage
frameImage = NSImage(size: frameSize, flipped: false, drawingHandler: {
NSColor.red.setFill()
$0.fill()
return true
}).cgImage(forProposedRect: nil, context: nil, hints: nil)!
let pixelBufferAttributes: [CFString: Any]
let outputSettings: [String: Any]
pixelBufferAttributes = [
kCVPixelBufferPixelFormatTypeKey: Int(kCVPixelFormatType_32ARGB),
kCVPixelBufferWidthKey: Float(frameSize.width),
kCVPixelBufferHeightKey: Float(frameSize.height),
kCVPixelBufferMetalCompatibilityKey: true,
kCVPixelBufferCGImageCompatibilityKey: true,
kCVPixelBufferCGBitmapContextCompatibilityKey: true,
]
outputSettings = [
AVVideoCodecKey: AVVideoCodecType.h264,
AVVideoWidthKey: Int(frameSize.width),
AVVideoHeightKey: Int(frameSize.height),
]
let writer: AVAssetWriter = try! AVAssetWriter(outputURL: url, fileType: .mov)
let input: AVAssetWriterInput = AVAssetWriterInput(mediaType: .video, outputSettings: outputSettings)
let pixelBufferAdaptor: AVAssetWriterInputPixelBufferAdaptor = AVAssetWriterInputPixelBufferAdaptor(assetWriterInput: input, sourcePixelBufferAttributes: pixelBufferAttributes as [String: Any])
input.expectsMediaDataInRealTime = true
precondition(writer.canAdd(input))
writer.add(input)
precondition(writer.startWriting())
writer.startSession(atSourceTime: CMTime.zero)
let colorSpace: CGColorSpace = CGColorSpace(name: CGColorSpace.sRGB) ?? CGColorSpaceCreateDeviceRGB()
let context = CIContext(mtlDevice: MTLCreateSystemDefaultDevice()!)
Swift.print("Starting the render…")
// Preferred scenario: using CoreImage to fill the buffer from the pixel buffer adapter. Shows that
// CIImage + AVAssetWriterInputPixelBufferAdaptor are not working together.
for frameNumber in 0 ..< frameCount {
var pixelBuffer: CVPixelBuffer?
guard let pixelBufferPool: CVPixelBufferPool = pixelBufferAdaptor.pixelBufferPool else { preconditionFailure() }
precondition(CVPixelBufferPoolCreatePixelBuffer(nil, pixelBufferPool, &pixelBuffer) == kCVReturnSuccess)
precondition(CVPixelBufferLockBaseAddress(pixelBuffer!, []) == kCVReturnSuccess)
defer { precondition(CVPixelBufferUnlockBaseAddress(pixelBuffer!, []) == kCVReturnSuccess) }
let ciImage = CIImage(cgImage: frameImage)
context.render(ciImage, to: pixelBuffer!)
// 💥 This fails – the pixel buffer doesn't get filled. AT ALL! Why? How to make it work?
let bytes = UnsafeBufferPointer(start: CVPixelBufferGetBaseAddress(pixelBuffer!)!.assumingMemoryBound(to: UInt8.self), count: CVPixelBufferGetDataSize(pixelBuffer!))
precondition(bytes.contains(where: { $0 != 0 }))
while !input.isReadyForMoreMediaData { Thread.sleep(forTimeInterval: 10 / 1000) }
precondition(pixelBufferAdaptor.append(pixelBuffer!, withPresentationTime: CMTime(seconds: Double(frameNumber) * frameRate, preferredTimescale: 600)))
}
// Unpreferred scenario: using CoreImage to fill the manually created buffer. Proves that CIImage
// can fill buffer and working.
// for frameNumber in 0 ..< frameCount {
// var pixelBuffer: CVPixelBuffer?
// precondition(CVPixelBufferCreate(nil, frameImage.width, frameImage.height, kCVPixelFormatType_32ARGB, pixelBufferAttributes as CFDictionary, &pixelBuffer) == kCVReturnSuccess)
//
// precondition(CVPixelBufferLockBaseAddress(pixelBuffer!, []) == kCVReturnSuccess)
// defer { precondition(CVPixelBufferUnlockBaseAddress(pixelBuffer!, []) == kCVReturnSuccess) }
//
// let ciImage = CIImage(cgImage: frameImage)
// context.render(ciImage, to: pixelBuffer!)
//
// // ✅ This passes.
// let bytes = UnsafeBufferPointer(start: CVPixelBufferGetBaseAddress(pixelBuffer!)!.assumingMemoryBound(to: UInt8.self), count: CVPixelBufferGetDataSize(pixelBuffer!))
// precondition(bytes.contains(where: { $0 != 0 }))
//
// while !input.isReadyForMoreMediaData { Thread.sleep(forTimeInterval: 10 / 1000) }
// precondition(pixelBufferAdaptor.append(pixelBuffer!, withPresentationTime: CMTime(seconds: Double(frameNumber) * frameRate, preferredTimescale: 600)))
// }
// Unpreferred scenario: using CoreGraphics to fill the buffer from the pixel buffer adapter. Shows that
// buffer from pixel buffer adapter can be filled and working.
// for frameNumber in 0 ..< frameCount {
// var pixelBuffer: CVPixelBuffer?
// guard let pixelBufferPool: CVPixelBufferPool = pixelBufferAdaptor.pixelBufferPool else { preconditionFailure() }
// precondition(CVPixelBufferPoolCreatePixelBuffer(nil, pixelBufferPool, &pixelBuffer) == kCVReturnSuccess)
//
// precondition(CVPixelBufferLockBaseAddress(pixelBuffer!, []) == kCVReturnSuccess)
// defer { precondition(CVPixelBufferUnlockBaseAddress(pixelBuffer!, []) == kCVReturnSuccess) }
//
// guard let context: CGContext = CGContext(data: CVPixelBufferGetBaseAddress(pixelBuffer!), width: frameImage.width, height: frameImage.height, bitsPerComponent: 8, bytesPerRow: CVPixelBufferGetBytesPerRow(pixelBuffer!), space: colorSpace, bitmapInfo: CGImageAlphaInfo.premultipliedFirst.rawValue) else { preconditionFailure() }
// context.clear(CGRect(origin: .zero, size: frameSize))
// context.draw(frameImage, in: CGRect(origin: .zero, size: frameSize))
//
// // ✅ This passes.
// let bytes = UnsafeBufferPointer(start: CVPixelBufferGetBaseAddress(pixelBuffer!)!.assumingMemoryBound(to: UInt8.self), count: CVPixelBufferGetDataSize(pixelBuffer!))
// precondition(bytes.contains(where: { $0 != 0 }))
//
// while !input.isReadyForMoreMediaData { Thread.sleep(forTimeInterval: 10 / 1000) }
// precondition(pixelBufferAdaptor.append(pixelBuffer!, withPresentationTime: CMTime(seconds: Double(frameNumber) * frameRate, preferredTimescale: 600)))
// }
let semaphore = DispatchSemaphore(value: 0)
input.markAsFinished()
writer.endSession(atSourceTime: CMTime(seconds: Double(frameCount) * frameRate, preferredTimescale: 600))
writer.finishWriting(completionHandler: { semaphore.signal() })
semaphore.wait()
Swift.print("Successfully finished rendering to (url.path)")
但是,以下内容适用于CGContext
,但我需要CIContext
才能使用GPU。问题似乎出在AVAssetWriterInputPixelBufferAdaptor
的缓冲池提供的像素缓冲区上。将CIContext
渲染到单独创建的缓冲区并将其附加到适配器上是可行的,但效率非常低。渲染CIContext
到适配器的池提供的缓冲区会导致根本没有数据写入缓冲区,其字面意思是全零,就好像两个不兼容!但是,使用CGImage
渲染是可行的,因此可以手动复制数据。<
主要观察到CIContext.render
似乎异步工作,或者在填充缓冲区和将数据写入视频流之间出现问题。换句话说,当缓冲区被刷新时,缓冲区中没有数据。以下内容有点指向那个方向:
- 删除缓冲区锁定会导致几乎所有帧都被写入,除了前几个帧之外,上面的代码实际上会生成correct output,但对于实际数据,其行为如所述。
- 使用不同的编解码器(如ProRes422)会导致几乎所有帧都被正确写入,只有几个空白-此外,上面的代码也会生成correct output,但较大且复杂的图像会导致跳过帧。
此代码有什么问题,正确的操作方法是什么?
附注:大多数iOS示例使用几乎相同的实现,并且看起来工作得非常好。我发现一个hint对于MacOS可能有所不同,但看不到任何有关此问题的官方文档。推荐答案
与苹果开发者技术支持联系后发现:
所以解决方案就是在调用CIContext.render
后,简单地做CVPixelBufferLockBaseAddress
,如下图所示:
for frameNumber in 0 ..< frameCount {
var pixelBuffer: CVPixelBuffer?
guard let pixelBufferPool: CVPixelBufferPool = pixelBufferAdaptor.pixelBufferPool else { preconditionFailure() }
precondition(CVPixelBufferPoolCreatePixelBuffer(nil, pixelBufferPool, &pixelBuffer) == kCVReturnSuccess)
let ciImage = CIImage(cgImage: frameImage)
context.render(ciImage, to: pixelBuffer!)
precondition(CVPixelBufferLockBaseAddress(pixelBuffer!, []) == kCVReturnSuccess)
defer { precondition(CVPixelBufferUnlockBaseAddress(pixelBuffer!, []) == kCVReturnSuccess) }
let bytes = UnsafeBufferPointer(start: CVPixelBufferGetBaseAddress(pixelBuffer!)!.assumingMemoryBound(to: UInt8.self), count: CVPixelBufferGetDataSize(pixelBuffer!))
precondition(bytes.contains(where: { $0 != 0 }))
while !input.isReadyForMoreMediaData { Thread.sleep(forTimeInterval: 10 / 1000) }
precondition(pixelBufferAdaptor.append(pixelBuffer!, withPresentationTime: CMTime(seconds: Double(frameNumber) * frameRate, preferredTimescale: 600)))
}
这篇关于使CIContext.Render(CIImage,CVPixelBuffer)使用AVAssetWriter的文章就介绍到这了,希望我们推荐的答案对大家有所帮助,也希望大家多多支持!