fix: metal renderer packed format crash on emulator and lower iOS version
This commit is contained in:
Acaki committed
1 parent
0d32b8f06f
commit
34eb2ccd31
3 files changed
+145
-38
No files matched your search
@@ -142,12 +142,18 @@ CFStringRef __currentColorSpace;
|
||||
dispatch_queue_attr_make_with_qos_class(DISPATCH_QUEUE_SERIAL, QOS_CLASS_USER_INTERACTIVE, 0));
|
||||
_averageGPUTime = (1.0f / framerate) / 2;
|
||||
_device = device;
|
||||
_colorPixelFormat = MTLPixelFormatBGR10A2Unorm;
|
||||
_colorPixelFormat = drawablePixelFormat;
|
||||
_framerate = framerate;
|
||||
_commandQueue = [_device newCommandQueue];
|
||||
_currentEDRHeadroom = 1.0f;
|
||||
_lastColorSpace = -1;
|
||||
_lastFullRange = NO;
|
||||
|
||||
// Initialize default CSC parameters buffer with BT.601 limited range
|
||||
struct ParamBuffer defaultParamBuffer;
|
||||
defaultParamBuffer.cscParams = k_CscParams_Bt601Lim;
|
||||
MTLResourceOptions bufferOptions = MTLResourceStorageModeShared;
|
||||
_CscParamsBuffer = [_device newBufferWithBytes:(void *)&defaultParamBuffer length:sizeof(defaultParamBuffer) options:bufferOptions];
|
||||
_lastPresented = 0.0f;
|
||||
_inFlightSemaphore = dispatch_semaphore_create(MaxFramesInFlight);
|
||||
_isStopping = NO;
|
||||
@@ -512,6 +518,15 @@ CFStringRef __currentColorSpace;
|
||||
}
|
||||
|
||||
size_t planes = CVPixelBufferGetPlaneCount(frame.pixelBuffer);
|
||||
|
||||
// For packed formats like BGRA, plane count is 0 but we treat it as 1 plane
|
||||
OSType pixelFormatType = CVPixelBufferGetPixelFormatType(frame.pixelBuffer);
|
||||
BOOL isPackedFormat = (pixelFormatType == kCVPixelFormatType_32BGRA ||
|
||||
pixelFormatType == kCVPixelFormatType_32ARGB);
|
||||
if (isPackedFormat) {
|
||||
planes = 1; // Treat packed formats as single plane
|
||||
}
|
||||
|
||||
assert(planes <= MAX_VIDEO_PLANES);
|
||||
|
||||
if (layerDidChange && frame.frameNumber > 1) {
|
||||
@@ -548,16 +563,26 @@ CFStringRef __currentColorSpace;
|
||||
// linear shaders
|
||||
id<MTLFunction> yuvToLinear = [defaultLibrary newFunctionWithName:@"yuvToLinear"];
|
||||
|
||||
// Determine if this is 10-bit based on the framebuffer's pixel format
|
||||
BOOL is10Bit = (framebufferPixelFormat == MTLPixelFormatBGR10A2Unorm);
|
||||
// Determine if this is 10-bit based on the input CVPixelBuffer format, not the output framebuffer format
|
||||
// pixelFormatType is already declared above
|
||||
BOOL is10BitInput = (pixelFormatType == kCVPixelFormatType_420YpCbCr10BiPlanarFullRange ||
|
||||
pixelFormatType == kCVPixelFormatType_444YpCbCr10BiPlanarFullRange ||
|
||||
pixelFormatType == kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange ||
|
||||
pixelFormatType == kCVPixelFormatType_444YpCbCr10BiPlanarVideoRange);
|
||||
|
||||
NSString *fragmentShaderName;
|
||||
if (planes == 2) {
|
||||
fragmentShaderName = is10Bit ? @"ps_draw_biplanar_10bit" : @"ps_draw_biplanar_8bit";
|
||||
if (isPackedFormat) {
|
||||
// BGRA/ARGB packed formats don't need color space conversion
|
||||
fragmentShaderName = @"ps_draw_bgra";
|
||||
} else if (planes == 2) {
|
||||
fragmentShaderName = is10BitInput ? @"ps_draw_biplanar_10bit" : @"ps_draw_biplanar_8bit";
|
||||
} else {
|
||||
fragmentShaderName = is10Bit ? @"ps_draw_triplanar_10bit" : @"ps_draw_triplanar_8bit";
|
||||
fragmentShaderName = is10BitInput ? @"ps_draw_triplanar_10bit" : @"ps_draw_triplanar_8bit";
|
||||
}
|
||||
|
||||
Log(LOG_I, @"DEBUG: CVPixelBuffer format: 0x%X, planes: %zu, is10BitInput: %d, shader: %@, framebuffer format: %lu",
|
||||
pixelFormatType, planes, is10BitInput, fragmentShaderName, (unsigned long)framebufferPixelFormat);
|
||||
|
||||
pipelineDesc.colorAttachments[0].pixelFormat = framebufferPixelFormat;
|
||||
pipelineDesc.vertexBuffers[0].mutability = MTLMutabilityImmutable;
|
||||
|
||||
@@ -585,41 +610,84 @@ CFStringRef __currentColorSpace;
|
||||
_videoPipelinePixelFormat[planes] = framebufferPixelFormat;
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < planes; i++) {
|
||||
MTLPixelFormat fmt;
|
||||
|
||||
switch (CVPixelBufferGetPixelFormatType(frame.pixelBuffer)) {
|
||||
case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange:
|
||||
case kCVPixelFormatType_444YpCbCr8BiPlanarVideoRange:
|
||||
case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange:
|
||||
case kCVPixelFormatType_444YpCbCr8BiPlanarFullRange:
|
||||
fmt = (i == 0) ? MTLPixelFormatR8Unorm : MTLPixelFormatRG8Unorm;
|
||||
break;
|
||||
|
||||
case kCVPixelFormatType_420YpCbCr10BiPlanarFullRange:
|
||||
case kCVPixelFormatType_444YpCbCr10BiPlanarFullRange:
|
||||
case kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange:
|
||||
case kCVPixelFormatType_444YpCbCr10BiPlanarVideoRange:
|
||||
fmt = (i == 0) ? MTLPixelFormatR16Unorm : MTLPixelFormatRG16Unorm;
|
||||
break;
|
||||
|
||||
default:
|
||||
Log(LOG_E, @"Unknown pixel format: %@", CVPixelBufferGetPixelFormatType(frame.pixelBuffer));
|
||||
return;
|
||||
if (isPackedFormat) {
|
||||
// Handle packed BGRA format - iOS/macOS uses BGRA internally
|
||||
// Check if the pixel buffer has IOSurface backing
|
||||
CFTypeRef ioSurface = CVPixelBufferGetIOSurface(frame.pixelBuffer);
|
||||
if (!ioSurface) {
|
||||
Log(LOG_E, @"CVPixelBuffer does not have IOSurface backing - cannot create Metal texture");
|
||||
return;
|
||||
}
|
||||
|
||||
CVReturn err = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault,
|
||||
_textureCache,
|
||||
frame.pixelBuffer,
|
||||
NULL,
|
||||
fmt,
|
||||
CVPixelBufferGetWidthOfPlane(frame.pixelBuffer, i),
|
||||
CVPixelBufferGetHeightOfPlane(frame.pixelBuffer, i),
|
||||
i,
|
||||
&_cvMetalTextures[i]);
|
||||
MTLPixelFormatBGRA8Unorm,
|
||||
CVPixelBufferGetWidth(frame.pixelBuffer),
|
||||
CVPixelBufferGetHeight(frame.pixelBuffer),
|
||||
0, // planeIndex must be 0 for non-planar
|
||||
&_cvMetalTextures[0]);
|
||||
if (err != kCVReturnSuccess) {
|
||||
Log(LOG_E, @"CVMetalTextureCacheCreateTextureFromImage() failed: %d", err);
|
||||
Log(LOG_E, @"CVMetalTextureCacheCreateTextureFromImage() failed for BGRA: %d", err);
|
||||
Log(LOG_E, @"PixelBuffer info - format: 0x%X, width: %zu, height: %zu, IOSurface: %p",
|
||||
pixelFormatType,
|
||||
CVPixelBufferGetWidth(frame.pixelBuffer),
|
||||
CVPixelBufferGetHeight(frame.pixelBuffer),
|
||||
ioSurface);
|
||||
return;
|
||||
} else {
|
||||
id<MTLTexture> texture = CVMetalTextureGetTexture(_cvMetalTextures[0]);
|
||||
Log(LOG_I, @"DEBUG: Created BGRA texture: format=%lu, width=%zu, height=%zu",
|
||||
(unsigned long)MTLPixelFormatBGRA8Unorm,
|
||||
CVPixelBufferGetWidth(frame.pixelBuffer),
|
||||
CVPixelBufferGetHeight(frame.pixelBuffer));
|
||||
}
|
||||
} else {
|
||||
// Handle planar YUV formats
|
||||
size_t actualPlanes = CVPixelBufferGetPlaneCount(frame.pixelBuffer);
|
||||
for (size_t i = 0; i < actualPlanes; i++) {
|
||||
MTLPixelFormat fmt;
|
||||
|
||||
switch (CVPixelBufferGetPixelFormatType(frame.pixelBuffer)) {
|
||||
case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange:
|
||||
case kCVPixelFormatType_444YpCbCr8BiPlanarVideoRange:
|
||||
case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange:
|
||||
case kCVPixelFormatType_444YpCbCr8BiPlanarFullRange:
|
||||
fmt = (i == 0) ? MTLPixelFormatR8Unorm : MTLPixelFormatRG8Unorm;
|
||||
break;
|
||||
|
||||
case kCVPixelFormatType_420YpCbCr10BiPlanarFullRange:
|
||||
case kCVPixelFormatType_444YpCbCr10BiPlanarFullRange:
|
||||
case kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange:
|
||||
case kCVPixelFormatType_444YpCbCr10BiPlanarVideoRange:
|
||||
fmt = (i == 0) ? MTLPixelFormatR16Unorm : MTLPixelFormatRG16Unorm;
|
||||
break;
|
||||
|
||||
default:
|
||||
Log(LOG_E, @"Unknown pixel format: %@", CVPixelBufferGetPixelFormatType(frame.pixelBuffer));
|
||||
return;
|
||||
}
|
||||
|
||||
CVReturn err = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault,
|
||||
_textureCache,
|
||||
frame.pixelBuffer,
|
||||
NULL,
|
||||
fmt,
|
||||
CVPixelBufferGetWidthOfPlane(frame.pixelBuffer, i),
|
||||
CVPixelBufferGetHeightOfPlane(frame.pixelBuffer, i),
|
||||
i,
|
||||
&_cvMetalTextures[i]);
|
||||
if (err != kCVReturnSuccess) {
|
||||
Log(LOG_E, @"CVMetalTextureCacheCreateTextureFromImage() failed: %d", err);
|
||||
return;
|
||||
} else {
|
||||
id<MTLTexture> texture = CVMetalTextureGetTexture(_cvMetalTextures[i]);
|
||||
Log(LOG_I, @"DEBUG: Created texture for plane %zu: format=%lu, width=%zu, height=%zu",
|
||||
i, (unsigned long)fmt,
|
||||
CVPixelBufferGetWidthOfPlane(frame.pixelBuffer, i),
|
||||
CVPixelBufferGetHeightOfPlane(frame.pixelBuffer, i));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -629,12 +697,24 @@ CFStringRef __currentColorSpace;
|
||||
id<MTLRenderCommandEncoder> renderEncoder = [commandBuffer renderCommandEncoderWithDescriptor:_renderPassDescriptor];
|
||||
|
||||
[renderEncoder setRenderPipelineState:_videoPipelineState[planes]];
|
||||
for (size_t i = 0; i < planes; i++) {
|
||||
[renderEncoder setFragmentTexture:CVMetalTextureGetTexture(_cvMetalTextures[i]) atIndex:i];
|
||||
|
||||
if (isPackedFormat) {
|
||||
// For packed formats, we only have one texture
|
||||
[renderEncoder setFragmentTexture:CVMetalTextureGetTexture(_cvMetalTextures[0]) atIndex:0];
|
||||
} else {
|
||||
// For planar formats, set multiple textures
|
||||
size_t actualPlanes = CVPixelBufferGetPlaneCount(frame.pixelBuffer);
|
||||
for (size_t i = 0; i < actualPlanes; i++) {
|
||||
[renderEncoder setFragmentTexture:CVMetalTextureGetTexture(_cvMetalTextures[i]) atIndex:i];
|
||||
}
|
||||
}
|
||||
|
||||
[renderEncoder setVertexBuffer:_VideoVertexBuffer offset:0 atIndex:0];
|
||||
[renderEncoder setFragmentBuffer:_CscParamsBuffer offset:0 atIndex:0];
|
||||
|
||||
// Only set CSC params buffer for YUV formats that need color space conversion
|
||||
if (!isPackedFormat) {
|
||||
[renderEncoder setFragmentBuffer:_CscParamsBuffer offset:0 atIndex:0];
|
||||
}
|
||||
#if !TARGET_OS_TV
|
||||
if (layer.pixelFormat == MTLPixelFormatRGBA16Float) {
|
||||
[self pollCurrentEDRHeadroom];
|
||||
@@ -657,6 +737,7 @@ CFStringRef __currentColorSpace;
|
||||
|
||||
// signal semaphore, compute GPU time average, and clear textures
|
||||
__block dispatch_semaphore_t block_semaphore = _inFlightSemaphore;
|
||||
__block size_t texturesToClean = isPackedFormat ? 1 : CVPixelBufferGetPlaneCount(frame.pixelBuffer);
|
||||
[commandBuffer addCompletedHandler:^(id<MTLCommandBuffer> cb) {
|
||||
dispatch_semaphore_signal(block_semaphore);
|
||||
|
||||
@@ -665,7 +746,7 @@ CFStringRef __currentColorSpace;
|
||||
self->_averageGPUTime = (GPUTime * alpha) + (self->_averageGPUTime * (1.0 - alpha));
|
||||
|
||||
// Free textures after completion of rendering
|
||||
for (size_t i = 0; i < planes; i++) {
|
||||
for (size_t i = 0; i < texturesToClean; i++) {
|
||||
if (self->_cvMetalTextures[i]) {
|
||||
CFRelease(self->_cvMetalTextures[i]);
|
||||
self->_cvMetalTextures[i] = nil;
|
||||
|
||||
Reference in new issue
Block a user