hermes-agent/VoidLink
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: metal renderer packed format crash on emulator and lower iOS version

Browse Source
This commit is contained in:
Acaki
2025-08-28 00:07:56 +08:00
parent 49ed0e43e9
commit c7dd7a2c21
3 changed files with 140 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
VoidLink/Metal/MetalVideoRenderer.m
+112 -37
View File
@@ -142,13 +142,19 @@ CFStringRef __currentColorSpace;
dispatch_queue_attr_make_with_qos_class(DISPATCH_QUEUE_SERIAL, QOS_CLASS_USER_INTERACTIVE, 0));
_averageGPUTime = (1.0f / framerate) / 2;
_device = device;
_colorPixelFormat = MTLPixelFormatBGR10A2Unorm;
_colorPixelFormat = drawablePixelFormat;
_framerate = framerate;
_hdrEnabled = hdrEnabled;
_commandQueue = [_device newCommandQueue];
_currentEDRHeadroom = 1.0f;
_lastColorSpace = -1;
_lastFullRange = NO;
// Initialize default CSC parameters buffer with BT.601 limited range
struct ParamBuffer defaultParamBuffer;
defaultParamBuffer.cscParams = k_CscParams_Bt601Lim;
MTLResourceOptions bufferOptions = MTLResourceStorageModeShared;
_CscParamsBuffer = [_device newBufferWithBytes:(void *)&defaultParamBuffer length:sizeof(defaultParamBuffer) options:bufferOptions];
_lastPresented = 0.0f;
_inFlightSemaphore = dispatch_semaphore_create(MaxFramesInFlight);
_isStopping = NO;
@@ -583,6 +589,15 @@ CFStringRef __currentColorSpace;
}
size_t planes = CVPixelBufferGetPlaneCount(frame.pixelBuffer);
// For packed formats like BGRA, plane count is 0 but we treat it as 1 plane
OSType pixelFormatType = CVPixelBufferGetPixelFormatType(frame.pixelBuffer);
BOOL isPackedFormat = (pixelFormatType == kCVPixelFormatType_32BGRA ||
pixelFormatType == kCVPixelFormatType_32ARGB);
if (isPackedFormat) {
planes = 1; // Treat packed formats as single plane
}
assert(planes <= MAX_VIDEO_PLANES);
if (layerDidChange && frame.frameNumber > 1) {
@@ -621,16 +636,26 @@ CFStringRef __currentColorSpace;
// linear shaders
id<MTLFunction> yuvToLinear = [defaultLibrary newFunctionWithName:@"yuvToLinear"];
// Determine if this is 10-bit based on the framebuffer's pixel format
BOOL is10Bit = (framebufferPixelFormat == MTLPixelFormatBGR10A2Unorm);
// Determine if this is 10-bit based on the input CVPixelBuffer format, not the output framebuffer format
// pixelFormatType is already declared above
BOOL is10BitInput = (pixelFormatType == kCVPixelFormatType_420YpCbCr10BiPlanarFullRange ||
pixelFormatType == kCVPixelFormatType_444YpCbCr10BiPlanarFullRange ||
pixelFormatType == kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange ||
pixelFormatType == kCVPixelFormatType_444YpCbCr10BiPlanarVideoRange);
NSString *fragmentShaderName;
if (planes == 2) {
fragmentShaderName = is10Bit ? @"ps_draw_biplanar_10bit" : @"ps_draw_biplanar_8bit";
if (isPackedFormat) {
// BGRA/ARGB packed formats don't need color space conversion
fragmentShaderName = @"ps_draw_bgra";
} else if (planes == 2) {
fragmentShaderName = is10BitInput ? @"ps_draw_biplanar_10bit" : @"ps_draw_biplanar_8bit";
} else {
fragmentShaderName = is10Bit ? @"ps_draw_triplanar_10bit" : @"ps_draw_triplanar_8bit";
fragmentShaderName = is10BitInput ? @"ps_draw_triplanar_10bit" : @"ps_draw_triplanar_8bit";
}
Log(LOG_I, @"DEBUG: CVPixelBuffer format: 0x%X, planes: %zu, is10BitInput: %d, shader: %@, framebuffer format: %lu",
pixelFormatType, planes, is10BitInput, fragmentShaderName, (unsigned long)framebufferPixelFormat);
pipelineDesc.colorAttachments[0].pixelFormat = framebufferPixelFormat;
pipelineDesc.vertexBuffers[0].mutability = MTLMutabilityImmutable;
@@ -658,41 +683,78 @@ CFStringRef __currentColorSpace;
_videoPipelinePixelFormat[planes] = framebufferPixelFormat;
}
for (size_t i = 0; i < planes; i++) {
MTLPixelFormat fmt;
switch (CVPixelBufferGetPixelFormatType(frame.pixelBuffer)) {
case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange:
case kCVPixelFormatType_444YpCbCr8BiPlanarVideoRange:
case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange:
case kCVPixelFormatType_444YpCbCr8BiPlanarFullRange:
fmt = (i == 0) ? MTLPixelFormatR8Unorm : MTLPixelFormatRG8Unorm;
break;
case kCVPixelFormatType_420YpCbCr10BiPlanarFullRange:
case kCVPixelFormatType_444YpCbCr10BiPlanarFullRange:
case kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange:
case kCVPixelFormatType_444YpCbCr10BiPlanarVideoRange:
fmt = (i == 0) ? MTLPixelFormatR16Unorm : MTLPixelFormatRG16Unorm;
break;
default:
Log(LOG_E, @"Unknown pixel format: %@", CVPixelBufferGetPixelFormatType(frame.pixelBuffer));
return;
if (isPackedFormat) {
// Handle packed BGRA format - iOS/macOS uses BGRA internally
// Check if the pixel buffer has IOSurface backing
CFTypeRef ioSurface = CVPixelBufferGetIOSurface(frame.pixelBuffer);
if (!ioSurface) {
Log(LOG_E, @"CVPixelBuffer does not have IOSurface backing - cannot create Metal texture");
return;
}
CVReturn err = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault,
_textureCache,
frame.pixelBuffer,
NULL,
fmt,
CVPixelBufferGetWidthOfPlane(frame.pixelBuffer, i),
CVPixelBufferGetHeightOfPlane(frame.pixelBuffer, i),
i,
&_cvMetalTextures[i]);
MTLPixelFormatBGRA8Unorm,
CVPixelBufferGetWidth(frame.pixelBuffer),
CVPixelBufferGetHeight(frame.pixelBuffer),
0, // planeIndex must be 0 for non-planar
&_cvMetalTextures[0]);
if (err != kCVReturnSuccess) {
Log(LOG_E, @"CVMetalTextureCacheCreateTextureFromImage() failed: %d", err);
Log(LOG_E, @"CVMetalTextureCacheCreateTextureFromImage() failed for BGRA: %d", err);
Log(LOG_E, @"PixelBuffer info - format: 0x%X, width: %zu, height: %zu, IOSurface: %p",
pixelFormatType,
CVPixelBufferGetWidth(frame.pixelBuffer),
CVPixelBufferGetHeight(frame.pixelBuffer),
ioSurface);
return;
} else {
id<MTLTexture> texture = CVMetalTextureGetTexture(_cvMetalTextures[0]);
Log(LOG_I, @"DEBUG: Created BGRA texture: format=%lu, width=%zu, height=%zu",
(unsigned long)MTLPixelFormatBGRA8Unorm,
CVPixelBufferGetWidth(frame.pixelBuffer),
CVPixelBufferGetHeight(frame.pixelBuffer));
}
} else {
// Handle planar YUV formats
size_t actualPlanes = CVPixelBufferGetPlaneCount(frame.pixelBuffer);
for (size_t i = 0; i < actualPlanes; i++) {
MTLPixelFormat fmt;
switch (CVPixelBufferGetPixelFormatType(frame.pixelBuffer)) {
case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange:
case kCVPixelFormatType_444YpCbCr8BiPlanarVideoRange:
case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange:
case kCVPixelFormatType_444YpCbCr8BiPlanarFullRange:
fmt = (i == 0) ? MTLPixelFormatR8Unorm : MTLPixelFormatRG8Unorm;
break;
case kCVPixelFormatType_420YpCbCr10BiPlanarFullRange:
case kCVPixelFormatType_444YpCbCr10BiPlanarFullRange:
case kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange:
case kCVPixelFormatType_444YpCbCr10BiPlanarVideoRange:
fmt = (i == 0) ? MTLPixelFormatR16Unorm : MTLPixelFormatRG16Unorm;
break;
default:
Log(LOG_E, @"Unknown pixel format: %@", CVPixelBufferGetPixelFormatType(frame.pixelBuffer));
return;
}
CVReturn err = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault,
_textureCache,
frame.pixelBuffer,
NULL,
fmt,
CVPixelBufferGetWidthOfPlane(frame.pixelBuffer, i),
CVPixelBufferGetHeightOfPlane(frame.pixelBuffer, i),
i,
&_cvMetalTextures[i]);
if (err != kCVReturnSuccess) {
Log(LOG_E, @"CVMetalTextureCacheCreateTextureFromImage() failed: %d", err);
return;
}
}
}
@@ -702,12 +764,24 @@ CFStringRef __currentColorSpace;
id<MTLRenderCommandEncoder> renderEncoder = [commandBuffer renderCommandEncoderWithDescriptor:_renderPassDescriptor];
[renderEncoder setRenderPipelineState:_videoPipelineState[planes]];
for (size_t i = 0; i < planes; i++) {
[renderEncoder setFragmentTexture:CVMetalTextureGetTexture(_cvMetalTextures[i]) atIndex:i];
if (isPackedFormat) {
// For packed formats, we only have one texture
[renderEncoder setFragmentTexture:CVMetalTextureGetTexture(_cvMetalTextures[0]) atIndex:0];
} else {
// For planar formats, set multiple textures
size_t actualPlanes = CVPixelBufferGetPlaneCount(frame.pixelBuffer);
for (size_t i = 0; i < actualPlanes; i++) {
[renderEncoder setFragmentTexture:CVMetalTextureGetTexture(_cvMetalTextures[i]) atIndex:i];
}
}
[renderEncoder setVertexBuffer:_VideoVertexBuffer offset:0 atIndex:0];
[renderEncoder setFragmentBuffer:_CscParamsBuffer offset:0 atIndex:0];
// Only set CSC params buffer for YUV formats that need color space conversion
if (!isPackedFormat) {
[renderEncoder setFragmentBuffer:_CscParamsBuffer offset:0 atIndex:0];
}
#if !TARGET_OS_TV
if (layer.pixelFormat == MTLPixelFormatRGBA16Float) {
[self pollCurrentEDRHeadroom];
@@ -730,6 +804,7 @@ CFStringRef __currentColorSpace;
// signal semaphore, compute GPU time average, and clear textures
__block dispatch_semaphore_t block_semaphore = _inFlightSemaphore;
__block size_t texturesToClean = isPackedFormat ? 1 : CVPixelBufferGetPlaneCount(frame.pixelBuffer);
[commandBuffer addCompletedHandler:^(id<MTLCommandBuffer> cb) {
dispatch_semaphore_signal(block_semaphore);
@@ -738,7 +813,7 @@ CFStringRef __currentColorSpace;
self->_averageGPUTime = (GPUTime * alpha) + (self->_averageGPUTime * (1.0 - alpha));
// Free textures after completion of rendering
for (size_t i = 0; i < planes; i++) {
for (size_t i = 0; i < texturesToClean; i++) {
if (self->_cvMetalTextures[i]) {
CFRelease(self->_cvMetalTextures[i]);
self->_cvMetalTextures[i] = NULL;
VoidLink/Metal/MetalViewController.m
+20 -2
View File
@@ -59,9 +59,27 @@
}
view.metalLayer.device = device;
// Determine supported pixel format before initializing renderer
// Use TARGET_OS_SIMULATOR to detect simulator environment
MTLPixelFormat pixelFormat;
#if TARGET_OS_SIMULATOR
// iOS Simulator doesn't support BGR10A2Unorm
pixelFormat = MTLPixelFormatBGRA8Unorm;
Log(LOG_W, @"Running on iOS Simulator, using BGRA8Unorm pixel format");
#else
// On real devices, check if we should enable HDR
if (_enableHdr) {
pixelFormat = MTLPixelFormatBGR10A2Unorm;
Log(LOG_I, @"HDR enabled, using BGR10A2Unorm pixel format");
} else {
pixelFormat = MTLPixelFormatBGRA8Unorm;
Log(LOG_I, @"HDR disabled, using BGRA8Unorm pixel format");
}
#endif
// Initialize the renderer.
MetalVideoRenderer *renderer = [[MetalVideoRenderer alloc] initWithMetalDevice:device
drawablePixelFormat:MTLPixelFormatBGR10A2Unorm
drawablePixelFormat:pixelFormat
framerate:self->_framerate
hdrEnabled:self->_enableHdr];
if (!renderer) {
@@ -162,7 +180,7 @@
[_renderer shutdown];
_renderer = nil;
}
if (_metalView) {
_metalView.delegate = nil;
[_metalView shutdown];
VoidLink/Metal/Shaders.metal
+8
View File
@@ -127,3 +127,11 @@ fragment float4 ps_draw_triplanar_10bit(Vertex v [[ stage_in ]],
rgb.b = dot(yuv_corrected, cscParams.matrix[2]);
return float4(rgb, 1.0f);
}
// Shader for packed BGRA format (no color space conversion needed)
fragment float4 ps_draw_bgra(Vertex v [[ stage_in ]],
texture2d<float> bgraTexture [[ texture(0) ]])
{
// BGRA format is already in RGB color space, just sample and return
return bgraTexture.sample(s, v.texCoords);
}