From 34eb2ccd31292c034eec9d7e17d612fe29b3593d Mon Sep 17 00:00:00 2001 From: Acaki Date: Fri, 15 Aug 2025 04:32:48 +0800 Subject: [PATCH] fix: metal renderer packed format crash on emulator and lower iOS version --- VoidLink/Metal/MetalVideoRenderer.m | 155 ++++++++++++++++++++------- VoidLink/Metal/MetalViewController.m | 20 +++- VoidLink/Metal/Shaders.metal | 8 ++ 3 files changed, 145 insertions(+), 38 deletions(-) diff --git a/VoidLink/Metal/MetalVideoRenderer.m b/VoidLink/Metal/MetalVideoRenderer.m index fce060df..43b3e82a 100644 --- a/VoidLink/Metal/MetalVideoRenderer.m +++ b/VoidLink/Metal/MetalVideoRenderer.m @@ -142,12 +142,18 @@ CFStringRef __currentColorSpace; dispatch_queue_attr_make_with_qos_class(DISPATCH_QUEUE_SERIAL, QOS_CLASS_USER_INTERACTIVE, 0)); _averageGPUTime = (1.0f / framerate) / 2; _device = device; - _colorPixelFormat = MTLPixelFormatBGR10A2Unorm; + _colorPixelFormat = drawablePixelFormat; _framerate = framerate; _commandQueue = [_device newCommandQueue]; _currentEDRHeadroom = 1.0f; _lastColorSpace = -1; _lastFullRange = NO; + + // Initialize default CSC parameters buffer with BT.601 limited range + struct ParamBuffer defaultParamBuffer; + defaultParamBuffer.cscParams = k_CscParams_Bt601Lim; + MTLResourceOptions bufferOptions = MTLResourceStorageModeShared; + _CscParamsBuffer = [_device newBufferWithBytes:(void *)&defaultParamBuffer length:sizeof(defaultParamBuffer) options:bufferOptions]; _lastPresented = 0.0f; _inFlightSemaphore = dispatch_semaphore_create(MaxFramesInFlight); _isStopping = NO; @@ -512,6 +518,15 @@ CFStringRef __currentColorSpace; } size_t planes = CVPixelBufferGetPlaneCount(frame.pixelBuffer); + + // For packed formats like BGRA, plane count is 0 but we treat it as 1 plane + OSType pixelFormatType = CVPixelBufferGetPixelFormatType(frame.pixelBuffer); + BOOL isPackedFormat = (pixelFormatType == kCVPixelFormatType_32BGRA || + pixelFormatType == kCVPixelFormatType_32ARGB); + if (isPackedFormat) { + planes = 1; // Treat packed formats as single plane + } + assert(planes <= MAX_VIDEO_PLANES); if (layerDidChange && frame.frameNumber > 1) { @@ -548,16 +563,26 @@ CFStringRef __currentColorSpace; // linear shaders id yuvToLinear = [defaultLibrary newFunctionWithName:@"yuvToLinear"]; - // Determine if this is 10-bit based on the framebuffer's pixel format - BOOL is10Bit = (framebufferPixelFormat == MTLPixelFormatBGR10A2Unorm); + // Determine if this is 10-bit based on the input CVPixelBuffer format, not the output framebuffer format + // pixelFormatType is already declared above + BOOL is10BitInput = (pixelFormatType == kCVPixelFormatType_420YpCbCr10BiPlanarFullRange || + pixelFormatType == kCVPixelFormatType_444YpCbCr10BiPlanarFullRange || + pixelFormatType == kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange || + pixelFormatType == kCVPixelFormatType_444YpCbCr10BiPlanarVideoRange); NSString *fragmentShaderName; - if (planes == 2) { - fragmentShaderName = is10Bit ? @"ps_draw_biplanar_10bit" : @"ps_draw_biplanar_8bit"; + if (isPackedFormat) { + // BGRA/ARGB packed formats don't need color space conversion + fragmentShaderName = @"ps_draw_bgra"; + } else if (planes == 2) { + fragmentShaderName = is10BitInput ? @"ps_draw_biplanar_10bit" : @"ps_draw_biplanar_8bit"; } else { - fragmentShaderName = is10Bit ? @"ps_draw_triplanar_10bit" : @"ps_draw_triplanar_8bit"; + fragmentShaderName = is10BitInput ? @"ps_draw_triplanar_10bit" : @"ps_draw_triplanar_8bit"; } + Log(LOG_I, @"DEBUG: CVPixelBuffer format: 0x%X, planes: %zu, is10BitInput: %d, shader: %@, framebuffer format: %lu", + pixelFormatType, planes, is10BitInput, fragmentShaderName, (unsigned long)framebufferPixelFormat); + pipelineDesc.colorAttachments[0].pixelFormat = framebufferPixelFormat; pipelineDesc.vertexBuffers[0].mutability = MTLMutabilityImmutable; @@ -585,41 +610,84 @@ CFStringRef __currentColorSpace; _videoPipelinePixelFormat[planes] = framebufferPixelFormat; } - for (size_t i = 0; i < planes; i++) { - MTLPixelFormat fmt; - - switch (CVPixelBufferGetPixelFormatType(frame.pixelBuffer)) { - case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange: - case kCVPixelFormatType_444YpCbCr8BiPlanarVideoRange: - case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange: - case kCVPixelFormatType_444YpCbCr8BiPlanarFullRange: - fmt = (i == 0) ? MTLPixelFormatR8Unorm : MTLPixelFormatRG8Unorm; - break; - - case kCVPixelFormatType_420YpCbCr10BiPlanarFullRange: - case kCVPixelFormatType_444YpCbCr10BiPlanarFullRange: - case kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange: - case kCVPixelFormatType_444YpCbCr10BiPlanarVideoRange: - fmt = (i == 0) ? MTLPixelFormatR16Unorm : MTLPixelFormatRG16Unorm; - break; - - default: - Log(LOG_E, @"Unknown pixel format: %@", CVPixelBufferGetPixelFormatType(frame.pixelBuffer)); - return; + if (isPackedFormat) { + // Handle packed BGRA format - iOS/macOS uses BGRA internally + // Check if the pixel buffer has IOSurface backing + CFTypeRef ioSurface = CVPixelBufferGetIOSurface(frame.pixelBuffer); + if (!ioSurface) { + Log(LOG_E, @"CVPixelBuffer does not have IOSurface backing - cannot create Metal texture"); + return; } CVReturn err = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, _textureCache, frame.pixelBuffer, NULL, - fmt, - CVPixelBufferGetWidthOfPlane(frame.pixelBuffer, i), - CVPixelBufferGetHeightOfPlane(frame.pixelBuffer, i), - i, - &_cvMetalTextures[i]); + MTLPixelFormatBGRA8Unorm, + CVPixelBufferGetWidth(frame.pixelBuffer), + CVPixelBufferGetHeight(frame.pixelBuffer), + 0, // planeIndex must be 0 for non-planar + &_cvMetalTextures[0]); if (err != kCVReturnSuccess) { - Log(LOG_E, @"CVMetalTextureCacheCreateTextureFromImage() failed: %d", err); + Log(LOG_E, @"CVMetalTextureCacheCreateTextureFromImage() failed for BGRA: %d", err); + Log(LOG_E, @"PixelBuffer info - format: 0x%X, width: %zu, height: %zu, IOSurface: %p", + pixelFormatType, + CVPixelBufferGetWidth(frame.pixelBuffer), + CVPixelBufferGetHeight(frame.pixelBuffer), + ioSurface); return; + } else { + id texture = CVMetalTextureGetTexture(_cvMetalTextures[0]); + Log(LOG_I, @"DEBUG: Created BGRA texture: format=%lu, width=%zu, height=%zu", + (unsigned long)MTLPixelFormatBGRA8Unorm, + CVPixelBufferGetWidth(frame.pixelBuffer), + CVPixelBufferGetHeight(frame.pixelBuffer)); + } + } else { + // Handle planar YUV formats + size_t actualPlanes = CVPixelBufferGetPlaneCount(frame.pixelBuffer); + for (size_t i = 0; i < actualPlanes; i++) { + MTLPixelFormat fmt; + + switch (CVPixelBufferGetPixelFormatType(frame.pixelBuffer)) { + case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange: + case kCVPixelFormatType_444YpCbCr8BiPlanarVideoRange: + case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange: + case kCVPixelFormatType_444YpCbCr8BiPlanarFullRange: + fmt = (i == 0) ? MTLPixelFormatR8Unorm : MTLPixelFormatRG8Unorm; + break; + + case kCVPixelFormatType_420YpCbCr10BiPlanarFullRange: + case kCVPixelFormatType_444YpCbCr10BiPlanarFullRange: + case kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange: + case kCVPixelFormatType_444YpCbCr10BiPlanarVideoRange: + fmt = (i == 0) ? MTLPixelFormatR16Unorm : MTLPixelFormatRG16Unorm; + break; + + default: + Log(LOG_E, @"Unknown pixel format: %@", CVPixelBufferGetPixelFormatType(frame.pixelBuffer)); + return; + } + + CVReturn err = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, + _textureCache, + frame.pixelBuffer, + NULL, + fmt, + CVPixelBufferGetWidthOfPlane(frame.pixelBuffer, i), + CVPixelBufferGetHeightOfPlane(frame.pixelBuffer, i), + i, + &_cvMetalTextures[i]); + if (err != kCVReturnSuccess) { + Log(LOG_E, @"CVMetalTextureCacheCreateTextureFromImage() failed: %d", err); + return; + } else { + id texture = CVMetalTextureGetTexture(_cvMetalTextures[i]); + Log(LOG_I, @"DEBUG: Created texture for plane %zu: format=%lu, width=%zu, height=%zu", + i, (unsigned long)fmt, + CVPixelBufferGetWidthOfPlane(frame.pixelBuffer, i), + CVPixelBufferGetHeightOfPlane(frame.pixelBuffer, i)); + } } } @@ -629,12 +697,24 @@ CFStringRef __currentColorSpace; id renderEncoder = [commandBuffer renderCommandEncoderWithDescriptor:_renderPassDescriptor]; [renderEncoder setRenderPipelineState:_videoPipelineState[planes]]; - for (size_t i = 0; i < planes; i++) { - [renderEncoder setFragmentTexture:CVMetalTextureGetTexture(_cvMetalTextures[i]) atIndex:i]; + + if (isPackedFormat) { + // For packed formats, we only have one texture + [renderEncoder setFragmentTexture:CVMetalTextureGetTexture(_cvMetalTextures[0]) atIndex:0]; + } else { + // For planar formats, set multiple textures + size_t actualPlanes = CVPixelBufferGetPlaneCount(frame.pixelBuffer); + for (size_t i = 0; i < actualPlanes; i++) { + [renderEncoder setFragmentTexture:CVMetalTextureGetTexture(_cvMetalTextures[i]) atIndex:i]; + } } [renderEncoder setVertexBuffer:_VideoVertexBuffer offset:0 atIndex:0]; - [renderEncoder setFragmentBuffer:_CscParamsBuffer offset:0 atIndex:0]; + + // Only set CSC params buffer for YUV formats that need color space conversion + if (!isPackedFormat) { + [renderEncoder setFragmentBuffer:_CscParamsBuffer offset:0 atIndex:0]; + } #if !TARGET_OS_TV if (layer.pixelFormat == MTLPixelFormatRGBA16Float) { [self pollCurrentEDRHeadroom]; @@ -657,6 +737,7 @@ CFStringRef __currentColorSpace; // signal semaphore, compute GPU time average, and clear textures __block dispatch_semaphore_t block_semaphore = _inFlightSemaphore; + __block size_t texturesToClean = isPackedFormat ? 1 : CVPixelBufferGetPlaneCount(frame.pixelBuffer); [commandBuffer addCompletedHandler:^(id cb) { dispatch_semaphore_signal(block_semaphore); @@ -665,7 +746,7 @@ CFStringRef __currentColorSpace; self->_averageGPUTime = (GPUTime * alpha) + (self->_averageGPUTime * (1.0 - alpha)); // Free textures after completion of rendering - for (size_t i = 0; i < planes; i++) { + for (size_t i = 0; i < texturesToClean; i++) { if (self->_cvMetalTextures[i]) { CFRelease(self->_cvMetalTextures[i]); self->_cvMetalTextures[i] = nil; diff --git a/VoidLink/Metal/MetalViewController.m b/VoidLink/Metal/MetalViewController.m index 549285f4..e5e6ffb4 100644 --- a/VoidLink/Metal/MetalViewController.m +++ b/VoidLink/Metal/MetalViewController.m @@ -59,9 +59,27 @@ } view.metalLayer.device = device; + // Determine supported pixel format before initializing renderer + // Use TARGET_OS_SIMULATOR to detect simulator environment + MTLPixelFormat pixelFormat; +#if TARGET_OS_SIMULATOR + // iOS Simulator doesn't support BGR10A2Unorm + pixelFormat = MTLPixelFormatBGRA8Unorm; + Log(LOG_W, @"Running on iOS Simulator, using BGRA8Unorm pixel format"); +#else + // On real devices, check if we should enable HDR + if (_enableHdr) { + pixelFormat = MTLPixelFormatBGR10A2Unorm; + Log(LOG_I, @"HDR enabled, using BGR10A2Unorm pixel format"); + } else { + pixelFormat = MTLPixelFormatBGRA8Unorm; + Log(LOG_I, @"HDR disabled, using BGRA8Unorm pixel format"); + } +#endif + // Initialize the renderer. MetalVideoRenderer *renderer = [[MetalVideoRenderer alloc] initWithMetalDevice:device - drawablePixelFormat:MTLPixelFormatBGR10A2Unorm + drawablePixelFormat:pixelFormat framerate:self->_framerate]; if (!renderer) { Log(LOG_E, @"The renderer couldn't be initialized."); diff --git a/VoidLink/Metal/Shaders.metal b/VoidLink/Metal/Shaders.metal index 538adfd6..58c46b0e 100644 --- a/VoidLink/Metal/Shaders.metal +++ b/VoidLink/Metal/Shaders.metal @@ -127,3 +127,11 @@ fragment float4 ps_draw_triplanar_10bit(Vertex v [[ stage_in ]], rgb.b = dot(yuv_corrected, cscParams.matrix[2]); return float4(rgb, 1.0f); } + +// Shader for packed BGRA format (no color space conversion needed) +fragment float4 ps_draw_bgra(Vertex v [[ stage_in ]], + texture2d bgraTexture [[ texture(0) ]]) +{ + // BGRA format is already in RGB color space, just sample and return + return bgraTexture.sample(s, v.texCoords); +}