diff --git a/VoidLink/Metal/MetalVideoRenderer.m b/VoidLink/Metal/MetalVideoRenderer.m index 82c96a4e..715499e4 100644 --- a/VoidLink/Metal/MetalVideoRenderer.m +++ b/VoidLink/Metal/MetalVideoRenderer.m @@ -303,9 +303,10 @@ CFStringRef __currentColorSpace; // CAEDRMetadata is only available on iOS 16.0+ if (@available(iOS 16.0, tvOS 16.0, *)) { - if (frame.pixelBuffer) { - CFDataRef masteringDisplayColorVolume = CVBufferCopyAttachment(frame.pixelBuffer, kCVImageBufferMasteringDisplayColorVolumeKey, nil); - CFDataRef contentLightLevel = CVBufferCopyAttachment(frame.pixelBuffer, kCVImageBufferContentLightLevelInfoKey, nil); + CVPixelBufferRef imageBuffer = frame.imageBuffer; + if (imageBuffer) { + CFDataRef masteringDisplayColorVolume = CVBufferCopyAttachment(imageBuffer, kCVImageBufferMasteringDisplayColorVolumeKey, nil); + CFDataRef contentLightLevel = CVBufferCopyAttachment(imageBuffer, kCVImageBufferContentLightLevelInfoKey, nil); // Configure optical output scale (100.0f is standard for HDR10) const float opticalOutputScale = 100.0f; @@ -592,15 +593,16 @@ CFStringRef __currentColorSpace; FQLog(LOG_I, @"[%d / %.3f ms] Metal frame rendering", frame.frameNumber, frame.pts); - if (!frame.pixelBuffer) { - Log(LOG_W, @"Frame pixelBuffer is NULL, skipping render"); + CVPixelBufferRef imageBuffer = frame.imageBuffer; + if (!imageBuffer) { + Log(LOG_W, @"Frame imageBuffer is NULL, skipping render"); return; } - size_t planes = CVPixelBufferGetPlaneCount(frame.pixelBuffer); + size_t planes = CVPixelBufferGetPlaneCount(imageBuffer); // For packed formats like BGRA, plane count is 0 but we treat it as 1 plane - OSType pixelFormatType = CVPixelBufferGetPixelFormatType(frame.pixelBuffer); + OSType pixelFormatType = CVPixelBufferGetPixelFormatType(imageBuffer); BOOL isPackedFormat = (pixelFormatType == kCVPixelFormatType_32BGRA || pixelFormatType == kCVPixelFormatType_32ARGB); if (isPackedFormat) { @@ -695,7 +697,7 @@ CFStringRef __currentColorSpace; if (isPackedFormat) { // Handle packed BGRA format - iOS/macOS uses BGRA internally // Check if the pixel buffer has IOSurface backing - CFTypeRef ioSurface = CVPixelBufferGetIOSurface(frame.pixelBuffer); + CFTypeRef ioSurface = CVPixelBufferGetIOSurface(imageBuffer); if (!ioSurface) { Log(LOG_E, @"CVPixelBuffer does not have IOSurface backing - cannot create Metal texture"); return; @@ -703,35 +705,35 @@ CFStringRef __currentColorSpace; CVReturn err = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, _textureCache, - frame.pixelBuffer, + imageBuffer, NULL, MTLPixelFormatBGRA8Unorm, - CVPixelBufferGetWidth(frame.pixelBuffer), - CVPixelBufferGetHeight(frame.pixelBuffer), + CVPixelBufferGetWidth(imageBuffer), + CVPixelBufferGetHeight(imageBuffer), 0, // planeIndex must be 0 for non-planar &_cvMetalTextures[0]); if (err != kCVReturnSuccess) { Log(LOG_E, @"CVMetalTextureCacheCreateTextureFromImage() failed for BGRA: %d", err); Log(LOG_E, @"PixelBuffer info - format: 0x%X, width: %zu, height: %zu, IOSurface: %p", pixelFormatType, - CVPixelBufferGetWidth(frame.pixelBuffer), - CVPixelBufferGetHeight(frame.pixelBuffer), + CVPixelBufferGetWidth(imageBuffer), + CVPixelBufferGetHeight(imageBuffer), ioSurface); return; } else { id texture = CVMetalTextureGetTexture(_cvMetalTextures[0]); Log(LOG_I, @"DEBUG: Created BGRA texture: format=%lu, width=%zu, height=%zu", (unsigned long)MTLPixelFormatBGRA8Unorm, - CVPixelBufferGetWidth(frame.pixelBuffer), - CVPixelBufferGetHeight(frame.pixelBuffer)); + CVPixelBufferGetWidth(imageBuffer), + CVPixelBufferGetHeight(imageBuffer)); } } else { // Handle planar YUV formats - size_t actualPlanes = CVPixelBufferGetPlaneCount(frame.pixelBuffer); + size_t actualPlanes = CVPixelBufferGetPlaneCount(imageBuffer); for (size_t i = 0; i < actualPlanes; i++) { MTLPixelFormat fmt; - switch (CVPixelBufferGetPixelFormatType(frame.pixelBuffer)) { + switch (CVPixelBufferGetPixelFormatType(imageBuffer)) { case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange: case kCVPixelFormatType_444YpCbCr8BiPlanarVideoRange: case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange: @@ -747,17 +749,17 @@ CFStringRef __currentColorSpace; break; default: - Log(LOG_E, @"Unknown pixel format: %@", CVPixelBufferGetPixelFormatType(frame.pixelBuffer)); + Log(LOG_E, @"Unknown pixel format: %@", CVPixelBufferGetPixelFormatType(imageBuffer)); return; } CVReturn err = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, _textureCache, - frame.pixelBuffer, + imageBuffer, NULL, fmt, - CVPixelBufferGetWidthOfPlane(frame.pixelBuffer, i), - CVPixelBufferGetHeightOfPlane(frame.pixelBuffer, i), + CVPixelBufferGetWidthOfPlane(imageBuffer, i), + CVPixelBufferGetHeightOfPlane(imageBuffer, i), i, &_cvMetalTextures[i]); if (err != kCVReturnSuccess) { @@ -779,7 +781,7 @@ CFStringRef __currentColorSpace; [renderEncoder setFragmentTexture:CVMetalTextureGetTexture(_cvMetalTextures[0]) atIndex:0]; } else { // For planar formats, set multiple textures - size_t actualPlanes = CVPixelBufferGetPlaneCount(frame.pixelBuffer); + size_t actualPlanes = CVPixelBufferGetPlaneCount(imageBuffer); for (size_t i = 0; i < actualPlanes; i++) { [renderEncoder setFragmentTexture:CVMetalTextureGetTexture(_cvMetalTextures[i]) atIndex:i]; } @@ -813,7 +815,7 @@ CFStringRef __currentColorSpace; // signal semaphore, compute GPU time average, and clear textures __block dispatch_semaphore_t block_semaphore = _inFlightSemaphore; - __block size_t texturesToClean = isPackedFormat ? 1 : CVPixelBufferGetPlaneCount(frame.pixelBuffer); + __block size_t texturesToClean = isPackedFormat ? 1 : CVPixelBufferGetPlaneCount(imageBuffer); [commandBuffer addCompletedHandler:^(id cb) { dispatch_semaphore_signal(block_semaphore); diff --git a/VoidLink/Stream/FrameInterpolator.swift b/VoidLink/Stream/FrameInterpolator.swift index 61da8387..f8f9b4c5 100644 --- a/VoidLink/Stream/FrameInterpolator.swift +++ b/VoidLink/Stream/FrameInterpolator.swift @@ -94,11 +94,8 @@ final class FrameInterpolator: NSObject { return } - guard - let currentSampleBuffer = frame.sampleBuffer, - let currentPixelBuffer = CMSampleBufferGetImageBuffer(currentSampleBuffer) - else { - logOnce("missing-image-buffer", "input CMSampleBuffer has no CVPixelBuffer") + guard let currentPixelBuffer = frame.imageBuffer else { + logOnce("missing-image-buffer", "input Frame has no CVPixelBuffer") previousFrame = frame completion([frame]) drainPendingFrames() @@ -123,11 +120,7 @@ final class FrameInterpolator: NSObject { return } - guard - let previousFrame, - let previousSampleBuffer = previousFrame.sampleBuffer, - let previousPixelBuffer = CMSampleBufferGetImageBuffer(previousSampleBuffer) - else { + guard let previousFrame, let previousPixelBuffer = previousFrame.imageBuffer else { previousFrame = frame completion([frame]) drainPendingFrames() diff --git a/VoidLink/Utility/Frame.h b/VoidLink/Utility/Frame.h index c923663d..8758932b 100644 --- a/VoidLink/Utility/Frame.h +++ b/VoidLink/Utility/Frame.h @@ -11,6 +11,7 @@ @property (nonatomic, assign) CMVideoFormatDescriptionRef formatDesc; @property (nonatomic) CMSampleBufferRef sampleBuffer; @property (nonatomic) CVPixelBufferRef pixelBuffer; +@property (nonatomic, readonly) CVPixelBufferRef imageBuffer; - (instancetype)initWithPixelBufffer:(CVPixelBufferRef)pixelBuffer frameNumber:(int)frameNumber frameType:(int)frameType pts:(CMTime)pts; - (instancetype)initWithSampleBuffer:(CMSampleBufferRef)sampleBuffer frameNumber:(int)frameNumber frameType:(int)frameType; diff --git a/VoidLink/Utility/Frame.m b/VoidLink/Utility/Frame.m index 50aa5f94..f69c25af 100644 --- a/VoidLink/Utility/Frame.m +++ b/VoidLink/Utility/Frame.m @@ -21,6 +21,16 @@ static char FrameInterpolatedKey; OBJC_ASSOCIATION_RETAIN_NONATOMIC); } +- (CVPixelBufferRef)imageBuffer { + if (_pixelBuffer) { + return _pixelBuffer; + } + if (_sampleBuffer) { + return CMSampleBufferGetImageBuffer(_sampleBuffer); + } + return nil; +} + - (instancetype)initWithPixelBufffer:(CVPixelBufferRef)pixelBuffer frameNumber:(int)frameNumber frameType:(int)frameType