hermes-agent/VoidLink
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

make pixelBuffer available to metalRenderer

Browse Source
This commit is contained in:
True Zhuanjia
2026-08-07 16:39:46 +08:00
parent a92010f5b1
commit 9c9ef4a13f
4 changed files with 39 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
VoidLink/Metal/MetalVideoRenderer.m
+25 -23
View File
@@ -303,9 +303,10 @@ CFStringRef __currentColorSpace;
// CAEDRMetadata is only available on iOS 16.0+
if (@available(iOS 16.0, tvOS 16.0, *)) {
if (frame.pixelBuffer) {
CFDataRef masteringDisplayColorVolume = CVBufferCopyAttachment(frame.pixelBuffer, kCVImageBufferMasteringDisplayColorVolumeKey, nil);
CFDataRef contentLightLevel = CVBufferCopyAttachment(frame.pixelBuffer, kCVImageBufferContentLightLevelInfoKey, nil);
CVPixelBufferRef imageBuffer = frame.imageBuffer;
if (imageBuffer) {
CFDataRef masteringDisplayColorVolume = CVBufferCopyAttachment(imageBuffer, kCVImageBufferMasteringDisplayColorVolumeKey, nil);
CFDataRef contentLightLevel = CVBufferCopyAttachment(imageBuffer, kCVImageBufferContentLightLevelInfoKey, nil);
// Configure optical output scale (100.0f is standard for HDR10)
const float opticalOutputScale = 100.0f;
@@ -592,15 +593,16 @@ CFStringRef __currentColorSpace;
FQLog(LOG_I, @"[%d / %.3f ms] Metal frame rendering", frame.frameNumber, frame.pts);
if (!frame.pixelBuffer) {
Log(LOG_W, @"Frame pixelBuffer is NULL, skipping render");
CVPixelBufferRef imageBuffer = frame.imageBuffer;
if (!imageBuffer) {
Log(LOG_W, @"Frame imageBuffer is NULL, skipping render");
return;
}
size_t planes = CVPixelBufferGetPlaneCount(frame.pixelBuffer);
size_t planes = CVPixelBufferGetPlaneCount(imageBuffer);
// For packed formats like BGRA, plane count is 0 but we treat it as 1 plane
OSType pixelFormatType = CVPixelBufferGetPixelFormatType(frame.pixelBuffer);
OSType pixelFormatType = CVPixelBufferGetPixelFormatType(imageBuffer);
BOOL isPackedFormat = (pixelFormatType == kCVPixelFormatType_32BGRA ||
pixelFormatType == kCVPixelFormatType_32ARGB);
if (isPackedFormat) {
@@ -695,7 +697,7 @@ CFStringRef __currentColorSpace;
if (isPackedFormat) {
// Handle packed BGRA format - iOS/macOS uses BGRA internally
// Check if the pixel buffer has IOSurface backing
CFTypeRef ioSurface = CVPixelBufferGetIOSurface(frame.pixelBuffer);
CFTypeRef ioSurface = CVPixelBufferGetIOSurface(imageBuffer);
if (!ioSurface) {
Log(LOG_E, @"CVPixelBuffer does not have IOSurface backing - cannot create Metal texture");
return;
@@ -703,35 +705,35 @@ CFStringRef __currentColorSpace;
CVReturn err = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault,
_textureCache,
frame.pixelBuffer,
imageBuffer,
NULL,
MTLPixelFormatBGRA8Unorm,
CVPixelBufferGetWidth(frame.pixelBuffer),
CVPixelBufferGetHeight(frame.pixelBuffer),
CVPixelBufferGetWidth(imageBuffer),
CVPixelBufferGetHeight(imageBuffer),
0, // planeIndex must be 0 for non-planar
&_cvMetalTextures[0]);
if (err != kCVReturnSuccess) {
Log(LOG_E, @"CVMetalTextureCacheCreateTextureFromImage() failed for BGRA: %d", err);
Log(LOG_E, @"PixelBuffer info - format: 0x%X, width: %zu, height: %zu, IOSurface: %p",
pixelFormatType,
CVPixelBufferGetWidth(frame.pixelBuffer),
CVPixelBufferGetHeight(frame.pixelBuffer),
CVPixelBufferGetWidth(imageBuffer),
CVPixelBufferGetHeight(imageBuffer),
ioSurface);
return;
} else {
id<MTLTexture> texture = CVMetalTextureGetTexture(_cvMetalTextures[0]);
Log(LOG_I, @"DEBUG: Created BGRA texture: format=%lu, width=%zu, height=%zu",
(unsigned long)MTLPixelFormatBGRA8Unorm,
CVPixelBufferGetWidth(frame.pixelBuffer),
CVPixelBufferGetHeight(frame.pixelBuffer));
CVPixelBufferGetWidth(imageBuffer),
CVPixelBufferGetHeight(imageBuffer));
}
} else {
// Handle planar YUV formats
size_t actualPlanes = CVPixelBufferGetPlaneCount(frame.pixelBuffer);
size_t actualPlanes = CVPixelBufferGetPlaneCount(imageBuffer);
for (size_t i = 0; i < actualPlanes; i++) {
MTLPixelFormat fmt;
switch (CVPixelBufferGetPixelFormatType(frame.pixelBuffer)) {
switch (CVPixelBufferGetPixelFormatType(imageBuffer)) {
case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange:
case kCVPixelFormatType_444YpCbCr8BiPlanarVideoRange:
case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange:
@@ -747,17 +749,17 @@ CFStringRef __currentColorSpace;
break;
default:
Log(LOG_E, @"Unknown pixel format: %@", CVPixelBufferGetPixelFormatType(frame.pixelBuffer));
Log(LOG_E, @"Unknown pixel format: %@", CVPixelBufferGetPixelFormatType(imageBuffer));
return;
}
CVReturn err = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault,
_textureCache,
frame.pixelBuffer,
imageBuffer,
NULL,
fmt,
CVPixelBufferGetWidthOfPlane(frame.pixelBuffer, i),
CVPixelBufferGetHeightOfPlane(frame.pixelBuffer, i),
CVPixelBufferGetWidthOfPlane(imageBuffer, i),
CVPixelBufferGetHeightOfPlane(imageBuffer, i),
i,
&_cvMetalTextures[i]);
if (err != kCVReturnSuccess) {
@@ -779,7 +781,7 @@ CFStringRef __currentColorSpace;
[renderEncoder setFragmentTexture:CVMetalTextureGetTexture(_cvMetalTextures[0]) atIndex:0];
} else {
// For planar formats, set multiple textures
size_t actualPlanes = CVPixelBufferGetPlaneCount(frame.pixelBuffer);
size_t actualPlanes = CVPixelBufferGetPlaneCount(imageBuffer);
for (size_t i = 0; i < actualPlanes; i++) {
[renderEncoder setFragmentTexture:CVMetalTextureGetTexture(_cvMetalTextures[i]) atIndex:i];
}
@@ -813,7 +815,7 @@ CFStringRef __currentColorSpace;
// signal semaphore, compute GPU time average, and clear textures
__block dispatch_semaphore_t block_semaphore = _inFlightSemaphore;
__block size_t texturesToClean = isPackedFormat ? 1 : CVPixelBufferGetPlaneCount(frame.pixelBuffer);
__block size_t texturesToClean = isPackedFormat ? 1 : CVPixelBufferGetPlaneCount(imageBuffer);
[commandBuffer addCompletedHandler:^(id<MTLCommandBuffer> cb) {
dispatch_semaphore_signal(block_semaphore);
VoidLink/Stream/FrameInterpolator.swift
+3 -10
View File
@@ -94,11 +94,8 @@ final class FrameInterpolator: NSObject {
return
}
guard
let currentSampleBuffer = frame.sampleBuffer,
let currentPixelBuffer = CMSampleBufferGetImageBuffer(currentSampleBuffer)
else {
logOnce("missing-image-buffer", "input CMSampleBuffer has no CVPixelBuffer")
guard let currentPixelBuffer = frame.imageBuffer else {
logOnce("missing-image-buffer", "input Frame has no CVPixelBuffer")
previousFrame = frame
completion([frame])
drainPendingFrames()
@@ -123,11 +120,7 @@ final class FrameInterpolator: NSObject {
return
}
guard
let previousFrame,
let previousSampleBuffer = previousFrame.sampleBuffer,
let previousPixelBuffer = CMSampleBufferGetImageBuffer(previousSampleBuffer)
else {
guard let previousFrame, let previousPixelBuffer = previousFrame.imageBuffer else {
previousFrame = frame
completion([frame])
drainPendingFrames()
VoidLink/Utility/Frame.h
+1
View File
@@ -11,6 +11,7 @@
@property (nonatomic, assign) CMVideoFormatDescriptionRef formatDesc;
@property (nonatomic) CMSampleBufferRef sampleBuffer;
@property (nonatomic) CVPixelBufferRef pixelBuffer;
@property (nonatomic, readonly) CVPixelBufferRef imageBuffer;
- (instancetype)initWithPixelBufffer:(CVPixelBufferRef)pixelBuffer frameNumber:(int)frameNumber frameType:(int)frameType pts:(CMTime)pts;
- (instancetype)initWithSampleBuffer:(CMSampleBufferRef)sampleBuffer frameNumber:(int)frameNumber frameType:(int)frameType;
VoidLink/Utility/Frame.m
+10
View File
@@ -21,6 +21,16 @@ static char FrameInterpolatedKey;
OBJC_ASSOCIATION_RETAIN_NONATOMIC);
}
- (CVPixelBufferRef)imageBuffer {
if (_pixelBuffer) {
return _pixelBuffer;
}
if (_sampleBuffer) {
return CMSampleBufferGetImageBuffer(_sampleBuffer);
}
return nil;
}
- (instancetype)initWithPixelBufffer:(CVPixelBufferRef)pixelBuffer
frameNumber:(int)frameNumber
frameType:(int)frameType