hermes-agent/VoidLink
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

make pixelBuffer available to metalRenderer

Browse Source
This commit is contained in:
True Zhuanjia
2026-08-07 16:39:46 +08:00
parent a92010f5b1
commit 9c9ef4a13f
4 changed files with 39 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
VoidLink/Metal/MetalVideoRenderer.m
+25 -23
View File
@@ -303,9 +303,10 @@ CFStringRef __currentColorSpace;
// CAEDRMetadata is only available on iOS 16.0+ // CAEDRMetadata is only available on iOS 16.0+
if (@available(iOS 16.0, tvOS 16.0, *)) { if (@available(iOS 16.0, tvOS 16.0, *)) {
if (frame.pixelBuffer) { CVPixelBufferRef imageBuffer = frame.imageBuffer;
CFDataRef masteringDisplayColorVolume = CVBufferCopyAttachment(frame.pixelBuffer, kCVImageBufferMasteringDisplayColorVolumeKey, nil); if (imageBuffer) {
CFDataRef contentLightLevel = CVBufferCopyAttachment(frame.pixelBuffer, kCVImageBufferContentLightLevelInfoKey, nil); CFDataRef masteringDisplayColorVolume = CVBufferCopyAttachment(imageBuffer, kCVImageBufferMasteringDisplayColorVolumeKey, nil);
CFDataRef contentLightLevel = CVBufferCopyAttachment(imageBuffer, kCVImageBufferContentLightLevelInfoKey, nil);
// Configure optical output scale (100.0f is standard for HDR10) // Configure optical output scale (100.0f is standard for HDR10)
const float opticalOutputScale = 100.0f; const float opticalOutputScale = 100.0f;
@@ -592,15 +593,16 @@ CFStringRef __currentColorSpace;
FQLog(LOG_I, @"[%d / %.3f ms] Metal frame rendering", frame.frameNumber, frame.pts); FQLog(LOG_I, @"[%d / %.3f ms] Metal frame rendering", frame.frameNumber, frame.pts);
if (!frame.pixelBuffer) { CVPixelBufferRef imageBuffer = frame.imageBuffer;
Log(LOG_W, @"Frame pixelBuffer is NULL, skipping render"); if (!imageBuffer) {
Log(LOG_W, @"Frame imageBuffer is NULL, skipping render");
return; return;
} }
size_t planes = CVPixelBufferGetPlaneCount(frame.pixelBuffer); size_t planes = CVPixelBufferGetPlaneCount(imageBuffer);
// For packed formats like BGRA, plane count is 0 but we treat it as 1 plane // For packed formats like BGRA, plane count is 0 but we treat it as 1 plane
OSType pixelFormatType = CVPixelBufferGetPixelFormatType(frame.pixelBuffer); OSType pixelFormatType = CVPixelBufferGetPixelFormatType(imageBuffer);
BOOL isPackedFormat = (pixelFormatType == kCVPixelFormatType_32BGRA || BOOL isPackedFormat = (pixelFormatType == kCVPixelFormatType_32BGRA ||
pixelFormatType == kCVPixelFormatType_32ARGB); pixelFormatType == kCVPixelFormatType_32ARGB);
if (isPackedFormat) { if (isPackedFormat) {
@@ -695,7 +697,7 @@ CFStringRef __currentColorSpace;
if (isPackedFormat) { if (isPackedFormat) {
// Handle packed BGRA format - iOS/macOS uses BGRA internally // Handle packed BGRA format - iOS/macOS uses BGRA internally
// Check if the pixel buffer has IOSurface backing // Check if the pixel buffer has IOSurface backing
CFTypeRef ioSurface = CVPixelBufferGetIOSurface(frame.pixelBuffer); CFTypeRef ioSurface = CVPixelBufferGetIOSurface(imageBuffer);
if (!ioSurface) { if (!ioSurface) {
Log(LOG_E, @"CVPixelBuffer does not have IOSurface backing - cannot create Metal texture"); Log(LOG_E, @"CVPixelBuffer does not have IOSurface backing - cannot create Metal texture");
return; return;
@@ -703,35 +705,35 @@ CFStringRef __currentColorSpace;
CVReturn err = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, CVReturn err = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault,
_textureCache, _textureCache,
frame.pixelBuffer, imageBuffer,
NULL, NULL,
MTLPixelFormatBGRA8Unorm, MTLPixelFormatBGRA8Unorm,
CVPixelBufferGetWidth(frame.pixelBuffer), CVPixelBufferGetWidth(imageBuffer),
CVPixelBufferGetHeight(frame.pixelBuffer), CVPixelBufferGetHeight(imageBuffer),
0, // planeIndex must be 0 for non-planar 0, // planeIndex must be 0 for non-planar
&_cvMetalTextures[0]); &_cvMetalTextures[0]);
if (err != kCVReturnSuccess) { if (err != kCVReturnSuccess) {
Log(LOG_E, @"CVMetalTextureCacheCreateTextureFromImage() failed for BGRA: %d", err); Log(LOG_E, @"CVMetalTextureCacheCreateTextureFromImage() failed for BGRA: %d", err);
Log(LOG_E, @"PixelBuffer info - format: 0x%X, width: %zu, height: %zu, IOSurface: %p", Log(LOG_E, @"PixelBuffer info - format: 0x%X, width: %zu, height: %zu, IOSurface: %p",
pixelFormatType, pixelFormatType,
CVPixelBufferGetWidth(frame.pixelBuffer), CVPixelBufferGetWidth(imageBuffer),
CVPixelBufferGetHeight(frame.pixelBuffer), CVPixelBufferGetHeight(imageBuffer),
ioSurface); ioSurface);
return; return;
} else { } else {
id<MTLTexture> texture = CVMetalTextureGetTexture(_cvMetalTextures[0]); id<MTLTexture> texture = CVMetalTextureGetTexture(_cvMetalTextures[0]);
Log(LOG_I, @"DEBUG: Created BGRA texture: format=%lu, width=%zu, height=%zu", Log(LOG_I, @"DEBUG: Created BGRA texture: format=%lu, width=%zu, height=%zu",
(unsigned long)MTLPixelFormatBGRA8Unorm, (unsigned long)MTLPixelFormatBGRA8Unorm,
CVPixelBufferGetWidth(frame.pixelBuffer), CVPixelBufferGetWidth(imageBuffer),
CVPixelBufferGetHeight(frame.pixelBuffer)); CVPixelBufferGetHeight(imageBuffer));
} }
} else { } else {
// Handle planar YUV formats // Handle planar YUV formats
size_t actualPlanes = CVPixelBufferGetPlaneCount(frame.pixelBuffer); size_t actualPlanes = CVPixelBufferGetPlaneCount(imageBuffer);
for (size_t i = 0; i < actualPlanes; i++) { for (size_t i = 0; i < actualPlanes; i++) {
MTLPixelFormat fmt; MTLPixelFormat fmt;
switch (CVPixelBufferGetPixelFormatType(frame.pixelBuffer)) { switch (CVPixelBufferGetPixelFormatType(imageBuffer)) {
case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange: case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange:
case kCVPixelFormatType_444YpCbCr8BiPlanarVideoRange: case kCVPixelFormatType_444YpCbCr8BiPlanarVideoRange:
case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange: case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange:
@@ -747,17 +749,17 @@ CFStringRef __currentColorSpace;
break; break;
default: default:
Log(LOG_E, @"Unknown pixel format: %@", CVPixelBufferGetPixelFormatType(frame.pixelBuffer)); Log(LOG_E, @"Unknown pixel format: %@", CVPixelBufferGetPixelFormatType(imageBuffer));
return; return;
} }
CVReturn err = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, CVReturn err = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault,
_textureCache, _textureCache,
frame.pixelBuffer, imageBuffer,
NULL, NULL,
fmt, fmt,
CVPixelBufferGetWidthOfPlane(frame.pixelBuffer, i), CVPixelBufferGetWidthOfPlane(imageBuffer, i),
CVPixelBufferGetHeightOfPlane(frame.pixelBuffer, i), CVPixelBufferGetHeightOfPlane(imageBuffer, i),
i, i,
&_cvMetalTextures[i]); &_cvMetalTextures[i]);
if (err != kCVReturnSuccess) { if (err != kCVReturnSuccess) {
@@ -779,7 +781,7 @@ CFStringRef __currentColorSpace;
[renderEncoder setFragmentTexture:CVMetalTextureGetTexture(_cvMetalTextures[0]) atIndex:0]; [renderEncoder setFragmentTexture:CVMetalTextureGetTexture(_cvMetalTextures[0]) atIndex:0];
} else { } else {
// For planar formats, set multiple textures // For planar formats, set multiple textures
size_t actualPlanes = CVPixelBufferGetPlaneCount(frame.pixelBuffer); size_t actualPlanes = CVPixelBufferGetPlaneCount(imageBuffer);
for (size_t i = 0; i < actualPlanes; i++) { for (size_t i = 0; i < actualPlanes; i++) {
[renderEncoder setFragmentTexture:CVMetalTextureGetTexture(_cvMetalTextures[i]) atIndex:i]; [renderEncoder setFragmentTexture:CVMetalTextureGetTexture(_cvMetalTextures[i]) atIndex:i];
} }
@@ -813,7 +815,7 @@ CFStringRef __currentColorSpace;
// signal semaphore, compute GPU time average, and clear textures // signal semaphore, compute GPU time average, and clear textures
__block dispatch_semaphore_t block_semaphore = _inFlightSemaphore; __block dispatch_semaphore_t block_semaphore = _inFlightSemaphore;
__block size_t texturesToClean = isPackedFormat ? 1 : CVPixelBufferGetPlaneCount(frame.pixelBuffer); __block size_t texturesToClean = isPackedFormat ? 1 : CVPixelBufferGetPlaneCount(imageBuffer);
[commandBuffer addCompletedHandler:^(id<MTLCommandBuffer> cb) { [commandBuffer addCompletedHandler:^(id<MTLCommandBuffer> cb) {
dispatch_semaphore_signal(block_semaphore); dispatch_semaphore_signal(block_semaphore);
VoidLink/Stream/FrameInterpolator.swift
+3 -10
View File
@@ -94,11 +94,8 @@ final class FrameInterpolator: NSObject {
return return
} }
guard guard let currentPixelBuffer = frame.imageBuffer else {
let currentSampleBuffer = frame.sampleBuffer, logOnce("missing-image-buffer", "input Frame has no CVPixelBuffer")
let currentPixelBuffer = CMSampleBufferGetImageBuffer(currentSampleBuffer)
else {
logOnce("missing-image-buffer", "input CMSampleBuffer has no CVPixelBuffer")
previousFrame = frame previousFrame = frame
completion([frame]) completion([frame])
drainPendingFrames() drainPendingFrames()
@@ -123,11 +120,7 @@ final class FrameInterpolator: NSObject {
return return
} }
guard guard let previousFrame, let previousPixelBuffer = previousFrame.imageBuffer else {
let previousFrame,
let previousSampleBuffer = previousFrame.sampleBuffer,
let previousPixelBuffer = CMSampleBufferGetImageBuffer(previousSampleBuffer)
else {
previousFrame = frame previousFrame = frame
completion([frame]) completion([frame])
drainPendingFrames() drainPendingFrames()
VoidLink/Utility/Frame.h
+1
View File
@@ -11,6 +11,7 @@
@property (nonatomic, assign) CMVideoFormatDescriptionRef formatDesc; @property (nonatomic, assign) CMVideoFormatDescriptionRef formatDesc;
@property (nonatomic) CMSampleBufferRef sampleBuffer; @property (nonatomic) CMSampleBufferRef sampleBuffer;
@property (nonatomic) CVPixelBufferRef pixelBuffer; @property (nonatomic) CVPixelBufferRef pixelBuffer;
@property (nonatomic, readonly) CVPixelBufferRef imageBuffer;
- (instancetype)initWithPixelBufffer:(CVPixelBufferRef)pixelBuffer frameNumber:(int)frameNumber frameType:(int)frameType pts:(CMTime)pts; - (instancetype)initWithPixelBufffer:(CVPixelBufferRef)pixelBuffer frameNumber:(int)frameNumber frameType:(int)frameType pts:(CMTime)pts;
- (instancetype)initWithSampleBuffer:(CMSampleBufferRef)sampleBuffer frameNumber:(int)frameNumber frameType:(int)frameType; - (instancetype)initWithSampleBuffer:(CMSampleBufferRef)sampleBuffer frameNumber:(int)frameNumber frameType:(int)frameType;
VoidLink/Utility/Frame.m
+10
View File
@@ -21,6 +21,16 @@ static char FrameInterpolatedKey;
OBJC_ASSOCIATION_RETAIN_NONATOMIC); OBJC_ASSOCIATION_RETAIN_NONATOMIC);
} }
- (CVPixelBufferRef)imageBuffer {
if (_pixelBuffer) {
return _pixelBuffer;
}
if (_sampleBuffer) {
return CMSampleBufferGetImageBuffer(_sampleBuffer);
}
return nil;
}
- (instancetype)initWithPixelBufffer:(CVPixelBufferRef)pixelBuffer - (instancetype)initWithPixelBufffer:(CVPixelBufferRef)pixelBuffer
frameNumber:(int)frameNumber frameNumber:(int)frameNumber
frameType:(int)frameType frameType:(int)frameType