hermes-agent/VoidLink
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Clean up render thread, plot method calls

Browse Source
This commit is contained in:
Andy Grundman
2025-07-15 02:06:13 +08:00
committed by Acaki
parent 6682e64519
commit 5e4ba088be
11 changed files with 61 additions and 157 deletions
Download Patch File Download Diff File Expand all files Collapse all files
VoidLink/ImGuiRenderer.h
-2
View File
@@ -31,6 +31,4 @@ typedef void (^MetricsHandler)(int plotId, CFTimeInterval value);
-(void) show;
-(void) hide;
-(void) stop;
-(void) observeFloat:(int)plotId value:(CFTimeInterval)value;
-(void) observeFloatReturnMetrics:(int)plotId value:(CFTimeInterval)value plotMetrics:(PlotMetrics * _Nullable)plotMetrics;
@end
VoidLink/ImGuiRenderer.mm
-13
View File
@@ -226,19 +226,6 @@
# endif
#endif
/// Stats Graphs, we can still track data this way even with ImGui disabled
- (void) observeFloat:(int)plotId value:(CFTimeInterval)value {
[self.plots[plotId].buffer addValue:(float)value];
}
- (void) observeFloatReturnMetrics:(int)plotId value:(CFTimeInterval)value plotMetrics:(PlotMetrics *)plotMetrics {
[self.plots[plotId].buffer addValue:(float)value];
if (plotMetrics != nil) {
[self.plots[plotId].buffer copyMetrics:plotMetrics];
}
}
#if !defined(IMGUI_DISABLE)
inline static float getValue(void *buffer, int idx) {
float *fbuffer = (float *)buffer;
VoidLink/Metal/MetalVideoRenderer.h
+1 -1
View File
@@ -18,6 +18,6 @@
- (void)renderFrame:(nonnull Frame *)frame toLayer:(nonnull CAMetalLayer *)layer;
- (void)waitToRenderTo:(nonnull CAMetalLayer *)layer;
- (void)drawableResize:(CGSize)drawableSize;
- (void)plotFrametime:(CFTimeInterval)presentedTime;
- (void)plotFrametime:(CFTimeInterval)presentedTime withPresentTime:(CFTimeInterval)presentTime;
@end
VoidLink/Metal/MetalVideoRenderer.m
+45 -31
View File
@@ -130,6 +130,13 @@ static const NSUInteger MaxFramesInFlight = 3;
return self;
}
- (void)dealloc {
if (_CscParamsBuffer) {
_CscParamsBuffer = nil;
}
}
#if !TARGET_OS_TV
- (void)applyEDRFromFrame:(Frame *)frame toLayer:(CAMetalLayer *)layer {
CFDictionaryRef ext = [frame getFormatDescExtensions];
@@ -253,10 +260,9 @@ static const NSUInteger MaxFramesInFlight = 3;
: newPixelFormat == MTLPixelFormatBGR10A2Unorm ? @"MTLPixelFormatBGR10A2Unorm"
: [NSString stringWithFormat:@"Unknown: %lu", (unsigned long)layer.pixelFormat]);
}
#if !RENDER_ON_MAIN_THREAD
// These can only be changed on the main thread
dispatch_sync(dispatch_get_main_queue(), ^{
#endif
#if !TARGET_OS_TV
if (isHDR) {
layer.wantsExtendedDynamicRangeContent = YES;
@@ -264,9 +270,7 @@ static const NSUInteger MaxFramesInFlight = 3;
#endif
layer.colorspace = newColorSpace;
layer.pixelFormat = newPixelFormat;
#if !RENDER_ON_MAIN_THREAD
});
#endif
CGColorSpaceRelease(newColorSpace);
}
@@ -351,7 +355,8 @@ static const NSUInteger MaxFramesInFlight = 3;
_nextDrawable = nil;
}
- (void)renderFrame:(Frame *)frame toLayer:(CAMetalLayer *)layer {
- (void)renderFrame:(Frame *)frame toLayer:(CAMetalLayer *)layer
{ @autoreleasepool {
// Handle changes to the frame's colorspace from last time we rendered
BOOL layerDidChange = NO;
if (![self updateColorSpaceForFrame:frame toLayer:layer layerDidChange:&layerDidChange]) {
@@ -377,7 +382,7 @@ static const NSUInteger MaxFramesInFlight = 3;
#endif
size_t planes = CVPixelBufferGetPlaneCount(frame.pixelBuffer);
assert(planes == 2 || planes == 3);
assert(planes <= MAX_VIDEO_PLANES);
MTLRenderPipelineDescriptor *pipelineDesc = [MTLRenderPipelineDescriptor new];
id<MTLLibrary> defaultLibrary = [_device newDefaultLibrary];
@@ -397,23 +402,23 @@ static const NSUInteger MaxFramesInFlight = 3;
MTLPixelFormat fmt;
switch (CVPixelBufferGetPixelFormatType(frame.pixelBuffer)) {
case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange:
case kCVPixelFormatType_444YpCbCr8BiPlanarVideoRange:
case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange:
case kCVPixelFormatType_444YpCbCr8BiPlanarFullRange:
fmt = (i == 0) ? MTLPixelFormatR8Unorm : MTLPixelFormatRG8Unorm;
break;
case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange:
case kCVPixelFormatType_444YpCbCr8BiPlanarVideoRange:
case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange:
case kCVPixelFormatType_444YpCbCr8BiPlanarFullRange:
fmt = (i == 0) ? MTLPixelFormatR8Unorm : MTLPixelFormatRG8Unorm;
break;
case kCVPixelFormatType_420YpCbCr10BiPlanarFullRange:
case kCVPixelFormatType_444YpCbCr10BiPlanarFullRange:
case kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange:
case kCVPixelFormatType_444YpCbCr10BiPlanarVideoRange:
fmt = (i == 0) ? MTLPixelFormatR16Unorm : MTLPixelFormatRG16Unorm;
break;
case kCVPixelFormatType_420YpCbCr10BiPlanarFullRange:
case kCVPixelFormatType_444YpCbCr10BiPlanarFullRange:
case kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange:
case kCVPixelFormatType_444YpCbCr10BiPlanarVideoRange:
fmt = (i == 0) ? MTLPixelFormatR16Unorm : MTLPixelFormatRG16Unorm;
break;
default:
Log(LOG_E, @"Unknown pixel format: %@", CVPixelBufferGetPixelFormatType(frame.pixelBuffer));
return;
default:
Log(LOG_E, @"Unknown pixel format: %@", CVPixelBufferGetPixelFormatType(frame.pixelBuffer));
return;
}
if (_cvMetalTextures[i]) {
@@ -434,6 +439,14 @@ static const NSUInteger MaxFramesInFlight = 3;
}
}
if (!_nextDrawable) {
Log(LOG_E, @"Lost nextDrawable, trying to get a new one");
_nextDrawable = [layer nextDrawable];
if (!_nextDrawable) {
Log(LOG_E, @"Failed to get nextDrawable");
return;
}
}
_renderPassDescriptor.colorAttachments[0].texture = _nextDrawable.texture;
id<MTLCommandBuffer> commandBuffer = [_commandQueue commandBuffer];
@@ -443,15 +456,21 @@ static const NSUInteger MaxFramesInFlight = 3;
for (size_t i = 0; i < planes; i++) {
[renderEncoder setFragmentTexture:CVMetalTextureGetTexture(_cvMetalTextures[i]) atIndex:i];
}
__block dispatch_semaphore_t block_semaphore = _inFlightSemaphore;
[commandBuffer addCompletedHandler:^(id<MTLCommandBuffer> cb) {
dispatch_semaphore_signal(block_semaphore);
const CFTimeInterval GPUTime = cb.GPUEndTime - cb.GPUStartTime;
const double alpha = 0.25f;
self->_averageGPUTime = (GPUTime * alpha) + (self->_averageGPUTime * (1.0 - alpha));
// Free textures after completion of rendering per CVMetalTextureCache requirements
// Free textures after completion of rendering
for (size_t i = 0; i < planes; i++) {
CVBufferRelease(self->_cvMetalTextures[i]);
self->_cvMetalTextures[i] = nil;
if (self->_cvMetalTextures[i]) {
CVBufferRelease(self->_cvMetalTextures[i]);
self->_cvMetalTextures[i] = nil;
}
}
CVMetalTextureCacheFlush(self->_textureCache, 0);
@@ -462,11 +481,6 @@ static const NSUInteger MaxFramesInFlight = 3;
[renderEncoder drawPrimitives:MTLPrimitiveTypeTriangleStrip vertexStart:0 vertexCount:4];
[renderEncoder endEncoding];
__block dispatch_semaphore_t block_semaphore = _inFlightSemaphore;
[commandBuffer addCompletedHandler:^(id<MTLCommandBuffer> buffer) {
dispatch_semaphore_signal(block_semaphore);
}];
__weak typeof(self) self_ = self;
[_nextDrawable addPresentedHandler:^(id<MTLDrawable> d) {
if (self_) {
@@ -487,7 +501,7 @@ static const NSUInteger MaxFramesInFlight = 3;
[commandBuffer waitUntilCompleted];
_nextDrawable = nil;
}
} }
- (void)plotFrametime:(CFTimeInterval)presentedTime {
if (_lastPresented > 0) {
@@ -518,7 +532,7 @@ static const NSUInteger MaxFramesInFlight = 3;
}
- (void)resize:(CGSize)size {
// TODO?
}
@end
VoidLink/Metal/MetalView.m
+3 -17
View File
@@ -6,13 +6,11 @@
#import "MetalConfig.h"
@implementation MetalView {
#if !RENDER_ON_MAIN_THREAD
// The secondary thread containing the render loop.
NSThread *_renderThread;
// The flag to indicate that rendering needs to cease on the main thread.
BOOL _continueRunLoop;
#endif
}
#pragma mark - Initialization and Setup.
@@ -64,7 +62,6 @@
#endif // END TARGET_OS_IOS || TARGET_OS_TV
- (void)movedToWindow {
#if !RENDER_ON_MAIN_THREAD
// Protect _continueRunLoop with a `@synchronized` block because it's accessed by the separate
// animation thread.
@synchronized(self) {
@@ -72,13 +69,12 @@
_continueRunLoop = NO;
}
// Create and start a secondary NSThread that has another run runloop. The NSThread
// Create and start a secondary NSThread that has another runloop. The NSThread
// class calls the 'runThread' method at the start of the secondary thread's execution.
_renderThread = [[NSThread alloc] initWithTarget:self selector:@selector(runThread) object:nil];
_continueRunLoop = YES;
_renderThread.qualityOfService = NSQualityOfServiceUserInteractive;
[_renderThread start];
#endif // END !RENDER_ON_MAIN_THREAD
// Perform any actions that need to know the size and scale of the drawable. When UIKit calls
// didMoveToWindow after the view initialization, this is the first opportunity to notify
@@ -98,7 +94,6 @@
#endif
}
#if !RENDER_ON_MAIN_THREAD
- (void)runThread {
// The system sets the '_continueRunLoop' ivar outside this thread, so it needs to synchronize. Create a
// 'continueRunLoop' local var that the system can set from the _continueRunLoop ivar in a @synchronized block.
@@ -124,7 +119,6 @@
}
}
}
#endif // END !RENDER_ON_MAIN_THREAD
#pragma mark - Resizing
@@ -178,15 +172,6 @@
return;
}
#if RENDER_ON_MAIN_THREAD
if (newSize.width == _metalLayer.drawableSize.width && newSize.height == _metalLayer.drawableSize.height) {
return;
}
_metalLayer.drawableSize = newSize;
[_delegate drawableResize:newSize];
#else
// The system calls all AppKit and UIKit calls that notify of a resize on the main thread. Use
// a synchronized block to ensure that resize notifications on the delegate are atomic.
@synchronized(_metalLayer) {
@@ -194,11 +179,12 @@
return;
}
Log(LOG_I, @"[MetalView] resizeDrawable: %.2f x %.2f", newSize.width, newSize.height);
_metalLayer.drawableSize = newSize;
[_delegate drawableResize:newSize];
}
#endif
}
#endif // END AUTOMATICALLY_RESIZE
VoidLink/Metal/MetalViewController.m
-4
View File
@@ -83,11 +83,9 @@ The implementation of the cross-platform game view controller.
- (void)waitToRenderTo:(nonnull CAMetalLayer *)layer {
if (!_stopping) {
// Renderer obtains a nextDrawable, waiting if necessary
FQLog(LOG_I, @"[MetalViewController] caling [_renderer waitToRenderTo:layer]");
[_renderer waitToRenderTo:layer];
// If we don't have a frame yet, wait on that too
FQLog(LOG_I, @"[MetalViewController] caling [_renderer waitForEnqueue]");
[_frameQueue waitForEnqueue];
}
}
@@ -99,10 +97,8 @@ The implementation of the cross-platform game view controller.
}
CFTimeInterval timeout = (1.0f / _framerate) - _renderer.averageGPUTime;
FQLog(LOG_I, @"[MetalViewController] caling [_frameQueue dequeueWithTimeout:%f]", timeout);
Frame *frame = [_frameQueue dequeueWithTimeout:timeout];
if (frame) {
FQLog(LOG_I, @"[MetalViewController] calling [_renderer renderFrame] %@", frame);
[_renderer renderFrame:frame toLayer:layer];
}
}
VoidLink/Stream/ConnectionCallbacks.h
-3
View File
@@ -24,7 +24,4 @@
- (void) setControllerLed:(uint16_t)controllerNumber r:(uint8_t)r g:(uint8_t)g b:(uint8_t)b;
- (void) videoContentShown;
- (void) observeFloat:(int)plotId value:(CFTimeInterval)value;
- (void) observeFloatReturnMetrics:(int)plotId value:(CFTimeInterval)value plotMetrics:(PlotMetrics *)plotMetrics;
@end
VoidLink/Stream/StreamManager.m
+1 -1
View File
@@ -196,7 +196,7 @@
return [NSString stringWithFormat:@"Video stream: %dx%d %.2f FPS (Codec: %@)\n"
"Bitrate: %.1f Mbps, Peak: %.1f, Renderer: %@\n"
"%@"
"Frames buffered: %.1f\n"
"Frames buffered: %.1f, Present \n"
"Frames dropped by network/pacing jitter: %.1f%% / %.1f%%\n"
"Average network latency: %@\n"
"Decode time: %.2f/%.2f/%.2f ms",
VoidLink/Stream/VideoDecoderRenderer.m
+10 -74
View File
@@ -17,6 +17,7 @@
#import "Plot.h"
#import "PlatformThreads.h"
#import "MetalViewController.h"
#import "ImGuiPlots.h"
#include <libavcodec/avcodec.h>
#include <libavcodec/cbs.h>
@@ -275,7 +276,7 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit);
// we missed a callback
// Log(LOG_W, @"*** slow frametime %.3f ms", frametime * 1000.0);
}
[self->_callbacks observeFloat:PLOT_FRAMETIME value:frametime * 1000.0];
[[ImGuiPlots sharedInstance] observeFloat:PLOT_FRAMETIME value:frametime * 1000.0];
}
lastTargetLocal = targetLocal;
@@ -801,18 +802,10 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit);
return DR_NEED_IDR;
}
OSStatus decodeStatus;
if (0 && _renderingBackend == RENDER_METAL) {
decodeStatus = [self decodeFrameToLinearColorspaceWithSampleBuffer:sampleBuffer
frameNumber:du->frameNumber
frameType:du->frameType
decodeStartTime:decodeStartTime];
} else {
decodeStatus = [self decodeFrameWithSampleBuffer:sampleBuffer
frameNumber:du->frameNumber
frameType:du->frameType
decodeStartTime:decodeStartTime];
}
OSStatus decodeStatus = [self decodeFrameWithSampleBuffer:sampleBuffer
frameNumber:du->frameNumber
frameType:du->frameType
decodeStartTime:decodeStartTime];
// Dereference the buffers
CFRelease(dataBlockBuffer);
CFRelease(frameBlockBuffer);
@@ -821,63 +814,6 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit);
return DR_OK;
}
// For experimenting with improved HDR tone-mapping by directly decoding to linear so we can tonemap using
// the host's metadata.
- (OSStatus)decodeFrameToLinearColorspaceWithSampleBuffer:(CMSampleBufferRef)sampleBuffer
frameNumber:(int)frameNumber
frameType:(int)frameType
decodeStartTime:(CFTimeInterval)decodeStartTime
{
NSDictionary *linearAttributes = @{
(id)kCVPixelBufferPixelFormatTypeKey : @(kCVPixelFormatType_64RGBAHalf), // half-float RGBA
(id)kCVImageBufferColorPrimariesKey : (id)kCVImageBufferColorPrimaries_ITU_R_2020, // wide-gamut primaries (Rec.2020)
(id)kCVImageBufferTransferFunctionKey : (id)kCVImageBufferTransferFunction_Linear, // linear transfer function
(id)kCVImageBufferYCbCrMatrixKey : (id)kCVImageBufferYCbCrMatrix_ITU_R_2020, // Y′CbCr matrix → RGB matrix (Rec.2020)
(id)kCVPixelBufferMetalCompatibilityKey : @YES // make it GPU-compatible
};
if (frameType == FRAME_TYPE_IDR || _decompressionSession == nil) {
[self setupDecompressionSessionWithAttributes:linearAttributes];
}
OSStatus status = VTDecompressionSessionDecodeFrameWithOutputHandler(
_decompressionSession, sampleBuffer, 0, NULL,
^(OSStatus status, VTDecodeInfoFlags infoFlags, CVImageBufferRef _Nullable imageBuffer, CMTime pts, CMTime duration) {
CVPixelBufferRef pixelBuffer = CVPixelBufferRetain((CVPixelBufferRef)imageBuffer);
Log(LOG_D, @"Decoded to PixelBuffer %@", pixelBuffer); // dumps full frame details
// Dispatch onto our higher priority queue
dispatch_async(self->_vtq, ^{
Frame *frame = [[Frame alloc] initWithPixelBufffer:pixelBuffer
frameNumber:frameNumber
frameType:frameType
pts:pts];
[frame setFormatDesc:self->_formatDesc];
int framesDropped = [self->_frameQueue enqueue:frame withSlackSize:3];
static PlotMetrics frameQueueMetrics = {};
[self->_callbacks observeFloatReturnMetrics:PLOT_QUEUED_FRAMES value:[self->_frameQueue count] plotMetrics:&frameQueueMetrics];
[self safeCopyMetricsTo:&self->_frameQueueMetrics from:&frameQueueMetrics];
[self->_callbacks observeFloat:PLOT_DROPPED value:framesDropped];
// It's important we capture host metrics on the incoming thread, as this frame object
// may have been dropped by the above enqueue
static CFTimeInterval lastHostFrame = 0.0f;
if (lastHostFrame != 0) {
[self->_callbacks observeFloat:PLOT_HOST_FRAMETIME value:(frame.pts - lastHostFrame) * 1000.0];
}
lastHostFrame = frame.pts;
// Decode time is not graphed because it is marked as hidden, but we can use the same mechanism for the value used by stats
static PlotMetrics decodeMetrics = {};
[self->_callbacks observeFloatReturnMetrics:PLOT_DECODE value:(CACurrentMediaTime() - decodeStartTime) * 1000.0 plotMetrics:&decodeMetrics];
[self safeCopyMetricsTo:&self->_decodeMetrics from:&decodeMetrics];
});
});
return status;
}
- (OSStatus)decodeFrameWithSampleBuffer:(CMSampleBufferRef)sampleBuffer
frameNumber:(int)frameNumber
frameType:(int)frameType
@@ -954,22 +890,22 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit);
int framesDropped = [self->_frameQueue enqueue:frame withSlackSize:3];
static PlotMetrics frameQueueMetrics = {};
[self->_callbacks observeFloatReturnMetrics:PLOT_QUEUED_FRAMES value:[self->_frameQueue count] plotMetrics:&frameQueueMetrics];
[[ImGuiPlots sharedInstance] observeFloatReturnMetrics:PLOT_QUEUED_FRAMES value:[self->_frameQueue count] plotMetrics:&frameQueueMetrics];
[self safeCopyMetricsTo:&self->_frameQueueMetrics from:&frameQueueMetrics];
[self->_callbacks observeFloat:PLOT_DROPPED value:framesDropped];
[[ImGuiPlots sharedInstance] observeFloat:PLOT_DROPPED value:framesDropped];
// It's important we capture host metrics on the incoming thread, as this frame object
// may have been dropped by the above enqueue
static CFTimeInterval lastHostFrame = 0.0f;
if (lastHostFrame != 0) {
[self->_callbacks observeFloat:PLOT_HOST_FRAMETIME value:(frame.pts - lastHostFrame) * 1000.0];
[[ImGuiPlots sharedInstance] observeFloat:PLOT_HOST_FRAMETIME value:(frame.pts - lastHostFrame) * 1000.0];
}
lastHostFrame = frame.pts;
// Decode time is not graphed because it is marked as hidden, but we can use the same mechanism for the value used by stats
static PlotMetrics decodeMetrics = {};
[self->_callbacks observeFloatReturnMetrics:PLOT_DECODE value:(CACurrentMediaTime() - decodeStartTime) * 1000.0 plotMetrics:&decodeMetrics];
[[ImGuiPlots sharedInstance] observeFloatReturnMetrics:PLOT_DECODE value:(CACurrentMediaTime() - decodeStartTime) * 1000.0 plotMetrics:&decodeMetrics];
[self safeCopyMetricsTo:&self->_decodeMetrics from:&decodeMetrics];
});
});
VoidLink/Utility/FrameQueue.m
+1 -2
View File
@@ -226,8 +226,7 @@
// Allows the render loop to wait if the queue is empty
- (void)waitForEnqueue {
while ([self isEmpty]) {
FQLog(LOG_I, @"waitForEnqueue...");
long result = dispatch_semaphore_wait(_frameSemaphore, DISPATCH_TIME_FOREVER);
dispatch_semaphore_wait(_frameSemaphore, DISPATCH_TIME_FOREVER);
}
}
VoidLink/ViewControllers/StreamFrameViewController.m
-9
View File
@@ -661,7 +661,6 @@
if ([_settings.renderingBackend intValue] == RENDER_METAL) {
// Metal view for video
// TODO: refactor the way things access observeFloat for stats
self.metalViewController = [[MetalViewController alloc] initWithFrame:self.view.bounds
framerate:[self->_settings.framerate floatValue]
enableHdr:self->_settings.enableHdr
@@ -1267,14 +1266,6 @@
});
}
- (void) observeFloat:(int)plotId value:(CFTimeInterval)value {
[self.imguiView observeFloat:plotId value:value];
}
- (void) observeFloatReturnMetrics:(int)plotId value:(CFTimeInterval)value plotMetrics:(PlotMetrics *)plotMetrics {
return [self.imguiView observeFloatReturnMetrics:plotId value:value plotMetrics:plotMetrics];
}
- (void)didReceiveMemoryWarning
{
[super didReceiveMemoryWarning];