hermes-agent/VoidLink
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: use CAMetalDisplayLink for metal renderer to fix refresh rate issues

Browse Source 
- update imgui to use stream fps instead of hardcoded 60
- hide renderer setting for iOS < 17 as CAMetalDisplayLink only supports later versions
This commit is contained in:
Acaki
2025-08-10 20:58:56 +08:00
parent 8d05e8e670
commit 628ce0f12e
9 changed files with 147 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
VoidLink/Database/TemporarySettings.m
+9 -1
View File
@@ -52,7 +52,15 @@
self.statsOverlay = [[NSUserDefaults standardUserDefaults] boolForKey:@"statsOverlay"];
self.enableGraphs = [[NSUserDefaults standardUserDefaults] boolForKey:@"enableGraphs"];
self.graphOpacity = [NSNumber numberWithInteger:[[NSUserDefaults standardUserDefaults] integerForKey:@"graphOpacity"]];
self.renderingBackend = [NSNumber numberWithInteger:[[NSUserDefaults standardUserDefaults] integerForKey:@"renderingBackend"]];
// Set rendering backend based on iOS version and user preference
NSInteger savedBackend = [[NSUserDefaults standardUserDefaults] integerForKey:@"renderingBackend"];
if (@available(iOS 17.0, *)) {
// iOS 17+ can use Metal renderer (CAMetalDisplayLink available)
self.renderingBackend = [NSNumber numberWithInteger:savedBackend];
} else {
// iOS < 17 must use AVSB renderer (no CAMetalDisplayLink)
self.renderingBackend = [NSNumber numberWithInteger:1]; // RENDER_AVSB = 1
}
NSInteger _screenSize = [[NSUserDefaults standardUserDefaults] integerForKey:@"streamResolution"];
switch (_screenSize) {
VoidLink/ImGuiRenderer.h
+1
View File
@@ -21,6 +21,7 @@ typedef void (^MetricsHandler)(int plotId, CFTimeInterval value);
@property (nonatomic) BOOL enableGraphs;
@property (nonatomic) float graphOpacity;
@property (nonatomic) BOOL imguiRunning;
@property (nonatomic) int streamFps;
@property (nonatomic) MetricsHandler _Nonnull metricsHandler;
-(nonnull instancetype) initWithFrame:(CGRect)bounds
VoidLink/ImGuiRenderer.mm
+5 -2
View File
@@ -35,6 +35,9 @@
_plots = [ImGuiPlots sharedInstance].plots;
// Store stream FPS to synchronize MTKView
_streamFps = streamFps;
return self;
}
@@ -83,7 +86,7 @@
self.mtkView.device = self.device;
self.mtkView.delegate = self;
self.mtkView.preferredFramesPerSecond = 60; // ImGui overlay will always render at this rate
self.mtkView.preferredFramesPerSecond = _streamFps; // Synchronize with stream framerate
self.mtkView.opaque = NO;
self.mtkView.enableSetNeedsDisplay = NO;
@@ -166,7 +169,7 @@
#if TARGET_OS_SIMULATOR
[commandBuffer presentDrawable:view.currentDrawable];
#else
[commandBuffer presentDrawable:view.currentDrawable afterMinimumDuration:1.0 / view.preferredFramesPerSecond];
[commandBuffer presentDrawable:view.currentDrawable afterMinimumDuration:1.0 / _streamFps];
#endif
[commandBuffer commit];
#endif
VoidLink/Metal/MetalVideoRenderer.h
+2 -1
View File
@@ -15,7 +15,8 @@
@property (atomic) BOOL isStopping;
- (nonnull instancetype)initWithMetalDevice:(nonnull id<MTLDevice>)device drawablePixelFormat:(MTLPixelFormat)drawablePixelFormat framerate:(float)framerate;
- (void)renderFrame:(nonnull Frame *)frame toLayer:(nonnull CAMetalLayer *)layer;
- (void)renderFrame:(nonnull Frame *)frame withDrawable:(nonnull id<CAMetalDrawable>)drawable API_AVAILABLE(ios(17.0));
- (void)renderFrame:(nonnull Frame *)frame withDrawable:(nonnull id<CAMetalDrawable>)drawable targetPresentationTimestamp:(CFTimeInterval)targetPresentationTimestamp API_AVAILABLE(ios(17.0));
- (void)waitToRenderTo:(nonnull CAMetalLayer *)layer;
- (void)drawableResize:(CGSize)drawableSize;
- (void)shutdown;
VoidLink/Metal/MetalVideoRenderer.m
+19 -28
View File
@@ -443,21 +443,30 @@ CFStringRef __currentColorSpace;
return YES;
}
- (void)renderFrame:(Frame *)frame toLayer:(CAMetalLayer *)layer {
// Enhanced method that receives timing information from CAMetalDisplayLink
- (void)renderFrame:(Frame *)frame withDrawable:(id<CAMetalDrawable>)drawable targetPresentationTimestamp:(CFTimeInterval)targetPresentationTimestamp API_AVAILABLE(ios(17.0)) {
@autoreleasepool {
if (self.isStopping) {
Log(LOG_I, @"[MetalVideoRenderer] isStopping");
return;
}
// Calculate accurate frametime using CAMetalDisplayLink timing
if (self.lastPresented > 0.0f) {
CFTimeInterval frametime = targetPresentationTimestamp - self.lastPresented;
[[ImGuiPlots sharedInstance] observeFloat:PLOT_FRAMETIME value:(frametime * 1000.0)];
}
self.lastPresented = targetPresentationTimestamp;
// Handle changes to the frame's colorspace from last time we rendered
BOOL layerDidChange = NO;
if (![self updateColorSpaceForFrame:frame toLayer:layer layerDidChange:&layerDidChange]) {
if (![self updateColorSpaceForFrame:frame toLayer:(CAMetalLayer *)drawable.layer layerDidChange:&layerDidChange]) {
return;
}
// Handle changes to the video size or drawable size
if (![self updateVideoRegionSizeForFrame:frame toLayer:layer]) {
if (![self updateVideoRegionSizeForFrame:frame toLayer:(CAMetalLayer *)drawable.layer]) {
return;
}
@@ -483,7 +492,7 @@ CFStringRef __currentColorSpace;
id<MTLFunction> yuvToLinear = [defaultLibrary newFunctionWithName:@"yuvToLinear"];
// Determine if this is 10-bit based on the layer's pixel format (after colorspace update)
BOOL is10Bit = (layer.pixelFormat == MTLPixelFormatBGR10A2Unorm);
BOOL is10Bit = (((CAMetalLayer *)drawable.layer).pixelFormat == MTLPixelFormatBGR10A2Unorm);
NSString *fragmentShaderName;
if (planes == 2) {
@@ -492,13 +501,13 @@ CFStringRef __currentColorSpace;
fragmentShaderName = is10Bit ? @"ps_draw_triplanar_10bit" : @"ps_draw_triplanar_8bit";
}
pipelineDesc.colorAttachments[0].pixelFormat = layer.pixelFormat;
pipelineDesc.colorAttachments[0].pixelFormat = ((CAMetalLayer *)drawable.layer).pixelFormat;
pipelineDesc.vertexBuffers[0].mutability = MTLMutabilityImmutable;
Log(LOG_I, @"Metal pipeline state for %zu planes with pixel format %@",
planes, layer.pixelFormat == MTLPixelFormatRGBA16Float ? @"MTLPixelFormatRGBA16Float" : @"MTLPixelFormatBGR10A2Unorm");
planes, ((CAMetalLayer *)drawable.layer).pixelFormat == MTLPixelFormatRGBA16Float ? @"MTLPixelFormatRGBA16Float" : @"MTLPixelFormatBGR10A2Unorm");
if (layer.pixelFormat == MTLPixelFormatRGBA16Float) {
if (((CAMetalLayer *)drawable.layer).pixelFormat == MTLPixelFormatRGBA16Float) {
// 4:2:0 or 4:4:4 YUV -> BT.2020 RGB -> linear float
pipelineDesc.vertexFunction = vertexVsDraw;
pipelineDesc.fragmentFunction = yuvToLinear;
@@ -554,11 +563,7 @@ CFStringRef __currentColorSpace;
}
}
id<CAMetalDrawable> drawable = [layer nextDrawable];
if (!drawable) {
Log(LOG_E, @"Failed to get nextDrawable");
return;
}
// Use the provided drawable instead of getting nextDrawable from layer
_renderPassDescriptor.colorAttachments[0].texture = drawable.texture;
id<MTLCommandBuffer> commandBuffer = [_commandQueue commandBuffer];
@@ -572,7 +577,7 @@ CFStringRef __currentColorSpace;
[renderEncoder setVertexBuffer:_VideoVertexBuffer offset:0 atIndex:0];
[renderEncoder setFragmentBuffer:_CscParamsBuffer offset:0 atIndex:0];
#if !TARGET_OS_TV
if (layer.pixelFormat == MTLPixelFormatRGBA16Float) {
if (((CAMetalLayer *)drawable.layer).pixelFormat == MTLPixelFormatRGBA16Float) {
[self pollCurrentEDRHeadroom];
[renderEncoder setFragmentBytes:&_currentEDRHeadroom length:sizeof(float) atIndex:1];
}
@@ -580,15 +585,6 @@ CFStringRef __currentColorSpace;
[renderEncoder drawPrimitives:MTLPrimitiveTypeTriangleStrip vertexStart:0 vertexCount:4];
[renderEncoder endEncoding];
__block MetalVideoRenderer *strongSelf = self;
[drawable addPresentedHandler:^(id<MTLDrawable> d) {
if (strongSelf.lastPresented > 0.0f) {
CFTimeInterval frametime = d.presentedTime - strongSelf.lastPresented;
[[ImGuiPlots sharedInstance] observeFloat:PLOT_FRAMETIME value:(frametime * 1000.0)];
}
strongSelf.lastPresented = d.presentedTime;
}];
// signal semaphore, compute GPU time average, and clear textures
__block dispatch_semaphore_t block_semaphore = _inFlightSemaphore;
[commandBuffer addCompletedHandler:^(id<MTLCommandBuffer> cb) {
@@ -609,15 +605,10 @@ CFStringRef __currentColorSpace;
CVMetalTextureCacheFlush(self->_textureCache, 0);
}];
#if TARGET_OS_SIMULATOR
// CAMetalDisplayLink handles timing automatically, so we don't use afterMinimumDuration
[commandBuffer presentDrawable:drawable];
#else
// present for a minimum duration for best frame pacing
[commandBuffer presentDrawable:drawable afterMinimumDuration:1.0f / _framerate];
#endif
[commandBuffer commit];
[commandBuffer waitUntilCompleted];
}
}
VoidLink/Metal/MetalView.h
+3 -2
View File
@@ -8,13 +8,14 @@
@protocol MetalViewDelegate <NSObject>
- (void)drawableResize:(CGSize)size;
- (void)renderTo:(nonnull CAMetalLayer *)layer;
- (void)renderWithDrawable:(nonnull id<CAMetalDrawable>)drawable toLayer:(nonnull CAMetalLayer *)layer API_AVAILABLE(ios(17.0));
- (void)renderWithDrawable:(nonnull id<CAMetalDrawable>)drawable toLayer:(nonnull CAMetalLayer *)layer targetPresentationTimestamp:(CFTimeInterval)targetPresentationTimestamp API_AVAILABLE(ios(17.0));
- (void)waitToRenderTo:(nonnull CAMetalLayer *)layer;
@end
// The Metal game view base class.
@interface MetalView : UIView <CALayerDelegate>
@interface MetalView : UIView <CALayerDelegate, CAMetalDisplayLinkDelegate>
@property (nonatomic, nonnull, readonly) CAMetalLayer *metalLayer;
@property (nonatomic, nullable) id<MetalViewDelegate> delegate;
VoidLink/Metal/MetalView.m
+82 -8
View File
@@ -4,10 +4,13 @@
#import "MetalView.h"
#import "MetalConfig.h"
#import <QuartzCore/CAMetalDisplayLink.h>
@implementation MetalView {
// The secondary thread containing the render loop.
NSThread *_renderThread;
// Metal display link for vsync synchronization (iOS 17+)
CAMetalDisplayLink *_metalDisplayLink API_AVAILABLE(ios(17.0));
}
#pragma mark - Initialization and Setup.
@@ -33,15 +36,43 @@
self.layer.delegate = self;
}
- (void)setFramerate:(float)framerate {
_framerate = framerate;
if (@available(iOS 17.0, *)) {
if (_metalDisplayLink) {
_metalDisplayLink.preferredFrameRateRange = CAFrameRateRangeMake(framerate, framerate, framerate);
}
}
}
- (void)shutdown {
// First cancel the thread to stop the run loop
if (_renderThread) {
Log(LOG_I, @"[MetalView] sending renderThread a cancel message");
[_renderThread cancel];
// Invalidate metal display link to stop callbacks
if (@available(iOS 17.0, *)) {
if (_metalDisplayLink) {
Log(LOG_I, @"[MetalView] invalidating metal display link");
[_metalDisplayLink invalidate];
_metalDisplayLink = nil;
}
}
// Now wait for thread to finish
Log(LOG_I, @"[MetalView] waiting on renderThread to finish");
while (!_renderThread.isFinished) {
usleep(100);
NSDate *timeout = [NSDate dateWithTimeIntervalSinceNow:2.0]; // 2 second timeout
while (!_renderThread.isFinished && [timeout timeIntervalSinceNow] > 0) {
usleep(1000);
}
if (_renderThread.isFinished) {
Log(LOG_I, @"[MetalView] renderThread has finished");
} else {
Log(LOG_E, @"[MetalView] renderThread failed to finish in time");
}
Log(LOG_I, @"[MetalView] renderThread has finished");
_renderThread = nil;
}
}
@@ -54,6 +85,28 @@
[self movedToWindow];
}
// CAMetalDisplayLinkDelegate method (iOS 17+)
- (void)metalDisplayLink:(CAMetalDisplayLink *)link needsUpdate:(CAMetalDisplayLinkUpdate *)update API_AVAILABLE(ios(17.0)) {
// Skip rendering if we're shutting down
if ([NSThread currentThread].isCancelled) {
return;
}
@autoreleasepool {
[self.delegate waitToRenderTo:self.metalLayer];
// Get the drawable from the update object
id<CAMetalDrawable> drawable = update.drawable;
if (drawable) {
// Pass timing information to the renderer
if ([self.delegate respondsToSelector:@selector(renderWithDrawable:toLayer:targetPresentationTimestamp:)]) {
[self.delegate renderWithDrawable:drawable toLayer:self.metalLayer targetPresentationTimestamp:update.targetPresentationTimestamp];
} else {
[self.delegate renderWithDrawable:drawable toLayer:self.metalLayer];
}
}
}
}
- (void)movedToWindow {
if (!self.window) {
Log(LOG_I, @"[MetalView] movedToWindow(nil): shutting down...");
@@ -61,12 +114,33 @@
return;
}
// Render on a new thread
// Create CAMetalDisplayLink for vsync-synchronized rendering (iOS 17+ only)
if (@available(iOS 17.0, *)) {
_metalDisplayLink = [[CAMetalDisplayLink alloc] initWithMetalLayer:_metalLayer];
_metalDisplayLink.delegate = self;
_metalDisplayLink.preferredFrameRateRange = CAFrameRateRangeMake(_framerate, _framerate, _framerate);
Log(LOG_I, @"[MetalView] Using CAMetalDisplayLink for optimal Metal rendering");
}
// Start the display link on a background thread
_renderThread = [[NSThread alloc] initWithBlock:^{
while (![NSThread currentThread].isCancelled) {
// Add metal display link to this thread's run loop
if (@available(iOS 17.0, *)) {
[self->_metalDisplayLink addToRunLoop:[NSRunLoop currentRunLoop] forMode:NSDefaultRunLoopMode];
}
// Keep the run loop alive, checking for cancellation regularly
while (![NSThread currentThread].isCancelled && self->_metalDisplayLink) {
@autoreleasepool {
[self.delegate waitToRenderTo:self.metalLayer];
[self.delegate renderTo:self.metalLayer];
// Run the run loop for a short time to allow checking cancellation
[[NSRunLoop currentRunLoop] runMode:NSDefaultRunLoopMode beforeDate:[NSDate dateWithTimeIntervalSinceNow:0.1]];
}
}
// Remove from run loop if still valid
if (@available(iOS 17.0, *)) {
if (self->_metalDisplayLink) {
[self->_metalDisplayLink removeFromRunLoop:[NSRunLoop currentRunLoop] forMode:NSDefaultRunLoopMode];
}
}
Log(LOG_I, @"[MetalView] renderThread is exiting");
@@ -74,7 +148,7 @@
_renderThread.name = @"MetalVideoRenderer";
_renderThread.qualityOfService = NSQualityOfServiceUserInteractive;
[_renderThread start];
Log(LOG_I, @"[MetalView] started renderThread %@", _renderThread);
Log(LOG_I, @"[MetalView] started renderThread with CAMetalDisplayLink at %f fps", _framerate);
// Perform any actions that need to know the size and scale of the drawable. When UIKit calls
// didMoveToWindow after the view initialization, this is the first opportunity to notify
VoidLink/Metal/MetalViewController.m
+21 -3
View File
@@ -81,13 +81,25 @@ The implementation of the cross-platform game view controller.
[_frameQueue waitForEnqueue];
}
/// Draw frame (used by manual loop)
- (void)renderTo:(nonnull CAMetalLayer *)layer {
// New method for CAMetalDisplayLink that provides the drawable
- (void)renderWithDrawable:(nonnull id<CAMetalDrawable>)drawable toLayer:(nonnull CAMetalLayer *)layer API_AVAILABLE(ios(17.0)) {
if (!_renderer.isStopping) {
CFTimeInterval timeout = (1.0f / _framerate) - _renderer.averageGPUTime;
Frame *frame = [_frameQueue dequeueWithTimeout:timeout];
if (frame) {
[_renderer renderFrame:frame toLayer:layer];
[_renderer renderFrame:frame withDrawable:drawable];
}
}
}
// Enhanced method that receives timing information from CAMetalDisplayLink
- (void)renderWithDrawable:(nonnull id<CAMetalDrawable>)drawable toLayer:(nonnull CAMetalLayer *)layer targetPresentationTimestamp:(CFTimeInterval)targetPresentationTimestamp API_AVAILABLE(ios(17.0)) {
if (!_renderer.isStopping) {
CFTimeInterval timeout = (1.0f / _framerate) - _renderer.averageGPUTime;
Frame *frame = [_frameQueue dequeueWithTimeout:timeout];
if (frame) {
[_renderer renderFrame:frame withDrawable:drawable targetPresentationTimestamp:targetPresentationTimestamp];
}
}
}
@@ -101,8 +113,14 @@ The implementation of the cross-platform game view controller.
Log(LOG_I, @"[MetalViewController] viewDidDisappear");
// Shutdown the renderer first
[_renderer shutdown];
_renderer = nil;
// Then shutdown the view's display link
if (_metalView) {
[_metalView shutdown];
}
}
#if TARGET_OS_IOS
VoidLink/ViewControllers/SettingsViewController.m
+5 -1
View File
@@ -535,7 +535,11 @@ BOOL isCustomResolution(int resolutionSelected) {
[self addSetting:self.pipStack ofId:@"pipStack" withInfoTag:YES withDynamicLabel:NO to:videoSection];
[self addSetting:self.pipStack ofId:@"pipStack" withInfoTag:YES withDynamicLabel:NO to:videoSection];
[self addSetting:self.frameQueueSizeStack ofId:@"frameQueueSizeStack" withInfoTag:NO withDynamicLabel:NO to:videoSection];
[self addSetting:self.renderingBackendStack ofId:@"renderingBackendStack" withInfoTag:YES withDynamicLabel:NO to:videoSection];
// Only show Metal renderer option on iOS 17+ where CAMetalDisplayLink is available
if (@available(iOS 17.0, *)) {
[self addSetting:self.renderingBackendStack ofId:@"renderingBackendStack" withInfoTag:YES withDynamicLabel:NO to:videoSection];
}
[videoSection addToParentStack:_parentStack];
[videoSection setExpanded:YES];