hermes-agent/VoidLink
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: use CAMetalDisplayLink for metal renderer to fix refresh rate issues

Browse Source 
This reverts commit 868288e7
Using CAMetalDisplayLink with CAMetalLayer is required for ProMotion to work consistently and make graphs and touch widgets work smoothly with Metal (especially wasdpad)
With CADisplayLink (no-op or not) the widgets and/or graphs can cause fps drops to the stream
This commit is contained in:
Acaki
2025-08-19 18:47:41 +08:00
parent 9e32c1da3d
commit 9e20efff4b
7 changed files with 130 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
VoidLink/Database/TemporarySettings.m
+9 -1
View File
@@ -52,7 +52,15 @@
self.statsOverlay = [[NSUserDefaults standardUserDefaults] boolForKey:@"statsOverlay"];
self.enableGraphs = [[NSUserDefaults standardUserDefaults] boolForKey:@"enableGraphs"];
self.graphOpacity = [NSNumber numberWithInteger:[[NSUserDefaults standardUserDefaults] integerForKey:@"graphOpacity"]];
self.renderingBackend = [NSNumber numberWithInteger:[[NSUserDefaults standardUserDefaults] integerForKey:@"renderingBackend"]];
// Set rendering backend based on iOS version and user preference
NSInteger savedBackend = [[NSUserDefaults standardUserDefaults] integerForKey:@"renderingBackend"];
if (@available(iOS 17.0, *)) {
// iOS 17+ can use Performance mode (Metal renderer)
self.renderingBackend = [NSNumber numberWithInteger:savedBackend];
} else {
// iOS < 17 must use Balanced mode (AVSB renderer)
self.renderingBackend = [NSNumber numberWithInteger:RENDER_AVSB];
}
NSInteger _screenSize = [[NSUserDefaults standardUserDefaults] integerForKey:@"streamResolution"];
switch (_screenSize) {
VoidLink/Metal/MetalVideoRenderer.h
+3 -2
View File
@@ -23,8 +23,9 @@
@property (atomic) BOOL isStopping;
- (nonnull instancetype)initWithMetalDevice:(nonnull id<MTLDevice>)device drawablePixelFormat:(MTLPixelFormat)drawablePixelFormat framerate:(float)framerate;
- (void)renderFrame:(nonnull Frame *)frame toLayer:(nonnull CAMetalLayer *)layer API_AVAILABLE(ios(13.0));
- (void)waitToRenderTo:(nonnull CAMetalLayer *)layer API_AVAILABLE(ios(13.0));
- (void)renderFrame:(nonnull Frame *)frame withDrawable:(nonnull id<CAMetalDrawable>)drawable API_AVAILABLE(ios(17.0));
- (void)renderFrame:(nonnull Frame *)frame withDrawable:(nonnull id<CAMetalDrawable>)drawable targetPresentationTimestamp:(CFTimeInterval)targetPresentationTimestamp API_AVAILABLE(ios(17.0));
- (void)waitToRenderTo:(nonnull CAMetalLayer *)layer;
- (void)drawableResize:(CGSize)drawableSize;
- (void)shutdown;
VoidLink/Metal/MetalVideoRenderer.m
+13 -27
View File
@@ -486,21 +486,29 @@ CFStringRef __currentColorSpace;
return YES;
}
- (void)renderFrame:(Frame *)frame toLayer:(CAMetalLayer *)layer {
- (void)renderFrame:(Frame *)frame withDrawable:(id<CAMetalDrawable>)drawable targetPresentationTimestamp:(CFTimeInterval)targetPresentationTimestamp API_AVAILABLE(ios(17.0)) {
@autoreleasepool {
if (self.isStopping) {
Log(LOG_I, @"[MetalVideoRenderer] isStopping");
return;
}
// Calculate accurate frametime using CAMetalDisplayLink timing
if (self.lastPresented > 0.0f) {
CFTimeInterval frametime = targetPresentationTimestamp - self.lastPresented;
[[ImGuiPlots sharedInstance] observeFloat:PLOT_FRAMETIME value:(frametime * 1000.0)];
}
self.lastPresented = targetPresentationTimestamp;
// Handle changes to the frame's colorspace from last time we rendered
BOOL layerDidChange = NO;
if (![self updateColorSpaceForFrame:frame toLayer:layer layerDidChange:&layerDidChange]) {
if (![self updateColorSpaceForFrame:frame toLayer:(CAMetalLayer *)drawable.layer layerDidChange:&layerDidChange]) {
return;
}
// Handle changes to the video size or drawable size
if (![self updateVideoRegionSizeForFrame:frame toLayer:layer]) {
if (![self updateVideoRegionSizeForFrame:frame toLayer:(CAMetalLayer *)drawable.layer]) {
return;
}
@@ -523,13 +531,6 @@ CFStringRef __currentColorSpace;
}
}
// Get the next drawable early to get its pixel format
id<CAMetalDrawable> drawable = [layer nextDrawable];
if (!drawable) {
Log(LOG_E, @"Failed to get nextDrawable");
return;
}
// Get the framebuffer pixel format for pipeline creation
MTLPixelFormat framebufferPixelFormat = drawable.texture.pixelFormat;
@@ -636,7 +637,7 @@ CFStringRef __currentColorSpace;
[renderEncoder setVertexBuffer:_VideoVertexBuffer offset:0 atIndex:0];
[renderEncoder setFragmentBuffer:_CscParamsBuffer offset:0 atIndex:0];
#if !TARGET_OS_TV
if (layer.pixelFormat == MTLPixelFormatRGBA16Float) {
if (((CAMetalLayer *)drawable.layer).pixelFormat == MTLPixelFormatRGBA16Float) {
[self pollCurrentEDRHeadroom];
[renderEncoder setFragmentBytes:&_currentEDRHeadroom length:sizeof(float) atIndex:1];
}
@@ -644,17 +645,6 @@ CFStringRef __currentColorSpace;
[renderEncoder drawPrimitives:MTLPrimitiveTypeTriangleStrip vertexStart:0 vertexCount:4];
[renderEncoder endEncoding];
__block MetalVideoRenderer *strongSelf = self;
#if !TARGET_OS_SIMULATOR
[drawable addPresentedHandler:^(id<MTLDrawable> d) {
if (strongSelf.lastPresented > 0.0f) {
CFTimeInterval frametime = d.presentedTime - strongSelf.lastPresented;
[[ImGuiPlots sharedInstance] observeFloat:PLOT_FRAMETIME value:(frametime * 1000.0)];
}
strongSelf.lastPresented = d.presentedTime;
}];
#endif
// signal semaphore, compute GPU time average, and clear textures
__block dispatch_semaphore_t block_semaphore = _inFlightSemaphore;
[commandBuffer addCompletedHandler:^(id<MTLCommandBuffer> cb) {
@@ -675,12 +665,8 @@ CFStringRef __currentColorSpace;
CVMetalTextureCacheFlush(self->_textureCache, 0);
}];
#if TARGET_OS_SIMULATOR
// CAMetalDisplayLink handles timing automatically, so we don't use afterMinimumDuration
[commandBuffer presentDrawable:drawable];
#else
// present for a minimum duration for best frame pacing
[commandBuffer presentDrawable:drawable afterMinimumDuration:1.0f / _framerate];
#endif
[commandBuffer commit];
}
VoidLink/Metal/MetalView.h
+3 -2
View File
@@ -15,13 +15,14 @@
@protocol MetalViewDelegate <NSObject>
- (void)drawableResize:(CGSize)size;
- (void)renderTo:(nonnull CAMetalLayer *)layer;
- (void)renderWithDrawable:(nonnull id<CAMetalDrawable>)drawable toLayer:(nonnull CAMetalLayer *)layer API_AVAILABLE(ios(17.0));
- (void)renderWithDrawable:(nonnull id<CAMetalDrawable>)drawable toLayer:(nonnull CAMetalLayer *)layer targetPresentationTimestamp:(CFTimeInterval)targetPresentationTimestamp API_AVAILABLE(ios(17.0));
- (void)waitToRenderTo:(nonnull CAMetalLayer *)layer;
@end
// The Metal game view base class.
@interface MetalView : UIView <CALayerDelegate>
@interface MetalView : UIView <CALayerDelegate, CAMetalDisplayLinkDelegate>
@property (nonatomic, nonnull, readonly) CAMetalLayer *metalLayer;
@property (nonatomic, nullable) id<MetalViewDelegate> delegate;
VoidLink/Metal/MetalView.m
+78 -8
View File
@@ -11,10 +11,12 @@
#import "MetalView.h"
#import "MetalConfig.h"
#import <QuartzCore/CAMetalDisplayLink.h>
@implementation MetalView {
// The secondary thread containing the render loop.
NSThread *_renderThread;
CAMetalDisplayLink *_metalDisplayLink API_AVAILABLE(ios(17.0));
}
#pragma mark - Initialization and Setup.
@@ -40,15 +42,43 @@
self.layer.delegate = self;
}
- (void)setFramerate:(float)framerate {
_framerate = framerate;
if (@available(iOS 17.0, *)) {
if (_metalDisplayLink) {
_metalDisplayLink.preferredFrameRateRange = CAFrameRateRangeMake(framerate, framerate, framerate);
}
}
}
- (void)shutdown {
// First cancel the thread to stop the run loop
if (_renderThread) {
Log(LOG_I, @"[MetalView] sending renderThread a cancel message");
[_renderThread cancel];
// Invalidate metal display link to stop callbacks
if (@available(iOS 17.0, *)) {
if (_metalDisplayLink) {
Log(LOG_I, @"[MetalView] invalidating metal display link");
[_metalDisplayLink invalidate];
_metalDisplayLink = nil;
}
}
// Now wait for thread to finish
Log(LOG_I, @"[MetalView] waiting on renderThread to finish");
while (!_renderThread.isFinished) {
usleep(100);
NSDate *timeout = [NSDate dateWithTimeIntervalSinceNow:2.0]; // 2 second timeout
while (!_renderThread.isFinished && [timeout timeIntervalSinceNow] > 0) {
usleep(1000);
}
if (_renderThread.isFinished) {
Log(LOG_I, @"[MetalView] renderThread has finished");
} else {
Log(LOG_E, @"[MetalView] renderThread failed to finish in time");
}
Log(LOG_I, @"[MetalView] renderThread has finished");
_renderThread = nil;
}
}
@@ -61,6 +91,26 @@
[self movedToWindow];
}
- (void)metalDisplayLink:(CAMetalDisplayLink *)link needsUpdate:(CAMetalDisplayLinkUpdate *)update API_AVAILABLE(ios(17.0)) {
// Skip rendering if we're shutting down
if ([NSThread currentThread].isCancelled) {
return;
}
@autoreleasepool {
[self.delegate waitToRenderTo:self.metalLayer];
id<CAMetalDrawable> drawable = update.drawable;
if (drawable) {
// Pass timing information to the renderer
if ([self.delegate respondsToSelector:@selector(renderWithDrawable:toLayer:targetPresentationTimestamp:)]) {
[self.delegate renderWithDrawable:drawable toLayer:self.metalLayer targetPresentationTimestamp:update.targetPresentationTimestamp];
} else {
[self.delegate renderWithDrawable:drawable toLayer:self.metalLayer];
}
}
}
}
- (void)movedToWindow {
if (!self.window) {
Log(LOG_I, @"[MetalView] movedToWindow(nil): shutting down...");
@@ -68,12 +118,32 @@
return;
}
// Render on a new thread
if (@available(iOS 17.0, *)) {
_metalDisplayLink = [[CAMetalDisplayLink alloc] initWithMetalLayer:_metalLayer];
_metalDisplayLink.delegate = self;
_metalDisplayLink.preferredFrameRateRange = CAFrameRateRangeMake(_framerate, _framerate, _framerate);
Log(LOG_I, @"[MetalView] Using CAMetalDisplayLink for optimal Metal rendering");
}
// Start the display link on a background thread
_renderThread = [[NSThread alloc] initWithBlock:^{
while (![NSThread currentThread].isCancelled) {
// Add metal display link to this thread's run loop
if (@available(iOS 17.0, *)) {
[self->_metalDisplayLink addToRunLoop:[NSRunLoop currentRunLoop] forMode:NSDefaultRunLoopMode];
}
// Keep the run loop alive, checking for cancellation regularly
while (![NSThread currentThread].isCancelled && self->_metalDisplayLink) {
@autoreleasepool {
[self.delegate waitToRenderTo:self.metalLayer];
[self.delegate renderTo:self.metalLayer];
// Run the run loop for a short time to allow checking cancellation
[[NSRunLoop currentRunLoop] runMode:NSDefaultRunLoopMode beforeDate:[NSDate dateWithTimeIntervalSinceNow:0.1]];
}
}
// Remove from run loop if still valid
if (@available(iOS 17.0, *)) {
if (self->_metalDisplayLink) {
[self->_metalDisplayLink removeFromRunLoop:[NSRunLoop currentRunLoop] forMode:NSDefaultRunLoopMode];
}
}
Log(LOG_I, @"[MetalView] renderThread is exiting");
@@ -81,7 +151,7 @@
_renderThread.name = @"MetalVideoRenderer";
_renderThread.qualityOfService = NSQualityOfServiceUserInteractive;
[_renderThread start];
Log(LOG_I, @"[MetalView] started renderThread %@", _renderThread);
Log(LOG_I, @"[MetalView] started renderThread with CAMetalDisplayLink at %f fps", _framerate);
// Perform any actions that need to know the size and scale of the drawable. When UIKit calls
// didMoveToWindow after the view initialization, this is the first opportunity to notify
VoidLink/Metal/MetalViewController.m
+19 -5
View File
@@ -84,15 +84,23 @@
[_frameQueue waitForEnqueue];
}
/// Draw frame (used by manual loop)
- (void)renderTo:(nonnull CAMetalLayer *)layer {
- (void)renderWithDrawable:(nonnull id<CAMetalDrawable>)drawable toLayer:(nonnull CAMetalLayer *)layer API_AVAILABLE(ios(17.0)) {
if (!_renderer.isStopping) {
CFTimeInterval timeout = (1.0f / _framerate) - _renderer.averageGPUTime;
Frame *frame = [_frameQueue dequeueWithTimeout:timeout];
if (frame) {
if (@available(iOS 13.0, *)) {
[_renderer renderFrame:frame toLayer:layer];
}
[_renderer renderFrame:frame withDrawable:drawable];
}
}
}
- (void)renderWithDrawable:(nonnull id<CAMetalDrawable>)drawable toLayer:(nonnull CAMetalLayer *)layer targetPresentationTimestamp:(CFTimeInterval)targetPresentationTimestamp API_AVAILABLE(ios(17.0)) {
if (!_renderer.isStopping) {
CFTimeInterval timeout = (1.0f / _framerate) - _renderer.averageGPUTime;
Frame *frame = [_frameQueue dequeueWithTimeout:timeout];
if (frame) {
[_renderer renderFrame:frame withDrawable:drawable targetPresentationTimestamp:targetPresentationTimestamp];
}
}
}
@@ -106,8 +114,14 @@
Log(LOG_I, @"[MetalViewController] viewDidDisappear");
// Shutdown the renderer first
[_renderer shutdown];
_renderer = nil;
// Then shutdown the view's display link
if (_metalView) {
[_metalView shutdown];
}
}
#if TARGET_OS_IOS
VoidLink/ViewControllers/SettingsViewController.m
+5 -1
View File
@@ -535,7 +535,11 @@ BOOL isCustomResolution(int resolutionSelected) {
[self addSetting:self.pipStack ofId:@"pipStack" withInfoTag:YES withDynamicLabel:NO to:videoSection];
[self addSetting:self.pipStack ofId:@"pipStack" withInfoTag:YES withDynamicLabel:NO to:videoSection];
[self addSetting:self.frameQueueSizeStack ofId:@"frameQueueSizeStack" withInfoTag:NO withDynamicLabel:NO to:videoSection];
[self addSetting:self.renderingBackendStack ofId:@"renderingBackendStack" withInfoTag:YES withDynamicLabel:NO to:videoSection];
// Only show Metal renderer option on iOS 17+ where CAMetalDisplayLink is available
if (@available(iOS 17.0, *)) {
[self addSetting:self.renderingBackendStack ofId:@"renderingBackendStack" withInfoTag:YES withDynamicLabel:NO to:videoSection];
}
[videoSection addToParentStack:_parentStack];
[videoSection setExpanded:YES];