feat: use CAMetalDisplayLink for metal renderer to fix refresh rate issues

This reverts commit 868288e7
Using CAMetalDisplayLink with CAMetalLayer is required for ProMotion to work consistently and make graphs and touch widgets work smoothly with Metal (especially wasdpad)
With CADisplayLink (no-op or not) the widgets and/or graphs can cause fps drops to the stream

VoidLink/Database/TemporarySettings.m

@@ -52,7 +52,15 @@
     self.statsOverlay = [[NSUserDefaults standardUserDefaults] boolForKey:@"statsOverlay"];
     self.enableGraphs = [[NSUserDefaults standardUserDefaults] boolForKey:@"enableGraphs"];
     self.graphOpacity = [NSNumber numberWithInteger:[[NSUserDefaults standardUserDefaults] integerForKey:@"graphOpacity"]];
-    self.renderingBackend = [NSNumber numberWithInteger:[[NSUserDefaults standardUserDefaults] integerForKey:@"renderingBackend"]];
+    // Set rendering backend based on iOS version and user preference
+    NSInteger savedBackend = [[NSUserDefaults standardUserDefaults] integerForKey:@"renderingBackend"];
+    if (@available(iOS 17.0, *)) {
+        // iOS 17+ can use Performance mode (Metal renderer)
+        self.renderingBackend = [NSNumber numberWithInteger:savedBackend];
+    } else {
+        // iOS < 17 must use Balanced mode (AVSB renderer)
+        self.renderingBackend = [NSNumber numberWithInteger:RENDER_AVSB];
+    }
 
     NSInteger _screenSize = [[NSUserDefaults standardUserDefaults] integerForKey:@"streamResolution"];
     switch (_screenSize) {

VoidLink/Metal/MetalVideoRenderer.h

@@ -23,8 +23,9 @@
 @property (atomic) BOOL isStopping;
 
 - (nonnull instancetype)initWithMetalDevice:(nonnull id<MTLDevice>)device drawablePixelFormat:(MTLPixelFormat)drawablePixelFormat framerate:(float)framerate;
-- (void)renderFrame:(nonnull Frame *)frame toLayer:(nonnull CAMetalLayer *)layer API_AVAILABLE(ios(13.0));
+- (void)renderFrame:(nonnull Frame *)frame withDrawable:(nonnull id<CAMetalDrawable>)drawable API_AVAILABLE(ios(17.0));
-- (void)waitToRenderTo:(nonnull CAMetalLayer *)layer API_AVAILABLE(ios(13.0));
+- (void)renderFrame:(nonnull Frame *)frame withDrawable:(nonnull id<CAMetalDrawable>)drawable targetPresentationTimestamp:(CFTimeInterval)targetPresentationTimestamp API_AVAILABLE(ios(17.0));
+- (void)waitToRenderTo:(nonnull CAMetalLayer *)layer;
 - (void)drawableResize:(CGSize)drawableSize;
 - (void)shutdown;
 

VoidLink/Metal/MetalVideoRenderer.m

@@ -486,21 +486,29 @@ CFStringRef __currentColorSpace;
     return YES;
 }
 
-- (void)renderFrame:(Frame *)frame toLayer:(CAMetalLayer *)layer {
+
+- (void)renderFrame:(Frame *)frame withDrawable:(id<CAMetalDrawable>)drawable targetPresentationTimestamp:(CFTimeInterval)targetPresentationTimestamp API_AVAILABLE(ios(17.0)) {
     @autoreleasepool {
         if (self.isStopping) {
             Log(LOG_I, @"[MetalVideoRenderer] isStopping");
             return;
         }
 
+        // Calculate accurate frametime using CAMetalDisplayLink timing
+        if (self.lastPresented > 0.0f) {
+            CFTimeInterval frametime = targetPresentationTimestamp - self.lastPresented;
+            [[ImGuiPlots sharedInstance] observeFloat:PLOT_FRAMETIME value:(frametime * 1000.0)];
+        }
+        self.lastPresented = targetPresentationTimestamp;
+
         // Handle changes to the frame's colorspace from last time we rendered
         BOOL layerDidChange = NO;
-        if (![self updateColorSpaceForFrame:frame toLayer:layer layerDidChange:&layerDidChange]) {
+        if (![self updateColorSpaceForFrame:frame toLayer:(CAMetalLayer *)drawable.layer layerDidChange:&layerDidChange]) {
             return;
         }
 
         // Handle changes to the video size or drawable size
-        if (![self updateVideoRegionSizeForFrame:frame toLayer:layer]) {
+        if (![self updateVideoRegionSizeForFrame:frame toLayer:(CAMetalLayer *)drawable.layer]) {
             return;
         }
 
@@ -523,13 +531,6 @@ CFStringRef __currentColorSpace;
             }
         }
 
-        // Get the next drawable early to get its pixel format
-        id<CAMetalDrawable> drawable = [layer nextDrawable];
-        if (!drawable) {
-            Log(LOG_E, @"Failed to get nextDrawable");
-            return;
-        }
-        
         // Get the framebuffer pixel format for pipeline creation
         MTLPixelFormat framebufferPixelFormat = drawable.texture.pixelFormat;
 
@@ -636,7 +637,7 @@ CFStringRef __currentColorSpace;
         [renderEncoder setVertexBuffer:_VideoVertexBuffer offset:0 atIndex:0];
         [renderEncoder setFragmentBuffer:_CscParamsBuffer offset:0 atIndex:0];
 #if !TARGET_OS_TV
-        if (layer.pixelFormat == MTLPixelFormatRGBA16Float) {
+        if (((CAMetalLayer *)drawable.layer).pixelFormat == MTLPixelFormatRGBA16Float) {
             [self pollCurrentEDRHeadroom];
             [renderEncoder setFragmentBytes:&_currentEDRHeadroom length:sizeof(float) atIndex:1];
         }
@@ -644,17 +645,6 @@ CFStringRef __currentColorSpace;
         [renderEncoder drawPrimitives:MTLPrimitiveTypeTriangleStrip vertexStart:0 vertexCount:4];
         [renderEncoder endEncoding];
 
-        __block MetalVideoRenderer *strongSelf = self;
-#if !TARGET_OS_SIMULATOR
-        [drawable addPresentedHandler:^(id<MTLDrawable> d) {
-            if (strongSelf.lastPresented > 0.0f) {
-                CFTimeInterval frametime = d.presentedTime - strongSelf.lastPresented;
-                [[ImGuiPlots sharedInstance] observeFloat:PLOT_FRAMETIME value:(frametime * 1000.0)];
-            }
-            strongSelf.lastPresented = d.presentedTime;
-        }];
-#endif
-
         // signal semaphore, compute GPU time average, and clear textures
         __block dispatch_semaphore_t block_semaphore = _inFlightSemaphore;
         [commandBuffer addCompletedHandler:^(id<MTLCommandBuffer> cb) {
@@ -675,12 +665,8 @@ CFStringRef __currentColorSpace;
             CVMetalTextureCacheFlush(self->_textureCache, 0);
         }];
 
-#if TARGET_OS_SIMULATOR
+        // CAMetalDisplayLink handles timing automatically, so we don't use afterMinimumDuration
         [commandBuffer presentDrawable:drawable];
-#else
-        // present for a minimum duration for best frame pacing
-        [commandBuffer presentDrawable:drawable afterMinimumDuration:1.0f / _framerate];
-#endif
 
         [commandBuffer commit];
     }

VoidLink/Metal/MetalView.h

@@ -15,13 +15,14 @@
 @protocol MetalViewDelegate <NSObject>
 
 - (void)drawableResize:(CGSize)size;
-- (void)renderTo:(nonnull CAMetalLayer *)layer;
+- (void)renderWithDrawable:(nonnull id<CAMetalDrawable>)drawable toLayer:(nonnull CAMetalLayer *)layer API_AVAILABLE(ios(17.0));
+- (void)renderWithDrawable:(nonnull id<CAMetalDrawable>)drawable toLayer:(nonnull CAMetalLayer *)layer targetPresentationTimestamp:(CFTimeInterval)targetPresentationTimestamp API_AVAILABLE(ios(17.0));
 - (void)waitToRenderTo:(nonnull CAMetalLayer *)layer;
 
 @end
 
 // The Metal game view base class.
-@interface MetalView : UIView <CALayerDelegate>
+@interface MetalView : UIView <CALayerDelegate, CAMetalDisplayLinkDelegate>
 
 @property (nonatomic, nonnull, readonly) CAMetalLayer *metalLayer;
 @property (nonatomic, nullable) id<MetalViewDelegate> delegate;

VoidLink/Metal/MetalView.m

@@ -11,10 +11,12 @@
 
 #import "MetalView.h"
 #import "MetalConfig.h"
+#import <QuartzCore/CAMetalDisplayLink.h>
 
 @implementation MetalView {
     // The secondary thread containing the render loop.
     NSThread *_renderThread;
+    CAMetalDisplayLink *_metalDisplayLink API_AVAILABLE(ios(17.0));
 }
 
 #pragma mark - Initialization and Setup.
@@ -40,15 +42,43 @@
     self.layer.delegate = self;
 }
 
+- (void)setFramerate:(float)framerate {
+    _framerate = framerate;
+    
+    if (@available(iOS 17.0, *)) {
+        if (_metalDisplayLink) {
+            _metalDisplayLink.preferredFrameRateRange = CAFrameRateRangeMake(framerate, framerate, framerate);
+        }
+    }
+}
+
 - (void)shutdown {
+    // First cancel the thread to stop the run loop
     if (_renderThread) {
         Log(LOG_I, @"[MetalView] sending renderThread a cancel message");
         [_renderThread cancel];
+        
+        // Invalidate metal display link to stop callbacks
+        if (@available(iOS 17.0, *)) {
+            if (_metalDisplayLink) {
+                Log(LOG_I, @"[MetalView] invalidating metal display link");
+                [_metalDisplayLink invalidate];
+                _metalDisplayLink = nil;
+            }
+        }
+        
+        // Now wait for thread to finish
         Log(LOG_I, @"[MetalView] waiting on renderThread to finish");
-        while (!_renderThread.isFinished) {
+        NSDate *timeout = [NSDate dateWithTimeIntervalSinceNow:2.0]; // 2 second timeout
-            usleep(100);
+        while (!_renderThread.isFinished && [timeout timeIntervalSinceNow] > 0) {
+            usleep(1000);
+        }
+        
+        if (_renderThread.isFinished) {
+            Log(LOG_I, @"[MetalView] renderThread has finished");
+        } else {
+            Log(LOG_E, @"[MetalView] renderThread failed to finish in time");
         }
-        Log(LOG_I, @"[MetalView] renderThread has finished");
         _renderThread = nil;
     }
 }
@@ -61,6 +91,26 @@
     [self movedToWindow];
 }
 
+- (void)metalDisplayLink:(CAMetalDisplayLink *)link needsUpdate:(CAMetalDisplayLinkUpdate *)update API_AVAILABLE(ios(17.0)) {
+    // Skip rendering if we're shutting down
+    if ([NSThread currentThread].isCancelled) {
+        return;
+    }
+    @autoreleasepool {
+        [self.delegate waitToRenderTo:self.metalLayer];
+        
+        id<CAMetalDrawable> drawable = update.drawable;
+        if (drawable) {
+            // Pass timing information to the renderer
+            if ([self.delegate respondsToSelector:@selector(renderWithDrawable:toLayer:targetPresentationTimestamp:)]) {
+                [self.delegate renderWithDrawable:drawable toLayer:self.metalLayer targetPresentationTimestamp:update.targetPresentationTimestamp];
+            } else {
+                [self.delegate renderWithDrawable:drawable toLayer:self.metalLayer];
+            }
+        }
+    }
+}
+
 - (void)movedToWindow {
     if (!self.window) {
         Log(LOG_I, @"[MetalView] movedToWindow(nil): shutting down...");
@@ -68,12 +118,32 @@
         return;
     }
 
-    // Render on a new thread
+    if (@available(iOS 17.0, *)) {
+        _metalDisplayLink = [[CAMetalDisplayLink alloc] initWithMetalLayer:_metalLayer];
+        _metalDisplayLink.delegate = self;
+        _metalDisplayLink.preferredFrameRateRange = CAFrameRateRangeMake(_framerate, _framerate, _framerate);
+        Log(LOG_I, @"[MetalView] Using CAMetalDisplayLink for optimal Metal rendering");
+    }
+    
+    // Start the display link on a background thread
     _renderThread = [[NSThread alloc] initWithBlock:^{
-        while (![NSThread currentThread].isCancelled) {
+        // Add metal display link to this thread's run loop
+        if (@available(iOS 17.0, *)) {
+            [self->_metalDisplayLink addToRunLoop:[NSRunLoop currentRunLoop] forMode:NSDefaultRunLoopMode];
+        }
+        
+        // Keep the run loop alive, checking for cancellation regularly
+        while (![NSThread currentThread].isCancelled && self->_metalDisplayLink) {
             @autoreleasepool {
-                [self.delegate waitToRenderTo:self.metalLayer];
+                // Run the run loop for a short time to allow checking cancellation
-                [self.delegate renderTo:self.metalLayer];
+                [[NSRunLoop currentRunLoop] runMode:NSDefaultRunLoopMode beforeDate:[NSDate dateWithTimeIntervalSinceNow:0.1]];
+            }
+        }
+        
+        // Remove from run loop if still valid
+        if (@available(iOS 17.0, *)) {
+            if (self->_metalDisplayLink) {
+                [self->_metalDisplayLink removeFromRunLoop:[NSRunLoop currentRunLoop] forMode:NSDefaultRunLoopMode];
             }
         }
         Log(LOG_I, @"[MetalView] renderThread is exiting");
@@ -81,7 +151,7 @@
     _renderThread.name = @"MetalVideoRenderer";
     _renderThread.qualityOfService = NSQualityOfServiceUserInteractive;
     [_renderThread start];
-    Log(LOG_I, @"[MetalView] started renderThread %@", _renderThread);
+    Log(LOG_I, @"[MetalView] started renderThread with CAMetalDisplayLink at %f fps", _framerate);
 
     // Perform any actions that need to know the size and scale of the drawable. When UIKit calls
     // didMoveToWindow after the view initialization, this is the first opportunity to notify

VoidLink/Metal/MetalViewController.m

@@ -84,15 +84,23 @@
     [_frameQueue waitForEnqueue];
 }
 
-/// Draw frame (used by manual loop)
+
-- (void)renderTo:(nonnull CAMetalLayer *)layer {
+- (void)renderWithDrawable:(nonnull id<CAMetalDrawable>)drawable toLayer:(nonnull CAMetalLayer *)layer API_AVAILABLE(ios(17.0)) {
     if (!_renderer.isStopping) {
         CFTimeInterval timeout = (1.0f / _framerate) - _renderer.averageGPUTime;
         Frame *frame = [_frameQueue dequeueWithTimeout:timeout];
         if (frame) {
-            if (@available(iOS 13.0, *)) {
+            [_renderer renderFrame:frame withDrawable:drawable];
-                [_renderer renderFrame:frame toLayer:layer];
+        }
-            }
+    }
+}
+
+- (void)renderWithDrawable:(nonnull id<CAMetalDrawable>)drawable toLayer:(nonnull CAMetalLayer *)layer targetPresentationTimestamp:(CFTimeInterval)targetPresentationTimestamp API_AVAILABLE(ios(17.0)) {
+    if (!_renderer.isStopping) {
+        CFTimeInterval timeout = (1.0f / _framerate) - _renderer.averageGPUTime;
+        Frame *frame = [_frameQueue dequeueWithTimeout:timeout];
+        if (frame) {
+            [_renderer renderFrame:frame withDrawable:drawable targetPresentationTimestamp:targetPresentationTimestamp];
         }
     }
 }
@@ -106,8 +114,14 @@
 
     Log(LOG_I, @"[MetalViewController] viewDidDisappear");
 
+    // Shutdown the renderer first
     [_renderer shutdown];
     _renderer = nil;
+
+    // Then shutdown the view's display link
+    if (_metalView) {
+        [_metalView shutdown];
+    }
 }
 
 #if TARGET_OS_IOS

VoidLink/ViewControllers/SettingsViewController.m

@@ -535,7 +535,11 @@ BOOL isCustomResolution(int resolutionSelected) {
     [self addSetting:self.pipStack ofId:@"pipStack" withInfoTag:YES withDynamicLabel:NO to:videoSection];
     [self addSetting:self.pipStack ofId:@"pipStack" withInfoTag:YES withDynamicLabel:NO to:videoSection];
     [self addSetting:self.frameQueueSizeStack ofId:@"frameQueueSizeStack" withInfoTag:NO withDynamicLabel:NO to:videoSection];
-    [self addSetting:self.renderingBackendStack ofId:@"renderingBackendStack" withInfoTag:YES withDynamicLabel:NO to:videoSection];
+    
+    // Only show Metal renderer option on iOS 17+ where CAMetalDisplayLink is available
+    if (@available(iOS 17.0, *)) {
+        [self addSetting:self.renderingBackendStack ofId:@"renderingBackendStack" withInfoTag:YES withDynamicLabel:NO to:videoSection];
+    }
 
     [videoSection addToParentStack:_parentStack];
     [videoSection setExpanded:YES];