feat: use CAMetalDisplayLink for metal renderer to fix refresh rate issues

- update imgui to use stream fps instead of hardcoded 60
- hide renderer setting for iOS < 17 as CAMetalDisplayLink only supports later versions

VoidLink/Database/TemporarySettings.m

@@ -52,7 +52,15 @@
     self.statsOverlay = [[NSUserDefaults standardUserDefaults] boolForKey:@"statsOverlay"];
     self.enableGraphs = [[NSUserDefaults standardUserDefaults] boolForKey:@"enableGraphs"];
     self.graphOpacity = [NSNumber numberWithInteger:[[NSUserDefaults standardUserDefaults] integerForKey:@"graphOpacity"]];
-    self.renderingBackend = [NSNumber numberWithInteger:[[NSUserDefaults standardUserDefaults] integerForKey:@"renderingBackend"]];
+    // Set rendering backend based on iOS version and user preference
+    NSInteger savedBackend = [[NSUserDefaults standardUserDefaults] integerForKey:@"renderingBackend"];
+    if (@available(iOS 17.0, *)) {
+        // iOS 17+ can use Metal renderer (CAMetalDisplayLink available)
+        self.renderingBackend = [NSNumber numberWithInteger:savedBackend];
+    } else {
+        // iOS < 17 must use AVSB renderer (no CAMetalDisplayLink)
+        self.renderingBackend = [NSNumber numberWithInteger:1]; // RENDER_AVSB = 1
+    }
 
     NSInteger _screenSize = [[NSUserDefaults standardUserDefaults] integerForKey:@"streamResolution"];
     switch (_screenSize) {

VoidLink/ImGuiRenderer.h

@@ -21,6 +21,7 @@ typedef void (^MetricsHandler)(int plotId, CFTimeInterval value);
 @property (nonatomic) BOOL enableGraphs;
 @property (nonatomic) float graphOpacity;
 @property (nonatomic) BOOL imguiRunning;
+@property (nonatomic) int streamFps;
 @property (nonatomic) MetricsHandler _Nonnull metricsHandler;
 
 -(nonnull instancetype) initWithFrame:(CGRect)bounds

VoidLink/ImGuiRenderer.mm

@@ -35,6 +35,9 @@
 
     _plots = [ImGuiPlots sharedInstance].plots;
 
+    // Store stream FPS to synchronize MTKView
+    _streamFps = streamFps;
+
     return self;
 }
 
@@ -83,7 +86,7 @@
 
     self.mtkView.device = self.device;
     self.mtkView.delegate = self;
-    self.mtkView.preferredFramesPerSecond = 60; // ImGui overlay will always render at this rate
+    self.mtkView.preferredFramesPerSecond = _streamFps; // Synchronize with stream framerate
     self.mtkView.opaque = NO;
     self.mtkView.enableSetNeedsDisplay = NO;
 
@@ -166,7 +169,7 @@
 #if TARGET_OS_SIMULATOR
     [commandBuffer presentDrawable:view.currentDrawable];
 #else
-    [commandBuffer presentDrawable:view.currentDrawable afterMinimumDuration:1.0 / view.preferredFramesPerSecond];
+    [commandBuffer presentDrawable:view.currentDrawable afterMinimumDuration:1.0 / _streamFps];
 #endif
     [commandBuffer commit];
 #endif

VoidLink/Metal/MetalVideoRenderer.h

@@ -15,7 +15,8 @@
 @property (atomic) BOOL isStopping;
 
 - (nonnull instancetype)initWithMetalDevice:(nonnull id<MTLDevice>)device drawablePixelFormat:(MTLPixelFormat)drawablePixelFormat framerate:(float)framerate;
-- (void)renderFrame:(nonnull Frame *)frame toLayer:(nonnull CAMetalLayer *)layer;
+- (void)renderFrame:(nonnull Frame *)frame withDrawable:(nonnull id<CAMetalDrawable>)drawable API_AVAILABLE(ios(17.0));
+- (void)renderFrame:(nonnull Frame *)frame withDrawable:(nonnull id<CAMetalDrawable>)drawable targetPresentationTimestamp:(CFTimeInterval)targetPresentationTimestamp API_AVAILABLE(ios(17.0));
 - (void)waitToRenderTo:(nonnull CAMetalLayer *)layer;
 - (void)drawableResize:(CGSize)drawableSize;
 - (void)shutdown;

VoidLink/Metal/MetalVideoRenderer.m

@@ -443,21 +443,30 @@ CFStringRef __currentColorSpace;
     return YES;
 }
 
-- (void)renderFrame:(Frame *)frame toLayer:(CAMetalLayer *)layer {
+
+// Enhanced method that receives timing information from CAMetalDisplayLink
+- (void)renderFrame:(Frame *)frame withDrawable:(id<CAMetalDrawable>)drawable targetPresentationTimestamp:(CFTimeInterval)targetPresentationTimestamp API_AVAILABLE(ios(17.0)) {
     @autoreleasepool {
         if (self.isStopping) {
             Log(LOG_I, @"[MetalVideoRenderer] isStopping");
             return;
         }
 
+        // Calculate accurate frametime using CAMetalDisplayLink timing
+        if (self.lastPresented > 0.0f) {
+            CFTimeInterval frametime = targetPresentationTimestamp - self.lastPresented;
+            [[ImGuiPlots sharedInstance] observeFloat:PLOT_FRAMETIME value:(frametime * 1000.0)];
+        }
+        self.lastPresented = targetPresentationTimestamp;
+
         // Handle changes to the frame's colorspace from last time we rendered
         BOOL layerDidChange = NO;
-        if (![self updateColorSpaceForFrame:frame toLayer:layer layerDidChange:&layerDidChange]) {
+        if (![self updateColorSpaceForFrame:frame toLayer:(CAMetalLayer *)drawable.layer layerDidChange:&layerDidChange]) {
             return;
         }
 
         // Handle changes to the video size or drawable size
-        if (![self updateVideoRegionSizeForFrame:frame toLayer:layer]) {
+        if (![self updateVideoRegionSizeForFrame:frame toLayer:(CAMetalLayer *)drawable.layer]) {
             return;
         }
 
@@ -483,7 +492,7 @@ CFStringRef __currentColorSpace;
             id<MTLFunction> yuvToLinear = [defaultLibrary newFunctionWithName:@"yuvToLinear"];
 
             // Determine if this is 10-bit based on the layer's pixel format (after colorspace update)
-            BOOL is10Bit = (layer.pixelFormat == MTLPixelFormatBGR10A2Unorm);
+            BOOL is10Bit = (((CAMetalLayer *)drawable.layer).pixelFormat == MTLPixelFormatBGR10A2Unorm);
 
             NSString *fragmentShaderName;
             if (planes == 2) {
@@ -492,13 +501,13 @@ CFStringRef __currentColorSpace;
                 fragmentShaderName = is10Bit ? @"ps_draw_triplanar_10bit" : @"ps_draw_triplanar_8bit";
             }
 
-            pipelineDesc.colorAttachments[0].pixelFormat = layer.pixelFormat;
+            pipelineDesc.colorAttachments[0].pixelFormat = ((CAMetalLayer *)drawable.layer).pixelFormat;
             pipelineDesc.vertexBuffers[0].mutability = MTLMutabilityImmutable;
             
             Log(LOG_I, @"Metal pipeline state for %zu planes with pixel format %@",
-                planes, layer.pixelFormat == MTLPixelFormatRGBA16Float ? @"MTLPixelFormatRGBA16Float" : @"MTLPixelFormatBGR10A2Unorm");
+                planes, ((CAMetalLayer *)drawable.layer).pixelFormat == MTLPixelFormatRGBA16Float ? @"MTLPixelFormatRGBA16Float" : @"MTLPixelFormatBGR10A2Unorm");
 
-            if (layer.pixelFormat == MTLPixelFormatRGBA16Float) {
+            if (((CAMetalLayer *)drawable.layer).pixelFormat == MTLPixelFormatRGBA16Float) {
                 // 4:2:0 or 4:4:4 YUV -> BT.2020 RGB -> linear float
                 pipelineDesc.vertexFunction = vertexVsDraw;
                 pipelineDesc.fragmentFunction = yuvToLinear;
@@ -554,11 +563,7 @@ CFStringRef __currentColorSpace;
             }
         }
 
-        id<CAMetalDrawable> drawable = [layer nextDrawable];
+        // Use the provided drawable instead of getting nextDrawable from layer
-        if (!drawable) {
-            Log(LOG_E, @"Failed to get nextDrawable");
-            return;
-        }
         _renderPassDescriptor.colorAttachments[0].texture = drawable.texture;
 
         id<MTLCommandBuffer> commandBuffer = [_commandQueue commandBuffer];
@@ -572,7 +577,7 @@ CFStringRef __currentColorSpace;
         [renderEncoder setVertexBuffer:_VideoVertexBuffer offset:0 atIndex:0];
         [renderEncoder setFragmentBuffer:_CscParamsBuffer offset:0 atIndex:0];
 #if !TARGET_OS_TV
-        if (layer.pixelFormat == MTLPixelFormatRGBA16Float) {
+        if (((CAMetalLayer *)drawable.layer).pixelFormat == MTLPixelFormatRGBA16Float) {
             [self pollCurrentEDRHeadroom];
             [renderEncoder setFragmentBytes:&_currentEDRHeadroom length:sizeof(float) atIndex:1];
         }
@@ -580,15 +585,6 @@ CFStringRef __currentColorSpace;
         [renderEncoder drawPrimitives:MTLPrimitiveTypeTriangleStrip vertexStart:0 vertexCount:4];
         [renderEncoder endEncoding];
 
-        __block MetalVideoRenderer *strongSelf = self;
-        [drawable addPresentedHandler:^(id<MTLDrawable> d) {
-            if (strongSelf.lastPresented > 0.0f) {
-                CFTimeInterval frametime = d.presentedTime - strongSelf.lastPresented;
-                [[ImGuiPlots sharedInstance] observeFloat:PLOT_FRAMETIME value:(frametime * 1000.0)];
-            }
-            strongSelf.lastPresented = d.presentedTime;
-        }];
-
         // signal semaphore, compute GPU time average, and clear textures
         __block dispatch_semaphore_t block_semaphore = _inFlightSemaphore;
         [commandBuffer addCompletedHandler:^(id<MTLCommandBuffer> cb) {
@@ -609,15 +605,10 @@ CFStringRef __currentColorSpace;
             CVMetalTextureCacheFlush(self->_textureCache, 0);
         }];
 
-#if TARGET_OS_SIMULATOR
+        // CAMetalDisplayLink handles timing automatically, so we don't use afterMinimumDuration
         [commandBuffer presentDrawable:drawable];
-#else
-        // present for a minimum duration for best frame pacing
-        [commandBuffer presentDrawable:drawable afterMinimumDuration:1.0f / _framerate];
-#endif
 
         [commandBuffer commit];
-        [commandBuffer waitUntilCompleted];
     }
 }
 

VoidLink/Metal/MetalView.h

@@ -8,13 +8,14 @@
 @protocol MetalViewDelegate <NSObject>
 
 - (void)drawableResize:(CGSize)size;
-- (void)renderTo:(nonnull CAMetalLayer *)layer;
+- (void)renderWithDrawable:(nonnull id<CAMetalDrawable>)drawable toLayer:(nonnull CAMetalLayer *)layer API_AVAILABLE(ios(17.0));
+- (void)renderWithDrawable:(nonnull id<CAMetalDrawable>)drawable toLayer:(nonnull CAMetalLayer *)layer targetPresentationTimestamp:(CFTimeInterval)targetPresentationTimestamp API_AVAILABLE(ios(17.0));
 - (void)waitToRenderTo:(nonnull CAMetalLayer *)layer;
 
 @end
 
 // The Metal game view base class.
-@interface MetalView : UIView <CALayerDelegate>
+@interface MetalView : UIView <CALayerDelegate, CAMetalDisplayLinkDelegate>
 
 @property (nonatomic, nonnull, readonly) CAMetalLayer *metalLayer;
 @property (nonatomic, nullable) id<MetalViewDelegate> delegate;

VoidLink/Metal/MetalView.m

@@ -4,10 +4,13 @@
 
 #import "MetalView.h"
 #import "MetalConfig.h"
+#import <QuartzCore/CAMetalDisplayLink.h>
 
 @implementation MetalView {
     // The secondary thread containing the render loop.
     NSThread *_renderThread;
+    // Metal display link for vsync synchronization (iOS 17+)
+    CAMetalDisplayLink *_metalDisplayLink API_AVAILABLE(ios(17.0));
 }
 
 #pragma mark - Initialization and Setup.
@@ -33,15 +36,43 @@
     self.layer.delegate = self;
 }
 
+- (void)setFramerate:(float)framerate {
+    _framerate = framerate;
+    
+    if (@available(iOS 17.0, *)) {
+        if (_metalDisplayLink) {
+            _metalDisplayLink.preferredFrameRateRange = CAFrameRateRangeMake(framerate, framerate, framerate);
+        }
+    }
+}
+
 - (void)shutdown {
+    // First cancel the thread to stop the run loop
     if (_renderThread) {
         Log(LOG_I, @"[MetalView] sending renderThread a cancel message");
         [_renderThread cancel];
+        
+        // Invalidate metal display link to stop callbacks
+        if (@available(iOS 17.0, *)) {
+            if (_metalDisplayLink) {
+                Log(LOG_I, @"[MetalView] invalidating metal display link");
+                [_metalDisplayLink invalidate];
+                _metalDisplayLink = nil;
+            }
+        }
+        
+        // Now wait for thread to finish
         Log(LOG_I, @"[MetalView] waiting on renderThread to finish");
-        while (!_renderThread.isFinished) {
+        NSDate *timeout = [NSDate dateWithTimeIntervalSinceNow:2.0]; // 2 second timeout
-            usleep(100);
+        while (!_renderThread.isFinished && [timeout timeIntervalSinceNow] > 0) {
+            usleep(1000);
+        }
+        
+        if (_renderThread.isFinished) {
+            Log(LOG_I, @"[MetalView] renderThread has finished");
+        } else {
+            Log(LOG_E, @"[MetalView] renderThread failed to finish in time");
         }
-        Log(LOG_I, @"[MetalView] renderThread has finished");
         _renderThread = nil;
     }
 }
@@ -54,6 +85,28 @@
     [self movedToWindow];
 }
 
+// CAMetalDisplayLinkDelegate method (iOS 17+)
+- (void)metalDisplayLink:(CAMetalDisplayLink *)link needsUpdate:(CAMetalDisplayLinkUpdate *)update API_AVAILABLE(ios(17.0)) {
+    // Skip rendering if we're shutting down
+    if ([NSThread currentThread].isCancelled) {
+        return;
+    }
+    @autoreleasepool {
+        [self.delegate waitToRenderTo:self.metalLayer];
+        
+        // Get the drawable from the update object
+        id<CAMetalDrawable> drawable = update.drawable;
+        if (drawable) {
+            // Pass timing information to the renderer
+            if ([self.delegate respondsToSelector:@selector(renderWithDrawable:toLayer:targetPresentationTimestamp:)]) {
+                [self.delegate renderWithDrawable:drawable toLayer:self.metalLayer targetPresentationTimestamp:update.targetPresentationTimestamp];
+            } else {
+                [self.delegate renderWithDrawable:drawable toLayer:self.metalLayer];
+            }
+        }
+    }
+}
+
 - (void)movedToWindow {
     if (!self.window) {
         Log(LOG_I, @"[MetalView] movedToWindow(nil): shutting down...");
@@ -61,12 +114,33 @@
         return;
     }
 
-    // Render on a new thread
+    // Create CAMetalDisplayLink for vsync-synchronized rendering (iOS 17+ only)
+    if (@available(iOS 17.0, *)) {
+        _metalDisplayLink = [[CAMetalDisplayLink alloc] initWithMetalLayer:_metalLayer];
+        _metalDisplayLink.delegate = self;
+        _metalDisplayLink.preferredFrameRateRange = CAFrameRateRangeMake(_framerate, _framerate, _framerate);
+        Log(LOG_I, @"[MetalView] Using CAMetalDisplayLink for optimal Metal rendering");
+    }
+    
+    // Start the display link on a background thread
     _renderThread = [[NSThread alloc] initWithBlock:^{
-        while (![NSThread currentThread].isCancelled) {
+        // Add metal display link to this thread's run loop
+        if (@available(iOS 17.0, *)) {
+            [self->_metalDisplayLink addToRunLoop:[NSRunLoop currentRunLoop] forMode:NSDefaultRunLoopMode];
+        }
+        
+        // Keep the run loop alive, checking for cancellation regularly
+        while (![NSThread currentThread].isCancelled && self->_metalDisplayLink) {
             @autoreleasepool {
-                [self.delegate waitToRenderTo:self.metalLayer];
+                // Run the run loop for a short time to allow checking cancellation
-                [self.delegate renderTo:self.metalLayer];
+                [[NSRunLoop currentRunLoop] runMode:NSDefaultRunLoopMode beforeDate:[NSDate dateWithTimeIntervalSinceNow:0.1]];
+            }
+        }
+        
+        // Remove from run loop if still valid
+        if (@available(iOS 17.0, *)) {
+            if (self->_metalDisplayLink) {
+                [self->_metalDisplayLink removeFromRunLoop:[NSRunLoop currentRunLoop] forMode:NSDefaultRunLoopMode];
             }
         }
         Log(LOG_I, @"[MetalView] renderThread is exiting");
@@ -74,7 +148,7 @@
     _renderThread.name = @"MetalVideoRenderer";
     _renderThread.qualityOfService = NSQualityOfServiceUserInteractive;
     [_renderThread start];
-    Log(LOG_I, @"[MetalView] started renderThread %@", _renderThread);
+    Log(LOG_I, @"[MetalView] started renderThread with CAMetalDisplayLink at %f fps", _framerate);
 
     // Perform any actions that need to know the size and scale of the drawable. When UIKit calls
     // didMoveToWindow after the view initialization, this is the first opportunity to notify

VoidLink/Metal/MetalViewController.m

@@ -81,13 +81,25 @@ The implementation of the cross-platform game view controller.
     [_frameQueue waitForEnqueue];
 }
 
-/// Draw frame (used by manual loop)
+
-- (void)renderTo:(nonnull CAMetalLayer *)layer {
+// New method for CAMetalDisplayLink that provides the drawable
+- (void)renderWithDrawable:(nonnull id<CAMetalDrawable>)drawable toLayer:(nonnull CAMetalLayer *)layer API_AVAILABLE(ios(17.0)) {
     if (!_renderer.isStopping) {
         CFTimeInterval timeout = (1.0f / _framerate) - _renderer.averageGPUTime;
         Frame *frame = [_frameQueue dequeueWithTimeout:timeout];
         if (frame) {
-            [_renderer renderFrame:frame toLayer:layer];
+            [_renderer renderFrame:frame withDrawable:drawable];
+        }
+    }
+}
+
+// Enhanced method that receives timing information from CAMetalDisplayLink
+- (void)renderWithDrawable:(nonnull id<CAMetalDrawable>)drawable toLayer:(nonnull CAMetalLayer *)layer targetPresentationTimestamp:(CFTimeInterval)targetPresentationTimestamp API_AVAILABLE(ios(17.0)) {
+    if (!_renderer.isStopping) {
+        CFTimeInterval timeout = (1.0f / _framerate) - _renderer.averageGPUTime;
+        Frame *frame = [_frameQueue dequeueWithTimeout:timeout];
+        if (frame) {
+            [_renderer renderFrame:frame withDrawable:drawable targetPresentationTimestamp:targetPresentationTimestamp];
         }
     }
 }
@@ -101,8 +113,14 @@ The implementation of the cross-platform game view controller.
 
     Log(LOG_I, @"[MetalViewController] viewDidDisappear");
 
+    // Shutdown the renderer first
     [_renderer shutdown];
     _renderer = nil;
+    
+    // Then shutdown the view's display link
+    if (_metalView) {
+        [_metalView shutdown];
+    }
 }
 
 #if TARGET_OS_IOS

VoidLink/ViewControllers/SettingsViewController.m

@@ -535,7 +535,11 @@ BOOL isCustomResolution(int resolutionSelected) {
     [self addSetting:self.pipStack ofId:@"pipStack" withInfoTag:YES withDynamicLabel:NO to:videoSection];
     [self addSetting:self.pipStack ofId:@"pipStack" withInfoTag:YES withDynamicLabel:NO to:videoSection];
     [self addSetting:self.frameQueueSizeStack ofId:@"frameQueueSizeStack" withInfoTag:NO withDynamicLabel:NO to:videoSection];
-    [self addSetting:self.renderingBackendStack ofId:@"renderingBackendStack" withInfoTag:YES withDynamicLabel:NO to:videoSection];
+    
+    // Only show Metal renderer option on iOS 17+ where CAMetalDisplayLink is available
+    if (@available(iOS 17.0, *)) {
+        [self addSetting:self.renderingBackendStack ofId:@"renderingBackendStack" withInfoTag:YES withDynamicLabel:NO to:videoSection];
+    }
 
     [videoSection addToParentStack:_parentStack];
     [videoSection setExpanded:YES];