From ee67ff56c78d3ba72d9a2d80824080bbfe3492ec Mon Sep 17 00:00:00 2001 From: Acaki Date: Mon, 18 Aug 2025 19:27:52 +0800 Subject: [PATCH] feat: add proper CADisplayLink handling for iOS15 where commandBuffer presentDrawable:drawable afterMinimumDuration doesn't seem to work well --- VoidLink/Metal/MetalVideoRenderer.h | 9 +++ VoidLink/Metal/MetalVideoRenderer.m | 65 +++++++++++++++++-- VoidLink/Metal/MetalViewController.h | 3 + VoidLink/Metal/MetalViewController.m | 52 +++++++++++++-- .../StreamFrameViewController.m | 6 ++ 5 files changed, 126 insertions(+), 9 deletions(-) diff --git a/VoidLink/Metal/MetalVideoRenderer.h b/VoidLink/Metal/MetalVideoRenderer.h index 1c35c786..19e6f4d1 100644 --- a/VoidLink/Metal/MetalVideoRenderer.h +++ b/VoidLink/Metal/MetalVideoRenderer.h @@ -22,12 +22,21 @@ @property (nonatomic) id _Nullable nextDrawable; @property (atomic) BOOL isStopping; +// iOS 15.0 and below frame timing properties +@property (nonatomic) BOOL useLegacyFrameTiming; +@property (nonatomic) CFTimeInterval lastDisplayLinkTime; +@property (nonatomic) CFTimeInterval displayLinkFrametime; + - (nonnull instancetype)initWithMetalDevice:(nonnull id)device drawablePixelFormat:(MTLPixelFormat)drawablePixelFormat framerate:(float)framerate; - (void)renderFrame:(nonnull Frame *)frame toLayer:(nonnull CAMetalLayer *)layer API_AVAILABLE(ios(13.0)); - (void)waitToRenderTo:(nonnull CAMetalLayer *)layer API_AVAILABLE(ios(13.0)); - (void)drawableResize:(CGSize)drawableSize; - (void)shutdown; +// iOS 15.0 and below legacy timing support +- (void)updateLegacyFrameTiming:(CFTimeInterval)displayLinkTimestamp; +- (void)resetFrameTiming; + + (NSString *_Nullable)currentColorSpace; @end diff --git a/VoidLink/Metal/MetalVideoRenderer.m b/VoidLink/Metal/MetalVideoRenderer.m index d2347014..4787c50a 100644 --- a/VoidLink/Metal/MetalVideoRenderer.m +++ b/VoidLink/Metal/MetalVideoRenderer.m @@ -157,6 +157,17 @@ CFStringRef __currentColorSpace; _lastPresented = 0.0f; _inFlightSemaphore = dispatch_semaphore_create(MaxFramesInFlight); _isStopping = NO; + + // Enable legacy frame timing for iOS 15.0 and below to fix timing issues + if (@available(iOS 16.0, *)) { + _useLegacyFrameTiming = NO; + Log(LOG_I, @"MetalVideoRenderer: Using iOS 16+ timing (Metal presentedTime)"); + } else { + _useLegacyFrameTiming = YES; + _lastDisplayLinkTime = 0.0f; + _displayLinkFrametime = 1.0f / framerate * 1000.0f; // Expected frametime in ms + Log(LOG_I, @"MetalVideoRenderer: Using legacy timing for iOS 15.0 and below (CADisplayLink)"); + } CFStringRef keys[1] = {kCVMetalTextureUsage}; NSUInteger values[1] = {MTLTextureUsageShaderRead}; @@ -713,14 +724,21 @@ CFStringRef __currentColorSpace; [renderEncoder drawPrimitives:MTLPrimitiveTypeTriangleStrip vertexStart:0 vertexCount:4]; [renderEncoder endEncoding]; - __block MetalVideoRenderer *strongSelf = self; #if !TARGET_OS_SIMULATOR + __block MetalVideoRenderer *strongSelf = self; [drawable addPresentedHandler:^(id d) { - if (strongSelf.lastPresented > 0.0f) { - CFTimeInterval frametime = d.presentedTime - strongSelf.lastPresented; - [[ImGuiPlots sharedInstance] observeFloat:PLOT_FRAMETIME value:(frametime * 1000.0)]; + // Only use Metal's presentedTime on iOS 16+ due to timing bugs on iOS 15.0 + if (!strongSelf.useLegacyFrameTiming) { + if (strongSelf.lastPresented > 0.0f) { + CFTimeInterval frametime = d.presentedTime - strongSelf.lastPresented; + // Sanity check: reject negative or impossibly large frametimes + if (frametime > 0 && frametime < 0.1) { // Less than 100ms + [[ImGuiPlots sharedInstance] observeFloat:PLOT_FRAMETIME value:(frametime * 1000.0)]; + } + } + strongSelf.lastPresented = d.presentedTime; } - strongSelf.lastPresented = d.presentedTime; + // For iOS 15.0 and below, frametime is calculated in updateLegacyFrameTiming }]; #endif @@ -777,6 +795,43 @@ CFStringRef __currentColorSpace; } } +#pragma mark - iOS 15.0 and below legacy timing support + +- (void)updateLegacyFrameTiming:(CFTimeInterval)displayLinkTimestamp { + if (!_useLegacyFrameTiming) { + return; // Only for iOS 15.0 and below + } + + if (_lastDisplayLinkTime > 0.0f) { + CFTimeInterval actualFrametime = (displayLinkTimestamp - _lastDisplayLinkTime) * 1000.0; // Convert to ms + + // Sanity check: reject negative or impossibly large frametimes + if (actualFrametime > 0 && actualFrametime < 100.0) { // Less than 100ms + [[ImGuiPlots sharedInstance] observeFloat:PLOT_FRAMETIME value:actualFrametime]; + _displayLinkFrametime = actualFrametime; + } else { + // Use expected frametime if actual is unreliable + Log(LOG_W, @"MetalVideoRenderer: Rejecting invalid frametime %.2f ms, using expected %.2f ms", actualFrametime, _displayLinkFrametime); + [[ImGuiPlots sharedInstance] observeFloat:PLOT_FRAMETIME value:_displayLinkFrametime]; + } + } + + _lastDisplayLinkTime = displayLinkTimestamp; +} + +- (void)resetFrameTiming { + if (_useLegacyFrameTiming) { + // Reset legacy timing state for iOS 15.0 and below + _lastDisplayLinkTime = 0.0f; + _displayLinkFrametime = 1.0f / _framerate * 1000.0f; + Log(LOG_I, @"MetalVideoRenderer: Reset legacy frame timing for iOS 15.0 and below"); + } else { + // Reset Metal timing state for iOS 16+ + _lastPresented = 0.0f; + Log(LOG_I, @"MetalVideoRenderer: Reset Metal frame timing for iOS 16+"); + } +} + /// Responds to the drawable's size or orientation changes. - (void)drawableResize:(CGSize)drawableSize { [self resize:drawableSize]; diff --git a/VoidLink/Metal/MetalViewController.h b/VoidLink/Metal/MetalViewController.h index 29e51cdc..f06ebea3 100644 --- a/VoidLink/Metal/MetalViewController.h +++ b/VoidLink/Metal/MetalViewController.h @@ -22,4 +22,7 @@ enableHdr:(BOOL)enableHdr metricsHandler:(MetricsHandler _Nonnull)metricsHandler; +// Reset frame timing for iOS 15.0 and below compatibility +- (void)resetFrameTiming; + @end diff --git a/VoidLink/Metal/MetalViewController.m b/VoidLink/Metal/MetalViewController.m index e5e6ffb4..73ce2983 100644 --- a/VoidLink/Metal/MetalViewController.m +++ b/VoidLink/Metal/MetalViewController.m @@ -92,19 +92,35 @@ view.metalLayer.pixelFormat = renderer.colorPixelFormat; view.metalLayer.maximumDrawableCount = 3; - // We need a no-op displaylink timer or iOS can decide to run at 60fps - // The overhead from this should be minimal. + // We need a displaylink timer for iOS 15.0 and below to provide accurate frame timing, + // or just as a no-op for iOS 16+ to prevent iOS from dropping to 60fps _displayLink = [CADisplayLink displayLinkWithTarget:self selector:@selector(displayLinkHandler:)]; - if (@available(iOS 15.0, tvOS 15.0, *)) { + if (@available(iOS 16.0, tvOS 16.0, *)) { + // iOS 16+: Use CAFrameRateRange, displaylink is just a no-op _displayLink.preferredFrameRateRange = CAFrameRateRangeMake(_framerate, _framerate, _framerate); + Log(LOG_I, @"MetalViewController: DisplayLink using iOS 16+ CAFrameRateRange"); + } else if (@available(iOS 15.0, tvOS 15.0, *)) { + // iOS 15.0: Use legacy preferredFramesPerSecond to avoid CAFrameRateRange bugs + _displayLink.preferredFramesPerSecond = _framerate; + Log(LOG_I, @"MetalViewController: DisplayLink using iOS 15.0 legacy preferredFramesPerSecond"); } else { + // iOS 14 and below _displayLink.preferredFramesPerSecond = _framerate; } [_displayLink addToRunLoop:[NSRunLoop currentRunLoop] forMode:NSRunLoopCommonModes]; } - (void)displayLinkHandler:(CADisplayLink *)link { - // Rendering does not use DisplayLink, this exists to fool iOS into keeping us running at the desired framerate + // For iOS 16+: This is just a no-op to fool iOS into keeping us running at the desired framerate + // For iOS 15.0 and below: Update renderer with accurate frame timing to fix negative framerates + if (@available(iOS 16.0, *)) { + // No-op for iOS 16+, Metal's presentedTime is reliable + } else { + // iOS 15.0 and below: Use DisplayLink timing since Metal's presentedTime is unreliable + if (_renderer) { + [_renderer updateLegacyFrameTiming:link.timestamp]; + } + } } - (void)waitToRenderTo:(nonnull CAMetalLayer *)layer { @@ -154,4 +170,32 @@ } #endif +- (void)resetFrameTiming { + // Reset the renderer's frame timing state for iOS 15.0 and below compatibility + if (_renderer) { + [_renderer resetFrameTiming]; + Log(LOG_I, @"MetalViewController: Reset frame timing called"); + } + + // Reset DisplayLink timing for iOS 15.0 and below + if (@available(iOS 16.0, *)) { + // No additional DisplayLink reset needed for iOS 16+ + } else { + // For iOS 15.0 and below, pause and resume DisplayLink to re-engage timing + if (_displayLink) { + _displayLink.paused = YES; + + // Re-apply frame rate (avoid CAFrameRateRange on iOS 15.0) + if (@available(iOS 15.0, *)) { + _displayLink.preferredFramesPerSecond = _framerate; + } else { + _displayLink.preferredFramesPerSecond = _framerate; + } + + _displayLink.paused = NO; + Log(LOG_I, @"MetalViewController: Reset DisplayLink timing for iOS 15.0 and below"); + } + } +} + @end diff --git a/VoidLink/ViewControllers/StreamFrameViewController.m b/VoidLink/ViewControllers/StreamFrameViewController.m index 3d46e3f0..cf9138f4 100644 --- a/VoidLink/ViewControllers/StreamFrameViewController.m +++ b/VoidLink/ViewControllers/StreamFrameViewController.m @@ -1043,6 +1043,12 @@ } [self->_streamMan.videoRenderer resetFramePacing]; + + // Also reset Metal renderer timing for iOS 15.0 and below compatibility + if (self.metalViewController) { + [self.metalViewController resetFrameTiming]; + } + _isRestoringFromPiP = NO; }