diff --git a/VoidLink/Database/TemporarySettings.m b/VoidLink/Database/TemporarySettings.m index ddf234e4..160549ac 100644 --- a/VoidLink/Database/TemporarySettings.m +++ b/VoidLink/Database/TemporarySettings.m @@ -52,7 +52,15 @@ self.statsOverlay = [[NSUserDefaults standardUserDefaults] boolForKey:@"statsOverlay"]; self.enableGraphs = [[NSUserDefaults standardUserDefaults] boolForKey:@"enableGraphs"]; self.graphOpacity = [NSNumber numberWithInteger:[[NSUserDefaults standardUserDefaults] integerForKey:@"graphOpacity"]]; - self.renderingBackend = [NSNumber numberWithInteger:[[NSUserDefaults standardUserDefaults] integerForKey:@"renderingBackend"]]; + // Set rendering backend based on iOS version and user preference + NSInteger savedBackend = [[NSUserDefaults standardUserDefaults] integerForKey:@"renderingBackend"]; + if (@available(iOS 17.0, *)) { + // iOS 17+ can use Performance mode (Metal renderer) + self.renderingBackend = [NSNumber numberWithInteger:savedBackend]; + } else { + // iOS < 17 must use Balanced mode (AVSB renderer) + self.renderingBackend = [NSNumber numberWithInteger:RENDER_AVSB]; + } NSInteger _screenSize = [[NSUserDefaults standardUserDefaults] integerForKey:@"streamResolution"]; switch (_screenSize) { diff --git a/VoidLink/ImGuiRenderer.h b/VoidLink/ImGuiRenderer.h index ac5cc620..d65094a5 100644 --- a/VoidLink/ImGuiRenderer.h +++ b/VoidLink/ImGuiRenderer.h @@ -28,6 +28,7 @@ typedef void (^MetricsHandler)(int plotId, CFTimeInterval value); @property (nonatomic) BOOL enableGraphs; @property (nonatomic) float graphOpacity; @property (nonatomic) BOOL imguiRunning; +@property (nonatomic) int streamFps; @property (nonatomic) MetricsHandler _Nonnull metricsHandler; -(nonnull instancetype) initWithFrame:(CGRect)bounds diff --git a/VoidLink/ImGuiRenderer.mm b/VoidLink/ImGuiRenderer.mm index a1eba0a2..a8cc0dee 100644 --- a/VoidLink/ImGuiRenderer.mm +++ b/VoidLink/ImGuiRenderer.mm @@ -42,6 +42,9 @@ _plots = [ImGuiPlots sharedInstance].plots; + // Store stream FPS to synchronize MTKView + _streamFps = streamFps; + return self; } @@ -90,7 +93,7 @@ self.mtkView.device = self.device; self.mtkView.delegate = self; - self.mtkView.preferredFramesPerSecond = 60; // ImGui overlay will always render at this rate + self.mtkView.preferredFramesPerSecond = _streamFps; // Synchronize with stream framerate self.mtkView.opaque = NO; self.mtkView.enableSetNeedsDisplay = NO; @@ -179,7 +182,7 @@ #if TARGET_OS_SIMULATOR [commandBuffer presentDrawable:view.currentDrawable]; #else - [commandBuffer presentDrawable:view.currentDrawable afterMinimumDuration:1.0 / view.preferredFramesPerSecond]; + [commandBuffer presentDrawable:view.currentDrawable afterMinimumDuration:1.0 / _streamFps]; #endif [commandBuffer commit]; #endif diff --git a/VoidLink/Metal/MetalVideoRenderer.h b/VoidLink/Metal/MetalVideoRenderer.h index 954374bc..9392480f 100644 --- a/VoidLink/Metal/MetalVideoRenderer.h +++ b/VoidLink/Metal/MetalVideoRenderer.h @@ -24,8 +24,9 @@ @property (nonatomic) BOOL hdrEnabled; - (nonnull instancetype)initWithMetalDevice:(nonnull id)device drawablePixelFormat:(MTLPixelFormat)drawablePixelFormat framerate:(float)framerate hdrEnabled:(BOOL)hdrEnabled; -- (void)renderFrame:(nonnull Frame *)frame toLayer:(nonnull CAMetalLayer *)layer API_AVAILABLE(ios(13.0)); -- (void)waitToRenderTo:(nonnull CAMetalLayer *)layer API_AVAILABLE(ios(13.0)); +- (void)renderFrame:(nonnull Frame *)frame withDrawable:(nonnull id)drawable API_AVAILABLE(ios(17.0)); +- (void)renderFrame:(nonnull Frame *)frame withDrawable:(nonnull id)drawable targetPresentationTimestamp:(CFTimeInterval)targetPresentationTimestamp API_AVAILABLE(ios(17.0)); +- (void)waitToRenderTo:(nonnull CAMetalLayer *)layer; - (void)drawableResize:(CGSize)drawableSize; - (void)shutdown; diff --git a/VoidLink/Metal/MetalVideoRenderer.m b/VoidLink/Metal/MetalVideoRenderer.m index b860f88c..01603355 100644 --- a/VoidLink/Metal/MetalVideoRenderer.m +++ b/VoidLink/Metal/MetalVideoRenderer.m @@ -557,21 +557,29 @@ CFStringRef __currentColorSpace; return YES; } -- (void)renderFrame:(Frame *)frame toLayer:(CAMetalLayer *)layer { + +- (void)renderFrame:(Frame *)frame withDrawable:(id)drawable targetPresentationTimestamp:(CFTimeInterval)targetPresentationTimestamp API_AVAILABLE(ios(17.0)) { @autoreleasepool { if (self.isStopping) { Log(LOG_I, @"[MetalVideoRenderer] isStopping"); return; } + // Calculate accurate frametime using CAMetalDisplayLink timing + if (self.lastPresented > 0.0f) { + CFTimeInterval frametime = targetPresentationTimestamp - self.lastPresented; + [[ImGuiPlots sharedInstance] observeFloat:PLOT_FRAMETIME value:(frametime * 1000.0)]; + } + self.lastPresented = targetPresentationTimestamp; + // Handle changes to the frame's colorspace from last time we rendered BOOL layerDidChange = NO; - if (![self updateColorSpaceForFrame:frame toLayer:layer layerDidChange:&layerDidChange]) { + if (![self updateColorSpaceForFrame:frame toLayer:(CAMetalLayer *)drawable.layer layerDidChange:&layerDidChange]) { return; } // Handle changes to the video size or drawable size - if (![self updateVideoRegionSizeForFrame:frame toLayer:layer]) { + if (![self updateVideoRegionSizeForFrame:frame toLayer:(CAMetalLayer *)drawable.layer]) { return; } @@ -596,13 +604,6 @@ CFStringRef __currentColorSpace; } } - // Get the next drawable early to get its pixel format - id drawable = [layer nextDrawable]; - if (!drawable) { - Log(LOG_E, @"Failed to get nextDrawable"); - return; - } - // Get the framebuffer pixel format for pipeline creation MTLPixelFormat framebufferPixelFormat = drawable.texture.pixelFormat; @@ -709,7 +710,7 @@ CFStringRef __currentColorSpace; [renderEncoder setVertexBuffer:_VideoVertexBuffer offset:0 atIndex:0]; [renderEncoder setFragmentBuffer:_CscParamsBuffer offset:0 atIndex:0]; #if !TARGET_OS_TV - if (layer.pixelFormat == MTLPixelFormatRGBA16Float) { + if (((CAMetalLayer *)drawable.layer).pixelFormat == MTLPixelFormatRGBA16Float) { [self pollCurrentEDRHeadroom]; [renderEncoder setFragmentBytes:&_currentEDRHeadroom length:sizeof(float) atIndex:1]; } @@ -717,25 +718,19 @@ CFStringRef __currentColorSpace; [renderEncoder drawPrimitives:MTLPrimitiveTypeTriangleStrip vertexStart:0 vertexCount:4]; [renderEncoder endEncoding]; - __block MetalVideoRenderer *strongSelf = self; -#if !TARGET_OS_SIMULATOR - [drawable addPresentedHandler:^(id d) { - if (strongSelf.lastPresented > 0.0f) { - CFTimeInterval frametime = d.presentedTime - strongSelf.lastPresented; - [[ImGuiPlots sharedInstance] observeFloat:PLOT_FRAMETIME value:(frametime * 1000.0)]; - } - strongSelf.lastPresented = d.presentedTime; - }]; -#endif - // signal semaphore, compute GPU time average, and clear textures __block dispatch_semaphore_t block_semaphore = _inFlightSemaphore; [commandBuffer addCompletedHandler:^(id cb) { dispatch_semaphore_signal(block_semaphore); - const CFTimeInterval GPUTime = cb.GPUEndTime - cb.GPUStartTime; - const double alpha = 0.25f; - self->_averageGPUTime = (GPUTime * alpha) + (self->_averageGPUTime * (1.0 - alpha)); + if (cb.error) { + // Request IDR frame to recover from rendering error + LiRequestIdrFrame(); + } else { + const CFTimeInterval GPUTime = cb.GPUEndTime - cb.GPUStartTime; + const double alpha = 0.25f; + self->_averageGPUTime = (GPUTime * alpha) + (self->_averageGPUTime * (1.0 - alpha)); + } // Free textures after completion of rendering for (size_t i = 0; i < planes; i++) { @@ -748,15 +743,10 @@ CFStringRef __currentColorSpace; CVMetalTextureCacheFlush(self->_textureCache, 0); }]; -#if TARGET_OS_SIMULATOR + // CAMetalDisplayLink handles timing automatically, so we don't use afterMinimumDuration [commandBuffer presentDrawable:drawable]; -#else - // present for a minimum duration for best frame pacing - [commandBuffer presentDrawable:drawable afterMinimumDuration:1.0f / _framerate]; -#endif [commandBuffer commit]; - [commandBuffer waitUntilCompleted]; } } @@ -765,7 +755,11 @@ CFStringRef __currentColorSpace; // by any stage in the Metal pipeline (CPU, GPU, Metal, Drivers, etc.). if (!self.isStopping) { dispatch_time_t timeout = dispatch_time(DISPATCH_TIME_NOW, (int64_t)(0.1f * NSEC_PER_SEC)); // 100ms - dispatch_semaphore_wait(_inFlightSemaphore, timeout); + long result = dispatch_semaphore_wait(_inFlightSemaphore, timeout); + if (result != 0) { + Log(LOG_W, @"MetalVideoRenderer semaphore timeout - requesting IDR frame"); + LiRequestIdrFrame(); // Request recovery + } } } diff --git a/VoidLink/Metal/MetalView.h b/VoidLink/Metal/MetalView.h index 4564f51d..304be491 100644 --- a/VoidLink/Metal/MetalView.h +++ b/VoidLink/Metal/MetalView.h @@ -15,13 +15,14 @@ @protocol MetalViewDelegate - (void)drawableResize:(CGSize)size; -- (void)renderTo:(nonnull CAMetalLayer *)layer; +- (void)renderWithDrawable:(nonnull id)drawable toLayer:(nonnull CAMetalLayer *)layer API_AVAILABLE(ios(17.0)); +- (void)renderWithDrawable:(nonnull id)drawable toLayer:(nonnull CAMetalLayer *)layer targetPresentationTimestamp:(CFTimeInterval)targetPresentationTimestamp API_AVAILABLE(ios(17.0)); - (void)waitToRenderTo:(nonnull CAMetalLayer *)layer; @end // The Metal game view base class. -@interface MetalView : UIView +@interface MetalView : UIView @property (nonatomic, nonnull, readonly) CAMetalLayer *metalLayer; @property (nonatomic, nullable) id delegate; diff --git a/VoidLink/Metal/MetalView.m b/VoidLink/Metal/MetalView.m index ca967738..3835be44 100644 --- a/VoidLink/Metal/MetalView.m +++ b/VoidLink/Metal/MetalView.m @@ -11,10 +11,12 @@ #import "MetalView.h" #import "MetalConfig.h" +#import @implementation MetalView { // The secondary thread containing the render loop. NSThread *_renderThread; + CAMetalDisplayLink *_metalDisplayLink API_AVAILABLE(ios(17.0)); } #pragma mark - Initialization and Setup. @@ -40,15 +42,43 @@ self.layer.delegate = self; } +- (void)setFramerate:(float)framerate { + _framerate = framerate; + + if (@available(iOS 17.0, *)) { + if (_metalDisplayLink) { + _metalDisplayLink.preferredFrameRateRange = CAFrameRateRangeMake(framerate, framerate, framerate); + } + } +} + - (void)shutdown { + // First cancel the thread to stop the run loop if (_renderThread) { Log(LOG_I, @"[MetalView] sending renderThread a cancel message"); [_renderThread cancel]; + + // Invalidate metal display link to stop callbacks + if (@available(iOS 17.0, *)) { + if (_metalDisplayLink) { + Log(LOG_I, @"[MetalView] invalidating metal display link"); + [_metalDisplayLink invalidate]; + _metalDisplayLink = nil; + } + } + + // Now wait for thread to finish Log(LOG_I, @"[MetalView] waiting on renderThread to finish"); - while (!_renderThread.isFinished) { - usleep(100); + NSDate *timeout = [NSDate dateWithTimeIntervalSinceNow:2.0]; // 2 second timeout + while (!_renderThread.isFinished && [timeout timeIntervalSinceNow] > 0) { + usleep(1000); + } + + if (_renderThread.isFinished) { + Log(LOG_I, @"[MetalView] renderThread has finished"); + } else { + Log(LOG_E, @"[MetalView] renderThread failed to finish in time"); } - Log(LOG_I, @"[MetalView] renderThread has finished"); _renderThread = nil; } } @@ -61,6 +91,26 @@ [self movedToWindow]; } +- (void)metalDisplayLink:(CAMetalDisplayLink *)link needsUpdate:(CAMetalDisplayLinkUpdate *)update API_AVAILABLE(ios(17.0)) { + // Skip rendering if we're shutting down + if ([NSThread currentThread].isCancelled) { + return; + } + @autoreleasepool { + [self.delegate waitToRenderTo:self.metalLayer]; + + id drawable = update.drawable; + if (drawable) { + // Pass timing information to the renderer + if ([self.delegate respondsToSelector:@selector(renderWithDrawable:toLayer:targetPresentationTimestamp:)]) { + [self.delegate renderWithDrawable:drawable toLayer:self.metalLayer targetPresentationTimestamp:update.targetPresentationTimestamp]; + } else { + [self.delegate renderWithDrawable:drawable toLayer:self.metalLayer]; + } + } + } +} + - (void)movedToWindow { if (!self.window) { Log(LOG_I, @"[MetalView] movedToWindow(nil): shutting down..."); @@ -68,12 +118,32 @@ return; } - // Render on a new thread + if (@available(iOS 17.0, *)) { + _metalDisplayLink = [[CAMetalDisplayLink alloc] initWithMetalLayer:_metalLayer]; + _metalDisplayLink.delegate = self; + _metalDisplayLink.preferredFrameRateRange = CAFrameRateRangeMake(_framerate, _framerate, _framerate); + Log(LOG_I, @"[MetalView] Using CAMetalDisplayLink for optimal Metal rendering"); + } + + // Start the display link on a background thread _renderThread = [[NSThread alloc] initWithBlock:^{ - while (![NSThread currentThread].isCancelled) { + // Add metal display link to this thread's run loop + if (@available(iOS 17.0, *)) { + [self->_metalDisplayLink addToRunLoop:[NSRunLoop currentRunLoop] forMode:NSDefaultRunLoopMode]; + } + + // Keep the run loop alive, checking for cancellation regularly + while (![NSThread currentThread].isCancelled && self->_metalDisplayLink) { @autoreleasepool { - [self.delegate waitToRenderTo:self.metalLayer]; - [self.delegate renderTo:self.metalLayer]; + // Run the run loop for a short time to allow checking cancellation + [[NSRunLoop currentRunLoop] runMode:NSDefaultRunLoopMode beforeDate:[NSDate dateWithTimeIntervalSinceNow:0.1]]; + } + } + + // Remove from run loop if still valid + if (@available(iOS 17.0, *)) { + if (self->_metalDisplayLink) { + [self->_metalDisplayLink removeFromRunLoop:[NSRunLoop currentRunLoop] forMode:NSDefaultRunLoopMode]; } } Log(LOG_I, @"[MetalView] renderThread is exiting"); @@ -81,7 +151,7 @@ _renderThread.name = @"MetalVideoRenderer"; _renderThread.qualityOfService = NSQualityOfServiceUserInteractive; [_renderThread start]; - Log(LOG_I, @"[MetalView] started renderThread %@", _renderThread); + Log(LOG_I, @"[MetalView] started renderThread with CAMetalDisplayLink at %f fps", _framerate); // Perform any actions that need to know the size and scale of the drawable. When UIKit calls // didMoveToWindow after the view initialization, this is the first opportunity to notify diff --git a/VoidLink/Metal/MetalViewController.m b/VoidLink/Metal/MetalViewController.m index 46868bcd..96d4342d 100644 --- a/VoidLink/Metal/MetalViewController.m +++ b/VoidLink/Metal/MetalViewController.m @@ -18,7 +18,6 @@ MetalView *_metalView; MetalVideoRenderer *_renderer; MetricsHandler _metricsHandler; - CADisplayLink *_displayLink; } - (nonnull instancetype)initWithFrame:(CGRect)bounds framerate:(float)framerate enableHdr:(BOOL)enableHdr metricsHandler:(MetricsHandler)metricsHandler { @@ -74,20 +73,6 @@ // Initialize the renderer-dependent view properties. view.metalLayer.pixelFormat = renderer.colorPixelFormat; view.metalLayer.maximumDrawableCount = 3; - - // We need a no-op displaylink timer or iOS can decide to run at 60fps - // The overhead from this should be minimal. - _displayLink = [CADisplayLink displayLinkWithTarget:self selector:@selector(displayLinkHandler:)]; - if (@available(iOS 15.0, tvOS 15.0, *)) { - _displayLink.preferredFrameRateRange = CAFrameRateRangeMake(_framerate, _framerate, _framerate); - } else { - _displayLink.preferredFramesPerSecond = _framerate; - } - [_displayLink addToRunLoop:[NSRunLoop currentRunLoop] forMode:NSRunLoopCommonModes]; -} - -- (void)displayLinkHandler:(CADisplayLink *)link { - // Rendering does not use DisplayLink, this exists to fool iOS into keeping us running at the desired framerate } - (void)waitToRenderTo:(nonnull CAMetalLayer *)layer { @@ -105,17 +90,23 @@ [_frameQueue waitForEnqueue]; } -/// Draw frame (used by manual loop) -- (void)renderTo:(nonnull CAMetalLayer *)layer { +- (void)renderWithDrawable:(nonnull id)drawable toLayer:(nonnull CAMetalLayer *)layer API_AVAILABLE(ios(17.0)) { CFTimeInterval timeout = (1.0f / _framerate) - _renderer.averageGPUTime; Frame *frame = [_frameQueue dequeueWithTimeout:timeout]; - if (!_renderer.isStopping) { // Only render if not paused if (frame) { - if (@available(iOS 13.0, *)) { - [_renderer renderFrame:frame toLayer:layer]; - } + [_renderer renderFrame:frame withDrawable:drawable]; + } + } +} + +- (void)renderWithDrawable:(nonnull id)drawable toLayer:(nonnull CAMetalLayer *)layer targetPresentationTimestamp:(CFTimeInterval)targetPresentationTimestamp API_AVAILABLE(ios(17.0)) { + if (!_renderer.isStopping) { + CFTimeInterval timeout = (1.0f / _framerate) - _renderer.averageGPUTime; + Frame *frame = [_frameQueue dequeueWithTimeout:timeout]; + if (frame) { + [_renderer renderFrame:frame withDrawable:drawable targetPresentationTimestamp:targetPresentationTimestamp]; } } else { // When paused, we still dequeue frames to prevent accumulation @@ -129,9 +120,6 @@ } - (void)pauseRendering { - if (_displayLink) { - _displayLink.paused = YES; - } if (_renderer) { _renderer.isStopping = YES; } @@ -142,9 +130,6 @@ if (_renderer) { _renderer.isStopping = NO; } - if (_displayLink) { - _displayLink.paused = NO; - } Log(LOG_I, @"[MetalViewController] Rendering resumed"); } @@ -153,16 +138,13 @@ Log(LOG_I, @"[MetalViewController] viewDidDisappear"); - if (_displayLink) { - [_displayLink invalidate]; - _displayLink = nil; - } - + // Shutdown the renderer first if (_renderer) { [_renderer shutdown]; _renderer = nil; } + // Then shutdown the view's display link if (_metalView) { _metalView.delegate = nil; [_metalView shutdown]; diff --git a/VoidLink/ViewControllers/SettingsViewController.m b/VoidLink/ViewControllers/SettingsViewController.m index 1ef044bc..9262ceef 100644 --- a/VoidLink/ViewControllers/SettingsViewController.m +++ b/VoidLink/ViewControllers/SettingsViewController.m @@ -560,7 +560,11 @@ BOOL isCustomResolution(int resolutionSelected) { [self addSetting:self.pipStack ofId:@"pipStack" withInfoTag:YES withDynamicLabel:NO to:videoSection]; [self addSetting:self.pipStack ofId:@"pipStack" withInfoTag:YES withDynamicLabel:NO to:videoSection]; [self addSetting:self.frameQueueSizeStack ofId:@"frameQueueSizeStack" withInfoTag:NO withDynamicLabel:NO to:videoSection]; - [self addSetting:self.renderingBackendStack ofId:@"renderingBackendStack" withInfoTag:YES withDynamicLabel:NO to:videoSection]; + + // Only show Metal renderer option on iOS 17+ where CAMetalDisplayLink is available + if (@available(iOS 17.0, *)) { + [self addSetting:self.renderingBackendStack ofId:@"renderingBackendStack" withInfoTag:YES withDynamicLabel:NO to:videoSection]; + } [videoSection addToParentStack:_parentStack]; [videoSection setExpanded:YES];