From 686418997b750fa8fced09370055462f1a786a7f Mon Sep 17 00:00:00 2001 From: Acaki Date: Tue, 15 Jul 2025 18:08:54 +0800 Subject: [PATCH] fix: metal renderer teardown and resume --- VoidLink/Metal/MetalVideoRenderer.h | 2 + VoidLink/Metal/MetalVideoRenderer.m | 66 +++++++------- VoidLink/Metal/MetalView.h | 4 + VoidLink/Metal/MetalView.m | 85 +++++++++++++------ VoidLink/Metal/MetalViewController.h | 4 + VoidLink/Metal/MetalViewController.m | 52 +++++++++++- VoidLink/Utility/FrameQueue.h | 1 + VoidLink/Utility/FrameQueue.m | 45 ++++++++++ .../StreamFrameViewController.m | 15 ++++ 9 files changed, 216 insertions(+), 58 deletions(-) diff --git a/VoidLink/Metal/MetalVideoRenderer.h b/VoidLink/Metal/MetalVideoRenderer.h index 8b2f0337..23f76b56 100644 --- a/VoidLink/Metal/MetalVideoRenderer.h +++ b/VoidLink/Metal/MetalVideoRenderer.h @@ -19,5 +19,7 @@ - (void)waitToRenderTo:(nonnull CAMetalLayer *)layer; - (void)drawableResize:(CGSize)drawableSize; - (void)plotFrametime:(CFTimeInterval)presentedTime withPresentTime:(CFTimeInterval)presentTime; +- (void)stop; +- (void)discardNextDrawable; @end diff --git a/VoidLink/Metal/MetalVideoRenderer.m b/VoidLink/Metal/MetalVideoRenderer.m index 27a9511e..b0ca860b 100644 --- a/VoidLink/Metal/MetalVideoRenderer.m +++ b/VoidLink/Metal/MetalVideoRenderer.m @@ -95,8 +95,6 @@ static const NSUInteger MaxFramesInFlight = 3; id _VideoVertexBuffer; CFTimeInterval _lastPresented; - // https://developer.apple.com/documentation/metal/synchronizing-cpu-and-gpu-work?language=objc - dispatch_semaphore_t _inFlightSemaphore; } - (instancetype)initWithMetalDevice:(id)device drawablePixelFormat:(MTLPixelFormat)drawablePixelFormat framerate:(float)framerate { @@ -114,7 +112,6 @@ static const NSUInteger MaxFramesInFlight = 3; _lastColorSpace = -1; _lastFullRange = NO; _lastPresented = 0; - _inFlightSemaphore = dispatch_semaphore_create(MaxFramesInFlight); CFStringRef keys[1] = {kCVMetalTextureUsage}; NSUInteger values[1] = {MTLTextureUsageShaderRead}; @@ -131,10 +128,21 @@ static const NSUInteger MaxFramesInFlight = 3; } - (void)dealloc { - if (_CscParamsBuffer) { - _CscParamsBuffer = nil; + // Release the Core Foundation texture cache, which is not managed by ARC. + if (_textureCache) { + CFRelease(_textureCache); + _textureCache = NULL; } + // Safely release any textures that might still be referenced. + for (int i = 0; i < MAX_VIDEO_PLANES; i++) { + if (_cvMetalTextures[i]) { + CFRelease(_cvMetalTextures[i]); + _cvMetalTextures[i] = NULL; + } + } + + // ARC will handle the rest of the Objective-C objects like _CscParamsBuffer. } #if !TARGET_OS_TV @@ -453,38 +461,21 @@ static const NSUInteger MaxFramesInFlight = 3; id renderEncoder = [commandBuffer renderCommandEncoderWithDescriptor:_renderPassDescriptor]; [renderEncoder setRenderPipelineState:_videoPipelineState]; + [renderEncoder setVertexBuffer:_VideoVertexBuffer offset:0 atIndex:0]; + [renderEncoder setFragmentBuffer:_CscParamsBuffer offset:0 atIndex:0]; + for (size_t i = 0; i < planes; i++) { [renderEncoder setFragmentTexture:CVMetalTextureGetTexture(_cvMetalTextures[i]) atIndex:i]; } - __block dispatch_semaphore_t block_semaphore = _inFlightSemaphore; - [commandBuffer addCompletedHandler:^(id cb) { - dispatch_semaphore_signal(block_semaphore); - - const CFTimeInterval GPUTime = cb.GPUEndTime - cb.GPUStartTime; - const double alpha = 0.25f; - self->_averageGPUTime = (GPUTime * alpha) + (self->_averageGPUTime * (1.0 - alpha)); - - // Free textures after completion of rendering - for (size_t i = 0; i < planes; i++) { - if (self->_cvMetalTextures[i]) { - CVBufferRelease(self->_cvMetalTextures[i]); - self->_cvMetalTextures[i] = nil; - } - } - - CVMetalTextureCacheFlush(self->_textureCache, 0); - }]; - - [renderEncoder setFragmentBuffer:_CscParamsBuffer offset:0 atIndex:0]; - [renderEncoder setVertexBuffer:_VideoVertexBuffer offset:0 atIndex:0]; [renderEncoder drawPrimitives:MTLPrimitiveTypeTriangleStrip vertexStart:0 vertexCount:4]; [renderEncoder endEncoding]; - __weak typeof(self) self_ = self; + __weak typeof(self) weakSelf = self; [_nextDrawable addPresentedHandler:^(id d) { - if (self_) { - [self_ plotFrametime:d.presentedTime]; + __strong typeof(weakSelf) strongSelf = weakSelf; + if (strongSelf) { + [strongSelf plotFrametime:d.presentedTime]; } }]; @@ -499,6 +490,19 @@ static const NSUInteger MaxFramesInFlight = 3; // Wait for the command buffer to complete and free our CVMetalTextureCache references [commandBuffer waitUntilCompleted]; + + // Perform cleanup + const CFTimeInterval GPUTime = commandBuffer.GPUEndTime - commandBuffer.GPUStartTime; + const double alpha = 0.25f; + _averageGPUTime = (GPUTime * alpha) + (_averageGPUTime * (1.0 - alpha)); + + for (size_t i = 0; i < planes; i++) { + if (_cvMetalTextures[i]) { + CVBufferRelease(_cvMetalTextures[i]); + _cvMetalTextures[i] = nil; + } + } + CVMetalTextureCacheFlush(_textureCache, 0); _nextDrawable = nil; } } @@ -522,7 +526,6 @@ static const NSUInteger MaxFramesInFlight = 3; // Wait to ensure only `MaxFramesInFlight` number of frames are getting processed // by any stage in the Metal pipeline (CPU, GPU, Metal, Drivers, etc.). - dispatch_semaphore_wait(_inFlightSemaphore, DISPATCH_TIME_FOREVER); } } @@ -535,4 +538,7 @@ static const NSUInteger MaxFramesInFlight = 3; } +- (void)stop { +} + @end diff --git a/VoidLink/Metal/MetalView.h b/VoidLink/Metal/MetalView.h index 603b97ca..bace20ea 100644 --- a/VoidLink/Metal/MetalView.h +++ b/VoidLink/Metal/MetalView.h @@ -33,4 +33,8 @@ - (void)resizeDrawable:(CGFloat)scaleFactor; #endif +- (void)stop; +- (void)pause; +- (void)resume; + @end diff --git a/VoidLink/Metal/MetalView.m b/VoidLink/Metal/MetalView.m index efa6e45b..d24b3521 100644 --- a/VoidLink/Metal/MetalView.m +++ b/VoidLink/Metal/MetalView.m @@ -11,6 +11,7 @@ // The flag to indicate that rendering needs to cease on the main thread. BOOL _continueRunLoop; + dispatch_semaphore_t _renderThreadSemaphore; } #pragma mark - Initialization and Setup. @@ -41,6 +42,7 @@ _metalLayer = (CAMetalLayer *)self.layer; self.layer.delegate = self; + _renderThreadSemaphore = dispatch_semaphore_create(0); } #if TARGET_OS_IOS || TARGET_OS_TV @@ -61,37 +63,44 @@ } #endif // END TARGET_OS_IOS || TARGET_OS_TV -- (void)movedToWindow { - // Protect _continueRunLoop with a `@synchronized` block because it's accessed by the separate - // animation thread. +- (void)startRenderThread { @synchronized(self) { - // Stop the animation loop, allowing it to complete if it's in progress. - _continueRunLoop = NO; + // Don't start a new thread if one is already running + if (_renderThread) { + return; + } + + _continueRunLoop = YES; + _renderThread = [[NSThread alloc] initWithTarget:self selector:@selector(runThread) object:nil]; + _renderThread.qualityOfService = NSQualityOfServiceUserInteractive; + [_renderThread start]; } +} - // Create and start a secondary NSThread that has another runloop. The NSThread - // class calls the 'runThread' method at the start of the secondary thread's execution. - _renderThread = [[NSThread alloc] initWithTarget:self selector:@selector(runThread) object:nil]; - _continueRunLoop = YES; - _renderThread.qualityOfService = NSQualityOfServiceUserInteractive; - [_renderThread start]; +- (void)movedToWindow { + if (self.window) { + // The view was added to a window, so start rendering. + [self resume]; + + // Notify the delegate of the drawable's size. + CGSize defaultDrawableSize = self.bounds.size; + defaultDrawableSize.width *= self.layer.contentsScale; + defaultDrawableSize.height *= self.layer.contentsScale; + [self.delegate drawableResize:defaultDrawableSize]; + } else { + // The view was removed from a window, so stop rendering. + [self pause]; + } +} - // Perform any actions that need to know the size and scale of the drawable. When UIKit calls - // didMoveToWindow after the view initialization, this is the first opportunity to notify - // components of the drawable's size. -#if AUTOMATICALLY_RESIZE -#if TARGET_OS_IOS || TARGET_OS_TV - [self resizeDrawable:self.window.screen.nativeScale]; -#else - [self resizeDrawable:self.window.screen.backingScaleFactor]; -#endif -#else - // Notify the delegate of the default drawable size when the system can calculate it. - CGSize defaultDrawableSize = self.bounds.size; - defaultDrawableSize.width *= self.layer.contentsScale; - defaultDrawableSize.height *= self.layer.contentsScale; - [self.delegate drawableResize:defaultDrawableSize]; -#endif +- (void)pause { + // Pausing is implemented by simply stopping the render thread. + [self stop]; +} + +- (void)resume { + // Resuming is implemented by starting a new render thread. + [self startRenderThread]; } - (void)runThread { @@ -118,6 +127,7 @@ } } } + dispatch_semaphore_signal(self->_renderThreadSemaphore); } #pragma mark - Resizing @@ -188,4 +198,25 @@ } #endif // END AUTOMATICALLY_RESIZE +- (void)stop { + @synchronized(self) { + // If already stopping, do nothing. + if (!_continueRunLoop) { + return; + } + _continueRunLoop = NO; + } + + if (_renderThread && _renderThread != [NSThread currentThread]) { + // Wait for the render thread to finish its loop and signal the semaphore. + // We use a 1-second timeout to prevent the app from hanging indefinitely + // if the thread gets stuck for some reason. + long timeoutResult = dispatch_semaphore_wait(_renderThreadSemaphore, dispatch_time(DISPATCH_TIME_NOW, 1 * NSEC_PER_SEC)); + if (timeoutResult != 0) { + Log(LOG_E, @"MetalView render thread failed to stop gracefully within 1 second."); + } + } + + _renderThread = nil; +} @end diff --git a/VoidLink/Metal/MetalViewController.h b/VoidLink/Metal/MetalViewController.h index 76903d64..1610cd5b 100644 --- a/VoidLink/Metal/MetalViewController.h +++ b/VoidLink/Metal/MetalViewController.h @@ -21,4 +21,8 @@ enableHdr:(BOOL)enableHdr metricsHandler:(MetricsHandler _Nonnull)metricsHandler; +- (void)stop; +- (void)pause; +- (void)resume; + @end diff --git a/VoidLink/Metal/MetalViewController.m b/VoidLink/Metal/MetalViewController.m index a88782d5..b8f47c73 100644 --- a/VoidLink/Metal/MetalViewController.m +++ b/VoidLink/Metal/MetalViewController.m @@ -31,6 +31,7 @@ The implementation of the cross-platform game view controller. _enableHdr = enableHdr; _metricsHandler = metricsHandler; _stopping = NO; + [_frameQueue clear]; } return self; } @@ -86,7 +87,9 @@ The implementation of the cross-platform game view controller. [_renderer waitToRenderTo:layer]; // If we don't have a frame yet, wait on that too - [_frameQueue waitForEnqueue]; + if (!_stopping) { + [_frameQueue waitForEnqueue]; + } } } @@ -137,4 +140,51 @@ The implementation of the cross-platform game view controller. } #endif +- (void)stop { + // 1. Signal that we are stopping to prevent new work from starting. + _stopping = YES; + + // 2. Invalidate the CAMetalLayer by removing its device. + // This should cause any blocking calls like `nextDrawable` on the render thread to fail + // and return immediately, breaking the deadlock. + if (_metalView) { + _metalView.metalLayer.device = nil; + } + + // 3. Unblock the render thread from any other potential waiting points. + if (_renderer) { + [_renderer stop]; + } + [_frameQueue stop]; + + // 4. Now that the thread is unblocked, wait for it to finish its execution. + if (_metalView) { + [_metalView stop]; + } + + // 5. Once the thread has terminated, it's safe to deallocate all resources. + _renderer = nil; + _metalView = nil; +} + +- (void)pause { + if (_renderer) { + // Add this line to discard any stale drawable before pausing the thread. + // This forces the renderer to get a fresh one on resume. + [_renderer discardNextDrawable]; + } + + if (_metalView) { + [_metalView pause]; + Log(LOG_I, @"Metal rendering paused."); + } +} + +- (void)resume { + if (_metalView) { + [_metalView resume]; + Log(LOG_I, @"Metal rendering resumed."); + } +} + @end diff --git a/VoidLink/Utility/FrameQueue.h b/VoidLink/Utility/FrameQueue.h index 33cd3654..06a47385 100644 --- a/VoidLink/Utility/FrameQueue.h +++ b/VoidLink/Utility/FrameQueue.h @@ -26,6 +26,7 @@ NS_ASSUME_NONNULL_BEGIN - (CFTimeInterval)estimatedFramerate; - (int)currentSoftCap; - (void)waitForEnqueue; +- (void)stop; @end diff --git a/VoidLink/Utility/FrameQueue.m b/VoidLink/Utility/FrameQueue.m index 229b9f8a..f9db0c77 100644 --- a/VoidLink/Utility/FrameQueue.m +++ b/VoidLink/Utility/FrameQueue.m @@ -16,6 +16,7 @@ int _count; BOOL _droppedLast; + BOOL _stopping; int _framesIn; CMTime _ptsCorrection; os_unfair_lock _lock; @@ -75,6 +76,7 @@ // ping estimatedFramerate to set initial last value [self estimatedFramerate]; + _stopping = NO; } return self; } @@ -171,6 +173,10 @@ // enqueue with simple alternate-drop logic - (int)enqueue:(Frame *)frame { os_unfair_lock_lock(&_lock); + if (_stopping) { + os_unfair_lock_unlock(&_lock); + return 0; // Don't enqueue if stopping + } int dropCount = [self _unsafeEnqueue:frame withDropTarget:_highWaterMark]; os_unfair_lock_unlock(&_lock); return dropCount; @@ -179,6 +185,10 @@ // enqueue that is a bit more flexixble, using the same 500ms queue size history method as moonlight-qt. - (int)enqueue:(Frame *)frame withSlackSize:(int)slack { os_unfair_lock_lock(&_lock); + if (_stopping) { + os_unfair_lock_unlock(&_lock); + return 0; // Don't enqueue if stopping + } CFTimeInterval now = CACurrentMediaTime(); // new data point for queue health @@ -226,7 +236,13 @@ // Allows the render loop to wait if the queue is empty - (void)waitForEnqueue { while ([self isEmpty]) { + if (_stopping) { + return; + } dispatch_semaphore_wait(_frameSemaphore, DISPATCH_TIME_FOREVER); + if (_stopping) { + return; + } } } @@ -256,6 +272,9 @@ // Always attempt to dequeue at least once do { + if (_stopping) { + return nil; + } if (round > 0) { usleep(100); // 0.1ms } @@ -285,6 +304,7 @@ os_unfair_lock_lock(&_lock); _head = _tail = _count = 0; _frameDropMetrics = [[FloatBuffer alloc] initWithCapacity:512]; + _stopping = NO; os_unfair_lock_unlock(&_lock); } @@ -327,4 +347,29 @@ return [NSString stringWithFormat:@"[%@]", [parts componentsJoinedByString:@",\n"]]; } +- (void)stop { + os_unfair_lock_lock(&_lock); + // Prevent redundant calls + if (_stopping) { + os_unfair_lock_unlock(&_lock); + return; + } + + _stopping = YES; + + // Clear the buffer's contents + _head = _tail = _count = 0; + for (int i = 0; i < _capacity; i++) { + [_buffer replaceObjectAtIndex:i withObject:[NSNull null]]; + } + + os_unfair_lock_unlock(&_lock); + + // Unblock any threads waiting on the semaphore. + // Signaling multiple times ensures all potential consumer threads are released. + for (int i = 0; i < _maxCapacity; i++) { + dispatch_semaphore_signal(_frameSemaphore); + } +} + @end diff --git a/VoidLink/ViewControllers/StreamFrameViewController.m b/VoidLink/ViewControllers/StreamFrameViewController.m index 466fde0e..b511ef6e 100644 --- a/VoidLink/ViewControllers/StreamFrameViewController.m +++ b/VoidLink/ViewControllers/StreamFrameViewController.m @@ -668,7 +668,9 @@ enableHdr:self->_settings.enableHdr metricsHandler:self.imguiView.metricsHandler]; self.metalViewController.view.userInteractionEnabled = NO; + [self addChildViewController:self.metalViewController]; [self.view addSubview:self.metalViewController.view]; + [self.metalViewController didMoveToParentViewController:self]; [self.view bringSubviewToFront:self.metalViewController.view]; } } @@ -742,6 +744,12 @@ [_inactivityTimer invalidate]; _inactivityTimer = nil; } + if (self.metalViewController) { + [self.metalViewController stop]; + [self.metalViewController.view removeFromSuperview]; + self.metalViewController = nil; + NSLog(@"Metal renderer stopped and cleaned up."); + } [[NSNotificationCenter defaultCenter] removeObserver:self]; } } @@ -918,6 +926,10 @@ // This will fire if the user opens control center or gets a low battery message - (void)applicationWillResignActive:(NSNotification *)notification { + if (self.metalViewController) { + [self.metalViewController pause]; + } + //[self.pipController startPictureInPicture]; //sleep(1); @@ -935,6 +947,9 @@ } - (void)applicationDidBecomeActive:(NSNotification *)notification { + if (self.metalViewController) { + [self.metalViewController resume]; + } // Stop the background timer, since we're foregrounded again if (_inactivityTimer != nil) { Log(LOG_I, @"Stopping inactivity timer after becoming active again");