From e5fc439da2b46cc6cae44014552db41ffacbeaee Mon Sep 17 00:00:00 2001 From: Andy Grundman Date: Wed, 23 Jul 2025 02:43:38 +0800 Subject: [PATCH] Misc: fix bugs that were preventing starting a new stream after disconnecting. Known issue: this leaks a bit of memory. This also removes all OSX-specific code from the Apple example I used, as I have no intention of using this on Mac. --- VoidLink/Metal/MetalVideoRenderer.m | 40 ++++++------ VoidLink/Metal/MetalView.h | 13 +--- VoidLink/Metal/MetalView.m | 62 +++---------------- VoidLink/Metal/MetalViewController.h | 11 +--- VoidLink/Metal/MetalViewController.m | 41 +++--------- VoidLink/Stats/ImGuiPlots.h | 1 + VoidLink/Stats/ImGuiPlots.mm | 8 +++ VoidLink/Stream/VideoDecoderRenderer.m | 32 +++++----- VoidLink/Utility/FrameQueue.h | 4 +- VoidLink/Utility/FrameQueue.m | 29 ++++----- .../StreamFrameViewController.m | 2 +- 11 files changed, 81 insertions(+), 162 deletions(-) diff --git a/VoidLink/Metal/MetalVideoRenderer.m b/VoidLink/Metal/MetalVideoRenderer.m index 5f7f8c8e..add9a520 100644 --- a/VoidLink/Metal/MetalVideoRenderer.m +++ b/VoidLink/Metal/MetalVideoRenderer.m @@ -158,6 +158,11 @@ CFStringRef __currentColorSpace; } - (void)dealloc { + Log(LOG_I, @"MetalVideoRenderer dealloc"); + + if (_commandQueue) { + _commandQueue = nil; + } if (_CscParamsBuffer) { _CscParamsBuffer = nil; } @@ -172,6 +177,9 @@ CFStringRef __currentColorSpace; if (_renderPassDescriptor) { _renderPassDescriptor = nil; } + if (_textureCache) { + _textureCache = nil; + } if (__currentColorSpace) { CFRelease(__currentColorSpace); } @@ -179,12 +187,7 @@ CFStringRef __currentColorSpace; #if !TARGET_OS_TV - (void)reportMaxEDRHeadroom { - CGFloat maxHeadroom = 1.0f; -#if TARGET_OS_OSX - maxHeadroom = [[NSScreen mainScreen] maximumPotentialExtendedDynamicRangeColorComponentValue]; -#else - maxHeadroom = [[UIScreen mainScreen] potentialEDRHeadroom]; -#endif + CGFloat maxHeadroom = [[UIScreen mainScreen] potentialEDRHeadroom]; if (maxHeadroom > 1.0) { LogOnce(LOG_I, @"Display supports EDR with a max headroom of %.1f", maxHeadroom); } else { @@ -193,12 +196,7 @@ CFStringRef __currentColorSpace; } - (void)pollCurrentEDRHeadroom { - CGFloat headroom = 1.0f; -#if TARGET_OS_OSX - headroom = [[NSScreen mainScreen] maximumExtendedDynamicRangeColorComponentValue]; -#else - headroom = [[UIScreen mainScreen] currentEDRHeadroom]; -#endif + CGFloat headroom = [[UIScreen mainScreen] currentEDRHeadroom]; if (headroom != _currentEDRHeadroom) { Log(LOG_I, @"EDR headroom changed to %.1f", headroom); _currentEDRHeadroom = (float)headroom; @@ -448,7 +446,7 @@ CFStringRef __currentColorSpace; - (void)renderFrame:(Frame *)frame toLayer:(CAMetalLayer *)layer { @autoreleasepool { if (self.isStopping) { - Log(LOG_I, @"XXX Metal renderThread is stopping. returning from renderFrame"); + Log(LOG_I, @"[MetalVideoRenderer] isStopping"); return; } @@ -573,10 +571,12 @@ CFStringRef __currentColorSpace; [renderEncoder setVertexBuffer:_VideoVertexBuffer offset:0 atIndex:0]; [renderEncoder setFragmentBuffer:_CscParamsBuffer offset:0 atIndex:0]; +#if !TARGET_OS_TV if (layer.pixelFormat == MTLPixelFormatRGBA16Float) { [self pollCurrentEDRHeadroom]; [renderEncoder setFragmentBytes:&_currentEDRHeadroom length:sizeof(float) atIndex:1]; } +#endif [renderEncoder drawPrimitives:MTLPrimitiveTypeTriangleStrip vertexStart:0 vertexCount:4]; [renderEncoder endEncoding]; @@ -617,8 +617,6 @@ CFStringRef __currentColorSpace; #endif [commandBuffer commit]; - - // Wait for the command buffer to complete and free our CVMetalTextureCache references [commandBuffer waitUntilCompleted]; } } @@ -633,12 +631,14 @@ CFStringRef __currentColorSpace; } - (void)shutdown { - Log(LOG_I, @"XXX MetalVideoRenderer shutodwn"); - self.isStopping = YES; + if (!self.isStopping) { + self.isStopping = YES; + Log(LOG_I, @"[MetalVideoRenderer] shutdown"); - // Ensure no rendering is in flight - for (NSUInteger i = 0; i < MaxFramesInFlight; i++) { - dispatch_semaphore_signal(_inFlightSemaphore); + // Ensure no rendering is in flight + for (NSUInteger i = 0; i < MaxFramesInFlight; i++) { + dispatch_semaphore_signal(_inFlightSemaphore); + } } } diff --git a/VoidLink/Metal/MetalView.h b/VoidLink/Metal/MetalView.h index 39124ff4..27944bf3 100644 --- a/VoidLink/Metal/MetalView.h +++ b/VoidLink/Metal/MetalView.h @@ -1,15 +1,8 @@ #import #import #import -#import "MetalConfig.h" - -#if TARGET_OS_IOS || TARGET_OS_TV #import -#define PlatformView UIView -#else -#import -#define PlatformView NSView -#endif +#import "MetalConfig.h" // The protocol to provide resize and redraw callbacks to a delegate. @protocol MetalViewDelegate @@ -17,15 +10,13 @@ - (void)drawableResize:(CGSize)size; - (void)renderTo:(nonnull CAMetalLayer *)layer; - (void)waitToRenderTo:(nonnull CAMetalLayer *)layer; -- (void)shutdown; @end // The Metal game view base class. -@interface MetalView : PlatformView +@interface MetalView : UIView @property (nonatomic, nonnull, readonly) CAMetalLayer *metalLayer; -@property (nonatomic, getter=isPaused) BOOL paused; @property (nonatomic, nullable) id delegate; @property (nonatomic) float framerate; diff --git a/VoidLink/Metal/MetalView.m b/VoidLink/Metal/MetalView.m index 87a95cbc..734900a1 100644 --- a/VoidLink/Metal/MetalView.m +++ b/VoidLink/Metal/MetalView.m @@ -1,4 +1,4 @@ -// This is based on the following Apple example: +// This is based on the following Apple example // https://developer.apple.com/documentation/metal/achieving-smooth-frame-rates-with-a-metal-display-link?language=objc // https://developer.apple.com/wwdc23/10123/ @@ -29,29 +29,23 @@ } - (void)initCommon { -#if TARGET_OS_OSX - self.wantsLayer = YES; - - self.layerContentsRedrawPolicy = NSViewLayerContentsRedrawDuringViewResize; -#endif - _metalLayer = (CAMetalLayer *)self.layer; - self.layer.delegate = self; } - (void)shutdown { if (_renderThread) { + Log(LOG_I, @"[MetalView] sending renderThread a cancel message"); [_renderThread cancel]; - // wait for thread to exist + Log(LOG_I, @"[MetalView] waiting on renderThread to finish"); while (!_renderThread.isFinished) { - Log(LOG_I, @"XXX MetalView waiting on renderThread to finish"); usleep(100); } + Log(LOG_I, @"[MetalView] renderThread has finished"); + _renderThread = nil; } } -#if TARGET_OS_IOS || TARGET_OS_TV + (Class)layerClass { return [CAMetalLayer class]; } @@ -59,29 +53,11 @@ - (void)didMoveToWindow { [self movedToWindow]; } -#else -- (CALayer *)makeBackingLayer { - return [CAMetalLayer layer]; -} - -- (void)viewDidMoveToWindow { - [self movedToWindow]; -} -#endif // END TARGET_OS_IOS || TARGET_OS_TV - (void)movedToWindow { if (!self.window) { - return; - - // We have been removed - if (_renderThread) { - [_renderThread cancel]; - // wait for thread to exist - while (!_renderThread.isFinished) { - Log(LOG_I, @"XXX MetalView waiting on renderThread to finish"); - usleep(100); - } - } + Log(LOG_I, @"[MetalView] movedToWindow(nil): shutting down..."); + [self shutdown]; return; } @@ -93,21 +69,18 @@ [self.delegate renderTo:self.metalLayer]; } } - Log(LOG_I, @"XXX Metal renderThread shutting down"); + Log(LOG_I, @"[MetalView] renderThread is exiting"); }]; _renderThread.name = @"MetalVideoRenderer"; _renderThread.qualityOfService = NSQualityOfServiceUserInteractive; [_renderThread start]; + Log(LOG_I, @"[MetalView] started renderThread %@", _renderThread); // Perform any actions that need to know the size and scale of the drawable. When UIKit calls // didMoveToWindow after the view initialization, this is the first opportunity to notify // components of the drawable's size. #if AUTOMATICALLY_RESIZE -#if TARGET_OS_IOS || TARGET_OS_TV [self resizeDrawable:self.window.screen.nativeScale]; -#else - [self resizeDrawable:self.window.screen.backingScaleFactor]; -#endif #else // Notify the delegate of the default drawable size when the system can calculate it. CGSize defaultDrawableSize = self.bounds.size; @@ -123,7 +96,6 @@ // Override all methods that indicate the view's size has changed. -#if TARGET_OS_IOS || TARGET_OS_TV - (void)setContentScaleFactor:(CGFloat)contentScaleFactor { [super setContentScaleFactor:contentScaleFactor]; [self resizeDrawable:self.window.screen.nativeScale]; @@ -143,22 +115,6 @@ [super setBounds:bounds]; [self resizeDrawable:self.window.screen.nativeScale]; } -#else -- (void)viewDidChangeBackingProperties { - [super viewDidChangeBackingProperties]; - [self resizeDrawable:self.window.screen.backingScaleFactor]; -} - -- (void)setFrameSize:(NSSize)size { - [super setFrameSize:size]; - [self resizeDrawable:self.window.screen.backingScaleFactor]; -} - -- (void)setBoundsSize:(NSSize)size { - [super setBoundsSize:size]; - [self resizeDrawable:self.window.screen.backingScaleFactor]; -} -#endif - (void)resizeDrawable:(CGFloat)scaleFactor { CGSize newSize = self.bounds.size; diff --git a/VoidLink/Metal/MetalViewController.h b/VoidLink/Metal/MetalViewController.h index 76903d64..40cbb8b2 100644 --- a/VoidLink/Metal/MetalViewController.h +++ b/VoidLink/Metal/MetalViewController.h @@ -1,18 +1,11 @@ #import +#import #import "FrameQueue.h" #import "ImGuiRenderer.h" #import "MetalVideoRenderer.h" #import "MetalView.h" -#if TARGET_OS_IOS || TARGET_OS_TV -#import -#define PlatformViewController UIViewController -#else -#import -#define PlatformViewController NSViewController -#endif - -@interface MetalViewController : PlatformViewController +@interface MetalViewController : UIViewController @property (nonatomic) CGRect bounds; diff --git a/VoidLink/Metal/MetalViewController.m b/VoidLink/Metal/MetalViewController.m index 135746b8..8a67c91e 100644 --- a/VoidLink/Metal/MetalViewController.m +++ b/VoidLink/Metal/MetalViewController.m @@ -33,6 +33,7 @@ The implementation of the cross-platform game view controller. - (void)loadView { self.view = [[MetalView alloc] initWithFrame:_bounds]; + Log(LOG_I, @"[MetalViewController] created MetalView %@", (MetalView *)self.view); } - (void)viewDidLoad { @@ -51,7 +52,7 @@ The implementation of the cross-platform game view controller. id device = MTLCreateSystemDefaultDevice(); if (!device) { Log(LOG_E, @"Metal isn't supported on this device."); - self.view = [[PlatformView alloc] initWithFrame:self.view.frame]; + self.view = [[UIView alloc] initWithFrame:self.view.frame]; return; } view.metalLayer.device = device; @@ -64,12 +65,12 @@ The implementation of the cross-platform game view controller. Log(LOG_E, @"The renderer couldn't be initialized."); return; } + self->_renderer = renderer; + Log(LOG_I, @"[MetalViewController] viewDidLoad, created renderer: %@", renderer); // Initialize the renderer-dependent view properties. view.metalLayer.pixelFormat = renderer.colorPixelFormat; view.metalLayer.maximumDrawableCount = 3; - - self->_renderer = renderer; } - (void)waitToRenderTo:(nonnull CAMetalLayer *)layer { @@ -95,46 +96,20 @@ The implementation of the cross-platform game view controller. [_renderer drawableResize:size]; } -- (void)shutdown { - [_renderer shutdown]; -} - - (void)viewDidDisappear:(BOOL)animated { [super viewDidDisappear:animated]; - Log(LOG_I, @"XXX MetalViewController viewDidDisappear"); + Log(LOG_I, @"[MetalViewController] viewDidDisappear"); - [_metalView shutdown]; + [_renderer shutdown]; + _renderer = nil; } #if TARGET_OS_IOS -/// Hides the Home indicator button automatically. +// Hides the Home indicator button automatically. - (BOOL)prefersHomeIndicatorAutoHidden { return YES; } #endif -#if TARGET_OS_OSX -/// Makes the view controller the first responder to receive keyboard events. -- (void)viewDidAppear { - [_metalView.window makeFirstResponder:self]; -} - -/// Receives the keydown events to avoid system beeps. -/// -/// The `GameInputKeyboardMouse` class handles keyboard events. -- (void)keyDown:(NSEvent *)event { - // Reference the parameter to avoid an unused parameter warning. - (void)(event); -} - -/// Receives the keyup events to avoid system beeps. -/// -/// The `GameInputKeyboardMouse` class handles keyboard events. -- (void)keyUp:(NSEvent *)event { - // Reference the parameter to avoid an unused parameter warning. - (void)(event); -} -#endif - @end diff --git a/VoidLink/Stats/ImGuiPlots.h b/VoidLink/Stats/ImGuiPlots.h index f94620c6..5a102f8a 100644 --- a/VoidLink/Stats/ImGuiPlots.h +++ b/VoidLink/Stats/ImGuiPlots.h @@ -24,6 +24,7 @@ struct PlotDef { - (instancetype _Nonnull)init NS_UNAVAILABLE; + (instancetype _Nonnull)new NS_UNAVAILABLE; +- (void)clearData; - (void)observeFloat:(int)plotId value:(CFTimeInterval)value; - (void)observeFloatReturnMetrics:(int)plotId value:(CFTimeInterval)value plotMetrics:(PlotMetrics * _Nullable)plotMetrics; diff --git a/VoidLink/Stats/ImGuiPlots.mm b/VoidLink/Stats/ImGuiPlots.mm index 9212d0ec..d1aa5302 100644 --- a/VoidLink/Stats/ImGuiPlots.mm +++ b/VoidLink/Stats/ImGuiPlots.mm @@ -92,4 +92,12 @@ } } +- (void) clearData { + for (int i = 0; i < PlotCount; i++) { + if (_plots[i].buffer) { + [_plots[i].buffer clear]; + } + } +} + @end diff --git a/VoidLink/Stream/VideoDecoderRenderer.m b/VoidLink/Stream/VideoDecoderRenderer.m index 56240fd5..53e45489 100644 --- a/VoidLink/Stream/VideoDecoderRenderer.m +++ b/VoidLink/Stream/VideoDecoderRenderer.m @@ -65,7 +65,7 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size, _displayLayer.videoGravity = AVLayerVideoGravityResize; [_view.layer addSublayer:_displayLayer]; } - + // Ensure the AVSampleBufferDisplayLayer is sized to preserve the aspect ratio // of the video stream. We used to use AVLayerVideoGravityResizeAspect, but that // respects the PAR encoded in the SPS which causes our computed video-relative @@ -77,30 +77,30 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size, } else { videoSize = CGSizeMake(_view.bounds.size.width, _view.bounds.size.width / _streamAspectRatio); } - + [CATransaction begin]; [CATransaction setDisableActions:YES]; _displayLayer.position = CGPointMake(CGRectGetMidX(_view.bounds), CGRectGetMidY(_view.bounds)); _displayLayer.bounds = CGRectMake(0, 0, videoSize.width, videoSize.height); [CATransaction commit]; - + // Hide the layer until we get an IDR frame. This ensures we // can see the loading progress label as the stream is starting. _displayLayer.hidden = YES; - + if (_formatDesc != nil) { CFRelease(_formatDesc); _formatDesc = nil; } - + if (_formatDescImageBuffer != nil) { CFRelease(_formatDescImageBuffer); _formatDescImageBuffer = nil; } - + if (_decompressionSession != nil){ VTDecompressionSessionWaitForAsynchronousFrames(_decompressionSession); - + VTDecompressionSessionInvalidate(_decompressionSession); CFRelease(_decompressionSession); _decompressionSession = nil; @@ -122,14 +122,13 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size, _view = view; _callbacks = callbacks; _streamAspectRatio = aspectRatio; - - _parameterSetBuffers = [[NSMutableArray alloc] init]; - _frameQueue = [FrameQueue sharedInstance]; - [_frameQueue start]; _maxRefreshRate = [[UIScreen mainScreen] maximumFramesPerSecond]; + _parameterSetBuffers = [[NSMutableArray alloc] init]; DataManager* dataMan = [[DataManager alloc] init]; + _frameQueue = [FrameQueue sharedInstance]; + [_frameQueue start]; [_frameQueue setHighWaterMark:(int)[[dataMan getSettings].frameQueueSize integerValue]]; [self reinitializeDisplayLayer]; @@ -149,6 +148,9 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size, self->_videoFormat = videoFormat; self->_frameRate = frameRate; + // reset plot data in case we've already used it for a previous renderer + [[ImGuiPlots sharedInstance] clearData]; + DataManager* dataMan = [[DataManager alloc] init]; if ([[dataMan getSettings].renderingBackend integerValue] == RENDER_AVSB) { // PACING_MODE_VSYNC: @@ -221,7 +223,7 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size, // so this is safe to do right here. [self->_displayLayer flushAndRemoveImage]; [self reinitializeDisplayLayer]; - + // Request an IDR frame to initialize the new decoder LiRequestIdrFrame(); } @@ -349,8 +351,8 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit); } - (void)cleanup { - [_frameQueue shutdown]; - + [_frameQueue stop]; + if (_renderingBackend == RENDER_AVSB) { [_displayLink invalidate]; } @@ -745,7 +747,7 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit); free(data); return DR_NEED_IDR; } - + // Now we're decoding actual frame data here CMBlockBufferRef frameBlockBuffer; CMBlockBufferRef dataBlockBuffer; diff --git a/VoidLink/Utility/FrameQueue.h b/VoidLink/Utility/FrameQueue.h index e680b910..cfca4ce2 100644 --- a/VoidLink/Utility/FrameQueue.h +++ b/VoidLink/Utility/FrameQueue.h @@ -12,7 +12,7 @@ NS_ASSUME_NONNULL_BEGIN @property (nonatomic) FloatBuffer *frameDropMetrics; @property (nonatomic) int highWaterMark; @property (nonatomic, readonly) int maxCapacity; -@property (atomic) BOOL isStopping; +@property (atomic) BOOL paused; + (instancetype)sharedInstance; @@ -27,8 +27,8 @@ NS_ASSUME_NONNULL_BEGIN - (CFTimeInterval)estimatedFramerate; - (int)currentSoftCap; - (void)waitForEnqueue; -- (void)shutdown; - (void)start; +- (void)stop; @end diff --git a/VoidLink/Utility/FrameQueue.m b/VoidLink/Utility/FrameQueue.m index 532c3269..0113441e 100644 --- a/VoidLink/Utility/FrameQueue.m +++ b/VoidLink/Utility/FrameQueue.m @@ -60,7 +60,7 @@ _ptsCorrection = CMTimeMake(0, 90000); _queueSizeHistory = [[FloatBuffer alloc] initWithCapacity:64]; _lock = OS_UNFAIR_LOCK_INIT; - _isStopping = NO; + _paused = YES; // caller will call start() // ring buffer _capacity = _maxCapacity; @@ -226,7 +226,7 @@ // Allows the render loop to wait if the queue is empty - (void)waitForEnqueue { - while (!self.isStopping && [self isEmpty]) { + while (!self.paused && [self isEmpty]) { dispatch_time_t timeout = dispatch_time(DISPATCH_TIME_NOW, (int64_t)(0.1f * NSEC_PER_SEC)); // 100ms dispatch_semaphore_wait(_frameSemaphore, timeout); } @@ -256,7 +256,7 @@ CFTimeInterval deadline = start + timeout; int round = 0; - if (self.isStopping) { + if (self.paused) { return nil; } @@ -287,15 +287,10 @@ return [self count] == 0; } -- (void)_clear_unsafe { - _head = _tail = _count = 0; - _frameDropMetrics = [[FloatBuffer alloc] initWithCapacity:512]; -} - -// The public clear method now safely calls the unsafe version. - (void)clear { os_unfair_lock_lock(&_lock); - [self _clear_unsafe]; + _head = _tail = _count = 0; + _frameDropMetrics = [[FloatBuffer alloc] initWithCapacity:512]; os_unfair_lock_unlock(&_lock); } @@ -329,19 +324,17 @@ return cap; } -- (void)shutdown { +- (void)stop { // new frames will no longer be coming in, make sure consumer side is not left waiting - self.isStopping = YES; - Log(LOG_I, @"XXX FrameQueue shutting down"); + self.paused = YES; + Log(LOG_I, @"FrameQueue stopped"); dispatch_semaphore_signal(_frameSemaphore); } - (void)start { - os_unfair_lock_lock(&_lock); - [self _clear_unsafe]; // Use the version without the lock - self.isStopping = NO; - os_unfair_lock_unlock(&_lock); - Log(LOG_I, @"FrameQueue (re)started"); + // (re)start for a new renderer + self.paused = NO; + Log(LOG_I, @"FrameQueue started"); } // For use with NSLog("%@", franeQueue); diff --git a/VoidLink/ViewControllers/StreamFrameViewController.m b/VoidLink/ViewControllers/StreamFrameViewController.m index 06838968..669b6d0e 100644 --- a/VoidLink/ViewControllers/StreamFrameViewController.m +++ b/VoidLink/ViewControllers/StreamFrameViewController.m @@ -663,6 +663,7 @@ if ([_settings.renderingBackend intValue] == RENDER_METAL) { // Metal view for video + Log(LOG_I, @"StreamFrameViewController creating MetalViewController"); self.metalViewController = [[MetalViewController alloc] initWithFrame:self.view.bounds framerate:[self->_settings.framerate floatValue] enableHdr:self->_settings.enableHdr @@ -745,7 +746,6 @@ _inactivityTimer = nil; } if (self.metalViewController) { - [self.metalViewController shutdown]; [self.metalViewController.view removeFromSuperview]; [self.metalViewController removeFromParentViewController]; self.metalViewController = nil;