diff --git a/VoidLink.xcodeproj/project.pbxproj b/VoidLink.xcodeproj/project.pbxproj index f68c64cc..730cc37d 100644 --- a/VoidLink.xcodeproj/project.pbxproj +++ b/VoidLink.xcodeproj/project.pbxproj @@ -40,7 +40,7 @@ 84E1CDFA2C4FE83400547904 /* SwiftLocalizationHelper.swift in Sources */ = {isa = PBXBuildFile; fileRef = 84E1CDF92C4FE83400547904 /* SwiftLocalizationHelper.swift */; }; 84FD0B512C31338D006B5BE5 /* LocalizationHelper.m in Sources */ = {isa = PBXBuildFile; fileRef = 84FD0B502C31338D006B5BE5 /* LocalizationHelper.m */; }; 84FD0B542C3134A6006B5BE5 /* Localizable.xcstrings in Resources */ = {isa = PBXBuildFile; fileRef = 84FD0B532C3134A6006B5BE5 /* Localizable.xcstrings */; }; - 97E20DBF2E21637B000D5DCA /* SceneDelegate.m in Sources */ = {isa = PBXBuildFile; fileRef = 97E20DBE2E21637B000D5DCA /* SceneDelegate.m */; }; + 9777AAF32E2CF7A400D7FBAF /* Shaders-Linear.metal in Sources */ = {isa = PBXBuildFile; fileRef = 9777AAF22E2CF7A400D7FBAF /* Shaders-Linear.metal */; }; 97E20DCF2E21647A000D5DCA /* Shaders.metal in Sources */ = {isa = PBXBuildFile; fileRef = 97E20DCD2E21647A000D5DCA /* Shaders.metal */; }; 97E20DD02E21647A000D5DCA /* MetalViewController.m in Sources */ = {isa = PBXBuildFile; fileRef = 97E20DCC2E21647A000D5DCA /* MetalViewController.m */; }; 97E20DD12E21647A000D5DCA /* MetalView.m in Sources */ = {isa = PBXBuildFile; fileRef = 97E20DCA2E21647A000D5DCA /* MetalView.m */; }; @@ -211,6 +211,7 @@ 84FD0B4F2C313343006B5BE5 /* LocalizationHelper.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = LocalizationHelper.h; sourceTree = ""; }; 84FD0B502C31338D006B5BE5 /* LocalizationHelper.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = LocalizationHelper.m; sourceTree = ""; }; 84FD0B532C3134A6006B5BE5 /* Localizable.xcstrings */ = {isa = PBXFileReference; lastKnownFileType = text.json.xcstrings; path = Localizable.xcstrings; sourceTree = ""; }; + 9777AAF22E2CF7A400D7FBAF /* Shaders-Linear.metal */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.metal; path = "Shaders-Linear.metal"; sourceTree = ""; }; 97E20DBD2E21637B000D5DCA /* SceneDelegate.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = SceneDelegate.h; sourceTree = ""; }; 97E20DBE2E21637B000D5DCA /* SceneDelegate.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = SceneDelegate.m; sourceTree = ""; }; 97E20DC62E21647A000D5DCA /* MetalConfig.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = MetalConfig.h; sourceTree = ""; }; @@ -469,6 +470,7 @@ 97E20DCE2E21647A000D5DCA /* Metal */ = { isa = PBXGroup; children = ( + 9777AAF22E2CF7A400D7FBAF /* Shaders-Linear.metal */, 97E20DC62E21647A000D5DCA /* MetalConfig.h */, 97E20DC72E21647A000D5DCA /* MetalVideoRenderer.h */, 97E20DC82E21647A000D5DCA /* MetalVideoRenderer.m */, @@ -1022,6 +1024,7 @@ FBD3495319FF36FB002D2A60 /* SWRevealViewController.m in Sources */, 84203F232C5F24C00032EEED /* OnScreenWidgetView.swift in Sources */, 97E21BB52E2186A3000D5DCA /* imgui_impl_metal.mm in Sources */, + 9777AAF32E2CF7A400D7FBAF /* Shaders-Linear.metal in Sources */, 84FD0B512C31338D006B5BE5 /* LocalizationHelper.m in Sources */, 97E20DEA2E216532000D5DCA /* SceneDelegate.m in Sources */, FB1D59971BBCCB6400F482CA /* ComputerScrollView.m in Sources */, diff --git a/VoidLink/Metal/MetalVideoRenderer.h b/VoidLink/Metal/MetalVideoRenderer.h index 95489fe1..7f5d77ad 100644 --- a/VoidLink/Metal/MetalVideoRenderer.h +++ b/VoidLink/Metal/MetalVideoRenderer.h @@ -1,5 +1,6 @@ #import #import +#import #import "ConnectionCallbacks.h" #import "Frame.h" #import "Plot.h" @@ -9,17 +10,14 @@ @property (atomic) CFTimeInterval averageGPUTime; @property (nonatomic) NSUInteger sampleCount; @property (nonatomic) MTLPixelFormat colorPixelFormat; -@property (nonatomic, nonnull) CGColorSpaceRef colorspace; +@property (nonatomic) CFTimeInterval lastPresented; @property (nonatomic) id _Nullable nextDrawable; +@property (atomic) BOOL isStopping; -- (nonnull instancetype)initWithMetalDevice:(nonnull id)device - drawablePixelFormat:(MTLPixelFormat)drawablePixelFormat - framerate:(float)framerate; +- (nonnull instancetype)initWithMetalDevice:(nonnull id)device drawablePixelFormat:(MTLPixelFormat)drawablePixelFormat framerate:(float)framerate; - (void)renderFrame:(nonnull Frame *)frame toLayer:(nonnull CAMetalLayer *)layer; - (void)waitToRenderTo:(nonnull CAMetalLayer *)layer; - (void)drawableResize:(CGSize)drawableSize; -- (void)plotFrametime:(CFTimeInterval)presentedTime; -- (void)stop; -- (void)discardNextDrawable; +- (void)shutdown; @end diff --git a/VoidLink/Metal/MetalVideoRenderer.m b/VoidLink/Metal/MetalVideoRenderer.m index 4e6e394b..8f74c914 100644 --- a/VoidLink/Metal/MetalVideoRenderer.m +++ b/VoidLink/Metal/MetalVideoRenderer.m @@ -1,7 +1,10 @@ #import "MetalVideoRenderer.h" +#import #import +#import #import #import +#import #import "ImGuiPlots.h" #include @@ -81,7 +84,6 @@ static const struct CscParams k_CscParams_Bt2020Full_10bit = { {1.0f, -0.1646f, -0.5714f}, {1.0f, 1.8814f, 0.0f}, }, - {0.0f, 512.0f / 1023.0f, 512.0f / 1023.0f}, }; @@ -101,10 +103,10 @@ static const NSUInteger MaxFramesInFlight = 3; id _shaderLibrary; id _videoPipelineState[MAX_VIDEO_PLANES]; MTLRenderPassDescriptor *_renderPassDescriptor; - id _videoTexture; CVMetalTextureCacheRef _textureCache; CVMetalTextureRef _cvMetalTextures[MAX_VIDEO_PLANES]; + CGFloat _currentEDRHeadroom; int _lastColorSpace; BOOL _lastFullRange; size_t _lastFrameWidth; @@ -113,8 +115,9 @@ static const NSUInteger MaxFramesInFlight = 3; size_t _lastDrawableHeight; id _CscParamsBuffer; id _VideoVertexBuffer; - CFTimeInterval _lastPresented; + // https://developer.apple.com/documentation/metal/synchronizing-cpu-and-gpu-work?language=objc + dispatch_semaphore_t _inFlightSemaphore; } - (instancetype)initWithMetalDevice:(id)device drawablePixelFormat:(MTLPixelFormat)drawablePixelFormat framerate:(float)framerate { @@ -124,14 +127,15 @@ static const NSUInteger MaxFramesInFlight = 3; dispatch_queue_attr_make_with_qos_class(DISPATCH_QUEUE_SERIAL, QOS_CLASS_USER_INTERACTIVE, 0)); _averageGPUTime = (1.0f / framerate) / 2; _device = device; - _nextDrawable = nil; _colorPixelFormat = MTLPixelFormatBGR10A2Unorm; - _colorspace = CGColorSpaceCreateWithName(kCGColorSpaceITUR_2100_PQ); _framerate = framerate; _commandQueue = [_device newCommandQueue]; + _currentEDRHeadroom = 1.0f; _lastColorSpace = -1; _lastFullRange = NO; - _lastPresented = 0; + _lastPresented = 0.0f; + _inFlightSemaphore = dispatch_semaphore_create(MaxFramesInFlight); + _isStopping = NO; CFStringRef keys[1] = {kCVMetalTextureUsage}; NSUInteger values[1] = {MTLTextureUsageShaderRead}; @@ -148,63 +152,94 @@ static const NSUInteger MaxFramesInFlight = 3; } - (void)dealloc { - // Release the Core Foundation texture cache, which is not managed by ARC. - if (_textureCache) { - CFRelease(_textureCache); - _textureCache = NULL; + if (_CscParamsBuffer) { + _CscParamsBuffer = nil; + } + if (_VideoVertexBuffer) { + _VideoVertexBuffer = nil; } - - // Safely release any textures that might still be referenced. for (int i = 0; i < MAX_VIDEO_PLANES; i++) { - if (_cvMetalTextures[i]) { - CFRelease(_cvMetalTextures[i]); - _cvMetalTextures[i] = NULL; + if (_videoPipelineState[i]) { + _videoPipelineState[i] = nil; } } - - // ARC will handle the rest of the Objective-C objects like _CscParamsBuffer. + if (_renderPassDescriptor) { + _renderPassDescriptor = nil; + } } #if !TARGET_OS_TV -- (void)applyEDRFromFrame:(Frame *)frame toLayer:(CAMetalLayer *)layer { - CFDictionaryRef ext = [frame getFormatDescExtensions]; - - Log(LOG_I, @"ext: %@", ext); - - CFDataRef masteringData = CFDictionaryGetValue(ext, kCMFormatDescriptionExtension_MasteringDisplayColorVolume); - CFDataRef contentDataRef = CFDictionaryGetValue(ext, kCMFormatDescriptionExtension_ContentLightLevelInfo); - - if (masteringData) { - Log(LOG_I, @"ext MDCV %@", masteringData); - } - if (contentDataRef) { - Log(LOG_I, @"ext CLLI %@", contentDataRef); - } - - if (masteringData && CFDataGetLength(masteringData) == 24 && contentDataRef && CFDataGetLength(contentDataRef) == 4) { - NSData *displayData = (__bridge NSData *)masteringData; - NSData *contentData = (__bridge NSData *)contentDataRef; - - layer.wantsExtendedDynamicRangeContent = YES; - layer.pixelFormat = MTLPixelFormatRGBA16Float; - CFStringRef name = kCGColorSpaceExtendedLinearITUR_2020; - CGColorSpaceRef colorspace = CGColorSpaceCreateWithName(name); - layer.colorspace = colorspace; - - layer.EDRMetadata = [CAEDRMetadata HDR10MetadataWithDisplayInfo:displayData contentInfo:contentData opticalOutputScale:100.0f]; - - Log(LOG_I, @"EDRMetadata set from MDCV %@ and CLLI %@", displayData, contentData); +- (void)reportMaxEDRHeadroom { + CGFloat maxHeadroom = 1.0f; +#if TARGET_OS_OSX + maxHeadroom = [[NSScreen mainScreen] maximumPotentialExtendedDynamicRangeColorComponentValue]; +#else + maxHeadroom = [[UIScreen mainScreen] potentialEDRHeadroom]; +#endif + if (maxHeadroom > 1.0) { + LogOnce(LOG_I, @"Display supports EDR with a max headroom of %.1f", maxHeadroom); } else { + LogOnce(LOG_I, @"Display does not support EDR"); + } +} + +- (void)pollCurrentEDRHeadroom { + CGFloat headroom = 1.0f; +#if TARGET_OS_OSX + headroom = [[NSScreen mainScreen] maximumExtendedDynamicRangeColorComponentValue]; +#else + headroom = [[UIScreen mainScreen] currentEDRHeadroom]; +#endif + if (headroom != _currentEDRHeadroom) { + Log(LOG_I, @"EDR headroom changed to %.1f", headroom); + _currentEDRHeadroom = headroom; + } +} + +- (void)setInitialEDRMetadata { +} + +- (void)applyEDRFromFrame:(Frame *)frame withColorspace:(int)colorspace toLayer:(CAMetalLayer *)layer { + [self reportMaxEDRHeadroom]; + [self setInitialEDRMetadata]; + + CFDictionaryRef ext = [frame getFormatDescExtensions]; + CFStringRef frame_trc = CFDictionaryGetValue(ext, kCVImageBufferTransferFunctionKey); + + // These can only be changed on the main thread + dispatch_sync(dispatch_get_main_queue(), ^{ layer.wantsExtendedDynamicRangeContent = YES; layer.pixelFormat = MTLPixelFormatRGBA16Float; - CFStringRef name = kCGColorSpaceExtendedLinearITUR_2020; + + CFStringRef name; + switch (colorspace) { + case COLORSPACE_REC_2020: + name = kCGColorSpaceExtendedLinearITUR_2020; + break; + case COLORSPACE_REC_601: + name = kCGColorSpaceExtendedLinearSRGB; + break; + case COLORSPACE_REC_709: + default: + name = kCGColorSpaceExtendedLinearSRGB; + break; + } + CGColorSpaceRef colorspace = CGColorSpaceCreateWithName(name); layer.colorspace = colorspace; + CGColorSpaceRelease(colorspace); - layer.EDRMetadata = [CAEDRMetadata HDR10MetadataWithMinLuminance:0.0005f maxLuminance:1000.0f opticalOutputScale:100.0f]; - - Log(LOG_I, @"EDRMetadata set for 1000 nits"); - } + CFDataRef masteringDisplayColorVolume = CVBufferCopyAttachment(frame.pixelBuffer, kCVImageBufferMasteringDisplayColorVolumeKey, nil); + CFDataRef contentLightLevel = CVBufferCopyAttachment(frame.pixelBuffer, kCVImageBufferContentLightLevelInfoKey, nil); + if (masteringDisplayColorVolume) { + layer.EDRMetadata = [CAEDRMetadata HDR10MetadataWithDisplayInfo:(__bridge NSData *)masteringDisplayColorVolume + contentInfo:contentLightLevel ? (__bridge NSData *)contentLightLevel : nil + opticalOutputScale:100.0f]; + } else { + layer.EDRMetadata = [CAEDRMetadata HDR10MetadataWithMinLuminance:0.005f maxLuminance:1000.0f opticalOutputScale:100.0f]; + } + LogOnce(LOG_I, @"EDRMetadata set to colorspace %@, transfer function %@, %@", layer.colorspace, frame_trc, layer.EDRMetadata); + }); } #endif @@ -293,17 +328,21 @@ static const NSUInteger MaxFramesInFlight = 3; : [NSString stringWithFormat:@"Unknown: %lu", (unsigned long)layer.pixelFormat]); } - // These can only be changed on the main thread - dispatch_sync(dispatch_get_main_queue(), ^{ + if ([CAEDRMetadata isAvailable]) { + [self applyEDRFromFrame:frame withColorspace:colorspace toLayer:layer]; + } else { + // These can only be changed on the main thread + dispatch_sync(dispatch_get_main_queue(), ^{ #if !TARGET_OS_TV - if (isHDR) { - layer.wantsExtendedDynamicRangeContent = YES; - } + if (isHDR) { + layer.wantsExtendedDynamicRangeContent = YES; + } #endif - layer.colorspace = newColorSpace; - layer.pixelFormat = newPixelFormat; - }); - CGColorSpaceRelease(newColorSpace); + layer.colorspace = newColorSpace; + layer.pixelFormat = newPixelFormat; + }); + CGColorSpaceRelease(newColorSpace); + } } // Create the new colorspace parameter buffer for our fragment shader @@ -383,12 +422,13 @@ static const NSUInteger MaxFramesInFlight = 3; return YES; } -- (void)discardNextDrawable { - _nextDrawable = nil; -} +- (void)renderFrame:(Frame *)frame toLayer:(CAMetalLayer *)layer { + @autoreleasepool { + if (self.isStopping) { + Log(LOG_I, @"XXX Metal renderThread is stopping. returning from renderFrame"); + return; + } -- (void)renderFrame:(Frame *)frame toLayer:(CAMetalLayer *)layer -{ @autoreleasepool { // Handle changes to the frame's colorspace from last time we rendered BOOL layerDidChange = NO; if (![self updateColorSpaceForFrame:frame toLayer:layer layerDidChange:&layerDidChange]) { @@ -402,11 +442,6 @@ static const NSUInteger MaxFramesInFlight = 3; FQLog(LOG_I, @"[%d / %.3f ms] Metal frame rendering", frame.frameNumber, frame.pts); -#if !TARGET_OS_TV - // Experimental EDR handling based on frame metadata - //[self applyEDRFromFrame:frame toLayer:layer]; -#endif - size_t planes = CVPixelBufferGetPlaneCount(frame.pixelBuffer); assert(planes <= MAX_VIDEO_PLANES); @@ -419,23 +454,38 @@ static const NSUInteger MaxFramesInFlight = 3; if (!_videoPipelineState[planes]) { MTLRenderPipelineDescriptor *pipelineDesc = [MTLRenderPipelineDescriptor new]; id defaultLibrary = [_device newDefaultLibrary]; - pipelineDesc.vertexFunction = [defaultLibrary newFunctionWithName:@"vs_draw"]; - + + // RGB shaders + id vertexVsDraw = [defaultLibrary newFunctionWithName:@"vs_draw"]; + + // linear shaders + id yuvToLinear = [defaultLibrary newFunctionWithName:@"yuvToLinear"]; + // Determine if this is 10-bit based on the layer's pixel format (after colorspace update) BOOL is10Bit = (layer.pixelFormat == MTLPixelFormatBGR10A2Unorm); - Log(LOG_I, @"Layer pixel format: %lu, is10Bit: %@", (unsigned long)layer.pixelFormat, is10Bit ? @"YES" : @"NO"); - + NSString *fragmentShaderName; if (planes == 2) { fragmentShaderName = is10Bit ? @"ps_draw_biplanar_10bit" : @"ps_draw_biplanar_8bit"; } else { fragmentShaderName = is10Bit ? @"ps_draw_triplanar_10bit" : @"ps_draw_triplanar_8bit"; } - - Log(LOG_I, @"Rendering frame with %zu planes, using shader: %@", planes, fragmentShaderName); - pipelineDesc.fragmentFunction = [defaultLibrary newFunctionWithName:fragmentShaderName]; + pipelineDesc.colorAttachments[0].pixelFormat = layer.pixelFormat; pipelineDesc.vertexBuffers[0].mutability = MTLMutabilityImmutable; + + Log(LOG_I, @"Metal pipeline state for %zu planes with pixel format %@", + planes, layer.pixelFormat == MTLPixelFormatRGBA16Float ? @"MTLPixelFormatRGBA16Float" : @"MTLPixelFormatBGR10A2Unorm"); + + if (layer.pixelFormat == MTLPixelFormatRGBA16Float) { + // 4:2:0 or 4:4:4 YUV -> BT.2020 RGB -> linear float + pipelineDesc.vertexFunction = vertexVsDraw; + pipelineDesc.fragmentFunction = yuvToLinear; + } else { + // 4:2:0 or 4:4:4 YUV -> BT.2020 RGB + pipelineDesc.vertexFunction = vertexVsDraw; + pipelineDesc.fragmentFunction = [defaultLibrary newFunctionWithName:fragmentShaderName]; + } NSError *error = nil; _videoPipelineState[planes] = [_device newRenderPipelineStateWithDescriptor:pipelineDesc error:&error]; @@ -468,18 +518,15 @@ static const NSUInteger MaxFramesInFlight = 3; return; } - if (_cvMetalTextures[i]) { - CVBufferRelease(_cvMetalTextures[i]); - } - CVReturn err = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, - _textureCache, - frame.pixelBuffer, - NULL, - fmt, - CVPixelBufferGetWidthOfPlane(frame.pixelBuffer, i), - CVPixelBufferGetHeightOfPlane(frame.pixelBuffer, i), - i, - &_cvMetalTextures[i]); + CVReturn err = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, + _textureCache, + frame.pixelBuffer, + NULL, + fmt, + CVPixelBufferGetWidthOfPlane(frame.pixelBuffer, i), + CVPixelBufferGetHeightOfPlane(frame.pixelBuffer, i), + i, + &_cvMetalTextures[i]); if (err != kCVReturnSuccess) { Log(LOG_E, @"CVMetalTextureCacheCreateTextureFromImage() failed: %d", err); return; @@ -501,11 +548,45 @@ static const NSUInteger MaxFramesInFlight = 3; [renderEncoder setFragmentTexture:CVMetalTextureGetTexture(_cvMetalTextures[i]) atIndex:i]; } - [renderEncoder setFragmentBuffer:_CscParamsBuffer offset:0 atIndex:0]; +// if (layer.pixelFormat == MTLPixelFormatRGBA16Float) { +// [self pollCurrentEDRHeadroom]; +// [renderEncoder setFragmentBytes:&_currentEDRHeadroom length:sizeof(CGFloat) atIndex:0]; +// } + [renderEncoder setVertexBuffer:_VideoVertexBuffer offset:0 atIndex:0]; + [renderEncoder setFragmentBuffer:_CscParamsBuffer offset:0 atIndex:0]; [renderEncoder drawPrimitives:MTLPrimitiveTypeTriangleStrip vertexStart:0 vertexCount:4]; [renderEncoder endEncoding]; + __block MetalVideoRenderer *strongSelf = self; + [drawable addPresentedHandler:^(id d) { + if (strongSelf.lastPresented > 0.0f) { + CFTimeInterval frametime = d.presentedTime - strongSelf.lastPresented; + [[ImGuiPlots sharedInstance] observeFloat:PLOT_FRAMETIME value:(frametime * 1000.0)]; + } + strongSelf.lastPresented = d.presentedTime; + }]; + + // signal semaphore, compute GPU time average, and clear textures + __block dispatch_semaphore_t block_semaphore = _inFlightSemaphore; + [commandBuffer addCompletedHandler:^(id cb) { + dispatch_semaphore_signal(block_semaphore); + + const CFTimeInterval GPUTime = cb.GPUEndTime - cb.GPUStartTime; + const double alpha = 0.25f; + self->_averageGPUTime = (GPUTime * alpha) + (self->_averageGPUTime * (1.0 - alpha)); + + // Free textures after completion of rendering + for (size_t i = 0; i < planes; i++) { + if (self->_cvMetalTextures[i]) { + CFRelease(self->_cvMetalTextures[i]); + self->_cvMetalTextures[i] = nil; + } + } + + CVMetalTextureCacheFlush(self->_textureCache, 0); + }]; + #if TARGET_OS_SIMULATOR [commandBuffer presentDrawable:drawable]; #else @@ -515,39 +596,28 @@ static const NSUInteger MaxFramesInFlight = 3; [commandBuffer commit]; - __weak typeof(self) self_ = self; - [drawable addPresentedHandler:^(id d) { - if (self_) { - [self_ plotFrametime:d.presentedTime]; - } - }]; - - - const CFTimeInterval GPUTime = commandBuffer.GPUEndTime - commandBuffer.GPUStartTime; - const double alpha = 0.25f; - _averageGPUTime = (GPUTime * alpha) + (_averageGPUTime * (1.0 - alpha)); - - // Free textures after completion of rendering - for (size_t i = 0; i < planes; i++) { - if (_cvMetalTextures[i]) { - CVBufferRelease(_cvMetalTextures[i]); - _cvMetalTextures[i] = nil; - } - } - CVMetalTextureCacheFlush(_textureCache, 0); + // Wait for the command buffer to complete and free our CVMetalTextureCache references [commandBuffer waitUntilCompleted]; } } -- (void)plotFrametime:(CFTimeInterval)presentedTime { - if (_lastPresented > 0) { - CFTimeInterval frametime = presentedTime - _lastPresented; - [[ImGuiPlots sharedInstance] observeFloat:PLOT_FRAMETIME value:(frametime * 1000.0)]; +- (void)waitToRenderTo:(nonnull CAMetalLayer *)layer { + // Wait to ensure only `MaxFramesInFlight` number of frames are getting processed + // by any stage in the Metal pipeline (CPU, GPU, Metal, Drivers, etc.). + if (!self.isStopping) { + dispatch_time_t timeout = dispatch_time(DISPATCH_TIME_NOW, (int64_t)(0.1f * NSEC_PER_SEC)); // 100ms + dispatch_semaphore_wait(_inFlightSemaphore, timeout); } - _lastPresented = presentedTime; } -- (void)waitToRenderTo:(nonnull CAMetalLayer *)layer { +- (void)shutdown { + Log(LOG_I, @"XXX MetalVideoRenderer shutodwn"); + self.isStopping = YES; + + // Ensure no rendering is in flight + for (NSUInteger i = 0; i < MaxFramesInFlight; i++) { + dispatch_semaphore_signal(_inFlightSemaphore); + } } /// Responds to the drawable's size or orientation changes. @@ -558,7 +628,4 @@ static const NSUInteger MaxFramesInFlight = 3; - (void)resize:(CGSize)size { } -- (void)stop { -} - @end diff --git a/VoidLink/Metal/MetalView.h b/VoidLink/Metal/MetalView.h index bace20ea..39124ff4 100644 --- a/VoidLink/Metal/MetalView.h +++ b/VoidLink/Metal/MetalView.h @@ -17,6 +17,7 @@ - (void)drawableResize:(CGSize)size; - (void)renderTo:(nonnull CAMetalLayer *)layer; - (void)waitToRenderTo:(nonnull CAMetalLayer *)layer; +- (void)shutdown; @end @@ -29,12 +30,9 @@ @property (nonatomic) float framerate; - (void)initCommon; +- (void)shutdown; #if AUTOMATICALLY_RESIZE - (void)resizeDrawable:(CGFloat)scaleFactor; #endif -- (void)stop; -- (void)pause; -- (void)resume; - @end diff --git a/VoidLink/Metal/MetalView.m b/VoidLink/Metal/MetalView.m index d24b3521..87a95cbc 100644 --- a/VoidLink/Metal/MetalView.m +++ b/VoidLink/Metal/MetalView.m @@ -8,10 +8,6 @@ @implementation MetalView { // The secondary thread containing the render loop. NSThread *_renderThread; - - // The flag to indicate that rendering needs to cease on the main thread. - BOOL _continueRunLoop; - dispatch_semaphore_t _renderThreadSemaphore; } #pragma mark - Initialization and Setup. @@ -42,7 +38,17 @@ _metalLayer = (CAMetalLayer *)self.layer; self.layer.delegate = self; - _renderThreadSemaphore = dispatch_semaphore_create(0); +} + +- (void)shutdown { + if (_renderThread) { + [_renderThread cancel]; + // wait for thread to exist + while (!_renderThread.isFinished) { + Log(LOG_I, @"XXX MetalView waiting on renderThread to finish"); + usleep(100); + } + } } #if TARGET_OS_IOS || TARGET_OS_TV @@ -63,71 +69,52 @@ } #endif // END TARGET_OS_IOS || TARGET_OS_TV -- (void)startRenderThread { - @synchronized(self) { - // Don't start a new thread if one is already running - if (_renderThread) { - return; - } - - _continueRunLoop = YES; - _renderThread = [[NSThread alloc] initWithTarget:self selector:@selector(runThread) object:nil]; - _renderThread.qualityOfService = NSQualityOfServiceUserInteractive; - [_renderThread start]; - } -} - - (void)movedToWindow { - if (self.window) { - // The view was added to a window, so start rendering. - [self resume]; - - // Notify the delegate of the drawable's size. - CGSize defaultDrawableSize = self.bounds.size; - defaultDrawableSize.width *= self.layer.contentsScale; - defaultDrawableSize.height *= self.layer.contentsScale; - [self.delegate drawableResize:defaultDrawableSize]; - } else { - // The view was removed from a window, so stop rendering. - [self pause]; - } -} + if (!self.window) { + return; -- (void)pause { - // Pausing is implemented by simply stopping the render thread. - [self stop]; -} - -- (void)resume { - // Resuming is implemented by starting a new render thread. - [self startRenderThread]; -} - -- (void)runThread { - // The system sets the '_continueRunLoop' ivar outside this thread, so it needs to synchronize. Create a - // 'continueRunLoop' local var that the system can set from the _continueRunLoop ivar in a @synchronized block. - BOOL continueRunLoop = YES; - - // Begin the run loop. - while (continueRunLoop) { - @autoreleasepool { - [_delegate waitToRenderTo:_metalLayer]; - - @synchronized(self) { - continueRunLoop = _continueRunLoop; - } - if (!continueRunLoop) { - break; - } - - [_delegate renderTo:_metalLayer]; - - @synchronized(self) { - continueRunLoop = _continueRunLoop; + // We have been removed + if (_renderThread) { + [_renderThread cancel]; + // wait for thread to exist + while (!_renderThread.isFinished) { + Log(LOG_I, @"XXX MetalView waiting on renderThread to finish"); + usleep(100); } } + return; } - dispatch_semaphore_signal(self->_renderThreadSemaphore); + + // Render on a new thread + _renderThread = [[NSThread alloc] initWithBlock:^{ + while (![NSThread currentThread].isCancelled) { + @autoreleasepool { + [self.delegate waitToRenderTo:self.metalLayer]; + [self.delegate renderTo:self.metalLayer]; + } + } + Log(LOG_I, @"XXX Metal renderThread shutting down"); + }]; + _renderThread.name = @"MetalVideoRenderer"; + _renderThread.qualityOfService = NSQualityOfServiceUserInteractive; + [_renderThread start]; + + // Perform any actions that need to know the size and scale of the drawable. When UIKit calls + // didMoveToWindow after the view initialization, this is the first opportunity to notify + // components of the drawable's size. +#if AUTOMATICALLY_RESIZE +#if TARGET_OS_IOS || TARGET_OS_TV + [self resizeDrawable:self.window.screen.nativeScale]; +#else + [self resizeDrawable:self.window.screen.backingScaleFactor]; +#endif +#else + // Notify the delegate of the default drawable size when the system can calculate it. + CGSize defaultDrawableSize = self.bounds.size; + defaultDrawableSize.width *= self.layer.contentsScale; + defaultDrawableSize.height *= self.layer.contentsScale; + [self.delegate drawableResize:defaultDrawableSize]; +#endif } #pragma mark - Resizing @@ -198,25 +185,4 @@ } #endif // END AUTOMATICALLY_RESIZE -- (void)stop { - @synchronized(self) { - // If already stopping, do nothing. - if (!_continueRunLoop) { - return; - } - _continueRunLoop = NO; - } - - if (_renderThread && _renderThread != [NSThread currentThread]) { - // Wait for the render thread to finish its loop and signal the semaphore. - // We use a 1-second timeout to prevent the app from hanging indefinitely - // if the thread gets stuck for some reason. - long timeoutResult = dispatch_semaphore_wait(_renderThreadSemaphore, dispatch_time(DISPATCH_TIME_NOW, 1 * NSEC_PER_SEC)); - if (timeoutResult != 0) { - Log(LOG_E, @"MetalView render thread failed to stop gracefully within 1 second."); - } - } - - _renderThread = nil; -} @end diff --git a/VoidLink/Metal/MetalViewController.h b/VoidLink/Metal/MetalViewController.h index 1610cd5b..76903d64 100644 --- a/VoidLink/Metal/MetalViewController.h +++ b/VoidLink/Metal/MetalViewController.h @@ -21,8 +21,4 @@ enableHdr:(BOOL)enableHdr metricsHandler:(MetricsHandler _Nonnull)metricsHandler; -- (void)stop; -- (void)pause; -- (void)resume; - @end diff --git a/VoidLink/Metal/MetalViewController.m b/VoidLink/Metal/MetalViewController.m index b8f47c73..135746b8 100644 --- a/VoidLink/Metal/MetalViewController.m +++ b/VoidLink/Metal/MetalViewController.m @@ -11,15 +11,12 @@ The implementation of the cross-platform game view controller. #import "MetalVideoRenderer.h" @implementation MetalViewController { - /// A queue to initialize the renderer asynchronously from the main thread. - dispatch_queue_t _dispatch_queue; FrameQueue *_frameQueue; float _framerate; BOOL _enableHdr; MetalView *_metalView; MetalVideoRenderer *_renderer; MetricsHandler _metricsHandler; - BOOL _stopping; } - (nonnull instancetype)initWithFrame:(CGRect)bounds framerate:(float)framerate enableHdr:(BOOL)enableHdr metricsHandler:(MetricsHandler)metricsHandler { @@ -30,8 +27,6 @@ The implementation of the cross-platform game view controller. _framerate = framerate; _enableHdr = enableHdr; _metricsHandler = metricsHandler; - _stopping = NO; - [_frameQueue clear]; } return self; } @@ -43,9 +38,6 @@ The implementation of the cross-platform game view controller. - (void)viewDidLoad { [super viewDidLoad]; - /// A queue to initialize the renderer asynchronously from the main thread. - _dispatch_queue = dispatch_queue_create("com.moonlight.Metal", DISPATCH_QUEUE_CONCURRENT); - __block MetalView *view = (MetalView *)self.view; if (!view) { Log(LOG_E, @"The view attached to MetalViewController isn't a MetalView."); @@ -75,34 +67,27 @@ The implementation of the cross-platform game view controller. // Initialize the renderer-dependent view properties. view.metalLayer.pixelFormat = renderer.colorPixelFormat; - view.metalLayer.colorspace = renderer.colorspace; view.metalLayer.maximumDrawableCount = 3; self->_renderer = renderer; } - (void)waitToRenderTo:(nonnull CAMetalLayer *)layer { - if (!_stopping) { - // Renderer obtains a nextDrawable, waiting if necessary - [_renderer waitToRenderTo:layer]; + // Renderer obtains a nextDrawable, waiting if necessary + [_renderer waitToRenderTo:layer]; - // If we don't have a frame yet, wait on that too - if (!_stopping) { - [_frameQueue waitForEnqueue]; - } - } + // If we don't have a frame yet, wait on that too + [_frameQueue waitForEnqueue]; } /// Draw frame (used by manual loop) - (void)renderTo:(nonnull CAMetalLayer *)layer { - if (!_renderer) { - return; - } - - CFTimeInterval timeout = (1.0f / _framerate) - _renderer.averageGPUTime; - Frame *frame = [_frameQueue dequeueWithTimeout:timeout]; - if (frame) { - [_renderer renderFrame:frame toLayer:layer]; + if (!_renderer.isStopping) { + CFTimeInterval timeout = (1.0f / _framerate) - _renderer.averageGPUTime; + Frame *frame = [_frameQueue dequeueWithTimeout:timeout]; + if (frame) { + [_renderer renderFrame:frame toLayer:layer]; + } } } @@ -110,6 +95,18 @@ The implementation of the cross-platform game view controller. [_renderer drawableResize:size]; } +- (void)shutdown { + [_renderer shutdown]; +} + +- (void)viewDidDisappear:(BOOL)animated { + [super viewDidDisappear:animated]; + + Log(LOG_I, @"XXX MetalViewController viewDidDisappear"); + + [_metalView shutdown]; +} + #if TARGET_OS_IOS /// Hides the Home indicator button automatically. - (BOOL)prefersHomeIndicatorAutoHidden { @@ -140,51 +137,4 @@ The implementation of the cross-platform game view controller. } #endif -- (void)stop { - // 1. Signal that we are stopping to prevent new work from starting. - _stopping = YES; - - // 2. Invalidate the CAMetalLayer by removing its device. - // This should cause any blocking calls like `nextDrawable` on the render thread to fail - // and return immediately, breaking the deadlock. - if (_metalView) { - _metalView.metalLayer.device = nil; - } - - // 3. Unblock the render thread from any other potential waiting points. - if (_renderer) { - [_renderer stop]; - } - [_frameQueue stop]; - - // 4. Now that the thread is unblocked, wait for it to finish its execution. - if (_metalView) { - [_metalView stop]; - } - - // 5. Once the thread has terminated, it's safe to deallocate all resources. - _renderer = nil; - _metalView = nil; -} - -- (void)pause { - if (_renderer) { - // Add this line to discard any stale drawable before pausing the thread. - // This forces the renderer to get a fresh one on resume. - [_renderer discardNextDrawable]; - } - - if (_metalView) { - [_metalView pause]; - Log(LOG_I, @"Metal rendering paused."); - } -} - -- (void)resume { - if (_metalView) { - [_metalView resume]; - Log(LOG_I, @"Metal rendering resumed."); - } -} - @end diff --git a/VoidLink/Metal/Shaders-Linear.metal b/VoidLink/Metal/Shaders-Linear.metal new file mode 100644 index 00000000..e7275a97 --- /dev/null +++ b/VoidLink/Metal/Shaders-Linear.metal @@ -0,0 +1,84 @@ +#include +#include + +using namespace metal; + +struct Vertex +{ + float4 position [[ position ]]; + float2 texCoords; +}; + +struct CscParams +{ + float3 matrix[3]; + float3 offsets; +}; + +// PQ (SMPTE ST 2084) constants for inverse EOTF +constant float PQ_M1 = 0.1593017578125; // 2610/16384 +constant float PQ_M2 = 78.84375; // 2523/32 * 1000/1000 +constant float PQ_C1 = 0.8359375; // 3424/4096 +constant float PQ_C2 = 18.8515625; // 2413/128 +constant float PQ_C3 = 18.6875; // 2392/128 + +// BT.2020 to Rec.709/sRGB color space conversion matrix +constant float3x3 bt2020_to_rec709 = float3x3( + float3( 1.7166511, -0.3556708, -0.2533663), + float3(-0.6666844, 1.6164812, 0.0157685), + float3( 0.0176399, -0.0427706, 0.9421031) +); + +constexpr sampler s(coord::normalized, address::clamp_to_edge, filter::linear); + +// Convert from PQ curve to linear light +float pq_to_linear(float pq) { + if (pq <= 0.0) return 0.0; + + float pq_pow_inv_m2 = pow(pq, 1.0 / PQ_M2); + float numerator = max(pq_pow_inv_m2 - PQ_C1, 0.0); + float denominator = PQ_C2 - PQ_C3 * pq_pow_inv_m2; + + if (denominator <= 0.0) return 0.0; + + return pow(numerator / denominator, 1.0 / PQ_M1); +} + +// Apply PQ inverse EOTF to RGB components +float3 pq_to_linear_rgb(float3 pq_rgb) { + return float3( + pq_to_linear(pq_rgb.r), + pq_to_linear(pq_rgb.g), + pq_to_linear(pq_rgb.b) + ); +} + +fragment float4 yuvToLinear(Vertex v [[ stage_in ]], + constant CscParams &cscParams [[ buffer(0) ]], + texture2d luminancePlane [[ texture(0) ]], + texture2d chrominancePlane [[ texture(1) ]]) +{ + float3 yuv = float3(luminancePlane.sample(s, v.texCoords).r, + chrominancePlane.sample(s, v.texCoords).rg); + yuv -= cscParams.offsets; + + float3 rgb; + rgb.r = dot(yuv, cscParams.matrix[0]); + rgb.g = dot(yuv, cscParams.matrix[1]); + rgb.b = dot(yuv, cscParams.matrix[2]); + + // Clamp RGB to valid range [0, 1] + rgb = clamp(rgb, 0.0, 1.0); + + // Apply PQ inverse EOTF to convert from gamma-encoded to linear light + // This converts from 0-1 PQ range to 0-10000 nits linear + float3 linear_rgb = pq_to_linear_rgb(rgb); + + // Scale for EDR (1.0 = 100 nits SDR white) + linear_rgb = linear_rgb * 100.0; + + // TODO: support tonemapping to Rec.709 for non-HDR viewers + //linear_rgb = bt2020_to_rec709 * linear_rgb; + + return float4(linear_rgb, 1.0f); +} diff --git a/VoidLink/Metal/Shaders.metal b/VoidLink/Metal/Shaders.metal index 8167c469..5fd1adf6 100644 --- a/VoidLink/Metal/Shaders.metal +++ b/VoidLink/Metal/Shaders.metal @@ -119,9 +119,3 @@ fragment float4 ps_draw_triplanar_10bit(Vertex v [[ stage_in ]], rgb.b = dot(yuv_corrected, cscParams.matrix[2]); return float4(rgb, 1.0f); } - -fragment float4 ps_draw_rgb(Vertex v [[ stage_in ]], - texture2d rgbTexture [[ texture(0) ]]) -{ - return rgbTexture.sample(s, v.texCoords); -} \ No newline at end of file diff --git a/VoidLink/Stream/StreamManager.m b/VoidLink/Stream/StreamManager.m index 3fd32341..50f1e42f 100644 --- a/VoidLink/Stream/StreamManager.m +++ b/VoidLink/Stream/StreamManager.m @@ -192,15 +192,15 @@ double avgVideoMbps = [_connection getBwTracker].averageMbps; double peakVideoMbps = [_connection getBwTracker].peakMbps; - + if(overlayLevel == 1) return [LocalizationHelper localizedStringForKey:@"FPS: %5.2f Network dropped frames: %.2f%% Network latency: %@", stats.totalFrames / interval, stats.networkDroppedFrames / interval, latencyString]; else return [LocalizationHelper localizedStringForKey:@"Video stream: %dx%d %.2f FPS (Codec: %@)\n" - "Bitrate: %.1f Mbps, Peak: %.1f, Renderer: %@\n" + "Bitrate: %.1f Mbps, Peak: %.1f, Frames buffered: %.1f\n" "%@" - "Frames buffered: %.1f, Present \n" + "Renderer: %@\n" "Frames dropped by network/pacing jitter: %.1f%% / %.1f%%\n" "Average network latency: %@\n" "Decode time: %.2f/%.2f/%.2f ms", @@ -208,9 +208,9 @@ _config.height, fps, [_connection getActiveCodecName], - avgVideoMbps, peakVideoMbps, (stats.renderingBackend == RENDER_METAL) ? @"Metal" : @"AVSampleBuffer", + avgVideoMbps, peakVideoMbps, stats.frameQueueMetrics.avg, hostProcessingString, - stats.frameQueueMetrics.avg, + stats.renderingBackendString, (stats.networkDroppedFrames / stats.totalFrames) * 100.0, stats.frameDropMetrics.nsamples > 0 ? (stats.frameDropMetrics.total / stats.frameDropMetrics.nsamples) * 100.0 : 0.0f, latencyString, diff --git a/VoidLink/Stream/VideoDecoderRenderer.m b/VoidLink/Stream/VideoDecoderRenderer.m index 0542de4b..1b8eec60 100644 --- a/VoidLink/Stream/VideoDecoderRenderer.m +++ b/VoidLink/Stream/VideoDecoderRenderer.m @@ -201,7 +201,11 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size, } #endif - NSDictionary *destinationPixelBufferAttributes = @{(id)kCVPixelBufferPixelFormatTypeKey : pixelFormat}; + NSDictionary *destinationPixelBufferAttributes = @{ + (id)kCVPixelBufferPixelFormatTypeKey : pixelFormat, + (id)kVTDecompressionPropertyKey_GeneratePerFrameHDRDisplayMetadata : @YES, + (id)kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder : @YES + }; return [self setupDecompressionSessionWithAttributes:destinationPixelBufferAttributes]; } @@ -344,6 +348,8 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit); } - (void)cleanup { + [_frameQueue shutdown]; + if (_renderingBackend == RENDER_AVSB) { [_displayLink invalidate]; } @@ -607,7 +613,8 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit); formatDesc = NULL; } - Log(LOG_I, @"AV1 extensions: %@, format description: %@", extensions, formatDesc); + LogOnce(LOG_I, @"AV1 extensions: %@", extensions); + LogOnce(LOG_I, @"AV1 format description: %@", formatDesc); ff_cbs_fragment_free(&cbsFrag); ff_cbs_close(&cbsCtx); @@ -674,6 +681,8 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit); _formatDesc = NULL; } + LogOnce(LOG_I, @"H264 format description: %@", _formatDesc); + // Free parameter set buffers after submission [_parameterSetBuffers removeAllObjects]; } @@ -713,6 +722,8 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit); _formatDesc = NULL; } + LogOnce(LOG_I, @"HEVC format description: %@", _formatDesc); + // Free parameter set buffers after submission [_parameterSetBuffers removeAllObjects]; } @@ -1003,8 +1014,39 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit); } - (void)getAllStats:(video_stats_t *)stats { + if (_renderingBackend == RENDER_METAL) { +#if TARGET_OS_OSX + float edrHeadroom = [[NSScreen mainScreen] maximumExtendedDynamicRangeColorComponentValue]; +#else + float edrHeadroom = [[UIScreen mainScreen] currentEDRHeadroom]; + UIScreenReferenceDisplayModeStatus referenceStatus = [[UIScreen mainScreen] referenceDisplayModeStatus]; +#endif + if (edrHeadroom > 1.0) { + NSString *ref; + // Device has a reference display that may or may not be enabled + switch (referenceStatus) { + case UIScreenReferenceDisplayModeStatusLimited: + ref = @"(Reference mode limited),"; + break; + case UIScreenReferenceDisplayModeStatusEnabled: + ref = @"(Reference mode),"; + break; + default: + ref = @","; + break; + } + int peakNits = 1000; + stats->renderingBackendString = [NSString stringWithFormat:@"Metal, EDR %.1f %@ tone-mapped: %d nits", + edrHeadroom, ref, peakNits]; + } else { + // if HDR + stats->renderingBackendString = [NSString stringWithFormat:@"Metal, tone-mapped: HDR->sRGB"]; + } + } else { + stats->renderingBackendString = @"AVSampleBuffer"; + } + dispatch_sync(_sq, ^{ - stats->renderingBackend = _renderingBackend; memcpy(&stats->decodeMetrics, &_decodeMetrics, sizeof(PlotMetrics)); memcpy(&stats->frameQueueMetrics, &_frameQueueMetrics, sizeof(PlotMetrics)); [_frameQueue.frameDropMetrics copyMetrics:&stats->frameDropMetrics]; diff --git a/VoidLink/Utility/FloatBuffer.m b/VoidLink/Utility/FloatBuffer.m index 9a537c54..8722f834 100644 --- a/VoidLink/Utility/FloatBuffer.m +++ b/VoidLink/Utility/FloatBuffer.m @@ -36,7 +36,7 @@ _count = 0; _minValue = 0.0f; _maxValue = 0.0f; - _sum = 0.0; + _sum = 0.0f; _sq = dispatch_queue_create("com.floatbuffer.serial", DISPATCH_QUEUE_SERIAL); } diff --git a/VoidLink/Utility/FrameQueue.h b/VoidLink/Utility/FrameQueue.h index 06a47385..4230e67d 100644 --- a/VoidLink/Utility/FrameQueue.h +++ b/VoidLink/Utility/FrameQueue.h @@ -12,6 +12,7 @@ NS_ASSUME_NONNULL_BEGIN @property (nonatomic) FloatBuffer *frameDropMetrics; @property (nonatomic) int highWaterMark; @property (nonatomic, readonly) int maxCapacity; +@property (atomic) BOOL isStopping; + (instancetype)sharedInstance; @@ -26,7 +27,7 @@ NS_ASSUME_NONNULL_BEGIN - (CFTimeInterval)estimatedFramerate; - (int)currentSoftCap; - (void)waitForEnqueue; -- (void)stop; +- (void)shutdown; @end diff --git a/VoidLink/Utility/FrameQueue.m b/VoidLink/Utility/FrameQueue.m index f9db0c77..a6ac6cd1 100644 --- a/VoidLink/Utility/FrameQueue.m +++ b/VoidLink/Utility/FrameQueue.m @@ -16,7 +16,6 @@ int _count; BOOL _droppedLast; - BOOL _stopping; int _framesIn; CMTime _ptsCorrection; os_unfair_lock _lock; @@ -61,6 +60,7 @@ _ptsCorrection = CMTimeMake(0, 90000); _queueSizeHistory = [[FloatBuffer alloc] initWithCapacity:64]; _lock = OS_UNFAIR_LOCK_INIT; + _isStopping = NO; // ring buffer _capacity = _maxCapacity; @@ -76,7 +76,6 @@ // ping estimatedFramerate to set initial last value [self estimatedFramerate]; - _stopping = NO; } return self; } @@ -173,10 +172,6 @@ // enqueue with simple alternate-drop logic - (int)enqueue:(Frame *)frame { os_unfair_lock_lock(&_lock); - if (_stopping) { - os_unfair_lock_unlock(&_lock); - return 0; // Don't enqueue if stopping - } int dropCount = [self _unsafeEnqueue:frame withDropTarget:_highWaterMark]; os_unfair_lock_unlock(&_lock); return dropCount; @@ -185,10 +180,6 @@ // enqueue that is a bit more flexixble, using the same 500ms queue size history method as moonlight-qt. - (int)enqueue:(Frame *)frame withSlackSize:(int)slack { os_unfair_lock_lock(&_lock); - if (_stopping) { - os_unfair_lock_unlock(&_lock); - return 0; // Don't enqueue if stopping - } CFTimeInterval now = CACurrentMediaTime(); // new data point for queue health @@ -235,14 +226,9 @@ // Allows the render loop to wait if the queue is empty - (void)waitForEnqueue { - while ([self isEmpty]) { - if (_stopping) { - return; - } - dispatch_semaphore_wait(_frameSemaphore, DISPATCH_TIME_FOREVER); - if (_stopping) { - return; - } + while (!self.isStopping && [self isEmpty]) { + dispatch_time_t timeout = dispatch_time(DISPATCH_TIME_NOW, (int64_t)(0.1f * NSEC_PER_SEC)); // 100ms + dispatch_semaphore_wait(_frameSemaphore, timeout); } } @@ -270,11 +256,12 @@ CFTimeInterval deadline = start + timeout; int round = 0; + if (self.isStopping) { + return nil; + } + // Always attempt to dequeue at least once do { - if (_stopping) { - return nil; - } if (round > 0) { usleep(100); // 0.1ms } @@ -304,7 +291,6 @@ os_unfair_lock_lock(&_lock); _head = _tail = _count = 0; _frameDropMetrics = [[FloatBuffer alloc] initWithCapacity:512]; - _stopping = NO; os_unfair_lock_unlock(&_lock); } @@ -338,6 +324,13 @@ return cap; } +- (void)shutdown { + // new frames will no longer be coming in, make sure consumer side is not left waiting + self.isStopping = YES; + Log(LOG_I, @"XXX FrameQueue shutting down"); + dispatch_semaphore_signal(_frameSemaphore); +} + // For use with NSLog("%@", franeQueue); - (NSString *)description { __block NSMutableArray *parts = [NSMutableArray arrayWithCapacity:_count]; @@ -347,29 +340,4 @@ return [NSString stringWithFormat:@"[%@]", [parts componentsJoinedByString:@",\n"]]; } -- (void)stop { - os_unfair_lock_lock(&_lock); - // Prevent redundant calls - if (_stopping) { - os_unfair_lock_unlock(&_lock); - return; - } - - _stopping = YES; - - // Clear the buffer's contents - _head = _tail = _count = 0; - for (int i = 0; i < _capacity; i++) { - [_buffer replaceObjectAtIndex:i withObject:[NSNull null]]; - } - - os_unfair_lock_unlock(&_lock); - - // Unblock any threads waiting on the semaphore. - // Signaling multiple times ensures all potential consumer threads are released. - for (int i = 0; i < _maxCapacity; i++) { - dispatch_semaphore_signal(_frameSemaphore); - } -} - @end diff --git a/VoidLink/Utility/Logger.m b/VoidLink/Utility/Logger.m index b448b505..ef3546ba 100644 --- a/VoidLink/Utility/Logger.m +++ b/VoidLink/Utility/Logger.m @@ -11,7 +11,9 @@ #if defined(NDEBUG) static LogLevel LoggerLogLevel = LOG_I; #else -static LogLevel LoggerLogLevel = LOG_D; +// Debug is too spammy during discovery +// static LogLevel LoggerLogLevel = LOG_D; +static LogLevel LoggerLogLevel = LOG_I; #endif void LogTagv(LogLevel level, NSString* tag, NSString* fmt, va_list args) { diff --git a/VoidLink/Utility/Plot.h b/VoidLink/Utility/Plot.h index 2c4af42d..a6e28982 100644 --- a/VoidLink/Utility/Plot.h +++ b/VoidLink/Utility/Plot.h @@ -52,5 +52,5 @@ typedef struct { PlotMetrics decodeMetrics; PlotMetrics frameQueueMetrics; PlotMetrics frameDropMetrics; - RenderingBackend renderingBackend; + NSString *renderingBackendString; } video_stats_t; diff --git a/VoidLink/ViewControllers/StreamFrameViewController.m b/VoidLink/ViewControllers/StreamFrameViewController.m index b511ef6e..56bd4aeb 100644 --- a/VoidLink/ViewControllers/StreamFrameViewController.m +++ b/VoidLink/ViewControllers/StreamFrameViewController.m @@ -745,7 +745,6 @@ _inactivityTimer = nil; } if (self.metalViewController) { - [self.metalViewController stop]; [self.metalViewController.view removeFromSuperview]; self.metalViewController = nil; NSLog(@"Metal renderer stopped and cleaned up."); @@ -926,11 +925,6 @@ // This will fire if the user opens control center or gets a low battery message - (void)applicationWillResignActive:(NSNotification *)notification { - if (self.metalViewController) { - [self.metalViewController pause]; - } - - //[self.pipController startPictureInPicture]; //sleep(1); @@ -947,9 +941,6 @@ } - (void)applicationDidBecomeActive:(NSNotification *)notification { - if (self.metalViewController) { - [self.metalViewController resume]; - } // Stop the background timer, since we're foregrounded again if (_inactivityTimer != nil) { Log(LOG_I, @"Stopping inactivity timer after becoming active again");