diff --git a/VoidLink/Stream/StreamManager.h b/VoidLink/Stream/StreamManager.h index ad95b6f0..50345496 100644 --- a/VoidLink/Stream/StreamManager.h +++ b/VoidLink/Stream/StreamManager.h @@ -16,8 +16,6 @@ @property (nonatomic, strong, readonly) VideoDecoderRenderer *videoRenderer; -@property (nonatomic, strong) AVSampleBufferDisplayLayer *pipLayer; - - (id) initWithConfig:(StreamConfiguration*)config renderView:(UIView*)view connectionCallbacks:(id)callback; - (void) stopStream; diff --git a/VoidLink/Stream/StreamManager.m b/VoidLink/Stream/StreamManager.m index 011d5d56..3fd32341 100644 --- a/VoidLink/Stream/StreamManager.m +++ b/VoidLink/Stream/StreamManager.m @@ -106,7 +106,7 @@ // Initializing the renderer must be done on the main thread dispatch_async(dispatch_get_main_queue(), ^{ self->_videoRenderer = [[VideoDecoderRenderer alloc] initWithView:self->_renderView callbacks:self->_callbacks streamAspectRatio:(float)self->_config.width / (float)self->_config.height]; - self->_videoRenderer.pipLayer = self.pipLayer; + self->_connection = [[Connection alloc] initWithConfig:self->_config renderer:self->_videoRenderer connectionCallbacks:self->_callbacks]; NSOperationQueue* opQueue = [[NSOperationQueue alloc] init]; [opQueue addOperation:self->_connection]; diff --git a/VoidLink/Stream/VideoDecoderRenderer.h b/VoidLink/Stream/VideoDecoderRenderer.h index 3888d97d..afc36a47 100644 --- a/VoidLink/Stream/VideoDecoderRenderer.h +++ b/VoidLink/Stream/VideoDecoderRenderer.h @@ -20,7 +20,6 @@ @property (atomic, readonly) PlotMetrics frameQueueMetrics; @property (nonatomic, strong, readonly) AVSampleBufferDisplayLayer *displayLayer; -@property (nonatomic, weak) AVSampleBufferDisplayLayer *pipLayer; - (id)initWithView:(UIView*)view callbacks:(id)callbacks streamAspectRatio:(float)aspectRatio; diff --git a/VoidLink/Stream/VideoDecoderRenderer.m b/VoidLink/Stream/VideoDecoderRenderer.m index dbf8fe05..0542de4b 100644 --- a/VoidLink/Stream/VideoDecoderRenderer.m +++ b/VoidLink/Stream/VideoDecoderRenderer.m @@ -47,16 +47,14 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size, NSData *_masteringDisplayColorVolume; NSData *_contentLightLevelInfo; CMVideoFormatDescriptionRef _formatDesc; - CMVideoFormatDescriptionRef _pipFormatDesc; - CMVideoFormatDescriptionRef _avsbFormatDesc; + CMVideoFormatDescriptionRef _formatDescImageBuffer; VTDecompressionSessionRef _decompressionSession; CADisplayLink *_displayLink; FrameQueue *_frameQueue; NSInteger _maxRefreshRate; RenderingBackend _renderingBackend; - - BOOL _hasSentVideoContentShown; + } - (void)reinitializeDisplayLayer @@ -95,13 +93,9 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size, _formatDesc = nil; } - if (_pipFormatDesc != nil) { - CFRelease(_pipFormatDesc); - _pipFormatDesc = nil; - } - if (_avsbFormatDesc != nil) { - CFRelease(_avsbFormatDesc); - _avsbFormatDesc = nil; + if (_formatDescImageBuffer != nil) { + CFRelease(_formatDescImageBuffer); + _formatDescImageBuffer = nil; } if (_decompressionSession != nil){ @@ -338,6 +332,9 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit); if (frame.frameType == FRAME_TYPE_IDR) { // Ensure the layer is visible now self->_displayLayer.hidden = NO; + + // Tell our parent VC to hide the progress indicator + [self->_callbacks videoContentShown]; } } @@ -827,12 +824,6 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit); if (frameType == FRAME_TYPE_IDR || _decompressionSession == nil) { [self setupDecompressionSession]; } - - if (frameType == FRAME_TYPE_IDR && !_hasSentVideoContentShown) { - // Tell our parent VC to hide the progress indicator and set up PiP - [_callbacks videoContentShown]; - _hasSentVideoContentShown = YES; - } OSStatus status = VTDecompressionSessionDecodeFrameWithOutputHandler( _decompressionSession, @@ -855,116 +846,71 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit); LiRequestIdrFrame(); // Request an IDR to restart the stream return; } - if (self->_renderingBackend == RENDER_METAL && self.pipLayer) { -// NSLog(@"[Metal PiP Path] Entered for frame number %d.", frameNumber); - CVPixelBufferRetain(imageBuffer); - dispatch_async(dispatch_get_main_queue(), ^{ -// NSLog(@"[Metal PiP Path] Executing on main thread for frame %d.", frameNumber); - @synchronized(self) { - if (self.pipLayer.controlTimebase == NULL) { - NSLog(@"[Metal PiP Path] Timebase is NULL. Setting it now."); - CMTimebaseRef timebase; - CMTimebaseCreateWithSourceClock(kCFAllocatorDefault, CMClockGetHostTimeClock(), &timebase); - if (timebase) { - CMTimebaseSetTime(timebase, presentationTimestamp); - CMTimebaseSetRate(timebase, 1.0); - [self.pipLayer setControlTimebase:timebase]; - CFRelease(timebase); - } - } - - if (self->_pipFormatDesc == NULL || !CMVideoFormatDescriptionMatchesImageBuffer(self->_pipFormatDesc, imageBuffer)) { - NSLog(@"[Metal PiP Path] Format description mismatch or NULL. Creating new one."); - if (self->_pipFormatDesc) { CFRelease(self->_pipFormatDesc); } - OSStatus res = CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, imageBuffer, &(self->_pipFormatDesc)); - if (res == noErr) { - NSLog(@"[Metal PiP Path] New format description created: %@", self->_pipFormatDesc); - } else { - NSLog(@"[Metal PiP Path] FAILED to create format description. Error: %d", (int)res); - } - } - - if (self->_pipFormatDesc) { - CMSampleBufferRef sampleBufferForPip = NULL; - CMSampleTimingInfo sampleTiming = {kCMTimeInvalid, presentationTimestamp, presentationDuration}; - OSStatus err = CMSampleBufferCreateReadyWithImageBuffer(kCFAllocatorDefault, imageBuffer, self->_pipFormatDesc, &sampleTiming, &sampleBufferForPip); - if (err == noErr && sampleBufferForPip) { -// NSLog(@"[Metal PiP Path] Sample buffer created. Enqueuing to _pipLayer."); - [self.pipLayer enqueueSampleBuffer:sampleBufferForPip]; - CFRelease(sampleBufferForPip); - } else { - NSLog(@"[Metal PiP Path] FAILED to create sample buffer. Error: %d", (int)err); - } - } - } - CVPixelBufferRelease(imageBuffer); - }); - } - // --- MAIN RENDERER PATH --- + + CMSampleBufferRef sampleBufferOut = nil; + CVPixelBufferRef pixelBuffer = nil; + + // AVSampleBuffer path: package into a SampleBuffer if (self->_renderingBackend == RENDER_AVSB) { -// self.pipLayer.hidden = YES; - if (self->_avsbFormatDesc == NULL || !CMVideoFormatDescriptionMatchesImageBuffer(self->_avsbFormatDesc, imageBuffer)) { - if (self->_avsbFormatDesc) { - CFRelease(self->_avsbFormatDesc); - } - CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, imageBuffer, &(self->_avsbFormatDesc)); + if (self->_formatDescImageBuffer == NULL || !CMVideoFormatDescriptionMatchesImageBuffer(self->_formatDescImageBuffer, imageBuffer)) { + OSStatus res = CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, imageBuffer, &(self->_formatDescImageBuffer)); + if (res != noErr) { + Log(LOG_E, @"Failed to create video format description from imageBuffer"); + return; } - - CMSampleBufferRef sampleBufferForAvsb = NULL; - CMSampleTimingInfo sampleTiming = {kCMTimeInvalid, presentationTimestamp, presentationDuration}; - OSStatus err = CMSampleBufferCreateReadyWithImageBuffer(kCFAllocatorDefault, imageBuffer, self->_avsbFormatDesc, &sampleTiming, &sampleBufferForAvsb); - - if (err == noErr && sampleBufferForAvsb) { - dispatch_async(self->_vtq, ^{ - Frame *frame = [[Frame alloc] initWithSampleBuffer:sampleBufferForAvsb frameNumber:frameNumber frameType:frameType]; - int framesDropped = [self->_frameQueue enqueue:frame withSlackSize:3]; - // ... stats and metrics updates - }); - } else { - if (sampleBufferForAvsb) { - CFRelease(sampleBufferForAvsb); - } + } + + if (self->_formatDescImageBuffer == NULL || !CMVideoFormatDescriptionMatchesImageBuffer(self->_formatDescImageBuffer, imageBuffer)) { + OSStatus res = CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, imageBuffer, &(self->_formatDescImageBuffer)); + if (res != noErr) { + Log(LOG_E, @"Failed to create video format description from imageBuffer"); + return; } + } + + CMSampleTimingInfo sampleTiming = {kCMTimeInvalid, presentationTimestamp, presentationDuration}; + + OSStatus err = + CMSampleBufferCreateReadyWithImageBuffer(kCFAllocatorDefault, imageBuffer, self->_formatDescImageBuffer, &sampleTiming, &sampleBufferOut); + if (err != noErr) { + Log(LOG_E, @"Error creating sample buffer for decompressed image buffer %d", (int)err); + return; + } } else if (self->_renderingBackend == RENDER_METAL) { - self.pipLayer.hidden = NO; - // Safely capture the format description for the background block. - __block CMVideoFormatDescriptionRef capturedFormatDesc = NULL; - @synchronized(self) { - if (self->_formatDesc) { - capturedFormatDesc = (CMVideoFormatDescriptionRef)CFRetain(self->_formatDesc); - } - } - - if (!capturedFormatDesc) { - Log(LOG_E, @"Failed to capture format description for Metal frame, dropping."); - return; - } - - CVPixelBufferRef pixelBuffer = CVPixelBufferRetain((CVPixelBufferRef)imageBuffer); - dispatch_async(self->_vtq, ^{ - Frame *frame = [[Frame alloc] initWithPixelBufffer:pixelBuffer frameNumber:frameNumber frameType:frameType pts:presentationTimestamp]; - [frame setFormatDesc:capturedFormatDesc]; - CFRelease(capturedFormatDesc); - - int framesDropped = [self->_frameQueue enqueue:frame withSlackSize:3]; - - static PlotMetrics frameQueueMetrics = {}; - [[ImGuiPlots sharedInstance] observeFloatReturnMetrics:PLOT_QUEUED_FRAMES value:[self->_frameQueue count] plotMetrics:&frameQueueMetrics]; - [self safeCopyMetricsTo:&self->_frameQueueMetrics from:&frameQueueMetrics]; - - [[ImGuiPlots sharedInstance] observeFloat:PLOT_DROPPED value:framesDropped]; - - static CFTimeInterval lastHostFrame = 0.0f; - if (lastHostFrame != 0) { - [[ImGuiPlots sharedInstance] observeFloat:PLOT_HOST_FRAMETIME value:(frame.pts - lastHostFrame) * 1000.0]; - } - lastHostFrame = frame.pts; - - static PlotMetrics decodeMetrics = {}; - [[ImGuiPlots sharedInstance] observeFloatReturnMetrics:PLOT_DECODE value:(CACurrentMediaTime() - decodeStartTime) * 1000.0 plotMetrics:&decodeMetrics]; - [self safeCopyMetricsTo:&self->_decodeMetrics from:&decodeMetrics]; - }); + // Metal path: retain the pixelBuffer here so it survives the dispatch + pixelBuffer = CVPixelBufferRetain((CVPixelBufferRef)imageBuffer); } + + // Dispatch onto our higher priority queue + dispatch_async(self->_vtq, ^{ + Frame *frame = nil; + if (self->_renderingBackend == RENDER_AVSB) { + frame = [[Frame alloc] initWithSampleBuffer:sampleBufferOut frameNumber:frameNumber frameType:frameType]; + } else { + frame = [[Frame alloc] initWithPixelBufffer:pixelBuffer frameNumber:frameNumber frameType:frameType pts:presentationTimestamp]; + [frame setFormatDesc:self->_formatDesc]; + } + int framesDropped = [self->_frameQueue enqueue:frame withSlackSize:3]; + + static PlotMetrics frameQueueMetrics = {}; + [[ImGuiPlots sharedInstance] observeFloatReturnMetrics:PLOT_QUEUED_FRAMES value:[self->_frameQueue count] plotMetrics:&frameQueueMetrics]; + [self safeCopyMetricsTo:&self->_frameQueueMetrics from:&frameQueueMetrics]; + + [[ImGuiPlots sharedInstance] observeFloat:PLOT_DROPPED value:framesDropped]; + + // It's important we capture host metrics on the incoming thread, as this frame object + // may have been dropped by the above enqueue + static CFTimeInterval lastHostFrame = 0.0f; + if (lastHostFrame != 0) { + [[ImGuiPlots sharedInstance] observeFloat:PLOT_HOST_FRAMETIME value:(frame.pts - lastHostFrame) * 1000.0]; + } + lastHostFrame = frame.pts; + + // Decode time is not graphed because it is marked as hidden, but we can use the same mechanism for the value used by stats + static PlotMetrics decodeMetrics = {}; + [[ImGuiPlots sharedInstance] observeFloatReturnMetrics:PLOT_DECODE value:(CACurrentMediaTime() - decodeStartTime) * 1000.0 plotMetrics:&decodeMetrics]; + [self safeCopyMetricsTo:&self->_decodeMetrics from:&decodeMetrics]; + }); }); return status; @@ -1123,11 +1069,11 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit); // Ensure this only runs for the AVSampleBuffer rendering backend and that the display link exists. if (_renderingBackend == RENDER_AVSB && _displayLink) { Log(LOG_I, @"Frame pacing is being reset to %d FPS...", self->_frameRate); - + // Toggling the paused state can help re-engage the display link with the // run loop correctly after the app resumes from a background state like PiP. _displayLink.paused = YES; - + // Re-apply the desired frame rate range. This is the critical hint for ProMotion // that may have been lost or ignored during the PiP transition. if (@available(iOS 15.0, *)) { @@ -1135,25 +1081,10 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit); } else { _displayLink.preferredFramesPerSecond = self->_frameRate; } - + // Resume the display link immediately. _displayLink.paused = NO; - } - else if (_renderingBackend == RENDER_METAL) { - Log(LOG_I, @"Resetting video pipeline for Metal renderer after resuming..."); - - // 1. Reset the receiver (the PiP Layer) on the main thread - dispatch_async(dispatch_get_main_queue(), ^{ - if (self.pipLayer) { - // Flush stale buffers - [self.pipLayer flushAndRemoveImage]; - - // Destroy the old, stuck timebase - [self.pipLayer setControlTimebase:NULL]; - } - }); - - // 2. Reset the sender (the Decompression Session) + } else if (_renderingBackend == RENDER_METAL) { @synchronized(self) { if (_decompressionSession != nil) { VTDecompressionSessionInvalidate(_decompressionSession); @@ -1161,8 +1092,6 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit); _decompressionSession = nil; } } - - // 3. Request a new keyframe to restart the stream cleanly LiRequestIdrFrame(); } } diff --git a/VoidLink/ViewControllers/StreamFrameViewController.m b/VoidLink/ViewControllers/StreamFrameViewController.m index 51541778..b511ef6e 100644 --- a/VoidLink/ViewControllers/StreamFrameViewController.m +++ b/VoidLink/ViewControllers/StreamFrameViewController.m @@ -80,7 +80,6 @@ dispatch_block_t _delayedRemoveExtScreen; VideoDecoderRenderer *_videoRenderer; BOOL _isRestoringFromPiP; - AVSampleBufferDisplayLayer *_pipLayer; #if !TARGET_OS_TV CustomEdgeSlideGestureRecognizer *_slideToSettingsRecognizer; CustomEdgeSlideGestureRecognizer *_slideToCmdToolRecognizer; @@ -183,37 +182,30 @@ return; } Log(LOG_I, @"Setting up PiP controller..."); - - AVSampleBufferDisplayLayer* sourceLayer; - if ([_settings.renderingBackend intValue] == RENDER_METAL) { - sourceLayer = _pipLayer; - NSLog(@"[PiP LOG] Backend: Metal. Using _pipLayer: %p", sourceLayer); - if (!sourceLayer) { - NSLog(@"[PiP LOG] ERROR: _pipLayer is nil during setup."); - return; - } - } else { // RENDER_AVSB - sourceLayer = videoRenderer.displayLayer; - NSLog(@"[PiP LOG] Backend: AVSB. Using _displayLayer: %p", sourceLayer); - } - if (!_pipLayer) { - Log(LOG_E, @"PiP setup failed: Dedicated PiP layer not ready."); + if (!videoRenderer || !videoRenderer.displayLayer) { + Log(LOG_E, @"PiP setup failed: Video renderer or display layer not ready."); return; } + AVSampleBufferDisplayLayer *streamLayer = videoRenderer.displayLayer; + if ([AVPictureInPictureController isPictureInPictureSupported]) { if (@available(iOS 15.0, *)) { - self.pipContentSource = [[AVPictureInPictureControllerContentSource alloc] initWithSampleBufferDisplayLayer:sourceLayer playbackDelegate:(id)self]; + self.pipContentSource = [[AVPictureInPictureControllerContentSource alloc] initWithSampleBufferDisplayLayer:streamLayer playbackDelegate:(id)self]; self.pipController = [[AVPictureInPictureController alloc] initWithContentSource:self.pipContentSource]; - self.pipController.requiresLinearPlayback = YES; self.pipController.canStartPictureInPictureAutomaticallyFromInline = YES; - self.pipController.delegate = self; - NSLog(@"[PiP LOG] PiP Controller created successfully with source layer: %@", sourceLayer); } else { - Log(LOG_E, @"PiP is not fully supported on this iOS version."); + Log(LOG_E, @"PiP not fully supported on this device."); return; } + + if (self.pipController) { + self.pipController.delegate = self; + Log(LOG_I, @"PiP controller created successfully."); + } else { + Log(LOG_E, @"Failed to create PiP controller."); + } } else { Log(LOG_E, @"PiP not supported on this device."); } @@ -223,10 +215,10 @@ if (self.pipController) { self.pipController.delegate = nil; self.pipController = nil; - _pipLayer = nil; // Clean up the layer if (@available(iOS 15.0, *)) { self.pipContentSource = nil; } + Log(LOG_I, @"PiP controller cleaned up."); } } @@ -624,15 +616,9 @@ _tipLabel.textAlignment = NSTextAlignmentCenter; _tipLabel.center = CGPointMake(self.view.frame.size.width / 2, self.view.frame.size.height * 0.9); - _pipLayer = [[AVSampleBufferDisplayLayer alloc] init]; - _pipLayer.videoGravity = AVLayerVideoGravityResizeAspect; - _pipLayer.frame = self.view.bounds; - _pipLayer.hidden = YES; - [self.view.layer addSublayer:_pipLayer]; _streamMan = [[StreamManager alloc] initWithConfig:self.streamConfig renderView:_streamVideoRenderView connectionCallbacks:self]; - _streamMan.pipLayer = _pipLayer; NSOperationQueue* opQueue = [[NSOperationQueue alloc] init]; [opQueue addOperation:_streamMan]; @@ -723,14 +709,7 @@ } - (void)enterPip{ - NSLog(@"[PiP LOG] enterPip called."); - if (self.pipController) { - NSLog(@"[PiP LOG] Controller exists. Firing startPictureInPicture."); - NSLog(@"[PiP LOG] Layer state before start: hidden=%d, frame=%@, timebase=%@", _pipLayer.hidden, NSStringFromCGRect(_pipLayer.frame), _pipLayer.controlTimebase); - [self.pipController startPictureInPicture]; - } else { - NSLog(@"[PiP LOG] ERROR: enterPip called, but pipController is nil!"); - } + [self.pipController startPictureInPicture]; } - (void)oscLayoutClosed{ @@ -977,14 +956,13 @@ [_inactivityTimer invalidate]; _inactivityTimer = nil; } - - [self->_streamMan.videoRenderer resetFramePacing]; // Check if we were in PiP if (self.pipController && self.pipController.isPictureInPictureActive) { [self.pipController stopPictureInPicture]; } + [self->_streamMan.videoRenderer resetFramePacing]; _isRestoringFromPiP = NO; }