feat: pip for metal renderer

VoidLink/Stream/StreamManager.h

@@ -16,6 +16,8 @@
 
 @property (nonatomic, strong, readonly) VideoDecoderRenderer *videoRenderer;
 
+@property (nonatomic, strong) AVSampleBufferDisplayLayer *pipLayer;
+
 - (id) initWithConfig:(StreamConfiguration*)config renderView:(UIView*)view connectionCallbacks:(id<ConnectionCallbacks>)callback;
 
 - (void) stopStream;

VoidLink/Stream/StreamManager.m

@@ -106,7 +106,7 @@
     // Initializing the renderer must be done on the main thread
     dispatch_async(dispatch_get_main_queue(), ^{
         self->_videoRenderer = [[VideoDecoderRenderer alloc] initWithView:self->_renderView callbacks:self->_callbacks streamAspectRatio:(float)self->_config.width / (float)self->_config.height];
-
+        self->_videoRenderer.pipLayer = self.pipLayer;
         self->_connection = [[Connection alloc] initWithConfig:self->_config renderer:self->_videoRenderer connectionCallbacks:self->_callbacks];
         NSOperationQueue* opQueue = [[NSOperationQueue alloc] init];
         [opQueue addOperation:self->_connection];

VoidLink/Stream/VideoDecoderRenderer.h

@@ -20,6 +20,7 @@
 @property (atomic, readonly) PlotMetrics frameQueueMetrics;
 
 @property (nonatomic, strong, readonly) AVSampleBufferDisplayLayer *displayLayer;
+@property (nonatomic, weak) AVSampleBufferDisplayLayer *pipLayer;
 
 - (id)initWithView:(UIView*)view callbacks:(id<ConnectionCallbacks>)callbacks streamAspectRatio:(float)aspectRatio;
 

VoidLink/Stream/VideoDecoderRenderer.m

@@ -47,7 +47,8 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size,
     NSData *_masteringDisplayColorVolume;
     NSData *_contentLightLevelInfo;
     CMVideoFormatDescriptionRef _formatDesc;
-    CMVideoFormatDescriptionRef _formatDescImageBuffer;
+    CMVideoFormatDescriptionRef _pipFormatDesc;
+    CMVideoFormatDescriptionRef _avsbFormatDesc;
     VTDecompressionSessionRef _decompressionSession;
 
     CADisplayLink *_displayLink;
@@ -55,6 +56,7 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size,
     NSInteger _maxRefreshRate;
     RenderingBackend _renderingBackend;
     
+    BOOL _hasSentVideoContentShown;
 }
 
 - (void)reinitializeDisplayLayer
@@ -93,9 +95,13 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size,
         _formatDesc = nil;
     }
     
-    if (_formatDescImageBuffer != nil) {
+    if (_pipFormatDesc != nil) {
-        CFRelease(_formatDescImageBuffer);
+        CFRelease(_pipFormatDesc);
-        _formatDescImageBuffer = nil;
+        _pipFormatDesc = nil;
+    }
+    if (_avsbFormatDesc != nil) {
+        CFRelease(_avsbFormatDesc);
+        _avsbFormatDesc = nil;
     }
     
     if (_decompressionSession != nil){
@@ -329,9 +335,6 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit);
     if (frame.frameType == FRAME_TYPE_IDR) {
         // Ensure the layer is visible now
         self->_displayLayer.hidden = NO;
-
-        // Tell our parent VC to hide the progress indicator
-        [self->_callbacks videoContentShown];
     }
 }
 
@@ -822,6 +825,12 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit);
     [self setupDecompressionSession];
   }
     
+  if (frameType == FRAME_TYPE_IDR && !_hasSentVideoContentShown) {
+      // Tell our parent VC to hide the progress indicator and set up PiP
+      [_callbacks videoContentShown];
+      _hasSentVideoContentShown = YES;
+  }
+
   OSStatus status = VTDecompressionSessionDecodeFrameWithOutputHandler(
       _decompressionSession,
       sampleBuffer,
@@ -843,71 +852,116 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit);
             LiRequestIdrFrame(); // Request an IDR to restart the stream
             return;
           }
+          if (self->_renderingBackend == RENDER_METAL && self.pipLayer) {
+//              NSLog(@"[Metal PiP Path] Entered for frame number %d.", frameNumber);
+              CVPixelBufferRetain(imageBuffer);
+              dispatch_async(dispatch_get_main_queue(), ^{
+//                  NSLog(@"[Metal PiP Path] Executing on main thread for frame %d.", frameNumber);
+                  @synchronized(self) {
+                      if (self.pipLayer.controlTimebase == NULL) {
+                          NSLog(@"[Metal PiP Path] Timebase is NULL. Setting it now.");
+                          CMTimebaseRef timebase;
+                          CMTimebaseCreateWithSourceClock(kCFAllocatorDefault, CMClockGetHostTimeClock(), &timebase);
+                          if (timebase) {
+                              CMTimebaseSetTime(timebase, presentationTimestamp);
+                              CMTimebaseSetRate(timebase, 1.0);
+                              [self.pipLayer setControlTimebase:timebase];
+                              CFRelease(timebase);
+                          }
+                      }
                       
-          CMSampleBufferRef sampleBufferOut = nil;
+                      if (self->_pipFormatDesc == NULL || !CMVideoFormatDescriptionMatchesImageBuffer(self->_pipFormatDesc, imageBuffer)) {
-          CVPixelBufferRef pixelBuffer = nil;
+                          NSLog(@"[Metal PiP Path] Format description mismatch or NULL. Creating new one.");
+                          if (self->_pipFormatDesc) { CFRelease(self->_pipFormatDesc); }
+                          OSStatus res = CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, imageBuffer, &(self->_pipFormatDesc));
+                          if (res == noErr) {
+                              NSLog(@"[Metal PiP Path] New format description created: %@", self->_pipFormatDesc);
+                          } else {
+                              NSLog(@"[Metal PiP Path] FAILED to create format description. Error: %d", (int)res);
+                          }
+                      }
                       
-          // AVSampleBuffer path: package into a SampleBuffer
+                      if (self->_pipFormatDesc) {
-          if (self->_renderingBackend == RENDER_AVSB) {
+                          CMSampleBufferRef sampleBufferForPip = NULL;
-            if (self->_formatDescImageBuffer == NULL || !CMVideoFormatDescriptionMatchesImageBuffer(self->_formatDescImageBuffer, imageBuffer)) {
+                          CMSampleTimingInfo sampleTiming = {kCMTimeInvalid, presentationTimestamp, presentationDuration};
-              OSStatus res = CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, imageBuffer, &(self->_formatDescImageBuffer));
+                          OSStatus err = CMSampleBufferCreateReadyWithImageBuffer(kCFAllocatorDefault, imageBuffer, self->_pipFormatDesc, &sampleTiming, &sampleBufferForPip);
-              if (res != noErr) {
+                          if (err == noErr && sampleBufferForPip) {
-                Log(LOG_E, @"Failed to create video format description from imageBuffer");
+//                              NSLog(@"[Metal PiP Path] Sample buffer created. Enqueuing to _pipLayer.");
-                return;
+                              [self.pipLayer enqueueSampleBuffer:sampleBufferForPip];
-              }
+                              CFRelease(sampleBufferForPip);
-            }
+                          } else {
-
+                              NSLog(@"[Metal PiP Path] FAILED to create sample buffer. Error: %d", (int)err);
-            if (self->_formatDescImageBuffer == NULL || !CMVideoFormatDescriptionMatchesImageBuffer(self->_formatDescImageBuffer, imageBuffer)) {
+                          }
-              OSStatus res = CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, imageBuffer, &(self->_formatDescImageBuffer));
+                      }
-              if (res != noErr) {
+                  }
-                Log(LOG_E, @"Failed to create video format description from imageBuffer");
+                  CVPixelBufferRelease(imageBuffer);
-                return;
+              });
-              }
-            }
-
-            CMSampleTimingInfo sampleTiming = {kCMTimeInvalid, presentationTimestamp, presentationDuration};
-
-            OSStatus err =
-                CMSampleBufferCreateReadyWithImageBuffer(kCFAllocatorDefault, imageBuffer, self->_formatDescImageBuffer, &sampleTiming, &sampleBufferOut);
-            if (err != noErr) {
-              Log(LOG_E, @"Error creating sample buffer for decompressed image buffer %d", (int)err);
-              return;
-            }
-          } else if (self->_renderingBackend == RENDER_METAL) {
-            // Metal path: retain the pixelBuffer here so it survives the dispatch
-            pixelBuffer = CVPixelBufferRetain((CVPixelBufferRef)imageBuffer);
           }
+          // --- MAIN RENDERER PATH ---
+          if (self->_renderingBackend == RENDER_AVSB) {
+//              self.pipLayer.hidden = YES;
+              if (self->_avsbFormatDesc == NULL || !CMVideoFormatDescriptionMatchesImageBuffer(self->_avsbFormatDesc, imageBuffer)) {
+                  if (self->_avsbFormatDesc) {
+                      CFRelease(self->_avsbFormatDesc);
+                  }
+                  CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, imageBuffer, &(self->_avsbFormatDesc));
+              }
               
-          // Dispatch onto our higher priority queue
+              CMSampleBufferRef sampleBufferForAvsb = NULL;
-          dispatch_async(self->_vtq, ^{
+              CMSampleTimingInfo sampleTiming = {kCMTimeInvalid, presentationTimestamp, presentationDuration};
-              Frame *frame = nil;
+              OSStatus err = CMSampleBufferCreateReadyWithImageBuffer(kCFAllocatorDefault, imageBuffer, self->_avsbFormatDesc, &sampleTiming, &sampleBufferForAvsb);
-              if (self->_renderingBackend == RENDER_AVSB) {
+              
-                frame = [[Frame alloc] initWithSampleBuffer:sampleBufferOut frameNumber:frameNumber frameType:frameType];
+              if (err == noErr && sampleBufferForAvsb) {
+                  dispatch_async(self->_vtq, ^{
+                      Frame *frame = [[Frame alloc] initWithSampleBuffer:sampleBufferForAvsb frameNumber:frameNumber frameType:frameType];
+                      int framesDropped = [self->_frameQueue enqueue:frame withSlackSize:3];
+                      // ... stats and metrics updates
+                  });
               } else {
-                frame = [[Frame alloc] initWithPixelBufffer:pixelBuffer frameNumber:frameNumber frameType:frameType pts:presentationTimestamp];
+                  if (sampleBufferForAvsb) {
-                [frame setFormatDesc:self->_formatDesc];
+                      CFRelease(sampleBufferForAvsb);
+                  }
               }
-              int framesDropped = [self->_frameQueue enqueue:frame withSlackSize:3];
+          } else if (self->_renderingBackend == RENDER_METAL) {
-
+              self.pipLayer.hidden = NO;
-              static PlotMetrics frameQueueMetrics = {};
+              // Safely capture the format description for the background block.
-              [[ImGuiPlots sharedInstance] observeFloatReturnMetrics:PLOT_QUEUED_FRAMES value:[self->_frameQueue count] plotMetrics:&frameQueueMetrics];
+              __block CMVideoFormatDescriptionRef capturedFormatDesc = NULL;
-              [self safeCopyMetricsTo:&self->_frameQueueMetrics from:&frameQueueMetrics];
+              @synchronized(self) {
-
+                  if (self->_formatDesc) {
-              [[ImGuiPlots sharedInstance] observeFloat:PLOT_DROPPED value:framesDropped];
+                      capturedFormatDesc = (CMVideoFormatDescriptionRef)CFRetain(self->_formatDesc);
-
+                  }
-              // It's important we capture host metrics on the incoming thread, as this frame object
-              // may have been dropped by the above enqueue
-              static CFTimeInterval lastHostFrame = 0.0f;
-              if (lastHostFrame != 0) {
-                [[ImGuiPlots sharedInstance] observeFloat:PLOT_HOST_FRAMETIME value:(frame.pts - lastHostFrame) * 1000.0];
               }
-              lastHostFrame = frame.pts;
               
-              // Decode time is not graphed because it is marked as hidden, but we can use the same mechanism for the value used by stats
+              if (!capturedFormatDesc) {
-              static PlotMetrics decodeMetrics = {};
+                  Log(LOG_E, @"Failed to capture format description for Metal frame, dropping.");
-              [[ImGuiPlots sharedInstance] observeFloatReturnMetrics:PLOT_DECODE value:(CACurrentMediaTime() - decodeStartTime) * 1000.0 plotMetrics:&decodeMetrics];
+                  return;
-              [self safeCopyMetricsTo:&self->_decodeMetrics from:&decodeMetrics];
+              }
-          });
+              
+              CVPixelBufferRef pixelBuffer = CVPixelBufferRetain((CVPixelBufferRef)imageBuffer);
+              dispatch_async(self->_vtq, ^{
+                  Frame *frame = [[Frame alloc] initWithPixelBufffer:pixelBuffer frameNumber:frameNumber frameType:frameType pts:presentationTimestamp];
+                  [frame setFormatDesc:capturedFormatDesc];
+                  CFRelease(capturedFormatDesc);
+                  
+                  int framesDropped = [self->_frameQueue enqueue:frame withSlackSize:3];
+                  
+                  static PlotMetrics frameQueueMetrics = {};
+                  [[ImGuiPlots sharedInstance] observeFloatReturnMetrics:PLOT_QUEUED_FRAMES value:[self->_frameQueue count] plotMetrics:&frameQueueMetrics];
+                  [self safeCopyMetricsTo:&self->_frameQueueMetrics from:&frameQueueMetrics];
+                  
+                  [[ImGuiPlots sharedInstance] observeFloat:PLOT_DROPPED value:framesDropped];
+                  
+                  static CFTimeInterval lastHostFrame = 0.0f;
+                  if (lastHostFrame != 0) {
+                      [[ImGuiPlots sharedInstance] observeFloat:PLOT_HOST_FRAMETIME value:(frame.pts - lastHostFrame) * 1000.0];
+                  }
+                  lastHostFrame = frame.pts;
+                  
+                  static PlotMetrics decodeMetrics = {};
+                  [[ImGuiPlots sharedInstance] observeFloatReturnMetrics:PLOT_DECODE value:(CACurrentMediaTime() - decodeStartTime) * 1000.0 plotMetrics:&decodeMetrics];
+                  [self safeCopyMetricsTo:&self->_decodeMetrics from:&decodeMetrics];
+              });
+          }
       });
 
   return status;
@@ -1082,6 +1136,32 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit);
         // Resume the display link immediately.
         _displayLink.paused = NO;
     }
+    else if (_renderingBackend == RENDER_METAL) {
+        Log(LOG_I, @"Resetting video pipeline for Metal renderer after resuming...");
+
+        // 1. Reset the receiver (the PiP Layer) on the main thread
+        dispatch_async(dispatch_get_main_queue(), ^{
+            if (self.pipLayer) {
+                // Flush stale buffers
+                [self.pipLayer flushAndRemoveImage];
+
+                // Destroy the old, stuck timebase
+                [self.pipLayer setControlTimebase:NULL];
+            }
+        });
+
+        // 2. Reset the sender (the Decompression Session)
+        @synchronized(self) {
+            if (_decompressionSession != nil) {
+                VTDecompressionSessionInvalidate(_decompressionSession);
+                CFRelease(_decompressionSession);
+                _decompressionSession = nil;
+            }
+        }
+        
+        // 3. Request a new keyframe to restart the stream cleanly
+        LiRequestIdrFrame();
+    }
 }
 
 @end

VoidLink/ViewControllers/StreamFrameViewController.m

@@ -80,6 +80,7 @@
     dispatch_block_t _delayedRemoveExtScreen;
     VideoDecoderRenderer *_videoRenderer;
     BOOL _isRestoringFromPiP;
+    AVSampleBufferDisplayLayer *_pipLayer;
 #if !TARGET_OS_TV
     CustomEdgeSlideGestureRecognizer *_slideToSettingsRecognizer;
     CustomEdgeSlideGestureRecognizer *_slideToCmdToolRecognizer;
@@ -183,28 +184,35 @@
     }
     Log(LOG_I, @"Setting up PiP controller...");
     
-    if (!videoRenderer || !videoRenderer.displayLayer) {
+    AVSampleBufferDisplayLayer* sourceLayer;
-        Log(LOG_E, @"PiP setup failed: Video renderer or display layer not ready.");
+    if ([_settings.renderingBackend intValue] == RENDER_METAL) {
+        sourceLayer = _pipLayer;
+        NSLog(@"[PiP LOG] Backend: Metal. Using _pipLayer: %p", sourceLayer);
+        if (!sourceLayer) {
+            NSLog(@"[PiP LOG] ERROR: _pipLayer is nil during setup.");
+            return;
+        }
+    } else { // RENDER_AVSB
+        sourceLayer = videoRenderer.displayLayer;
+        NSLog(@"[PiP LOG] Backend: AVSB. Using _displayLayer: %p", sourceLayer);
+    }
+
+    if (!_pipLayer) {
+        Log(LOG_E, @"PiP setup failed: Dedicated PiP layer not ready.");
         return;
     }
 
-    AVSampleBufferDisplayLayer *streamLayer = videoRenderer.displayLayer;
-
     if ([AVPictureInPictureController isPictureInPictureSupported]) {
         if (@available(iOS 15.0, *)) {
-            self.pipContentSource = [[AVPictureInPictureControllerContentSource alloc] initWithSampleBufferDisplayLayer:streamLayer playbackDelegate:(id<AVPictureInPictureSampleBufferPlaybackDelegate>)self];
+            self.pipContentSource = [[AVPictureInPictureControllerContentSource alloc] initWithSampleBufferDisplayLayer:sourceLayer playbackDelegate:(id<AVPictureInPictureSampleBufferPlaybackDelegate>)self];
             self.pipController = [[AVPictureInPictureController alloc] initWithContentSource:self.pipContentSource];
+            self.pipController.requiresLinearPlayback = YES;
             self.pipController.canStartPictureInPictureAutomaticallyFromInline = YES;
-        } else {
-            Log(LOG_E, @"PiP not fully supported on this device.");
-            return;
-        }
-
-        if (self.pipController) {
             self.pipController.delegate = self;
-            Log(LOG_I, @"PiP controller created successfully.");
+            NSLog(@"[PiP LOG] PiP Controller created successfully with source layer: %@", sourceLayer);
         } else {
-            Log(LOG_E, @"Failed to create PiP controller.");
+            Log(LOG_E, @"PiP is not fully supported on this iOS version.");
+            return;
         }
     } else {
         Log(LOG_E, @"PiP not supported on this device.");
@@ -215,10 +223,10 @@
     if (self.pipController) {
         self.pipController.delegate = nil;
         self.pipController = nil;
+        _pipLayer = nil; // Clean up the layer
         if (@available(iOS 15.0, *)) {
             self.pipContentSource = nil;
         }
-
         Log(LOG_I, @"PiP controller cleaned up.");
     }
 }
@@ -616,9 +624,15 @@
     _tipLabel.textAlignment = NSTextAlignmentCenter;
     _tipLabel.center = CGPointMake(self.view.frame.size.width / 2, self.view.frame.size.height * 0.9);
     
+    _pipLayer = [[AVSampleBufferDisplayLayer alloc] init];
+    _pipLayer.videoGravity = AVLayerVideoGravityResizeAspect;
+    _pipLayer.frame = self.view.bounds;
+    _pipLayer.hidden = YES;
+    [self.view.layer addSublayer:_pipLayer];
     _streamMan = [[StreamManager alloc] initWithConfig:self.streamConfig
                                             renderView:_streamVideoRenderView
                                    connectionCallbacks:self];
+    _streamMan.pipLayer = _pipLayer;
     NSOperationQueue* opQueue = [[NSOperationQueue alloc] init];
     [opQueue addOperation:_streamMan];
     
@@ -709,7 +723,14 @@
 }
 
 - (void)enterPip{
-    [self.pipController startPictureInPicture];
+    NSLog(@"[PiP LOG] enterPip called.");
+    if (self.pipController) {
+        NSLog(@"[PiP LOG] Controller exists. Firing startPictureInPicture.");
+        NSLog(@"[PiP LOG] Layer state before start: hidden=%d, frame=%@, timebase=%@", _pipLayer.hidden, NSStringFromCGRect(_pipLayer.frame), _pipLayer.controlTimebase);
+        [self.pipController startPictureInPicture];
+    } else {
+        NSLog(@"[PiP LOG] ERROR: enterPip called, but pipController is nil!");
+    }
 }
 
 - (void)oscLayoutClosed{
@@ -957,12 +978,13 @@
         _inactivityTimer = nil;
     }
     
+    [self->_streamMan.videoRenderer resetFramePacing];
+
     // Check if we were in PiP
     if (self.pipController && self.pipController.isPictureInPictureActive) {
         [self.pipController stopPictureInPicture];
     }
     
-    [self->_streamMan.videoRenderer resetFramePacing];
     _isRestoringFromPiP = NO;
 }