Revert "feat: pip for metal renderer"

This reverts commit fa7710ce44.

VoidLink/Stream/StreamManager.h

@@ -16,8 +16,6 @@
 
 @property (nonatomic, strong, readonly) VideoDecoderRenderer *videoRenderer;
 
-@property (nonatomic, strong) AVSampleBufferDisplayLayer *pipLayer;
-
 - (id) initWithConfig:(StreamConfiguration*)config renderView:(UIView*)view connectionCallbacks:(id<ConnectionCallbacks>)callback;
 
 - (void) stopStream;

VoidLink/Stream/StreamManager.m

@@ -106,7 +106,7 @@
     // Initializing the renderer must be done on the main thread
     dispatch_async(dispatch_get_main_queue(), ^{
         self->_videoRenderer = [[VideoDecoderRenderer alloc] initWithView:self->_renderView callbacks:self->_callbacks streamAspectRatio:(float)self->_config.width / (float)self->_config.height];
-        self->_videoRenderer.pipLayer = self.pipLayer;
+
         self->_connection = [[Connection alloc] initWithConfig:self->_config renderer:self->_videoRenderer connectionCallbacks:self->_callbacks];
         NSOperationQueue* opQueue = [[NSOperationQueue alloc] init];
         [opQueue addOperation:self->_connection];

VoidLink/Stream/VideoDecoderRenderer.h

@@ -20,7 +20,6 @@
 @property (atomic, readonly) PlotMetrics frameQueueMetrics;
 
 @property (nonatomic, strong, readonly) AVSampleBufferDisplayLayer *displayLayer;
-@property (nonatomic, weak) AVSampleBufferDisplayLayer *pipLayer;
 
 - (id)initWithView:(UIView*)view callbacks:(id<ConnectionCallbacks>)callbacks streamAspectRatio:(float)aspectRatio;
 

VoidLink/Stream/VideoDecoderRenderer.m

@@ -47,16 +47,14 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size,
     NSData *_masteringDisplayColorVolume;
     NSData *_contentLightLevelInfo;
     CMVideoFormatDescriptionRef _formatDesc;
-    CMVideoFormatDescriptionRef _pipFormatDesc;
+    CMVideoFormatDescriptionRef _formatDescImageBuffer;
-    CMVideoFormatDescriptionRef _avsbFormatDesc;
     VTDecompressionSessionRef _decompressionSession;
 
     CADisplayLink *_displayLink;
     FrameQueue *_frameQueue;
     NSInteger _maxRefreshRate;
     RenderingBackend _renderingBackend;
-    
+
-    BOOL _hasSentVideoContentShown;
 }
 
 - (void)reinitializeDisplayLayer
@@ -95,13 +93,9 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size,
         _formatDesc = nil;
     }
     
-    if (_pipFormatDesc != nil) {
+    if (_formatDescImageBuffer != nil) {
-        CFRelease(_pipFormatDesc);
+        CFRelease(_formatDescImageBuffer);
-        _pipFormatDesc = nil;
+        _formatDescImageBuffer = nil;
-    }
-    if (_avsbFormatDesc != nil) {
-        CFRelease(_avsbFormatDesc);
-        _avsbFormatDesc = nil;
     }
     
     if (_decompressionSession != nil){
@@ -338,6 +332,9 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit);
     if (frame.frameType == FRAME_TYPE_IDR) {
         // Ensure the layer is visible now
         self->_displayLayer.hidden = NO;
+
+        // Tell our parent VC to hide the progress indicator
+        [self->_callbacks videoContentShown];
     }
 }
 
@@ -827,12 +824,6 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit);
   if (frameType == FRAME_TYPE_IDR || _decompressionSession == nil) {
     [self setupDecompressionSession];
   }
-    
-  if (frameType == FRAME_TYPE_IDR && !_hasSentVideoContentShown) {
-      // Tell our parent VC to hide the progress indicator and set up PiP
-      [_callbacks videoContentShown];
-      _hasSentVideoContentShown = YES;
-  }
 
   OSStatus status = VTDecompressionSessionDecodeFrameWithOutputHandler(
       _decompressionSession,
@@ -855,116 +846,71 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit);
             LiRequestIdrFrame(); // Request an IDR to restart the stream
             return;
           }
-          if (self->_renderingBackend == RENDER_METAL && self.pipLayer) {
+
-//              NSLog(@"[Metal PiP Path] Entered for frame number %d.", frameNumber);
+          CMSampleBufferRef sampleBufferOut = nil;
-              CVPixelBufferRetain(imageBuffer);
+          CVPixelBufferRef pixelBuffer = nil;
-              dispatch_async(dispatch_get_main_queue(), ^{
+
-//                  NSLog(@"[Metal PiP Path] Executing on main thread for frame %d.", frameNumber);
+          // AVSampleBuffer path: package into a SampleBuffer
-                  @synchronized(self) {
-                      if (self.pipLayer.controlTimebase == NULL) {
-                          NSLog(@"[Metal PiP Path] Timebase is NULL. Setting it now.");
-                          CMTimebaseRef timebase;
-                          CMTimebaseCreateWithSourceClock(kCFAllocatorDefault, CMClockGetHostTimeClock(), &timebase);
-                          if (timebase) {
-                              CMTimebaseSetTime(timebase, presentationTimestamp);
-                              CMTimebaseSetRate(timebase, 1.0);
-                              [self.pipLayer setControlTimebase:timebase];
-                              CFRelease(timebase);
-                          }
-                      }
-                      
-                      if (self->_pipFormatDesc == NULL || !CMVideoFormatDescriptionMatchesImageBuffer(self->_pipFormatDesc, imageBuffer)) {
-                          NSLog(@"[Metal PiP Path] Format description mismatch or NULL. Creating new one.");
-                          if (self->_pipFormatDesc) { CFRelease(self->_pipFormatDesc); }
-                          OSStatus res = CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, imageBuffer, &(self->_pipFormatDesc));
-                          if (res == noErr) {
-                              NSLog(@"[Metal PiP Path] New format description created: %@", self->_pipFormatDesc);
-                          } else {
-                              NSLog(@"[Metal PiP Path] FAILED to create format description. Error: %d", (int)res);
-                          }
-                      }
-                      
-                      if (self->_pipFormatDesc) {
-                          CMSampleBufferRef sampleBufferForPip = NULL;
-                          CMSampleTimingInfo sampleTiming = {kCMTimeInvalid, presentationTimestamp, presentationDuration};
-                          OSStatus err = CMSampleBufferCreateReadyWithImageBuffer(kCFAllocatorDefault, imageBuffer, self->_pipFormatDesc, &sampleTiming, &sampleBufferForPip);
-                          if (err == noErr && sampleBufferForPip) {
-//                              NSLog(@"[Metal PiP Path] Sample buffer created. Enqueuing to _pipLayer.");
-                              [self.pipLayer enqueueSampleBuffer:sampleBufferForPip];
-                              CFRelease(sampleBufferForPip);
-                          } else {
-                              NSLog(@"[Metal PiP Path] FAILED to create sample buffer. Error: %d", (int)err);
-                          }
-                      }
-                  }
-                  CVPixelBufferRelease(imageBuffer);
-              });
-          }
-          // --- MAIN RENDERER PATH ---
           if (self->_renderingBackend == RENDER_AVSB) {
-//              self.pipLayer.hidden = YES;
+            if (self->_formatDescImageBuffer == NULL || !CMVideoFormatDescriptionMatchesImageBuffer(self->_formatDescImageBuffer, imageBuffer)) {
-              if (self->_avsbFormatDesc == NULL || !CMVideoFormatDescriptionMatchesImageBuffer(self->_avsbFormatDesc, imageBuffer)) {
+              OSStatus res = CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, imageBuffer, &(self->_formatDescImageBuffer));
-                  if (self->_avsbFormatDesc) {
+              if (res != noErr) {
-                      CFRelease(self->_avsbFormatDesc);
+                Log(LOG_E, @"Failed to create video format description from imageBuffer");
-                  }
+                return;
-                  CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, imageBuffer, &(self->_avsbFormatDesc));
               }
-              
+            }
-              CMSampleBufferRef sampleBufferForAvsb = NULL;
+
-              CMSampleTimingInfo sampleTiming = {kCMTimeInvalid, presentationTimestamp, presentationDuration};
+            if (self->_formatDescImageBuffer == NULL || !CMVideoFormatDescriptionMatchesImageBuffer(self->_formatDescImageBuffer, imageBuffer)) {
-              OSStatus err = CMSampleBufferCreateReadyWithImageBuffer(kCFAllocatorDefault, imageBuffer, self->_avsbFormatDesc, &sampleTiming, &sampleBufferForAvsb);
+              OSStatus res = CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, imageBuffer, &(self->_formatDescImageBuffer));
-              
+              if (res != noErr) {
-              if (err == noErr && sampleBufferForAvsb) {
+                Log(LOG_E, @"Failed to create video format description from imageBuffer");
-                  dispatch_async(self->_vtq, ^{
+                return;
-                      Frame *frame = [[Frame alloc] initWithSampleBuffer:sampleBufferForAvsb frameNumber:frameNumber frameType:frameType];
-                      int framesDropped = [self->_frameQueue enqueue:frame withSlackSize:3];
-                      // ... stats and metrics updates
-                  });
-              } else {
-                  if (sampleBufferForAvsb) {
-                      CFRelease(sampleBufferForAvsb);
-                  }
               }
+            }
+
+            CMSampleTimingInfo sampleTiming = {kCMTimeInvalid, presentationTimestamp, presentationDuration};
+
+            OSStatus err =
+                CMSampleBufferCreateReadyWithImageBuffer(kCFAllocatorDefault, imageBuffer, self->_formatDescImageBuffer, &sampleTiming, &sampleBufferOut);
+            if (err != noErr) {
+              Log(LOG_E, @"Error creating sample buffer for decompressed image buffer %d", (int)err);
+              return;
+            }
           } else if (self->_renderingBackend == RENDER_METAL) {
-              self.pipLayer.hidden = NO;
+            // Metal path: retain the pixelBuffer here so it survives the dispatch
-              // Safely capture the format description for the background block.
+            pixelBuffer = CVPixelBufferRetain((CVPixelBufferRef)imageBuffer);
-              __block CMVideoFormatDescriptionRef capturedFormatDesc = NULL;
-              @synchronized(self) {
-                  if (self->_formatDesc) {
-                      capturedFormatDesc = (CMVideoFormatDescriptionRef)CFRetain(self->_formatDesc);
-                  }
-              }
-              
-              if (!capturedFormatDesc) {
-                  Log(LOG_E, @"Failed to capture format description for Metal frame, dropping.");
-                  return;
-              }
-              
-              CVPixelBufferRef pixelBuffer = CVPixelBufferRetain((CVPixelBufferRef)imageBuffer);
-              dispatch_async(self->_vtq, ^{
-                  Frame *frame = [[Frame alloc] initWithPixelBufffer:pixelBuffer frameNumber:frameNumber frameType:frameType pts:presentationTimestamp];
-                  [frame setFormatDesc:capturedFormatDesc];
-                  CFRelease(capturedFormatDesc);
-                  
-                  int framesDropped = [self->_frameQueue enqueue:frame withSlackSize:3];
-                  
-                  static PlotMetrics frameQueueMetrics = {};
-                  [[ImGuiPlots sharedInstance] observeFloatReturnMetrics:PLOT_QUEUED_FRAMES value:[self->_frameQueue count] plotMetrics:&frameQueueMetrics];
-                  [self safeCopyMetricsTo:&self->_frameQueueMetrics from:&frameQueueMetrics];
-                  
-                  [[ImGuiPlots sharedInstance] observeFloat:PLOT_DROPPED value:framesDropped];
-                  
-                  static CFTimeInterval lastHostFrame = 0.0f;
-                  if (lastHostFrame != 0) {
-                      [[ImGuiPlots sharedInstance] observeFloat:PLOT_HOST_FRAMETIME value:(frame.pts - lastHostFrame) * 1000.0];
-                  }
-                  lastHostFrame = frame.pts;
-                  
-                  static PlotMetrics decodeMetrics = {};
-                  [[ImGuiPlots sharedInstance] observeFloatReturnMetrics:PLOT_DECODE value:(CACurrentMediaTime() - decodeStartTime) * 1000.0 plotMetrics:&decodeMetrics];
-                  [self safeCopyMetricsTo:&self->_decodeMetrics from:&decodeMetrics];
-              });
           }
+
+          // Dispatch onto our higher priority queue
+          dispatch_async(self->_vtq, ^{
+              Frame *frame = nil;
+              if (self->_renderingBackend == RENDER_AVSB) {
+                frame = [[Frame alloc] initWithSampleBuffer:sampleBufferOut frameNumber:frameNumber frameType:frameType];
+              } else {
+                frame = [[Frame alloc] initWithPixelBufffer:pixelBuffer frameNumber:frameNumber frameType:frameType pts:presentationTimestamp];
+                [frame setFormatDesc:self->_formatDesc];
+              }
+              int framesDropped = [self->_frameQueue enqueue:frame withSlackSize:3];
+
+              static PlotMetrics frameQueueMetrics = {};
+              [[ImGuiPlots sharedInstance] observeFloatReturnMetrics:PLOT_QUEUED_FRAMES value:[self->_frameQueue count] plotMetrics:&frameQueueMetrics];
+              [self safeCopyMetricsTo:&self->_frameQueueMetrics from:&frameQueueMetrics];
+
+              [[ImGuiPlots sharedInstance] observeFloat:PLOT_DROPPED value:framesDropped];
+
+              // It's important we capture host metrics on the incoming thread, as this frame object
+              // may have been dropped by the above enqueue
+              static CFTimeInterval lastHostFrame = 0.0f;
+              if (lastHostFrame != 0) {
+                [[ImGuiPlots sharedInstance] observeFloat:PLOT_HOST_FRAMETIME value:(frame.pts - lastHostFrame) * 1000.0];
+              }
+              lastHostFrame = frame.pts;
+
+              // Decode time is not graphed because it is marked as hidden, but we can use the same mechanism for the value used by stats
+              static PlotMetrics decodeMetrics = {};
+              [[ImGuiPlots sharedInstance] observeFloatReturnMetrics:PLOT_DECODE value:(CACurrentMediaTime() - decodeStartTime) * 1000.0 plotMetrics:&decodeMetrics];
+              [self safeCopyMetricsTo:&self->_decodeMetrics from:&decodeMetrics];
+          });
       });
 
   return status;
@@ -1123,11 +1069,11 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit);
     // Ensure this only runs for the AVSampleBuffer rendering backend and that the display link exists.
     if (_renderingBackend == RENDER_AVSB && _displayLink) {
         Log(LOG_I, @"Frame pacing is being reset to %d FPS...", self->_frameRate);
-        
+
         // Toggling the paused state can help re-engage the display link with the
         // run loop correctly after the app resumes from a background state like PiP.
         _displayLink.paused = YES;
-        
+
         // Re-apply the desired frame rate range. This is the critical hint for ProMotion
         // that may have been lost or ignored during the PiP transition.
         if (@available(iOS 15.0, *)) {
@@ -1135,25 +1081,10 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit);
         } else {
             _displayLink.preferredFramesPerSecond = self->_frameRate;
         }
-        
+
         // Resume the display link immediately.
         _displayLink.paused = NO;
-    }
+    } else if (_renderingBackend == RENDER_METAL) {
-    else if (_renderingBackend == RENDER_METAL) {
-        Log(LOG_I, @"Resetting video pipeline for Metal renderer after resuming...");
-
-        // 1. Reset the receiver (the PiP Layer) on the main thread
-        dispatch_async(dispatch_get_main_queue(), ^{
-            if (self.pipLayer) {
-                // Flush stale buffers
-                [self.pipLayer flushAndRemoveImage];
-
-                // Destroy the old, stuck timebase
-                [self.pipLayer setControlTimebase:NULL];
-            }
-        });
-
-        // 2. Reset the sender (the Decompression Session)
         @synchronized(self) {
             if (_decompressionSession != nil) {
                 VTDecompressionSessionInvalidate(_decompressionSession);
@@ -1161,8 +1092,6 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit);
                 _decompressionSession = nil;
             }
         }
-        
-        // 3. Request a new keyframe to restart the stream cleanly
         LiRequestIdrFrame();
     }
 }

VoidLink/ViewControllers/StreamFrameViewController.m

@@ -80,7 +80,6 @@
     dispatch_block_t _delayedRemoveExtScreen;
     VideoDecoderRenderer *_videoRenderer;
     BOOL _isRestoringFromPiP;
-    AVSampleBufferDisplayLayer *_pipLayer;
 #if !TARGET_OS_TV
     CustomEdgeSlideGestureRecognizer *_slideToSettingsRecognizer;
     CustomEdgeSlideGestureRecognizer *_slideToCmdToolRecognizer;
@@ -183,37 +182,30 @@
         return;
     }
     Log(LOG_I, @"Setting up PiP controller...");
-    
-    AVSampleBufferDisplayLayer* sourceLayer;
-    if ([_settings.renderingBackend intValue] == RENDER_METAL) {
-        sourceLayer = _pipLayer;
-        NSLog(@"[PiP LOG] Backend: Metal. Using _pipLayer: %p", sourceLayer);
-        if (!sourceLayer) {
-            NSLog(@"[PiP LOG] ERROR: _pipLayer is nil during setup.");
-            return;
-        }
-    } else { // RENDER_AVSB
-        sourceLayer = videoRenderer.displayLayer;
-        NSLog(@"[PiP LOG] Backend: AVSB. Using _displayLayer: %p", sourceLayer);
-    }
 
-    if (!_pipLayer) {
+    if (!videoRenderer || !videoRenderer.displayLayer) {
-        Log(LOG_E, @"PiP setup failed: Dedicated PiP layer not ready.");
+        Log(LOG_E, @"PiP setup failed: Video renderer or display layer not ready.");
         return;
     }
 
+    AVSampleBufferDisplayLayer *streamLayer = videoRenderer.displayLayer;
+
     if ([AVPictureInPictureController isPictureInPictureSupported]) {
         if (@available(iOS 15.0, *)) {
-            self.pipContentSource = [[AVPictureInPictureControllerContentSource alloc] initWithSampleBufferDisplayLayer:sourceLayer playbackDelegate:(id<AVPictureInPictureSampleBufferPlaybackDelegate>)self];
+            self.pipContentSource = [[AVPictureInPictureControllerContentSource alloc] initWithSampleBufferDisplayLayer:streamLayer playbackDelegate:(id<AVPictureInPictureSampleBufferPlaybackDelegate>)self];
             self.pipController = [[AVPictureInPictureController alloc] initWithContentSource:self.pipContentSource];
-            self.pipController.requiresLinearPlayback = YES;
             self.pipController.canStartPictureInPictureAutomaticallyFromInline = YES;
-            self.pipController.delegate = self;
-            NSLog(@"[PiP LOG] PiP Controller created successfully with source layer: %@", sourceLayer);
         } else {
-            Log(LOG_E, @"PiP is not fully supported on this iOS version.");
+            Log(LOG_E, @"PiP not fully supported on this device.");
             return;
         }
+
+        if (self.pipController) {
+            self.pipController.delegate = self;
+            Log(LOG_I, @"PiP controller created successfully.");
+        } else {
+            Log(LOG_E, @"Failed to create PiP controller.");
+        }
     } else {
         Log(LOG_E, @"PiP not supported on this device.");
     }
@@ -223,10 +215,10 @@
     if (self.pipController) {
         self.pipController.delegate = nil;
         self.pipController = nil;
-        _pipLayer = nil; // Clean up the layer
         if (@available(iOS 15.0, *)) {
             self.pipContentSource = nil;
         }
+
         Log(LOG_I, @"PiP controller cleaned up.");
     }
 }
@@ -624,15 +616,9 @@
     _tipLabel.textAlignment = NSTextAlignmentCenter;
     _tipLabel.center = CGPointMake(self.view.frame.size.width / 2, self.view.frame.size.height * 0.9);
     
-    _pipLayer = [[AVSampleBufferDisplayLayer alloc] init];
-    _pipLayer.videoGravity = AVLayerVideoGravityResizeAspect;
-    _pipLayer.frame = self.view.bounds;
-    _pipLayer.hidden = YES;
-    [self.view.layer addSublayer:_pipLayer];
     _streamMan = [[StreamManager alloc] initWithConfig:self.streamConfig
                                             renderView:_streamVideoRenderView
                                    connectionCallbacks:self];
-    _streamMan.pipLayer = _pipLayer;
     NSOperationQueue* opQueue = [[NSOperationQueue alloc] init];
     [opQueue addOperation:_streamMan];
     
@@ -723,14 +709,7 @@
 }
 
 - (void)enterPip{
-    NSLog(@"[PiP LOG] enterPip called.");
+    [self.pipController startPictureInPicture];
-    if (self.pipController) {
-        NSLog(@"[PiP LOG] Controller exists. Firing startPictureInPicture.");
-        NSLog(@"[PiP LOG] Layer state before start: hidden=%d, frame=%@, timebase=%@", _pipLayer.hidden, NSStringFromCGRect(_pipLayer.frame), _pipLayer.controlTimebase);
-        [self.pipController startPictureInPicture];
-    } else {
-        NSLog(@"[PiP LOG] ERROR: enterPip called, but pipController is nil!");
-    }
 }
 
 - (void)oscLayoutClosed{
@@ -977,14 +956,13 @@
         [_inactivityTimer invalidate];
         _inactivityTimer = nil;
     }
-    
-    [self->_streamMan.videoRenderer resetFramePacing];
 
     // Check if we were in PiP
     if (self.pipController && self.pipController.isPictureInPictureActive) {
         [self.pipController stopPictureInPicture];
     }
     
+    [self->_streamMan.videoRenderer resetFramePacing];
     _isRestoringFromPiP = NO;
 }