hermes-agent/VoidLink
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Revert "feat: pip for metal renderer"

Browse Source 
This reverts commit fa7710ce44.
This commit is contained in:
Acaki
2025-07-19 20:05:03 +08:00
parent f368076aaf
commit 4d45c538cc
5 changed files with 89 additions and 185 deletions
Download Patch File Download Diff File Expand all files Collapse all files
VoidLink/Stream/StreamManager.h
-2
View File
@@ -16,8 +16,6 @@
@property (nonatomic, strong, readonly) VideoDecoderRenderer *videoRenderer;
@property (nonatomic, strong) AVSampleBufferDisplayLayer *pipLayer;
- (id) initWithConfig:(StreamConfiguration*)config renderView:(UIView*)view connectionCallbacks:(id<ConnectionCallbacks>)callback;
- (void) stopStream;
VoidLink/Stream/StreamManager.m
+1 -1
View File
@@ -106,7 +106,7 @@
// Initializing the renderer must be done on the main thread
dispatch_async(dispatch_get_main_queue(), ^{
self->_videoRenderer = [[VideoDecoderRenderer alloc] initWithView:self->_renderView callbacks:self->_callbacks streamAspectRatio:(float)self->_config.width / (float)self->_config.height];
self->_videoRenderer.pipLayer = self.pipLayer;
self->_connection = [[Connection alloc] initWithConfig:self->_config renderer:self->_videoRenderer connectionCallbacks:self->_callbacks];
NSOperationQueue* opQueue = [[NSOperationQueue alloc] init];
[opQueue addOperation:self->_connection];
VoidLink/Stream/VideoDecoderRenderer.h
-1
View File
@@ -20,7 +20,6 @@
@property (atomic, readonly) PlotMetrics frameQueueMetrics;
@property (nonatomic, strong, readonly) AVSampleBufferDisplayLayer *displayLayer;
@property (nonatomic, weak) AVSampleBufferDisplayLayer *pipLayer;
- (id)initWithView:(UIView*)view callbacks:(id<ConnectionCallbacks>)callbacks streamAspectRatio:(float)aspectRatio;
VoidLink/Stream/VideoDecoderRenderer.m
+72 -143
View File
@@ -47,16 +47,14 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size,
NSData *_masteringDisplayColorVolume;
NSData *_contentLightLevelInfo;
CMVideoFormatDescriptionRef _formatDesc;
CMVideoFormatDescriptionRef _pipFormatDesc;
CMVideoFormatDescriptionRef _avsbFormatDesc;
CMVideoFormatDescriptionRef _formatDescImageBuffer;
VTDecompressionSessionRef _decompressionSession;
CADisplayLink *_displayLink;
FrameQueue *_frameQueue;
NSInteger _maxRefreshRate;
RenderingBackend _renderingBackend;
BOOL _hasSentVideoContentShown;
}
- (void)reinitializeDisplayLayer
@@ -95,13 +93,9 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size,
_formatDesc = nil;
}
if (_pipFormatDesc != nil) {
CFRelease(_pipFormatDesc);
_pipFormatDesc = nil;
}
if (_avsbFormatDesc != nil) {
CFRelease(_avsbFormatDesc);
_avsbFormatDesc = nil;
if (_formatDescImageBuffer != nil) {
CFRelease(_formatDescImageBuffer);
_formatDescImageBuffer = nil;
}
if (_decompressionSession != nil){
@@ -338,6 +332,9 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit);
if (frame.frameType == FRAME_TYPE_IDR) {
// Ensure the layer is visible now
self->_displayLayer.hidden = NO;
// Tell our parent VC to hide the progress indicator
[self->_callbacks videoContentShown];
}
}
@@ -827,12 +824,6 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit);
if (frameType == FRAME_TYPE_IDR || _decompressionSession == nil) {
[self setupDecompressionSession];
}
if (frameType == FRAME_TYPE_IDR && !_hasSentVideoContentShown) {
// Tell our parent VC to hide the progress indicator and set up PiP
[_callbacks videoContentShown];
_hasSentVideoContentShown = YES;
}
OSStatus status = VTDecompressionSessionDecodeFrameWithOutputHandler(
_decompressionSession,
@@ -855,116 +846,71 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit);
LiRequestIdrFrame(); // Request an IDR to restart the stream
return;
}
if (self->_renderingBackend == RENDER_METAL && self.pipLayer) {
// NSLog(@"[Metal PiP Path] Entered for frame number %d.", frameNumber);
CVPixelBufferRetain(imageBuffer);
dispatch_async(dispatch_get_main_queue(), ^{
// NSLog(@"[Metal PiP Path] Executing on main thread for frame %d.", frameNumber);
@synchronized(self) {
if (self.pipLayer.controlTimebase == NULL) {
NSLog(@"[Metal PiP Path] Timebase is NULL. Setting it now.");
CMTimebaseRef timebase;
CMTimebaseCreateWithSourceClock(kCFAllocatorDefault, CMClockGetHostTimeClock(), &timebase);
if (timebase) {
CMTimebaseSetTime(timebase, presentationTimestamp);
CMTimebaseSetRate(timebase, 1.0);
[self.pipLayer setControlTimebase:timebase];
CFRelease(timebase);
}
}
if (self->_pipFormatDesc == NULL || !CMVideoFormatDescriptionMatchesImageBuffer(self->_pipFormatDesc, imageBuffer)) {
NSLog(@"[Metal PiP Path] Format description mismatch or NULL. Creating new one.");
if (self->_pipFormatDesc) { CFRelease(self->_pipFormatDesc); }
OSStatus res = CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, imageBuffer, &(self->_pipFormatDesc));
if (res == noErr) {
NSLog(@"[Metal PiP Path] New format description created: %@", self->_pipFormatDesc);
} else {
NSLog(@"[Metal PiP Path] FAILED to create format description. Error: %d", (int)res);
}
}
if (self->_pipFormatDesc) {
CMSampleBufferRef sampleBufferForPip = NULL;
CMSampleTimingInfo sampleTiming = {kCMTimeInvalid, presentationTimestamp, presentationDuration};
OSStatus err = CMSampleBufferCreateReadyWithImageBuffer(kCFAllocatorDefault, imageBuffer, self->_pipFormatDesc, &sampleTiming, &sampleBufferForPip);
if (err == noErr && sampleBufferForPip) {
// NSLog(@"[Metal PiP Path] Sample buffer created. Enqueuing to _pipLayer.");
[self.pipLayer enqueueSampleBuffer:sampleBufferForPip];
CFRelease(sampleBufferForPip);
} else {
NSLog(@"[Metal PiP Path] FAILED to create sample buffer. Error: %d", (int)err);
}
}
}
CVPixelBufferRelease(imageBuffer);
});
}
// --- MAIN RENDERER PATH ---
CMSampleBufferRef sampleBufferOut = nil;
CVPixelBufferRef pixelBuffer = nil;
// AVSampleBuffer path: package into a SampleBuffer
if (self->_renderingBackend == RENDER_AVSB) {
// self.pipLayer.hidden = YES;
if (self->_avsbFormatDesc == NULL || !CMVideoFormatDescriptionMatchesImageBuffer(self->_avsbFormatDesc, imageBuffer)) {
if (self->_avsbFormatDesc) {
CFRelease(self->_avsbFormatDesc);
}
CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, imageBuffer, &(self->_avsbFormatDesc));
if (self->_formatDescImageBuffer == NULL || !CMVideoFormatDescriptionMatchesImageBuffer(self->_formatDescImageBuffer, imageBuffer)) {
OSStatus res = CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, imageBuffer, &(self->_formatDescImageBuffer));
if (res != noErr) {
Log(LOG_E, @"Failed to create video format description from imageBuffer");
return;
}
CMSampleBufferRef sampleBufferForAvsb = NULL;
CMSampleTimingInfo sampleTiming = {kCMTimeInvalid, presentationTimestamp, presentationDuration};
OSStatus err = CMSampleBufferCreateReadyWithImageBuffer(kCFAllocatorDefault, imageBuffer, self->_avsbFormatDesc, &sampleTiming, &sampleBufferForAvsb);
if (err == noErr && sampleBufferForAvsb) {
dispatch_async(self->_vtq, ^{
Frame *frame = [[Frame alloc] initWithSampleBuffer:sampleBufferForAvsb frameNumber:frameNumber frameType:frameType];
int framesDropped = [self->_frameQueue enqueue:frame withSlackSize:3];
// ... stats and metrics updates
});
} else {
if (sampleBufferForAvsb) {
CFRelease(sampleBufferForAvsb);
}
}
if (self->_formatDescImageBuffer == NULL || !CMVideoFormatDescriptionMatchesImageBuffer(self->_formatDescImageBuffer, imageBuffer)) {
OSStatus res = CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, imageBuffer, &(self->_formatDescImageBuffer));
if (res != noErr) {
Log(LOG_E, @"Failed to create video format description from imageBuffer");
return;
}
}
CMSampleTimingInfo sampleTiming = {kCMTimeInvalid, presentationTimestamp, presentationDuration};
OSStatus err =
CMSampleBufferCreateReadyWithImageBuffer(kCFAllocatorDefault, imageBuffer, self->_formatDescImageBuffer, &sampleTiming, &sampleBufferOut);
if (err != noErr) {
Log(LOG_E, @"Error creating sample buffer for decompressed image buffer %d", (int)err);
return;
}
} else if (self->_renderingBackend == RENDER_METAL) {
self.pipLayer.hidden = NO;
// Safely capture the format description for the background block.
__block CMVideoFormatDescriptionRef capturedFormatDesc = NULL;
@synchronized(self) {
if (self->_formatDesc) {
capturedFormatDesc = (CMVideoFormatDescriptionRef)CFRetain(self->_formatDesc);
}
}
if (!capturedFormatDesc) {
Log(LOG_E, @"Failed to capture format description for Metal frame, dropping.");
return;
}
CVPixelBufferRef pixelBuffer = CVPixelBufferRetain((CVPixelBufferRef)imageBuffer);
dispatch_async(self->_vtq, ^{
Frame *frame = [[Frame alloc] initWithPixelBufffer:pixelBuffer frameNumber:frameNumber frameType:frameType pts:presentationTimestamp];
[frame setFormatDesc:capturedFormatDesc];
CFRelease(capturedFormatDesc);
int framesDropped = [self->_frameQueue enqueue:frame withSlackSize:3];
static PlotMetrics frameQueueMetrics = {};
[[ImGuiPlots sharedInstance] observeFloatReturnMetrics:PLOT_QUEUED_FRAMES value:[self->_frameQueue count] plotMetrics:&frameQueueMetrics];
[self safeCopyMetricsTo:&self->_frameQueueMetrics from:&frameQueueMetrics];
[[ImGuiPlots sharedInstance] observeFloat:PLOT_DROPPED value:framesDropped];
static CFTimeInterval lastHostFrame = 0.0f;
if (lastHostFrame != 0) {
[[ImGuiPlots sharedInstance] observeFloat:PLOT_HOST_FRAMETIME value:(frame.pts - lastHostFrame) * 1000.0];
}
lastHostFrame = frame.pts;
static PlotMetrics decodeMetrics = {};
[[ImGuiPlots sharedInstance] observeFloatReturnMetrics:PLOT_DECODE value:(CACurrentMediaTime() - decodeStartTime) * 1000.0 plotMetrics:&decodeMetrics];
[self safeCopyMetricsTo:&self->_decodeMetrics from:&decodeMetrics];
});
// Metal path: retain the pixelBuffer here so it survives the dispatch
pixelBuffer = CVPixelBufferRetain((CVPixelBufferRef)imageBuffer);
}
// Dispatch onto our higher priority queue
dispatch_async(self->_vtq, ^{
Frame *frame = nil;
if (self->_renderingBackend == RENDER_AVSB) {
frame = [[Frame alloc] initWithSampleBuffer:sampleBufferOut frameNumber:frameNumber frameType:frameType];
} else {
frame = [[Frame alloc] initWithPixelBufffer:pixelBuffer frameNumber:frameNumber frameType:frameType pts:presentationTimestamp];
[frame setFormatDesc:self->_formatDesc];
}
int framesDropped = [self->_frameQueue enqueue:frame withSlackSize:3];
static PlotMetrics frameQueueMetrics = {};
[[ImGuiPlots sharedInstance] observeFloatReturnMetrics:PLOT_QUEUED_FRAMES value:[self->_frameQueue count] plotMetrics:&frameQueueMetrics];
[self safeCopyMetricsTo:&self->_frameQueueMetrics from:&frameQueueMetrics];
[[ImGuiPlots sharedInstance] observeFloat:PLOT_DROPPED value:framesDropped];
// It's important we capture host metrics on the incoming thread, as this frame object
// may have been dropped by the above enqueue
static CFTimeInterval lastHostFrame = 0.0f;
if (lastHostFrame != 0) {
[[ImGuiPlots sharedInstance] observeFloat:PLOT_HOST_FRAMETIME value:(frame.pts - lastHostFrame) * 1000.0];
}
lastHostFrame = frame.pts;
// Decode time is not graphed because it is marked as hidden, but we can use the same mechanism for the value used by stats
static PlotMetrics decodeMetrics = {};
[[ImGuiPlots sharedInstance] observeFloatReturnMetrics:PLOT_DECODE value:(CACurrentMediaTime() - decodeStartTime) * 1000.0 plotMetrics:&decodeMetrics];
[self safeCopyMetricsTo:&self->_decodeMetrics from:&decodeMetrics];
});
});
return status;
@@ -1123,11 +1069,11 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit);
// Ensure this only runs for the AVSampleBuffer rendering backend and that the display link exists.
if (_renderingBackend == RENDER_AVSB && _displayLink) {
Log(LOG_I, @"Frame pacing is being reset to %d FPS...", self->_frameRate);
// Toggling the paused state can help re-engage the display link with the
// run loop correctly after the app resumes from a background state like PiP.
_displayLink.paused = YES;
// Re-apply the desired frame rate range. This is the critical hint for ProMotion
// that may have been lost or ignored during the PiP transition.
if (@available(iOS 15.0, *)) {
@@ -1135,25 +1081,10 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit);
} else {
_displayLink.preferredFramesPerSecond = self->_frameRate;
}
// Resume the display link immediately.
_displayLink.paused = NO;
}
else if (_renderingBackend == RENDER_METAL) {
Log(LOG_I, @"Resetting video pipeline for Metal renderer after resuming...");
// 1. Reset the receiver (the PiP Layer) on the main thread
dispatch_async(dispatch_get_main_queue(), ^{
if (self.pipLayer) {
// Flush stale buffers
[self.pipLayer flushAndRemoveImage];
// Destroy the old, stuck timebase
[self.pipLayer setControlTimebase:NULL];
}
});
// 2. Reset the sender (the Decompression Session)
} else if (_renderingBackend == RENDER_METAL) {
@synchronized(self) {
if (_decompressionSession != nil) {
VTDecompressionSessionInvalidate(_decompressionSession);
@@ -1161,8 +1092,6 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit);
_decompressionSession = nil;
}
}
// 3. Request a new keyframe to restart the stream cleanly
LiRequestIdrFrame();
}
}
VoidLink/ViewControllers/StreamFrameViewController.m
+16 -38
View File
@@ -80,7 +80,6 @@
dispatch_block_t _delayedRemoveExtScreen;
VideoDecoderRenderer *_videoRenderer;
BOOL _isRestoringFromPiP;
AVSampleBufferDisplayLayer *_pipLayer;
#if !TARGET_OS_TV
CustomEdgeSlideGestureRecognizer *_slideToSettingsRecognizer;
CustomEdgeSlideGestureRecognizer *_slideToCmdToolRecognizer;
@@ -183,37 +182,30 @@
return;
}
Log(LOG_I, @"Setting up PiP controller...");
AVSampleBufferDisplayLayer* sourceLayer;
if ([_settings.renderingBackend intValue] == RENDER_METAL) {
sourceLayer = _pipLayer;
NSLog(@"[PiP LOG] Backend: Metal. Using _pipLayer: %p", sourceLayer);
if (!sourceLayer) {
NSLog(@"[PiP LOG] ERROR: _pipLayer is nil during setup.");
return;
}
} else { // RENDER_AVSB
sourceLayer = videoRenderer.displayLayer;
NSLog(@"[PiP LOG] Backend: AVSB. Using _displayLayer: %p", sourceLayer);
}
if (!_pipLayer) {
Log(LOG_E, @"PiP setup failed: Dedicated PiP layer not ready.");
if (!videoRenderer || !videoRenderer.displayLayer) {
Log(LOG_E, @"PiP setup failed: Video renderer or display layer not ready.");
return;
}
AVSampleBufferDisplayLayer *streamLayer = videoRenderer.displayLayer;
if ([AVPictureInPictureController isPictureInPictureSupported]) {
if (@available(iOS 15.0, *)) {
self.pipContentSource = [[AVPictureInPictureControllerContentSource alloc] initWithSampleBufferDisplayLayer:sourceLayer playbackDelegate:(id<AVPictureInPictureSampleBufferPlaybackDelegate>)self];
self.pipContentSource = [[AVPictureInPictureControllerContentSource alloc] initWithSampleBufferDisplayLayer:streamLayer playbackDelegate:(id<AVPictureInPictureSampleBufferPlaybackDelegate>)self];
self.pipController = [[AVPictureInPictureController alloc] initWithContentSource:self.pipContentSource];
self.pipController.requiresLinearPlayback = YES;
self.pipController.canStartPictureInPictureAutomaticallyFromInline = YES;
self.pipController.delegate = self;
NSLog(@"[PiP LOG] PiP Controller created successfully with source layer: %@", sourceLayer);
} else {
Log(LOG_E, @"PiP is not fully supported on this iOS version.");
Log(LOG_E, @"PiP not fully supported on this device.");
return;
}
if (self.pipController) {
self.pipController.delegate = self;
Log(LOG_I, @"PiP controller created successfully.");
} else {
Log(LOG_E, @"Failed to create PiP controller.");
}
} else {
Log(LOG_E, @"PiP not supported on this device.");
}
@@ -223,10 +215,10 @@
if (self.pipController) {
self.pipController.delegate = nil;
self.pipController = nil;
_pipLayer = nil; // Clean up the layer
if (@available(iOS 15.0, *)) {
self.pipContentSource = nil;
}
Log(LOG_I, @"PiP controller cleaned up.");
}
}
@@ -624,15 +616,9 @@
_tipLabel.textAlignment = NSTextAlignmentCenter;
_tipLabel.center = CGPointMake(self.view.frame.size.width / 2, self.view.frame.size.height * 0.9);
_pipLayer = [[AVSampleBufferDisplayLayer alloc] init];
_pipLayer.videoGravity = AVLayerVideoGravityResizeAspect;
_pipLayer.frame = self.view.bounds;
_pipLayer.hidden = YES;
[self.view.layer addSublayer:_pipLayer];
_streamMan = [[StreamManager alloc] initWithConfig:self.streamConfig
renderView:_streamVideoRenderView
connectionCallbacks:self];
_streamMan.pipLayer = _pipLayer;
NSOperationQueue* opQueue = [[NSOperationQueue alloc] init];
[opQueue addOperation:_streamMan];
@@ -723,14 +709,7 @@
}
- (void)enterPip{
NSLog(@"[PiP LOG] enterPip called.");
if (self.pipController) {
NSLog(@"[PiP LOG] Controller exists. Firing startPictureInPicture.");
NSLog(@"[PiP LOG] Layer state before start: hidden=%d, frame=%@, timebase=%@", _pipLayer.hidden, NSStringFromCGRect(_pipLayer.frame), _pipLayer.controlTimebase);
[self.pipController startPictureInPicture];
} else {
NSLog(@"[PiP LOG] ERROR: enterPip called, but pipController is nil!");
}
[self.pipController startPictureInPicture];
}
- (void)oscLayoutClosed{
@@ -977,14 +956,13 @@
[_inactivityTimer invalidate];
_inactivityTimer = nil;
}
[self->_streamMan.videoRenderer resetFramePacing];
// Check if we were in PiP
if (self.pipController && self.pipController.isPictureInPictureActive) {
[self.pipController stopPictureInPicture];
}
[self->_streamMan.videoRenderer resetFramePacing];
_isRestoringFromPiP = NO;
}