hermes-agent/VoidLink
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Revert "refactor: use separate piplayer for AVSB pip function"

Browse Source 
This reverts commit bd9e2e3215.
This commit is contained in:
Acaki committed 2025-07-19 17:43:46 +08:00
1 parent 61c5b958f2
commit f368076aaf
2 files changed
+164 -212
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

VoidLink/Stream/VideoDecoderRenderer.m
+151 -200
View File
@@ -824,186 +824,150 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit);
frameNumber:(int)frameNumber frameNumber:(int)frameNumber
frameType:(int)frameType frameType:(int)frameType
decodeStartTime:(CFTimeInterval)decodeStartTime { decodeStartTime:(CFTimeInterval)decodeStartTime {
VTDecompressionSessionRef sessionForDecode = NULL; if (frameType == FRAME_TYPE_IDR || _decompressionSession == nil) {
@synchronized(self) { [self setupDecompressionSession];
// If the session is nil, it can only be created by an IDR frame. }
if (_decompressionSession == nil) {
if (frameType == FRAME_TYPE_IDR) { if (frameType == FRAME_TYPE_IDR && !_hasSentVideoContentShown) {
[self setupDecompressionSession]; // Tell our parent VC to hide the progress indicator and set up PiP
} else { [_callbacks videoContentShown];
// Not an IDR, so we can't create a session. Drop the frame. _hasSentVideoContentShown = YES;
return noErr; }
}
}
// If an IDR arrives during a valid session, re-create the session
// in case video parameters have changed.
else if (frameType == FRAME_TYPE_IDR) {
[self setupDecompressionSession];
}
// If, after trying, the session is still nil, we cannot proceed. OSStatus status = VTDecompressionSessionDecodeFrameWithOutputHandler(
if (_decompressionSession == nil) { _decompressionSession,
Log(LOG_E, @"Decompression session is nil after setup attempt. Dropping frame."); sampleBuffer,
return kVTInvalidSessionErr; 0,
} NULL,
^(OSStatus status, VTDecodeInfoFlags infoFlags, CVImageBufferRef _Nullable imageBuffer, CMTime presentationTimestamp, CMTime presentationDuration) {
// Capture a strong reference to the session that we can use outside the lock. if (status != noErr || !imageBuffer) {
// This prevents another thread from deallocating it while we're using it. NSError *error = [NSError errorWithDomain:NSOSStatusErrorDomain code:status userInfo:nil];
sessionForDecode = (VTDecompressionSessionRef)CFRetain(_decompressionSession); Log(LOG_E, @"Decompression session error: %@", error);
} if (status == kVTInvalidSessionErr) {
// The session was invalidated by the OS. Destroy our reference
if (frameType == FRAME_TYPE_IDR && !_hasSentVideoContentShown) { // so it gets recreated on the next IDR frame.
// Tell our parent VC to hide the progress indicator and set up PiP if (self->_decompressionSession) {
[_callbacks videoContentShown]; VTDecompressionSessionInvalidate(self->_decompressionSession);
_hasSentVideoContentShown = YES; CFRelease(self->_decompressionSession);
} self->_decompressionSession = nil;
OSStatus status = VTDecompressionSessionDecodeFrameWithOutputHandler(
_decompressionSession,
sampleBuffer,
0,
NULL,
^(OSStatus status, VTDecodeInfoFlags infoFlags, CVImageBufferRef _Nullable imageBuffer, CMTime presentationTimestamp, CMTime presentationDuration) {
if (status != noErr || !imageBuffer) {
NSError *error = [NSError errorWithDomain:NSOSStatusErrorDomain code:status userInfo:nil];
Log(LOG_E, @"Decompression session error: %@", error);
if (status == kVTInvalidSessionErr) {
// The session was invalidated by the OS. Destroy our reference
// so it gets recreated on the next IDR frame.
if (self->_decompressionSession) {
VTDecompressionSessionInvalidate(self->_decompressionSession);
CFRelease(self->_decompressionSession);
self->_decompressionSession = nil;
}
} }
LiRequestIdrFrame(); // Request an IDR to restart the stream }
return; LiRequestIdrFrame(); // Request an IDR to restart the stream
} return;
}
if (self.pipLayer) { if (self->_renderingBackend == RENDER_METAL && self.pipLayer) {
CVPixelBufferRetain(imageBuffer); // NSLog(@"[Metal PiP Path] Entered for frame number %d.", frameNumber);
dispatch_async(dispatch_get_main_queue(), ^{ CVPixelBufferRetain(imageBuffer);
@synchronized(self) { dispatch_async(dispatch_get_main_queue(), ^{
if (self.pipLayer.controlTimebase == NULL) { // NSLog(@"[Metal PiP Path] Executing on main thread for frame %d.", frameNumber);
NSLog(@"[Metal PiP Path] Timebase is NULL. Setting it now.");
CMTimebaseRef timebase;
CMTimebaseCreateWithSourceClock(kCFAllocatorDefault, CMClockGetHostTimeClock(), &timebase);
if (timebase) {
CMTimebaseSetTime(timebase, presentationTimestamp);
CMTimebaseSetRate(timebase, 1.0);
[self.pipLayer setControlTimebase:timebase];
CFRelease(timebase);
}
}
if (self->_pipFormatDesc == NULL || !CMVideoFormatDescriptionMatchesImageBuffer(self->_pipFormatDesc, imageBuffer)) {
NSLog(@"[Metal PiP Path] Format description mismatch or NULL. Creating new one.");
if (self->_pipFormatDesc) { CFRelease(self->_pipFormatDesc); }
OSStatus res = CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, imageBuffer, &(self->_pipFormatDesc));
if (res == noErr) {
NSLog(@"[Metal PiP Path] New format description created: %@", self->_pipFormatDesc);
} else {
NSLog(@"[Metal PiP Path] FAILED to create format description. Error: %d", (int)res);
}
}
if (self->_pipFormatDesc) {
CMSampleBufferRef sampleBufferForPip = NULL;
CMSampleTimingInfo sampleTiming = {kCMTimeInvalid, presentationTimestamp, presentationDuration};
OSStatus err = CMSampleBufferCreateReadyWithImageBuffer(kCFAllocatorDefault, imageBuffer, self->_pipFormatDesc, &sampleTiming, &sampleBufferForPip);
if (err == noErr && sampleBufferForPip) {
[self.pipLayer enqueueSampleBuffer:sampleBufferForPip];
CFRelease(sampleBufferForPip);
} else {
NSLog(@"[Metal PiP Path] FAILED to create sample buffer. Error: %d", (int)err);
}
}
}
CVPixelBufferRelease(imageBuffer);
});
}
// --- MAIN RENDERER PATH ---
if (self->_renderingBackend == RENDER_AVSB) {
self.pipLayer.hidden = NO;
if (self->_avsbFormatDesc == NULL || !CMVideoFormatDescriptionMatchesImageBuffer(self->_avsbFormatDesc, imageBuffer)) {
if (self->_avsbFormatDesc) {
CFRelease(self->_avsbFormatDesc);
}
CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, imageBuffer, &(self->_avsbFormatDesc));
}
CMSampleBufferRef sampleBufferForAvsb = NULL;
CMSampleTimingInfo sampleTiming = {kCMTimeInvalid, presentationTimestamp, presentationDuration};
OSStatus err = CMSampleBufferCreateReadyWithImageBuffer(kCFAllocatorDefault, imageBuffer, self->_avsbFormatDesc, &sampleTiming, &sampleBufferForAvsb);
if (err == noErr && sampleBufferForAvsb) {
dispatch_async(self->_vtq, ^{
Frame *frame = [[Frame alloc] initWithSampleBuffer:sampleBufferForAvsb frameNumber:frameNumber frameType:frameType];
int framesDropped = [self->_frameQueue enqueue:frame withSlackSize:3];
static PlotMetrics frameQueueMetrics = {};
[[ImGuiPlots sharedInstance] observeFloatReturnMetrics:PLOT_QUEUED_FRAMES value:[self->_frameQueue count] plotMetrics:&frameQueueMetrics];
[self safeCopyMetricsTo:&self->_frameQueueMetrics from:&frameQueueMetrics];
[[ImGuiPlots sharedInstance] observeFloat:PLOT_DROPPED value:framesDropped];
static CFTimeInterval lastHostFrame = 0.0f;
if (lastHostFrame != 0) {
[[ImGuiPlots sharedInstance] observeFloat:PLOT_HOST_FRAMETIME value:(frame.pts - lastHostFrame) * 1000.0];
}
lastHostFrame = frame.pts;
static PlotMetrics decodeMetrics = {};
[[ImGuiPlots sharedInstance] observeFloatReturnMetrics:PLOT_DECODE value:(CACurrentMediaTime() - decodeStartTime) * 1000.0 plotMetrics:&decodeMetrics];
[self safeCopyMetricsTo:&self->_decodeMetrics from:&decodeMetrics];
});
} else {
if (sampleBufferForAvsb) {
CFRelease(sampleBufferForAvsb);
}
}
} else if (self->_renderingBackend == RENDER_METAL) {
self.pipLayer.hidden = NO;
// Safely capture the format description for the background block.
__block CMVideoFormatDescriptionRef capturedFormatDesc = NULL;
@synchronized(self) { @synchronized(self) {
if (self->_formatDesc) { if (self.pipLayer.controlTimebase == NULL) {
capturedFormatDesc = (CMVideoFormatDescriptionRef)CFRetain(self->_formatDesc); NSLog(@"[Metal PiP Path] Timebase is NULL. Setting it now.");
CMTimebaseRef timebase;
CMTimebaseCreateWithSourceClock(kCFAllocatorDefault, CMClockGetHostTimeClock(), &timebase);
if (timebase) {
CMTimebaseSetTime(timebase, presentationTimestamp);
CMTimebaseSetRate(timebase, 1.0);
[self.pipLayer setControlTimebase:timebase];
CFRelease(timebase);
}
}
if (self->_pipFormatDesc == NULL || !CMVideoFormatDescriptionMatchesImageBuffer(self->_pipFormatDesc, imageBuffer)) {
NSLog(@"[Metal PiP Path] Format description mismatch or NULL. Creating new one.");
if (self->_pipFormatDesc) { CFRelease(self->_pipFormatDesc); }
OSStatus res = CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, imageBuffer, &(self->_pipFormatDesc));
if (res == noErr) {
NSLog(@"[Metal PiP Path] New format description created: %@", self->_pipFormatDesc);
} else {
NSLog(@"[Metal PiP Path] FAILED to create format description. Error: %d", (int)res);
}
}
if (self->_pipFormatDesc) {
CMSampleBufferRef sampleBufferForPip = NULL;
CMSampleTimingInfo sampleTiming = {kCMTimeInvalid, presentationTimestamp, presentationDuration};
OSStatus err = CMSampleBufferCreateReadyWithImageBuffer(kCFAllocatorDefault, imageBuffer, self->_pipFormatDesc, &sampleTiming, &sampleBufferForPip);
if (err == noErr && sampleBufferForPip) {
// NSLog(@"[Metal PiP Path] Sample buffer created. Enqueuing to _pipLayer.");
[self.pipLayer enqueueSampleBuffer:sampleBufferForPip];
CFRelease(sampleBufferForPip);
} else {
NSLog(@"[Metal PiP Path] FAILED to create sample buffer. Error: %d", (int)err);
}
} }
} }
CVPixelBufferRelease(imageBuffer);
if (!capturedFormatDesc) { });
Log(LOG_E, @"Failed to capture format description for Metal frame, dropping."); }
return; // --- MAIN RENDERER PATH ---
if (self->_renderingBackend == RENDER_AVSB) {
// self.pipLayer.hidden = YES;
if (self->_avsbFormatDesc == NULL || !CMVideoFormatDescriptionMatchesImageBuffer(self->_avsbFormatDesc, imageBuffer)) {
if (self->_avsbFormatDesc) {
CFRelease(self->_avsbFormatDesc);
} }
CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, imageBuffer, &(self->_avsbFormatDesc));
CVPixelBufferRef pixelBuffer = CVPixelBufferRetain((CVPixelBufferRef)imageBuffer);
dispatch_async(self->_vtq, ^{
Frame *frame = [[Frame alloc] initWithPixelBufffer:pixelBuffer frameNumber:frameNumber frameType:frameType pts:presentationTimestamp];
[frame setFormatDesc:capturedFormatDesc];
CFRelease(capturedFormatDesc);
int framesDropped = [self->_frameQueue enqueue:frame withSlackSize:3];
static PlotMetrics frameQueueMetrics = {};
[[ImGuiPlots sharedInstance] observeFloatReturnMetrics:PLOT_QUEUED_FRAMES value:[self->_frameQueue count] plotMetrics:&frameQueueMetrics];
[self safeCopyMetricsTo:&self->_frameQueueMetrics from:&frameQueueMetrics];
[[ImGuiPlots sharedInstance] observeFloat:PLOT_DROPPED value:framesDropped];
static CFTimeInterval lastHostFrame = 0.0f;
if (lastHostFrame != 0) {
[[ImGuiPlots sharedInstance] observeFloat:PLOT_HOST_FRAMETIME value:(frame.pts - lastHostFrame) * 1000.0];
}
lastHostFrame = frame.pts;
static PlotMetrics decodeMetrics = {};
[[ImGuiPlots sharedInstance] observeFloatReturnMetrics:PLOT_DECODE value:(CACurrentMediaTime() - decodeStartTime) * 1000.0 plotMetrics:&decodeMetrics];
[self safeCopyMetricsTo:&self->_decodeMetrics from:&decodeMetrics];
});
} }
});
CMSampleBufferRef sampleBufferForAvsb = NULL;
CMSampleTimingInfo sampleTiming = {kCMTimeInvalid, presentationTimestamp, presentationDuration};
OSStatus err = CMSampleBufferCreateReadyWithImageBuffer(kCFAllocatorDefault, imageBuffer, self->_avsbFormatDesc, &sampleTiming, &sampleBufferForAvsb);
if (err == noErr && sampleBufferForAvsb) {
dispatch_async(self->_vtq, ^{
Frame *frame = [[Frame alloc] initWithSampleBuffer:sampleBufferForAvsb frameNumber:frameNumber frameType:frameType];
int framesDropped = [self->_frameQueue enqueue:frame withSlackSize:3];
// ... stats and metrics updates
});
} else {
if (sampleBufferForAvsb) {
CFRelease(sampleBufferForAvsb);
}
}
} else if (self->_renderingBackend == RENDER_METAL) {
self.pipLayer.hidden = NO;
// Safely capture the format description for the background block.
__block CMVideoFormatDescriptionRef capturedFormatDesc = NULL;
@synchronized(self) {
if (self->_formatDesc) {
capturedFormatDesc = (CMVideoFormatDescriptionRef)CFRetain(self->_formatDesc);
}
}
if (!capturedFormatDesc) {
Log(LOG_E, @"Failed to capture format description for Metal frame, dropping.");
return;
}
CVPixelBufferRef pixelBuffer = CVPixelBufferRetain((CVPixelBufferRef)imageBuffer);
dispatch_async(self->_vtq, ^{
Frame *frame = [[Frame alloc] initWithPixelBufffer:pixelBuffer frameNumber:frameNumber frameType:frameType pts:presentationTimestamp];
[frame setFormatDesc:capturedFormatDesc];
CFRelease(capturedFormatDesc);
int framesDropped = [self->_frameQueue enqueue:frame withSlackSize:3];
static PlotMetrics frameQueueMetrics = {};
[[ImGuiPlots sharedInstance] observeFloatReturnMetrics:PLOT_QUEUED_FRAMES value:[self->_frameQueue count] plotMetrics:&frameQueueMetrics];
[self safeCopyMetricsTo:&self->_frameQueueMetrics from:&frameQueueMetrics];
[[ImGuiPlots sharedInstance] observeFloat:PLOT_DROPPED value:framesDropped];
static CFTimeInterval lastHostFrame = 0.0f;
if (lastHostFrame != 0) {
[[ImGuiPlots sharedInstance] observeFloat:PLOT_HOST_FRAMETIME value:(frame.pts - lastHostFrame) * 1000.0];
}
lastHostFrame = frame.pts;
static PlotMetrics decodeMetrics = {};
[[ImGuiPlots sharedInstance] observeFloatReturnMetrics:PLOT_DECODE value:(CACurrentMediaTime() - decodeStartTime) * 1000.0 plotMetrics:&decodeMetrics];
[self safeCopyMetricsTo:&self->_decodeMetrics from:&decodeMetrics];
});
}
});
return status; return status;
} }
- (void)setHdrMode:(BOOL)enabled { - (void)setHdrMode:(BOOL)enabled {
@@ -1157,38 +1121,25 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit);
- (void)resetFramePacing { - (void)resetFramePacing {
// Ensure this only runs for the AVSampleBuffer rendering backend and that the display link exists. // Ensure this only runs for the AVSampleBuffer rendering backend and that the display link exists.
if (_renderingBackend == RENDER_AVSB) { if (_renderingBackend == RENDER_AVSB && _displayLink) {
Log(LOG_I, @"Resetting video pipeline for AVSB renderer after resuming..."); Log(LOG_I, @"Frame pacing is being reset to %d FPS...", self->_frameRate);
// 1. Reset the receiver layers on the main thread // Toggling the paused state can help re-engage the display link with the
dispatch_async(dispatch_get_main_queue(), ^{ // run loop correctly after the app resumes from a background state like PiP.
// Reset the main display layer _displayLink.paused = YES;
if (self->_displayLayer) {
[self->_displayLayer flushAndRemoveImage]; // Re-apply the desired frame rate range. This is the critical hint for ProMotion
[self->_displayLayer setControlTimebase:NULL]; // that may have been lost or ignored during the PiP transition.
} if (@available(iOS 15.0, *)) {
_displayLink.preferredFrameRateRange = CAFrameRateRangeMake(self->_frameRate, self->_frameRate, self->_frameRate);
// FIX: Also reset the PiP layer if it exists } else {
if (self.pipLayer) { _displayLink.preferredFramesPerSecond = self->_frameRate;
[self.pipLayer flushAndRemoveImage];
[self.pipLayer setControlTimebase:NULL];
}
});
// 2. Reset the sender (the Decompression Session)
@synchronized(self) {
if (_decompressionSession != nil) {
VTDecompressionSessionInvalidate(_decompressionSession);
CFRelease(_decompressionSession);
_decompressionSession = nil;
}
} }
// 3. Request a new keyframe to restart the video stream cleanly // Resume the display link immediately.
LiRequestIdrFrame(); _displayLink.paused = NO;
} }
else if (_renderingBackend == RENDER_METAL) { else if (_renderingBackend == RENDER_METAL) {
// (The existing, correct logic for the Metal renderer remains here)
Log(LOG_I, @"Resetting video pipeline for Metal renderer after resuming..."); Log(LOG_I, @"Resetting video pipeline for Metal renderer after resuming...");
// 1. Reset the receiver (the PiP Layer) on the main thread // 1. Reset the receiver (the PiP Layer) on the main thread
VoidLink/ViewControllers/StreamFrameViewController.m
+13 -12
View File
@@ -97,7 +97,6 @@
- (void)pictureInPictureControllerWillStartPictureInPicture:(AVPictureInPictureController *)pictureInPictureController { - (void)pictureInPictureControllerWillStartPictureInPicture:(AVPictureInPictureController *)pictureInPictureController {
_streamView.hidden = YES; _streamView.hidden = YES;
_pipLayer.hidden = YES;
if (self.imguiView) { if (self.imguiView) {
self.imguiView.mtkView.hidden = YES; self.imguiView.mtkView.hidden = YES;
Log(LOG_I, @"Hiding ImGui view for PiP start."); Log(LOG_I, @"Hiding ImGui view for PiP start.");
@@ -117,7 +116,6 @@
- (void)pictureInPictureControllerDidStopPictureInPicture:(AVPictureInPictureController *)pictureInPictureController { - (void)pictureInPictureControllerDidStopPictureInPicture:(AVPictureInPictureController *)pictureInPictureController {
_streamView.hidden = NO; _streamView.hidden = NO;
_pipLayer.hidden = NO;
if (self.imguiView) { if (self.imguiView) {
self.imguiView.mtkView.hidden = NO; self.imguiView.mtkView.hidden = NO;
Log(LOG_I, @"Showing ImGui view after PiP stop."); Log(LOG_I, @"Showing ImGui view after PiP stop.");
@@ -131,7 +129,6 @@
- (void)pictureInPictureController:(AVPictureInPictureController *)pictureInPictureController restoreUserInterfaceForPictureInPictureStopWithCompletionHandler:(void (^)(BOOL restored))completionHandler { - (void)pictureInPictureController:(AVPictureInPictureController *)pictureInPictureController restoreUserInterfaceForPictureInPictureStopWithCompletionHandler:(void (^)(BOOL restored))completionHandler {
_isRestoringFromPiP = YES; _isRestoringFromPiP = YES;
_streamView.hidden = NO; _streamView.hidden = NO;
_pipLayer.hidden = NO;
if (self.imguiView) { if (self.imguiView) {
self.imguiView.mtkView.hidden = NO; self.imguiView.mtkView.hidden = NO;
Log(LOG_I, @"Showing ImGui view for PiP restore."); Log(LOG_I, @"Showing ImGui view for PiP restore.");
@@ -181,23 +178,27 @@
return (_settings.touchMode.intValue == RelativeTouch || _settings.touchMode.intValue == NativeTouch || _settings.touchMode.intValue == AbsoluteTouch) && _settings.onscreenControls.intValue == OnScreenControlsLevelCustom; return (_settings.touchMode.intValue == RelativeTouch || _settings.touchMode.intValue == NativeTouch || _settings.touchMode.intValue == AbsoluteTouch) && _settings.onscreenControls.intValue == OnScreenControlsLevelCustom;
} }
- (void)setupPiPControllerWithRenderer:(VideoDecoderRenderer *)videoRenderer { - (void)setupPiPControllerWithRenderer:(VideoDecoderRenderer *)videoRenderer { // Ensure we have the renderer and its layer
if (self.pipController) { if (self.pipController) {
return; return;
} }
Log(LOG_I, @"Setting up PiP controller..."); Log(LOG_I, @"Setting up PiP controller...");
// Both Metal and AVSB will use the dedicated _pipLayer for PiP. AVSampleBufferDisplayLayer* sourceLayer;
AVSampleBufferDisplayLayer* sourceLayer = _pipLayer;
if ([_settings.renderingBackend intValue] == RENDER_METAL) { if ([_settings.renderingBackend intValue] == RENDER_METAL) {
NSLog(@"[PiP LOG] Backend: Metal. Using dedicated _pipLayer: %p", sourceLayer); sourceLayer = _pipLayer;
NSLog(@"[PiP LOG] Backend: Metal. Using _pipLayer: %p", sourceLayer);
if (!sourceLayer) {
NSLog(@"[PiP LOG] ERROR: _pipLayer is nil during setup.");
return;
}
} else { // RENDER_AVSB } else { // RENDER_AVSB
NSLog(@"[PiP LOG] Backend: AVSB. Using dedicated _pipLayer: %p", sourceLayer); sourceLayer = videoRenderer.displayLayer;
NSLog(@"[PiP LOG] Backend: AVSB. Using _displayLayer: %p", sourceLayer);
} }
if (!sourceLayer) { if (!_pipLayer) {
Log(LOG_E, @"PiP setup failed: Dedicated PiP layer is nil."); Log(LOG_E, @"PiP setup failed: Dedicated PiP layer not ready.");
return; return;
} }