hermes-agent/VoidLink
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

revert: legacy frame pacing changes for VideoDecoderRenderer

Browse Source
This commit is contained in:
Acaki
2025-08-26 01:33:19 +08:00
parent 56729469bf
commit 8547cee1b9
1 changed files with 128 additions and 227 deletions
Download Patch File Download Diff File Expand all files Collapse all files
VoidLink/Stream/VideoDecoderRenderer.m
+128 -227
View File
@@ -55,13 +55,6 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size,
NSInteger _maxRefreshRate;
RenderingBackend _renderingBackend;
// Frame pacing mode
BOOL _useLegacyPacing;
// Legacy pacing frame counter and queue management
NSInteger _simplePacingFrameCount;
NSInteger _enqueuedSampleBuffers;
}
- (void)reinitializeDisplayLayer
@@ -122,11 +115,11 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size,
self = [super init];
_sq = dispatch_queue_create("com.moonlight.VideoDecoderRenderer",
dispatch_queue_attr_make_with_qos_class(DISPATCH_QUEUE_SERIAL, QOS_CLASS_USER_INTERACTIVE, 0));
dispatch_queue_attr_make_with_qos_class(DISPATCH_QUEUE_SERIAL, QOS_CLASS_USER_INTERACTIVE, 0));
// Video decoder needs to run at the highest priority since DisplayLink waits on it
_vtq = dispatch_queue_create("com.moonlight.VideoDecoderRenderer.VTDecoder",
dispatch_queue_attr_make_with_qos_class(DISPATCH_QUEUE_SERIAL, QOS_CLASS_USER_INTERACTIVE, 0));
dispatch_queue_attr_make_with_qos_class(DISPATCH_QUEUE_SERIAL, QOS_CLASS_USER_INTERACTIVE, 0));
_view = view;
_callbacks = callbacks;
@@ -136,22 +129,16 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size,
DataManager* dataMan = [[DataManager alloc] init];
// Check if we should use legacy pacing mode
_useLegacyPacing = [[dataMan getSettings].framePacingMode integerValue] == FramePacingModeLegacy;
if (!_useLegacyPacing) {
// Queue pacing mode uses FrameQueue
_frameQueue = [FrameQueue sharedInstance];
[_frameQueue start];
[_frameQueue setHighWaterMark:(int)[[dataMan getSettings].frameQueueSize integerValue]];
}
_frameQueue = [FrameQueue sharedInstance];
[_frameQueue start];
[_frameQueue setHighWaterMark:(int)[[dataMan getSettings].frameQueueSize integerValue]];
[self reinitializeDisplayLayer];
[[NSNotificationCenter defaultCenter] addObserver:self
selector:@selector(reinitializeDisplayLayer)
name:@"ScreenChanged"
object:nil];
selector:@selector(reinitializeDisplayLayer)
name:@"ScreenChanged"
object:nil];
return self;
}
@@ -168,17 +155,11 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size,
DataManager* dataMan = [[DataManager alloc] init];
if ([[dataMan getSettings].renderingBackend integerValue] == RENDER_AVSB) {
// PACING_MODE_VSYNC:
// Deliver 1 frame at each vsync interval. Ignores server pts timestamps.
// Drop frames intelligently to maintain chosen queue size.
_renderingBackend = RENDER_AVSB;
if (_useLegacyPacing) {
// Legacy pacing mode
_displayLink = [CADisplayLink displayLinkWithTarget:self selector:@selector(renderModeLegacy:)];
} else {
// PACING_MODE_VSYNC:
// Deliver 1 frame at each vsync interval. Ignores server pts timestamps.
// Drop frames intelligently to maintain chosen queue size.
_displayLink = [CADisplayLink displayLinkWithTarget:self selector:@selector(renderModeAVSB:)];
}
_displayLink = [CADisplayLink displayLinkWithTarget:self selector:@selector(renderModeAVSB:)];
if (@available(iOS 15.0, tvOS 15.0, *)) {
_displayLink.preferredFrameRateRange = CAFrameRateRangeMake(self->_frameRate, self->_frameRate, self->_frameRate);
@@ -195,6 +176,7 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size,
- (void)setupDecompressionSessionWithAttributes:(NSDictionary *)destinationPixelBufferAttributes {
// This method is called from within synchronized block, so no additional sync needed here
if (_decompressionSession != NULL) {
VTDecompressionSessionInvalidate(_decompressionSession);
CFRelease(_decompressionSession);
@@ -238,7 +220,7 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size,
destinationPixelBufferAttributes[(id)kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder] = @YES;
destinationPixelBufferAttributes[(id)kVTDecompressionPropertyKey_GeneratePerFrameHDRDisplayMetadata] = @YES;
}
return [self setupDecompressionSessionWithAttributes:destinationPixelBufferAttributes];
}
@@ -260,33 +242,7 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size,
int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit);
#pragma mark DisplayLink - Legacy Frame Pacing
// Legacy frame pacing method from upstream/Integration - direct polling without FrameQueue
- (void)renderModeLegacy:(CADisplayLink *)link {
VIDEO_FRAME_HANDLE handle;
PDECODE_UNIT du;
while (LiPollNextVideoFrame(&handle, &du)) {
LiCompleteVideoFrame(handle, DrSubmitDecodeUnit(du));
// Calculate the actual display refresh rate
double displayRefreshRate = 1 / (link.targetTimestamp - link.timestamp);
// Only pace frames if the display refresh rate is >= 90% of our stream frame rate.
// Battery saver, accessibility settings, or device thermals can cause the actual
// refresh rate of the display to drop below the physical maximum.
if (displayRefreshRate >= _frameRate * 0.9f) {
// Keep one pending frame to smooth out gaps due to
// network jitter at the cost of 1 frame of latency
if (LiGetPendingVideoFrames() == 1) {
break;
}
}
}
}
#pragma mark DisplayLink - Frame Pacing - Vsync with FrameQueue (Queue Pacing)
#pragma mark DisplayLink - Frame Pacing - Vsync with FrameQueue
// This frame pacing method was inspired by the behavior of moonlight-qt's Pacer class, although it has evolved
// a few additional features. Incoming frames from Sunshine are asynchronously processed into a queue by the VideoRecv thread.
@@ -338,11 +294,9 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit);
// we missed a callback
// Log(LOG_W, @"*** slow frametime %.3f ms", frametime * 1000.0);
}
dispatch_async(dispatch_get_main_queue(), ^{
if ([[UIApplication sharedApplication] applicationState] != UIApplicationStateBackground) {
[[ImGuiPlots sharedInstance] observeFloat:PLOT_FRAMETIME value:frametime * 1000.0];
}
});
if ([[UIApplication sharedApplication] applicationState] != UIApplicationStateBackground) {
[[ImGuiPlots sharedInstance] observeFloat:PLOT_FRAMETIME value:frametime * 1000.0];
}
}
lastTargetLocal = targetLocal;
@@ -410,9 +364,7 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit);
}
- (void)cleanup
{
if (_frameQueue) {
[_frameQueue stop];
}
[_frameQueue stop];
if (_renderingBackend == RENDER_AVSB) {
[_displayLink invalidate];
@@ -448,7 +400,7 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit);
// Write the length prefix to the new buffer
const int dataLength = nalLength - NALU_START_PREFIX_SIZE;
const uint8_t lengthBytes[] = {(uint8_t)(dataLength >> 24), (uint8_t)(dataLength >> 16),
(uint8_t)(dataLength >> 8), (uint8_t)dataLength};
(uint8_t)(dataLength >> 8), (uint8_t)dataLength};
status = CMBlockBufferReplaceDataBytes(lengthBytes, frameBuffer,
oldOffset, NAL_LENGTH_PREFIX_SIZE);
if (status != noErr) {
@@ -660,9 +612,9 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit);
// Referenced the VP9 code in Chrome that performs a similar function
// https://source.chromium.org/chromium/chromium/src/+/main:media/gpu/mac/vt_config_util.mm;drc=977dc02c431b4979e34c7792bc3d646f649dacb4;l=155
extensions[(__bridge NSString*)kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms] =
@{
@"av1C" : [self getAv1CodecConfigurationBox:frameData],
};
@{
@"av1C" : [self getAv1CodecConfigurationBox:frameData],
};
extensions[@"BitsPerComponent"] = @(bitstreamCtx->bit_depth);
#undef SET_EXTENSION
@@ -871,10 +823,10 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit);
CMSampleBufferRef sampleBuffer;
status = CMSampleBufferCreateReady(kCFAllocatorDefault,
frameBlockBuffer,
_formatDesc, 1, 1,
&sampleTiming, 0, NULL,
&sampleBuffer);
frameBlockBuffer,
_formatDesc, 1, 1,
&sampleTiming, 0, NULL,
&sampleBuffer);
if (status != noErr) {
Log(LOG_E, @"CMSampleBufferCreate failed: %d", (int)status);
CFRelease(dataBlockBuffer);
@@ -898,163 +850,115 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit);
frameNumber:(int)frameNumber
frameType:(int)frameType
decodeStartTime:(CFTimeInterval)decodeStartTime {
// Synchronize access to decompression session to prevent race conditions during background/foreground transitions
@synchronized(self) {
// Check if we need to create/recreate the decompression session
BOOL needsNewSession = (frameType == FRAME_TYPE_IDR || _decompressionSession == nil);
// Synchronize access to decompression session to prevent race conditions during background/foreground transitions
@synchronized(self) {
// Check if we need to create/recreate the decompression session
BOOL needsNewSession = (frameType == FRAME_TYPE_IDR || _decompressionSession == nil);
// Also check if the session might have been invalidated by iOS during background
if (!needsNewSession && _decompressionSession != nil) {
Boolean isValid = VTDecompressionSessionCanAcceptFormatDescription(_decompressionSession, _formatDesc);
if (!isValid) {
Log(LOG_W, @"Decompression session is invalid, needs recreation");
needsNewSession = YES;
}
}
// Also check if the session might have been invalidated by iOS during background
if (!needsNewSession && _decompressionSession != nil) {
Boolean isValid = VTDecompressionSessionCanAcceptFormatDescription(_decompressionSession, _formatDesc);
if (!isValid) {
Log(LOG_W, @"Decompression session is invalid, needs recreation");
needsNewSession = YES;
}
}
if (needsNewSession) {
[self setupDecompressionSession];
}
if (needsNewSession) {
[self setupDecompressionSession];
}
if (_decompressionSession == nil) {
Log(LOG_E, @"Failed to create decompression session");
return kVTInvalidSessionErr;
}
if (_decompressionSession == nil) {
Log(LOG_E, @"Failed to create decompression session");
return kVTInvalidSessionErr;
}
OSStatus status = VTDecompressionSessionDecodeFrameWithOutputHandler(
_decompressionSession,
sampleBuffer,
0,
NULL,
^(OSStatus status, VTDecodeInfoFlags infoFlags, CVImageBufferRef _Nullable imageBuffer, CMTime presentationTimestamp, CMTime presentationDuration) {
if (status != noErr || !imageBuffer) {
NSError *error = [NSError errorWithDomain:NSOSStatusErrorDomain code:status userInfo:nil];
Log(LOG_E, @"Decompression session error: %@", error);
if (status == kVTInvalidSessionErr) {
// The session was invalidated by the OS. Destroy our reference
// so it gets recreated on the next IDR frame.
@synchronized(self) {
if (self->_decompressionSession) {
VTDecompressionSessionInvalidate(self->_decompressionSession);
CFRelease(self->_decompressionSession);
self->_decompressionSession = nil;
OSStatus status = VTDecompressionSessionDecodeFrameWithOutputHandler(
_decompressionSession,
sampleBuffer,
0,
NULL,
^(OSStatus status, VTDecodeInfoFlags infoFlags, CVImageBufferRef _Nullable imageBuffer, CMTime presentationTimestamp, CMTime presentationDuration) {
if (status != noErr || !imageBuffer) {
NSError *error = [NSError errorWithDomain:NSOSStatusErrorDomain code:status userInfo:nil];
Log(LOG_E, @"Decompression session error: %@", error);
if (status == kVTInvalidSessionErr) {
// The session was invalidated by the OS. Destroy our reference
// so it gets recreated on the next IDR frame.
@synchronized(self) {
if (self->_decompressionSession) {
VTDecompressionSessionInvalidate(self->_decompressionSession);
CFRelease(self->_decompressionSession);
self->_decompressionSession = nil;
}
}
}
LiRequestIdrFrame(); // Request an IDR to restart the stream
return;
}
}
LiRequestIdrFrame(); // Request an IDR to restart the stream
return;
}
CMSampleBufferRef sampleBufferOut = nil;
CVPixelBufferRef pixelBuffer = nil;
CMSampleBufferRef sampleBufferOut = nil;
CVPixelBufferRef pixelBuffer = nil;
// AVSampleBuffer path: package into a SampleBuffer
if (self->_renderingBackend == RENDER_AVSB) {
if (self->_formatDescImageBuffer == NULL || !CMVideoFormatDescriptionMatchesImageBuffer(self->_formatDescImageBuffer, imageBuffer)) {
OSStatus res = CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, imageBuffer, &(self->_formatDescImageBuffer));
if (res != noErr) {
Log(LOG_E, @"Failed to create video format description from imageBuffer");
return;
}
}
// AVSampleBuffer path: package into a SampleBuffer
if (self->_renderingBackend == RENDER_AVSB) {
if (self->_formatDescImageBuffer == NULL || !CMVideoFormatDescriptionMatchesImageBuffer(self->_formatDescImageBuffer, imageBuffer)) {
OSStatus res = CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, imageBuffer, &(self->_formatDescImageBuffer));
if (res != noErr) {
Log(LOG_E, @"Failed to create video format description from imageBuffer");
return;
}
}
CMSampleTimingInfo sampleTiming = {kCMTimeInvalid, presentationTimestamp, presentationDuration};
CMSampleTimingInfo sampleTiming = {kCMTimeInvalid, presentationTimestamp, presentationDuration};
OSStatus err =
CMSampleBufferCreateReadyWithImageBuffer(kCFAllocatorDefault, imageBuffer, self->_formatDescImageBuffer, &sampleTiming, &sampleBufferOut);
if (err != noErr) {
Log(LOG_E, @"Error creating sample buffer for decompressed image buffer %d", (int)err);
return;
}
} else if (self->_renderingBackend == RENDER_METAL) {
// Metal path: retain the pixelBuffer here so it survives the dispatch
pixelBuffer = CVPixelBufferRetain((CVPixelBufferRef)imageBuffer);
}
OSStatus err =
CMSampleBufferCreateReadyWithImageBuffer(kCFAllocatorDefault, imageBuffer, self->_formatDescImageBuffer, &sampleTiming, &sampleBufferOut);
if (err != noErr) {
Log(LOG_E, @"Error creating sample buffer for decompressed image buffer %d", (int)err);
return;
}
} else if (self->_renderingBackend == RENDER_METAL) {
// Metal path: retain the pixelBuffer here so it survives the dispatch
pixelBuffer = CVPixelBufferRetain((CVPixelBufferRef)imageBuffer);
}
// Dispatch onto our higher priority queue
dispatch_async(self->_vtq, ^{
if (self->_useLegacyPacing && self->_renderingBackend == RENDER_AVSB) {
// Legacy pacing: Process all frames normally, but with immediate display timing
// Set presentation time to current time for immediate display
CMTime targetTime = CMTimeMakeWithSeconds(CACurrentMediaTime(), NSEC_PER_SEC);
CMSampleBufferSetOutputPresentationTimeStamp(sampleBufferOut, targetTime);
// Dispatch display layer operations to main thread to avoid CATransaction warnings
dispatch_async(dispatch_get_main_queue(), ^{
if (frameType == FRAME_TYPE_IDR) {
// Ensure the layer is visible now
self->_displayLayer.hidden = NO;
// Dispatch onto our higher priority queue
dispatch_async(self->_vtq, ^{
Frame *frame = nil;
if (self->_renderingBackend == RENDER_AVSB) {
frame = [[Frame alloc] initWithSampleBuffer:sampleBufferOut frameNumber:frameNumber frameType:frameType];
} else {
frame = [[Frame alloc] initWithPixelBufffer:pixelBuffer frameNumber:frameNumber frameType:frameType pts:presentationTimestamp];
[frame setFormatDesc:self->_formatDesc];
}
int framesDropped = [self->_frameQueue enqueue:frame withSlackSize:3];
// Tell our parent VC to hide the progress indicator
[self->_callbacks videoContentShown];
}
if ([[UIApplication sharedApplication] applicationState] != UIApplicationStateBackground) {
static PlotMetrics frameQueueMetrics = {};
[[ImGuiPlots sharedInstance] observeFloatReturnMetrics:PLOT_QUEUED_FRAMES value:[self->_frameQueue count] plotMetrics:&frameQueueMetrics];
[self safeCopyMetricsTo:&self->_frameQueueMetrics from:&frameQueueMetrics];
if ([self->_displayLayer controlTimebase] == NULL) {
// On first frame, set timebase
CMTimebaseRef timebase = NULL;
CMTimebaseCreateWithSourceClock(CFAllocatorGetDefault(), CMClockGetHostTimeClock(), &timebase);
[[ImGuiPlots sharedInstance] observeFloat:PLOT_DROPPED value:framesDropped];
CMTime pts = CMSampleBufferGetOutputPresentationTimeStamp(sampleBufferOut);
CMTimebaseSetTime(timebase, pts);
CMTimebaseSetRate(timebase, 1.0);
// It's important we capture host metrics on the incoming thread, as this frame object
// may have been dropped by the above enqueue
static CFTimeInterval lastHostFrame = 0.0f;
if (lastHostFrame != 0) {
[[ImGuiPlots sharedInstance] observeFloat:PLOT_HOST_FRAMETIME value:(frame.pts - lastHostFrame) * 1000.0];
}
lastHostFrame = frame.pts;
[self->_displayLayer setControlTimebase:timebase];
if (timebase) {
CFRelease(timebase);
}
Log(LOG_I, @"Setting timebase for legacy pacing to %d / %d", pts.value, pts.timescale);
}
// Decode time is not graphed because it is marked as hidden, but we can use the same mechanism for the value used by stats
static PlotMetrics decodeMetrics = {};
[[ImGuiPlots sharedInstance] observeFloatReturnMetrics:PLOT_DECODE value:(CACurrentMediaTime() - decodeStartTime) * 1000.0 plotMetrics:&decodeMetrics];
[self safeCopyMetricsTo:&self->_decodeMetrics from:&decodeMetrics];
}
});
});
[self->_displayLayer enqueueSampleBuffer:sampleBufferOut];
CFRelease(sampleBufferOut);
});
} else {
// Queue pacing: use FrameQueue
Frame *frame = nil;
if (self->_renderingBackend == RENDER_AVSB) {
frame = [[Frame alloc] initWithSampleBuffer:sampleBufferOut frameNumber:frameNumber frameType:frameType];
} else {
frame = [[Frame alloc] initWithPixelBufffer:pixelBuffer frameNumber:frameNumber frameType:frameType pts:presentationTimestamp];
[frame setFormatDesc:self->_formatDesc];
}
int framesDropped = [self->_frameQueue enqueue:frame withSlackSize:3];
// Capture metrics on main thread to avoid UIApplication thread warning
int frameQueueCount = [self->_frameQueue count];
CFTimeInterval capturedDecodeTime = CACurrentMediaTime() - decodeStartTime;
CFTimeInterval hostFramePts = frame.pts;
dispatch_async(dispatch_get_main_queue(), ^{
if ([[UIApplication sharedApplication] applicationState] != UIApplicationStateBackground) {
static PlotMetrics frameQueueMetrics = {};
[[ImGuiPlots sharedInstance] observeFloatReturnMetrics:PLOT_QUEUED_FRAMES value:frameQueueCount plotMetrics:&frameQueueMetrics];
[self safeCopyMetricsTo:&self->_frameQueueMetrics from:&frameQueueMetrics];
[[ImGuiPlots sharedInstance] observeFloat:PLOT_DROPPED value:framesDropped];
// It's important we capture host metrics on the incoming thread, as this frame object
// may have been dropped by the above enqueue
static CFTimeInterval lastHostFrame = 0.0f;
if (lastHostFrame != 0) {
[[ImGuiPlots sharedInstance] observeFloat:PLOT_HOST_FRAMETIME value:(hostFramePts - lastHostFrame) * 1000.0];
}
lastHostFrame = hostFramePts;
// Decode time is not graphed because it is marked as hidden, but we can use the same mechanism for the value used by stats
static PlotMetrics decodeMetrics = {};
[[ImGuiPlots sharedInstance] observeFloatReturnMetrics:PLOT_DECODE
value:capturedDecodeTime * 1000.0
plotMetrics:&decodeMetrics];
[self safeCopyMetricsTo:&self->_decodeMetrics from:&decodeMetrics];
}
});
}
});
});
return status;
}
return status;
}
}
- (void)setHdrMode:(BOOL)enabled {
@@ -1066,10 +970,10 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit);
if (hasMetadata && hdrMetadata.displayPrimaries[0].x != 0 && hdrMetadata.maxDisplayLuminance != 0) {
// This data is all in big-endian
struct {
vector_ushort2 primaries[3];
vector_ushort2 white_point;
uint32_t luminance_max;
uint32_t luminance_min;
vector_ushort2 primaries[3];
vector_ushort2 white_point;
uint32_t luminance_max;
uint32_t luminance_min;
} __attribute__((packed, aligned(4))) mdcv;
// mdcv is in GBR order while SS_HDR_METADATA is in RGB order
@@ -1147,16 +1051,13 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit);
if (_renderingBackend == RENDER_METAL) {
stats->renderingBackendString = [NSString stringWithFormat:@"Metal, colorspace: %@", [MetalVideoRenderer currentColorSpace]];
} else {
NSString *pacingMode = _useLegacyPacing ? @"Legacy" : @"Queue";
stats->renderingBackendString = [NSString stringWithFormat:@"AVSampleBuffer (%@ pacing)", pacingMode];
stats->renderingBackendString = @"AVSampleBuffer";
}
dispatch_sync(_sq, ^{
memcpy(&stats->decodeMetrics, &_decodeMetrics, sizeof(PlotMetrics));
if (_frameQueue) {
memcpy(&stats->frameQueueMetrics, &_frameQueueMetrics, sizeof(PlotMetrics));
[_frameQueue.frameDropMetrics copyMetrics:&stats->frameDropMetrics];
}
memcpy(&stats->frameQueueMetrics, &_frameQueueMetrics, sizeof(PlotMetrics));
[_frameQueue.frameDropMetrics copyMetrics:&stats->frameDropMetrics];
});
}
@@ -1168,7 +1069,7 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit);
static int lastTargetRate = 0;
int targetRate = (int)_maxRefreshRate;
if (_maxRefreshRate <= 60 || _maxRefreshRate == 90 || !_frameQueue) {
if (_maxRefreshRate <= 60 || _maxRefreshRate == 90) {
return;
}