diff --git a/VoidLink/Stream/VideoDecoderRenderer.m b/VoidLink/Stream/VideoDecoderRenderer.m index 48384b07..3996a48a 100644 --- a/VoidLink/Stream/VideoDecoderRenderer.m +++ b/VoidLink/Stream/VideoDecoderRenderer.m @@ -55,13 +55,6 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size, NSInteger _maxRefreshRate; RenderingBackend _renderingBackend; - // Frame pacing mode - BOOL _useLegacyPacing; - - // Legacy pacing frame counter and queue management - NSInteger _simplePacingFrameCount; - NSInteger _enqueuedSampleBuffers; - } - (void)reinitializeDisplayLayer @@ -122,11 +115,11 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size, self = [super init]; _sq = dispatch_queue_create("com.moonlight.VideoDecoderRenderer", - dispatch_queue_attr_make_with_qos_class(DISPATCH_QUEUE_SERIAL, QOS_CLASS_USER_INTERACTIVE, 0)); + dispatch_queue_attr_make_with_qos_class(DISPATCH_QUEUE_SERIAL, QOS_CLASS_USER_INTERACTIVE, 0)); // Video decoder needs to run at the highest priority since DisplayLink waits on it _vtq = dispatch_queue_create("com.moonlight.VideoDecoderRenderer.VTDecoder", - dispatch_queue_attr_make_with_qos_class(DISPATCH_QUEUE_SERIAL, QOS_CLASS_USER_INTERACTIVE, 0)); + dispatch_queue_attr_make_with_qos_class(DISPATCH_QUEUE_SERIAL, QOS_CLASS_USER_INTERACTIVE, 0)); _view = view; _callbacks = callbacks; @@ -136,22 +129,16 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size, DataManager* dataMan = [[DataManager alloc] init]; - // Check if we should use legacy pacing mode - _useLegacyPacing = [[dataMan getSettings].framePacingMode integerValue] == FramePacingModeLegacy; - - if (!_useLegacyPacing) { - // Queue pacing mode uses FrameQueue - _frameQueue = [FrameQueue sharedInstance]; - [_frameQueue start]; - [_frameQueue setHighWaterMark:(int)[[dataMan getSettings].frameQueueSize integerValue]]; - } + _frameQueue = [FrameQueue sharedInstance]; + [_frameQueue start]; + [_frameQueue setHighWaterMark:(int)[[dataMan getSettings].frameQueueSize integerValue]]; [self reinitializeDisplayLayer]; [[NSNotificationCenter defaultCenter] addObserver:self - selector:@selector(reinitializeDisplayLayer) - name:@"ScreenChanged" - object:nil]; + selector:@selector(reinitializeDisplayLayer) + name:@"ScreenChanged" + object:nil]; return self; } @@ -168,17 +155,11 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size, DataManager* dataMan = [[DataManager alloc] init]; if ([[dataMan getSettings].renderingBackend integerValue] == RENDER_AVSB) { + // PACING_MODE_VSYNC: + // Deliver 1 frame at each vsync interval. Ignores server pts timestamps. + // Drop frames intelligently to maintain chosen queue size. _renderingBackend = RENDER_AVSB; - - if (_useLegacyPacing) { - // Legacy pacing mode - _displayLink = [CADisplayLink displayLinkWithTarget:self selector:@selector(renderModeLegacy:)]; - } else { - // PACING_MODE_VSYNC: - // Deliver 1 frame at each vsync interval. Ignores server pts timestamps. - // Drop frames intelligently to maintain chosen queue size. - _displayLink = [CADisplayLink displayLinkWithTarget:self selector:@selector(renderModeAVSB:)]; - } + _displayLink = [CADisplayLink displayLinkWithTarget:self selector:@selector(renderModeAVSB:)]; if (@available(iOS 15.0, tvOS 15.0, *)) { _displayLink.preferredFrameRateRange = CAFrameRateRangeMake(self->_frameRate, self->_frameRate, self->_frameRate); @@ -195,6 +176,7 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size, - (void)setupDecompressionSessionWithAttributes:(NSDictionary *)destinationPixelBufferAttributes { + // This method is called from within synchronized block, so no additional sync needed here if (_decompressionSession != NULL) { VTDecompressionSessionInvalidate(_decompressionSession); CFRelease(_decompressionSession); @@ -238,7 +220,7 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size, destinationPixelBufferAttributes[(id)kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder] = @YES; destinationPixelBufferAttributes[(id)kVTDecompressionPropertyKey_GeneratePerFrameHDRDisplayMetadata] = @YES; } - + return [self setupDecompressionSessionWithAttributes:destinationPixelBufferAttributes]; } @@ -260,33 +242,7 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size, int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit); -#pragma mark DisplayLink - Legacy Frame Pacing - -// Legacy frame pacing method from upstream/Integration - direct polling without FrameQueue -- (void)renderModeLegacy:(CADisplayLink *)link { - VIDEO_FRAME_HANDLE handle; - PDECODE_UNIT du; - - while (LiPollNextVideoFrame(&handle, &du)) { - LiCompleteVideoFrame(handle, DrSubmitDecodeUnit(du)); - - // Calculate the actual display refresh rate - double displayRefreshRate = 1 / (link.targetTimestamp - link.timestamp); - - // Only pace frames if the display refresh rate is >= 90% of our stream frame rate. - // Battery saver, accessibility settings, or device thermals can cause the actual - // refresh rate of the display to drop below the physical maximum. - if (displayRefreshRate >= _frameRate * 0.9f) { - // Keep one pending frame to smooth out gaps due to - // network jitter at the cost of 1 frame of latency - if (LiGetPendingVideoFrames() == 1) { - break; - } - } - } -} - -#pragma mark DisplayLink - Frame Pacing - Vsync with FrameQueue (Queue Pacing) +#pragma mark DisplayLink - Frame Pacing - Vsync with FrameQueue // This frame pacing method was inspired by the behavior of moonlight-qt's Pacer class, although it has evolved // a few additional features. Incoming frames from Sunshine are asynchronously processed into a queue by the VideoRecv thread. @@ -338,11 +294,9 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit); // we missed a callback // Log(LOG_W, @"*** slow frametime %.3f ms", frametime * 1000.0); } - dispatch_async(dispatch_get_main_queue(), ^{ - if ([[UIApplication sharedApplication] applicationState] != UIApplicationStateBackground) { - [[ImGuiPlots sharedInstance] observeFloat:PLOT_FRAMETIME value:frametime * 1000.0]; - } - }); + if ([[UIApplication sharedApplication] applicationState] != UIApplicationStateBackground) { + [[ImGuiPlots sharedInstance] observeFloat:PLOT_FRAMETIME value:frametime * 1000.0]; + } } lastTargetLocal = targetLocal; @@ -410,9 +364,7 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit); } - (void)cleanup { - if (_frameQueue) { - [_frameQueue stop]; - } + [_frameQueue stop]; if (_renderingBackend == RENDER_AVSB) { [_displayLink invalidate]; @@ -448,7 +400,7 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit); // Write the length prefix to the new buffer const int dataLength = nalLength - NALU_START_PREFIX_SIZE; const uint8_t lengthBytes[] = {(uint8_t)(dataLength >> 24), (uint8_t)(dataLength >> 16), - (uint8_t)(dataLength >> 8), (uint8_t)dataLength}; + (uint8_t)(dataLength >> 8), (uint8_t)dataLength}; status = CMBlockBufferReplaceDataBytes(lengthBytes, frameBuffer, oldOffset, NAL_LENGTH_PREFIX_SIZE); if (status != noErr) { @@ -660,9 +612,9 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit); // Referenced the VP9 code in Chrome that performs a similar function // https://source.chromium.org/chromium/chromium/src/+/main:media/gpu/mac/vt_config_util.mm;drc=977dc02c431b4979e34c7792bc3d646f649dacb4;l=155 extensions[(__bridge NSString*)kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms] = - @{ - @"av1C" : [self getAv1CodecConfigurationBox:frameData], - }; + @{ + @"av1C" : [self getAv1CodecConfigurationBox:frameData], + }; extensions[@"BitsPerComponent"] = @(bitstreamCtx->bit_depth); #undef SET_EXTENSION @@ -871,10 +823,10 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit); CMSampleBufferRef sampleBuffer; status = CMSampleBufferCreateReady(kCFAllocatorDefault, - frameBlockBuffer, - _formatDesc, 1, 1, - &sampleTiming, 0, NULL, - &sampleBuffer); + frameBlockBuffer, + _formatDesc, 1, 1, + &sampleTiming, 0, NULL, + &sampleBuffer); if (status != noErr) { Log(LOG_E, @"CMSampleBufferCreate failed: %d", (int)status); CFRelease(dataBlockBuffer); @@ -898,163 +850,115 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit); frameNumber:(int)frameNumber frameType:(int)frameType decodeStartTime:(CFTimeInterval)decodeStartTime { - // Synchronize access to decompression session to prevent race conditions during background/foreground transitions - @synchronized(self) { - // Check if we need to create/recreate the decompression session - BOOL needsNewSession = (frameType == FRAME_TYPE_IDR || _decompressionSession == nil); + // Synchronize access to decompression session to prevent race conditions during background/foreground transitions + @synchronized(self) { + // Check if we need to create/recreate the decompression session + BOOL needsNewSession = (frameType == FRAME_TYPE_IDR || _decompressionSession == nil); - // Also check if the session might have been invalidated by iOS during background - if (!needsNewSession && _decompressionSession != nil) { - Boolean isValid = VTDecompressionSessionCanAcceptFormatDescription(_decompressionSession, _formatDesc); - if (!isValid) { - Log(LOG_W, @"Decompression session is invalid, needs recreation"); - needsNewSession = YES; - } - } + // Also check if the session might have been invalidated by iOS during background + if (!needsNewSession && _decompressionSession != nil) { + Boolean isValid = VTDecompressionSessionCanAcceptFormatDescription(_decompressionSession, _formatDesc); + if (!isValid) { + Log(LOG_W, @"Decompression session is invalid, needs recreation"); + needsNewSession = YES; + } + } - if (needsNewSession) { - [self setupDecompressionSession]; - } + if (needsNewSession) { + [self setupDecompressionSession]; + } - if (_decompressionSession == nil) { - Log(LOG_E, @"Failed to create decompression session"); - return kVTInvalidSessionErr; - } + if (_decompressionSession == nil) { + Log(LOG_E, @"Failed to create decompression session"); + return kVTInvalidSessionErr; + } - OSStatus status = VTDecompressionSessionDecodeFrameWithOutputHandler( - _decompressionSession, - sampleBuffer, - 0, - NULL, - ^(OSStatus status, VTDecodeInfoFlags infoFlags, CVImageBufferRef _Nullable imageBuffer, CMTime presentationTimestamp, CMTime presentationDuration) { - if (status != noErr || !imageBuffer) { - NSError *error = [NSError errorWithDomain:NSOSStatusErrorDomain code:status userInfo:nil]; - Log(LOG_E, @"Decompression session error: %@", error); - if (status == kVTInvalidSessionErr) { - // The session was invalidated by the OS. Destroy our reference - // so it gets recreated on the next IDR frame. - @synchronized(self) { - if (self->_decompressionSession) { - VTDecompressionSessionInvalidate(self->_decompressionSession); - CFRelease(self->_decompressionSession); - self->_decompressionSession = nil; + OSStatus status = VTDecompressionSessionDecodeFrameWithOutputHandler( + _decompressionSession, + sampleBuffer, + 0, + NULL, + ^(OSStatus status, VTDecodeInfoFlags infoFlags, CVImageBufferRef _Nullable imageBuffer, CMTime presentationTimestamp, CMTime presentationDuration) { + if (status != noErr || !imageBuffer) { + NSError *error = [NSError errorWithDomain:NSOSStatusErrorDomain code:status userInfo:nil]; + Log(LOG_E, @"Decompression session error: %@", error); + if (status == kVTInvalidSessionErr) { + // The session was invalidated by the OS. Destroy our reference + // so it gets recreated on the next IDR frame. + @synchronized(self) { + if (self->_decompressionSession) { + VTDecompressionSessionInvalidate(self->_decompressionSession); + CFRelease(self->_decompressionSession); + self->_decompressionSession = nil; + } + } } + LiRequestIdrFrame(); // Request an IDR to restart the stream + return; } - } - LiRequestIdrFrame(); // Request an IDR to restart the stream - return; - } - CMSampleBufferRef sampleBufferOut = nil; - CVPixelBufferRef pixelBuffer = nil; + CMSampleBufferRef sampleBufferOut = nil; + CVPixelBufferRef pixelBuffer = nil; - // AVSampleBuffer path: package into a SampleBuffer - if (self->_renderingBackend == RENDER_AVSB) { - if (self->_formatDescImageBuffer == NULL || !CMVideoFormatDescriptionMatchesImageBuffer(self->_formatDescImageBuffer, imageBuffer)) { - OSStatus res = CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, imageBuffer, &(self->_formatDescImageBuffer)); - if (res != noErr) { - Log(LOG_E, @"Failed to create video format description from imageBuffer"); - return; - } - } + // AVSampleBuffer path: package into a SampleBuffer + if (self->_renderingBackend == RENDER_AVSB) { + if (self->_formatDescImageBuffer == NULL || !CMVideoFormatDescriptionMatchesImageBuffer(self->_formatDescImageBuffer, imageBuffer)) { + OSStatus res = CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, imageBuffer, &(self->_formatDescImageBuffer)); + if (res != noErr) { + Log(LOG_E, @"Failed to create video format description from imageBuffer"); + return; + } + } - CMSampleTimingInfo sampleTiming = {kCMTimeInvalid, presentationTimestamp, presentationDuration}; + CMSampleTimingInfo sampleTiming = {kCMTimeInvalid, presentationTimestamp, presentationDuration}; - OSStatus err = - CMSampleBufferCreateReadyWithImageBuffer(kCFAllocatorDefault, imageBuffer, self->_formatDescImageBuffer, &sampleTiming, &sampleBufferOut); - if (err != noErr) { - Log(LOG_E, @"Error creating sample buffer for decompressed image buffer %d", (int)err); - return; - } - } else if (self->_renderingBackend == RENDER_METAL) { - // Metal path: retain the pixelBuffer here so it survives the dispatch - pixelBuffer = CVPixelBufferRetain((CVPixelBufferRef)imageBuffer); - } + OSStatus err = + CMSampleBufferCreateReadyWithImageBuffer(kCFAllocatorDefault, imageBuffer, self->_formatDescImageBuffer, &sampleTiming, &sampleBufferOut); + if (err != noErr) { + Log(LOG_E, @"Error creating sample buffer for decompressed image buffer %d", (int)err); + return; + } + } else if (self->_renderingBackend == RENDER_METAL) { + // Metal path: retain the pixelBuffer here so it survives the dispatch + pixelBuffer = CVPixelBufferRetain((CVPixelBufferRef)imageBuffer); + } - // Dispatch onto our higher priority queue - dispatch_async(self->_vtq, ^{ - if (self->_useLegacyPacing && self->_renderingBackend == RENDER_AVSB) { - // Legacy pacing: Process all frames normally, but with immediate display timing - - // Set presentation time to current time for immediate display - CMTime targetTime = CMTimeMakeWithSeconds(CACurrentMediaTime(), NSEC_PER_SEC); - CMSampleBufferSetOutputPresentationTimeStamp(sampleBufferOut, targetTime); - - // Dispatch display layer operations to main thread to avoid CATransaction warnings - dispatch_async(dispatch_get_main_queue(), ^{ - if (frameType == FRAME_TYPE_IDR) { - // Ensure the layer is visible now - self->_displayLayer.hidden = NO; + // Dispatch onto our higher priority queue + dispatch_async(self->_vtq, ^{ + Frame *frame = nil; + if (self->_renderingBackend == RENDER_AVSB) { + frame = [[Frame alloc] initWithSampleBuffer:sampleBufferOut frameNumber:frameNumber frameType:frameType]; + } else { + frame = [[Frame alloc] initWithPixelBufffer:pixelBuffer frameNumber:frameNumber frameType:frameType pts:presentationTimestamp]; + [frame setFormatDesc:self->_formatDesc]; + } + int framesDropped = [self->_frameQueue enqueue:frame withSlackSize:3]; - // Tell our parent VC to hide the progress indicator - [self->_callbacks videoContentShown]; - } + if ([[UIApplication sharedApplication] applicationState] != UIApplicationStateBackground) { + static PlotMetrics frameQueueMetrics = {}; + [[ImGuiPlots sharedInstance] observeFloatReturnMetrics:PLOT_QUEUED_FRAMES value:[self->_frameQueue count] plotMetrics:&frameQueueMetrics]; + [self safeCopyMetricsTo:&self->_frameQueueMetrics from:&frameQueueMetrics]; - if ([self->_displayLayer controlTimebase] == NULL) { - // On first frame, set timebase - CMTimebaseRef timebase = NULL; - CMTimebaseCreateWithSourceClock(CFAllocatorGetDefault(), CMClockGetHostTimeClock(), &timebase); + [[ImGuiPlots sharedInstance] observeFloat:PLOT_DROPPED value:framesDropped]; - CMTime pts = CMSampleBufferGetOutputPresentationTimeStamp(sampleBufferOut); - CMTimebaseSetTime(timebase, pts); - CMTimebaseSetRate(timebase, 1.0); + // It's important we capture host metrics on the incoming thread, as this frame object + // may have been dropped by the above enqueue + static CFTimeInterval lastHostFrame = 0.0f; + if (lastHostFrame != 0) { + [[ImGuiPlots sharedInstance] observeFloat:PLOT_HOST_FRAMETIME value:(frame.pts - lastHostFrame) * 1000.0]; + } + lastHostFrame = frame.pts; - [self->_displayLayer setControlTimebase:timebase]; - if (timebase) { - CFRelease(timebase); - } - Log(LOG_I, @"Setting timebase for legacy pacing to %d / %d", pts.value, pts.timescale); - } + // Decode time is not graphed because it is marked as hidden, but we can use the same mechanism for the value used by stats + static PlotMetrics decodeMetrics = {}; + [[ImGuiPlots sharedInstance] observeFloatReturnMetrics:PLOT_DECODE value:(CACurrentMediaTime() - decodeStartTime) * 1000.0 plotMetrics:&decodeMetrics]; + [self safeCopyMetricsTo:&self->_decodeMetrics from:&decodeMetrics]; + } + }); + }); - [self->_displayLayer enqueueSampleBuffer:sampleBufferOut]; - CFRelease(sampleBufferOut); - }); - } else { - // Queue pacing: use FrameQueue - Frame *frame = nil; - if (self->_renderingBackend == RENDER_AVSB) { - frame = [[Frame alloc] initWithSampleBuffer:sampleBufferOut frameNumber:frameNumber frameType:frameType]; - } else { - frame = [[Frame alloc] initWithPixelBufffer:pixelBuffer frameNumber:frameNumber frameType:frameType pts:presentationTimestamp]; - [frame setFormatDesc:self->_formatDesc]; - } - int framesDropped = [self->_frameQueue enqueue:frame withSlackSize:3]; - - // Capture metrics on main thread to avoid UIApplication thread warning - int frameQueueCount = [self->_frameQueue count]; - CFTimeInterval capturedDecodeTime = CACurrentMediaTime() - decodeStartTime; - CFTimeInterval hostFramePts = frame.pts; - - dispatch_async(dispatch_get_main_queue(), ^{ - if ([[UIApplication sharedApplication] applicationState] != UIApplicationStateBackground) { - static PlotMetrics frameQueueMetrics = {}; - [[ImGuiPlots sharedInstance] observeFloatReturnMetrics:PLOT_QUEUED_FRAMES value:frameQueueCount plotMetrics:&frameQueueMetrics]; - [self safeCopyMetricsTo:&self->_frameQueueMetrics from:&frameQueueMetrics]; - - [[ImGuiPlots sharedInstance] observeFloat:PLOT_DROPPED value:framesDropped]; - - // It's important we capture host metrics on the incoming thread, as this frame object - // may have been dropped by the above enqueue - static CFTimeInterval lastHostFrame = 0.0f; - if (lastHostFrame != 0) { - [[ImGuiPlots sharedInstance] observeFloat:PLOT_HOST_FRAMETIME value:(hostFramePts - lastHostFrame) * 1000.0]; - } - lastHostFrame = hostFramePts; - - // Decode time is not graphed because it is marked as hidden, but we can use the same mechanism for the value used by stats - static PlotMetrics decodeMetrics = {}; - [[ImGuiPlots sharedInstance] observeFloatReturnMetrics:PLOT_DECODE - value:capturedDecodeTime * 1000.0 - plotMetrics:&decodeMetrics]; - [self safeCopyMetricsTo:&self->_decodeMetrics from:&decodeMetrics]; - } - }); - } - }); - }); - - return status; - } + return status; + } } - (void)setHdrMode:(BOOL)enabled { @@ -1066,10 +970,10 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit); if (hasMetadata && hdrMetadata.displayPrimaries[0].x != 0 && hdrMetadata.maxDisplayLuminance != 0) { // This data is all in big-endian struct { - vector_ushort2 primaries[3]; - vector_ushort2 white_point; - uint32_t luminance_max; - uint32_t luminance_min; + vector_ushort2 primaries[3]; + vector_ushort2 white_point; + uint32_t luminance_max; + uint32_t luminance_min; } __attribute__((packed, aligned(4))) mdcv; // mdcv is in GBR order while SS_HDR_METADATA is in RGB order @@ -1147,16 +1051,13 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit); if (_renderingBackend == RENDER_METAL) { stats->renderingBackendString = [NSString stringWithFormat:@"Metal, colorspace: %@", [MetalVideoRenderer currentColorSpace]]; } else { - NSString *pacingMode = _useLegacyPacing ? @"Legacy" : @"Queue"; - stats->renderingBackendString = [NSString stringWithFormat:@"AVSampleBuffer (%@ pacing)", pacingMode]; + stats->renderingBackendString = @"AVSampleBuffer"; } dispatch_sync(_sq, ^{ memcpy(&stats->decodeMetrics, &_decodeMetrics, sizeof(PlotMetrics)); - if (_frameQueue) { - memcpy(&stats->frameQueueMetrics, &_frameQueueMetrics, sizeof(PlotMetrics)); - [_frameQueue.frameDropMetrics copyMetrics:&stats->frameDropMetrics]; - } + memcpy(&stats->frameQueueMetrics, &_frameQueueMetrics, sizeof(PlotMetrics)); + [_frameQueue.frameDropMetrics copyMetrics:&stats->frameDropMetrics]; }); } @@ -1168,7 +1069,7 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit); static int lastTargetRate = 0; int targetRate = (int)_maxRefreshRate; - if (_maxRefreshRate <= 60 || _maxRefreshRate == 90 || !_frameQueue) { + if (_maxRefreshRate <= 60 || _maxRefreshRate == 90) { return; }