diff --git a/VoidLink.xcodeproj/project.pbxproj b/VoidLink.xcodeproj/project.pbxproj index 06af66bf..d941a429 100644 --- a/VoidLink.xcodeproj/project.pbxproj +++ b/VoidLink.xcodeproj/project.pbxproj @@ -143,6 +143,7 @@ FB89463019F646E200339C8A /* StreamConfiguration.m in Sources */ = {isa = PBXBuildFile; fileRef = FB89461919F646E200339C8A /* StreamConfiguration.m */; }; FB89463119F646E200339C8A /* StreamManager.m in Sources */ = {isa = PBXBuildFile; fileRef = FB89461B19F646E200339C8A /* StreamManager.m */; }; FB89463219F646E200339C8A /* VideoDecoderRenderer.m in Sources */ = {isa = PBXBuildFile; fileRef = FB89461D19F646E200339C8A /* VideoDecoderRenderer.m */; }; + A1F100012FFB000100000001 /* FrameInterpolator.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1F100002FFB000100000001 /* FrameInterpolator.swift */; }; FB89463419F646E200339C8A /* Utils.m in Sources */ = {isa = PBXBuildFile; fileRef = FB89462219F646E200339C8A /* Utils.m */; }; FB89463519F646E200339C8A /* MainFrameViewController.m in Sources */ = {isa = PBXBuildFile; fileRef = FB89462519F646E200339C8A /* MainFrameViewController.m */; }; FB89463619F646E200339C8A /* StreamFrameViewController.m in Sources */ = {isa = PBXBuildFile; fileRef = FB89462719F646E200339C8A /* StreamFrameViewController.m */; }; @@ -393,6 +394,7 @@ FB89461B19F646E200339C8A /* StreamManager.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = StreamManager.m; sourceTree = ""; }; FB89461C19F646E200339C8A /* VideoDecoderRenderer.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = VideoDecoderRenderer.h; sourceTree = ""; }; FB89461D19F646E200339C8A /* VideoDecoderRenderer.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = VideoDecoderRenderer.m; sourceTree = ""; }; + A1F100002FFB000100000001 /* FrameInterpolator.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FrameInterpolator.swift; sourceTree = ""; }; FB89462119F646E200339C8A /* Utils.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = Utils.h; sourceTree = ""; }; FB89462219F646E200339C8A /* Utils.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = Utils.m; sourceTree = ""; }; FB89462419F646E200339C8A /* MainFrameViewController.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = MainFrameViewController.h; sourceTree = ""; }; @@ -821,6 +823,7 @@ FB89461B19F646E200339C8A /* StreamManager.m */, FB89461C19F646E200339C8A /* VideoDecoderRenderer.h */, FB89461D19F646E200339C8A /* VideoDecoderRenderer.m */, + A1F100002FFB000100000001 /* FrameInterpolator.swift */, 9803CCAB254F9EAF00EE185E /* ConnectionCallbacks.h */, ); path = Stream; @@ -1153,6 +1156,7 @@ 843CC7F62C1E7A1700BBC475 /* NativeTouchHandler.m in Sources */, 97E20DEE2E2165A5000D5DCA /* ImGuiPlots.mm in Sources */, FB89463219F646E200339C8A /* VideoDecoderRenderer.m in Sources */, + A1F100012FFB000100000001 /* FrameInterpolator.swift in Sources */, 2A263D4F2FF3BA2800154DEA /* PublicUtils.swift in Sources */, FB290D0419B2C406004C83CF /* AppDelegate.m in Sources */, FB9AFD401A7E127D00872C98 /* AppListResponse.m in Sources */, diff --git a/VoidLink/Stream/Connection.m b/VoidLink/Stream/Connection.m index 5720b46d..0354a405 100644 --- a/VoidLink/Stream/Connection.m +++ b/VoidLink/Stream/Connection.m @@ -59,6 +59,7 @@ static AVAudioFormat *audioFormat; static bool muteInBackground; static bool fullColorRange; +static bool request10BitCodec; static VideoDecoderRenderer* renderer; @@ -66,7 +67,7 @@ static BandwidthTracker *bwTracker; int DrDecoderSetup(int videoFormat, int width, int height, int redrawRate, void* context, int drFlags) { - [renderer setupWithVideoFormat:videoFormat width:width height:height frameRate:redrawRate fullRange:fullColorRange]; + [renderer setupWithVideoFormat:videoFormat width:width height:height frameRate:redrawRate fullRange:fullColorRange request10BitCodec:request10BitCodec]; lastFrameNumber = 0; activeVideoFormat = videoFormat; Log(LOG_I, @"Active video format: 0x%x", activeVideoFormat); @@ -657,6 +658,8 @@ void ClSetControllerLED(uint16_t controllerNumber, uint8_t r, uint8_t g, uint8_t LiInitializeStreamConfiguration(&_streamConfig); _streamConfig.colorRange = config.fullColorRange ? 1 : 0; fullColorRange = config.fullColorRange; + // request10BitCodec = config.enableHdr || config.sdrPerformanceWorkaround; + request10BitCodec = config.enableHdr; _streamConfig.width = config.width; _streamConfig.height = config.height; _streamConfig.fps = config.frameRate; diff --git a/VoidLink/Stream/FrameInterpolator.swift b/VoidLink/Stream/FrameInterpolator.swift new file mode 100644 index 00000000..61da8387 --- /dev/null +++ b/VoidLink/Stream/FrameInterpolator.swift @@ -0,0 +1,497 @@ +import AVFoundation +import CoreVideo +import ObjectiveC.runtime +import VideoToolbox + +@objcMembers +final class FrameInterpolator: NSObject { + typealias Completion = (NSArray) -> Void + + private struct PendingFrame { + let frame: Frame + let completion: Completion + } + + private let queue = DispatchQueue(label: "com.voidlink.FrameInterpolator", qos: .userInteractive) + private var processor: AnyObject? + private var pixelBufferPool: CVPixelBufferPool? + private var formatDescription: CMVideoFormatDescription? + private var previousFrame: Frame? + private var pendingFrames: [PendingFrame] = [] + private var isProcessing = false + private var width = 0 + private var height = 0 + private var slowUntil = CFTimeInterval(0) + private var completedProcessingCount = 0 + private var consecutiveSlowFrames = 0 + private var loggedEvents = Set() + + var isEnabled = false { + didSet { + if !isEnabled { + reset() + } + } + } + + @objc static let deviceSupportsInterpolation: Bool = { + #if targetEnvironment(simulator) + return false + #else + if #available(iOS 26.0, tvOS 26.0, *) { + VTLowLatencyFrameInterpolationConfiguration.printLimits() + return VTLowLatencyFrameInterpolationConfiguration.isSupported + } + return false + #endif + }() + + var isAvailable: Bool { + isEnabled && Self.deviceSupportsInterpolation + } + + func reset() { + queue.async { [weak self] in + self?.resetLocked() + } + } + + func processFrame(_ frame: Frame, completion: @escaping Completion) { + queue.async { [weak self] in + guard let self else { + completion([frame]) + return + } + + self.pendingFrames.append(PendingFrame(frame: frame, completion: completion)) + self.drainPendingFrames() + } + } + + private func drainPendingFrames() { + guard !isProcessing, !pendingFrames.isEmpty else { + return + } + + let pendingFrame = pendingFrames.removeFirst() + let frame = pendingFrame.frame + let completion = pendingFrame.completion + guard isAvailable, CACurrentMediaTime() >= slowUntil else { + if isEnabled && !Self.deviceSupportsInterpolation { + logOnce("unavailable", "disabled: VT low-latency frame interpolation is unsupported on this OS/device") + } + previousFrame = frame + completion([frame]) + drainPendingFrames() + return + } + + if frame.frameType == FRAME_TYPE_IDR { + resetLocked(keepPendingFrames: true) + previousFrame = frame + completion([frame]) + drainPendingFrames() + return + } + + guard + let currentSampleBuffer = frame.sampleBuffer, + let currentPixelBuffer = CMSampleBufferGetImageBuffer(currentSampleBuffer) + else { + logOnce("missing-image-buffer", "input CMSampleBuffer has no CVPixelBuffer") + previousFrame = frame + completion([frame]) + drainPendingFrames() + return + } + + // renderFrame() rewrites the sample buffer's output PTS for local display pacing. + // Frame.pts90 retains the original upstream PTS required for interpolation. + let currentPTS = frame.pts90 + guard CMTIME_IS_VALID(currentPTS) else { + logOnce("invalid-pts", "input frame has an invalid PTS") + previousFrame = frame + completion([frame]) + drainPendingFrames() + return + } + + guard prepareSessionIfNeeded(pixelBuffer: currentPixelBuffer) else { + previousFrame = frame + completion([frame]) + drainPendingFrames() + return + } + + guard + let previousFrame, + let previousSampleBuffer = previousFrame.sampleBuffer, + let previousPixelBuffer = CMSampleBufferGetImageBuffer(previousSampleBuffer) + else { + previousFrame = frame + completion([frame]) + drainPendingFrames() + return + } + + let previousPTS = previousFrame.pts90 + guard CMTIME_IS_VALID(previousPTS), CMTimeCompare(currentPTS, previousPTS) > 0 else { + logOnce( + "non-monotonic-pts", + "upstream PTS is not increasing: previous \(CMTimeGetSeconds(previousPTS)), current \(CMTimeGetSeconds(currentPTS))" + ) + self.previousFrame = frame + completion([frame]) + drainPendingFrames() + return + } + let sourceFrameInterval = CMTimeGetSeconds(CMTimeSubtract(currentPTS, previousPTS)) + + #if targetEnvironment(simulator) + self.previousFrame = frame + completion([frame]) + drainPendingFrames() + return + #else + guard #available(iOS 26.0, tvOS 26.0, *) else { + self.previousFrame = frame + completion([frame]) + drainPendingFrames() + return + } + + guard + let processor = processor as? VTFrameProcessor, + let sourceProcessorFrame = VTFrameProcessorFrame(buffer: currentPixelBuffer, presentationTimeStamp: currentPTS), + let previousProcessorFrame = VTFrameProcessorFrame(buffer: previousPixelBuffer, presentationTimeStamp: previousPTS), + let destinationPixelBuffer = makeDestinationPixelBuffer(), + let destinationProcessorFrame = VTFrameProcessorFrame( + buffer: destinationPixelBuffer, + presentationTimeStamp: midpointTime(previousPTS, currentPTS) + ), + let parameters = VTLowLatencyFrameInterpolationParameters( + sourceFrame: sourceProcessorFrame, + previousFrame: previousProcessorFrame, + interpolationPhase: [0.5], + destinationFrames: [destinationProcessorFrame] + ) + else { + logOnce( + "parameters-failed", + "could not create interpolation parameters; input pixel format is \(pixelFormatName(CVPixelBufferGetPixelFormatType(currentPixelBuffer)))" + ) + self.previousFrame = frame + completion([frame]) + drainPendingFrames() + return + } + + isProcessing = true + let startedAt = CACurrentMediaTime() + processor.process(parameters: parameters) { [weak self] _, error in + guard let self else { + completion([frame]) + return + } + + self.queue.async { + self.isProcessing = false + self.previousFrame = frame + + let elapsed = CACurrentMediaTime() - startedAt + guard + error == nil, + let interpolatedFrame = self.makeInterpolatedFrame( + from: frame, + imageBuffer: destinationPixelBuffer, + presentationTimeStamp: destinationProcessorFrame.presentationTimeStamp + ) + else { + if let error { + self.logOnce("process-error-\((error as NSError).code)", "processing failed: \(error)") + } else { + self.logOnce("sample-buffer-failed", "processing completed but the interpolated CMSampleBuffer could not be created") + } + completion([frame]) + self.drainPendingFrames() + return + } + + self.completedProcessingCount += 1 + if sourceFrameInterval.isFinite, sourceFrameInterval > 0, elapsed >= sourceFrameInterval { + self.consecutiveSlowFrames += 1 + } else { + self.consecutiveSlowFrames = 0 + } + if self.completedProcessingCount >= 10, self.consecutiveSlowFrames >= 3 { + self.slowUntil = CACurrentMediaTime() + 1.0 + self.consecutiveSlowFrames = 0 + self.logOnce( + "sustained-overrun", + "processing repeatedly exceeded the source-frame interval; temporarily falling back to original frames" + ) + } + + // NSLog("[FrameInterpolator] interpolated frame in %.2f ms",) + completion([interpolatedFrame, frame]) + self.drainPendingFrames() + } + } + #endif + } + + private func prepareSessionIfNeeded(pixelBuffer: CVPixelBuffer) -> Bool { + #if targetEnvironment(simulator) + return false + #else + let width = CVPixelBufferGetWidth(pixelBuffer) + let height = CVPixelBufferGetHeight(pixelBuffer) + if processor != nil, self.width == width, self.height == height { + return true + } + + resetLocked(keepPendingFrames: true) + + guard #available(iOS 26.0, tvOS 26.0, *) else { + return false + } + + guard + VTLowLatencyFrameInterpolationConfiguration.isSupported, + let configuration = VTLowLatencyFrameInterpolationConfiguration( + frameWidth: width, + frameHeight: height, + numberOfInterpolatedFrames: 1 + ) + else { + logOnce("configuration-failed", "could not create a configuration for \(width)x\(height)") + return false + } + + let inputPixelFormat = CVPixelBufferGetPixelFormatType(pixelBuffer) + if !configuration.supportedPixelFormats.contains(inputPixelFormat) { + let supportedFormats = configuration.supportedPixelFormats + .map(pixelFormatName) + .joined(separator: ", ") + logOnce( + "unsupported-pixel-format-\(inputPixelFormat)", + "input pixel format \(pixelFormatName(inputPixelFormat)) is unsupported; supported formats: \(supportedFormats)" + ) + return false + } + + let processor = VTFrameProcessor() + do { + try processor.startSession(configuration: configuration) + } catch { + logOnce("session-error-\((error as NSError).code)", "failed to start session: \(error)") + return false + } + + var pool: CVPixelBufferPool? + let status = CVPixelBufferPoolCreate( + kCFAllocatorDefault, + nil, + configuration.destinationPixelBufferAttributes as CFDictionary, + &pool + ) + guard status == kCVReturnSuccess, let pool else { + logOnce("pool-error-\(status)", "failed to create destination pixel buffer pool: \(status)") + return false + } + + self.width = width + self.height = height + self.processor = processor + self.pixelBufferPool = pool + logOnce( + "session-started-\(width)x\(height)", + "session started for \(width)x\(height), input pixel format \(pixelFormatName(inputPixelFormat))" + ) + return true + #endif + } + + private func makeDestinationPixelBuffer() -> CVPixelBuffer? { + guard let pixelBufferPool else { + return nil + } + + var pixelBuffer: CVPixelBuffer? + let status = CVPixelBufferPoolCreatePixelBuffer(kCFAllocatorDefault, pixelBufferPool, &pixelBuffer) + return status == kCVReturnSuccess ? pixelBuffer : nil + } + + private func makeInterpolatedFrame( + from sourceFrame: Frame, + imageBuffer: CVPixelBuffer, + presentationTimeStamp: CMTime + ) -> Frame? { + if formatDescription == nil || !CMVideoFormatDescriptionMatchesImageBuffer(formatDescription!, imageBuffer: imageBuffer) { + var newFormatDescription: CMVideoFormatDescription? + let status = CMVideoFormatDescriptionCreateForImageBuffer( + allocator: kCFAllocatorDefault, + imageBuffer: imageBuffer, + formatDescriptionOut: &newFormatDescription + ) + guard status == noErr else { + return nil + } + formatDescription = newFormatDescription + } + + let duration = sourceFrame.sampleBuffer.map { CMSampleBufferGetDuration($0) } ?? .invalid + var timing = CMSampleTimingInfo( + duration: duration, + presentationTimeStamp: presentationTimeStamp, + decodeTimeStamp: .invalid + ) + + var sampleBuffer: CMSampleBuffer? + let status = CMSampleBufferCreateReadyWithImageBuffer( + allocator: kCFAllocatorDefault, + imageBuffer: imageBuffer, + formatDescription: formatDescription!, + sampleTiming: &timing, + sampleBufferOut: &sampleBuffer + ) + guard status == noErr, let sampleBuffer else { + return nil + } + + let retainedSampleBuffer = Unmanaged.passRetained(sampleBuffer).takeUnretainedValue() + let frame = Frame( + sampleBuffer: retainedSampleBuffer, + frameNumber: sourceFrame.frameNumber, + frameType: FRAME_TYPE_PFRAME + ) + frame?.isInterpolated = true + return frame + } + + private func midpointTime(_ first: CMTime, _ second: CMTime) -> CMTime { + CMTimeAdd(first, CMTimeMultiplyByFloat64(CMTimeSubtract(second, first), multiplier: 0.5)) + } + + private func logOnce(_ event: String, _ message: String) { + guard loggedEvents.insert(event).inserted else { + return + } + NSLog("[FrameInterpolator] %@", message) + } + + private func pixelFormatName(_ pixelFormat: OSType) -> String { + let bytes: [UInt8] = [ + UInt8((pixelFormat >> 24) & 0xff), + UInt8((pixelFormat >> 16) & 0xff), + UInt8((pixelFormat >> 8) & 0xff), + UInt8(pixelFormat & 0xff), + ] + let fourCC = String(bytes: bytes, encoding: .ascii) ?? "????" + return "\(fourCC) (\(pixelFormat))" + } + + private func resetLocked(keepPendingFrames: Bool = false) { + processor = nil + pixelBufferPool = nil + formatDescription = nil + previousFrame = nil + isProcessing = false + width = 0 + height = 0 + slowUntil = 0 + completedProcessingCount = 0 + consecutiveSlowFrames = 0 + if !keepPendingFrames { + pendingFrames.removeAll() + } + } +} + +#if !targetEnvironment(simulator) +@available(iOS 26.0, tvOS 26.0, *) +extension VTLowLatencyFrameInterpolationConfiguration { + struct RuntimeDimensionLimits { + let minimumDimensions: CMVideoDimensions? + let maximumDimensions: CMVideoDimensions? + let maximumDimension: Int? + let maximumPixelCount: Int? + + func supports(width: Int, height: Int) -> Bool { + guard width > 0, height > 0 else { + return false + } + + if let minimumDimensions, + (width < Int(minimumDimensions.width) || height < Int(minimumDimensions.height)) { + return false + } + + if let maximumDimensions, + (width > Int(maximumDimensions.width) || height > Int(maximumDimensions.height)) { + return false + } + + if let maximumDimension, + (width > maximumDimension || height > maximumDimension) { + return false + } + + if let maximumPixelCount { + let (pixelCount, overflow) = width.multipliedReportingOverflow(by: height) + if overflow || pixelCount > maximumPixelCount { + return false + } + } + + return maximumDimensions != nil || maximumDimension != nil || maximumPixelCount != nil + } + } + + static func runtimeDimensionLimits(spatialScaleFactor: Int = 1) -> RuntimeDimensionLimits { + RuntimeDimensionLimits( + minimumDimensions: minimumDimensions, + maximumDimensions: maximumDimensions, + maximumDimension: callRuntimeIntegerClassMethod( + "maximumDimensionForSpatialScaleFactor:", + argument: spatialScaleFactor + ), + maximumPixelCount: callRuntimeIntegerClassMethod( + "maximumPixelCountForSpatialScaleFactor:", + argument: spatialScaleFactor + ) + ) + } + + private static func callRuntimeIntegerClassMethod( + _ selectorName: String, + argument: Int + ) -> Int? { + let configurationClass: AnyClass = self + let selector = NSSelectorFromString(selectorName) + guard let method = class_getClassMethod(configurationClass, selector) else { + return nil + } + + typealias Function = @convention(c) (AnyClass, Selector, Int) -> Int + let function = unsafeBitCast(method_getImplementation(method), to: Function.self) + let result = function(configurationClass, selector, argument) + return result > 0 ? result : nil + } + + static func printLimits() { + let limits = + VTLowLatencyFrameInterpolationConfiguration + .runtimeDimensionLimits(spatialScaleFactor: 1) + + print("minimum:", limits.minimumDimensions as Any) + print("legacy maximum:", limits.maximumDimensions as Any) + print("maximum dimension:", limits.maximumDimension as Any) + print("maximum pixel count:", limits.maximumPixelCount as Any) + + let supported = limits.supports(width: 1920, height: 1080) + print("1920x1080 supported:", supported) + } + +} +#endif diff --git a/VoidLink/Stream/StreamConfiguration.h b/VoidLink/Stream/StreamConfiguration.h index 1d3befe4..ca3ea07c 100644 --- a/VoidLink/Stream/StreamConfiguration.h +++ b/VoidLink/Stream/StreamConfiguration.h @@ -22,6 +22,8 @@ @property BOOL enableYUV444; @property BOOL enablePIP; @property BOOL fullColorRange; +@property BOOL enableHdr; +@property BOOL sdrPerformanceWorkaround; @property int width; @property int height; @property int frameRate; diff --git a/VoidLink/Stream/StreamConfiguration.m b/VoidLink/Stream/StreamConfiguration.m index 4812301f..42d00fd1 100644 --- a/VoidLink/Stream/StreamConfiguration.m +++ b/VoidLink/Stream/StreamConfiguration.m @@ -9,5 +9,5 @@ #import "StreamConfiguration.h" @implementation StreamConfiguration -@synthesize host, httpsPort, appID, width, height, frameRate, bitRate, riKeyId, riKey, gamepadMask, appName, optimizeGameSettings, playAudioOnPC, swapABXYButtons, buttonVisualFeedback, gyroMode, emulatedControllerType, audioConfiguration, supportedVideoFormats, multiController, serverCert, rtspSessionUrl, serverCodecModeSupport, enableYUV444, enablePIP, fullColorRange, localVolume; +@synthesize host, httpsPort, appID, width, height, frameRate, bitRate, riKeyId, riKey, gamepadMask, appName, optimizeGameSettings, playAudioOnPC, swapABXYButtons, buttonVisualFeedback, gyroMode, emulatedControllerType, audioConfiguration, supportedVideoFormats, multiController, serverCert, rtspSessionUrl, serverCodecModeSupport, enableYUV444, enablePIP, fullColorRange, enableHdr, sdrPerformanceWorkaround, localVolume; @end diff --git a/VoidLink/Stream/VideoDecoderRenderer.h b/VoidLink/Stream/VideoDecoderRenderer.h index 3dc14832..920ec247 100644 --- a/VoidLink/Stream/VideoDecoderRenderer.h +++ b/VoidLink/Stream/VideoDecoderRenderer.h @@ -26,7 +26,8 @@ - (id)initWithView:(UIView*)view callbacks:(id)callbacks streamAspectRatio:(float)aspectRatio; -- (void)setupWithVideoFormat:(int)videoFormat width:(int)videoWidth height:(int)videoHeight frameRate:(int)frameRate fullRange:(BOOL)fullRange; +- (void)setupWithVideoFormat:(int)videoFormat width:(int)videoWidth height:(int)videoHeight frameRate:(int)frameRate fullRange:(BOOL)fullRange request10BitCodec:(BOOL)enableHdr; + - (void)renderFrame:(Frame *)frame atTime:(CMTime)targetTime; - (void)cleanup; - (void)setHdrMode:(BOOL)enabled; @@ -47,5 +48,6 @@ decodeStartTime:(CFTimeInterval)decodeStartTime; - (void)invalidateDecompressionSession; ++ (void)setFrameInterpolationEnabled:(bool)enabled; @end diff --git a/VoidLink/Stream/VideoDecoderRenderer.m b/VoidLink/Stream/VideoDecoderRenderer.m index d108f355..65331876 100644 --- a/VoidLink/Stream/VideoDecoderRenderer.m +++ b/VoidLink/Stream/VideoDecoderRenderer.m @@ -22,6 +22,7 @@ #import "PlatformThreads.h" #import "MetalViewController.h" #import "ImGuiPlots.h" +#import "VoidLink-Swift.h" #include #include @@ -34,6 +35,8 @@ // Define for extra logging related to frame pacing //#define DISPLAYLINK_VERBOSE +static BOOL kEnableFrameInterpolation = false; + // Private libavformat API for writing the AV1 Codec Configuration Box extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size, int write_seq_header); @@ -48,6 +51,7 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size, int _videoFormat; int _frameRate; BOOL _fullRange; + BOOL _request10BitCodec; NSMutableArray *_parameterSetBuffers; NSData *_masteringDisplayColorVolume; @@ -55,6 +59,7 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size, CMVideoFormatDescriptionRef _formatDesc; CMVideoFormatDescriptionRef _formatDescImageBuffer; VTDecompressionSessionRef _decompressionSession; + FrameInterpolator *_frameInterpolator; CADisplayLink *_displayLink; NSInteger _maxRefreshRate; @@ -68,6 +73,10 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size, // NSTimeInterval previousLinkTime; } ++ (void)setFrameInterpolationEnabled:(bool)enabled { + kEnableFrameInterpolation = enabled; +} + - (void)reinitializeDisplayLayer { if (_displayLayer == nil) { @@ -185,11 +194,12 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size, # pragma mark DisplayLink vsync callback -- (void)setupWithVideoFormat:(int)videoFormat width:(int)videoWidth height:(int)videoHeight frameRate:(int)frameRate fullRange:(BOOL)fullRange +- (void)setupWithVideoFormat:(int)videoFormat width:(int)videoWidth height:(int)videoHeight frameRate:(int)frameRate fullRange:(BOOL)fullRange request10BitCodec:(BOOL)request10BitCodec { self->_videoFormat = videoFormat; self->_frameRate = frameRate; self->_fullRange = fullRange; + self->_request10BitCodec = request10BitCodec; // reset plot data in case we've already used it for a previous renderer [[ImGuiPlots sharedInstance] clearData]; @@ -197,6 +207,11 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size, DataManager* dataMan = [[DataManager alloc] init]; if ([[dataMan getSettings].renderingBackend integerValue] == RENDER_AVSB) { _renderingBackend = RENDER_AVSB; + + if (kEnableFrameInterpolation) { + _frameInterpolator = [[FrameInterpolator alloc] init]; + _frameInterpolator.isEnabled = YES; + } // Choose the appropriate selector based on frame pacing mode SEL displayLinkSelector; @@ -213,11 +228,16 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size, _displayLink = [CADisplayLink displayLinkWithTarget:self selector:displayLinkSelector]; + NSInteger targetFrameRate = self->_frameRate; + if (self->_frameInterpolator != nil) { + targetFrameRate = MIN(self->_frameRate * 2, self->_maxRefreshRate); + } + if (@available(iOS 15.0, tvOS 15.0, *)) { - _displayLink.preferredFrameRateRange = CAFrameRateRangeMake(self->_frameRate, self->_frameRate, self->_frameRate); + _displayLink.preferredFrameRateRange = CAFrameRateRangeMake(targetFrameRate, targetFrameRate, targetFrameRate); } else { - _displayLink.preferredFramesPerSecond = self->_frameRate; + _displayLink.preferredFramesPerSecond = targetFrameRate; } [_displayLink addToRunLoop:[NSRunLoop mainRunLoop] forMode:NSDefaultRunLoopMode]; } else { @@ -227,7 +247,7 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size, } -- (void)setupDecompressionSessionWithAttributes:(NSDictionary *)destinationPixelBufferAttributes { +- (OSStatus)setupDecompressionSessionWithAttributes:(NSDictionary *)destinationPixelBufferAttributes { // This method is called from within synchronized block, so no additional sync needed here if (_decompressionSession != NULL) { VTDecompressionSessionInvalidate(_decompressionSession); @@ -235,15 +255,16 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size, _decompressionSession = nil; } - int status = VTDecompressionSessionCreate(kCFAllocatorDefault, - _formatDesc, - nil, - (__bridge CFDictionaryRef)destinationPixelBufferAttributes, - nil, - &_decompressionSession); + OSStatus status = VTDecompressionSessionCreate(kCFAllocatorDefault, + _formatDesc, + nil, + (__bridge CFDictionaryRef)destinationPixelBufferAttributes, + nil, + &_decompressionSession); if (status != noErr) { Log(LOG_E, @"Failed to create VTDecompressionSession, status %d", status); } + return status; } - (void)setupDecompressionSession { @@ -256,11 +277,20 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size, }, } mutableCopy]; #else - NSNumber *pixelFormat = nil; + NSNumber *nativePixelFormat = nil; if (self->_videoFormat & VIDEO_FORMAT_MASK_YUV444) { - pixelFormat = self->_fullRange ? @(kCVPixelFormatType_444YpCbCr10BiPlanarFullRange) : @(kCVPixelFormatType_444YpCbCr10BiPlanarVideoRange); + NSNumber *frFormat = _request10BitCodec ? @(kCVPixelFormatType_444YpCbCr10BiPlanarFullRange) : @(kCVPixelFormatType_444YpCbCr8BiPlanarFullRange); + NSNumber *vrFormat = _request10BitCodec ? @(kCVPixelFormatType_444YpCbCr10BiPlanarVideoRange) : @(kCVPixelFormatType_444YpCbCr8BiPlanarVideoRange); + nativePixelFormat = self->_fullRange ? frFormat : vrFormat; } else { - pixelFormat = self->_fullRange ? @(kCVPixelFormatType_420YpCbCr10BiPlanarFullRange) : @(kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange); + NSNumber *frFormat = _request10BitCodec ? @(kCVPixelFormatType_420YpCbCr10BiPlanarFullRange) : @(kCVPixelFormatType_420YpCbCr8BiPlanarFullRange); + NSNumber *vrFormat = _request10BitCodec ? @(kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange) : @(kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange); + nativePixelFormat = self->_fullRange ? frFormat : vrFormat; + } + BOOL interpolationRequested = self->_frameInterpolator != nil; + NSNumber *pixelFormat = interpolationRequested ? @(kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange) : nativePixelFormat; + if (interpolationRequested) { + LogOnce(LOG_I, @"Frame interpolation enabled; requesting 420v output from VTDecompressionSession"); } NSMutableDictionary *destinationPixelBufferAttributes = [@{ (id)kCVPixelBufferPixelFormatTypeKey : pixelFormat @@ -272,7 +302,31 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size, destinationPixelBufferAttributes[(id)kVTDecompressionPropertyKey_GeneratePerFrameHDRDisplayMetadata] = @YES; } - return [self setupDecompressionSessionWithAttributes:destinationPixelBufferAttributes]; +#if !TARGET_OS_SIMULATOR + if (!interpolationRequested) { + [self setupDecompressionSessionWithAttributes:destinationPixelBufferAttributes]; + return; + } + + OSStatus status = [self setupDecompressionSessionWithAttributes:destinationPixelBufferAttributes]; + if (status != noErr) { + Log(LOG_W, @"Falling back from 420v frame-interpolation decode output to native pixel format %@", nativePixelFormat); + [self->_frameInterpolator reset]; + self->_frameInterpolator = nil; + destinationPixelBufferAttributes[(id)kCVPixelBufferPixelFormatTypeKey] = nativePixelFormat; + [self setupDecompressionSessionWithAttributes:destinationPixelBufferAttributes]; + + NSInteger targetFrameRate = MIN(self->_frameRate, self->_maxRefreshRate); + if (@available(iOS 15.0, tvOS 15.0, *)) { + self->_displayLink.preferredFrameRateRange = CAFrameRateRangeMake(targetFrameRate, targetFrameRate, targetFrameRate); + } else { + self->_displayLink.preferredFramesPerSecond = targetFrameRate; + } + } +#else + [self setupDecompressionSessionWithAttributes:destinationPixelBufferAttributes]; +#endif + return; } - (void) checkDisplayLayer { @@ -477,6 +531,9 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit); self->_decompressionSession = nil; } } + if (self->_frameInterpolator != nil) { + [self->_frameInterpolator reset]; + } }); } @@ -1073,6 +1130,37 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit); frame = [[Frame alloc] initWithPixelBufffer:pixelBuffer frameNumber:frameNumber frameType:frameType pts:presentationTimestamp]; [frame setFormatDesc:self->_formatDesc]; } + + if (self->_frameInterpolator != nil) { + [self->_frameInterpolator processFrame:frame completion:^(NSArray *frames) { + dispatch_async(self->_vtq, ^{ + int framesDropped = 0; + for (Frame *outputFrame in (NSArray *)frames) { + framesDropped += [self->_frameQueue enqueue:outputFrame withSlackSize:3]; + } + + if ([[UIApplication sharedApplication] applicationState] != UIApplicationStateBackground) { + static PlotMetrics frameQueueMetrics = {}; + [[ImGuiPlots sharedInstance] observeFloatReturnMetrics:PLOT_QUEUED_FRAMES value:[self->_frameQueue count] plotMetrics:&frameQueueMetrics]; + [self safeCopyMetricsTo:&self->_frameQueueMetrics from:&frameQueueMetrics]; + + [[ImGuiPlots sharedInstance] observeFloat:PLOT_DROPPED value:framesDropped]; + + static CFTimeInterval lastHostFrame = 0.0f; + if (lastHostFrame != 0) { + [[ImGuiPlots sharedInstance] observeFloat:PLOT_HOST_FRAMETIME value:(frame.pts - lastHostFrame) * 1000.0]; + } + lastHostFrame = frame.pts; + + static PlotMetrics decodeMetrics = {}; + [[ImGuiPlots sharedInstance] observeFloatReturnMetrics:PLOT_DECODE value:(CACurrentMediaTime() - decodeStartTime) * 1000.0 plotMetrics:&decodeMetrics]; + [self safeCopyMetricsTo:&self->_decodeMetrics from:&decodeMetrics]; + } + }); + }]; + return; + } + int framesDropped = [self->_frameQueue enqueue:frame withSlackSize:3]; if ([[UIApplication sharedApplication] applicationState] != UIApplicationStateBackground) { diff --git a/VoidLink/Utility/Frame.h b/VoidLink/Utility/Frame.h index a5967c4b..c923663d 100644 --- a/VoidLink/Utility/Frame.h +++ b/VoidLink/Utility/Frame.h @@ -4,6 +4,7 @@ @interface Frame : NSObject @property (nonatomic) int frameNumber; @property (nonatomic) int frameType; +@property (nonatomic) BOOL isInterpolated; @property (nonatomic) CFTimeInterval decodedAt; @property (nonatomic) CMTime pts90; @property (nonatomic) CMTime duration90; diff --git a/VoidLink/Utility/Frame.m b/VoidLink/Utility/Frame.m index e9609328..50aa5f94 100644 --- a/VoidLink/Utility/Frame.m +++ b/VoidLink/Utility/Frame.m @@ -1,4 +1,5 @@ #import +#import #import "Frame.h" #import "Logger.h" #include @@ -7,6 +8,19 @@ CFDictionaryRef _formatDescExt; } +static char FrameInterpolatedKey; + +- (BOOL)isInterpolated { + return [objc_getAssociatedObject(self, &FrameInterpolatedKey) boolValue]; +} + +- (void)setIsInterpolated:(BOOL)isInterpolated { + objc_setAssociatedObject(self, + &FrameInterpolatedKey, + isInterpolated ? @YES : nil, + OBJC_ASSOCIATION_RETAIN_NONATOMIC); +} + - (instancetype)initWithPixelBufffer:(CVPixelBufferRef)pixelBuffer frameNumber:(int)frameNumber frameType:(int)frameType diff --git a/VoidLink/ViewControllers/MainFrameViewController.m b/VoidLink/ViewControllers/MainFrameViewController.m index b3072b55..40988cc3 100644 --- a/VoidLink/ViewControllers/MainFrameViewController.m +++ b/VoidLink/ViewControllers/MainFrameViewController.m @@ -875,6 +875,8 @@ static NSMutableSet* hostList; _streamConfig.enableYUV444 = streamSettings.enableYUV444; _streamConfig.enablePIP = streamSettings.enablePIP; _streamConfig.fullColorRange = streamSettings.fullColorRange; + _streamConfig.enableHdr = streamSettings.enableHdr; + _streamConfig.sdrPerformanceWorkaround = streamSettings.sdrPerformanceWorkaround; _streamConfig.asyncNativeTouchPriority = streamSettings.asyncNativeTouchPriority; // new streamConfig segment _streamConfig.gyroMode = [streamSettings.gyroMode intValue]; _streamConfig.emulatedControllerType = streamSettings.emulatedControllerType.intValue; @@ -908,6 +910,7 @@ static NSMutableSet* hostList; Connection.useSystemAudioEngine = streamSettings.audioConfig.intValue == 2; + bool sdrPerformanceWorkaround = false; switch (streamSettings.preferredCodec) { case CODEC_PREF_AV1: #if defined(__IPHONE_16_0) || defined(__TVOS_16_0) @@ -917,7 +920,7 @@ static NSMutableSet* hostList; _streamConfig.supportedVideoFormats |= VIDEO_FORMAT_AV1_HIGH8_444; } else { - if(streamSettings.sdrPerformanceWorkaround && [Utils hdrSupported]) _streamConfig.supportedVideoFormats |= VIDEO_FORMAT_AV1_MAIN10; // 8bit performance degradation workaround for av1 + if(sdrPerformanceWorkaround && [Utils hdrSupported]) _streamConfig.supportedVideoFormats |= VIDEO_FORMAT_AV1_MAIN10; // 8bit performance degradation workaround for av1 else _streamConfig.supportedVideoFormats |= VIDEO_FORMAT_AV1_MAIN8; } } @@ -928,12 +931,12 @@ static NSMutableSet* hostList; case CODEC_PREF_HEVC: if (VTIsHardwareDecodeSupported(kCMVideoCodecType_HEVC)) { if (streamSettings.enableYUV444) { - if(streamSettings.sdrPerformanceWorkaround && [Utils hdrSupported]) _streamConfig.supportedVideoFormats |= VIDEO_FORMAT_H265_REXT10_444; // 8bit performance degradation workaround + if(sdrPerformanceWorkaround && [Utils hdrSupported]) _streamConfig.supportedVideoFormats |= VIDEO_FORMAT_H265_REXT10_444; // 8bit performance degradation workaround else _streamConfig.supportedVideoFormats |= VIDEO_FORMAT_H265_REXT8_444; } else { // _streamConfig.supportedVideoFormats |= VIDEO_FORMAT_H265; - if(streamSettings.sdrPerformanceWorkaround && [Utils hdrSupported]) _streamConfig.supportedVideoFormats |= VIDEO_FORMAT_H265_MAIN10; // 8bit performance degradation workaround + if(sdrPerformanceWorkaround && [Utils hdrSupported]) _streamConfig.supportedVideoFormats |= VIDEO_FORMAT_H265_MAIN10; // 8bit performance degradation workaround else _streamConfig.supportedVideoFormats |= VIDEO_FORMAT_H265; } } @@ -951,7 +954,7 @@ static NSMutableSet* hostList; // HEVC is supported if the user wants it (or it's required by the chosen resolution) and the SoC supports it if ((_streamConfig.width > 4096 || _streamConfig.height > 4096 || streamSettings.enableHdr) && VTIsHardwareDecodeSupported(kCMVideoCodecType_HEVC)) { - if(streamSettings.sdrPerformanceWorkaround && [Utils hdrSupported]) _streamConfig.supportedVideoFormats |= VIDEO_FORMAT_H265_MAIN10; // 8bit performance degradation workaround + if(sdrPerformanceWorkaround && [Utils hdrSupported]) _streamConfig.supportedVideoFormats |= VIDEO_FORMAT_H265_MAIN10; // 8bit performance degradation workaround else _streamConfig.supportedVideoFormats |= VIDEO_FORMAT_H265; // HEVC Main10 is supported if the user wants it and the display supports it diff --git a/VoidLink/ViewControllers/SettingsViewController.m b/VoidLink/ViewControllers/SettingsViewController.m index f9280584..0995bd63 100644 --- a/VoidLink/ViewControllers/SettingsViewController.m +++ b/VoidLink/ViewControllers/SettingsViewController.m @@ -877,8 +877,8 @@ BOOL isCustomResolution(int resolutionSelected) { self.yuv444Stack.hasInfoTag = YES; [self addSetting:self.yuv444Stack ofId:@"yuv444Stack" to:videoSection]; - self.sdrPerformanceWorkaroundStack.hasInfoTag = YES; - [self addSetting:self.sdrPerformanceWorkaroundStack ofId:@"sdrPerformanceWorkaroundStack" to:videoSection]; + // self.sdrPerformanceWorkaroundStack.hasInfoTag = YES; + // [self addSetting:self.sdrPerformanceWorkaroundStack ofId:@"sdrPerformanceWorkaroundStack" to:videoSection]; self.framePacingStack.hasInfoTag = YES; [self addSetting:self.framePacingStack ofId:@"framePacingStack" to:videoSection]; @@ -3592,6 +3592,7 @@ BOOL isCustomResolution(int resolutionSelected) { - (void)muteInBackgroundSwitchFlipped:(UISwitch* )sender{ Connection.muteInBackground = sender.isOn; + // [VideoDecoderRenderer setFrameInterpolationEnabled:sender.isOn]; } - (void)redirectMicSwitchFlipped:(UISwitch* )sender{ diff --git a/VoidLink/ViewControllers/StreamFrameViewController.m b/VoidLink/ViewControllers/StreamFrameViewController.m index da206327..af7043d8 100644 --- a/VoidLink/ViewControllers/StreamFrameViewController.m +++ b/VoidLink/ViewControllers/StreamFrameViewController.m @@ -1798,7 +1798,7 @@ static __weak StreamFrameViewController *VLSharedStreamFrameViewController = nil _micStreamInitialized = false; } - // 8bit 444 degration workaround + /* if(strcmp(stageName, "video stream establishment")==0){ NSLog(@"sendAutoReleaseComboCommandWithCmdStrings %f", CACurrentMediaTime()); if(!_settings.enableHdr @@ -1818,6 +1818,7 @@ static __weak StreamFrameViewController *VLSharedStreamFrameViewController = nil }); } } + */ } - (void) stageFailed:(const char*)stageName withError:(int)errorCode portTestFlags:(int)portTestFlags { diff --git a/VoidLink/VoidLink-Bridging-Header.h b/VoidLink/VoidLink-Bridging-Header.h index 3d6fe28a..e6166c71 100644 --- a/VoidLink/VoidLink-Bridging-Header.h +++ b/VoidLink/VoidLink-Bridging-Header.h @@ -16,6 +16,7 @@ #import "OSCProfile.h" #import "LayoutOnScreenControls.h" #import "ToolBarContainerView.h" +#import "Frame.h" #import "opus.h" #import "opus_defines.h" // #import "RelativeTouchHandler.h"