frame interpolation preparations

2026-08-06 19:57:24 +08:00
parent 11dd542cb3
commit 813005694a
13 changed files with 641 additions and 24 deletions
@@ -143,6 +143,7 @@
 		FB89463019F646E200339C8A /* StreamConfiguration.m in Sources */ = {isa = PBXBuildFile; fileRef = FB89461919F646E200339C8A /* StreamConfiguration.m */; };
 		FB89463119F646E200339C8A /* StreamManager.m in Sources */ = {isa = PBXBuildFile; fileRef = FB89461B19F646E200339C8A /* StreamManager.m */; };
 		FB89463219F646E200339C8A /* VideoDecoderRenderer.m in Sources */ = {isa = PBXBuildFile; fileRef = FB89461D19F646E200339C8A /* VideoDecoderRenderer.m */; };
+		A1F100012FFB000100000001 /* FrameInterpolator.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1F100002FFB000100000001 /* FrameInterpolator.swift */; };
 		FB89463419F646E200339C8A /* Utils.m in Sources */ = {isa = PBXBuildFile; fileRef = FB89462219F646E200339C8A /* Utils.m */; };
 		FB89463519F646E200339C8A /* MainFrameViewController.m in Sources */ = {isa = PBXBuildFile; fileRef = FB89462519F646E200339C8A /* MainFrameViewController.m */; };
 		FB89463619F646E200339C8A /* StreamFrameViewController.m in Sources */ = {isa = PBXBuildFile; fileRef = FB89462719F646E200339C8A /* StreamFrameViewController.m */; };
@@ -393,6 +394,7 @@
 		FB89461B19F646E200339C8A /* StreamManager.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = StreamManager.m; sourceTree = "<group>"; };
 		FB89461C19F646E200339C8A /* VideoDecoderRenderer.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = VideoDecoderRenderer.h; sourceTree = "<group>"; };
 		FB89461D19F646E200339C8A /* VideoDecoderRenderer.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = VideoDecoderRenderer.m; sourceTree = "<group>"; };
+		A1F100002FFB000100000001 /* FrameInterpolator.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FrameInterpolator.swift; sourceTree = "<group>"; };
 		FB89462119F646E200339C8A /* Utils.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = Utils.h; sourceTree = "<group>"; };
 		FB89462219F646E200339C8A /* Utils.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = Utils.m; sourceTree = "<group>"; };
 		FB89462419F646E200339C8A /* MainFrameViewController.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = MainFrameViewController.h; sourceTree = "<group>"; };
@@ -821,6 +823,7 @@
 				FB89461B19F646E200339C8A /* StreamManager.m */,
 				FB89461C19F646E200339C8A /* VideoDecoderRenderer.h */,
 				FB89461D19F646E200339C8A /* VideoDecoderRenderer.m */,
+				A1F100002FFB000100000001 /* FrameInterpolator.swift */,
 				9803CCAB254F9EAF00EE185E /* ConnectionCallbacks.h */,
 			);
 			path = Stream;
@@ -1153,6 +1156,7 @@
 				843CC7F62C1E7A1700BBC475 /* NativeTouchHandler.m in Sources */,
 				97E20DEE2E2165A5000D5DCA /* ImGuiPlots.mm in Sources */,
 				FB89463219F646E200339C8A /* VideoDecoderRenderer.m in Sources */,
+				A1F100012FFB000100000001 /* FrameInterpolator.swift in Sources */,
 				2A263D4F2FF3BA2800154DEA /* PublicUtils.swift in Sources */,
 				FB290D0419B2C406004C83CF /* AppDelegate.m in Sources */,
 				FB9AFD401A7E127D00872C98 /* AppListResponse.m in Sources */,
@@ -59,6 +59,7 @@ static AVAudioFormat *audioFormat;

 static bool muteInBackground;
 static bool fullColorRange;
+static bool request10BitCodec;

 static VideoDecoderRenderer* renderer;

@@ -66,7 +67,7 @@ static BandwidthTracker *bwTracker;

 int DrDecoderSetup(int videoFormat, int width, int height, int redrawRate, void* context, int drFlags)
 {
-    [renderer setupWithVideoFormat:videoFormat width:width height:height frameRate:redrawRate fullRange:fullColorRange];
+    [renderer setupWithVideoFormat:videoFormat width:width height:height frameRate:redrawRate fullRange:fullColorRange request10BitCodec:request10BitCodec];
    lastFrameNumber = 0;
    activeVideoFormat = videoFormat;
    Log(LOG_I, @"Active video format: 0x%x", activeVideoFormat);
@@ -657,6 +658,8 @@ void ClSetControllerLED(uint16_t controllerNumber, uint8_t r, uint8_t g, uint8_t
    LiInitializeStreamConfiguration(&_streamConfig);
    _streamConfig.colorRange = config.fullColorRange ? 1 : 0;
    fullColorRange = config.fullColorRange;
+    // request10BitCodec = config.enableHdr || config.sdrPerformanceWorkaround;
+    request10BitCodec = config.enableHdr;
    _streamConfig.width = config.width;
    _streamConfig.height = config.height;
    _streamConfig.fps = config.frameRate;
@@ -0,0 +1,497 @@
+import AVFoundation
+import CoreVideo
+import ObjectiveC.runtime
+import VideoToolbox
+
+@objcMembers
+final class FrameInterpolator: NSObject {
+    typealias Completion = (NSArray) -> Void
+
+    private struct PendingFrame {
+        let frame: Frame
+        let completion: Completion
+    }
+
+    private let queue = DispatchQueue(label: "com.voidlink.FrameInterpolator", qos: .userInteractive)
+    private var processor: AnyObject?
+    private var pixelBufferPool: CVPixelBufferPool?
+    private var formatDescription: CMVideoFormatDescription?
+    private var previousFrame: Frame?
+    private var pendingFrames: [PendingFrame] = []
+    private var isProcessing = false
+    private var width = 0
+    private var height = 0
+    private var slowUntil = CFTimeInterval(0)
+    private var completedProcessingCount = 0
+    private var consecutiveSlowFrames = 0
+    private var loggedEvents = Set<String>()
+
+    var isEnabled = false {
+        didSet {
+            if !isEnabled {
+                reset()
+            }
+        }
+    }
+
+    @objc static let deviceSupportsInterpolation: Bool = {
+        #if targetEnvironment(simulator)
+        return false
+        #else
+        if #available(iOS 26.0, tvOS 26.0, *) {
+            VTLowLatencyFrameInterpolationConfiguration.printLimits()
+            return VTLowLatencyFrameInterpolationConfiguration.isSupported
+        }
+        return false
+        #endif
+    }()
+
+    var isAvailable: Bool {
+        isEnabled && Self.deviceSupportsInterpolation
+    }
+
+    func reset() {
+        queue.async { [weak self] in
+            self?.resetLocked()
+        }
+    }
+
+    func processFrame(_ frame: Frame, completion: @escaping Completion) {
+        queue.async { [weak self] in
+            guard let self else {
+                completion([frame])
+                return
+            }
+
+            self.pendingFrames.append(PendingFrame(frame: frame, completion: completion))
+            self.drainPendingFrames()
+        }
+    }
+
+    private func drainPendingFrames() {
+        guard !isProcessing, !pendingFrames.isEmpty else {
+            return
+        }
+
+        let pendingFrame = pendingFrames.removeFirst()
+        let frame = pendingFrame.frame
+        let completion = pendingFrame.completion
+        guard isAvailable, CACurrentMediaTime() >= slowUntil else {
+            if isEnabled && !Self.deviceSupportsInterpolation {
+                logOnce("unavailable", "disabled: VT low-latency frame interpolation is unsupported on this OS/device")
+            }
+            previousFrame = frame
+            completion([frame])
+            drainPendingFrames()
+            return
+        }
+
+        if frame.frameType == FRAME_TYPE_IDR {
+            resetLocked(keepPendingFrames: true)
+            previousFrame = frame
+            completion([frame])
+            drainPendingFrames()
+            return
+        }
+
+        guard
+            let currentSampleBuffer = frame.sampleBuffer,
+            let currentPixelBuffer = CMSampleBufferGetImageBuffer(currentSampleBuffer)
+        else {
+            logOnce("missing-image-buffer", "input CMSampleBuffer has no CVPixelBuffer")
+            previousFrame = frame
+            completion([frame])
+            drainPendingFrames()
+            return
+        }
+
+        // renderFrame() rewrites the sample buffer's output PTS for local display pacing.
+        // Frame.pts90 retains the original upstream PTS required for interpolation.
+        let currentPTS = frame.pts90
+        guard CMTIME_IS_VALID(currentPTS) else {
+            logOnce("invalid-pts", "input frame has an invalid PTS")
+            previousFrame = frame
+            completion([frame])
+            drainPendingFrames()
+            return
+        }
+
+        guard prepareSessionIfNeeded(pixelBuffer: currentPixelBuffer) else {
+            previousFrame = frame
+            completion([frame])
+            drainPendingFrames()
+            return
+        }
+
+        guard
+            let previousFrame,
+            let previousSampleBuffer = previousFrame.sampleBuffer,
+            let previousPixelBuffer = CMSampleBufferGetImageBuffer(previousSampleBuffer)
+        else {
+            previousFrame = frame
+            completion([frame])
+            drainPendingFrames()
+            return
+        }
+
+        let previousPTS = previousFrame.pts90
+        guard CMTIME_IS_VALID(previousPTS), CMTimeCompare(currentPTS, previousPTS) > 0 else {
+            logOnce(
+                "non-monotonic-pts",
+                "upstream PTS is not increasing: previous \(CMTimeGetSeconds(previousPTS)), current \(CMTimeGetSeconds(currentPTS))"
+            )
+            self.previousFrame = frame
+            completion([frame])
+            drainPendingFrames()
+            return
+        }
+        let sourceFrameInterval = CMTimeGetSeconds(CMTimeSubtract(currentPTS, previousPTS))
+
+        #if targetEnvironment(simulator)
+        self.previousFrame = frame
+        completion([frame])
+        drainPendingFrames()
+        return
+        #else
+        guard #available(iOS 26.0, tvOS 26.0, *) else {
+            self.previousFrame = frame
+            completion([frame])
+            drainPendingFrames()
+            return
+        }
+
+        guard
+            let processor = processor as? VTFrameProcessor,
+            let sourceProcessorFrame = VTFrameProcessorFrame(buffer: currentPixelBuffer, presentationTimeStamp: currentPTS),
+            let previousProcessorFrame = VTFrameProcessorFrame(buffer: previousPixelBuffer, presentationTimeStamp: previousPTS),
+            let destinationPixelBuffer = makeDestinationPixelBuffer(),
+            let destinationProcessorFrame = VTFrameProcessorFrame(
+                buffer: destinationPixelBuffer,
+                presentationTimeStamp: midpointTime(previousPTS, currentPTS)
+            ),
+            let parameters = VTLowLatencyFrameInterpolationParameters(
+                sourceFrame: sourceProcessorFrame,
+                previousFrame: previousProcessorFrame,
+                interpolationPhase: [0.5],
+                destinationFrames: [destinationProcessorFrame]
+            )
+        else {
+            logOnce(
+                "parameters-failed",
+                "could not create interpolation parameters; input pixel format is \(pixelFormatName(CVPixelBufferGetPixelFormatType(currentPixelBuffer)))"
+            )
+            self.previousFrame = frame
+            completion([frame])
+            drainPendingFrames()
+            return
+        }
+
+        isProcessing = true
+        let startedAt = CACurrentMediaTime()
+        processor.process(parameters: parameters) { [weak self] _, error in
+            guard let self else {
+                completion([frame])
+                return
+            }
+
+            self.queue.async {
+                self.isProcessing = false
+                self.previousFrame = frame
+
+                let elapsed = CACurrentMediaTime() - startedAt
+                guard
+                    error == nil,
+                    let interpolatedFrame = self.makeInterpolatedFrame(
+                        from: frame,
+                        imageBuffer: destinationPixelBuffer,
+                        presentationTimeStamp: destinationProcessorFrame.presentationTimeStamp
+                    )
+                else {
+                    if let error {
+                        self.logOnce("process-error-\((error as NSError).code)", "processing failed: \(error)")
+                    } else {
+                        self.logOnce("sample-buffer-failed", "processing completed but the interpolated CMSampleBuffer could not be created")
+                    }
+                    completion([frame])
+                    self.drainPendingFrames()
+                    return
+                }
+
+                self.completedProcessingCount += 1
+                if sourceFrameInterval.isFinite, sourceFrameInterval > 0, elapsed >= sourceFrameInterval {
+                    self.consecutiveSlowFrames += 1
+                } else {
+                    self.consecutiveSlowFrames = 0
+                }
+                if self.completedProcessingCount >= 10, self.consecutiveSlowFrames >= 3 {
+                    self.slowUntil = CACurrentMediaTime() + 1.0
+                    self.consecutiveSlowFrames = 0
+                    self.logOnce(
+                        "sustained-overrun",
+                        "processing repeatedly exceeded the source-frame interval; temporarily falling back to original frames"
+                    )
+                }
+
+                // NSLog("[FrameInterpolator] interpolated frame in %.2f ms",)
+                completion([interpolatedFrame, frame])
+                self.drainPendingFrames()
+            }
+        }
+        #endif
+    }
+
+    private func prepareSessionIfNeeded(pixelBuffer: CVPixelBuffer) -> Bool {
+        #if targetEnvironment(simulator)
+        return false
+        #else
+        let width = CVPixelBufferGetWidth(pixelBuffer)
+        let height = CVPixelBufferGetHeight(pixelBuffer)
+        if processor != nil, self.width == width, self.height == height {
+            return true
+        }
+
+        resetLocked(keepPendingFrames: true)
+
+        guard #available(iOS 26.0, tvOS 26.0, *) else {
+            return false
+        }
+
+        guard
+            VTLowLatencyFrameInterpolationConfiguration.isSupported,
+            let configuration = VTLowLatencyFrameInterpolationConfiguration(
+                frameWidth: width,
+                frameHeight: height,
+                numberOfInterpolatedFrames: 1
+            )
+        else {
+            logOnce("configuration-failed", "could not create a configuration for \(width)x\(height)")
+            return false
+        }
+
+        let inputPixelFormat = CVPixelBufferGetPixelFormatType(pixelBuffer)
+        if !configuration.supportedPixelFormats.contains(inputPixelFormat) {
+            let supportedFormats = configuration.supportedPixelFormats
+                .map(pixelFormatName)
+                .joined(separator: ", ")
+            logOnce(
+                "unsupported-pixel-format-\(inputPixelFormat)",
+                "input pixel format \(pixelFormatName(inputPixelFormat)) is unsupported; supported formats: \(supportedFormats)"
+            )
+            return false
+        }
+
+        let processor = VTFrameProcessor()
+        do {
+            try processor.startSession(configuration: configuration)
+        } catch {
+            logOnce("session-error-\((error as NSError).code)", "failed to start session: \(error)")
+            return false
+        }
+
+        var pool: CVPixelBufferPool?
+        let status = CVPixelBufferPoolCreate(
+            kCFAllocatorDefault,
+            nil,
+            configuration.destinationPixelBufferAttributes as CFDictionary,
+            &pool
+        )
+        guard status == kCVReturnSuccess, let pool else {
+            logOnce("pool-error-\(status)", "failed to create destination pixel buffer pool: \(status)")
+            return false
+        }
+
+        self.width = width
+        self.height = height
+        self.processor = processor
+        self.pixelBufferPool = pool
+        logOnce(
+            "session-started-\(width)x\(height)",
+            "session started for \(width)x\(height), input pixel format \(pixelFormatName(inputPixelFormat))"
+        )
+        return true
+        #endif
+    }
+
+    private func makeDestinationPixelBuffer() -> CVPixelBuffer? {
+        guard let pixelBufferPool else {
+            return nil
+        }
+
+        var pixelBuffer: CVPixelBuffer?
+        let status = CVPixelBufferPoolCreatePixelBuffer(kCFAllocatorDefault, pixelBufferPool, &pixelBuffer)
+        return status == kCVReturnSuccess ? pixelBuffer : nil
+    }
+
+    private func makeInterpolatedFrame(
+        from sourceFrame: Frame,
+        imageBuffer: CVPixelBuffer,
+        presentationTimeStamp: CMTime
+    ) -> Frame? {
+        if formatDescription == nil || !CMVideoFormatDescriptionMatchesImageBuffer(formatDescription!, imageBuffer: imageBuffer) {
+            var newFormatDescription: CMVideoFormatDescription?
+            let status = CMVideoFormatDescriptionCreateForImageBuffer(
+                allocator: kCFAllocatorDefault,
+                imageBuffer: imageBuffer,
+                formatDescriptionOut: &newFormatDescription
+            )
+            guard status == noErr else {
+                return nil
+            }
+            formatDescription = newFormatDescription
+        }
+
+        let duration = sourceFrame.sampleBuffer.map { CMSampleBufferGetDuration($0) } ?? .invalid
+        var timing = CMSampleTimingInfo(
+            duration: duration,
+            presentationTimeStamp: presentationTimeStamp,
+            decodeTimeStamp: .invalid
+        )
+
+        var sampleBuffer: CMSampleBuffer?
+        let status = CMSampleBufferCreateReadyWithImageBuffer(
+            allocator: kCFAllocatorDefault,
+            imageBuffer: imageBuffer,
+            formatDescription: formatDescription!,
+            sampleTiming: &timing,
+            sampleBufferOut: &sampleBuffer
+        )
+        guard status == noErr, let sampleBuffer else {
+            return nil
+        }
+
+        let retainedSampleBuffer = Unmanaged.passRetained(sampleBuffer).takeUnretainedValue()
+        let frame = Frame(
+            sampleBuffer: retainedSampleBuffer,
+            frameNumber: sourceFrame.frameNumber,
+            frameType: FRAME_TYPE_PFRAME
+        )
+        frame?.isInterpolated = true
+        return frame
+    }
+
+    private func midpointTime(_ first: CMTime, _ second: CMTime) -> CMTime {
+        CMTimeAdd(first, CMTimeMultiplyByFloat64(CMTimeSubtract(second, first), multiplier: 0.5))
+    }
+
+    private func logOnce(_ event: String, _ message: String) {
+        guard loggedEvents.insert(event).inserted else {
+            return
+        }
+        NSLog("[FrameInterpolator] %@", message)
+    }
+
+    private func pixelFormatName(_ pixelFormat: OSType) -> String {
+        let bytes: [UInt8] = [
+            UInt8((pixelFormat >> 24) & 0xff),
+            UInt8((pixelFormat >> 16) & 0xff),
+            UInt8((pixelFormat >> 8) & 0xff),
+            UInt8(pixelFormat & 0xff),
+        ]
+        let fourCC = String(bytes: bytes, encoding: .ascii) ?? "????"
+        return "\(fourCC) (\(pixelFormat))"
+    }
+
+    private func resetLocked(keepPendingFrames: Bool = false) {
+        processor = nil
+        pixelBufferPool = nil
+        formatDescription = nil
+        previousFrame = nil
+        isProcessing = false
+        width = 0
+        height = 0
+        slowUntil = 0
+        completedProcessingCount = 0
+        consecutiveSlowFrames = 0
+        if !keepPendingFrames {
+            pendingFrames.removeAll()
+        }
+    }
+}
+
+#if !targetEnvironment(simulator)
+@available(iOS 26.0, tvOS 26.0, *)
+extension VTLowLatencyFrameInterpolationConfiguration {
+    struct RuntimeDimensionLimits {
+        let minimumDimensions: CMVideoDimensions?
+        let maximumDimensions: CMVideoDimensions?
+        let maximumDimension: Int?
+        let maximumPixelCount: Int?
+
+        func supports(width: Int, height: Int) -> Bool {
+            guard width > 0, height > 0 else {
+                return false
+            }
+
+            if let minimumDimensions,
+               (width < Int(minimumDimensions.width) || height < Int(minimumDimensions.height)) {
+                return false
+            }
+
+            if let maximumDimensions,
+               (width > Int(maximumDimensions.width) || height > Int(maximumDimensions.height)) {
+                return false
+            }
+
+            if let maximumDimension,
+               (width > maximumDimension || height > maximumDimension) {
+                return false
+            }
+
+            if let maximumPixelCount {
+                let (pixelCount, overflow) = width.multipliedReportingOverflow(by: height)
+                if overflow || pixelCount > maximumPixelCount {
+                    return false
+                }
+            }
+
+            return maximumDimensions != nil || maximumDimension != nil || maximumPixelCount != nil
+        }
+    }
+
+    static func runtimeDimensionLimits(spatialScaleFactor: Int = 1) -> RuntimeDimensionLimits {
+        RuntimeDimensionLimits(
+            minimumDimensions: minimumDimensions,
+            maximumDimensions: maximumDimensions,
+            maximumDimension: callRuntimeIntegerClassMethod(
+                "maximumDimensionForSpatialScaleFactor:",
+                argument: spatialScaleFactor
+            ),
+            maximumPixelCount: callRuntimeIntegerClassMethod(
+                "maximumPixelCountForSpatialScaleFactor:",
+                argument: spatialScaleFactor
+            )
+        )
+    }
+
+    private static func callRuntimeIntegerClassMethod(
+        _ selectorName: String,
+        argument: Int
+    ) -> Int? {
+        let configurationClass: AnyClass = self
+        let selector = NSSelectorFromString(selectorName)
+        guard let method = class_getClassMethod(configurationClass, selector) else {
+            return nil
+        }
+
+        typealias Function = @convention(c) (AnyClass, Selector, Int) -> Int
+        let function = unsafeBitCast(method_getImplementation(method), to: Function.self)
+        let result = function(configurationClass, selector, argument)
+        return result > 0 ? result : nil
+    }
+    
+    static func printLimits() {
+        let limits =
+            VTLowLatencyFrameInterpolationConfiguration
+                .runtimeDimensionLimits(spatialScaleFactor: 1)
+
+        print("minimum:", limits.minimumDimensions as Any)
+        print("legacy maximum:", limits.maximumDimensions as Any)
+        print("maximum dimension:", limits.maximumDimension as Any)
+        print("maximum pixel count:", limits.maximumPixelCount as Any)
+
+        let supported = limits.supports(width: 1920, height: 1080)
+        print("1920x1080 supported:", supported)
+    }
+    
+}
+#endif
@@ -22,6 +22,8 @@
@property BOOL enableYUV444;
@property BOOL enablePIP;
@property BOOL fullColorRange;
+@property BOOL enableHdr;
+@property BOOL sdrPerformanceWorkaround;
@property int width;
@property int height;
@property int frameRate;
@@ -9,5 +9,5 @@
 #import "StreamConfiguration.h"

@implementation StreamConfiguration
-@synthesize host, httpsPort, appID, width, height, frameRate, bitRate, riKeyId, riKey, gamepadMask, appName, optimizeGameSettings, playAudioOnPC, swapABXYButtons, buttonVisualFeedback, gyroMode, emulatedControllerType, audioConfiguration, supportedVideoFormats, multiController, serverCert, rtspSessionUrl, serverCodecModeSupport, enableYUV444, enablePIP, fullColorRange, localVolume;
+@synthesize host, httpsPort, appID, width, height, frameRate, bitRate, riKeyId, riKey, gamepadMask, appName, optimizeGameSettings, playAudioOnPC, swapABXYButtons, buttonVisualFeedback, gyroMode, emulatedControllerType, audioConfiguration, supportedVideoFormats, multiController, serverCert, rtspSessionUrl, serverCodecModeSupport, enableYUV444, enablePIP, fullColorRange, enableHdr, sdrPerformanceWorkaround, localVolume;
@end
@@ -26,7 +26,8 @@

 - (id)initWithView:(UIView*)view callbacks:(id<ConnectionCallbacks>)callbacks streamAspectRatio:(float)aspectRatio;

- (void)setupWithVideoFormat:(int)videoFormat width:(int)videoWidth height:(int)videoHeight frameRate:(int)frameRate fullRange:(BOOL)fullRange;
+- (void)setupWithVideoFormat:(int)videoFormat width:(int)videoWidth height:(int)videoHeight frameRate:(int)frameRate fullRange:(BOOL)fullRange request10BitCodec:(BOOL)enableHdr;
+
 - (void)renderFrame:(Frame *)frame atTime:(CMTime)targetTime;
 - (void)cleanup;
 - (void)setHdrMode:(BOOL)enabled;
@@ -47,5 +48,6 @@
                        decodeStartTime:(CFTimeInterval)decodeStartTime;

 - (void)invalidateDecompressionSession;
+ (void)setFrameInterpolationEnabled:(bool)enabled;

@end
@@ -22,6 +22,7 @@
 #import "PlatformThreads.h"
 #import "MetalViewController.h"
 #import "ImGuiPlots.h"
+#import "VoidLink-Swift.h"

 #include <libavcodec/avcodec.h>
 #include <libavcodec/cbs.h>
@@ -34,6 +35,8 @@
 // Define for extra logging related to frame pacing
 //#define DISPLAYLINK_VERBOSE

+static BOOL kEnableFrameInterpolation = false;
+
 // Private libavformat API for writing the AV1 Codec Configuration Box
 extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size,
                              int write_seq_header);
@@ -48,6 +51,7 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size,
    int _videoFormat;
    int _frameRate;
    BOOL _fullRange;
+    BOOL _request10BitCodec;

    NSMutableArray *_parameterSetBuffers;
    NSData *_masteringDisplayColorVolume;
@@ -55,6 +59,7 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size,
    CMVideoFormatDescriptionRef _formatDesc;
    CMVideoFormatDescriptionRef _formatDescImageBuffer;
    VTDecompressionSessionRef _decompressionSession;
+    FrameInterpolator *_frameInterpolator;

    CADisplayLink *_displayLink;
    NSInteger _maxRefreshRate;
@@ -68,6 +73,10 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size,
    // NSTimeInterval previousLinkTime;
 }

+ (void)setFrameInterpolationEnabled:(bool)enabled {
+    kEnableFrameInterpolation = enabled;
+}
+
 - (void)reinitializeDisplayLayer
 {
    if (_displayLayer == nil) {
@@ -185,11 +194,12 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size,

 # pragma mark DisplayLink vsync callback

- (void)setupWithVideoFormat:(int)videoFormat width:(int)videoWidth height:(int)videoHeight frameRate:(int)frameRate fullRange:(BOOL)fullRange
+- (void)setupWithVideoFormat:(int)videoFormat width:(int)videoWidth height:(int)videoHeight frameRate:(int)frameRate fullRange:(BOOL)fullRange request10BitCodec:(BOOL)request10BitCodec
 {
    self->_videoFormat = videoFormat;
    self->_frameRate = frameRate;
    self->_fullRange = fullRange;
+    self->_request10BitCodec = request10BitCodec;

    // reset plot data in case we've already used it for a previous renderer
    [[ImGuiPlots sharedInstance] clearData];
@@ -197,6 +207,11 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size,
    DataManager* dataMan = [[DataManager alloc] init];
    if ([[dataMan getSettings].renderingBackend integerValue] == RENDER_AVSB) {
        _renderingBackend = RENDER_AVSB;
+
+        if (kEnableFrameInterpolation) {
+            _frameInterpolator = [[FrameInterpolator alloc] init];
+            _frameInterpolator.isEnabled = YES;
+        }
        
        // Choose the appropriate selector based on frame pacing mode
        SEL displayLinkSelector;
@@ -213,11 +228,16 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size,
        
        _displayLink = [CADisplayLink displayLinkWithTarget:self selector:displayLinkSelector];

+        NSInteger targetFrameRate = self->_frameRate;
+        if (self->_frameInterpolator != nil) {
+            targetFrameRate = MIN(self->_frameRate * 2, self->_maxRefreshRate);
+        }
+
        if (@available(iOS 15.0, tvOS 15.0, *)) {
-            _displayLink.preferredFrameRateRange = CAFrameRateRangeMake(self->_frameRate, self->_frameRate, self->_frameRate);
+            _displayLink.preferredFrameRateRange = CAFrameRateRangeMake(targetFrameRate, targetFrameRate, targetFrameRate);
        }
        else {
-            _displayLink.preferredFramesPerSecond = self->_frameRate;
+            _displayLink.preferredFramesPerSecond = targetFrameRate;
        }
        [_displayLink addToRunLoop:[NSRunLoop mainRunLoop] forMode:NSDefaultRunLoopMode];
    } else {
@@ -227,7 +247,7 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size,
 }


- (void)setupDecompressionSessionWithAttributes:(NSDictionary *)destinationPixelBufferAttributes {
+- (OSStatus)setupDecompressionSessionWithAttributes:(NSDictionary *)destinationPixelBufferAttributes {
    // This method is called from within synchronized block, so no additional sync needed here
    if (_decompressionSession != NULL) {
        VTDecompressionSessionInvalidate(_decompressionSession);
@@ -235,15 +255,16 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size,
        _decompressionSession = nil;
    }

-    int status = VTDecompressionSessionCreate(kCFAllocatorDefault,
-                                              _formatDesc,
-                                              nil,
-                                              (__bridge CFDictionaryRef)destinationPixelBufferAttributes,
-                                              nil,
-                                              &_decompressionSession);
+    OSStatus status = VTDecompressionSessionCreate(kCFAllocatorDefault,
+                                                   _formatDesc,
+                                                   nil,
+                                                   (__bridge CFDictionaryRef)destinationPixelBufferAttributes,
+                                                   nil,
+                                                   &_decompressionSession);
    if (status != noErr) {
        Log(LOG_E, @"Failed to create VTDecompressionSession, status %d", status);
    }
+    return status;
 }

 - (void)setupDecompressionSession {
@@ -256,11 +277,20 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size,
        },
    } mutableCopy];
 #else
-    NSNumber *pixelFormat = nil;
+    NSNumber *nativePixelFormat = nil;
    if (self->_videoFormat & VIDEO_FORMAT_MASK_YUV444) {
-        pixelFormat = self->_fullRange ? @(kCVPixelFormatType_444YpCbCr10BiPlanarFullRange) : @(kCVPixelFormatType_444YpCbCr10BiPlanarVideoRange);
+        NSNumber *frFormat = _request10BitCodec ? @(kCVPixelFormatType_444YpCbCr10BiPlanarFullRange) : @(kCVPixelFormatType_444YpCbCr8BiPlanarFullRange);
+        NSNumber *vrFormat = _request10BitCodec ? @(kCVPixelFormatType_444YpCbCr10BiPlanarVideoRange) : @(kCVPixelFormatType_444YpCbCr8BiPlanarVideoRange);
+        nativePixelFormat = self->_fullRange ? frFormat : vrFormat;
    } else {
-        pixelFormat = self->_fullRange ? @(kCVPixelFormatType_420YpCbCr10BiPlanarFullRange) : @(kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange);
+        NSNumber *frFormat = _request10BitCodec ? @(kCVPixelFormatType_420YpCbCr10BiPlanarFullRange) : @(kCVPixelFormatType_420YpCbCr8BiPlanarFullRange);
+        NSNumber *vrFormat = _request10BitCodec ? @(kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange) : @(kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange);
+        nativePixelFormat = self->_fullRange ? frFormat : vrFormat;
+    }
+    BOOL interpolationRequested = self->_frameInterpolator != nil;
+    NSNumber *pixelFormat = interpolationRequested ? @(kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange) : nativePixelFormat;
+    if (interpolationRequested) {
+        LogOnce(LOG_I, @"Frame interpolation enabled; requesting 420v output from VTDecompressionSession");
    }
    NSMutableDictionary *destinationPixelBufferAttributes = [@{
        (id)kCVPixelBufferPixelFormatTypeKey : pixelFormat
@@ -272,7 +302,31 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size,
        destinationPixelBufferAttributes[(id)kVTDecompressionPropertyKey_GeneratePerFrameHDRDisplayMetadata] = @YES;
    }

-    return [self setupDecompressionSessionWithAttributes:destinationPixelBufferAttributes];
+#if !TARGET_OS_SIMULATOR
+    if (!interpolationRequested) {
+        [self setupDecompressionSessionWithAttributes:destinationPixelBufferAttributes];
+        return;
+    }
+
+    OSStatus status = [self setupDecompressionSessionWithAttributes:destinationPixelBufferAttributes];
+    if (status != noErr) {
+        Log(LOG_W, @"Falling back from 420v frame-interpolation decode output to native pixel format %@", nativePixelFormat);
+        [self->_frameInterpolator reset];
+        self->_frameInterpolator = nil;
+        destinationPixelBufferAttributes[(id)kCVPixelBufferPixelFormatTypeKey] = nativePixelFormat;
+        [self setupDecompressionSessionWithAttributes:destinationPixelBufferAttributes];
+
+        NSInteger targetFrameRate = MIN(self->_frameRate, self->_maxRefreshRate);
+        if (@available(iOS 15.0, tvOS 15.0, *)) {
+            self->_displayLink.preferredFrameRateRange = CAFrameRateRangeMake(targetFrameRate, targetFrameRate, targetFrameRate);
+        } else {
+            self->_displayLink.preferredFramesPerSecond = targetFrameRate;
+        }
+    }
+#else
+    [self setupDecompressionSessionWithAttributes:destinationPixelBufferAttributes];
+#endif
+    return;
 }

 - (void) checkDisplayLayer {
@@ -477,6 +531,9 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit);
                self->_decompressionSession = nil;
            }
        }
+        if (self->_frameInterpolator != nil) {
+            [self->_frameInterpolator reset];
+        }
    });
 }

@@ -1073,6 +1130,37 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit);
                        frame = [[Frame alloc] initWithPixelBufffer:pixelBuffer frameNumber:frameNumber frameType:frameType pts:presentationTimestamp];
                        [frame setFormatDesc:self->_formatDesc];
                    }
+
+                    if (self->_frameInterpolator != nil) {
+                        [self->_frameInterpolator processFrame:frame completion:^(NSArray *frames) {
+                            dispatch_async(self->_vtq, ^{
+                                int framesDropped = 0;
+                                for (Frame *outputFrame in (NSArray<Frame *> *)frames) {
+                                    framesDropped += [self->_frameQueue enqueue:outputFrame withSlackSize:3];
+                                }
+
+                                if ([[UIApplication sharedApplication] applicationState] != UIApplicationStateBackground) {
+                                    static PlotMetrics frameQueueMetrics = {};
+                                    [[ImGuiPlots sharedInstance] observeFloatReturnMetrics:PLOT_QUEUED_FRAMES value:[self->_frameQueue count] plotMetrics:&frameQueueMetrics];
+                                    [self safeCopyMetricsTo:&self->_frameQueueMetrics from:&frameQueueMetrics];
+
+                                    [[ImGuiPlots sharedInstance] observeFloat:PLOT_DROPPED value:framesDropped];
+
+                                    static CFTimeInterval lastHostFrame = 0.0f;
+                                    if (lastHostFrame != 0) {
+                                        [[ImGuiPlots sharedInstance] observeFloat:PLOT_HOST_FRAMETIME value:(frame.pts - lastHostFrame) * 1000.0];
+                                    }
+                                    lastHostFrame = frame.pts;
+
+                                    static PlotMetrics decodeMetrics = {};
+                                    [[ImGuiPlots sharedInstance] observeFloatReturnMetrics:PLOT_DECODE value:(CACurrentMediaTime() - decodeStartTime) * 1000.0 plotMetrics:&decodeMetrics];
+                                    [self safeCopyMetricsTo:&self->_decodeMetrics from:&decodeMetrics];
+                                }
+                            });
+                        }];
+                        return;
+                    }
+
                    int framesDropped = [self->_frameQueue enqueue:frame withSlackSize:3];

                    if ([[UIApplication sharedApplication] applicationState] != UIApplicationStateBackground) {
@@ -4,6 +4,7 @@
@interface Frame : NSObject
@property (nonatomic) int frameNumber;
@property (nonatomic) int frameType;
+@property (nonatomic) BOOL isInterpolated;
@property (nonatomic) CFTimeInterval decodedAt;
@property (nonatomic) CMTime pts90;
@property (nonatomic) CMTime duration90;
@@ -1,4 +1,5 @@
 #import <CoreVideo/CVImageBuffer.h>
+#import <objc/runtime.h>
 #import "Frame.h"
 #import "Logger.h"
 #include <Limelight.h>
@@ -7,6 +8,19 @@
    CFDictionaryRef _formatDescExt;
 }

+static char FrameInterpolatedKey;
+
+- (BOOL)isInterpolated {
+    return [objc_getAssociatedObject(self, &FrameInterpolatedKey) boolValue];
+}
+
+- (void)setIsInterpolated:(BOOL)isInterpolated {
+    objc_setAssociatedObject(self,
+                             &FrameInterpolatedKey,
+                             isInterpolated ? @YES : nil,
+                             OBJC_ASSOCIATION_RETAIN_NONATOMIC);
+}
+
 - (instancetype)initWithPixelBufffer:(CVPixelBufferRef)pixelBuffer
                         frameNumber:(int)frameNumber
                           frameType:(int)frameType
@@ -875,6 +875,8 @@ static NSMutableSet* hostList;
    _streamConfig.enableYUV444 = streamSettings.enableYUV444;
    _streamConfig.enablePIP = streamSettings.enablePIP;
    _streamConfig.fullColorRange = streamSettings.fullColorRange;
+    _streamConfig.enableHdr = streamSettings.enableHdr;
+    _streamConfig.sdrPerformanceWorkaround = streamSettings.sdrPerformanceWorkaround;
    _streamConfig.asyncNativeTouchPriority = streamSettings.asyncNativeTouchPriority; // new streamConfig segment
    _streamConfig.gyroMode = [streamSettings.gyroMode intValue];
    _streamConfig.emulatedControllerType = streamSettings.emulatedControllerType.intValue;
@@ -908,6 +910,7 @@ static NSMutableSet* hostList;
    
    Connection.useSystemAudioEngine = streamSettings.audioConfig.intValue == 2;
    
+    bool sdrPerformanceWorkaround = false;
    switch (streamSettings.preferredCodec) {
        case CODEC_PREF_AV1:
 #if defined(__IPHONE_16_0) || defined(__TVOS_16_0)
@@ -917,7 +920,7 @@ static NSMutableSet* hostList;
                    _streamConfig.supportedVideoFormats |= VIDEO_FORMAT_AV1_HIGH8_444;
                }
                else {
-                    if(streamSettings.sdrPerformanceWorkaround && [Utils hdrSupported]) _streamConfig.supportedVideoFormats |= VIDEO_FORMAT_AV1_MAIN10; // 8bit performance degradation workaround for av1
+                    if(sdrPerformanceWorkaround && [Utils hdrSupported]) _streamConfig.supportedVideoFormats |= VIDEO_FORMAT_AV1_MAIN10; // 8bit performance degradation workaround for av1
                    else _streamConfig.supportedVideoFormats |= VIDEO_FORMAT_AV1_MAIN8;
                }
            }
@@ -928,12 +931,12 @@ static NSMutableSet* hostList;
        case CODEC_PREF_HEVC:
            if (VTIsHardwareDecodeSupported(kCMVideoCodecType_HEVC)) {
                if (streamSettings.enableYUV444) {
-                    if(streamSettings.sdrPerformanceWorkaround && [Utils hdrSupported]) _streamConfig.supportedVideoFormats |= VIDEO_FORMAT_H265_REXT10_444; // 8bit performance degradation workaround
+                    if(sdrPerformanceWorkaround && [Utils hdrSupported]) _streamConfig.supportedVideoFormats |= VIDEO_FORMAT_H265_REXT10_444; // 8bit performance degradation workaround
                    else _streamConfig.supportedVideoFormats |= VIDEO_FORMAT_H265_REXT8_444;
                }
                else {
                    // _streamConfig.supportedVideoFormats |= VIDEO_FORMAT_H265;
-                    if(streamSettings.sdrPerformanceWorkaround && [Utils hdrSupported]) _streamConfig.supportedVideoFormats |= VIDEO_FORMAT_H265_MAIN10; // 8bit performance degradation workaround
+                    if(sdrPerformanceWorkaround && [Utils hdrSupported]) _streamConfig.supportedVideoFormats |= VIDEO_FORMAT_H265_MAIN10; // 8bit performance degradation workaround
                    else _streamConfig.supportedVideoFormats |= VIDEO_FORMAT_H265;
                }
            }
@@ -951,7 +954,7 @@ static NSMutableSet* hostList;
    // HEVC is supported if the user wants it (or it's required by the chosen resolution) and the SoC supports it
    if ((_streamConfig.width > 4096 || _streamConfig.height > 4096 || streamSettings.enableHdr) && VTIsHardwareDecodeSupported(kCMVideoCodecType_HEVC)) {
        
-        if(streamSettings.sdrPerformanceWorkaround && [Utils hdrSupported]) _streamConfig.supportedVideoFormats |= VIDEO_FORMAT_H265_MAIN10; // 8bit performance degradation workaround
+        if(sdrPerformanceWorkaround && [Utils hdrSupported]) _streamConfig.supportedVideoFormats |= VIDEO_FORMAT_H265_MAIN10; // 8bit performance degradation workaround
        else _streamConfig.supportedVideoFormats |= VIDEO_FORMAT_H265;

        // HEVC Main10 is supported if the user wants it and the display supports it
@@ -877,8 +877,8 @@ BOOL isCustomResolution(int resolutionSelected) {
    self.yuv444Stack.hasInfoTag = YES;
    [self addSetting:self.yuv444Stack ofId:@"yuv444Stack" to:videoSection];

-    self.sdrPerformanceWorkaroundStack.hasInfoTag = YES;
-    [self addSetting:self.sdrPerformanceWorkaroundStack ofId:@"sdrPerformanceWorkaroundStack" to:videoSection];
+    // self.sdrPerformanceWorkaroundStack.hasInfoTag = YES;
+    // [self addSetting:self.sdrPerformanceWorkaroundStack ofId:@"sdrPerformanceWorkaroundStack" to:videoSection];

    self.framePacingStack.hasInfoTag = YES;
    [self addSetting:self.framePacingStack ofId:@"framePacingStack" to:videoSection];
@@ -3592,6 +3592,7 @@ BOOL isCustomResolution(int resolutionSelected) {

 - (void)muteInBackgroundSwitchFlipped:(UISwitch* )sender{
    Connection.muteInBackground = sender.isOn;
+    // [VideoDecoderRenderer setFrameInterpolationEnabled:sender.isOn];
 }

 - (void)redirectMicSwitchFlipped:(UISwitch* )sender{
@@ -1798,7 +1798,7 @@ static __weak StreamFrameViewController *VLSharedStreamFrameViewController = nil
        _micStreamInitialized = false;
    }
    
-    // 8bit 444 degration workaround
+    /*
    if(strcmp(stageName, "video stream establishment")==0){
        NSLog(@"sendAutoReleaseComboCommandWithCmdStrings %f", CACurrentMediaTime());
        if(!_settings.enableHdr
@@ -1818,6 +1818,7 @@ static __weak StreamFrameViewController *VLSharedStreamFrameViewController = nil
            });
        }
    }
+    */
 }

 - (void) stageFailed:(const char*)stageName withError:(int)errorCode portTestFlags:(int)portTestFlags {
@@ -16,6 +16,7 @@
 #import "OSCProfile.h"
 #import "LayoutOnScreenControls.h"
 #import "ToolBarContainerView.h"
+#import "Frame.h"
 #import "opus.h"
 #import "opus_defines.h"
 // #import "RelativeTouchHandler.h"