From 7b7bb3837c67788ac4a081d6428cc092d9bee3da Mon Sep 17 00:00:00 2001 From: True Zhuanjia Date: Sun, 9 Aug 2026 20:34:39 +0800 Subject: [PATCH] frameInterpolation beta ready --- VoidLink/Base.lproj/iPad.storyboard | 40 + VoidLink/Base.lproj/iPhone.storyboard | 40 + VoidLink/Database/DataManager.h | 3 + VoidLink/Database/DataManager.m | 7 +- VoidLink/Database/TemporarySettings.h | 3 + VoidLink/Database/TemporarySettings.m | 6 + .../VoidLink v1.0.xcdatamodel/contents | 3 + VoidLink/Localization/Localizable.xcstrings | 63 ++ VoidLink/Stream/FrameInterpolator.swift | 771 ++++++++++++++++-- VoidLink/Stream/VideoDecoderRenderer.h | 2 + VoidLink/Stream/VideoDecoderRenderer.m | 260 ++++-- VoidLink/Utility/GenericUtils.swift | 17 + .../ViewControllers/MainFrameViewController.m | 32 +- .../ViewControllers/SettingsViewController.h | 4 + .../ViewControllers/SettingsViewController.m | 124 ++- VoidLink/mul.lproj/iPad.xcstrings | 24 + VoidLink/mul.lproj/iPhone.xcstrings | 24 + 17 files changed, 1261 insertions(+), 162 deletions(-) diff --git a/VoidLink/Base.lproj/iPad.storyboard b/VoidLink/Base.lproj/iPad.storyboard index 0abcef6f..4d7da8ef 100644 --- a/VoidLink/Base.lproj/iPad.storyboard +++ b/VoidLink/Base.lproj/iPad.storyboard @@ -411,6 +411,42 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -2302,6 +2338,10 @@ + + + + diff --git a/VoidLink/Base.lproj/iPhone.storyboard b/VoidLink/Base.lproj/iPhone.storyboard index dc6824b5..93a44458 100644 --- a/VoidLink/Base.lproj/iPhone.storyboard +++ b/VoidLink/Base.lproj/iPhone.storyboard @@ -2548,6 +2548,42 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -3569,6 +3605,10 @@ + + + + diff --git a/VoidLink/Database/DataManager.h b/VoidLink/Database/DataManager.h index 4fbef2e1..9340c8a4 100644 --- a/VoidLink/Database/DataManager.h +++ b/VoidLink/Database/DataManager.h @@ -130,6 +130,9 @@ typedef NS_ENUM(NSInteger, PencilTickMode) { graphOpacity:(NSInteger)graphOpacity renderingBackend:(NSInteger)renderingBackend framePacingMode:(NSInteger)framePacingMode + interpolationMaximumDimension:(NSInteger)interpolationMaximumDimension + interpolationMaximumPixelCount:(NSInteger)interpolationMaximumPixelCount + streamDimensionScale:(CGFloat)streamDimensionScale sendDummyEvent:(BOOL)sendDummyEvent rememberFoldState:(BOOL)rememberFoldState singleTapSensitivy:(CGFloat)singleTapSensitivy diff --git a/VoidLink/Database/DataManager.m b/VoidLink/Database/DataManager.m index a41c6a15..299ce92c 100644 --- a/VoidLink/Database/DataManager.m +++ b/VoidLink/Database/DataManager.m @@ -106,6 +106,9 @@ graphOpacity:(NSInteger)graphOpacity renderingBackend:(NSInteger)renderingBackend framePacingMode:(NSInteger)framePacingMode + interpolationMaximumDimension:(NSInteger)interpolationMaximumDimension + interpolationMaximumPixelCount:(NSInteger)interpolationMaximumPixelCount + streamDimensionScale:(CGFloat)streamDimensionScale sendDummyEvent:(BOOL)sendDummyEvent rememberFoldState:(BOOL)rememberFoldState singleTapSensitivy:(CGFloat)singleTapSensitivy @@ -193,6 +196,9 @@ settingsToSave.graphOpacity = [NSNumber numberWithInteger:graphOpacity]; settingsToSave.renderingBackend = [NSNumber numberWithInteger:renderingBackend]; settingsToSave.framePacingMode = [NSNumber numberWithInteger:framePacingMode]; + settingsToSave.interpolationMaximumDimension = [NSNumber numberWithInteger:interpolationMaximumDimension]; + settingsToSave.interpolationMaximumPixelCount = [NSNumber numberWithInteger:interpolationMaximumPixelCount]; + settingsToSave.streamDimensionScale = [NSNumber numberWithDouble:streamDimensionScale]; settingsToSave.sendDummyEvent = sendDummyEvent; settingsToSave.singleTapSensitivity = [NSNumber numberWithDouble:singleTapSensitivy]; settingsToSave.hapticEngine = [NSNumber numberWithInteger:hapticEngine]; @@ -370,4 +376,3 @@ } @end - diff --git a/VoidLink/Database/TemporarySettings.h b/VoidLink/Database/TemporarySettings.h index 7fb00ff5..80fd3fe9 100644 --- a/VoidLink/Database/TemporarySettings.h +++ b/VoidLink/Database/TemporarySettings.h @@ -99,6 +99,9 @@ @property (nonatomic) BOOL ctrlDownForPinch; @property (nonatomic) BOOL passthroughGestures; @property (nonatomic, retain) NSNumber * framePacingMode; +@property (nonatomic, retain) NSNumber * interpolationMaximumDimension; +@property (nonatomic, retain) NSNumber * interpolationMaximumPixelCount; +@property (nonatomic, retain) NSNumber * streamDimensionScale; @property (nonatomic, retain) NSNumber * scrollSensitivity; @property (nonatomic, retain) NSNumber * pinchSensitivity; @property (nonatomic, retain) NSNumber * relativeTouchSlideThreshold; diff --git a/VoidLink/Database/TemporarySettings.m b/VoidLink/Database/TemporarySettings.m index 62db404f..9e4b639e 100644 --- a/VoidLink/Database/TemporarySettings.m +++ b/VoidLink/Database/TemporarySettings.m @@ -55,6 +55,9 @@ self.graphOpacity = [NSNumber numberWithInteger:[[NSUserDefaults standardUserDefaults] integerForKey:@"graphOpacity"]]; self.renderingBackend = [NSNumber numberWithInteger:[[NSUserDefaults standardUserDefaults] integerForKey:@"renderingBackend"]]; self.framePacingMode = [NSNumber numberWithInteger:[[NSUserDefaults standardUserDefaults] integerForKey:@"framePacingMode"]]; + self.interpolationMaximumDimension = [NSNumber numberWithInteger:[[NSUserDefaults standardUserDefaults] integerForKey:@"interpolationMaximumDimension"]]; + self.interpolationMaximumPixelCount = [NSNumber numberWithInteger:[[NSUserDefaults standardUserDefaults] integerForKey:@"interpolationMaximumPixelCount"]]; + self.streamDimensionScale = [NSNumber numberWithDouble:[[NSUserDefaults standardUserDefaults] doubleForKey:@"streamDimensionScale"]]; NSInteger _screenSize = [[NSUserDefaults standardUserDefaults] integerForKey:@"streamResolution"]; switch (_screenSize) { @@ -135,6 +138,9 @@ self.graphOpacity = settings.graphOpacity; self.renderingBackend = settings.renderingBackend; self.framePacingMode = settings.framePacingMode; + self.interpolationMaximumDimension = settings.interpolationMaximumDimension ?: @(0); + self.interpolationMaximumPixelCount = settings.interpolationMaximumPixelCount ?: @(0); + self.streamDimensionScale = settings.streamDimensionScale ?: @(1); self.sendDummyEvent = settings.sendDummyEvent; self.rememberFoldState = settings.rememberFoldState; self.gyroBiasX = settings.gyroBiasX; diff --git a/VoidLink/Limelight.xcdatamodeld/VoidLink v1.0.xcdatamodel/contents b/VoidLink/Limelight.xcdatamodeld/VoidLink v1.0.xcdatamodel/contents index 083893b0..d9366ed1 100644 --- a/VoidLink/Limelight.xcdatamodeld/VoidLink v1.0.xcdatamodel/contents +++ b/VoidLink/Limelight.xcdatamodeld/VoidLink v1.0.xcdatamodel/contents @@ -74,6 +74,8 @@ + + @@ -115,6 +117,7 @@ + diff --git a/VoidLink/Localization/Localizable.xcstrings b/VoidLink/Localization/Localizable.xcstrings index 7d58ea11..19880a6e 100644 --- a/VoidLink/Localization/Localizable.xcstrings +++ b/VoidLink/Localization/Localizable.xcstrings @@ -9304,6 +9304,46 @@ } } }, + "Interpolation Resolution" : { + "extractionState" : "manual", + "localizations" : { + "zh-Hans" : { + "stringUnit" : { + "state" : "translated", + "value" : "插帧分辨率" + } + }, + "zh-Hant" : { + "stringUnit" : { + "state" : "translated", + "value" : "影格插補解析度" + } + }, + "zh-HK" : { + "stringUnit" : { + "state" : "translated", + "value" : "影格插補解析度" + } + } + } + }, + "Most devices currently support frame interpolation only for 8-bit YUV 4:2:0 SDR video. In this case, if HDR or YUV 4:4:4 is enabled, intermediate frames will still be generated in 8-bit YUV 4:2:0 SDR." : { + "extractionState" : "manual", + "localizations" : { + "zh-Hans" : { + "stringUnit" : { + "state" : "translated", + "value" : "目前多数设备仅支持 8-bit YUV420 SDR 视频插帧。这种情况下，如果您打开了 HDR 或 YUV444 选项，中间帧将依然按 8-bit YUV420 SDR 格式生成。" + } + }, + "zh-Hant" : { + "stringUnit" : { + "state" : "translated", + "value" : "目前多數裝置僅支援 8-bit YUV420 SDR 影片影格插補。在這種情況下，如果您開啟 HDR 或 YUV444 選項，中間影格仍會以 8-bit YUV420 SDR 格式產生。" + } + } + } + }, "Interpolation failed for an unknown reason." : { "extractionState" : "manual", "localizations" : { @@ -17209,6 +17249,29 @@ } } }, + "Scaled Stream Resolution" : { + "extractionState" : "manual", + "localizations" : { + "zh-Hans" : { + "stringUnit" : { + "state" : "translated", + "value" : "串流分辨率缩放" + } + }, + "zh-Hant" : { + "stringUnit" : { + "state" : "translated", + "value" : "串流解析度縮放" + } + }, + "zh-HK" : { + "stringUnit" : { + "state" : "translated", + "value" : "串流解析度縮放" + } + } + } + }, "Slow connection to PC, Reduce your bitrate" : { "extractionState" : "manual", "localizations" : { diff --git a/VoidLink/Stream/FrameInterpolator.swift b/VoidLink/Stream/FrameInterpolator.swift index 37762080..0fb0d23d 100644 --- a/VoidLink/Stream/FrameInterpolator.swift +++ b/VoidLink/Stream/FrameInterpolator.swift @@ -11,10 +11,40 @@ import CoreVideo import ObjectiveC.runtime import VideoToolbox +@objcMembers +final class InterpolationResolutionConfiguration: NSObject { + let maximumDimension: Int + let maximumPixelCount: Int + let dimensions: CMVideoDimensions + + init( + maximumDimension: Int, + maximumPixelCount: Int, + dimensions: CMVideoDimensions + ) { + self.maximumDimension = maximumDimension + self.maximumPixelCount = maximumPixelCount + self.dimensions = dimensions + } +} + @objcMembers final class FrameInterpolator: NSObject { typealias Completion = (NSArray) -> Void + enum DimensionSelectionStrategy { + case maximumScale + case aspectRatioMatched + } + + @objc enum ResolutionTier:Int { + case system + case p480 + case p720 + case p1080 + case disabled + } + private struct PendingFrame { let frame: Frame let completion: Completion @@ -22,15 +52,25 @@ final class FrameInterpolator: NSObject { private let queue = DispatchQueue(label: "com.voidlink.FrameInterpolator", qos: .userInteractive) private var processor: AnyObject? + private var pixelTransferSession: AnyObject? + private var interpolationSourcePixelBufferPool: CVPixelBufferPool? private var pixelBufferPool: CVPixelBufferPool? private var formatDescription: CMVideoFormatDescription? - private var previousFrame: Frame? + private var previousInterpolationPixelBuffer: CVPixelBuffer? + private var previousInterpolationPTS = CMTime.invalid private var pendingFrames: [PendingFrame] = [] private var isProcessing = false + private var isPaused = false private var resetRequested = false + private var resetResolutionTierRequested = false + private var configuredResolutionTier = ResolutionTier.system + private var resolutionTier = ResolutionTier.system + private var configuredMaximumDimension = 0 + private var configuredMaximumPixelCount = 0 private var width = 0 private var height = 0 - private var slowUntil = CFTimeInterval(0) + private var sourceWidth = 0 + private var sourceHeight = 0 private var completedProcessingCount = 0 private var consecutiveSlowFrames = 0 private var loggedEvents = Set() @@ -44,6 +84,21 @@ final class FrameInterpolator: NSObject { } } + @objc(initWithResolutionTierRawValue:) + convenience init(resolutionTierRawValue: Int) { + self.init() + let tier = ResolutionTier(rawValue: resolutionTierRawValue) ?? .system + configuredResolutionTier = tier == .disabled ? .system : tier + resolutionTier = configuredResolutionTier + } + + @objc(initWithMaximumDimension:maximumPixelCount:) + convenience init(maximumDimension: Int, maximumPixelCount: Int) { + self.init() + configuredMaximumDimension = max(0, maximumDimension) + configuredMaximumPixelCount = max(0, maximumPixelCount) + } + @objc static let deviceSupportsInterpolation: Bool = { #if targetEnvironment(simulator) return false @@ -55,6 +110,72 @@ final class FrameInterpolator: NSObject { return false #endif }() + + private static let runtimeSupportedPixelFormats: [OSType] = { + #if targetEnvironment(simulator) + return [] + #else + guard + #available(iOS 26.0, tvOS 26.0, *), + VTLowLatencyFrameInterpolationConfiguration.isSupported, + let configuration = VTLowLatencyFrameInterpolationConfiguration( + frameWidth: 1280, + frameHeight: 720, + numberOfInterpolatedFrames: 1 + ) + else { + return [] + } + + return configuration.supportedPixelFormats + #endif + }() + + private static let pixelFormatTraits: [OSType: (bitDepth: Int, range: Int, chroma: Int)] = [ + kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange: (8, 0, 0), + kCVPixelFormatType_420YpCbCr8BiPlanarFullRange: (8, 1, 0), + kCVPixelFormatType_422YpCbCr8BiPlanarVideoRange: (8, 0, 1), + kCVPixelFormatType_422YpCbCr8BiPlanarFullRange: (8, 1, 1), + kCVPixelFormatType_444YpCbCr8BiPlanarVideoRange: (8, 0, 2), + kCVPixelFormatType_444YpCbCr8BiPlanarFullRange: (8, 1, 2), + kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange: (10, 0, 0), + kCVPixelFormatType_420YpCbCr10BiPlanarFullRange: (10, 1, 0), + kCVPixelFormatType_422YpCbCr10BiPlanarVideoRange: (10, 0, 1), + kCVPixelFormatType_422YpCbCr10BiPlanarFullRange: (10, 1, 1), + kCVPixelFormatType_444YpCbCr10BiPlanarVideoRange: (10, 0, 2), + kCVPixelFormatType_444YpCbCr10BiPlanarFullRange: (10, 1, 2), + ] + + @objc static let hdrSupported: Bool = { + runtimeSupportedPixelFormats.contains { + pixelFormatTraits[$0]?.bitDepth == 10 + } + }() + + @objc(supportedPixelFormatClosestTo:) + static func supportedPixelFormat(closestTo nativePixelFormat: OSType) -> OSType { + guard let firstSupportedFormat = runtimeSupportedPixelFormats.first else { + return 0 + } + if runtimeSupportedPixelFormats.contains(nativePixelFormat) { + return nativePixelFormat + } + guard let nativeTraits = pixelFormatTraits[nativePixelFormat] else { + return firstSupportedFormat + } + + return runtimeSupportedPixelFormats.min { lhs, rhs in + func distance(from format: OSType) -> Int { + guard let traits = pixelFormatTraits[format] else { + return Int.max + } + return abs(traits.bitDepth - nativeTraits.bitDepth) * 100 + + abs(traits.range - nativeTraits.range) * 10 + + abs(traits.chroma - nativeTraits.chroma) + } + return distance(from: lhs) < distance(from: rhs) + } ?? firstSupportedFormat + } static var dimensionLimitsAreKnown: Bool = { #if targetEnvironment(simulator) @@ -77,7 +198,48 @@ final class FrameInterpolator: NSObject { #endif }() + static func runtimeMaximumInterpolationDimensions() -> CMVideoDimensions { + #if targetEnvironment(simulator) + return CMVideoDimensions(width: 0, height: 0) + #else + guard #available(iOS 26.0, tvOS 26.0, *) else { + return CMVideoDimensions(width: 0, height: 0) + } + + let limits = VTLowLatencyFrameInterpolationConfiguration + .runtimeDimensionLimits(spatialScaleFactor: 1) + guard + let maximumDimension = limits.maximumDimension, + let maximumPixelCount = limits.maximumPixelCount, + maximumDimension > 0, + maximumPixelCount > 0 + else { + return CMVideoDimensions(width: 0, height: 0) + } + + return CMVideoDimensions( + width: Int32(maximumDimension), + height: Int32(maximumPixelCount / maximumDimension) + ) + #endif + } + + static func runtimeMaximumInterpolationPixelCount() -> Int { + #if targetEnvironment(simulator) + return 0 + #else + guard #available(iOS 26.0, tvOS 26.0, *) else { + return 0 + } + + return VTLowLatencyFrameInterpolationConfiguration + .runtimeDimensionLimits(spatialScaleFactor: 1) + .maximumPixelCount ?? 0 + #endif + } + static func interpolatableDimensionsBy(_ dimensions: CMVideoDimensions) -> CMVideoDimensions { + guard deviceSupportsInterpolation else { return CMVideoDimensions(width: 0, height: 0) } @@ -132,6 +294,139 @@ final class FrameInterpolator: NSObject { ) } + static func interpolatableDimensionsBy480p(_ dimensions: CMVideoDimensions) -> CMVideoDimensions { + guard deviceSupportsInterpolation else { + return CMVideoDimensions(width: 0, height: 0) + } + + return constrainedDimensions( + dimensions, + maximumDimension: 854, + maximumPixelCount: 854 * 480 + ) + } + + @objc(interpolatableDimensionsBy:maximumDimension:maximumPixelCount:) + static func interpolatableDimensionsBy( + _ dimensions: CMVideoDimensions, + maximumDimension: Int, + maximumPixelCount: Int + ) -> CMVideoDimensions { + constrainedDimensions( + dimensions, + maximumDimension: maximumDimension, + maximumPixelCount: maximumPixelCount + ) + } + + @objc(resolutionConfigurationForDimensions:level:) + static func resolutionConfiguration( + for dimensions: CMVideoDimensions, + level: Float + ) -> InterpolationResolutionConfiguration { + let sliderLimits = interpolationSliderLimits(for: dimensions) + let clampedLevel = min(1, max(0, Double(level))) + let maximumDimension = Int( + (Double(sliderLimits.minimumDimension) + + Double(sliderLimits.maximumDimension - sliderLimits.minimumDimension) * clampedLevel) + .rounded() + ) + + let maximumPixelCount: Int + if clampedLevel <= 0 { + maximumPixelCount = 854 * 480 + } else if clampedLevel >= 1 { + maximumPixelCount = sliderLimits.maximumPixelCount + } else { + maximumPixelCount = min( + Int((Double(maximumDimension * maximumDimension) / sliderLimits.aspectRatio).rounded()), + sliderLimits.maximumPixelCount + ) + } + + return InterpolationResolutionConfiguration( + maximumDimension: maximumDimension, + maximumPixelCount: maximumPixelCount, + dimensions: constrainedDimensions( + dimensions, + maximumDimension: maximumDimension, + maximumPixelCount: maximumPixelCount + ) + ) + } + + @objc(interpolationLevelForDimensions:savedMaximumDimension:) + static func interpolationLevel( + for dimensions: CMVideoDimensions, + savedMaximumDimension: Int + ) -> Float { + let limits = interpolationSliderLimits(for: dimensions) + guard savedMaximumDimension > 0, + limits.maximumDimension != limits.minimumDimension else { + return 1 + } + + return Float(min(1, max(0, + Double(savedMaximumDimension - limits.minimumDimension) / + Double(limits.maximumDimension - limits.minimumDimension) + ))) + } + + @objc(scaledStreamDimensionsWithPresetDimensions:interpolationDimensions:scale:) + static func scaledStreamDimensions( + presetDimensions: CMVideoDimensions, + interpolationDimensions: CMVideoDimensions, + scale: Float + ) -> CMVideoDimensions { + let clampedScale = min(1, max(0, Double(scale))) + let width = Int( + (Double(interpolationDimensions.width) + + Double(presetDimensions.width - interpolationDimensions.width) * clampedScale) + .rounded() + ) + let height = Int( + (Double(interpolationDimensions.height) + + Double(presetDimensions.height - interpolationDimensions.height) * clampedScale) + .rounded() + ) + + return videoDimensions( + width: max(4, width & ~3), + height: max(4, height & ~3) + ) + } + + private static func interpolationSliderLimits( + for dimensions: CMVideoDimensions + ) -> ( + minimumDimension: Int, + maximumDimension: Int, + maximumPixelCount: Int, + aspectRatio: Double + ) { + let systemDimensions = interpolatableDimensionsBy(dimensions) + let systemDimensionsAreValid = systemDimensions.width > 0 && systemDimensions.height > 0 + let maximumDimension = systemDimensionsAreValid + ? max(Int(systemDimensions.width), Int(systemDimensions.height)) + : 1280 + let maximumPixelCount = systemDimensionsAreValid + ? Int(systemDimensions.width) * Int(systemDimensions.height) + : 1280 * 720 + let aspectRatio = Double(maximumDimension * maximumDimension) / + Double(maximumPixelCount) + let minimumDimension = min( + Int((Double(854 * 480) * aspectRatio).squareRoot().rounded()), + maximumDimension + ) + + return ( + minimumDimension, + maximumDimension, + maximumPixelCount, + aspectRatio + ) + } + @objc(sourcePixelBufferAttributesForDimensions:) static func sourcePixelBufferAttributes(for dimensions: CMVideoDimensions) -> NSDictionary? { #if targetEnvironment(simulator) @@ -158,7 +453,8 @@ final class FrameInterpolator: NSObject { private static func constrainedDimensions( _ dimensions: CMVideoDimensions, maximumDimension: Int, - maximumPixelCount: Int + maximumPixelCount: Int, + strategy: DimensionSelectionStrategy = .aspectRatioMatched ) -> CMVideoDimensions { guard dimensions.width > 0, @@ -190,28 +486,142 @@ final class FrameInterpolator: NSObject { return dimensions } + let constrainedDimensions: CMVideoDimensions + switch strategy { + case .maximumScale: + constrainedDimensions = maximumScaleDimensions( + width: width, + height: height, + maximumDimension: maximumDimension, + maximumPixelCount: maximumPixelCount + ) + case .aspectRatioMatched: + constrainedDimensions = aspectRatioMatchedDimensions( + width: width, + height: height, + maximumDimension: maximumDimension, + maximumPixelCount: maximumPixelCount + ) + } + + guard + constrainedDimensions.width > 0, + constrainedDimensions.height > 0 + else { + return CMVideoDimensions(width: 0, height: 0) + } + + return constrainedDimensions + } + + private static func maximumScaleDimensions( + width: Int, + height: Int, + maximumDimension: Int, + maximumPixelCount: Int + ) -> CMVideoDimensions { let sourceWidth = Double(width) let sourceHeight = Double(height) let dimensionScale = Double(maximumDimension) / max(sourceWidth, sourceHeight) let pixelCountScale = sqrt(Double(maximumPixelCount) / (sourceWidth * sourceHeight)) let scale = min(dimensionScale, pixelCountScale) - - let scaledWidth = Int((sourceWidth * scale).rounded(.down)) & ~(31) - let scaledHeight = Int((sourceHeight * scale).rounded(.down)) & ~(31) + let interpolationAlignment = 16 + let scaledWidth = Int((sourceWidth * scale).rounded(.down)) & ~(interpolationAlignment - 1) + let scaledHeight = Int((sourceHeight * scale).rounded(.down)) & ~(interpolationAlignment - 1) + + return videoDimensions(width: scaledWidth, height: scaledHeight) + } + + private static func aspectRatioMatchedDimensions( + width: Int, + height: Int, + maximumDimension: Int, + maximumPixelCount: Int + ) -> CMVideoDimensions { + let interpolationAlignment = 16 + let sourceWidth = Double(width) + let sourceHeight = Double(height) + let sourceAspectRatio = sourceWidth / sourceHeight + let dimensionScale = Double(maximumDimension) / max(sourceWidth, sourceHeight) + let pixelCountScale = sqrt(Double(maximumPixelCount) / (sourceWidth * sourceHeight)) + let idealScale = min(1, dimensionScale, pixelCountScale) + let minimumCandidateArea = sourceWidth * idealScale * sourceHeight * idealScale * 0.9 + let maximumCandidateWidth = min(width, maximumDimension) & ~(interpolationAlignment - 1) + let maximumCandidateHeight = min(height, maximumDimension) + + var bestWidth = 0 + var bestHeight = 0 + var bestArea = 0 + var bestAspectRatioError = Double.greatestFiniteMagnitude + + for candidateWidth in stride( + from: interpolationAlignment, + through: maximumCandidateWidth, + by: interpolationAlignment + ) { + let exactHeight = Double(candidateWidth) / sourceAspectRatio + let lowerHeight = Int(exactHeight.rounded(.down)) & ~(interpolationAlignment - 1) + + for candidateHeight in [lowerHeight, lowerHeight + interpolationAlignment] { + guard + candidateHeight > 0, + candidateHeight <= maximumCandidateHeight + else { + continue + } + + let (candidateArea, areaOverflow) = candidateWidth.multipliedReportingOverflow( + by: candidateHeight + ) + guard + !areaOverflow, + candidateArea <= maximumPixelCount, + Double(candidateArea) >= minimumCandidateArea + else { + continue + } + + let candidateAspectRatio = Double(candidateWidth) / Double(candidateHeight) + let aspectRatioError = abs(candidateAspectRatio / sourceAspectRatio - 1) + let hasBetterAspectRatio = aspectRatioError < bestAspectRatioError + let hasEqualAspectRatio = abs(aspectRatioError - bestAspectRatioError) < 1e-12 + + if hasBetterAspectRatio || (hasEqualAspectRatio && candidateArea > bestArea) { + bestWidth = candidateWidth + bestHeight = candidateHeight + bestArea = candidateArea + bestAspectRatioError = aspectRatioError + } + } + } + + guard bestWidth > 0, bestHeight > 0 else { + return maximumScaleDimensions( + width: width, + height: height, + maximumDimension: maximumDimension, + maximumPixelCount: maximumPixelCount + ) + } + + return videoDimensions(width: bestWidth, height: bestHeight) + } + + private static func videoDimensions(width: Int, height: Int) -> CMVideoDimensions { guard - scaledWidth > 0, - scaledHeight > 0, - scaledWidth <= Int(Int32.max), - scaledHeight <= Int(Int32.max) + width > 0, + height > 0, + width <= Int(Int32.max), + height <= Int(Int32.max) else { return CMVideoDimensions(width: 0, height: 0) } - return CMVideoDimensions(width: Int32(scaledWidth), height: Int32(scaledHeight)) + return CMVideoDimensions(width: Int32(width), height: Int32(height)) } var isAvailable: Bool { - isEnabled && Self.deviceSupportsInterpolation + isEnabled && Self.deviceSupportsInterpolation && resolutionTier != .disabled } func reset() { @@ -220,6 +630,31 @@ final class FrameInterpolator: NSObject { } } + func setPaused(_ paused: Bool) { + queue.async { + guard self.isPaused != paused else { + return + } + + self.isPaused = paused + guard paused else { + return + } + + let pendingFrames = self.pendingFrames + self.pendingFrames.removeAll() + for pendingFrame in pendingFrames { + pendingFrame.completion([pendingFrame.frame]) + } + + if self.isProcessing { + self.resetRequested = true + } else { + self.resetLocked() + } + } + } + func processFrame(_ frame: Frame, completion: @escaping Completion) { queue.async { [weak self] in guard let self else { @@ -227,7 +662,7 @@ final class FrameInterpolator: NSObject { return } - guard !self.resetRequested else { + guard !self.isPaused, !self.resetRequested else { completion([frame]) return } @@ -245,11 +680,12 @@ final class FrameInterpolator: NSObject { let pendingFrame = pendingFrames.removeFirst() let frame = pendingFrame.frame let completion = pendingFrame.completion - guard isAvailable, CACurrentMediaTime() >= slowUntil else { + guard isAvailable else { if isEnabled && !Self.deviceSupportsInterpolation { logOnce("unavailable", "disabled: VT low-latency frame interpolation is unsupported on this OS/device") } - previousFrame = frame + previousInterpolationPixelBuffer = nil + previousInterpolationPTS = .invalid completion([frame]) drainPendingFrames() return @@ -257,15 +693,12 @@ final class FrameInterpolator: NSObject { if frame.frameType == FRAME_TYPE_IDR { resetLocked(keepPendingFrames: true) - previousFrame = frame - completion([frame]) - drainPendingFrames() - return } guard let currentPixelBuffer = frame.imageBuffer else { logOnce("missing-image-buffer", "input Frame has no CVPixelBuffer") - previousFrame = frame + previousInterpolationPixelBuffer = nil + previousInterpolationPTS = .invalid completion([frame]) drainPendingFrames() return @@ -276,33 +709,46 @@ final class FrameInterpolator: NSObject { let currentPTS = frame.pts90 guard CMTIME_IS_VALID(currentPTS) else { logOnce("invalid-pts", "input frame has an invalid PTS") - previousFrame = frame + previousInterpolationPixelBuffer = nil + previousInterpolationPTS = .invalid completion([frame]) drainPendingFrames() return } - guard prepareSessionIfNeeded(pixelBuffer: currentPixelBuffer) else { - previousFrame = frame + let startedAt = CACurrentMediaTime() + guard + prepareSessionIfNeeded(pixelBuffer: currentPixelBuffer), + let currentInterpolationPixelBuffer = makeInterpolationSourcePixelBuffer( + from: currentPixelBuffer + ) + else { + previousInterpolationPixelBuffer = nil + previousInterpolationPTS = .invalid completion([frame]) drainPendingFrames() return } - guard let previousFrame, let previousPixelBuffer = previousFrame.imageBuffer else { - previousFrame = frame + guard + let previousPixelBuffer = previousInterpolationPixelBuffer, + CMTIME_IS_VALID(previousInterpolationPTS) + else { + previousInterpolationPixelBuffer = currentInterpolationPixelBuffer + previousInterpolationPTS = currentPTS completion([frame]) drainPendingFrames() return } - let previousPTS = previousFrame.pts90 + let previousPTS = previousInterpolationPTS guard CMTIME_IS_VALID(previousPTS), CMTimeCompare(currentPTS, previousPTS) > 0 else { logOnce( "non-monotonic-pts", "upstream PTS is not increasing: previous \(CMTimeGetSeconds(previousPTS)), current \(CMTimeGetSeconds(currentPTS))" ) - self.previousFrame = frame + previousInterpolationPixelBuffer = currentInterpolationPixelBuffer + previousInterpolationPTS = currentPTS completion([frame]) drainPendingFrames() return @@ -310,13 +756,15 @@ final class FrameInterpolator: NSObject { let sourceFrameInterval = CMTimeGetSeconds(CMTimeSubtract(currentPTS, previousPTS)) #if targetEnvironment(simulator) - self.previousFrame = frame + previousInterpolationPixelBuffer = currentInterpolationPixelBuffer + previousInterpolationPTS = currentPTS completion([frame]) drainPendingFrames() return #else guard #available(iOS 26.0, tvOS 26.0, *) else { - self.previousFrame = frame + previousInterpolationPixelBuffer = currentInterpolationPixelBuffer + previousInterpolationPTS = currentPTS completion([frame]) drainPendingFrames() return @@ -324,7 +772,7 @@ final class FrameInterpolator: NSObject { guard let processor = processor as? VTFrameProcessor, - let sourceProcessorFrame = VTFrameProcessorFrame(buffer: currentPixelBuffer, presentationTimeStamp: currentPTS), + let sourceProcessorFrame = VTFrameProcessorFrame(buffer: currentInterpolationPixelBuffer, presentationTimeStamp: currentPTS), let previousProcessorFrame = VTFrameProcessorFrame(buffer: previousPixelBuffer, presentationTimeStamp: previousPTS), let destinationPixelBuffer = makeDestinationPixelBuffer(), let destinationProcessorFrame = VTFrameProcessorFrame( @@ -340,27 +788,35 @@ final class FrameInterpolator: NSObject { else { logOnce( "parameters-failed", - "could not create interpolation parameters; input pixel format is \(pixelFormatName(CVPixelBufferGetPixelFormatType(currentPixelBuffer)))" + "could not create interpolation parameters; input pixel format is \(pixelFormatName(CVPixelBufferGetPixelFormatType(currentInterpolationPixelBuffer)))" ) - self.previousFrame = frame + previousInterpolationPixelBuffer = currentInterpolationPixelBuffer + previousInterpolationPTS = currentPTS completion([frame]) drainPendingFrames() return } + CVBufferPropagateAttachments( + currentInterpolationPixelBuffer, + destinationPixelBuffer + ) + isProcessing = true - let startedAt = CACurrentMediaTime() processor.process(parameters: parameters) { [self] _, error in self.queue.async { self.isProcessing = false if self.resetRequested { - self.resetLocked() + self.resetLocked( + resetResolutionTier: self.resetResolutionTierRequested + ) completion([frame]) return } - self.previousFrame = frame + self.previousInterpolationPixelBuffer = currentInterpolationPixelBuffer + self.previousInterpolationPTS = currentPTS let elapsed = CACurrentMediaTime() - startedAt guard @@ -396,14 +852,8 @@ final class FrameInterpolator: NSObject { } else { self.consecutiveSlowFrames = 0 } - if self.completedProcessingCount >= 10, self.consecutiveSlowFrames >= 3 { - self.slowUntil = CACurrentMediaTime() + 1.0 - self.consecutiveSlowFrames = 0 - self.logOnce( - "sustained-overrun", - "processing repeatedly exceeded the source-frame interval; temporarily falling back to original frames", - overlayText: "Interpolation overhead is too high. Falling back to normal mode.".localized - ) + if self.completedProcessingCount >= 10, self.consecutiveSlowFrames >= 30 { + // self.handleSustainedOverrun() // disable degradation for now } // NSLog("[FrameInterpolator] interpolated frame in %.2f ms",) @@ -418,9 +868,11 @@ final class FrameInterpolator: NSObject { #if targetEnvironment(simulator) return false #else - let width = CVPixelBufferGetWidth(pixelBuffer) - let height = CVPixelBufferGetHeight(pixelBuffer) - if processor != nil, self.width == width, self.height == height { + let sourceWidth = CVPixelBufferGetWidth(pixelBuffer) + let sourceHeight = CVPixelBufferGetHeight(pixelBuffer) + if processor != nil, + self.sourceWidth == sourceWidth, + self.sourceHeight == sourceHeight { return true } @@ -430,7 +882,17 @@ final class FrameInterpolator: NSObject { return false } + let sourceDimensions = CMVideoDimensions( + width: Int32(sourceWidth), + height: Int32(sourceHeight) + ) + let interpolationDimensions = interpolationDimensions(for: sourceDimensions) + let width = Int(interpolationDimensions.width) + let height = Int(interpolationDimensions.height) + guard + width > 0, + height > 0, VTLowLatencyFrameInterpolationConfiguration.isSupported, let configuration = VTLowLatencyFrameInterpolationConfiguration( frameWidth: width, @@ -455,8 +917,8 @@ final class FrameInterpolator: NSObject { &extendedBottom ) logOnce( - "input-layout-\(width)x\(height)", - "input pixel buffer layout for \(width)x\(height): " + + "input-layout-\(sourceWidth)x\(sourceHeight)", + "input pixel buffer layout for \(sourceWidth)x\(sourceHeight): " + "extended left=\(extendedLeft), right=\(extendedRight), " + "top=\(extendedTop), bottom=\(extendedBottom)" ) @@ -477,6 +939,7 @@ final class FrameInterpolator: NSObject { "destination-attributes-\(width)x\(height)", "destination pixel buffer attributes for \(width)x\(height): \(configuration.destinationPixelBufferAttributes)" ) + print("configuration.supportedPixelFormats \(configuration.supportedPixelFormats)") if !configuration.supportedPixelFormats.contains(inputPixelFormat) { let supportedFormats = configuration.supportedPixelFormats .map(pixelFormatName) @@ -488,6 +951,44 @@ final class FrameInterpolator: NSObject { return false } + guard let sourcePool = makeInterpolationSourcePixelBufferPool( + configuration: configuration, + pixelFormat: inputPixelFormat, + width: width, + height: height + ) else { + logOnce("source-pool-error", "failed to create the interpolation source pixel buffer pool") + return false + } + + var transferSession: VTPixelTransferSession? + let transferStatus = VTPixelTransferSessionCreate( + allocator: kCFAllocatorDefault, + pixelTransferSessionOut: &transferSession + ) + guard transferStatus == noErr, let transferSession else { + logOnce( + "pixel-transfer-session-error-\(transferStatus)", + "failed to create pixel transfer session: \(transferStatus)" + ) + return false + } + VTSessionSetProperty( + transferSession, + key: kVTPixelTransferPropertyKey_ScalingMode, + value: kVTScalingMode_Normal + ) + VTSessionSetProperty( + transferSession, + key: kVTPixelTransferPropertyKey_RealTime, + value: kCFBooleanTrue + ) + VTSessionSetProperty( + transferSession, + key: kVTPixelTransferPropertyKey_DownsamplingMode, + value: kVTDownsamplingMode_Average + ) + let processor = VTFrameProcessor() do { try processor.startSession(configuration: configuration) @@ -509,18 +1010,162 @@ final class FrameInterpolator: NSObject { return false } + self.sourceWidth = sourceWidth + self.sourceHeight = sourceHeight self.width = width self.height = height self.processor = processor + self.pixelTransferSession = transferSession + self.interpolationSourcePixelBufferPool = sourcePool self.pixelBufferPool = pool logOnce( "session-started-\(width)x\(height)", - "session started for \(width)x\(height), input pixel format \(pixelFormatName(inputPixelFormat))" + "session started for \(width)x\(height) from \(sourceWidth)x\(sourceHeight), input pixel format \(pixelFormatName(inputPixelFormat))" ) return true #endif } + private func interpolationDimensions( + for sourceDimensions: CMVideoDimensions + ) -> CMVideoDimensions { + switch resolutionTier { + case .system: + if configuredMaximumDimension > 0, configuredMaximumPixelCount > 0 { + return Self.constrainedDimensions( + sourceDimensions, + maximumDimension: configuredMaximumDimension, + maximumPixelCount: configuredMaximumPixelCount + ) + } + let dimensions = Self.interpolatableDimensionsBy(sourceDimensions) + if dimensions.width > 0, dimensions.height > 0 { + return dimensions + } + return Self.interpolatableDimensionsBy720p(sourceDimensions) + case .p1080: + return Self.interpolatableDimensionsBy1080p(sourceDimensions) + case .p720: + return Self.interpolatableDimensionsBy720p(sourceDimensions) + case .p480: + return Self.interpolatableDimensionsBy480p(sourceDimensions) + case .disabled: + return CMVideoDimensions(width: 0, height: 0) + } + } + + private func handleSustainedOverrun() { + switch resolutionTier { + case .system: + resolutionTier = .p720 + logOnce( + "sustained-overrun-720p", + "processing repeatedly exceeded the source-frame interval; reducing interpolation resolution to the 720p limit", + overlayText: "Interpolation overhead is too high. Reducing interpolation resolution to 720p.".localized + ) + case .p1080: + resolutionTier = .p720 + logOnce( + "sustained-overrun-1080p", + "processing repeatedly exceeded the source-frame interval at the 1080p limit; reducing interpolation resolution to the 720p limit", + overlayText: "Interpolation overhead is too high. Reducing interpolation resolution to 480p.".localized + ) + case .p720: + resolutionTier = .p480 + logOnce( + "sustained-overrun-480p", + "processing repeatedly exceeded the source-frame interval at the 720p limit; reducing interpolation resolution to the 480p limit", + overlayText: "Interpolation overhead is too high. Reducing interpolation resolution to 480p.".localized + ) + case .p480: + resolutionTier = .disabled + logOnce( + "sustained-overrun-disabled", + "processing repeatedly exceeded the source-frame interval at the 480p limit; falling back to original frames", + overlayText: "Interpolation overhead is too high. Falling back to normal mode.".localized + ) + case .disabled: + return + } + + resetLocked(keepPendingFrames: true) + } + + #if !targetEnvironment(simulator) + @available(iOS 26.0, tvOS 26.0, *) + private func makeInterpolationSourcePixelBufferPool( + configuration: VTLowLatencyFrameInterpolationConfiguration, + pixelFormat: OSType, + width: Int, + height: Int + ) -> CVPixelBufferPool? { + let requestedAttributes: [CFString: Any] = [ + kCVPixelBufferPixelFormatTypeKey: pixelFormat, + kCVPixelBufferWidthKey: width, + kCVPixelBufferHeightKey: height, + kCVPixelBufferIOSurfacePropertiesKey: [:] + ] + var resolvedAttributes: CFDictionary? + let resolveStatus = CVPixelBufferCreateResolvedAttributesDictionary( + kCFAllocatorDefault, + [requestedAttributes as CFDictionary, + configuration.sourcePixelBufferAttributes as CFDictionary] as CFArray, + &resolvedAttributes + ) + guard resolveStatus == kCVReturnSuccess, let resolvedAttributes else { + return nil + } + + var pool: CVPixelBufferPool? + let poolStatus = CVPixelBufferPoolCreate( + kCFAllocatorDefault, + nil, + resolvedAttributes, + &pool + ) + return poolStatus == kCVReturnSuccess ? pool : nil + } + #endif + + private func makeInterpolationSourcePixelBuffer( + from sourcePixelBuffer: CVPixelBuffer + ) -> CVPixelBuffer? { + #if targetEnvironment(simulator) + return nil + #else + guard + #available(iOS 16.0, tvOS 16.0, *), + let pixelTransferSession, + let interpolationSourcePixelBufferPool + else { + return nil + } + let transferSession = pixelTransferSession as! VTPixelTransferSession + + var destinationPixelBuffer: CVPixelBuffer? + let createStatus = CVPixelBufferPoolCreatePixelBuffer( + kCFAllocatorDefault, + interpolationSourcePixelBufferPool, + &destinationPixelBuffer + ) + guard createStatus == kCVReturnSuccess, let destinationPixelBuffer else { + logOnce("downsize-buffer-error-\(createStatus)", "failed to allocate downsize buffer: \(createStatus)") + return nil + } + + let transferStatus = VTPixelTransferSessionTransferImage( + transferSession, + from: sourcePixelBuffer, + to: destinationPixelBuffer + ) + guard transferStatus == noErr else { + logOnce("downsize-error-\(transferStatus)", "failed to downsize frame: \(transferStatus)") + return nil + } + return destinationPixelBuffer + #endif + } + private func makeDestinationPixelBuffer() -> CVPixelBuffer? { guard let pixelBufferPool else { return nil @@ -588,8 +1233,8 @@ final class FrameInterpolator: NSObject { } if let overlayText { showTransientHUDText(overlayText) - NSLog("[FrameInterpolator] %@", message) } + NSLog("[FrameInterpolator] %@", message) } private func logOnce(_ event: String, error: Error) { @@ -644,13 +1289,17 @@ final class FrameInterpolator: NSObject { pendingFrames.removeAll() guard !isProcessing else { resetRequested = true + resetResolutionTierRequested = true return } - resetLocked() + resetLocked(resetResolutionTier: true) } - private func resetLocked(keepPendingFrames: Bool = false) { + private func resetLocked( + keepPendingFrames: Bool = false, + resetResolutionTier: Bool = false + ) { #if !targetEnvironment(simulator) if #available(iOS 26.0, tvOS 26.0, *), let processor = processor as? VTFrameProcessor { @@ -659,16 +1308,30 @@ final class FrameInterpolator: NSObject { #endif processor = nil + #if !targetEnvironment(simulator) + if #available(iOS 16.0, tvOS 16.0, *), + let pixelTransferSession { + VTPixelTransferSessionInvalidate(pixelTransferSession as! VTPixelTransferSession) + } + #endif + pixelTransferSession = nil + interpolationSourcePixelBufferPool = nil pixelBufferPool = nil formatDescription = nil - previousFrame = nil + previousInterpolationPixelBuffer = nil + previousInterpolationPTS = .invalid isProcessing = false resetRequested = false + resetResolutionTierRequested = false width = 0 height = 0 - slowUntil = 0 + sourceWidth = 0 + sourceHeight = 0 completedProcessingCount = 0 consecutiveSlowFrames = 0 + if resetResolutionTier { + resolutionTier = configuredResolutionTier + } if !keepPendingFrames { pendingFrames.removeAll() } diff --git a/VoidLink/Stream/VideoDecoderRenderer.h b/VoidLink/Stream/VideoDecoderRenderer.h index 920ec247..c6f55f96 100644 --- a/VoidLink/Stream/VideoDecoderRenderer.h +++ b/VoidLink/Stream/VideoDecoderRenderer.h @@ -49,5 +49,7 @@ - (void)invalidateDecompressionSession; + (void)setFrameInterpolationEnabled:(bool)enabled; ++ (void)startOrRestartFrameInterpolation; ++ (void)stopFrameInterpolation; @end diff --git a/VoidLink/Stream/VideoDecoderRenderer.m b/VoidLink/Stream/VideoDecoderRenderer.m index c812a425..9c7e9099 100644 --- a/VoidLink/Stream/VideoDecoderRenderer.m +++ b/VoidLink/Stream/VideoDecoderRenderer.m @@ -36,11 +36,19 @@ //#define DISPLAYLINK_VERBOSE static BOOL kEnableFrameInterpolation = false; +static __weak VideoDecoderRenderer *sActiveRenderer = nil; // Private libavformat API for writing the AV1 Codec Configuration Box extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size, int write_seq_header); +@interface VideoDecoderRenderer () +- (void)startOrRestartFrameInterpolation; +- (void)stopFrameInterpolation; +- (NSInteger)displayLinkFrameRateForInterpolationEnabled:(BOOL)enabled; +- (void)restartDisplayLinkForInterpolationEnabled:(BOOL)enabled; +@end + @implementation VideoDecoderRenderer { dispatch_queue_t _sq, _vtq; StreamView* _view; @@ -60,6 +68,7 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size, CMVideoFormatDescriptionRef _formatDescImageBuffer; VTDecompressionSessionRef _decompressionSession; FrameInterpolator *_frameInterpolator; + BOOL _frameInterpolationPaused; CADisplayLink *_displayLink; NSInteger _maxRefreshRate; @@ -77,6 +86,26 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size, kEnableFrameInterpolation = enabled; } ++ (void)startOrRestartFrameInterpolation { + kEnableFrameInterpolation = true; + + VideoDecoderRenderer *renderer; + @synchronized(self) { + renderer = sActiveRenderer; + } + [renderer startOrRestartFrameInterpolation]; +} + ++ (void)stopFrameInterpolation { + kEnableFrameInterpolation = false; + + VideoDecoderRenderer *renderer; + @synchronized(self) { + renderer = sActiveRenderer; + } + [renderer stopFrameInterpolation]; +} + - (void)reinitializeDisplayLayer { if (_displayLayer == nil) { @@ -174,7 +203,7 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size, _asyncFrameDequeue = tempSettings.asyncFrameDequeue; NSLog(@"_asyncFrameDequeue %d", _asyncFrameDequeue); _enableTimebase = false; - _queueSize = kEnableFrameInterpolation ? 4 : tempSettings.frameQueueSize.intValue; + _queueSize = kEnableFrameInterpolation ? 8 : tempSettings.frameQueueSize.intValue; _needRequeuing = _queueSize>0; _frameQueue = [FrameQueue sharedInstance]; @@ -189,9 +218,115 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size, name:@"ScreenChanged" object:nil]; + @synchronized([VideoDecoderRenderer class]) { + sActiveRenderer = self; + } + return self; } +- (NSInteger)displayLinkFrameRateForInterpolationEnabled:(BOOL)enabled { + NSInteger multiplier = enabled ? 2 : 1; + return MIN(self->_frameRate * multiplier, self->_maxRefreshRate); +} + +- (void)restartDisplayLinkForInterpolationEnabled:(BOOL)enabled { + dispatch_block_t restartBlock = ^{ + if (self->_renderingBackend != RENDER_AVSB || self->_displayLink == nil) { + return; + } + + NSInteger targetFrameRate = [self displayLinkFrameRateForInterpolationEnabled:enabled]; + BOOL shouldRemainPaused = appDidEnterBackgroundWithoutPip; + self->_displayLink.paused = YES; + if (@available(iOS 15.0, tvOS 15.0, *)) { + self->_displayLink.preferredFrameRateRange = CAFrameRateRangeMake(targetFrameRate, targetFrameRate, targetFrameRate); + } + else { + self->_displayLink.preferredFramesPerSecond = targetFrameRate; + } + self->_displayLink.paused = shouldRemainPaused; + Log(LOG_I, @"Display link restarted at %ld FPS (interpolation %@)", + (long)targetFrameRate, enabled ? @"enabled" : @"disabled"); + }; + + if ([NSThread isMainThread]) { + restartBlock(); + } + else { + dispatch_async(dispatch_get_main_queue(), restartBlock); + } +} + +- (void)invalidateDecompressionSession { + @synchronized(self) { + if (self->_decompressionSession != NULL) { + VTDecompressionSessionInvalidate(self->_decompressionSession); + CFRelease(self->_decompressionSession); + self->_decompressionSession = nil; + } + } +} + +- (void)startOrRestartFrameInterpolation { + if (self->_displayLink == nil || self->_renderingBackend != RENDER_AVSB) { + return; + } + + DataManager *dataMan = [[DataManager alloc] init]; + TemporarySettings *settings = [dataMan getSettings]; + NSInteger maximumDimension = settings.interpolationMaximumDimension.integerValue; + NSInteger maximumPixelCount = settings.interpolationMaximumPixelCount.integerValue; + + dispatch_async(self->_vtq, ^{ + FrameInterpolator *oldInterpolator = self->_frameInterpolator; + self->_frameInterpolator = [[FrameInterpolator alloc] + initWithMaximumDimension:maximumDimension + maximumPixelCount:maximumPixelCount]; + self->_frameInterpolator.isEnabled = YES; + self->_frameInterpolationPaused = NO; + [oldInterpolator reset]; + + self->_queueSize = 8; + self->_needRequeuing = YES; + [self->_frameQueue setHighWaterMark:self->_queueSize]; + [self->_frameQueue clear]; + [self invalidateDecompressionSession]; + [self restartDisplayLinkForInterpolationEnabled:YES]; + LiRequestIdrFrame(); + + Log(LOG_I, @"Frame interpolation started or restarted with limits %ld / %ld pixels", + (long)maximumDimension, (long)maximumPixelCount); + }); +} + +- (void)stopFrameInterpolation { + if (self->_displayLink == nil || self->_renderingBackend != RENDER_AVSB) { + return; + } + + DataManager *dataMan = [[DataManager alloc] init]; + TemporarySettings *settings = [dataMan getSettings]; + NSInteger normalQueueSize = settings.frameQueueSize.integerValue; + + dispatch_async(self->_vtq, ^{ + FrameInterpolator *oldInterpolator = self->_frameInterpolator; + self->_frameInterpolator = nil; + self->_frameInterpolationPaused = NO; + [oldInterpolator reset]; + + self->_queueSize = (int32_t)normalQueueSize; + self->_needRequeuing = self->_queueSize > 0; + [self->_frameQueue setHighWaterMark:MAX(1, self->_queueSize)]; + [self->_frameQueue clear]; + [self invalidateDecompressionSession]; + [self restartDisplayLinkForInterpolationEnabled:NO]; + LiRequestIdrFrame(); + + Log(LOG_I, @"Frame interpolation stopped"); + }); +} + # pragma mark DisplayLink vsync callback - (void)setupWithVideoFormat:(int)videoFormat width:(int)videoWidth height:(int)videoHeight frameRate:(int)frameRate fullRange:(BOOL)fullRange request10BitCodec:(BOOL)request10BitCodec @@ -210,7 +345,9 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size, _renderingBackend = RENDER_AVSB; if (kEnableFrameInterpolation) { - _frameInterpolator = [[FrameInterpolator alloc] init]; + _frameInterpolator = [[FrameInterpolator alloc] + initWithMaximumDimension:settings.interpolationMaximumDimension.integerValue + maximumPixelCount:settings.interpolationMaximumPixelCount.integerValue]; _frameInterpolator.isEnabled = YES; } @@ -229,10 +366,7 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size, _displayLink = [CADisplayLink displayLinkWithTarget:self selector:displayLinkSelector]; - NSInteger targetFrameRate = self->_frameRate; - if (self->_frameInterpolator != nil) { - targetFrameRate = MIN(self->_frameRate * 2, self->_maxRefreshRate); - } + NSInteger targetFrameRate = [self displayLinkFrameRateForInterpolationEnabled:self->_frameInterpolator != nil]; if (@available(iOS 15.0, tvOS 15.0, *)) { _displayLink.preferredFrameRateRange = CAFrameRateRangeMake(targetFrameRate, targetFrameRate, targetFrameRate); @@ -289,61 +423,26 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size, nativePixelFormat = self->_fullRange ? frFormat : vrFormat; } BOOL interpolationRequested = self->_frameInterpolator != nil; - NSNumber *pixelFormat = interpolationRequested ? @(kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange) : nativePixelFormat; + OSType interpolationPixelFormat = 0; if (interpolationRequested) { - LogOnce(LOG_I, @"Frame interpolation enabled; requesting 420v output from VTDecompressionSession"); + interpolationPixelFormat = [FrameInterpolator supportedPixelFormatClosestTo:nativePixelFormat.unsignedIntValue]; + if (interpolationPixelFormat == 0) { + LogOnce(LOG_W, @"Frame interpolation disabled because the device reports no supported pixel format"); + [self->_frameInterpolator reset]; + self->_frameInterpolator = nil; + interpolationRequested = NO; + [self restartDisplayLinkForInterpolationEnabled:NO]; + } + } + NSNumber *pixelFormat = interpolationRequested ? @(interpolationPixelFormat) : nativePixelFormat; + if (interpolationRequested) { + LogOnce(LOG_I, @"Frame interpolation enabled; requesting pixel format %@ from VTDecompressionSession", pixelFormat); } NSMutableDictionary *destinationPixelBufferAttributes = [@{ (id)kCVPixelBufferPixelFormatTypeKey : pixelFormat } mutableCopy]; -#endif - -#if !TARGET_OS_SIMULATOR if (interpolationRequested) { - CMVideoDimensions dimensions = CMVideoFormatDescriptionGetDimensions(_formatDesc); - NSDictionary *interpolationSourceAttributes = - [FrameInterpolator sourcePixelBufferAttributesForDimensions:dimensions]; - CFDictionaryRef resolvedAttributes = NULL; - CVReturn resolveStatus = kCVReturnInvalidArgument; - - if (interpolationSourceAttributes != nil) { - NSArray *attributeRequirements = @[ - destinationPixelBufferAttributes, - interpolationSourceAttributes - ]; - resolveStatus = CVPixelBufferCreateResolvedAttributesDictionary( - kCFAllocatorDefault, - (__bridge CFArrayRef)attributeRequirements, - &resolvedAttributes); - } - - if (resolveStatus == kCVReturnSuccess && resolvedAttributes != NULL) { - destinationPixelBufferAttributes = - [(__bridge NSDictionary *)resolvedAttributes mutableCopy]; - CFRelease(resolvedAttributes); - } else { - if (resolvedAttributes != NULL) { - CFRelease(resolvedAttributes); - } - Log(LOG_W, - @"Unable to resolve frame-interpolation source attributes for %dx%d (status %d); using native decode output", - dimensions.width, - dimensions.height, - resolveStatus); - [self->_frameInterpolator reset]; - self->_frameInterpolator = nil; - interpolationRequested = NO; - destinationPixelBufferAttributes = [@{ - (id)kCVPixelBufferPixelFormatTypeKey : nativePixelFormat - } mutableCopy]; - - NSInteger targetFrameRate = MIN(self->_frameRate, self->_maxRefreshRate); - if (@available(iOS 15.0, tvOS 15.0, *)) { - self->_displayLink.preferredFrameRateRange = CAFrameRateRangeMake(targetFrameRate, targetFrameRate, targetFrameRate); - } else { - self->_displayLink.preferredFramesPerSecond = targetFrameRate; - } - } + destinationPixelBufferAttributes[(id)kCVPixelBufferIOSurfacePropertiesKey] = @{}; } #endif @@ -360,7 +459,7 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size, OSStatus status = [self setupDecompressionSessionWithAttributes:destinationPixelBufferAttributes]; if (status != noErr) { - Log(LOG_W, @"Falling back from 420v frame-interpolation decode output to native pixel format %@", nativePixelFormat); + Log(LOG_W, @"Falling back from frame-interpolation decode output %@ to native pixel format %@", pixelFormat, nativePixelFormat); [self->_frameInterpolator reset]; self->_frameInterpolator = nil; destinationPixelBufferAttributes = [@{ @@ -372,12 +471,7 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size, } [self setupDecompressionSessionWithAttributes:destinationPixelBufferAttributes]; - NSInteger targetFrameRate = MIN(self->_frameRate, self->_maxRefreshRate); - if (@available(iOS 15.0, tvOS 15.0, *)) { - self->_displayLink.preferredFrameRateRange = CAFrameRateRangeMake(targetFrameRate, targetFrameRate, targetFrameRate); - } else { - self->_displayLink.preferredFramesPerSecond = targetFrameRate; - } + [self restartDisplayLinkForInterpolationEnabled:NO]; } #else [self setupDecompressionSessionWithAttributes:destinationPixelBufferAttributes]; @@ -574,6 +668,12 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit); } - (void)cleanup{ + @synchronized([VideoDecoderRenderer class]) { + if (sActiveRenderer == self) { + sActiveRenderer = nil; + } + } + dispatch_async(dispatch_get_main_queue(), ^{ [self->_frameQueue stop]; @@ -1187,9 +1287,24 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit); [frame setFormatDesc:self->_formatDesc]; } - if (self->_frameInterpolator != nil) { - [self->_frameInterpolator processFrame:frame completion:^(NSArray *frames) { + FrameInterpolator *interpolator = self->_frameInterpolator; + if (interpolator != nil && appDidEnterBackgroundWithoutPip) { + if (!self->_frameInterpolationPaused) { + self->_frameInterpolationPaused = YES; + [interpolator setPaused:YES]; + } + } + else if (interpolator != nil) { + if (self->_frameInterpolationPaused) { + self->_frameInterpolationPaused = NO; + [interpolator setPaused:NO]; + } + [interpolator processFrame:frame completion:^(NSArray *frames) { dispatch_async(self->_vtq, ^{ + if (self->_frameInterpolator != interpolator) { + return; + } + int framesDropped = 0; for (Frame *outputFrame in (NSArray *)frames) { framesDropped += [self->_frameQueue enqueue:outputFrame withSlackSize:3]; @@ -1403,22 +1518,7 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit); - (void)resetFramePacing { // Ensure this only runs for the AVSampleBuffer rendering backend and that the display link exists. if (_renderingBackend == RENDER_AVSB && _displayLink) { - Log(LOG_I, @"Frame pacing is being reset to %d FPS...", self->_frameRate); - - // Toggling the paused state can help re-engage the display link with the - // run loop correctly after the app resumes from a background state like PiP. - _displayLink.paused = YES; - - // Re-apply the desired frame rate range. This is the critical hint for ProMotion - // that may have been lost or ignored during the PiP transition. - if (@available(iOS 15.0, *)) { - _displayLink.preferredFrameRateRange = CAFrameRateRangeMake(self->_frameRate, self->_frameRate, self->_frameRate); - } else { - _displayLink.preferredFramesPerSecond = self->_frameRate; - } - - // Resume the display link immediately. - _displayLink.paused = NO; + [self restartDisplayLinkForInterpolationEnabled:self->_frameInterpolator.isEnabled]; } else if (_renderingBackend == RENDER_METAL) { @synchronized(self) { if (_decompressionSession != nil) { diff --git a/VoidLink/Utility/GenericUtils.swift b/VoidLink/Utility/GenericUtils.swift index 3160f760..2029873b 100644 --- a/VoidLink/Utility/GenericUtils.swift +++ b/VoidLink/Utility/GenericUtils.swift @@ -55,6 +55,23 @@ import UIKit completion: {}) } } + + @objc public static func handleFrameInterpolationPixelFormatTip(in vc: UIViewController?) { + let key = "hasShownFrameInterpolationPixelFormatTip" + guard !UserDefaults.standard.bool(forKey: key) else { + return + } + UserDefaults.standard.set(true, forKey: key) + + AlertControllerUtil.showAlert( + in: vc, + title: LocalizationHelper.localizedString(forKey: "Tips"), + message: "\n\(LocalizationHelper.localizedString(forKey: "Most devices currently support frame interpolation only for 8-bit YUV 4:2:0 SDR video. In this case, if HDR or YUV 4:4:4 is enabled, intermediate frames will still be generated in 8-bit YUV 4:2:0 SDR."))", + withCancel: false, + buttonTitle: LocalizationHelper.localizedString(forKey: "Got it!"), + countdown: 5 + ) + } @objc public static func needUpdateDefaultSettings() -> Bool { // let key = "needUpdateDefaultSettings20260226-1" diff --git a/VoidLink/ViewControllers/MainFrameViewController.m b/VoidLink/ViewControllers/MainFrameViewController.m index 412a91c3..99daca9c 100644 --- a/VoidLink/ViewControllers/MainFrameViewController.m +++ b/VoidLink/ViewControllers/MainFrameViewController.m @@ -852,8 +852,8 @@ static NSMutableSet* hostList; } } - _streamConfig.height = [streamSettings.height intValue]; - _streamConfig.width = [streamSettings.width intValue]; + _streamConfig.height = streamSettings.height.intValue; + _streamConfig.width = streamSettings.width.intValue; NSLog(@"saveSettings.width %d, %d", _streamConfig.width, _streamConfig.height); #if TARGET_OS_TV @@ -1399,11 +1399,16 @@ static NSMutableSet* hostList; [self.settingsViewController setHidden:_settingsExpandedInStreamView forStack:self.settingsViewController.appThemeStack]; [self.settingsViewController.renderingBackendSelector setEnabled:!_settingsExpandedInStreamView]; // Enable frame pacing mode selector only if not in stream view AND not in performance mode - BOOL shouldEnableFramePacingSelector = !_settingsExpandedInStreamView && (self.settingsViewController.renderingBackendSelector.selectedSegmentIndex != RENDER_METAL); - [self.settingsViewController.framePacingModeSelector setEnabled:shouldEnableFramePacingSelector]; + BOOL shouldEnableFrameQueueSettings = !_settingsExpandedInStreamView && (self.settingsViewController.renderingBackendSelector.selectedSegmentIndex != RENDER_METAL); + // [self.settingsViewController.framePacingModeSelector setEnabled:shouldEnableFrameQueueSettings]; + [self.settingsViewController.framePacingModeSelector setEnabled:!_settingsExpandedInStreamView forSegmentAtIndex:0]; + [self.settingsViewController.framePacingModeSelector setEnabled:!_settingsExpandedInStreamView forSegmentAtIndex:1]; + // [self.settingsViewController.frameTimebaseSwitch setEnabled:shouldEnableFramePacing]; - [self.settingsViewController.asyncFrameDequeueSwitch setEnabled:shouldEnableFramePacingSelector]; - [self.settingsViewController setHidden:_settingsExpandedInStreamView || !(shouldEnableFramePacingSelector && self.settingsViewController.framePacingModeSelector.selectedSegmentIndex == FramePacingModeQueue) forStack:self.settingsViewController.frameQueueSizeStack]; + [self.settingsViewController.asyncFrameDequeueSwitch setEnabled:shouldEnableFrameQueueSettings]; + [self.settingsViewController setHidden:_settingsExpandedInStreamView || !(shouldEnableFrameQueueSettings && self.settingsViewController.framePacingModeSelector.selectedSegmentIndex == FramePacingModeQueue) forStack:self.settingsViewController.frameQueueSizeStack]; + [self.settingsViewController setHidden:!(self.settingsViewController.framePacingModeSelector.selectedSegmentIndex == FramePacingModeInterpolation) forStack:self.settingsViewController.interpolationLevelStack]; + [self.settingsViewController setHidden:_settingsExpandedInStreamView || !(self.settingsViewController.framePacingModeSelector.selectedSegmentIndex == FramePacingModeInterpolation) forStack:self.settingsViewController.streamDimensionScaleStack]; // Disable mic switch if sunshine does not support mic redirection [self.settingsViewController.redirectMicSwitch setEnabled:!_settingsExpandedInStreamView||streamFrameViewController.micStreamInitialized]; @@ -2145,24 +2150,19 @@ static NSMutableSet* hostList; .height = (int32_t)currentSettings.height.intValue, }; [self fillResolutionTable:tempResolutionTable externalDisplayMode:currentSettings.externalDisplayMode.intValue]; - + return; + + /* int selectedIndex = currentSettings.resolutionSelected.intValue; if (selectedIndex >= 0 && selectedIndex < 6) { CMVideoDimensions originalDimensions = tempResolutionTable[selectedIndex]; - - CMVideoDimensions targetDimensions; - if(currentSettings.framePacingMode.intValue == FramePacingModeInterpolation) { - targetDimensions = [FrameInterpolator interpolatableDimensionsBy:originalDimensions]; - if(!(targetDimensions.width>0 && targetDimensions.height>0)) targetDimensions = [FrameInterpolator interpolatableDimensionsBy720p:originalDimensions]; - } - else targetDimensions = originalDimensions; - + CMVideoDimensions targetDimensions = originalDimensions; currentSettings.width = @(targetDimensions.width); currentSettings.height = @(targetDimensions.height); NSLog(@"Updated resolution to: %@ x %@", currentSettings.width, currentSettings.height); } - [dataMan saveData]; + */ } #if TARGET_OS_TV diff --git a/VoidLink/ViewControllers/SettingsViewController.h b/VoidLink/ViewControllers/SettingsViewController.h index de512833..cfa029ef 100644 --- a/VoidLink/ViewControllers/SettingsViewController.h +++ b/VoidLink/ViewControllers/SettingsViewController.h @@ -128,6 +128,10 @@ @property (strong, nonatomic) IBOutlet UIStackView *graphOpacityStack; @property (strong, nonatomic) IBOutlet UISegmentedControl *renderingBackendSelector; @property (strong, nonatomic) IBOutlet UISegmentedControl *framePacingModeSelector; +@property (strong, nonatomic) IBOutlet UIStackView *interpolationLevelStack; +@property (strong, nonatomic) IBOutlet UISlider *interpolationLevelSlider; +@property (strong, nonatomic) IBOutlet UIStackView *streamDimensionScaleStack; +@property (strong, nonatomic) IBOutlet UISlider *streamDimensionScaleSlider; @property (strong, nonatomic) IBOutlet UIStackView *backgroundSessionTimerStack; @property (strong, nonatomic) IBOutlet UISlider *backgroundSessionTimerSlider; diff --git a/VoidLink/ViewControllers/SettingsViewController.m b/VoidLink/ViewControllers/SettingsViewController.m index f40f9f24..19b89071 100644 --- a/VoidLink/ViewControllers/SettingsViewController.m +++ b/VoidLink/ViewControllers/SettingsViewController.m @@ -633,6 +633,12 @@ BOOL isCustomResolution(int resolutionSelected) { if(!settingsViewAlreadyAppeared) { if(![self contentOffsetRestored]) _scrollView.contentOffset = CGPointMake(_scrollView.contentOffset.x, tempSettings.settingsMenuOffset.floatValue); + dispatch_async(dispatch_get_main_queue(), ^{ + [self.view layoutIfNeeded]; + [self updateResolutionTable]; + [self updateInterpolationLevelSliderWithMaximumDimension:self->tempSettings.interpolationMaximumDimension.integerValue]; + [self.streamDimensionScaleSlider sendActionsForControlEvents:UIControlEventValueChanged]; + }); } if(@available(iOS 13.0, *)) if(ControllerUtil.primaryGCController) [self restoreControllerNavigationHighlight]; @@ -656,6 +662,13 @@ BOOL isCustomResolution(int resolutionSelected) { bool unlockDisplayOrientationFlipped = tempSettings.unlockDisplayOrientation != (_unlockDisplayOrientationSelector.selectedSegmentIndex == 1); if(unlockDisplayOrientationFlipped) [_mainFrameViewController setNeedsUpdateAllowedOrientation]; // handle allow portratit on & off if (@available(iOS 13.0, *)) if(ControllerNavigator.radialMenuView.superview) [ControllerNavigator updateRadialMenu]; + + if(self.framePacingModeSelector.selectedSegmentIndex == FramePacingModeInterpolation) { + [VideoDecoderRenderer startOrRestartFrameInterpolation]; + } + else { + [VideoDecoderRenderer stopFrameInterpolation]; + } } @@ -884,6 +897,12 @@ BOOL isCustomResolution(int resolutionSelected) { [self addSetting:self.framePacingStack ofId:@"framePacingStack" to:videoSection]; if(!FrameInterpolator.deviceSupportsInterpolation) [self.framerateSelector setEnabled:false forSegmentAtIndex:FramePacingModeInterpolation]; + self.interpolationLevelStack.hasDynamicLabel = YES; + [self addSetting:self.interpolationLevelStack ofId:@"interpolationLevelStack" to:videoSection]; + + self.streamDimensionScaleStack.hasDynamicLabel = YES; + [self addSetting:self.streamDimensionScaleStack ofId:@"streamDimensionScaleStack" to:videoSection]; + self.frameQueueSizeStack.hasDynamicLabel = YES; [self addSetting:self.frameQueueSizeStack ofId:@"frameQueueSizeStack" to:videoSection]; @@ -2235,6 +2254,12 @@ BOOL isCustomResolution(int resolutionSelected) { NSInteger framePacingMode = [self->tempSettings.framePacingMode integerValue]; self.framePacingModeSelector.selectedSegmentIndex = framePacingMode; + ((UILabel *)self.interpolationLevelStack.arrangedSubviews.firstObject).text = + [LocalizationHelper localizedStringForKey:@"Interpolation Resolution"]; + [self.interpolationLevelSlider addTarget:self action:@selector(interpolationLevelSliderMoved:) forControlEvents:UIControlEventValueChanged]; + ((UILabel *)self.streamDimensionScaleStack.arrangedSubviews.firstObject).text = + [LocalizationHelper localizedStringForKey:@"Scaled Stream Resolution"]; + [self.streamDimensionScaleSlider addTarget:self action:@selector(streamDimensionScaleSliderMoved:) forControlEvents:UIControlEventValueChanged]; [self.framePacingModeSelector addTarget:self action:@selector(framePacingModeChanged:) forControlEvents:UIControlEventValueChanged]; [self.framePacingModeSelector sendActionsForControlEvents:UIControlEventValueChanged]; @@ -2311,6 +2336,11 @@ BOOL isCustomResolution(int resolutionSelected) { self->_lastSelectedResolutionIndex = resolution; self.resolutionSelector.selectedSegmentIndex = resolution; + [self updateInterpolationLevelSliderWithMaximumDimension:self->tempSettings.interpolationMaximumDimension.integerValue]; + self.streamDimensionScaleSlider.minimumValue = 0; + self.streamDimensionScaleSlider.maximumValue = 1; + self.streamDimensionScaleSlider.value = MIN(1, MAX(0, self->tempSettings.streamDimensionScale.floatValue)); + [self.streamDimensionScaleSlider sendActionsForControlEvents:UIControlEventValueChanged]; [self.resolutionSelector addTarget:self action:@selector(newResolutionChosen) forControlEvents:UIControlEventValueChanged]; self.framerateSelector.selectedSegmentIndex = framerate; @@ -2690,10 +2720,11 @@ BOOL isCustomResolution(int resolutionSelected) { - (void)framePacingModeChanged:(UISegmentedControl *)sender { // Hide frame queue size for Off and Legacy modes [GenericUtils handleLegacyFramePacingTipIn:self with:sender passAlert:settingsViewJustExpanded uiAction:^{ - [self setHidden:(sender.selectedSegmentIndex == FramePacingModeOff || sender.selectedSegmentIndex == FramePacingModeLegacy) forStack:self.frameQueueSizeStack]; - // [self setHidden:(sender.selectedSegmentIndex != FramePacingModeQueue) forStack:self.frameTimebaseStack]; [self setHidden:(sender.selectedSegmentIndex != FramePacingModeQueue - || sender.selectedSegmentIndex != FramePacingModeInterpolation + || self->_mainFrameViewController.isStreaming) forStack:self.frameQueueSizeStack]; + // [self setHidden:(sender.selectedSegmentIndex != FramePacingModeQueue) forStack:self.frameTimebaseStack]; + [self setHidden:((sender.selectedSegmentIndex != FramePacingModeQueue + && sender.selectedSegmentIndex != FramePacingModeInterpolation) || self.renderingBackendSelector.selectedSegmentIndex != 0) forStack:self.asyncFrameDequeueStack]; if(sender.selectedSegmentIndex == FramePacingModeOff || sender.selectedSegmentIndex == FramePacingModeLegacy){ @@ -2705,6 +2736,8 @@ BOOL isCustomResolution(int resolutionSelected) { [self setHidden:(sender.selectedSegmentIndex == FramePacingModeOff || sender.selectedSegmentIndex == FramePacingModeLegacy) forStack:self.performanceGraphStack]; bool enableInterpolation = sender.selectedSegmentIndex == FramePacingModeInterpolation; + [self setHidden:!enableInterpolation forStack:self.interpolationLevelStack]; + [self setHidden:!enableInterpolation || self->_mainFrameViewController.isStreaming forStack:self.streamDimensionScaleStack]; [self widget:self.frameQueueSizeSlider setEnabled:!enableInterpolation]; [self updateResolutionDisplayLabel]; if(enableInterpolation){ @@ -2712,6 +2745,9 @@ BOOL isCustomResolution(int resolutionSelected) { if(!FrameInterpolator.dimensionLimitsAreKnown){ NSLog(@"FrameInterpolator dimensionLimitsUnknown"); } + if(!self->settingsViewJustExpanded){ + [GenericUtils handleFrameInterpolationPixelFormatTipIn:self]; + } } }]; @@ -3663,7 +3699,7 @@ BOOL isCustomResolution(int resolutionSelected) { [[NSUserDefaults standardUserDefaults] setBool:YES forKey:key]; [[NSUserDefaults standardUserDefaults] synchronize]; - if(self.framerateSelector.selectedSegmentIndex == 2 + if(self.framerateSelector.selectedSegmentIndex == self.framerateSelector.numberOfSegments - 1 && self.framePacingModeSelector.selectedSegmentIndex == FramePacingModeInterpolation) self.framerateSelector.selectedSegmentIndex = self.framerateSelector.previousSelectedSegmentIndex; // NSInteger fps = [self getChosenFrameRate]; @@ -3735,10 +3771,13 @@ BOOL isCustomResolution(int resolutionSelected) { } - (void) newResolutionChosen { - [self updateBitrate]; - [self updateResolutionDisplayLabel]; _lastSelectedResolutionIndex = self.resolutionSelector.selectedSegmentIndex; - [self updateResolutionTable]; + if (self.mainFrameViewController.settingsExpandedInStreamView) { + [self updateResolutionDisplayLabel]; + } else { + [self updateResolutionTable]; + } + [self updateBitrate]; } - (void)customResolutionSwitched:(UISwitch* )sender{ @@ -3836,11 +3875,44 @@ BOOL isCustomResolution(int resolutionSelected) { } - (void) updateResolutionDisplayLabel { - CMVideoDimensions dimensions = [self getChosenStreamDimensions]; + CMVideoDimensions dimensions = [self getChosenPresetStreamDimensions]; NSInteger width = dimensions.width; NSInteger height = dimensions.height; [self findDynamicLabelFromStack:_resolutionSelectorStack].text = [NSString stringWithFormat:@"%ld × %ld", (long)width, (long)height]; + if (self.interpolationLevelSlider != nil) { + [self.interpolationLevelSlider sendActionsForControlEvents:UIControlEventValueChanged]; + } +} + +- (void)updateInterpolationLevelSliderWithMaximumDimension:(NSInteger)savedMaximumDimension { + self.interpolationLevelSlider.minimumValue = 0; + self.interpolationLevelSlider.maximumValue = 1; + self.interpolationLevelSlider.value = [FrameInterpolator + interpolationLevelForDimensions:[self getChosenPresetStreamDimensions] + savedMaximumDimension:savedMaximumDimension]; + [self.interpolationLevelSlider sendActionsForControlEvents:UIControlEventValueChanged]; +} + +- (void)interpolationLevelSliderMoved:(UISlider *)sender { + InterpolationResolutionConfiguration *configuration = [self getCurrentInterpolationResolutionConfiguration]; + [self findDynamicLabelFromStack:self.interpolationLevelStack].text = + [NSString stringWithFormat:@" %d × %d ", configuration.dimensions.width, configuration.dimensions.height]; + if (self.streamDimensionScaleSlider != nil) { + [self.streamDimensionScaleSlider sendActionsForControlEvents:UIControlEventValueChanged]; + } +} + +- (InterpolationResolutionConfiguration *)getCurrentInterpolationResolutionConfiguration { + return [FrameInterpolator + resolutionConfigurationForDimensions:[self getChosenPresetStreamDimensions] + level:self.interpolationLevelSlider.value]; +} + +- (void)streamDimensionScaleSliderMoved:(UISlider *)sender { + CMVideoDimensions dimensions = [self getChosenStreamDimensions]; + [self findDynamicLabelFromStack:self.streamDimensionScaleStack].text = + [NSString stringWithFormat:@" %d × %d ", dimensions.width, dimensions.height]; } - (void) touchMoveEventIntervalSliderMoved:(UISlider* )sender{ @@ -3911,7 +3983,7 @@ BOOL isCustomResolution(int resolutionSelected) { } } -- (CMVideoDimensions)getChosenStreamDimensions { +- (CMVideoDimensions)getChosenPresetStreamDimensions { // because the 4k resolution can be removed CMVideoDimensions originalDimensions; if (self.customResolutionSwitch.isOn) { @@ -3920,6 +3992,12 @@ BOOL isCustomResolution(int resolutionSelected) { originalDimensions = resolutionTable[self.resolutionSelector.selectedSegmentIndex]; } + if(originalDimensions.width == 0 || originalDimensions.height == 0) { + originalDimensions.width = tempSettings.width.intValue; + originalDimensions.height = tempSettings.height.intValue; + } + + /* CMVideoDimensions targetDimensions = self.framePacingModeSelector.selectedSegmentIndex == FramePacingModeInterpolation ? [FrameInterpolator interpolatableDimensionsBy:originalDimensions] : originalDimensions; @@ -3929,8 +4007,25 @@ BOOL isCustomResolution(int resolutionSelected) { } NSLog(@"dimensions: %d, %d", targetDimensions.width, targetDimensions.height); + */ + + return originalDimensions; +} - return targetDimensions; +- (CMVideoDimensions)getChosenStreamDimensions { + CMVideoDimensions presetDimensions = [self getChosenPresetStreamDimensions]; + if (self.streamDimensionScaleSlider == nil) { + return [FrameInterpolator + scaledStreamDimensionsWithPresetDimensions:presetDimensions + interpolationDimensions:presetDimensions + scale:1]; + } + + InterpolationResolutionConfiguration *configuration = [self getCurrentInterpolationResolutionConfiguration]; + return [FrameInterpolator + scaledStreamDimensionsWithPresetDimensions:presetDimensions + interpolationDimensions:configuration.dimensions + scale:self.streamDimensionScaleSlider.value]; } - (UIStackView *)findFlatStackViewFrom:(UIView *)view { @@ -4299,6 +4394,11 @@ BOOL isCustomResolution(int resolutionSelected) { NSInteger renderingBackend = self.renderingBackendSelector.selectedSegmentIndex; NSInteger framePacingMode = self.framePacingModeSelector.selectedSegmentIndex; + InterpolationResolutionConfiguration *interpolationConfiguration = + [self getCurrentInterpolationResolutionConfiguration]; + NSInteger interpolationMaximumDimension = interpolationConfiguration.maximumDimension; + NSInteger interpolationMaximumPixelCount = interpolationConfiguration.maximumPixelCount; + CGFloat streamDimensionScale = self.streamDimensionScaleSlider.value; NSInteger onscreenControls = self.onScreenWidgetSelector.selectedSegmentIndex; NSInteger keyboardToggleFingers = self.softKeyboardGestureSelector.selectedSegmentIndex == 3 ? 20 : self.softKeyboardGestureSelector.selectedSegmentIndex+3; NSInteger oscLayoutToolFingers = (uint16_t)self->oswLayoutFingers; @@ -4428,6 +4528,9 @@ BOOL isCustomResolution(int resolutionSelected) { graphOpacity:graphOpacity renderingBackend:renderingBackend framePacingMode:framePacingMode + interpolationMaximumDimension:interpolationMaximumDimension + interpolationMaximumPixelCount:interpolationMaximumPixelCount + streamDimensionScale:streamDimensionScale sendDummyEvent:sendDummyEvent rememberFoldState:rememberFoldState singleTapSensitivy:singleTapSensitivity @@ -4466,7 +4569,6 @@ BOOL isCustomResolution(int resolutionSelected) { }]; }); } - } - (void)didReceiveMemoryWarning { diff --git a/VoidLink/mul.lproj/iPad.xcstrings b/VoidLink/mul.lproj/iPad.xcstrings index c2c964ea..2067037b 100644 --- a/VoidLink/mul.lproj/iPad.xcstrings +++ b/VoidLink/mul.lproj/iPad.xcstrings @@ -3056,6 +3056,30 @@ } } }, + "FIp-Lv-Lbl.text" : { + "comment" : "Class = \"UILabel\"; text = \"Interpolation Resolution\"; ObjectID = \"FIp-Lv-Lbl\";", + "extractionState" : "extracted_with_value", + "localizations" : { + "en" : { + "stringUnit" : { + "state" : "new", + "value" : "Interpolation Resolution" + } + } + } + }, + "FIp-Sc-Lbl.text" : { + "comment" : "Class = \"UILabel\"; text = \"Scaled Stream Resolution\"; ObjectID = \"FIp-Sc-Lbl\";", + "extractionState" : "extracted_with_value", + "localizations" : { + "en" : { + "stringUnit" : { + "state" : "new", + "value" : "Scaled Stream Resolution" + } + } + } + }, "FJv-HJ-UYJ.text" : { "comment" : "Class = \"UILabel\"; text = \"Touch Move Event Interval\"; ObjectID = \"FJv-HJ-UYJ\";", "extractionState" : "extracted_with_value", diff --git a/VoidLink/mul.lproj/iPhone.xcstrings b/VoidLink/mul.lproj/iPhone.xcstrings index 3c649beb..630bd2ca 100644 --- a/VoidLink/mul.lproj/iPhone.xcstrings +++ b/VoidLink/mul.lproj/iPhone.xcstrings @@ -2557,6 +2557,30 @@ } } }, + "FIi-Lv-Lbl.text" : { + "comment" : "Class = \"UILabel\"; text = \"Interpolation Resolution\"; ObjectID = \"FIi-Lv-Lbl\";", + "extractionState" : "extracted_with_value", + "localizations" : { + "en" : { + "stringUnit" : { + "state" : "new", + "value" : "Interpolation Resolution" + } + } + } + }, + "FIi-Sc-Lbl.text" : { + "comment" : "Class = \"UILabel\"; text = \"Scaled Stream Resolution\"; ObjectID = \"FIi-Sc-Lbl\";", + "extractionState" : "extracted_with_value", + "localizations" : { + "en" : { + "stringUnit" : { + "state" : "new", + "value" : "Scaled Stream Resolution" + } + } + } + }, "fRl-Ra-Lbl.text" : { "comment" : "Class = \"UILabel\"; text = \"Full Color Range\"; ObjectID = \"fRl-Ra-Lbl\";", "extractionState" : "extracted_with_value",