From e57f0cd7230e552bad344f1ffb6ac53c1f6c4a19 Mon Sep 17 00:00:00 2001 From: Andy Grundman Date: Mon, 21 Jul 2025 20:41:06 -0400 Subject: [PATCH] Metal: after experimenting with EDR tone-mapping, it's here but disabled. It's unclear how best to render content from Windows. We get the system calibration peak nits in the Mastering Display Color Volume metadata, but this won't apply to most content. --- VoidLink/Metal/MetalVideoRenderer.h | 2 + VoidLink/Metal/MetalVideoRenderer.m | 53 ++++++++++++++++++++------ VoidLink/Metal/Shaders-Linear.metal | 35 ++++++++++++----- VoidLink/Stream/VideoDecoderRenderer.m | 28 +------------- 4 files changed, 70 insertions(+), 48 deletions(-) diff --git a/VoidLink/Metal/MetalVideoRenderer.h b/VoidLink/Metal/MetalVideoRenderer.h index 7f5d77ad..44dec7a7 100644 --- a/VoidLink/Metal/MetalVideoRenderer.h +++ b/VoidLink/Metal/MetalVideoRenderer.h @@ -20,4 +20,6 @@ - (void)drawableResize:(CGSize)drawableSize; - (void)shutdown; ++ (NSString *_Nullable)currentColorSpace; + @end diff --git a/VoidLink/Metal/MetalVideoRenderer.m b/VoidLink/Metal/MetalVideoRenderer.m index 8f74c914..5f7f8c8e 100644 --- a/VoidLink/Metal/MetalVideoRenderer.m +++ b/VoidLink/Metal/MetalVideoRenderer.m @@ -94,6 +94,12 @@ struct Vertex { static const NSUInteger MaxFramesInFlight = 3; +// EDR tone-mapping is possible but it's unclear how +// best to tone-map Windows content +static BOOL useEDR = NO; + +CFStringRef __currentColorSpace; + @implementation MetalVideoRenderer { dispatch_queue_t _sq; id _device; @@ -106,7 +112,7 @@ static const NSUInteger MaxFramesInFlight = 3; CVMetalTextureCacheRef _textureCache; CVMetalTextureRef _cvMetalTextures[MAX_VIDEO_PLANES]; - CGFloat _currentEDRHeadroom; + float _currentEDRHeadroom; int _lastColorSpace; BOOL _lastFullRange; size_t _lastFrameWidth; @@ -166,6 +172,9 @@ static const NSUInteger MaxFramesInFlight = 3; if (_renderPassDescriptor) { _renderPassDescriptor = nil; } + if (__currentColorSpace) { + CFRelease(__currentColorSpace); + } } #if !TARGET_OS_TV @@ -192,7 +201,7 @@ static const NSUInteger MaxFramesInFlight = 3; #endif if (headroom != _currentEDRHeadroom) { Log(LOG_I, @"EDR headroom changed to %.1f", headroom); - _currentEDRHeadroom = headroom; + _currentEDRHeadroom = (float)headroom; } } @@ -232,11 +241,15 @@ static const NSUInteger MaxFramesInFlight = 3; CFDataRef masteringDisplayColorVolume = CVBufferCopyAttachment(frame.pixelBuffer, kCVImageBufferMasteringDisplayColorVolumeKey, nil); CFDataRef contentLightLevel = CVBufferCopyAttachment(frame.pixelBuffer, kCVImageBufferContentLightLevelInfoKey, nil); if (masteringDisplayColorVolume) { + // This tone-mapper does receive the host's max nits in MDCV when connecting to Windows, + // but I think using this for tone-mapping would break apps not using system calibration (most apps). layer.EDRMetadata = [CAEDRMetadata HDR10MetadataWithDisplayInfo:(__bridge NSData *)masteringDisplayColorVolume contentInfo:contentLightLevel ? (__bridge NSData *)contentLightLevel : nil opticalOutputScale:100.0f]; } else { - layer.EDRMetadata = [CAEDRMetadata HDR10MetadataWithMinLuminance:0.005f maxLuminance:1000.0f opticalOutputScale:100.0f]; + layer.EDRMetadata = [CAEDRMetadata HDR10MetadataWithMinLuminance:0.005f + maxLuminance:1000.0f + opticalOutputScale:100.0f]; } LogOnce(LOG_I, @"EDRMetadata set to colorspace %@, transfer function %@, %@", layer.colorspace, frame_trc, layer.EDRMetadata); }); @@ -328,21 +341,31 @@ static const NSUInteger MaxFramesInFlight = 3; : [NSString stringWithFormat:@"Unknown: %lu", (unsigned long)layer.pixelFormat]); } - if ([CAEDRMetadata isAvailable]) { +#if TARGET_OS_TV + // These can only be changed on the main thread + dispatch_sync(dispatch_get_main_queue(), ^{ + layer.colorspace = newColorSpace; + layer.pixelFormat = newPixelFormat; + }); +#else + if (useEDR && [CAEDRMetadata isAvailable]) { [self applyEDRFromFrame:frame withColorspace:colorspace toLayer:layer]; } else { // These can only be changed on the main thread dispatch_sync(dispatch_get_main_queue(), ^{ -#if !TARGET_OS_TV if (isHDR) { layer.wantsExtendedDynamicRangeContent = YES; } -#endif layer.colorspace = newColorSpace; layer.pixelFormat = newPixelFormat; }); - CGColorSpaceRelease(newColorSpace); } +#endif + if (__currentColorSpace) { + CFRelease(__currentColorSpace); + } + __currentColorSpace = CGColorSpaceCopyName(newColorSpace); + CGColorSpaceRelease(newColorSpace); } // Create the new colorspace parameter buffer for our fragment shader @@ -548,13 +571,12 @@ static const NSUInteger MaxFramesInFlight = 3; [renderEncoder setFragmentTexture:CVMetalTextureGetTexture(_cvMetalTextures[i]) atIndex:i]; } -// if (layer.pixelFormat == MTLPixelFormatRGBA16Float) { -// [self pollCurrentEDRHeadroom]; -// [renderEncoder setFragmentBytes:&_currentEDRHeadroom length:sizeof(CGFloat) atIndex:0]; -// } - [renderEncoder setVertexBuffer:_VideoVertexBuffer offset:0 atIndex:0]; [renderEncoder setFragmentBuffer:_CscParamsBuffer offset:0 atIndex:0]; + if (layer.pixelFormat == MTLPixelFormatRGBA16Float) { + [self pollCurrentEDRHeadroom]; + [renderEncoder setFragmentBytes:&_currentEDRHeadroom length:sizeof(float) atIndex:1]; + } [renderEncoder drawPrimitives:MTLPrimitiveTypeTriangleStrip vertexStart:0 vertexCount:4]; [renderEncoder endEncoding]; @@ -628,4 +650,11 @@ static const NSUInteger MaxFramesInFlight = 3; - (void)resize:(CGSize)size { } ++ (NSString *)currentColorSpace { + if (__currentColorSpace) { + return (__bridge NSString *)__currentColorSpace; + } + return nil; +} + @end diff --git a/VoidLink/Metal/Shaders-Linear.metal b/VoidLink/Metal/Shaders-Linear.metal index 221d4f47..2ff94a1b 100644 --- a/VoidLink/Metal/Shaders-Linear.metal +++ b/VoidLink/Metal/Shaders-Linear.metal @@ -16,11 +16,11 @@ struct CscParams }; // PQ (SMPTE ST 2084) constants for inverse EOTF -constant float PQ_M1 = 0.1593017578125; // 2610/16384 -constant float PQ_M2 = 78.84375; // 2523/32 * 1000/1000 constant float PQ_C1 = 0.8359375; // 3424/4096 constant float PQ_C2 = 18.8515625; // 2413/128 -constant float PQ_C3 = 18.6875; // 2392/128 +constant float PQ_C3 = 18.6875; // 299/16 +constant float PQ_M = 78.84375; // 2523/32 +constant float PQ_N = 0.1593017578125; // 1305/8192 // BT.2020 to Rec.709/sRGB color space conversion matrix constant float3x3 bt2020_to_rec709 = float3x3( @@ -31,17 +31,28 @@ constant float3x3 bt2020_to_rec709 = float3x3( constexpr sampler s(coord::normalized, address::clamp_to_edge, filter::linear); + +float3 scaleEDR(float3 rgb, float maxInput, float maxOutput) { + if (maxInput <= maxOutput) { + return rgb; + } + float a = maxOutput / (maxInput * maxInput); + float b = 1.0f / maxOutput; + float colorMax = max(rgb.r, max(rgb.g, rgb.b)); + return rgb * (1.0f + a * colorMax) / (1.0f + b * colorMax); +} + // Convert from PQ curve to linear light float pq_to_linear(float pq) { if (pq <= 0.0) return 0.0; - float pq_pow_inv_m2 = pow(pq, 1.0 / PQ_M2); - float numerator = max(pq_pow_inv_m2 - PQ_C1, 0.0); - float denominator = PQ_C2 - PQ_C3 * pq_pow_inv_m2; + float pq_pow_inv_m = pow(pq, 1.0 / PQ_M); + float numerator = max(pq_pow_inv_m - PQ_C1, 0.0); + float denominator = PQ_C2 - PQ_C3 * pq_pow_inv_m; if (denominator <= 0.0) return 0.0; - return pow(numerator / denominator, 1.0 / PQ_M1); + return 10000.0f * pow(numerator / denominator, 1.0 / PQ_N); } // Apply PQ inverse EOTF to RGB components @@ -55,6 +66,7 @@ float3 pq_to_linear_rgb(float3 pq_rgb) { fragment float4 yuvToLinear(Vertex v [[ stage_in ]], constant CscParams &cscParams [[ buffer(0) ]], + constant float &edrHeadroom [[ buffer(1) ]], texture2d luminancePlane [[ texture(0) ]], texture2d chrominancePlane [[ texture(1) ]]) { @@ -87,8 +99,13 @@ fragment float4 yuvToLinear(Vertex v [[ stage_in ]], // This converts from 0-1 PQ range to 0-10000 nits linear float3 linear_rgb = pq_to_linear_rgb(rgb); - // Scale for EDR (1.0 = 100 nits SDR white) - linear_rgb = linear_rgb * 100.0; + // Scale from PQ (0-10000) to EDR (1.0 = 100 nits SDR white) + const float referenceWhite = 100.0f; // must match opticalOutputScale in EDRMetadata + const float peakWhite = 10000.0f; + linear_rgb = linear_rgb / referenceWhite; + + // This will apply linear tone-mapping to scale all values to not exceed peak, not really what we want + //linear_rgb = scaleEDR(linear_rgb / referenceWhite, peakWhite / referenceWhite, edrHeadroom); // TODO: support tonemapping to Rec.709 for non-HDR viewers //linear_rgb = bt2020_to_rec709 * linear_rgb; diff --git a/VoidLink/Stream/VideoDecoderRenderer.m b/VoidLink/Stream/VideoDecoderRenderer.m index 27a8b3d9..56240fd5 100644 --- a/VoidLink/Stream/VideoDecoderRenderer.m +++ b/VoidLink/Stream/VideoDecoderRenderer.m @@ -1016,33 +1016,7 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit); - (void)getAllStats:(video_stats_t *)stats { if (_renderingBackend == RENDER_METAL) { -#if TARGET_OS_OSX - float edrHeadroom = [[NSScreen mainScreen] maximumExtendedDynamicRangeColorComponentValue]; -#else - float edrHeadroom = [[UIScreen mainScreen] currentEDRHeadroom]; - UIScreenReferenceDisplayModeStatus referenceStatus = [[UIScreen mainScreen] referenceDisplayModeStatus]; -#endif - if (edrHeadroom > 1.0) { - NSString *ref; - // Device has a reference display that may or may not be enabled - switch (referenceStatus) { - case UIScreenReferenceDisplayModeStatusLimited: - ref = @"(Reference mode limited),"; - break; - case UIScreenReferenceDisplayModeStatusEnabled: - ref = @"(Reference mode),"; - break; - default: - ref = @","; - break; - } - int peakNits = 1000; - stats->renderingBackendString = [NSString stringWithFormat:@"Metal, EDR %.1f %@ tone-mapped: %d nits", - edrHeadroom, ref, peakNits]; - } else { - // if HDR - stats->renderingBackendString = [NSString stringWithFormat:@"Metal, tone-mapped: HDR->sRGB"]; - } + stats->renderingBackendString = [NSString stringWithFormat:@"Metal, colorspace: %@", [MetalVideoRenderer currentColorSpace]]; } else { stats->renderingBackendString = @"AVSampleBuffer"; }