diff --git a/VoidLink/Metal/MetalVideoRenderer.h b/VoidLink/Metal/MetalVideoRenderer.h index 7f5d77ad..44dec7a7 100644 --- a/VoidLink/Metal/MetalVideoRenderer.h +++ b/VoidLink/Metal/MetalVideoRenderer.h @@ -20,4 +20,6 @@ - (void)drawableResize:(CGSize)drawableSize; - (void)shutdown; ++ (NSString *_Nullable)currentColorSpace; + @end diff --git a/VoidLink/Metal/MetalVideoRenderer.m b/VoidLink/Metal/MetalVideoRenderer.m index 8f74c914..5f7f8c8e 100644 --- a/VoidLink/Metal/MetalVideoRenderer.m +++ b/VoidLink/Metal/MetalVideoRenderer.m @@ -94,6 +94,12 @@ struct Vertex { static const NSUInteger MaxFramesInFlight = 3; +// EDR tone-mapping is possible but it's unclear how +// best to tone-map Windows content +static BOOL useEDR = NO; + +CFStringRef __currentColorSpace; + @implementation MetalVideoRenderer { dispatch_queue_t _sq; id _device; @@ -106,7 +112,7 @@ static const NSUInteger MaxFramesInFlight = 3; CVMetalTextureCacheRef _textureCache; CVMetalTextureRef _cvMetalTextures[MAX_VIDEO_PLANES]; - CGFloat _currentEDRHeadroom; + float _currentEDRHeadroom; int _lastColorSpace; BOOL _lastFullRange; size_t _lastFrameWidth; @@ -166,6 +172,9 @@ static const NSUInteger MaxFramesInFlight = 3; if (_renderPassDescriptor) { _renderPassDescriptor = nil; } + if (__currentColorSpace) { + CFRelease(__currentColorSpace); + } } #if !TARGET_OS_TV @@ -192,7 +201,7 @@ static const NSUInteger MaxFramesInFlight = 3; #endif if (headroom != _currentEDRHeadroom) { Log(LOG_I, @"EDR headroom changed to %.1f", headroom); - _currentEDRHeadroom = headroom; + _currentEDRHeadroom = (float)headroom; } } @@ -232,11 +241,15 @@ static const NSUInteger MaxFramesInFlight = 3; CFDataRef masteringDisplayColorVolume = CVBufferCopyAttachment(frame.pixelBuffer, kCVImageBufferMasteringDisplayColorVolumeKey, nil); CFDataRef contentLightLevel = CVBufferCopyAttachment(frame.pixelBuffer, kCVImageBufferContentLightLevelInfoKey, nil); if (masteringDisplayColorVolume) { + // This tone-mapper does receive the host's max nits in MDCV when connecting to Windows, + // but I think using this for tone-mapping would break apps not using system calibration (most apps). layer.EDRMetadata = [CAEDRMetadata HDR10MetadataWithDisplayInfo:(__bridge NSData *)masteringDisplayColorVolume contentInfo:contentLightLevel ? (__bridge NSData *)contentLightLevel : nil opticalOutputScale:100.0f]; } else { - layer.EDRMetadata = [CAEDRMetadata HDR10MetadataWithMinLuminance:0.005f maxLuminance:1000.0f opticalOutputScale:100.0f]; + layer.EDRMetadata = [CAEDRMetadata HDR10MetadataWithMinLuminance:0.005f + maxLuminance:1000.0f + opticalOutputScale:100.0f]; } LogOnce(LOG_I, @"EDRMetadata set to colorspace %@, transfer function %@, %@", layer.colorspace, frame_trc, layer.EDRMetadata); }); @@ -328,21 +341,31 @@ static const NSUInteger MaxFramesInFlight = 3; : [NSString stringWithFormat:@"Unknown: %lu", (unsigned long)layer.pixelFormat]); } - if ([CAEDRMetadata isAvailable]) { +#if TARGET_OS_TV + // These can only be changed on the main thread + dispatch_sync(dispatch_get_main_queue(), ^{ + layer.colorspace = newColorSpace; + layer.pixelFormat = newPixelFormat; + }); +#else + if (useEDR && [CAEDRMetadata isAvailable]) { [self applyEDRFromFrame:frame withColorspace:colorspace toLayer:layer]; } else { // These can only be changed on the main thread dispatch_sync(dispatch_get_main_queue(), ^{ -#if !TARGET_OS_TV if (isHDR) { layer.wantsExtendedDynamicRangeContent = YES; } -#endif layer.colorspace = newColorSpace; layer.pixelFormat = newPixelFormat; }); - CGColorSpaceRelease(newColorSpace); } +#endif + if (__currentColorSpace) { + CFRelease(__currentColorSpace); + } + __currentColorSpace = CGColorSpaceCopyName(newColorSpace); + CGColorSpaceRelease(newColorSpace); } // Create the new colorspace parameter buffer for our fragment shader @@ -548,13 +571,12 @@ static const NSUInteger MaxFramesInFlight = 3; [renderEncoder setFragmentTexture:CVMetalTextureGetTexture(_cvMetalTextures[i]) atIndex:i]; } -// if (layer.pixelFormat == MTLPixelFormatRGBA16Float) { -// [self pollCurrentEDRHeadroom]; -// [renderEncoder setFragmentBytes:&_currentEDRHeadroom length:sizeof(CGFloat) atIndex:0]; -// } - [renderEncoder setVertexBuffer:_VideoVertexBuffer offset:0 atIndex:0]; [renderEncoder setFragmentBuffer:_CscParamsBuffer offset:0 atIndex:0]; + if (layer.pixelFormat == MTLPixelFormatRGBA16Float) { + [self pollCurrentEDRHeadroom]; + [renderEncoder setFragmentBytes:&_currentEDRHeadroom length:sizeof(float) atIndex:1]; + } [renderEncoder drawPrimitives:MTLPrimitiveTypeTriangleStrip vertexStart:0 vertexCount:4]; [renderEncoder endEncoding]; @@ -628,4 +650,11 @@ static const NSUInteger MaxFramesInFlight = 3; - (void)resize:(CGSize)size { } ++ (NSString *)currentColorSpace { + if (__currentColorSpace) { + return (__bridge NSString *)__currentColorSpace; + } + return nil; +} + @end diff --git a/VoidLink/Metal/Shaders-Linear.metal b/VoidLink/Metal/Shaders-Linear.metal index 221d4f47..2ff94a1b 100644 --- a/VoidLink/Metal/Shaders-Linear.metal +++ b/VoidLink/Metal/Shaders-Linear.metal @@ -16,11 +16,11 @@ struct CscParams }; // PQ (SMPTE ST 2084) constants for inverse EOTF -constant float PQ_M1 = 0.1593017578125; // 2610/16384 -constant float PQ_M2 = 78.84375; // 2523/32 * 1000/1000 constant float PQ_C1 = 0.8359375; // 3424/4096 constant float PQ_C2 = 18.8515625; // 2413/128 -constant float PQ_C3 = 18.6875; // 2392/128 +constant float PQ_C3 = 18.6875; // 299/16 +constant float PQ_M = 78.84375; // 2523/32 +constant float PQ_N = 0.1593017578125; // 1305/8192 // BT.2020 to Rec.709/sRGB color space conversion matrix constant float3x3 bt2020_to_rec709 = float3x3( @@ -31,17 +31,28 @@ constant float3x3 bt2020_to_rec709 = float3x3( constexpr sampler s(coord::normalized, address::clamp_to_edge, filter::linear); + +float3 scaleEDR(float3 rgb, float maxInput, float maxOutput) { + if (maxInput <= maxOutput) { + return rgb; + } + float a = maxOutput / (maxInput * maxInput); + float b = 1.0f / maxOutput; + float colorMax = max(rgb.r, max(rgb.g, rgb.b)); + return rgb * (1.0f + a * colorMax) / (1.0f + b * colorMax); +} + // Convert from PQ curve to linear light float pq_to_linear(float pq) { if (pq <= 0.0) return 0.0; - float pq_pow_inv_m2 = pow(pq, 1.0 / PQ_M2); - float numerator = max(pq_pow_inv_m2 - PQ_C1, 0.0); - float denominator = PQ_C2 - PQ_C3 * pq_pow_inv_m2; + float pq_pow_inv_m = pow(pq, 1.0 / PQ_M); + float numerator = max(pq_pow_inv_m - PQ_C1, 0.0); + float denominator = PQ_C2 - PQ_C3 * pq_pow_inv_m; if (denominator <= 0.0) return 0.0; - return pow(numerator / denominator, 1.0 / PQ_M1); + return 10000.0f * pow(numerator / denominator, 1.0 / PQ_N); } // Apply PQ inverse EOTF to RGB components @@ -55,6 +66,7 @@ float3 pq_to_linear_rgb(float3 pq_rgb) { fragment float4 yuvToLinear(Vertex v [[ stage_in ]], constant CscParams &cscParams [[ buffer(0) ]], + constant float &edrHeadroom [[ buffer(1) ]], texture2d luminancePlane [[ texture(0) ]], texture2d chrominancePlane [[ texture(1) ]]) { @@ -87,8 +99,13 @@ fragment float4 yuvToLinear(Vertex v [[ stage_in ]], // This converts from 0-1 PQ range to 0-10000 nits linear float3 linear_rgb = pq_to_linear_rgb(rgb); - // Scale for EDR (1.0 = 100 nits SDR white) - linear_rgb = linear_rgb * 100.0; + // Scale from PQ (0-10000) to EDR (1.0 = 100 nits SDR white) + const float referenceWhite = 100.0f; // must match opticalOutputScale in EDRMetadata + const float peakWhite = 10000.0f; + linear_rgb = linear_rgb / referenceWhite; + + // This will apply linear tone-mapping to scale all values to not exceed peak, not really what we want + //linear_rgb = scaleEDR(linear_rgb / referenceWhite, peakWhite / referenceWhite, edrHeadroom); // TODO: support tonemapping to Rec.709 for non-HDR viewers //linear_rgb = bt2020_to_rec709 * linear_rgb; diff --git a/VoidLink/Stream/VideoDecoderRenderer.m b/VoidLink/Stream/VideoDecoderRenderer.m index 27a8b3d9..56240fd5 100644 --- a/VoidLink/Stream/VideoDecoderRenderer.m +++ b/VoidLink/Stream/VideoDecoderRenderer.m @@ -1016,33 +1016,7 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit); - (void)getAllStats:(video_stats_t *)stats { if (_renderingBackend == RENDER_METAL) { -#if TARGET_OS_OSX - float edrHeadroom = [[NSScreen mainScreen] maximumExtendedDynamicRangeColorComponentValue]; -#else - float edrHeadroom = [[UIScreen mainScreen] currentEDRHeadroom]; - UIScreenReferenceDisplayModeStatus referenceStatus = [[UIScreen mainScreen] referenceDisplayModeStatus]; -#endif - if (edrHeadroom > 1.0) { - NSString *ref; - // Device has a reference display that may or may not be enabled - switch (referenceStatus) { - case UIScreenReferenceDisplayModeStatusLimited: - ref = @"(Reference mode limited),"; - break; - case UIScreenReferenceDisplayModeStatusEnabled: - ref = @"(Reference mode),"; - break; - default: - ref = @","; - break; - } - int peakNits = 1000; - stats->renderingBackendString = [NSString stringWithFormat:@"Metal, EDR %.1f %@ tone-mapped: %d nits", - edrHeadroom, ref, peakNits]; - } else { - // if HDR - stats->renderingBackendString = [NSString stringWithFormat:@"Metal, tone-mapped: HDR->sRGB"]; - } + stats->renderingBackendString = [NSString stringWithFormat:@"Metal, colorspace: %@", [MetalVideoRenderer currentColorSpace]]; } else { stats->renderingBackendString = @"AVSampleBuffer"; }