hermes-agent/VoidLink
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Metal: after experimenting with EDR tone-mapping, it's here but disabled. It's unclear how best to render content from Windows. We get the system calibration peak nits in the Mastering Display Color Volume metadata, but this won't apply to most content.

Browse Source
This commit is contained in:
Andy Grundman
2025-07-22 17:01:57 +08:00
committed by Acaki
parent 4fb3f7fa97
commit e57f0cd723
4 changed files with 70 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
VoidLink/Metal/MetalVideoRenderer.h
+2
View File
@@ -20,4 +20,6 @@
- (void)drawableResize:(CGSize)drawableSize;
- (void)shutdown;
+ (NSString *_Nullable)currentColorSpace;
@end
VoidLink/Metal/MetalVideoRenderer.m
+41 -12
View File
@@ -94,6 +94,12 @@ struct Vertex {
static const NSUInteger MaxFramesInFlight = 3;
// EDR tone-mapping is possible but it's unclear how
// best to tone-map Windows content
static BOOL useEDR = NO;
CFStringRef __currentColorSpace;
@implementation MetalVideoRenderer {
dispatch_queue_t _sq;
id<MTLDevice> _device;
@@ -106,7 +112,7 @@ static const NSUInteger MaxFramesInFlight = 3;
CVMetalTextureCacheRef _textureCache;
CVMetalTextureRef _cvMetalTextures[MAX_VIDEO_PLANES];
CGFloat _currentEDRHeadroom;
float _currentEDRHeadroom;
int _lastColorSpace;
BOOL _lastFullRange;
size_t _lastFrameWidth;
@@ -166,6 +172,9 @@ static const NSUInteger MaxFramesInFlight = 3;
if (_renderPassDescriptor) {
_renderPassDescriptor = nil;
}
if (__currentColorSpace) {
CFRelease(__currentColorSpace);
}
}
#if !TARGET_OS_TV
@@ -192,7 +201,7 @@ static const NSUInteger MaxFramesInFlight = 3;
#endif
if (headroom != _currentEDRHeadroom) {
Log(LOG_I, @"EDR headroom changed to %.1f", headroom);
_currentEDRHeadroom = headroom;
_currentEDRHeadroom = (float)headroom;
}
}
@@ -232,11 +241,15 @@ static const NSUInteger MaxFramesInFlight = 3;
CFDataRef masteringDisplayColorVolume = CVBufferCopyAttachment(frame.pixelBuffer, kCVImageBufferMasteringDisplayColorVolumeKey, nil);
CFDataRef contentLightLevel = CVBufferCopyAttachment(frame.pixelBuffer, kCVImageBufferContentLightLevelInfoKey, nil);
if (masteringDisplayColorVolume) {
// This tone-mapper does receive the host's max nits in MDCV when connecting to Windows,
// but I think using this for tone-mapping would break apps not using system calibration (most apps).
layer.EDRMetadata = [CAEDRMetadata HDR10MetadataWithDisplayInfo:(__bridge NSData *)masteringDisplayColorVolume
contentInfo:contentLightLevel ? (__bridge NSData *)contentLightLevel : nil
opticalOutputScale:100.0f];
} else {
layer.EDRMetadata = [CAEDRMetadata HDR10MetadataWithMinLuminance:0.005f maxLuminance:1000.0f opticalOutputScale:100.0f];
layer.EDRMetadata = [CAEDRMetadata HDR10MetadataWithMinLuminance:0.005f
maxLuminance:1000.0f
opticalOutputScale:100.0f];
}
LogOnce(LOG_I, @"EDRMetadata set to colorspace %@, transfer function %@, %@", layer.colorspace, frame_trc, layer.EDRMetadata);
});
@@ -328,21 +341,31 @@ static const NSUInteger MaxFramesInFlight = 3;
: [NSString stringWithFormat:@"Unknown: %lu", (unsigned long)layer.pixelFormat]);
}
if ([CAEDRMetadata isAvailable]) {
#if TARGET_OS_TV
// These can only be changed on the main thread
dispatch_sync(dispatch_get_main_queue(), ^{
layer.colorspace = newColorSpace;
layer.pixelFormat = newPixelFormat;
});
#else
if (useEDR && [CAEDRMetadata isAvailable]) {
[self applyEDRFromFrame:frame withColorspace:colorspace toLayer:layer];
} else {
// These can only be changed on the main thread
dispatch_sync(dispatch_get_main_queue(), ^{
#if !TARGET_OS_TV
if (isHDR) {
layer.wantsExtendedDynamicRangeContent = YES;
}
#endif
layer.colorspace = newColorSpace;
layer.pixelFormat = newPixelFormat;
});
CGColorSpaceRelease(newColorSpace);
}
#endif
if (__currentColorSpace) {
CFRelease(__currentColorSpace);
}
__currentColorSpace = CGColorSpaceCopyName(newColorSpace);
CGColorSpaceRelease(newColorSpace);
}
// Create the new colorspace parameter buffer for our fragment shader
@@ -548,13 +571,12 @@ static const NSUInteger MaxFramesInFlight = 3;
[renderEncoder setFragmentTexture:CVMetalTextureGetTexture(_cvMetalTextures[i]) atIndex:i];
}
// if (layer.pixelFormat == MTLPixelFormatRGBA16Float) {
// [self pollCurrentEDRHeadroom];
// [renderEncoder setFragmentBytes:&_currentEDRHeadroom length:sizeof(CGFloat) atIndex:0];
// }
[renderEncoder setVertexBuffer:_VideoVertexBuffer offset:0 atIndex:0];
[renderEncoder setFragmentBuffer:_CscParamsBuffer offset:0 atIndex:0];
if (layer.pixelFormat == MTLPixelFormatRGBA16Float) {
[self pollCurrentEDRHeadroom];
[renderEncoder setFragmentBytes:&_currentEDRHeadroom length:sizeof(float) atIndex:1];
}
[renderEncoder drawPrimitives:MTLPrimitiveTypeTriangleStrip vertexStart:0 vertexCount:4];
[renderEncoder endEncoding];
@@ -628,4 +650,11 @@ static const NSUInteger MaxFramesInFlight = 3;
- (void)resize:(CGSize)size {
}
+ (NSString *)currentColorSpace {
if (__currentColorSpace) {
return (__bridge NSString *)__currentColorSpace;
}
return nil;
}
@end
VoidLink/Metal/Shaders-Linear.metal
+26 -9
View File
@@ -16,11 +16,11 @@ struct CscParams
};
// PQ (SMPTE ST 2084) constants for inverse EOTF
constant float PQ_M1 = 0.1593017578125; // 2610/16384
constant float PQ_M2 = 78.84375; // 2523/32 * 1000/1000
constant float PQ_C1 = 0.8359375; // 3424/4096
constant float PQ_C2 = 18.8515625; // 2413/128
constant float PQ_C3 = 18.6875; // 2392/128
constant float PQ_C3 = 18.6875; // 299/16
constant float PQ_M = 78.84375; // 2523/32
constant float PQ_N = 0.1593017578125; // 1305/8192
// BT.2020 to Rec.709/sRGB color space conversion matrix
constant float3x3 bt2020_to_rec709 = float3x3(
@@ -31,17 +31,28 @@ constant float3x3 bt2020_to_rec709 = float3x3(
constexpr sampler s(coord::normalized, address::clamp_to_edge, filter::linear);
float3 scaleEDR(float3 rgb, float maxInput, float maxOutput) {
if (maxInput <= maxOutput) {
return rgb;
}
float a = maxOutput / (maxInput * maxInput);
float b = 1.0f / maxOutput;
float colorMax = max(rgb.r, max(rgb.g, rgb.b));
return rgb * (1.0f + a * colorMax) / (1.0f + b * colorMax);
}
// Convert from PQ curve to linear light
float pq_to_linear(float pq) {
if (pq <= 0.0) return 0.0;
float pq_pow_inv_m2 = pow(pq, 1.0 / PQ_M2);
float numerator = max(pq_pow_inv_m2 - PQ_C1, 0.0);
float denominator = PQ_C2 - PQ_C3 * pq_pow_inv_m2;
float pq_pow_inv_m = pow(pq, 1.0 / PQ_M);
float numerator = max(pq_pow_inv_m - PQ_C1, 0.0);
float denominator = PQ_C2 - PQ_C3 * pq_pow_inv_m;
if (denominator <= 0.0) return 0.0;
return pow(numerator / denominator, 1.0 / PQ_M1);
return 10000.0f * pow(numerator / denominator, 1.0 / PQ_N);
}
// Apply PQ inverse EOTF to RGB components
@@ -55,6 +66,7 @@ float3 pq_to_linear_rgb(float3 pq_rgb) {
fragment float4 yuvToLinear(Vertex v [[ stage_in ]],
constant CscParams &cscParams [[ buffer(0) ]],
constant float &edrHeadroom [[ buffer(1) ]],
texture2d<float> luminancePlane [[ texture(0) ]],
texture2d<float> chrominancePlane [[ texture(1) ]])
{
@@ -87,8 +99,13 @@ fragment float4 yuvToLinear(Vertex v [[ stage_in ]],
// This converts from 0-1 PQ range to 0-10000 nits linear
float3 linear_rgb = pq_to_linear_rgb(rgb);
// Scale for EDR (1.0 = 100 nits SDR white)
linear_rgb = linear_rgb * 100.0;
// Scale from PQ (0-10000) to EDR (1.0 = 100 nits SDR white)
const float referenceWhite = 100.0f; // must match opticalOutputScale in EDRMetadata
const float peakWhite = 10000.0f;
linear_rgb = linear_rgb / referenceWhite;
// This will apply linear tone-mapping to scale all values to not exceed peak, not really what we want
//linear_rgb = scaleEDR(linear_rgb / referenceWhite, peakWhite / referenceWhite, edrHeadroom);
// TODO: support tonemapping to Rec.709 for non-HDR viewers
//linear_rgb = bt2020_to_rec709 * linear_rgb;
VoidLink/Stream/VideoDecoderRenderer.m
+1 -27
View File
@@ -1016,33 +1016,7 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit);
- (void)getAllStats:(video_stats_t *)stats {
if (_renderingBackend == RENDER_METAL) {
#if TARGET_OS_OSX
float edrHeadroom = [[NSScreen mainScreen] maximumExtendedDynamicRangeColorComponentValue];
#else
float edrHeadroom = [[UIScreen mainScreen] currentEDRHeadroom];
UIScreenReferenceDisplayModeStatus referenceStatus = [[UIScreen mainScreen] referenceDisplayModeStatus];
#endif
if (edrHeadroom > 1.0) {
NSString *ref;
// Device has a reference display that may or may not be enabled
switch (referenceStatus) {
case UIScreenReferenceDisplayModeStatusLimited:
ref = @"(Reference mode limited),";
break;
case UIScreenReferenceDisplayModeStatusEnabled:
ref = @"(Reference mode),";
break;
default:
ref = @",";
break;
}
int peakNits = 1000;
stats->renderingBackendString = [NSString stringWithFormat:@"Metal, EDR %.1f %@ tone-mapped: %d nits",
edrHeadroom, ref, peakNits];
} else {
// if HDR
stats->renderingBackendString = [NSString stringWithFormat:@"Metal, tone-mapped: HDR->sRGB"];
}
stats->renderingBackendString = [NSString stringWithFormat:@"Metal, colorspace: %@", [MetalVideoRenderer currentColorSpace]];
} else {
stats->renderingBackendString = @"AVSampleBuffer";
}