fix: green tinted image for metal 10bit yuv444

VoidLink/Metal/MetalVideoRenderer.m

@@ -56,6 +56,16 @@ static const struct CscParams k_CscParams_Bt2020Lim = {
     },
     {16.0f / 255.0f, 128.0f / 255.0f, 128.0f / 255.0f},
 };
+
+static const struct CscParams k_CscParams_Bt2020Lim_10bit = {
+    {
+        {1.1644f, 0.0f, 1.6781f},
+        {1.1644f, -0.1874f, -0.6505f},
+        {1.1644f, 2.1418f, 0.0f},
+    },
+    {64.0f / 1023.0f, 512.0f / 1023.0f, 512.0f / 1023.0f},
+};
+
 static const struct CscParams k_CscParams_Bt2020Full = {
     {
         {1.0f, 0.0f, 1.4746f},
@@ -65,6 +75,16 @@ static const struct CscParams k_CscParams_Bt2020Full = {
     {0.0f, 128.0f / 255.0f, 128.0f / 255.0f},
 };
 
+static const struct CscParams k_CscParams_Bt2020Full_10bit = {
+    {
+        {1.0f, 0.0f, 1.4746f},
+        {1.0f, -0.1646f, -0.5714f},
+        {1.0f, 1.8814f, 0.0f},
+    },
+
+    {0.0f, 512.0f / 1023.0f, 512.0f / 1023.0f},
+};
+
 struct Vertex {
     vector_float4 position;
     vector_float2 texCoord;
@@ -237,7 +257,11 @@ static const NSUInteger MaxFramesInFlight = 3;
                     newColorSpace = CGColorSpaceCreateWithName(kCGColorSpaceITUR_2020);
                     newPixelFormat = MTLPixelFormatBGR10A2Unorm;
                 }
-                paramBuffer.cscParams = (fullRange ? k_CscParams_Bt2020Full : k_CscParams_Bt2020Lim);
+                if (isHDR) {
+                    paramBuffer.cscParams = (fullRange ? k_CscParams_Bt2020Full_10bit : k_CscParams_Bt2020Lim_10bit);
+                } else {
+                    paramBuffer.cscParams = (fullRange ? k_CscParams_Bt2020Full : k_CscParams_Bt2020Lim);
+                }
                 break;
             }
             case COLORSPACE_REC_601:
@@ -387,7 +411,20 @@ static const NSUInteger MaxFramesInFlight = 3;
             MTLRenderPipelineDescriptor *pipelineDesc = [MTLRenderPipelineDescriptor new];
             id<MTLLibrary> defaultLibrary = [_device newDefaultLibrary];
             pipelineDesc.vertexFunction = [defaultLibrary newFunctionWithName:@"vs_draw"];
-            pipelineDesc.fragmentFunction = [defaultLibrary newFunctionWithName:planes == 2 ? @"ps_draw_biplanar" : @"ps_draw_triplanar"];
+            
+            // Determine if this is 10-bit based on the layer's pixel format (after colorspace update)
+            BOOL is10Bit = (layer.pixelFormat == MTLPixelFormatBGR10A2Unorm);
+            Log(LOG_I, @"Layer pixel format: %lu, is10Bit: %@", (unsigned long)layer.pixelFormat, is10Bit ? @"YES" : @"NO");
+            
+            NSString *fragmentShaderName;
+            if (planes == 2) {
+                fragmentShaderName = is10Bit ? @"ps_draw_biplanar_10bit" : @"ps_draw_biplanar_8bit";
+            } else {
+                fragmentShaderName = is10Bit ? @"ps_draw_triplanar_10bit" : @"ps_draw_triplanar_8bit";
+            }
+            
+            Log(LOG_I, @"Rendering frame with %zu planes, using shader: %@", planes, fragmentShaderName);
+            pipelineDesc.fragmentFunction = [defaultLibrary newFunctionWithName:fragmentShaderName];
             pipelineDesc.colorAttachments[0].pixelFormat = layer.pixelFormat;
             pipelineDesc.vertexBuffers[0].mutability = MTLMutabilityImmutable;
 

VoidLink/Metal/Shaders.metal

@@ -22,10 +22,10 @@ vertex Vertex vs_draw(constant Vertex *vertices [[ buffer(0) ]], uint id [[ vert
     return vertices[id];
 }
 
-fragment float4 ps_draw_biplanar(Vertex v [[ stage_in ]],
+fragment float4 ps_draw_biplanar_8bit(Vertex v [[ stage_in ]],
-                                 constant CscParams &cscParams [[ buffer(0) ]],
+                                     constant CscParams &cscParams [[ buffer(0) ]],
-                                 texture2d<float> luminancePlane [[ texture(0) ]],
+                                     texture2d<float> luminancePlane [[ texture(0) ]],
-                                 texture2d<float> chrominancePlane [[ texture(1) ]])
+                                     texture2d<float> chrominancePlane [[ texture(1) ]])
 {
     float3 yuv = float3(luminancePlane.sample(s, v.texCoords).r,
                         chrominancePlane.sample(s, v.texCoords).rg);
@@ -38,11 +38,11 @@ fragment float4 ps_draw_biplanar(Vertex v [[ stage_in ]],
     return float4(rgb, 1.0f);
 }
 
-fragment float4 ps_draw_triplanar(Vertex v [[ stage_in ]],
+fragment float4 ps_draw_triplanar_8bit(Vertex v [[ stage_in ]],
-                                  constant CscParams &cscParams [[ buffer(0) ]],
+                                      constant CscParams &cscParams [[ buffer(0) ]],
-                                  texture2d<float> luminancePlane [[ texture(0) ]],
+                                      texture2d<float> luminancePlane [[ texture(0) ]],
-                                  texture2d<float> chrominancePlaneU [[ texture(1) ]],
+                                      texture2d<float> chrominancePlaneU [[ texture(1) ]],
-                                  texture2d<float> chrominancePlaneV [[ texture(2) ]])
+                                      texture2d<float> chrominancePlaneV [[ texture(2) ]])
 {
     float3 yuv = float3(luminancePlane.sample(s, v.texCoords).r,
                         chrominancePlaneU.sample(s, v.texCoords).r,
@@ -56,6 +56,70 @@ fragment float4 ps_draw_triplanar(Vertex v [[ stage_in ]],
     return float4(rgb, 1.0f);
 }
 
+fragment float4 ps_draw_biplanar_10bit(Vertex v [[ stage_in ]],
+                                      constant CscParams &cscParams [[ buffer(0) ]],
+                                      texture2d<float> luminancePlane [[ texture(0) ]],
+                                      texture2d<float> chrominancePlane [[ texture(1) ]])
+{
+    // 1. Sample the textures to get the normalized float value from the GPU hardware.
+    float3 yuv_hardware_normalized = float3(luminancePlane.sample(s, v.texCoords).r,
+                                          chrominancePlane.sample(s, v.texCoords).rg);
+
+    // 2. Reverse the normalization to get back to the approximate 10-bit integer value.
+    // This reverses the server's (v_10bit << 6) and the hardware's (/ 65535.0) operations.
+    float y_10bit = (yuv_hardware_normalized.r * 65535.0) / 64.0;
+    float2 uv_10bit = (yuv_hardware_normalized.gb * 65535.0) / 64.0;
+
+    // 3. Re-normalize the 10-bit value using the correct 1023.0 divisor that the CSC constants expect.
+    float3 yuv_corrected;
+    yuv_corrected.r = y_10bit / 1023.0;
+    yuv_corrected.gb = uv_10bit / 1023.0;
+
+    // 4. Use this perfectly scaled YUV value with the original CSC parameters.
+    yuv_corrected -= cscParams.offsets;
+
+    // 5. Perform the final color space conversion.
+    float3 rgb;
+    rgb.r = dot(yuv_corrected, cscParams.matrix[0]);
+    rgb.g = dot(yuv_corrected, cscParams.matrix[1]);
+    rgb.b = dot(yuv_corrected, cscParams.matrix[2]);
+    return float4(rgb, 1.0f);
+}
+
+fragment float4 ps_draw_triplanar_10bit(Vertex v [[ stage_in ]],
+                                       constant CscParams &cscParams [[ buffer(0) ]],
+                                       texture2d<float> luminancePlane [[ texture(0) ]],
+                                       texture2d<float> chrominancePlaneU [[ texture(1) ]],
+                                       texture2d<float> chrominancePlaneV [[ texture(2) ]])
+{
+    // 1. Sample the textures to get the normalized float value from the GPU hardware.
+    float y_hardware_normalized = luminancePlane.sample(s, v.texCoords).r;
+    float u_hardware_normalized = chrominancePlaneU.sample(s, v.texCoords).r;
+    float v_hardware_normalized = chrominancePlaneV.sample(s, v.texCoords).r;
+
+    // 2. Reverse the normalization to get back to the approximate 10-bit integer value.
+    // This reverses the server's (v_10bit << 6) and the hardware's (/ 65535.0) operations.
+    float y_10bit = (y_hardware_normalized * 65535.0) / 64.0;
+    float u_10bit = (u_hardware_normalized * 65535.0) / 64.0;
+    float v_10bit = (v_hardware_normalized * 65535.0) / 64.0;
+
+    // 3. Re-normalize the 10-bit value using the correct 1023.0 divisor that the CSC constants expect.
+    float3 yuv_corrected;
+    yuv_corrected.r = y_10bit / 1023.0;
+    yuv_corrected.g = u_10bit / 1023.0;
+    yuv_corrected.b = v_10bit / 1023.0;
+
+    // 4. Use this perfectly scaled YUV value with the original CSC parameters.
+    yuv_corrected -= cscParams.offsets;
+
+    // 5. Perform the final color space conversion.
+    float3 rgb;
+    rgb.r = dot(yuv_corrected, cscParams.matrix[0]);
+    rgb.g = dot(yuv_corrected, cscParams.matrix[1]);
+    rgb.b = dot(yuv_corrected, cscParams.matrix[2]);
+    return float4(rgb, 1.0f);
+}
+
 fragment float4 ps_draw_rgb(Vertex v [[ stage_in ]],
                             texture2d<float> rgbTexture [[ texture(0) ]])
 {