fix: build

VoidLink/Metal/MetalConfig.h

@@ -0,0 +1,22 @@
+//
+//  MetalConfig.h
+//  VoidLink
+//
+//  Created by Mason Chen on 2025/7/11.
+//  Copyright © 2025 Moonlight Game Streaming Project. All rights reserved.
+//
+
+
+// When enabled, rendering occurs on the main application thread.
+// This can make responding to UI events during redraw simpler
+// to manage because UI calls usually need to occur on the main thread.
+// When disabled, rendering occurs on a background thread, allowing
+// the UI to respond more quickly in some cases because events can
+// process asynchronously from potentially CPU-intensive rendering code.
+#define RENDER_ON_MAIN_THREAD 0
+
+// When enabled, the drawable's size updates automatically whenever
+// the view resizes. When disabled, you can update the drawable's
+// size explicitly outside the view class.
+#define AUTOMATICALLY_RESIZE 1
+

VoidLink/Metal/MetalVideoRenderer.h

@@ -0,0 +1,32 @@
+//
+//  MetalVideoRenderer.h
+//  VoidLink
+//
+//  Created by Mason Chen on 2025/7/11.
+//  Copyright © 2025 Moonlight Game Streaming Project. All rights reserved.
+//
+
+
+#import <QuartzCore/CAMetalLayer.h>
+#import <QuartzCore/QuartzCore.h>
+#import "ConnectionCallbacks.h"
+#import "Frame.h"
+#import "Plot.h"
+
+@interface MetalVideoRenderer : NSObject
+
+@property (atomic) CFTimeInterval averageGPUTime;
+@property (nonatomic) NSUInteger sampleCount;
+@property (nonatomic) MTLPixelFormat colorPixelFormat;
+@property (nonatomic, nonnull) CGColorSpaceRef colorspace;
+@property (nonatomic) id<CAMetalDrawable> _Nullable nextDrawable;
+
+- (nonnull instancetype)initWithMetalDevice:(nonnull id<MTLDevice>)device
+                        drawablePixelFormat:(MTLPixelFormat)drawablePixelFormat
+                                  framerate:(float)framerate;
+- (void)renderFrame:(nonnull Frame *)frame toLayer:(nonnull CAMetalLayer *)layer;
+- (void)waitToRenderTo:(nonnull CAMetalLayer *)layer;
+- (void)drawableResize:(CGSize)drawableSize;
+- (void)plotFrametime:(CFTimeInterval)presentedTime;
+
+@end

VoidLink/Metal/MetalVideoRenderer.m

@@ -0,0 +1,533 @@
+//
+//  MetalVideoRenderer.m
+//  VoidLink
+//
+//  Created by Mason Chen on 2025/7/11.
+//  Copyright © 2025 Moonlight Game Streaming Project. All rights reserved.
+//
+
+
+#import "MetalVideoRenderer.h"
+#import <CoreVideo/CoreVideo.h>
+#import <Metal/Metal.h>
+#import <MetalKit/MetalKit.h>
+#import "ImGuiPlots.h"
+
+#include <Limelight.h>
+
+#define MAX_VIDEO_PLANES 3
+
+struct CscParams {
+    vector_float3 matrix[3];
+    vector_float3 offsets;
+};
+
+struct ParamBuffer {
+    struct CscParams cscParams;
+};
+
+static const struct CscParams k_CscParams_Bt601Lim = {
+    // CSC Matrix
+    {{1.1644f, 0.0f, 1.5960f}, {1.1644f, -0.3917f, -0.8129f}, {1.1644f, 2.0172f, 0.0f}},
+
+    // Offsets
+    {16.0f / 255.0f, 128.0f / 255.0f, 128.0f / 255.0f},
+};
+static const struct CscParams k_CscParams_Bt601Full = {
+    {
+        {1.0f, 0.0f, 1.4020f},
+        {1.0f, -0.3441f, -0.7141f},
+        {1.0f, 1.7720f, 0.0f},
+    },
+    {0.0f, 128.0f / 255.0f, 128.0f / 255.0f},
+};
+static const struct CscParams k_CscParams_Bt709Lim = {
+    {
+        {1.1644f, 0.0f, 1.7927f},
+        {1.1644f, -0.2132f, -0.5329f},
+        {1.1644f, 2.1124f, 0.0f},
+    },
+    {16.0f / 255.0f, 128.0f / 255.0f, 128.0f / 255.0f},
+};
+static const struct CscParams k_CscParams_Bt709Full = {
+    {
+        {1.0f, 0.0f, 1.5748f},
+        {1.0f, -0.1873f, -0.4681f},
+        {1.0f, 1.8556f, 0.0f},
+    },
+    {0.0f, 128.0f / 255.0f, 128.0f / 255.0f},
+};
+static const struct CscParams k_CscParams_Bt2020Lim = {
+    {
+        {1.1644f, 0.0f, 1.6781f},
+        {1.1644f, -0.1874f, -0.6505f},
+        {1.1644f, 2.1418f, 0.0f},
+    },
+    {16.0f / 255.0f, 128.0f / 255.0f, 128.0f / 255.0f},
+};
+static const struct CscParams k_CscParams_Bt2020Full = {
+    {
+        {1.0f, 0.0f, 1.4746f},
+        {1.0f, -0.1646f, -0.5714f},
+        {1.0f, 1.8814f, 0.0f},
+    },
+    {0.0f, 128.0f / 255.0f, 128.0f / 255.0f},
+};
+
+struct Vertex {
+    vector_float4 position;
+    vector_float2 texCoord;
+};
+
+static const NSUInteger MaxFramesInFlight = 3;
+
+@implementation MetalVideoRenderer {
+    dispatch_queue_t _sq;
+    id<MTLDevice> _device;
+    float _framerate;
+    id<ConnectionCallbacks> _callbacks;
+    id<MTLCommandQueue> _commandQueue;
+    id<MTLLibrary> _shaderLibrary;
+    id<MTLRenderPipelineState> _videoPipelineState;
+    MTLRenderPassDescriptor *_renderPassDescriptor;
+    id<MTLTexture> _videoTexture;
+    CVMetalTextureCacheRef _textureCache;
+    CVMetalTextureRef _cvMetalTextures[MAX_VIDEO_PLANES];
+
+    int _lastColorSpace;
+    BOOL _lastFullRange;
+    size_t _lastFrameWidth;
+    size_t _lastFrameHeight;
+    size_t _lastDrawableWidth;
+    size_t _lastDrawableHeight;
+    id<MTLBuffer> _CscParamsBuffer;
+    id<MTLBuffer> _VideoVertexBuffer;
+    CFTimeInterval _lastPresented;
+
+    // https://developer.apple.com/documentation/metal/synchronizing-cpu-and-gpu-work?language=objc
+    dispatch_semaphore_t _inFlightSemaphore;
+}
+
+- (instancetype)initWithMetalDevice:(id<MTLDevice>)device drawablePixelFormat:(MTLPixelFormat)drawablePixelFormat framerate:(float)framerate {
+    self = [super init];
+    if (self) {
+        _sq = dispatch_queue_create("com.moonlight.MetalVideoRenderer",
+                                    dispatch_queue_attr_make_with_qos_class(DISPATCH_QUEUE_SERIAL, QOS_CLASS_USER_INTERACTIVE, 0));
+        _averageGPUTime = (1.0f / framerate) / 2;
+        _device = device;
+        _nextDrawable = nil;
+        _colorPixelFormat = MTLPixelFormatBGR10A2Unorm;
+        _colorspace = CGColorSpaceCreateWithName(kCGColorSpaceITUR_2100_PQ);
+        _framerate = framerate;
+        _commandQueue = [_device newCommandQueue];
+        _lastColorSpace = -1;
+        _lastFullRange = NO;
+        _lastPresented = 0;
+        _inFlightSemaphore = dispatch_semaphore_create(MaxFramesInFlight);
+
+        CFStringRef keys[1] = {kCVMetalTextureUsage};
+        NSUInteger values[1] = {MTLTextureUsageShaderRead};
+        CFDictionaryRef cacheAttributes = CFDictionaryCreate(kCFAllocatorDefault, (const void **)keys, (const void **)values, 1, NULL, NULL);
+        CVMetalTextureCacheCreate(kCFAllocatorDefault, cacheAttributes, _device, NULL, &_textureCache);
+        CFRelease(cacheAttributes);
+
+        _renderPassDescriptor = [MTLRenderPassDescriptor new];
+        _renderPassDescriptor.colorAttachments[0].loadAction = MTLLoadActionClear;
+        _renderPassDescriptor.colorAttachments[0].clearColor = MTLClearColorMake(0, 0, 0, 0);
+        _renderPassDescriptor.colorAttachments[0].storeAction = MTLStoreActionStore;
+    }
+    return self;
+}
+
+#if !TARGET_OS_TV
+- (void)applyEDRFromFrame:(Frame *)frame toLayer:(CAMetalLayer *)layer {
+    CFDictionaryRef ext = [frame getFormatDescExtensions];
+
+    Log(LOG_I, @"ext: %@", ext);
+
+    CFDataRef masteringData = CFDictionaryGetValue(ext, kCMFormatDescriptionExtension_MasteringDisplayColorVolume);
+    CFDataRef contentDataRef = CFDictionaryGetValue(ext, kCMFormatDescriptionExtension_ContentLightLevelInfo);
+
+    if (masteringData) {
+        Log(LOG_I, @"ext MDCV %@", masteringData);
+    }
+    if (contentDataRef) {
+        Log(LOG_I, @"ext CLLI %@", contentDataRef);
+    }
+
+    if (masteringData && CFDataGetLength(masteringData) == 24 && contentDataRef && CFDataGetLength(contentDataRef) == 4) {
+        NSData *displayData = (__bridge NSData *)masteringData;
+        NSData *contentData = (__bridge NSData *)contentDataRef;
+
+        layer.wantsExtendedDynamicRangeContent = YES;
+        layer.pixelFormat = MTLPixelFormatRGBA16Float;
+        CFStringRef name = kCGColorSpaceExtendedLinearITUR_2020;
+        CGColorSpaceRef colorspace = CGColorSpaceCreateWithName(name);
+        layer.colorspace = colorspace;
+
+        layer.EDRMetadata = [CAEDRMetadata HDR10MetadataWithDisplayInfo:displayData contentInfo:contentData opticalOutputScale:100.0f];
+
+        Log(LOG_I, @"EDRMetadata set from MDCV %@ and CLLI %@", displayData, contentData);
+    } else {
+        layer.wantsExtendedDynamicRangeContent = YES;
+        layer.pixelFormat = MTLPixelFormatRGBA16Float;
+        CFStringRef name = kCGColorSpaceExtendedLinearITUR_2020;
+        CGColorSpaceRef colorspace = CGColorSpaceCreateWithName(name);
+        layer.colorspace = colorspace;
+
+        layer.EDRMetadata = [CAEDRMetadata HDR10MetadataWithMinLuminance:0.0005f maxLuminance:1000.0f opticalOutputScale:100.0f];
+
+        Log(LOG_I, @"EDRMetadata set for 1000 nits");
+    }
+}
+#endif
+
+- (int)getFrameColorspaceAndRange:(Frame *)frame isFullRange:(BOOL *)isFullRange {
+    CFDictionaryRef ext = [frame getFormatDescExtensions];
+
+    // FQLog(LOG_I, @"%@", ext);
+
+    // Full Range boolean
+    CFBooleanRef fullRangeRef = CFDictionaryGetValue(ext, kCMFormatDescriptionExtension_FullRangeVideo);
+    *isFullRange = NO;
+    if (fullRangeRef && CFGetTypeID(fullRangeRef) == CFBooleanGetTypeID()) {
+        *isFullRange = CFBooleanGetValue(fullRangeRef);
+    }
+
+    // Colorspace
+    CFStringRef frame_color = CFDictionaryGetValue(ext, kCVImageBufferColorPrimariesKey);
+    if (CFEqual(frame_color, kCVImageBufferColorPrimaries_ITU_R_709_2)) {
+        return COLORSPACE_REC_709;
+    } else if (CFEqual(frame_color, kCVImageBufferColorPrimaries_ITU_R_2020)) {
+        return COLORSPACE_REC_2020;
+    }
+    return COLORSPACE_REC_601;
+}
+
+- (BOOL)updateColorSpaceForFrame:(Frame *)frame toLayer:(CAMetalLayer *)layer layerDidChange:(BOOL *)layerDidChange {
+    BOOL fullRange = NO;
+    int colorspace = [self getFrameColorspaceAndRange:frame isFullRange:&fullRange];
+    if (colorspace != _lastColorSpace || fullRange != _lastFullRange) {
+        CGColorSpaceRef newColorSpace = nil;
+        MTLPixelFormat newPixelFormat = layer.pixelFormat;
+        BOOL isHDR = NO;
+        struct ParamBuffer paramBuffer;
+
+        switch (colorspace) {
+            case COLORSPACE_REC_709:
+                newColorSpace = CGColorSpaceCreateWithName(kCGColorSpaceITUR_709);
+                newPixelFormat = MTLPixelFormatBGRA8Unorm;
+                paramBuffer.cscParams = (fullRange ? k_CscParams_Bt709Full : k_CscParams_Bt709Lim);
+                break;
+            case COLORSPACE_REC_2020: {
+                CFDictionaryRef ext = [frame getFormatDescExtensions];
+                CFStringRef frame_trc = CFDictionaryGetValue(ext, kCVImageBufferTransferFunctionKey);
+                if (CFEqual(frame_trc, kCVImageBufferTransferFunction_SMPTE_ST_2084_PQ)) {
+                    isHDR = YES;
+                    newColorSpace = CGColorSpaceCreateWithName(kCGColorSpaceITUR_2100_PQ);
+                    newPixelFormat = MTLPixelFormatBGR10A2Unorm;
+                } else {
+                    // SDR 2020
+                    newColorSpace = CGColorSpaceCreateWithName(kCGColorSpaceITUR_2020);
+                    newPixelFormat = MTLPixelFormatBGR10A2Unorm;
+                }
+                paramBuffer.cscParams = (fullRange ? k_CscParams_Bt2020Full : k_CscParams_Bt2020Lim);
+                break;
+            }
+            case COLORSPACE_REC_601:
+                newColorSpace = CGColorSpaceCreateWithName(kCGColorSpaceSRGB);
+                newPixelFormat = MTLPixelFormatBGRA8Unorm;
+                paramBuffer.cscParams = (fullRange ? k_CscParams_Bt601Full : k_CscParams_Bt601Lim);
+        }
+
+        // The CAMetalLayer retains the CGColorSpace
+        if (newColorSpace || newPixelFormat != layer.pixelFormat) {
+            *layerDidChange = YES;
+            if (newColorSpace) {
+                Log(LOG_I,
+                    @"Frame colorspace %@ - changing MetalLayer's colorspace to %@",
+                    colorspace == COLORSPACE_REC_709        ? @"REC_709"
+                        : colorspace == COLORSPACE_REC_2020 ? @"REC_2020"
+                        : colorspace == COLORSPACE_REC_601  ? @"REC_601 (sRGB)"
+                                                            : [NSString stringWithFormat:@"Unknown: %d", colorspace],
+                    newColorSpace);
+            }
+            if (newPixelFormat != layer.pixelFormat) {
+                Log(LOG_I,
+                    @"Frame pixel format %@ - changing MetalLayer's pixel format to %@",
+                    layer.pixelFormat == MTLPixelFormatBGRA8Unorm         ? @"MTLPixelFormatBGRA8Unorm"
+                        : layer.pixelFormat == MTLPixelFormatBGR10A2Unorm ? @"MTLPixelFormatBGR10A2Unorm"
+                                                                          : [NSString stringWithFormat:@"Unknown: %lu", layer.pixelFormat],
+                    newPixelFormat == MTLPixelFormatBGRA8Unorm         ? @"MTLPixelFormatBGRA8Unorm"
+                        : newPixelFormat == MTLPixelFormatBGR10A2Unorm ? @"MTLPixelFormatBGR10A2Unorm"
+                                                                       : [NSString stringWithFormat:@"Unknown: %lu", (unsigned long)layer.pixelFormat]);
+            }
+#if !RENDER_ON_MAIN_THREAD
+            // These can only be changed on the main thread
+            dispatch_sync(dispatch_get_main_queue(), ^{
+#endif
+#if !TARGET_OS_TV
+                if (isHDR) {
+                    layer.wantsExtendedDynamicRangeContent = YES;
+                }
+#endif
+                layer.colorspace = newColorSpace;
+                layer.pixelFormat = newPixelFormat;
+#if !RENDER_ON_MAIN_THREAD
+            });
+#endif
+            CGColorSpaceRelease(newColorSpace);
+        }
+
+        // Create the new colorspace parameter buffer for our fragment shader
+        MTLResourceOptions bufferOptions = MTLResourceStorageModeShared;
+        _CscParamsBuffer = [_device newBufferWithBytes:(void *)&paramBuffer length:sizeof(paramBuffer) options:bufferOptions];
+        if (!_CscParamsBuffer) {
+            Log(LOG_E, @"Failed to create CSC parameters buffer");
+            return NO;
+        }
+
+        _lastColorSpace = colorspace;
+        _lastFullRange = fullRange;
+    }
+
+    return YES;
+}
+
+- (void)scaleSource:(CGRect *)src toDest:(CGRect *)dst {
+    int dstH = ceilf((float)dst->size.width * src->size.height / src->size.width);
+    int dstW = ceilf((float)dst->size.height * src->size.width / src->size.height);
+
+    if (dstH > dst->size.height) {
+        dst->origin.x += (dst->size.width - dstW) / 2;
+        dst->size.width = dstW;
+    } else {
+        dst->origin.y += (dst->size.height - dstH) / 2;
+        dst->size.height = dstH;
+    }
+}
+
+- (void)screenSpace:(CGRect *)src toNormalizedDeviceCoords:(CGRect *)dst withDrawableWidth:(int)viewportWidth drawableHeight:(int)viewportHeight {
+    dst->origin.x = ((float)src->origin.x / (viewportWidth / 2.0f)) - 1.0f;
+    dst->origin.y = ((float)src->origin.y / (viewportHeight / 2.0f)) - 1.0f;
+    dst->size.width = (float)src->size.width / (viewportWidth / 2.0f);
+    dst->size.height = (float)src->size.height / (viewportHeight / 2.0f);
+}
+
+- (BOOL)updateVideoRegionSizeForFrame:(Frame *)frame toLayer:(CAMetalLayer *)layer {
+    int drawableWidth = layer.drawableSize.width;
+    int drawableHeight = layer.drawableSize.height;
+
+    // Check if anything has changed since the last vertex buffer upload
+    if (_VideoVertexBuffer && [frame width] == _lastFrameWidth && [frame height] == _lastFrameHeight && drawableWidth == _lastDrawableWidth &&
+        drawableHeight == _lastDrawableHeight) {
+        // Nothing to do
+        return YES;
+    }
+
+    // Determine the correct scaled size for the video region
+    CGRect src = CGRectMake(0.0, 0.0, [frame width], [frame height]);
+    CGRect dst = CGRectMake(0.0, 0.0, drawableWidth, drawableHeight);
+    [self scaleSource:&src toDest:&dst];
+
+    // Convert screen space to normalized device coordinates
+    CGRect renderRect;
+    [self screenSpace:&dst toNormalizedDeviceCoords:&renderRect withDrawableWidth:drawableWidth drawableHeight:drawableHeight];
+
+    struct Vertex verts[] = {
+        {{renderRect.origin.x, renderRect.origin.y, 0.0f, 1.0f}, {0.0f, 1.0f}},
+        {{renderRect.origin.x, renderRect.origin.y + renderRect.size.height, 0.0f, 1.0f}, {0.0f, 0}},
+        {{renderRect.origin.x + renderRect.size.width, renderRect.origin.y, 0.0f, 1.0f}, {1.0f, 1.0f}},
+        {{renderRect.origin.x + renderRect.size.width, renderRect.origin.y + renderRect.size.height, 0.0f, 1.0f}, {1.0f, 0}},
+    };
+
+    MTLResourceOptions bufferOptions = MTLResourceStorageModeShared;
+    _VideoVertexBuffer = [_device newBufferWithBytes:verts length:sizeof(verts) options:bufferOptions];
+    if (!_VideoVertexBuffer) {
+        Log(LOG_E, @"Failed to create video vertex buffer");
+        return NO;
+    }
+
+    _lastFrameWidth = [frame width];
+    _lastFrameHeight = [frame height];
+    _lastDrawableWidth = drawableWidth;
+    _lastDrawableHeight = drawableHeight;
+
+    return YES;
+}
+
+- (void)discardNextDrawable {
+    _nextDrawable = nil;
+}
+
+- (void)renderFrame:(Frame *)frame toLayer:(CAMetalLayer *)layer {
+    // Handle changes to the frame's colorspace from last time we rendered
+    BOOL layerDidChange = NO;
+    if (![self updateColorSpaceForFrame:frame toLayer:layer layerDidChange:&layerDidChange]) {
+        return;
+    }
+
+    if (layerDidChange && frame.frameNumber > 1) {
+        Log(LOG_I, @"Metal frame changed layer's colorspace and/or pixel format, returning for new drawable");
+        [self discardNextDrawable];
+        return;
+    }
+
+    // Handle changes to the video size or drawable size
+    if (![self updateVideoRegionSizeForFrame:frame toLayer:layer]) {
+        return;
+    }
+
+    FQLog(LOG_I, @"[%d / %.3f ms] Metal frame rendering", frame.frameNumber, frame.pts);
+
+#if !TARGET_OS_TV
+    // Experimental EDR handling based on frame metadata
+    //[self applyEDRFromFrame:frame toLayer:layer];
+#endif
+
+    size_t planes = CVPixelBufferGetPlaneCount(frame.pixelBuffer);
+    assert(planes == 2 || planes == 3);
+
+    MTLRenderPipelineDescriptor *pipelineDesc = [MTLRenderPipelineDescriptor new];
+    id<MTLLibrary> defaultLibrary = [_device newDefaultLibrary];
+    pipelineDesc.vertexFunction = [defaultLibrary newFunctionWithName:@"vs_draw"];
+    pipelineDesc.fragmentFunction = [defaultLibrary newFunctionWithName:planes == 2 ? @"ps_draw_biplanar" : @"ps_draw_triplanar"];
+    pipelineDesc.colorAttachments[0].pixelFormat = layer.pixelFormat;
+    pipelineDesc.vertexBuffers[0].mutability = MTLMutabilityImmutable;
+
+    NSError *error = nil;
+    _videoPipelineState = [_device newRenderPipelineStateWithDescriptor:pipelineDesc error:&error];
+    if (!_videoPipelineState) {
+        Log(LOG_E, @"Failed to create video pipeline state: %@", error);
+        return;
+    }
+
+    for (size_t i = 0; i < planes; i++) {
+        MTLPixelFormat fmt;
+
+        switch (CVPixelBufferGetPixelFormatType(frame.pixelBuffer)) {
+            case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange:
+            case kCVPixelFormatType_444YpCbCr8BiPlanarVideoRange:
+            case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange:
+            case kCVPixelFormatType_444YpCbCr8BiPlanarFullRange:
+                fmt = (i == 0) ? MTLPixelFormatR8Unorm : MTLPixelFormatRG8Unorm;
+                break;
+
+            case kCVPixelFormatType_420YpCbCr10BiPlanarFullRange:
+            case kCVPixelFormatType_444YpCbCr10BiPlanarFullRange:
+            case kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange:
+            case kCVPixelFormatType_444YpCbCr10BiPlanarVideoRange:
+                fmt = (i == 0) ? MTLPixelFormatR16Unorm : MTLPixelFormatRG16Unorm;
+                break;
+
+            default:
+                Log(LOG_E, @"Unknown pixel format: %@", CVPixelBufferGetPixelFormatType(frame.pixelBuffer));
+                return;
+        }
+
+        if (_cvMetalTextures[i]) {
+            CVBufferRelease(_cvMetalTextures[i]);
+        }
+        CVReturn err = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault,
+                                                                 _textureCache,
+                                                                 frame.pixelBuffer,
+                                                                 NULL,
+                                                                 fmt,
+                                                                 CVPixelBufferGetWidthOfPlane(frame.pixelBuffer, i),
+                                                                 CVPixelBufferGetHeightOfPlane(frame.pixelBuffer, i),
+                                                                 i,
+                                                                 &_cvMetalTextures[i]);
+        if (err != kCVReturnSuccess) {
+            Log(LOG_E, @"CVMetalTextureCacheCreateTextureFromImage() failed: %d", err);
+            return;
+        }
+    }
+
+    _renderPassDescriptor.colorAttachments[0].texture = _nextDrawable.texture;
+
+    id<MTLCommandBuffer> commandBuffer = [_commandQueue commandBuffer];
+    id<MTLRenderCommandEncoder> renderEncoder = [commandBuffer renderCommandEncoderWithDescriptor:_renderPassDescriptor];
+
+    [renderEncoder setRenderPipelineState:_videoPipelineState];
+    for (size_t i = 0; i < planes; i++) {
+        [renderEncoder setFragmentTexture:CVMetalTextureGetTexture(_cvMetalTextures[i]) atIndex:i];
+    }
+    [commandBuffer addCompletedHandler:^(id<MTLCommandBuffer> cb) {
+        const CFTimeInterval GPUTime = cb.GPUEndTime - cb.GPUStartTime;
+        const double alpha = 0.25f;
+        self->_averageGPUTime = (GPUTime * alpha) + (self->_averageGPUTime * (1.0 - alpha));
+
+        // Free textures after completion of rendering per CVMetalTextureCache requirements
+        for (size_t i = 0; i < planes; i++) {
+            CVBufferRelease(self->_cvMetalTextures[i]);
+            self->_cvMetalTextures[i] = nil;
+        }
+
+        CVMetalTextureCacheFlush(self->_textureCache, 0);
+    }];
+
+    [renderEncoder setFragmentBuffer:_CscParamsBuffer offset:0 atIndex:0];
+    [renderEncoder setVertexBuffer:_VideoVertexBuffer offset:0 atIndex:0];
+    [renderEncoder drawPrimitives:MTLPrimitiveTypeTriangleStrip vertexStart:0 vertexCount:4];
+    [renderEncoder endEncoding];
+
+    __block dispatch_semaphore_t block_semaphore = _inFlightSemaphore;
+    [commandBuffer addCompletedHandler:^(id<MTLCommandBuffer> buffer) {
+        dispatch_semaphore_signal(block_semaphore);
+    }];
+
+    __weak typeof(self) self_ = self;
+    [_nextDrawable addPresentedHandler:^(id<MTLDrawable> d) {
+        if (self_) {
+            [self_ plotFrametime:d.presentedTime];
+        }
+    }];
+
+#if TARGET_OS_SIMULATOR
+    [commandBuffer presentDrawable:_nextDrawable];
+#else
+    // present for a minimum duration for best frame pacing
+    [commandBuffer presentDrawable:_nextDrawable afterMinimumDuration:1.0f / _framerate];
+#endif
+
+    [commandBuffer commit];
+
+    // Wait for the command buffer to complete and free our CVMetalTextureCache references
+    [commandBuffer waitUntilCompleted];
+
+    _nextDrawable = nil;
+}
+
+- (void)plotFrametime:(CFTimeInterval)presentedTime {
+    if (_lastPresented > 0) {
+        CFTimeInterval frametime = presentedTime - _lastPresented;
+        [[ImGuiPlots sharedInstance] observeFloat:PLOT_FRAMETIME value:(frametime * 1000.0)];
+    }
+    _lastPresented = presentedTime;
+}
+
+- (void)waitToRenderTo:(nonnull CAMetalLayer *)layer {
+    if (!_nextDrawable) {
+        // Wait for the next available drawable
+        _nextDrawable = [layer nextDrawable];
+        if (!_nextDrawable) {
+            Log(LOG_E, @"Error getting nextDrawable from CAMetalLayer");
+            return;
+        }
+
+        // Wait to ensure only `MaxFramesInFlight` number of frames are getting processed
+        // by any stage in the Metal pipeline (CPU, GPU, Metal, Drivers, etc.).
+        dispatch_semaphore_wait(_inFlightSemaphore, DISPATCH_TIME_FOREVER);
+    }
+}
+
+/// Responds to the drawable's size or orientation changes.
+- (void)drawableResize:(CGSize)drawableSize {
+    [self resize:drawableSize];
+}
+
+- (void)resize:(CGSize)size {
+    // TODO?
+}
+
+@end

VoidLink/Metal/MetalView.h

@@ -0,0 +1,45 @@
+//
+//  MetalView.h
+//  VoidLink
+//
+//  Created by Mason Chen on 2025/7/11.
+//  Copyright © 2025 Moonlight Game Streaming Project. All rights reserved.
+//
+
+
+#import <Metal/Metal.h>
+#import <QuartzCore/CAMetalDisplayLink.h>
+#import <QuartzCore/CAMetalLayer.h>
+#import "MetalConfig.h"
+
+#if TARGET_OS_IOS || TARGET_OS_TV
+#import <UIKit/UIKit.h>
+#define PlatformView UIView
+#else
+#import <AppKit/AppKit.h>
+#define PlatformView NSView
+#endif
+
+// The protocol to provide resize and redraw callbacks to a delegate.
+@protocol MetalViewDelegate <NSObject>
+
+- (void)drawableResize:(CGSize)size;
+- (void)renderTo:(nonnull CAMetalLayer *)layer;
+- (void)waitToRenderTo:(nonnull CAMetalLayer *)layer;
+
+@end
+
+// The Metal game view base class.
+@interface MetalView : PlatformView <CALayerDelegate>
+
+@property (nonatomic, nonnull, readonly) CAMetalLayer *metalLayer;
+@property (nonatomic, getter=isPaused) BOOL paused;
+@property (nonatomic, nullable) id<MetalViewDelegate> delegate;
+@property (nonatomic) float framerate;
+
+- (void)initCommon;
+#if AUTOMATICALLY_RESIZE
+- (void)resizeDrawable:(CGFloat)scaleFactor;
+#endif
+
+@end

VoidLink/Metal/MetalView.m

@@ -0,0 +1,214 @@
+//
+//  MetalView.m
+//  VoidLink
+//
+//  Created by Mason Chen on 2025/7/11.
+//  Copyright © 2025 Moonlight Game Streaming Project. All rights reserved.
+//
+
+
+// This is based on the following Apple example:
+// https://developer.apple.com/documentation/metal/achieving-smooth-frame-rates-with-a-metal-display-link?language=objc
+// https://developer.apple.com/wwdc23/10123/
+
+#import "MetalView.h"
+#import "MetalConfig.h"
+
+@implementation MetalView {
+#if !RENDER_ON_MAIN_THREAD
+    // The secondary thread containing the render loop.
+    NSThread *_renderThread;
+
+    // The flag to indicate that rendering needs to cease on the main thread.
+    BOOL _continueRunLoop;
+#endif
+}
+
+#pragma mark - Initialization and Setup.
+
+- (instancetype)initWithFrame:(CGRect)frame {
+    self = [super initWithFrame:frame];
+    if (self) {
+        [self initCommon];
+    }
+    return self;
+}
+
+- (instancetype)initWithCoder:(NSCoder *)aDecoder {
+    self = [super initWithCoder:aDecoder];
+    if (self) {
+        [self initCommon];
+    }
+    return self;
+}
+
+- (void)initCommon {
+#if TARGET_OS_OSX
+    self.wantsLayer = YES;
+
+    self.layerContentsRedrawPolicy = NSViewLayerContentsRedrawDuringViewResize;
+#endif
+
+    _metalLayer = (CAMetalLayer *)self.layer;
+
+    self.layer.delegate = self;
+}
+
+#if TARGET_OS_IOS || TARGET_OS_TV
++ (Class)layerClass {
+    return [CAMetalLayer class];
+}
+
+- (void)didMoveToWindow {
+    [self movedToWindow];
+}
+#else
+- (CALayer *)makeBackingLayer {
+    return [CAMetalLayer layer];
+}
+
+- (void)viewDidMoveToWindow {
+    [self movedToWindow];
+}
+#endif  // END TARGET_OS_IOS || TARGET_OS_TV
+
+- (void)movedToWindow {
+#if !RENDER_ON_MAIN_THREAD
+    // Protect _continueRunLoop with a `@synchronized` block because it's accessed by the separate
+    // animation thread.
+    @synchronized(self) {
+        // Stop the animation loop, allowing it to complete if it's in progress.
+        _continueRunLoop = NO;
+    }
+
+    // Create and start a secondary NSThread that has another run runloop. The NSThread
+    // class calls the 'runThread' method at the start of the secondary thread's execution.
+    _renderThread = [[NSThread alloc] initWithTarget:self selector:@selector(runThread) object:nil];
+    _continueRunLoop = YES;
+    _renderThread.qualityOfService = NSQualityOfServiceUserInteractive;
+    [_renderThread start];
+#endif  // END !RENDER_ON_MAIN_THREAD
+
+    // Perform any actions that need to know the size and scale of the drawable. When UIKit calls
+    // didMoveToWindow after the view initialization, this is the first opportunity to notify
+    // components of the drawable's size.
+#if AUTOMATICALLY_RESIZE
+#if TARGET_OS_IOS || TARGET_OS_TV
+    [self resizeDrawable:self.window.screen.nativeScale];
+#else
+    [self resizeDrawable:self.window.screen.backingScaleFactor];
+#endif
+#else
+    // Notify the delegate of the default drawable size when the system can calculate it.
+    CGSize defaultDrawableSize = self.bounds.size;
+    defaultDrawableSize.width *= self.layer.contentsScale;
+    defaultDrawableSize.height *= self.layer.contentsScale;
+    [self.delegate drawableResize:defaultDrawableSize];
+#endif
+}
+
+#if !RENDER_ON_MAIN_THREAD
+- (void)runThread {
+    // The system sets the '_continueRunLoop' ivar outside this thread, so it needs to synchronize. Create a
+    // 'continueRunLoop' local var that the system can set from the _continueRunLoop ivar in a @synchronized block.
+    BOOL continueRunLoop = YES;
+
+    // Begin the run loop.
+    while (continueRunLoop) {
+        @autoreleasepool {
+            [_delegate waitToRenderTo:_metalLayer];
+
+            @synchronized(self) {
+                continueRunLoop = _continueRunLoop;
+            }
+            if (!continueRunLoop) {
+                break;
+            }
+
+            [_delegate renderTo:_metalLayer];
+
+            @synchronized(self) {
+                continueRunLoop = _continueRunLoop;
+            }
+        }
+    }
+}
+#endif  // END !RENDER_ON_MAIN_THREAD
+
+#pragma mark - Resizing
+
+#if AUTOMATICALLY_RESIZE
+
+// Override all methods that indicate the view's size has changed.
+
+#if TARGET_OS_IOS || TARGET_OS_TV
+- (void)setContentScaleFactor:(CGFloat)contentScaleFactor {
+    [super setContentScaleFactor:contentScaleFactor];
+    [self resizeDrawable:self.window.screen.nativeScale];
+}
+
+- (void)layoutSubviews {
+    [super layoutSubviews];
+    [self resizeDrawable:self.window.screen.nativeScale];
+}
+
+- (void)setFrame:(CGRect)frame {
+    [super setFrame:frame];
+    [self resizeDrawable:self.window.screen.nativeScale];
+}
+
+- (void)setBounds:(CGRect)bounds {
+    [super setBounds:bounds];
+    [self resizeDrawable:self.window.screen.nativeScale];
+}
+#else
+- (void)viewDidChangeBackingProperties {
+    [super viewDidChangeBackingProperties];
+    [self resizeDrawable:self.window.screen.backingScaleFactor];
+}
+
+- (void)setFrameSize:(NSSize)size {
+    [super setFrameSize:size];
+    [self resizeDrawable:self.window.screen.backingScaleFactor];
+}
+
+- (void)setBoundsSize:(NSSize)size {
+    [super setBoundsSize:size];
+    [self resizeDrawable:self.window.screen.backingScaleFactor];
+}
+#endif
+
+- (void)resizeDrawable:(CGFloat)scaleFactor {
+    CGSize newSize = self.bounds.size;
+    newSize.width *= scaleFactor;
+    newSize.height *= scaleFactor;
+
+    if (newSize.width <= 0 || newSize.width <= 0) {
+        return;
+    }
+
+#if RENDER_ON_MAIN_THREAD
+    if (newSize.width == _metalLayer.drawableSize.width && newSize.height == _metalLayer.drawableSize.height) {
+        return;
+    }
+
+    _metalLayer.drawableSize = newSize;
+
+    [_delegate drawableResize:newSize];
+#else
+    // The system calls all AppKit and UIKit calls that notify of a resize on the main thread. Use
+    // a synchronized block to ensure that resize notifications on the delegate are atomic.
+    @synchronized(_metalLayer) {
+        if (newSize.width == _metalLayer.drawableSize.width && newSize.height == _metalLayer.drawableSize.height) {
+            return;
+        }
+
+        _metalLayer.drawableSize = newSize;
+
+        [_delegate drawableResize:newSize];
+    }
+#endif
+}
+#endif  // END AUTOMATICALLY_RESIZE
+
+@end

VoidLink/Metal/MetalViewController.h

@@ -0,0 +1,33 @@
+//
+//  MetalViewController.h
+//  VoidLink
+//
+//  Created by Mason Chen on 2025/7/11.
+//  Copyright © 2025 Moonlight Game Streaming Project. All rights reserved.
+//
+
+
+#import <Metal/Metal.h>
+#import "FrameQueue.h"
+#import "ImGuiRenderer.h"
+#import "MetalVideoRenderer.h"
+#import "MetalView.h"
+
+#if TARGET_OS_IOS || TARGET_OS_TV
+#import <UIKit/UIKit.h>
+#define PlatformViewController UIViewController
+#else
+#import <AppKit/AppKit.h>
+#define PlatformViewController NSViewController
+#endif
+
+@interface MetalViewController : PlatformViewController <MetalViewDelegate>
+
+@property (nonatomic) CGRect bounds;
+
+- (nonnull instancetype)initWithFrame:(CGRect)bounds
+                            framerate:(float)framerate
+                            enableHdr:(BOOL)enableHdr
+                       metricsHandler:(MetricsHandler _Nonnull)metricsHandler;
+
+@end

VoidLink/Metal/MetalViewController.m

@@ -0,0 +1,153 @@
+//
+//  MetalViewController.m
+//  VoidLink
+//
+//  Created by Mason Chen on 2025/7/11.
+//  Copyright © 2025 Moonlight Game Streaming Project. All rights reserved.
+//
+
+
+/*
+See the LICENSE.txt file for this sample’s licensing information.
+
+Abstract:
+The implementation of the cross-platform game view controller.
+*/
+
+#import "MetalViewController.h"
+#import "FrameQueue.h"
+#import "ImGuiRenderer.h"
+#import "MetalVideoRenderer.h"
+
+@implementation MetalViewController {
+    /// A queue to initialize the renderer asynchronously from the main thread.
+    dispatch_queue_t _dispatch_queue;
+    FrameQueue *_frameQueue;
+    float _framerate;
+    BOOL _enableHdr;
+    MetalView *_metalView;
+    MetalVideoRenderer *_renderer;
+    MetricsHandler _metricsHandler;
+    BOOL _stopping;
+}
+
+- (nonnull instancetype)initWithFrame:(CGRect)bounds framerate:(float)framerate enableHdr:(BOOL)enableHdr metricsHandler:(MetricsHandler)metricsHandler {
+    self = [super init];
+    if (self) {
+        _bounds = bounds;
+        _frameQueue = [FrameQueue sharedInstance];
+        _framerate = framerate;
+        _enableHdr = enableHdr;
+        _metricsHandler = metricsHandler;
+        _stopping = NO;
+    }
+    return self;
+}
+
+- (void)loadView {
+    self.view = [[MetalView alloc] initWithFrame:_bounds];
+}
+
+- (void)viewDidLoad {
+    [super viewDidLoad];
+
+    /// A queue to initialize the renderer asynchronously from the main thread.
+    _dispatch_queue = dispatch_queue_create("com.moonlight.Metal", DISPATCH_QUEUE_CONCURRENT);
+
+    __block MetalView *view = (MetalView *)self.view;
+    if (!view) {
+        Log(LOG_E, @"The view attached to MetalViewController isn't a MetalView.");
+        return;
+    }
+    _metalView = view;
+    _metalView.delegate = self;
+    _metalView.framerate = _framerate;
+
+    // Select the device to render with.
+    id<MTLDevice> device = MTLCreateSystemDefaultDevice();
+    if (!device) {
+        Log(LOG_E, @"Metal isn't supported on this device.");
+        self.view = [[PlatformView alloc] initWithFrame:self.view.frame];
+        return;
+    }
+    view.metalLayer.device = device;
+
+    // Initialize the renderer.
+    MetalVideoRenderer *renderer = [[MetalVideoRenderer alloc] initWithMetalDevice:device
+                                                               drawablePixelFormat:MTLPixelFormatBGR10A2Unorm
+                                                                         framerate:self->_framerate];
+    if (!renderer) {
+        Log(LOG_E, @"The renderer couldn't be initialized.");
+        return;
+    }
+
+    // Initialize the renderer-dependent view properties.
+    view.metalLayer.pixelFormat = renderer.colorPixelFormat;
+    view.metalLayer.colorspace = renderer.colorspace;
+    view.metalLayer.maximumDrawableCount = 3;
+
+    self->_renderer = renderer;
+}
+
+- (void)waitToRenderTo:(nonnull CAMetalLayer *)layer {
+    if (!_stopping) {
+        // Renderer obtains a nextDrawable, waiting if necessary
+        FQLog(LOG_I, @"[MetalViewController] caling [_renderer waitToRenderTo:layer]");
+        [_renderer waitToRenderTo:layer];
+
+        // If we don't have a frame yet, wait on that too
+        FQLog(LOG_I, @"[MetalViewController] caling [_renderer waitForEnqueue]");
+        [_frameQueue waitForEnqueue];
+    }
+}
+
+/// Draw frame (used by manual loop)
+- (void)renderTo:(nonnull CAMetalLayer *)layer {
+    if (!_renderer) {
+        return;
+    }
+
+    CFTimeInterval timeout = (1.0f / _framerate) - _renderer.averageGPUTime;
+    FQLog(LOG_I, @"[MetalViewController] caling [_frameQueue dequeueWithTimeout:%f]", timeout);
+    Frame *frame = [_frameQueue dequeueWithTimeout:timeout];
+    if (frame) {
+        FQLog(LOG_I, @"[MetalViewController] calling [_renderer renderFrame] %@", frame);
+        [_renderer renderFrame:frame toLayer:layer];
+    }
+}
+
+- (void)drawableResize:(CGSize)size {
+    [_renderer drawableResize:size];
+}
+
+#if TARGET_OS_IOS
+/// Hides the Home indicator button automatically.
+- (BOOL)prefersHomeIndicatorAutoHidden {
+    return YES;
+}
+#endif
+
+#if TARGET_OS_OSX
+/// Makes the view controller the first responder to receive keyboard events.
+- (void)viewDidAppear {
+    [_metalView.window makeFirstResponder:self];
+}
+
+/// Receives the keydown events to avoid system beeps.
+///
+/// The `GameInputKeyboardMouse` class handles keyboard events.
+- (void)keyDown:(NSEvent *)event {
+    // Reference the parameter to avoid an unused parameter warning.
+    (void)(event);
+}
+
+/// Receives the keyup events to avoid system beeps.
+///
+/// The `GameInputKeyboardMouse` class handles keyboard events.
+- (void)keyUp:(NSEvent *)event {
+    // Reference the parameter to avoid an unused parameter warning.
+    (void)(event);
+}
+#endif
+
+@end

VoidLink/Metal/Shaders.metal

@@ -0,0 +1,72 @@
+//
+//  Vertex.swift
+//  VoidLink
+//
+//  Created by Mason Chen on 2025/7/11.
+//  Copyright © 2025 Moonlight Game Streaming Project. All rights reserved.
+//
+
+
+#include <metal_stdlib>
+#include <simd/simd.h>
+
+using namespace metal;
+
+struct Vertex
+{
+    float4 position [[ position ]];
+    float2 texCoords;
+};
+
+struct CscParams
+{
+    float3 matrix[3];
+    float3 offsets;
+};
+
+constexpr sampler s(coord::normalized, address::clamp_to_edge, filter::linear);
+
+vertex Vertex vs_draw(constant Vertex *vertices [[ buffer(0) ]], uint id [[ vertex_id ]])
+{
+    return vertices[id];
+}
+
+fragment float4 ps_draw_biplanar(Vertex v [[ stage_in ]],
+                                 constant CscParams &cscParams [[ buffer(0) ]],
+                                 texture2d<float> luminancePlane [[ texture(0) ]],
+                                 texture2d<float> chrominancePlane [[ texture(1) ]])
+{
+    float3 yuv = float3(luminancePlane.sample(s, v.texCoords).r,
+                        chrominancePlane.sample(s, v.texCoords).rg);
+    yuv -= cscParams.offsets;
+
+    float3 rgb;
+    rgb.r = dot(yuv, cscParams.matrix[0]);
+    rgb.g = dot(yuv, cscParams.matrix[1]);
+    rgb.b = dot(yuv, cscParams.matrix[2]);
+    return float4(rgb, 1.0f);
+}
+
+fragment float4 ps_draw_triplanar(Vertex v [[ stage_in ]],
+                                  constant CscParams &cscParams [[ buffer(0) ]],
+                                  texture2d<float> luminancePlane [[ texture(0) ]],
+                                  texture2d<float> chrominancePlaneU [[ texture(1) ]],
+                                  texture2d<float> chrominancePlaneV [[ texture(2) ]])
+{
+    float3 yuv = float3(luminancePlane.sample(s, v.texCoords).r,
+                        chrominancePlaneU.sample(s, v.texCoords).r,
+                        chrominancePlaneV.sample(s, v.texCoords).r);
+    yuv -= cscParams.offsets;
+
+    float3 rgb;
+    rgb.r = dot(yuv, cscParams.matrix[0]);
+    rgb.g = dot(yuv, cscParams.matrix[1]);
+    rgb.b = dot(yuv, cscParams.matrix[2]);
+    return float4(rgb, 1.0f);
+}
+
+fragment float4 ps_draw_rgb(Vertex v [[ stage_in ]],
+                            texture2d<float> rgbTexture [[ texture(0) ]])
+{
+    return rgbTexture.sample(s, v.texCoords);
+}