Fix thread sync

Reformat Metal files

Rewrite Metal render loop to not use DisplayLink so we can use the amazing afterMinimumDuration API call

Improve frame drop algorithm

Misc fixes to HDR <-> SDR switching, deprecation warning cleanup

Fix AVSB mode

Fix renderingBackend selected value

Add Rendering Backend setting to iPhone

Let users choose which renderer to use. Refactor metrics into a singleton

Fix tvOS build

Port moonlight-qt's Metal renderer

Bump imgui to latest

Misc tweaks and refactoring

Switch iOS to use Opus float32 audio, in line with other clients.  It was already being sent from the server as float32, so this is simply decoding it with less loss of dynamic range.

Give FrameQueue a faster data structure and full test coverage. Remove pts frame pacing, it's not a fit for iOS.

Don't allow very transparent opacity for middle stats overlay box

Change wording to select button instead of remote button

Add tvOS gesture for double-clicking remote button to show/hide stats

Remove kVTDecodeFrame_1xRealTimePlayback, this was found to seriously impact decode times

Move decode time stat out of async block

Enabe FQ logging

Misc cleanup/fixes

iPad: 26 will require UIRequiresFullScreen to be false, and for all orientations to be supported

Updated iPhone and iPad settings panels

Add setting for graph opacity. Enable stats and graphs to be toggled via two finger swipe

Fix wait time in displaylink, fix settings spacing, stats fixes

Settings for frame queue size, graphs. Misc refactoring and cleanup

Bump moonlight-common-c

Improve logging readability

Fix host frametime and frame queue stats to be recorded during input not output. Show min/max/avg decode time

Add bytes per frame graph

Started on settings changes, both modes working very well

Misc refactoring, move graphs around, both pacing functions working well

Set frameDropTarget to 2 always

Use raw RTP timestamps without converting to floating point for even more accurate playback

Standard frame pacing: remove +3 logic

Switch to NSRunLoopCommonModes for displayLink

Add frame pacing algorithm based on moonlight-qt's Pacer class as a simpler baseline

Switch to os_unfair_lock, misc tweaks

Large refactor that now uses VTDecompressionSession based on PR535 from felipejfc. This now allows more control over queue size and allows dropping arbitrary frames, but introduces a lot of new problems

bump moonlight-common-c to fix 32-bit rollover issue

LiWait timing graph

Add displaylink interval graph

Graphs checkpoint

ImGui graph working checkpoint

ImGui working checkpoint

Add pending frames to stats, add a define for verbose logging

DisplayLink WIP

Add imgui & implot modules

Frame pacing: alternate method based on thieman's pullFrames thread

Frame pacing: simplify displayLink, support passing targetTimestamp down to the sample buffer

Point to my common-c with some timing changes.

Add stats gestures: 2-finger swipe down to show stats, and 2-finger swipe up to hide

Limit refresh rate to 60 when in low power mode

Display screen refresh rate in stats overlay, remove decode time and explain why it can't be measured

Adjust the refresh rate of the display to match the stream framerate and improve battery life

Don't use presentationTimeMs when creating CMSampleTimingInfo

Fix overlayView build for iOS

Refactor CADisplayLink callback for improved frame delivery. Support a user-configurable buffered frame count.

[opt] Use UILabel for stats overlay

Limelight/Metal/MetalConfig.h

@@ -0,0 +1,13 @@
+// When enabled, rendering occurs on the main application thread.
+// This can make responding to UI events during redraw simpler
+// to manage because UI calls usually need to occur on the main thread.
+// When disabled, rendering occurs on a background thread, allowing
+// the UI to respond more quickly in some cases because events can
+// process asynchronously from potentially CPU-intensive rendering code.
+#define RENDER_ON_MAIN_THREAD 0
+
+// When enabled, the drawable's size updates automatically whenever
+// the view resizes. When disabled, you can update the drawable's
+// size explicitly outside the view class.
+#define AUTOMATICALLY_RESIZE 1
+

Limelight/Metal/MetalVideoRenderer.h

@@ -0,0 +1,23 @@
+#import <QuartzCore/CAMetalLayer.h>
+#import <QuartzCore/QuartzCore.h>
+#import "ConnectionCallbacks.h"
+#import "Frame.h"
+#import "Plot.h"
+
+@interface MetalVideoRenderer : NSObject
+
+@property (atomic) CFTimeInterval averageGPUTime;
+@property (nonatomic) NSUInteger sampleCount;
+@property (nonatomic) MTLPixelFormat colorPixelFormat;
+@property (nonatomic, nonnull) CGColorSpaceRef colorspace;
+@property (nonatomic) id<CAMetalDrawable> _Nullable nextDrawable;
+
+- (nonnull instancetype)initWithMetalDevice:(nonnull id<MTLDevice>)device
+                        drawablePixelFormat:(MTLPixelFormat)drawablePixelFormat
+                                  framerate:(float)framerate;
+- (void)renderFrame:(nonnull Frame *)frame toLayer:(nonnull CAMetalLayer *)layer;
+- (void)waitToRenderTo:(nonnull CAMetalLayer *)layer;
+- (void)drawableResize:(CGSize)drawableSize;
+- (void)plotFrametime:(CFTimeInterval)presentedTime;
+
+@end

Limelight/Metal/MetalVideoRenderer.m

@@ -0,0 +1,524 @@
+#import "MetalVideoRenderer.h"
+#import <CoreVideo/CoreVideo.h>
+#import <Metal/Metal.h>
+#import <MetalKit/MetalKit.h>
+#import "ImGuiPlots.h"
+
+#include <Limelight.h>
+
+#define MAX_VIDEO_PLANES 3
+
+struct CscParams {
+    vector_float3 matrix[3];
+    vector_float3 offsets;
+};
+
+struct ParamBuffer {
+    struct CscParams cscParams;
+};
+
+static const struct CscParams k_CscParams_Bt601Lim = {
+    // CSC Matrix
+    {{1.1644f, 0.0f, 1.5960f}, {1.1644f, -0.3917f, -0.8129f}, {1.1644f, 2.0172f, 0.0f}},
+
+    // Offsets
+    {16.0f / 255.0f, 128.0f / 255.0f, 128.0f / 255.0f},
+};
+static const struct CscParams k_CscParams_Bt601Full = {
+    {
+        {1.0f, 0.0f, 1.4020f},
+        {1.0f, -0.3441f, -0.7141f},
+        {1.0f, 1.7720f, 0.0f},
+    },
+    {0.0f, 128.0f / 255.0f, 128.0f / 255.0f},
+};
+static const struct CscParams k_CscParams_Bt709Lim = {
+    {
+        {1.1644f, 0.0f, 1.7927f},
+        {1.1644f, -0.2132f, -0.5329f},
+        {1.1644f, 2.1124f, 0.0f},
+    },
+    {16.0f / 255.0f, 128.0f / 255.0f, 128.0f / 255.0f},
+};
+static const struct CscParams k_CscParams_Bt709Full = {
+    {
+        {1.0f, 0.0f, 1.5748f},
+        {1.0f, -0.1873f, -0.4681f},
+        {1.0f, 1.8556f, 0.0f},
+    },
+    {0.0f, 128.0f / 255.0f, 128.0f / 255.0f},
+};
+static const struct CscParams k_CscParams_Bt2020Lim = {
+    {
+        {1.1644f, 0.0f, 1.6781f},
+        {1.1644f, -0.1874f, -0.6505f},
+        {1.1644f, 2.1418f, 0.0f},
+    },
+    {16.0f / 255.0f, 128.0f / 255.0f, 128.0f / 255.0f},
+};
+static const struct CscParams k_CscParams_Bt2020Full = {
+    {
+        {1.0f, 0.0f, 1.4746f},
+        {1.0f, -0.1646f, -0.5714f},
+        {1.0f, 1.8814f, 0.0f},
+    },
+    {0.0f, 128.0f / 255.0f, 128.0f / 255.0f},
+};
+
+struct Vertex {
+    vector_float4 position;
+    vector_float2 texCoord;
+};
+
+static const NSUInteger MaxFramesInFlight = 3;
+
+@implementation MetalVideoRenderer {
+    dispatch_queue_t _sq;
+    id<MTLDevice> _device;
+    float _framerate;
+    id<ConnectionCallbacks> _callbacks;
+    id<MTLCommandQueue> _commandQueue;
+    id<MTLLibrary> _shaderLibrary;
+    id<MTLRenderPipelineState> _videoPipelineState;
+    MTLRenderPassDescriptor *_renderPassDescriptor;
+    id<MTLTexture> _videoTexture;
+    CVMetalTextureCacheRef _textureCache;
+    CVMetalTextureRef _cvMetalTextures[MAX_VIDEO_PLANES];
+
+    int _lastColorSpace;
+    BOOL _lastFullRange;
+    size_t _lastFrameWidth;
+    size_t _lastFrameHeight;
+    size_t _lastDrawableWidth;
+    size_t _lastDrawableHeight;
+    id<MTLBuffer> _CscParamsBuffer;
+    id<MTLBuffer> _VideoVertexBuffer;
+    CFTimeInterval _lastPresented;
+
+    // https://developer.apple.com/documentation/metal/synchronizing-cpu-and-gpu-work?language=objc
+    dispatch_semaphore_t _inFlightSemaphore;
+}
+
+- (instancetype)initWithMetalDevice:(id<MTLDevice>)device drawablePixelFormat:(MTLPixelFormat)drawablePixelFormat framerate:(float)framerate {
+    self = [super init];
+    if (self) {
+        _sq = dispatch_queue_create("com.moonlight.MetalVideoRenderer",
+                                    dispatch_queue_attr_make_with_qos_class(DISPATCH_QUEUE_SERIAL, QOS_CLASS_USER_INTERACTIVE, 0));
+        _averageGPUTime = (1.0f / framerate) / 2;
+        _device = device;
+        _nextDrawable = nil;
+        _colorPixelFormat = MTLPixelFormatBGR10A2Unorm;
+        _colorspace = CGColorSpaceCreateWithName(kCGColorSpaceITUR_2100_PQ);
+        _framerate = framerate;
+        _commandQueue = [_device newCommandQueue];
+        _lastColorSpace = -1;
+        _lastFullRange = NO;
+        _lastPresented = 0;
+        _inFlightSemaphore = dispatch_semaphore_create(MaxFramesInFlight);
+
+        CFStringRef keys[1] = {kCVMetalTextureUsage};
+        NSUInteger values[1] = {MTLTextureUsageShaderRead};
+        CFDictionaryRef cacheAttributes = CFDictionaryCreate(kCFAllocatorDefault, (const void **)keys, (const void **)values, 1, NULL, NULL);
+        CVMetalTextureCacheCreate(kCFAllocatorDefault, cacheAttributes, _device, NULL, &_textureCache);
+        CFRelease(cacheAttributes);
+
+        _renderPassDescriptor = [MTLRenderPassDescriptor new];
+        _renderPassDescriptor.colorAttachments[0].loadAction = MTLLoadActionClear;
+        _renderPassDescriptor.colorAttachments[0].clearColor = MTLClearColorMake(0, 0, 0, 0);
+        _renderPassDescriptor.colorAttachments[0].storeAction = MTLStoreActionStore;
+    }
+    return self;
+}
+
+#if !TARGET_OS_TV
+- (void)applyEDRFromFrame:(Frame *)frame toLayer:(CAMetalLayer *)layer {
+    CFDictionaryRef ext = [frame getFormatDescExtensions];
+
+    Log(LOG_I, @"ext: %@", ext);
+
+    CFDataRef masteringData = CFDictionaryGetValue(ext, kCMFormatDescriptionExtension_MasteringDisplayColorVolume);
+    CFDataRef contentDataRef = CFDictionaryGetValue(ext, kCMFormatDescriptionExtension_ContentLightLevelInfo);
+
+    if (masteringData) {
+        Log(LOG_I, @"ext MDCV %@", masteringData);
+    }
+    if (contentDataRef) {
+        Log(LOG_I, @"ext CLLI %@", contentDataRef);
+    }
+
+    if (masteringData && CFDataGetLength(masteringData) == 24 && contentDataRef && CFDataGetLength(contentDataRef) == 4) {
+        NSData *displayData = (__bridge NSData *)masteringData;
+        NSData *contentData = (__bridge NSData *)contentDataRef;
+
+        layer.wantsExtendedDynamicRangeContent = YES;
+        layer.pixelFormat = MTLPixelFormatRGBA16Float;
+        CFStringRef name = kCGColorSpaceExtendedLinearITUR_2020;
+        CGColorSpaceRef colorspace = CGColorSpaceCreateWithName(name);
+        layer.colorspace = colorspace;
+
+        layer.EDRMetadata = [CAEDRMetadata HDR10MetadataWithDisplayInfo:displayData contentInfo:contentData opticalOutputScale:100.0f];
+
+        Log(LOG_I, @"EDRMetadata set from MDCV %@ and CLLI %@", displayData, contentData);
+    } else {
+        layer.wantsExtendedDynamicRangeContent = YES;
+        layer.pixelFormat = MTLPixelFormatRGBA16Float;
+        CFStringRef name = kCGColorSpaceExtendedLinearITUR_2020;
+        CGColorSpaceRef colorspace = CGColorSpaceCreateWithName(name);
+        layer.colorspace = colorspace;
+
+        layer.EDRMetadata = [CAEDRMetadata HDR10MetadataWithMinLuminance:0.0005f maxLuminance:1000.0f opticalOutputScale:100.0f];
+
+        Log(LOG_I, @"EDRMetadata set for 1000 nits");
+    }
+}
+#endif
+
+- (int)getFrameColorspaceAndRange:(Frame *)frame isFullRange:(BOOL *)isFullRange {
+    CFDictionaryRef ext = [frame getFormatDescExtensions];
+
+    // FQLog(LOG_I, @"%@", ext);
+
+    // Full Range boolean
+    CFBooleanRef fullRangeRef = CFDictionaryGetValue(ext, kCMFormatDescriptionExtension_FullRangeVideo);
+    *isFullRange = NO;
+    if (fullRangeRef && CFGetTypeID(fullRangeRef) == CFBooleanGetTypeID()) {
+        *isFullRange = CFBooleanGetValue(fullRangeRef);
+    }
+
+    // Colorspace
+    CFStringRef frame_color = CFDictionaryGetValue(ext, kCVImageBufferColorPrimariesKey);
+    if (CFEqual(frame_color, kCVImageBufferColorPrimaries_ITU_R_709_2)) {
+        return COLORSPACE_REC_709;
+    } else if (CFEqual(frame_color, kCVImageBufferColorPrimaries_ITU_R_2020)) {
+        return COLORSPACE_REC_2020;
+    }
+    return COLORSPACE_REC_601;
+}
+
+- (BOOL)updateColorSpaceForFrame:(Frame *)frame toLayer:(CAMetalLayer *)layer layerDidChange:(BOOL *)layerDidChange {
+    BOOL fullRange = NO;
+    int colorspace = [self getFrameColorspaceAndRange:frame isFullRange:&fullRange];
+    if (colorspace != _lastColorSpace || fullRange != _lastFullRange) {
+        CGColorSpaceRef newColorSpace = nil;
+        MTLPixelFormat newPixelFormat = layer.pixelFormat;
+        BOOL isHDR = NO;
+        struct ParamBuffer paramBuffer;
+
+        switch (colorspace) {
+            case COLORSPACE_REC_709:
+                newColorSpace = CGColorSpaceCreateWithName(kCGColorSpaceITUR_709);
+                newPixelFormat = MTLPixelFormatBGRA8Unorm;
+                paramBuffer.cscParams = (fullRange ? k_CscParams_Bt709Full : k_CscParams_Bt709Lim);
+                break;
+            case COLORSPACE_REC_2020: {
+                CFDictionaryRef ext = [frame getFormatDescExtensions];
+                CFStringRef frame_trc = CFDictionaryGetValue(ext, kCVImageBufferTransferFunctionKey);
+                if (CFEqual(frame_trc, kCVImageBufferTransferFunction_SMPTE_ST_2084_PQ)) {
+                    isHDR = YES;
+                    newColorSpace = CGColorSpaceCreateWithName(kCGColorSpaceITUR_2100_PQ);
+                    newPixelFormat = MTLPixelFormatBGR10A2Unorm;
+                } else {
+                    // SDR 2020
+                    newColorSpace = CGColorSpaceCreateWithName(kCGColorSpaceITUR_2020);
+                    newPixelFormat = MTLPixelFormatBGR10A2Unorm;
+                }
+                paramBuffer.cscParams = (fullRange ? k_CscParams_Bt2020Full : k_CscParams_Bt2020Lim);
+                break;
+            }
+            case COLORSPACE_REC_601:
+                newColorSpace = CGColorSpaceCreateWithName(kCGColorSpaceSRGB);
+                newPixelFormat = MTLPixelFormatBGRA8Unorm;
+                paramBuffer.cscParams = (fullRange ? k_CscParams_Bt601Full : k_CscParams_Bt601Lim);
+        }
+
+        // The CAMetalLayer retains the CGColorSpace
+        if (newColorSpace || newPixelFormat != layer.pixelFormat) {
+            *layerDidChange = YES;
+            if (newColorSpace) {
+                Log(LOG_I,
+                    @"Frame colorspace %@ - changing MetalLayer's colorspace to %@",
+                    colorspace == COLORSPACE_REC_709        ? @"REC_709"
+                        : colorspace == COLORSPACE_REC_2020 ? @"REC_2020"
+                        : colorspace == COLORSPACE_REC_601  ? @"REC_601 (sRGB)"
+                                                            : [NSString stringWithFormat:@"Unknown: %d", colorspace],
+                    newColorSpace);
+            }
+            if (newPixelFormat != layer.pixelFormat) {
+                Log(LOG_I,
+                    @"Frame pixel format %@ - changing MetalLayer's pixel format to %@",
+                    layer.pixelFormat == MTLPixelFormatBGRA8Unorm         ? @"MTLPixelFormatBGRA8Unorm"
+                        : layer.pixelFormat == MTLPixelFormatBGR10A2Unorm ? @"MTLPixelFormatBGR10A2Unorm"
+                                                                          : [NSString stringWithFormat:@"Unknown: %lu", layer.pixelFormat],
+                    newPixelFormat == MTLPixelFormatBGRA8Unorm         ? @"MTLPixelFormatBGRA8Unorm"
+                        : newPixelFormat == MTLPixelFormatBGR10A2Unorm ? @"MTLPixelFormatBGR10A2Unorm"
+                                                                       : [NSString stringWithFormat:@"Unknown: %lu", (unsigned long)layer.pixelFormat]);
+            }
+#if !RENDER_ON_MAIN_THREAD
+            // These can only be changed on the main thread
+            dispatch_sync(dispatch_get_main_queue(), ^{
+#endif
+#if !TARGET_OS_TV
+                if (isHDR) {
+                    layer.wantsExtendedDynamicRangeContent = YES;
+                }
+#endif
+                layer.colorspace = newColorSpace;
+                layer.pixelFormat = newPixelFormat;
+#if !RENDER_ON_MAIN_THREAD
+            });
+#endif
+            CGColorSpaceRelease(newColorSpace);
+        }
+
+        // Create the new colorspace parameter buffer for our fragment shader
+        MTLResourceOptions bufferOptions = MTLResourceStorageModeShared;
+        _CscParamsBuffer = [_device newBufferWithBytes:(void *)&paramBuffer length:sizeof(paramBuffer) options:bufferOptions];
+        if (!_CscParamsBuffer) {
+            Log(LOG_E, @"Failed to create CSC parameters buffer");
+            return NO;
+        }
+
+        _lastColorSpace = colorspace;
+        _lastFullRange = fullRange;
+    }
+
+    return YES;
+}
+
+- (void)scaleSource:(CGRect *)src toDest:(CGRect *)dst {
+    int dstH = ceilf((float)dst->size.width * src->size.height / src->size.width);
+    int dstW = ceilf((float)dst->size.height * src->size.width / src->size.height);
+
+    if (dstH > dst->size.height) {
+        dst->origin.x += (dst->size.width - dstW) / 2;
+        dst->size.width = dstW;
+    } else {
+        dst->origin.y += (dst->size.height - dstH) / 2;
+        dst->size.height = dstH;
+    }
+}
+
+- (void)screenSpace:(CGRect *)src toNormalizedDeviceCoords:(CGRect *)dst withDrawableWidth:(int)viewportWidth drawableHeight:(int)viewportHeight {
+    dst->origin.x = ((float)src->origin.x / (viewportWidth / 2.0f)) - 1.0f;
+    dst->origin.y = ((float)src->origin.y / (viewportHeight / 2.0f)) - 1.0f;
+    dst->size.width = (float)src->size.width / (viewportWidth / 2.0f);
+    dst->size.height = (float)src->size.height / (viewportHeight / 2.0f);
+}
+
+- (BOOL)updateVideoRegionSizeForFrame:(Frame *)frame toLayer:(CAMetalLayer *)layer {
+    int drawableWidth = layer.drawableSize.width;
+    int drawableHeight = layer.drawableSize.height;
+
+    // Check if anything has changed since the last vertex buffer upload
+    if (_VideoVertexBuffer && [frame width] == _lastFrameWidth && [frame height] == _lastFrameHeight && drawableWidth == _lastDrawableWidth &&
+        drawableHeight == _lastDrawableHeight) {
+        // Nothing to do
+        return YES;
+    }
+
+    // Determine the correct scaled size for the video region
+    CGRect src = CGRectMake(0.0, 0.0, [frame width], [frame height]);
+    CGRect dst = CGRectMake(0.0, 0.0, drawableWidth, drawableHeight);
+    [self scaleSource:&src toDest:&dst];
+
+    // Convert screen space to normalized device coordinates
+    CGRect renderRect;
+    [self screenSpace:&dst toNormalizedDeviceCoords:&renderRect withDrawableWidth:drawableWidth drawableHeight:drawableHeight];
+
+    struct Vertex verts[] = {
+        {{renderRect.origin.x, renderRect.origin.y, 0.0f, 1.0f}, {0.0f, 1.0f}},
+        {{renderRect.origin.x, renderRect.origin.y + renderRect.size.height, 0.0f, 1.0f}, {0.0f, 0}},
+        {{renderRect.origin.x + renderRect.size.width, renderRect.origin.y, 0.0f, 1.0f}, {1.0f, 1.0f}},
+        {{renderRect.origin.x + renderRect.size.width, renderRect.origin.y + renderRect.size.height, 0.0f, 1.0f}, {1.0f, 0}},
+    };
+
+    MTLResourceOptions bufferOptions = MTLResourceStorageModeShared;
+    _VideoVertexBuffer = [_device newBufferWithBytes:verts length:sizeof(verts) options:bufferOptions];
+    if (!_VideoVertexBuffer) {
+        Log(LOG_E, @"Failed to create video vertex buffer");
+        return NO;
+    }
+
+    _lastFrameWidth = [frame width];
+    _lastFrameHeight = [frame height];
+    _lastDrawableWidth = drawableWidth;
+    _lastDrawableHeight = drawableHeight;
+
+    return YES;
+}
+
+- (void)discardNextDrawable {
+    _nextDrawable = nil;
+}
+
+- (void)renderFrame:(Frame *)frame toLayer:(CAMetalLayer *)layer {
+    // Handle changes to the frame's colorspace from last time we rendered
+    BOOL layerDidChange = NO;
+    if (![self updateColorSpaceForFrame:frame toLayer:layer layerDidChange:&layerDidChange]) {
+        return;
+    }
+
+    if (layerDidChange && frame.frameNumber > 1) {
+        Log(LOG_I, @"Metal frame changed layer's colorspace and/or pixel format, returning for new drawable");
+        [self discardNextDrawable];
+        return;
+    }
+
+    // Handle changes to the video size or drawable size
+    if (![self updateVideoRegionSizeForFrame:frame toLayer:layer]) {
+        return;
+    }
+
+    FQLog(LOG_I, @"[%d / %.3f ms] Metal frame rendering", frame.frameNumber, frame.pts);
+
+#if !TARGET_OS_TV
+    // Experimental EDR handling based on frame metadata
+    //[self applyEDRFromFrame:frame toLayer:layer];
+#endif
+
+    size_t planes = CVPixelBufferGetPlaneCount(frame.pixelBuffer);
+    assert(planes == 2 || planes == 3);
+
+    MTLRenderPipelineDescriptor *pipelineDesc = [MTLRenderPipelineDescriptor new];
+    id<MTLLibrary> defaultLibrary = [_device newDefaultLibrary];
+    pipelineDesc.vertexFunction = [defaultLibrary newFunctionWithName:@"vs_draw"];
+    pipelineDesc.fragmentFunction = [defaultLibrary newFunctionWithName:planes == 2 ? @"ps_draw_biplanar" : @"ps_draw_triplanar"];
+    pipelineDesc.colorAttachments[0].pixelFormat = layer.pixelFormat;
+    pipelineDesc.vertexBuffers[0].mutability = MTLMutabilityImmutable;
+
+    NSError *error = nil;
+    _videoPipelineState = [_device newRenderPipelineStateWithDescriptor:pipelineDesc error:&error];
+    if (!_videoPipelineState) {
+        Log(LOG_E, @"Failed to create video pipeline state: %@", error);
+        return;
+    }
+
+    for (size_t i = 0; i < planes; i++) {
+        MTLPixelFormat fmt;
+
+        switch (CVPixelBufferGetPixelFormatType(frame.pixelBuffer)) {
+            case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange:
+            case kCVPixelFormatType_444YpCbCr8BiPlanarVideoRange:
+            case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange:
+            case kCVPixelFormatType_444YpCbCr8BiPlanarFullRange:
+                fmt = (i == 0) ? MTLPixelFormatR8Unorm : MTLPixelFormatRG8Unorm;
+                break;
+
+            case kCVPixelFormatType_420YpCbCr10BiPlanarFullRange:
+            case kCVPixelFormatType_444YpCbCr10BiPlanarFullRange:
+            case kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange:
+            case kCVPixelFormatType_444YpCbCr10BiPlanarVideoRange:
+                fmt = (i == 0) ? MTLPixelFormatR16Unorm : MTLPixelFormatRG16Unorm;
+                break;
+
+            default:
+                Log(LOG_E, @"Unknown pixel format: %@", CVPixelBufferGetPixelFormatType(frame.pixelBuffer));
+                return;
+        }
+
+        if (_cvMetalTextures[i]) {
+            CVBufferRelease(_cvMetalTextures[i]);
+        }
+        CVReturn err = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault,
+                                                                 _textureCache,
+                                                                 frame.pixelBuffer,
+                                                                 NULL,
+                                                                 fmt,
+                                                                 CVPixelBufferGetWidthOfPlane(frame.pixelBuffer, i),
+                                                                 CVPixelBufferGetHeightOfPlane(frame.pixelBuffer, i),
+                                                                 i,
+                                                                 &_cvMetalTextures[i]);
+        if (err != kCVReturnSuccess) {
+            Log(LOG_E, @"CVMetalTextureCacheCreateTextureFromImage() failed: %d", err);
+            return;
+        }
+    }
+
+    _renderPassDescriptor.colorAttachments[0].texture = _nextDrawable.texture;
+
+    id<MTLCommandBuffer> commandBuffer = [_commandQueue commandBuffer];
+    id<MTLRenderCommandEncoder> renderEncoder = [commandBuffer renderCommandEncoderWithDescriptor:_renderPassDescriptor];
+
+    [renderEncoder setRenderPipelineState:_videoPipelineState];
+    for (size_t i = 0; i < planes; i++) {
+        [renderEncoder setFragmentTexture:CVMetalTextureGetTexture(_cvMetalTextures[i]) atIndex:i];
+    }
+    [commandBuffer addCompletedHandler:^(id<MTLCommandBuffer> cb) {
+        const CFTimeInterval GPUTime = cb.GPUEndTime - cb.GPUStartTime;
+        const double alpha = 0.25f;
+        self->_averageGPUTime = (GPUTime * alpha) + (self->_averageGPUTime * (1.0 - alpha));
+
+        // Free textures after completion of rendering per CVMetalTextureCache requirements
+        for (size_t i = 0; i < planes; i++) {
+            CVBufferRelease(self->_cvMetalTextures[i]);
+            self->_cvMetalTextures[i] = nil;
+        }
+
+        CVMetalTextureCacheFlush(self->_textureCache, 0);
+    }];
+
+    [renderEncoder setFragmentBuffer:_CscParamsBuffer offset:0 atIndex:0];
+    [renderEncoder setVertexBuffer:_VideoVertexBuffer offset:0 atIndex:0];
+    [renderEncoder drawPrimitives:MTLPrimitiveTypeTriangleStrip vertexStart:0 vertexCount:4];
+    [renderEncoder endEncoding];
+
+    __block dispatch_semaphore_t block_semaphore = _inFlightSemaphore;
+    [commandBuffer addCompletedHandler:^(id<MTLCommandBuffer> buffer) {
+        dispatch_semaphore_signal(block_semaphore);
+    }];
+
+    __weak typeof(self) self_ = self;
+    [_nextDrawable addPresentedHandler:^(id<MTLDrawable> d) {
+        if (self_) {
+            [self_ plotFrametime:d.presentedTime];
+        }
+    }];
+
+#if TARGET_OS_SIMULATOR
+    [commandBuffer presentDrawable:_nextDrawable];
+#else
+    // present for a minimum duration for best frame pacing
+    [commandBuffer presentDrawable:_nextDrawable afterMinimumDuration:1.0f / _framerate];
+#endif
+
+    [commandBuffer commit];
+
+    // Wait for the command buffer to complete and free our CVMetalTextureCache references
+    [commandBuffer waitUntilCompleted];
+
+    _nextDrawable = nil;
+}
+
+- (void)plotFrametime:(CFTimeInterval)presentedTime {
+    if (_lastPresented > 0) {
+        CFTimeInterval frametime = presentedTime - _lastPresented;
+        [[ImGuiPlots sharedInstance] observeFloat:PLOT_FRAMETIME value:(frametime * 1000.0)];
+    }
+    _lastPresented = presentedTime;
+}
+
+- (void)waitToRenderTo:(nonnull CAMetalLayer *)layer {
+    if (!_nextDrawable) {
+        // Wait for the next available drawable
+        _nextDrawable = [layer nextDrawable];
+        if (!_nextDrawable) {
+            Log(LOG_E, @"Error getting nextDrawable from CAMetalLayer");
+            return;
+        }
+
+        // Wait to ensure only `MaxFramesInFlight` number of frames are getting processed
+        // by any stage in the Metal pipeline (CPU, GPU, Metal, Drivers, etc.).
+        dispatch_semaphore_wait(_inFlightSemaphore, DISPATCH_TIME_FOREVER);
+    }
+}
+
+/// Responds to the drawable's size or orientation changes.
+- (void)drawableResize:(CGSize)drawableSize {
+    [self resize:drawableSize];
+}
+
+- (void)resize:(CGSize)size {
+    // TODO?
+}
+
+@end

Limelight/Metal/MetalView.h

@@ -0,0 +1,36 @@
+#import <Metal/Metal.h>
+#import <QuartzCore/CAMetalDisplayLink.h>
+#import <QuartzCore/CAMetalLayer.h>
+#import "MetalConfig.h"
+
+#if TARGET_OS_IOS || TARGET_OS_TV
+#import <UIKit/UIKit.h>
+#define PlatformView UIView
+#else
+#import <AppKit/AppKit.h>
+#define PlatformView NSView
+#endif
+
+// The protocol to provide resize and redraw callbacks to a delegate.
+@protocol MetalViewDelegate <NSObject>
+
+- (void)drawableResize:(CGSize)size;
+- (void)renderTo:(nonnull CAMetalLayer *)layer;
+- (void)waitToRenderTo:(nonnull CAMetalLayer *)layer;
+
+@end
+
+// The Metal game view base class.
+@interface MetalView : PlatformView <CALayerDelegate>
+
+@property (nonatomic, nonnull, readonly) CAMetalLayer *metalLayer;
+@property (nonatomic, getter=isPaused) BOOL paused;
+@property (nonatomic, nullable) id<MetalViewDelegate> delegate;
+@property (nonatomic) float framerate;
+
+- (void)initCommon;
+#if AUTOMATICALLY_RESIZE
+- (void)resizeDrawable:(CGFloat)scaleFactor;
+#endif
+
+@end

Limelight/Metal/MetalView.m

@@ -0,0 +1,205 @@
+// This is based on the following Apple example:
+// https://developer.apple.com/documentation/metal/achieving-smooth-frame-rates-with-a-metal-display-link?language=objc
+// https://developer.apple.com/wwdc23/10123/
+
+#import "MetalView.h"
+#import "MetalConfig.h"
+
+@implementation MetalView {
+#if !RENDER_ON_MAIN_THREAD
+    // The secondary thread containing the render loop.
+    NSThread *_renderThread;
+
+    // The flag to indicate that rendering needs to cease on the main thread.
+    BOOL _continueRunLoop;
+#endif
+}
+
+#pragma mark - Initialization and Setup.
+
+- (instancetype)initWithFrame:(CGRect)frame {
+    self = [super initWithFrame:frame];
+    if (self) {
+        [self initCommon];
+    }
+    return self;
+}
+
+- (instancetype)initWithCoder:(NSCoder *)aDecoder {
+    self = [super initWithCoder:aDecoder];
+    if (self) {
+        [self initCommon];
+    }
+    return self;
+}
+
+- (void)initCommon {
+#if TARGET_OS_OSX
+    self.wantsLayer = YES;
+
+    self.layerContentsRedrawPolicy = NSViewLayerContentsRedrawDuringViewResize;
+#endif
+
+    _metalLayer = (CAMetalLayer *)self.layer;
+
+    self.layer.delegate = self;
+}
+
+#if TARGET_OS_IOS || TARGET_OS_TV
++ (Class)layerClass {
+    return [CAMetalLayer class];
+}
+
+- (void)didMoveToWindow {
+    [self movedToWindow];
+}
+#else
+- (CALayer *)makeBackingLayer {
+    return [CAMetalLayer layer];
+}
+
+- (void)viewDidMoveToWindow {
+    [self movedToWindow];
+}
+#endif  // END TARGET_OS_IOS || TARGET_OS_TV
+
+- (void)movedToWindow {
+#if !RENDER_ON_MAIN_THREAD
+    // Protect _continueRunLoop with a `@synchronized` block because it's accessed by the separate
+    // animation thread.
+    @synchronized(self) {
+        // Stop the animation loop, allowing it to complete if it's in progress.
+        _continueRunLoop = NO;
+    }
+
+    // Create and start a secondary NSThread that has another run runloop. The NSThread
+    // class calls the 'runThread' method at the start of the secondary thread's execution.
+    _renderThread = [[NSThread alloc] initWithTarget:self selector:@selector(runThread) object:nil];
+    _continueRunLoop = YES;
+    _renderThread.qualityOfService = NSQualityOfServiceUserInteractive;
+    [_renderThread start];
+#endif  // END !RENDER_ON_MAIN_THREAD
+
+    // Perform any actions that need to know the size and scale of the drawable. When UIKit calls
+    // didMoveToWindow after the view initialization, this is the first opportunity to notify
+    // components of the drawable's size.
+#if AUTOMATICALLY_RESIZE
+#if TARGET_OS_IOS || TARGET_OS_TV
+    [self resizeDrawable:self.window.screen.nativeScale];
+#else
+    [self resizeDrawable:self.window.screen.backingScaleFactor];
+#endif
+#else
+    // Notify the delegate of the default drawable size when the system can calculate it.
+    CGSize defaultDrawableSize = self.bounds.size;
+    defaultDrawableSize.width *= self.layer.contentsScale;
+    defaultDrawableSize.height *= self.layer.contentsScale;
+    [self.delegate drawableResize:defaultDrawableSize];
+#endif
+}
+
+#if !RENDER_ON_MAIN_THREAD
+- (void)runThread {
+    // The system sets the '_continueRunLoop' ivar outside this thread, so it needs to synchronize. Create a
+    // 'continueRunLoop' local var that the system can set from the _continueRunLoop ivar in a @synchronized block.
+    BOOL continueRunLoop = YES;
+
+    // Begin the run loop.
+    while (continueRunLoop) {
+        @autoreleasepool {
+            [_delegate waitToRenderTo:_metalLayer];
+
+            @synchronized(self) {
+                continueRunLoop = _continueRunLoop;
+            }
+            if (!continueRunLoop) {
+                break;
+            }
+
+            [_delegate renderTo:_metalLayer];
+
+            @synchronized(self) {
+                continueRunLoop = _continueRunLoop;
+            }
+        }
+    }
+}
+#endif  // END !RENDER_ON_MAIN_THREAD
+
+#pragma mark - Resizing
+
+#if AUTOMATICALLY_RESIZE
+
+// Override all methods that indicate the view's size has changed.
+
+#if TARGET_OS_IOS || TARGET_OS_TV
+- (void)setContentScaleFactor:(CGFloat)contentScaleFactor {
+    [super setContentScaleFactor:contentScaleFactor];
+    [self resizeDrawable:self.window.screen.nativeScale];
+}
+
+- (void)layoutSubviews {
+    [super layoutSubviews];
+    [self resizeDrawable:self.window.screen.nativeScale];
+}
+
+- (void)setFrame:(CGRect)frame {
+    [super setFrame:frame];
+    [self resizeDrawable:self.window.screen.nativeScale];
+}
+
+- (void)setBounds:(CGRect)bounds {
+    [super setBounds:bounds];
+    [self resizeDrawable:self.window.screen.nativeScale];
+}
+#else
+- (void)viewDidChangeBackingProperties {
+    [super viewDidChangeBackingProperties];
+    [self resizeDrawable:self.window.screen.backingScaleFactor];
+}
+
+- (void)setFrameSize:(NSSize)size {
+    [super setFrameSize:size];
+    [self resizeDrawable:self.window.screen.backingScaleFactor];
+}
+
+- (void)setBoundsSize:(NSSize)size {
+    [super setBoundsSize:size];
+    [self resizeDrawable:self.window.screen.backingScaleFactor];
+}
+#endif
+
+- (void)resizeDrawable:(CGFloat)scaleFactor {
+    CGSize newSize = self.bounds.size;
+    newSize.width *= scaleFactor;
+    newSize.height *= scaleFactor;
+
+    if (newSize.width <= 0 || newSize.width <= 0) {
+        return;
+    }
+
+#if RENDER_ON_MAIN_THREAD
+    if (newSize.width == _metalLayer.drawableSize.width && newSize.height == _metalLayer.drawableSize.height) {
+        return;
+    }
+
+    _metalLayer.drawableSize = newSize;
+
+    [_delegate drawableResize:newSize];
+#else
+    // The system calls all AppKit and UIKit calls that notify of a resize on the main thread. Use
+    // a synchronized block to ensure that resize notifications on the delegate are atomic.
+    @synchronized(_metalLayer) {
+        if (newSize.width == _metalLayer.drawableSize.width && newSize.height == _metalLayer.drawableSize.height) {
+            return;
+        }
+
+        _metalLayer.drawableSize = newSize;
+
+        [_delegate drawableResize:newSize];
+    }
+#endif
+}
+#endif  // END AUTOMATICALLY_RESIZE
+
+@end

Limelight/Metal/MetalViewController.h

@@ -0,0 +1,24 @@
+#import <Metal/Metal.h>
+#import "FrameQueue.h"
+#import "ImGuiRenderer.h"
+#import "MetalVideoRenderer.h"
+#import "MetalView.h"
+
+#if TARGET_OS_IOS || TARGET_OS_TV
+#import <UIKit/UIKit.h>
+#define PlatformViewController UIViewController
+#else
+#import <AppKit/AppKit.h>
+#define PlatformViewController NSViewController
+#endif
+
+@interface MetalViewController : PlatformViewController <MetalViewDelegate>
+
+@property (nonatomic) CGRect bounds;
+
+- (nonnull instancetype)initWithFrame:(CGRect)bounds
+                            framerate:(float)framerate
+                            enableHdr:(BOOL)enableHdr
+                       metricsHandler:(MetricsHandler _Nonnull)metricsHandler;
+
+@end

Limelight/Metal/MetalViewController.m

@@ -0,0 +1,144 @@
+/*
+See the LICENSE.txt file for this sample’s licensing information.
+
+Abstract:
+The implementation of the cross-platform game view controller.
+*/
+
+#import "MetalViewController.h"
+#import "FrameQueue.h"
+#import "ImGuiRenderer.h"
+#import "MetalVideoRenderer.h"
+
+@implementation MetalViewController {
+    /// A queue to initialize the renderer asynchronously from the main thread.
+    dispatch_queue_t _dispatch_queue;
+    FrameQueue *_frameQueue;
+    float _framerate;
+    BOOL _enableHdr;
+    MetalView *_metalView;
+    MetalVideoRenderer *_renderer;
+    MetricsHandler _metricsHandler;
+    BOOL _stopping;
+}
+
+- (nonnull instancetype)initWithFrame:(CGRect)bounds framerate:(float)framerate enableHdr:(BOOL)enableHdr metricsHandler:(MetricsHandler)metricsHandler {
+    self = [super init];
+    if (self) {
+        _bounds = bounds;
+        _frameQueue = [FrameQueue sharedInstance];
+        _framerate = framerate;
+        _enableHdr = enableHdr;
+        _metricsHandler = metricsHandler;
+        _stopping = NO;
+    }
+    return self;
+}
+
+- (void)loadView {
+    self.view = [[MetalView alloc] initWithFrame:_bounds];
+}
+
+- (void)viewDidLoad {
+    [super viewDidLoad];
+
+    /// A queue to initialize the renderer asynchronously from the main thread.
+    _dispatch_queue = dispatch_queue_create("com.moonlight.Metal", DISPATCH_QUEUE_CONCURRENT);
+
+    __block MetalView *view = (MetalView *)self.view;
+    if (!view) {
+        Log(LOG_E, @"The view attached to MetalViewController isn't a MetalView.");
+        return;
+    }
+    _metalView = view;
+    _metalView.delegate = self;
+    _metalView.framerate = _framerate;
+
+    // Select the device to render with.
+    id<MTLDevice> device = MTLCreateSystemDefaultDevice();
+    if (!device) {
+        Log(LOG_E, @"Metal isn't supported on this device.");
+        self.view = [[PlatformView alloc] initWithFrame:self.view.frame];
+        return;
+    }
+    view.metalLayer.device = device;
+
+    // Initialize the renderer.
+    MetalVideoRenderer *renderer = [[MetalVideoRenderer alloc] initWithMetalDevice:device
+                                                               drawablePixelFormat:MTLPixelFormatBGR10A2Unorm
+                                                                         framerate:self->_framerate];
+    if (!renderer) {
+        Log(LOG_E, @"The renderer couldn't be initialized.");
+        return;
+    }
+
+    // Initialize the renderer-dependent view properties.
+    view.metalLayer.pixelFormat = renderer.colorPixelFormat;
+    view.metalLayer.colorspace = renderer.colorspace;
+    view.metalLayer.maximumDrawableCount = 3;
+
+    self->_renderer = renderer;
+}
+
+- (void)waitToRenderTo:(nonnull CAMetalLayer *)layer {
+    if (!_stopping) {
+        // Renderer obtains a nextDrawable, waiting if necessary
+        FQLog(LOG_I, @"[MetalViewController] caling [_renderer waitToRenderTo:layer]");
+        [_renderer waitToRenderTo:layer];
+
+        // If we don't have a frame yet, wait on that too
+        FQLog(LOG_I, @"[MetalViewController] caling [_renderer waitForEnqueue]");
+        [_frameQueue waitForEnqueue];
+    }
+}
+
+/// Draw frame (used by manual loop)
+- (void)renderTo:(nonnull CAMetalLayer *)layer {
+    if (!_renderer) {
+        return;
+    }
+
+    CFTimeInterval timeout = (1.0f / _framerate) - _renderer.averageGPUTime;
+    FQLog(LOG_I, @"[MetalViewController] caling [_frameQueue dequeueWithTimeout:%f]", timeout);
+    Frame *frame = [_frameQueue dequeueWithTimeout:timeout];
+    if (frame) {
+        FQLog(LOG_I, @"[MetalViewController] calling [_renderer renderFrame] %@", frame);
+        [_renderer renderFrame:frame toLayer:layer];
+    }
+}
+
+- (void)drawableResize:(CGSize)size {
+    [_renderer drawableResize:size];
+}
+
+#if TARGET_OS_IOS
+/// Hides the Home indicator button automatically.
+- (BOOL)prefersHomeIndicatorAutoHidden {
+    return YES;
+}
+#endif
+
+#if TARGET_OS_OSX
+/// Makes the view controller the first responder to receive keyboard events.
+- (void)viewDidAppear {
+    [_metalView.window makeFirstResponder:self];
+}
+
+/// Receives the keydown events to avoid system beeps.
+///
+/// The `GameInputKeyboardMouse` class handles keyboard events.
+- (void)keyDown:(NSEvent *)event {
+    // Reference the parameter to avoid an unused parameter warning.
+    (void)(event);
+}
+
+/// Receives the keyup events to avoid system beeps.
+///
+/// The `GameInputKeyboardMouse` class handles keyboard events.
+- (void)keyUp:(NSEvent *)event {
+    // Reference the parameter to avoid an unused parameter warning.
+    (void)(event);
+}
+#endif
+
+@end

Limelight/Metal/Shaders.metal

@@ -0,0 +1,63 @@
+#include <metal_stdlib>
+#include <simd/simd.h>
+
+using namespace metal;
+
+struct Vertex
+{
+    float4 position [[ position ]];
+    float2 texCoords;
+};
+
+struct CscParams
+{
+    float3 matrix[3];
+    float3 offsets;
+};
+
+constexpr sampler s(coord::normalized, address::clamp_to_edge, filter::linear);
+
+vertex Vertex vs_draw(constant Vertex *vertices [[ buffer(0) ]], uint id [[ vertex_id ]])
+{
+    return vertices[id];
+}
+
+fragment float4 ps_draw_biplanar(Vertex v [[ stage_in ]],
+                                 constant CscParams &cscParams [[ buffer(0) ]],
+                                 texture2d<float> luminancePlane [[ texture(0) ]],
+                                 texture2d<float> chrominancePlane [[ texture(1) ]])
+{
+    float3 yuv = float3(luminancePlane.sample(s, v.texCoords).r,
+                        chrominancePlane.sample(s, v.texCoords).rg);
+    yuv -= cscParams.offsets;
+
+    float3 rgb;
+    rgb.r = dot(yuv, cscParams.matrix[0]);
+    rgb.g = dot(yuv, cscParams.matrix[1]);
+    rgb.b = dot(yuv, cscParams.matrix[2]);
+    return float4(rgb, 1.0f);
+}
+
+fragment float4 ps_draw_triplanar(Vertex v [[ stage_in ]],
+                                  constant CscParams &cscParams [[ buffer(0) ]],
+                                  texture2d<float> luminancePlane [[ texture(0) ]],
+                                  texture2d<float> chrominancePlaneU [[ texture(1) ]],
+                                  texture2d<float> chrominancePlaneV [[ texture(2) ]])
+{
+    float3 yuv = float3(luminancePlane.sample(s, v.texCoords).r,
+                        chrominancePlaneU.sample(s, v.texCoords).r,
+                        chrominancePlaneV.sample(s, v.texCoords).r);
+    yuv -= cscParams.offsets;
+
+    float3 rgb;
+    rgb.r = dot(yuv, cscParams.matrix[0]);
+    rgb.g = dot(yuv, cscParams.matrix[1]);
+    rgb.b = dot(yuv, cscParams.matrix[2]);
+    return float4(rgb, 1.0f);
+}
+
+fragment float4 ps_draw_rgb(Vertex v [[ stage_in ]],
+                            texture2d<float> rgbTexture [[ texture(0) ]])
+{
+    return rgbTexture.sample(s, v.texCoords);
+}