MetalVideoRenderer: don't recreate pipeline state every frame

VoidLink/Metal/MetalVideoRenderer.h

@@ -18,7 +18,7 @@
 - (void)renderFrame:(nonnull Frame *)frame toLayer:(nonnull CAMetalLayer *)layer;
 - (void)waitToRenderTo:(nonnull CAMetalLayer *)layer;
 - (void)drawableResize:(CGSize)drawableSize;
-- (void)plotFrametime:(CFTimeInterval)presentedTime withPresentTime:(CFTimeInterval)presentTime;
+- (void)plotFrametime:(CFTimeInterval)presentedTime;
 - (void)stop;
 - (void)discardNextDrawable;
 

VoidLink/Metal/MetalVideoRenderer.m

@@ -79,7 +79,7 @@ static const NSUInteger MaxFramesInFlight = 3;
     id<ConnectionCallbacks> _callbacks;
     id<MTLCommandQueue> _commandQueue;
     id<MTLLibrary> _shaderLibrary;
-    id<MTLRenderPipelineState> _videoPipelineState;
+    id<MTLRenderPipelineState> _videoPipelineState[MAX_VIDEO_PLANES];
     MTLRenderPassDescriptor *_renderPassDescriptor;
     id<MTLTexture> _videoTexture;
     CVMetalTextureCacheRef _textureCache;
@@ -371,27 +371,19 @@ static const NSUInteger MaxFramesInFlight = 3;
             return;
         }
 
-    if (layerDidChange && frame.frameNumber > 1) {
-        Log(LOG_I, @"Metal frame changed layer's colorspace and/or pixel format, returning for new drawable");
-        [self discardNextDrawable];
-        return;
-    }
-
-    // Handle changes to the video size or drawable size
         if (![self updateVideoRegionSizeForFrame:frame toLayer:layer]) {
             return;
         }
 
-    FQLog(LOG_I, @"[%d / %.3f ms] Metal frame rendering", frame.frameNumber, frame.pts);
-
-#if !TARGET_OS_TV
-    // Experimental EDR handling based on frame metadata
-    //[self applyEDRFromFrame:frame toLayer:layer];
-#endif
-
         size_t planes = CVPixelBufferGetPlaneCount(frame.pixelBuffer);
         assert(planes <= MAX_VIDEO_PLANES);
 
+        if (layerDidChange && frame.frameNumber > 1) {
+            Log(LOG_I, @"Metal frame changed layer's colorspace and/or pixel format, invalidating pipeline cache");
+            _videoPipelineState[planes] = nil;
+        }
+
+        if (!_videoPipelineState[planes]) {
             MTLRenderPipelineDescriptor *pipelineDesc = [MTLRenderPipelineDescriptor new];
             id<MTLLibrary> defaultLibrary = [_device newDefaultLibrary];
             pipelineDesc.vertexFunction = [defaultLibrary newFunctionWithName:@"vs_draw"];
@@ -400,11 +392,12 @@ static const NSUInteger MaxFramesInFlight = 3;
             pipelineDesc.vertexBuffers[0].mutability = MTLMutabilityImmutable;
 
             NSError *error = nil;
-    _videoPipelineState = [_device newRenderPipelineStateWithDescriptor:pipelineDesc error:&error];
+            _videoPipelineState[planes] = [_device newRenderPipelineStateWithDescriptor:pipelineDesc error:&error];
-    if (!_videoPipelineState) {
+            if (!_videoPipelineState[planes]) {
                 Log(LOG_E, @"Failed to create video pipeline state: %@", error);
                 return;
             }
+        }
 
         for (size_t i = 0; i < planes; i++) {
             MTLPixelFormat fmt;
@@ -447,23 +440,19 @@ static const NSUInteger MaxFramesInFlight = 3;
             }
         }
 
-    if (!_nextDrawable) {
+        id<CAMetalDrawable> drawable = [layer nextDrawable];
-        Log(LOG_E, @"Lost nextDrawable, trying to get a new one");
+        if (!drawable) {
-        _nextDrawable = [layer nextDrawable];
+            Log(LOG_E, @"Failed to get nextDrawable from layer");
-        if (!_nextDrawable) {
-            Log(LOG_E, @"Failed to get nextDrawable");
             return;
         }
-    }
+        _renderPassDescriptor.colorAttachments[0].texture = drawable.texture;
-    _renderPassDescriptor.colorAttachments[0].texture = _nextDrawable.texture;
 
         id<MTLCommandBuffer> commandBuffer = [_commandQueue commandBuffer];
         id<MTLRenderCommandEncoder> renderEncoder = [commandBuffer renderCommandEncoderWithDescriptor:_renderPassDescriptor];
 
-    [renderEncoder setRenderPipelineState:_videoPipelineState];
+        [renderEncoder setRenderPipelineState:_videoPipelineState[planes]];
         [renderEncoder setVertexBuffer:_VideoVertexBuffer offset:0 atIndex:0];
         [renderEncoder setFragmentBuffer:_CscParamsBuffer offset:0 atIndex:0];
-
         for (size_t i = 0; i < planes; i++) {
             [renderEncoder setFragmentTexture:CVMetalTextureGetTexture(_cvMetalTextures[i]) atIndex:i];
         }
@@ -471,27 +460,11 @@ static const NSUInteger MaxFramesInFlight = 3;
         [renderEncoder drawPrimitives:MTLPrimitiveTypeTriangleStrip vertexStart:0 vertexCount:4];
         [renderEncoder endEncoding];
 
-    __weak typeof(self) weakSelf = self;
+        [commandBuffer presentDrawable:drawable afterMinimumDuration:1.0f / _framerate];
-    [_nextDrawable addPresentedHandler:^(id<MTLDrawable> d) {
-        __strong typeof(weakSelf) strongSelf = weakSelf;
-        if (strongSelf) {
-            [strongSelf plotFrametime:d.presentedTime];
-        }
-    }];
-
-#if TARGET_OS_SIMULATOR
-    [commandBuffer presentDrawable:_nextDrawable];
-#else
-    // present for a minimum duration for best frame pacing
-    [commandBuffer presentDrawable:_nextDrawable afterMinimumDuration:1.0f / _framerate];
-#endif
-
         [commandBuffer commit];
 
-    // Wait for the command buffer to complete and free our CVMetalTextureCache references
         [commandBuffer waitUntilCompleted];
 
-    // Perform cleanup
         const CFTimeInterval GPUTime = commandBuffer.GPUEndTime - commandBuffer.GPUStartTime;
         const double alpha = 0.25f;
         _averageGPUTime = (GPUTime * alpha) + (_averageGPUTime * (1.0 - alpha));
@@ -503,9 +476,11 @@ static const NSUInteger MaxFramesInFlight = 3;
             }
         }
         CVMetalTextureCacheFlush(_textureCache, 0);
+    }
+}
 
-    _nextDrawable = nil;
+- (void)waitToRenderTo:(nonnull CAMetalLayer *)layer {
-} }
+}
 
 - (void)plotFrametime:(CFTimeInterval)presentedTime {
     if (_lastPresented > 0) {
@@ -515,27 +490,12 @@ static const NSUInteger MaxFramesInFlight = 3;
     _lastPresented = presentedTime;
 }
 
-- (void)waitToRenderTo:(nonnull CAMetalLayer *)layer {
-    if (!_nextDrawable) {
-        // Wait for the next available drawable
-        _nextDrawable = [layer nextDrawable];
-        if (!_nextDrawable) {
-            Log(LOG_E, @"Error getting nextDrawable from CAMetalLayer");
-            return;
-        }
-
-        // Wait to ensure only `MaxFramesInFlight` number of frames are getting processed
-        // by any stage in the Metal pipeline (CPU, GPU, Metal, Drivers, etc.).
-    }
-}
-
 /// Responds to the drawable's size or orientation changes.
 - (void)drawableResize:(CGSize)drawableSize {
     [self resize:drawableSize];
 }
 
 - (void)resize:(CGSize)size {
-
 }
 
 - (void)stop {