Clean up render thread, plot method calls

VoidLink/ImGuiRenderer.h

@@ -31,6 +31,4 @@ typedef void (^MetricsHandler)(int plotId, CFTimeInterval value);
 -(void) show;
 -(void) hide;
 -(void) stop;
--(void) observeFloat:(int)plotId value:(CFTimeInterval)value;
--(void) observeFloatReturnMetrics:(int)plotId value:(CFTimeInterval)value plotMetrics:(PlotMetrics * _Nullable)plotMetrics;
 @end

VoidLink/ImGuiRenderer.mm

@@ -226,19 +226,6 @@
 #  endif
 #endif
 
-/// Stats Graphs, we can still track data this way even with ImGui disabled
-
-- (void) observeFloat:(int)plotId value:(CFTimeInterval)value {
-    [self.plots[plotId].buffer addValue:(float)value];
-}
-
-- (void) observeFloatReturnMetrics:(int)plotId value:(CFTimeInterval)value plotMetrics:(PlotMetrics *)plotMetrics {
-    [self.plots[plotId].buffer addValue:(float)value];
-    if (plotMetrics != nil) {
-        [self.plots[plotId].buffer copyMetrics:plotMetrics];
-    }
-}
-
 #if !defined(IMGUI_DISABLE)
 inline static float getValue(void *buffer, int idx) {
     float *fbuffer = (float *)buffer;

VoidLink/Metal/MetalVideoRenderer.h

@@ -18,6 +18,6 @@
 - (void)renderFrame:(nonnull Frame *)frame toLayer:(nonnull CAMetalLayer *)layer;
 - (void)waitToRenderTo:(nonnull CAMetalLayer *)layer;
 - (void)drawableResize:(CGSize)drawableSize;
-- (void)plotFrametime:(CFTimeInterval)presentedTime;
+- (void)plotFrametime:(CFTimeInterval)presentedTime withPresentTime:(CFTimeInterval)presentTime;
 
 @end

VoidLink/Metal/MetalVideoRenderer.m

@@ -130,6 +130,13 @@ static const NSUInteger MaxFramesInFlight = 3;
     return self;
 }
 
+- (void)dealloc {
+    if (_CscParamsBuffer) {
+        _CscParamsBuffer = nil;
+    }
+
+}
+
 #if !TARGET_OS_TV
 - (void)applyEDRFromFrame:(Frame *)frame toLayer:(CAMetalLayer *)layer {
     CFDictionaryRef ext = [frame getFormatDescExtensions];
@@ -253,10 +260,9 @@ static const NSUInteger MaxFramesInFlight = 3;
                         : newPixelFormat == MTLPixelFormatBGR10A2Unorm ? @"MTLPixelFormatBGR10A2Unorm"
                                                                        : [NSString stringWithFormat:@"Unknown: %lu", (unsigned long)layer.pixelFormat]);
             }
-#if !RENDER_ON_MAIN_THREAD
+
             // These can only be changed on the main thread
             dispatch_sync(dispatch_get_main_queue(), ^{
-#endif
 #if !TARGET_OS_TV
                 if (isHDR) {
                     layer.wantsExtendedDynamicRangeContent = YES;
@@ -264,9 +270,7 @@ static const NSUInteger MaxFramesInFlight = 3;
 #endif
                 layer.colorspace = newColorSpace;
                 layer.pixelFormat = newPixelFormat;
-#if !RENDER_ON_MAIN_THREAD
             });
-#endif
             CGColorSpaceRelease(newColorSpace);
         }
 
@@ -351,7 +355,8 @@ static const NSUInteger MaxFramesInFlight = 3;
     _nextDrawable = nil;
 }
 
-- (void)renderFrame:(Frame *)frame toLayer:(CAMetalLayer *)layer {
+- (void)renderFrame:(Frame *)frame toLayer:(CAMetalLayer *)layer
+{ @autoreleasepool {
     // Handle changes to the frame's colorspace from last time we rendered
     BOOL layerDidChange = NO;
     if (![self updateColorSpaceForFrame:frame toLayer:layer layerDidChange:&layerDidChange]) {
@@ -377,7 +382,7 @@ static const NSUInteger MaxFramesInFlight = 3;
 #endif
 
     size_t planes = CVPixelBufferGetPlaneCount(frame.pixelBuffer);
-    assert(planes == 2 || planes == 3);
+    assert(planes <= MAX_VIDEO_PLANES);
 
     MTLRenderPipelineDescriptor *pipelineDesc = [MTLRenderPipelineDescriptor new];
     id<MTLLibrary> defaultLibrary = [_device newDefaultLibrary];
@@ -397,23 +402,23 @@ static const NSUInteger MaxFramesInFlight = 3;
         MTLPixelFormat fmt;
 
         switch (CVPixelBufferGetPixelFormatType(frame.pixelBuffer)) {
-            case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange:
+        case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange:
-            case kCVPixelFormatType_444YpCbCr8BiPlanarVideoRange:
+        case kCVPixelFormatType_444YpCbCr8BiPlanarVideoRange:
-            case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange:
+        case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange:
-            case kCVPixelFormatType_444YpCbCr8BiPlanarFullRange:
+        case kCVPixelFormatType_444YpCbCr8BiPlanarFullRange:
-                fmt = (i == 0) ? MTLPixelFormatR8Unorm : MTLPixelFormatRG8Unorm;
+            fmt = (i == 0) ? MTLPixelFormatR8Unorm : MTLPixelFormatRG8Unorm;
-                break;
+            break;
 
-            case kCVPixelFormatType_420YpCbCr10BiPlanarFullRange:
+        case kCVPixelFormatType_420YpCbCr10BiPlanarFullRange:
-            case kCVPixelFormatType_444YpCbCr10BiPlanarFullRange:
+        case kCVPixelFormatType_444YpCbCr10BiPlanarFullRange:
-            case kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange:
+        case kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange:
-            case kCVPixelFormatType_444YpCbCr10BiPlanarVideoRange:
+        case kCVPixelFormatType_444YpCbCr10BiPlanarVideoRange:
-                fmt = (i == 0) ? MTLPixelFormatR16Unorm : MTLPixelFormatRG16Unorm;
+            fmt = (i == 0) ? MTLPixelFormatR16Unorm : MTLPixelFormatRG16Unorm;
-                break;
+            break;
 
-            default:
+        default:
-                Log(LOG_E, @"Unknown pixel format: %@", CVPixelBufferGetPixelFormatType(frame.pixelBuffer));
+            Log(LOG_E, @"Unknown pixel format: %@", CVPixelBufferGetPixelFormatType(frame.pixelBuffer));
-                return;
+            return;
         }
 
         if (_cvMetalTextures[i]) {
@@ -434,6 +439,14 @@ static const NSUInteger MaxFramesInFlight = 3;
         }
     }
 
+    if (!_nextDrawable) {
+        Log(LOG_E, @"Lost nextDrawable, trying to get a new one");
+        _nextDrawable = [layer nextDrawable];
+        if (!_nextDrawable) {
+            Log(LOG_E, @"Failed to get nextDrawable");
+            return;
+        }
+    }
     _renderPassDescriptor.colorAttachments[0].texture = _nextDrawable.texture;
 
     id<MTLCommandBuffer> commandBuffer = [_commandQueue commandBuffer];
@@ -443,15 +456,21 @@ static const NSUInteger MaxFramesInFlight = 3;
     for (size_t i = 0; i < planes; i++) {
         [renderEncoder setFragmentTexture:CVMetalTextureGetTexture(_cvMetalTextures[i]) atIndex:i];
     }
+
+    __block dispatch_semaphore_t block_semaphore = _inFlightSemaphore;
     [commandBuffer addCompletedHandler:^(id<MTLCommandBuffer> cb) {
+        dispatch_semaphore_signal(block_semaphore);
+
         const CFTimeInterval GPUTime = cb.GPUEndTime - cb.GPUStartTime;
         const double alpha = 0.25f;
         self->_averageGPUTime = (GPUTime * alpha) + (self->_averageGPUTime * (1.0 - alpha));
 
-        // Free textures after completion of rendering per CVMetalTextureCache requirements
+        // Free textures after completion of rendering
         for (size_t i = 0; i < planes; i++) {
-            CVBufferRelease(self->_cvMetalTextures[i]);
+            if (self->_cvMetalTextures[i]) {
-            self->_cvMetalTextures[i] = nil;
+                CVBufferRelease(self->_cvMetalTextures[i]);
+                self->_cvMetalTextures[i] = nil;
+            }
         }
 
         CVMetalTextureCacheFlush(self->_textureCache, 0);
@@ -462,11 +481,6 @@ static const NSUInteger MaxFramesInFlight = 3;
     [renderEncoder drawPrimitives:MTLPrimitiveTypeTriangleStrip vertexStart:0 vertexCount:4];
     [renderEncoder endEncoding];
 
-    __block dispatch_semaphore_t block_semaphore = _inFlightSemaphore;
-    [commandBuffer addCompletedHandler:^(id<MTLCommandBuffer> buffer) {
-        dispatch_semaphore_signal(block_semaphore);
-    }];
-
     __weak typeof(self) self_ = self;
     [_nextDrawable addPresentedHandler:^(id<MTLDrawable> d) {
         if (self_) {
@@ -487,7 +501,7 @@ static const NSUInteger MaxFramesInFlight = 3;
     [commandBuffer waitUntilCompleted];
 
     _nextDrawable = nil;
-}
+} }
 
 - (void)plotFrametime:(CFTimeInterval)presentedTime {
     if (_lastPresented > 0) {
@@ -518,7 +532,7 @@ static const NSUInteger MaxFramesInFlight = 3;
 }
 
 - (void)resize:(CGSize)size {
-    // TODO?
+
 }
 
 @end

VoidLink/Metal/MetalView.m

@@ -6,13 +6,11 @@
 #import "MetalConfig.h"
 
 @implementation MetalView {
-#if !RENDER_ON_MAIN_THREAD
     // The secondary thread containing the render loop.
     NSThread *_renderThread;
 
     // The flag to indicate that rendering needs to cease on the main thread.
     BOOL _continueRunLoop;
-#endif
 }
 
 #pragma mark - Initialization and Setup.
@@ -64,7 +62,6 @@
 #endif  // END TARGET_OS_IOS || TARGET_OS_TV
 
 - (void)movedToWindow {
-#if !RENDER_ON_MAIN_THREAD
     // Protect _continueRunLoop with a `@synchronized` block because it's accessed by the separate
     // animation thread.
     @synchronized(self) {
@@ -72,13 +69,12 @@
         _continueRunLoop = NO;
     }
 
-    // Create and start a secondary NSThread that has another run runloop. The NSThread
+    // Create and start a secondary NSThread that has another runloop. The NSThread
     // class calls the 'runThread' method at the start of the secondary thread's execution.
     _renderThread = [[NSThread alloc] initWithTarget:self selector:@selector(runThread) object:nil];
     _continueRunLoop = YES;
     _renderThread.qualityOfService = NSQualityOfServiceUserInteractive;
     [_renderThread start];
-#endif  // END !RENDER_ON_MAIN_THREAD
 
     // Perform any actions that need to know the size and scale of the drawable. When UIKit calls
     // didMoveToWindow after the view initialization, this is the first opportunity to notify
@@ -98,7 +94,6 @@
 #endif
 }
 
-#if !RENDER_ON_MAIN_THREAD
 - (void)runThread {
     // The system sets the '_continueRunLoop' ivar outside this thread, so it needs to synchronize. Create a
     // 'continueRunLoop' local var that the system can set from the _continueRunLoop ivar in a @synchronized block.
@@ -124,7 +119,6 @@
         }
     }
 }
-#endif  // END !RENDER_ON_MAIN_THREAD
 
 #pragma mark - Resizing
 
@@ -178,15 +172,6 @@
         return;
     }
 
-#if RENDER_ON_MAIN_THREAD
-    if (newSize.width == _metalLayer.drawableSize.width && newSize.height == _metalLayer.drawableSize.height) {
-        return;
-    }
-
-    _metalLayer.drawableSize = newSize;
-
-    [_delegate drawableResize:newSize];
-#else
     // The system calls all AppKit and UIKit calls that notify of a resize on the main thread. Use
     // a synchronized block to ensure that resize notifications on the delegate are atomic.
     @synchronized(_metalLayer) {
@@ -194,11 +179,12 @@
             return;
         }
 
+        Log(LOG_I, @"[MetalView] resizeDrawable: %.2f x %.2f", newSize.width, newSize.height);
+
         _metalLayer.drawableSize = newSize;
 
         [_delegate drawableResize:newSize];
     }
-#endif
 }
 #endif  // END AUTOMATICALLY_RESIZE
 

VoidLink/Metal/MetalViewController.m

@@ -83,11 +83,9 @@ The implementation of the cross-platform game view controller.
 - (void)waitToRenderTo:(nonnull CAMetalLayer *)layer {
     if (!_stopping) {
         // Renderer obtains a nextDrawable, waiting if necessary
-        FQLog(LOG_I, @"[MetalViewController] caling [_renderer waitToRenderTo:layer]");
         [_renderer waitToRenderTo:layer];
 
         // If we don't have a frame yet, wait on that too
-        FQLog(LOG_I, @"[MetalViewController] caling [_renderer waitForEnqueue]");
         [_frameQueue waitForEnqueue];
     }
 }
@@ -99,10 +97,8 @@ The implementation of the cross-platform game view controller.
     }
 
     CFTimeInterval timeout = (1.0f / _framerate) - _renderer.averageGPUTime;
-    FQLog(LOG_I, @"[MetalViewController] caling [_frameQueue dequeueWithTimeout:%f]", timeout);
     Frame *frame = [_frameQueue dequeueWithTimeout:timeout];
     if (frame) {
-        FQLog(LOG_I, @"[MetalViewController] calling [_renderer renderFrame] %@", frame);
         [_renderer renderFrame:frame toLayer:layer];
     }
 }

VoidLink/Stream/ConnectionCallbacks.h

@@ -24,7 +24,4 @@
 - (void) setControllerLed:(uint16_t)controllerNumber r:(uint8_t)r g:(uint8_t)g b:(uint8_t)b;
 - (void) videoContentShown;
 
-- (void) observeFloat:(int)plotId value:(CFTimeInterval)value;
-- (void) observeFloatReturnMetrics:(int)plotId value:(CFTimeInterval)value plotMetrics:(PlotMetrics *)plotMetrics;
-
 @end

VoidLink/Stream/StreamManager.m

@@ -196,7 +196,7 @@
     return [NSString stringWithFormat:@"Video stream: %dx%d %.2f FPS (Codec: %@)\n"
             "Bitrate: %.1f Mbps, Peak: %.1f, Renderer: %@\n"
             "%@"
-            "Frames buffered: %.1f\n"
+            "Frames buffered: %.1f, Present \n"
             "Frames dropped by network/pacing jitter: %.1f%% / %.1f%%\n"
             "Average network latency: %@\n"
             "Decode time: %.2f/%.2f/%.2f ms",

VoidLink/Stream/VideoDecoderRenderer.m

@@ -17,6 +17,7 @@
 #import "Plot.h"
 #import "PlatformThreads.h"
 #import "MetalViewController.h"
+#import "ImGuiPlots.h"
 
 #include <libavcodec/avcodec.h>
 #include <libavcodec/cbs.h>
@@ -275,7 +276,7 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit);
                 // we missed a callback
                 // Log(LOG_W, @"*** slow frametime %.3f ms", frametime * 1000.0);
             }
-            [self->_callbacks observeFloat:PLOT_FRAMETIME value:frametime * 1000.0];
+            [[ImGuiPlots sharedInstance] observeFloat:PLOT_FRAMETIME value:frametime * 1000.0];
         }
         lastTargetLocal = targetLocal;
 
@@ -801,18 +802,10 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit);
         return DR_NEED_IDR;
     }
 
-    OSStatus decodeStatus;
+    OSStatus decodeStatus = [self decodeFrameWithSampleBuffer:sampleBuffer
-    if (0 && _renderingBackend == RENDER_METAL) {
+                                                  frameNumber:du->frameNumber
-        decodeStatus = [self decodeFrameToLinearColorspaceWithSampleBuffer:sampleBuffer
+                                                    frameType:du->frameType
-                                             frameNumber:du->frameNumber
+                                              decodeStartTime:decodeStartTime];
-                                               frameType:du->frameType
-                                         decodeStartTime:decodeStartTime];
-    } else {
-        decodeStatus = [self decodeFrameWithSampleBuffer:sampleBuffer
-                                             frameNumber:du->frameNumber
-                                               frameType:du->frameType
-                                         decodeStartTime:decodeStartTime];
-    }
     // Dereference the buffers
     CFRelease(dataBlockBuffer);
     CFRelease(frameBlockBuffer);
@@ -821,63 +814,6 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit);
     return DR_OK;
 }
 
-// For experimenting with improved HDR tone-mapping by directly decoding to linear so we can tonemap using
-// the host's metadata.
-- (OSStatus)decodeFrameToLinearColorspaceWithSampleBuffer:(CMSampleBufferRef)sampleBuffer
-                                              frameNumber:(int)frameNumber
-                                                frameType:(int)frameType
-                                          decodeStartTime:(CFTimeInterval)decodeStartTime
- {
-    NSDictionary *linearAttributes = @{
-      (id)kCVPixelBufferPixelFormatTypeKey    : @(kCVPixelFormatType_64RGBAHalf),  // half-float RGBA
-      (id)kCVImageBufferColorPrimariesKey     : (id)kCVImageBufferColorPrimaries_ITU_R_2020, // wide-gamut primaries (Rec.2020)
-      (id)kCVImageBufferTransferFunctionKey   : (id)kCVImageBufferTransferFunction_Linear, // linear transfer function
-      (id)kCVImageBufferYCbCrMatrixKey        : (id)kCVImageBufferYCbCrMatrix_ITU_R_2020, // Y′CbCr matrix → RGB matrix (Rec.2020)
-      (id)kCVPixelBufferMetalCompatibilityKey : @YES // make it GPU-compatible
-    };
-
-    if (frameType == FRAME_TYPE_IDR || _decompressionSession == nil) {
-        [self setupDecompressionSessionWithAttributes:linearAttributes];
-    }
-
-    OSStatus status = VTDecompressionSessionDecodeFrameWithOutputHandler(
-        _decompressionSession, sampleBuffer, 0, NULL,
-        ^(OSStatus status, VTDecodeInfoFlags infoFlags, CVImageBufferRef _Nullable imageBuffer, CMTime pts, CMTime duration) {
-            CVPixelBufferRef pixelBuffer = CVPixelBufferRetain((CVPixelBufferRef)imageBuffer);
-            Log(LOG_D, @"Decoded to PixelBuffer %@", pixelBuffer); // dumps full frame details
-
-            // Dispatch onto our higher priority queue
-            dispatch_async(self->_vtq, ^{
-                Frame *frame = [[Frame alloc] initWithPixelBufffer:pixelBuffer
-                                                       frameNumber:frameNumber
-                                                         frameType:frameType
-                                                               pts:pts];
-                [frame setFormatDesc:self->_formatDesc];
-                int framesDropped = [self->_frameQueue enqueue:frame withSlackSize:3];
-
-                static PlotMetrics frameQueueMetrics = {};
-                [self->_callbacks observeFloatReturnMetrics:PLOT_QUEUED_FRAMES value:[self->_frameQueue count] plotMetrics:&frameQueueMetrics];
-                [self safeCopyMetricsTo:&self->_frameQueueMetrics from:&frameQueueMetrics];
-
-                [self->_callbacks observeFloat:PLOT_DROPPED value:framesDropped];
-
-                // It's important we capture host metrics on the incoming thread, as this frame object
-                // may have been dropped by the above enqueue
-                static CFTimeInterval lastHostFrame = 0.0f;
-                if (lastHostFrame != 0) {
-                    [self->_callbacks observeFloat:PLOT_HOST_FRAMETIME value:(frame.pts - lastHostFrame) * 1000.0];
-                }
-                lastHostFrame = frame.pts;
-
-                // Decode time is not graphed because it is marked as hidden, but we can use the same mechanism for the value used by stats
-                static PlotMetrics decodeMetrics = {};
-                [self->_callbacks observeFloatReturnMetrics:PLOT_DECODE value:(CACurrentMediaTime() - decodeStartTime) * 1000.0 plotMetrics:&decodeMetrics];
-                [self safeCopyMetricsTo:&self->_decodeMetrics from:&decodeMetrics];
-            });
-        });
-     return status;
-}
-
 - (OSStatus)decodeFrameWithSampleBuffer:(CMSampleBufferRef)sampleBuffer
                             frameNumber:(int)frameNumber
                               frameType:(int)frameType
@@ -954,22 +890,22 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit);
               int framesDropped = [self->_frameQueue enqueue:frame withSlackSize:3];
 
               static PlotMetrics frameQueueMetrics = {};
-              [self->_callbacks observeFloatReturnMetrics:PLOT_QUEUED_FRAMES value:[self->_frameQueue count] plotMetrics:&frameQueueMetrics];
+              [[ImGuiPlots sharedInstance] observeFloatReturnMetrics:PLOT_QUEUED_FRAMES value:[self->_frameQueue count] plotMetrics:&frameQueueMetrics];
               [self safeCopyMetricsTo:&self->_frameQueueMetrics from:&frameQueueMetrics];
 
-              [self->_callbacks observeFloat:PLOT_DROPPED value:framesDropped];
+              [[ImGuiPlots sharedInstance] observeFloat:PLOT_DROPPED value:framesDropped];
 
               // It's important we capture host metrics on the incoming thread, as this frame object
               // may have been dropped by the above enqueue
               static CFTimeInterval lastHostFrame = 0.0f;
               if (lastHostFrame != 0) {
-                [self->_callbacks observeFloat:PLOT_HOST_FRAMETIME value:(frame.pts - lastHostFrame) * 1000.0];
+                [[ImGuiPlots sharedInstance] observeFloat:PLOT_HOST_FRAMETIME value:(frame.pts - lastHostFrame) * 1000.0];
               }
               lastHostFrame = frame.pts;
 
               // Decode time is not graphed because it is marked as hidden, but we can use the same mechanism for the value used by stats
               static PlotMetrics decodeMetrics = {};
-              [self->_callbacks observeFloatReturnMetrics:PLOT_DECODE value:(CACurrentMediaTime() - decodeStartTime) * 1000.0 plotMetrics:&decodeMetrics];
+              [[ImGuiPlots sharedInstance] observeFloatReturnMetrics:PLOT_DECODE value:(CACurrentMediaTime() - decodeStartTime) * 1000.0 plotMetrics:&decodeMetrics];
               [self safeCopyMetricsTo:&self->_decodeMetrics from:&decodeMetrics];
           });
       });

VoidLink/Utility/FrameQueue.m

@@ -226,8 +226,7 @@
 // Allows the render loop to wait if the queue is empty
 - (void)waitForEnqueue {
     while ([self isEmpty]) {
-        FQLog(LOG_I, @"waitForEnqueue...");
+        dispatch_semaphore_wait(_frameSemaphore, DISPATCH_TIME_FOREVER);
-        long result = dispatch_semaphore_wait(_frameSemaphore, DISPATCH_TIME_FOREVER);
     }
 }
 

VoidLink/ViewControllers/StreamFrameViewController.m

@@ -661,7 +661,6 @@
 
     if ([_settings.renderingBackend intValue] == RENDER_METAL) {
         // Metal view for video
-        // TODO: refactor the way things access observeFloat for stats
         self.metalViewController = [[MetalViewController alloc] initWithFrame:self.view.bounds
                                                                     framerate:[self->_settings.framerate floatValue]
                                                                     enableHdr:self->_settings.enableHdr
@@ -1267,14 +1266,6 @@
     });
 }
 
-- (void) observeFloat:(int)plotId value:(CFTimeInterval)value {
-    [self.imguiView observeFloat:plotId value:value];
-}
-
-- (void) observeFloatReturnMetrics:(int)plotId value:(CFTimeInterval)value plotMetrics:(PlotMetrics *)plotMetrics {
-    return [self.imguiView observeFloatReturnMetrics:plotId value:value plotMetrics:plotMetrics];
-}
-
 - (void)didReceiveMemoryWarning
 {
     [super didReceiveMemoryWarning];