hermes-agent/VoidLink
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Misc: fix bugs that were preventing starting a new stream after disconnecting. Known issue: this leaks a bit of memory. This also removes all OSX-specific code from the Apple example I used, as I have no intention of using this on Mac.

Browse Source
This commit is contained in:
Andy Grundman
2025-07-25 01:46:49 +08:00
committed by Acaki
parent e57f0cd723
commit e5fc439da2
11 changed files with 81 additions and 162 deletions
Download Patch File Download Diff File Expand all files Collapse all files
VoidLink/Metal/MetalVideoRenderer.m
+20 -20
View File
@@ -158,6 +158,11 @@ CFStringRef __currentColorSpace;
}
- (void)dealloc {
Log(LOG_I, @"MetalVideoRenderer dealloc");
if (_commandQueue) {
_commandQueue = nil;
}
if (_CscParamsBuffer) {
_CscParamsBuffer = nil;
}
@@ -172,6 +177,9 @@ CFStringRef __currentColorSpace;
if (_renderPassDescriptor) {
_renderPassDescriptor = nil;
}
if (_textureCache) {
_textureCache = nil;
}
if (__currentColorSpace) {
CFRelease(__currentColorSpace);
}
@@ -179,12 +187,7 @@ CFStringRef __currentColorSpace;
#if !TARGET_OS_TV
- (void)reportMaxEDRHeadroom {
CGFloat maxHeadroom = 1.0f;
#if TARGET_OS_OSX
maxHeadroom = [[NSScreen mainScreen] maximumPotentialExtendedDynamicRangeColorComponentValue];
#else
maxHeadroom = [[UIScreen mainScreen] potentialEDRHeadroom];
#endif
CGFloat maxHeadroom = [[UIScreen mainScreen] potentialEDRHeadroom];
if (maxHeadroom > 1.0) {
LogOnce(LOG_I, @"Display supports EDR with a max headroom of %.1f", maxHeadroom);
} else {
@@ -193,12 +196,7 @@ CFStringRef __currentColorSpace;
}
- (void)pollCurrentEDRHeadroom {
CGFloat headroom = 1.0f;
#if TARGET_OS_OSX
headroom = [[NSScreen mainScreen] maximumExtendedDynamicRangeColorComponentValue];
#else
headroom = [[UIScreen mainScreen] currentEDRHeadroom];
#endif
CGFloat headroom = [[UIScreen mainScreen] currentEDRHeadroom];
if (headroom != _currentEDRHeadroom) {
Log(LOG_I, @"EDR headroom changed to %.1f", headroom);
_currentEDRHeadroom = (float)headroom;
@@ -448,7 +446,7 @@ CFStringRef __currentColorSpace;
- (void)renderFrame:(Frame *)frame toLayer:(CAMetalLayer *)layer {
@autoreleasepool {
if (self.isStopping) {
Log(LOG_I, @"XXX Metal renderThread is stopping. returning from renderFrame");
Log(LOG_I, @"[MetalVideoRenderer] isStopping");
return;
}
@@ -573,10 +571,12 @@ CFStringRef __currentColorSpace;
[renderEncoder setVertexBuffer:_VideoVertexBuffer offset:0 atIndex:0];
[renderEncoder setFragmentBuffer:_CscParamsBuffer offset:0 atIndex:0];
#if !TARGET_OS_TV
if (layer.pixelFormat == MTLPixelFormatRGBA16Float) {
[self pollCurrentEDRHeadroom];
[renderEncoder setFragmentBytes:&_currentEDRHeadroom length:sizeof(float) atIndex:1];
}
#endif
[renderEncoder drawPrimitives:MTLPrimitiveTypeTriangleStrip vertexStart:0 vertexCount:4];
[renderEncoder endEncoding];
@@ -617,8 +617,6 @@ CFStringRef __currentColorSpace;
#endif
[commandBuffer commit];
// Wait for the command buffer to complete and free our CVMetalTextureCache references
[commandBuffer waitUntilCompleted];
}
}
@@ -633,12 +631,14 @@ CFStringRef __currentColorSpace;
}
- (void)shutdown {
Log(LOG_I, @"XXX MetalVideoRenderer shutodwn");
self.isStopping = YES;
if (!self.isStopping) {
self.isStopping = YES;
Log(LOG_I, @"[MetalVideoRenderer] shutdown");
// Ensure no rendering is in flight
for (NSUInteger i = 0; i < MaxFramesInFlight; i++) {
dispatch_semaphore_signal(_inFlightSemaphore);
// Ensure no rendering is in flight
for (NSUInteger i = 0; i < MaxFramesInFlight; i++) {
dispatch_semaphore_signal(_inFlightSemaphore);
}
}
}
VoidLink/Metal/MetalView.h
+2 -11
View File
@@ -1,15 +1,8 @@
#import <Metal/Metal.h>
#import <QuartzCore/CAMetalDisplayLink.h>
#import <QuartzCore/CAMetalLayer.h>
#import "MetalConfig.h"
#if TARGET_OS_IOS || TARGET_OS_TV
#import <UIKit/UIKit.h>
#define PlatformView UIView
#else
#import <AppKit/AppKit.h>
#define PlatformView NSView
#endif
#import "MetalConfig.h"
// The protocol to provide resize and redraw callbacks to a delegate.
@protocol MetalViewDelegate <NSObject>
@@ -17,15 +10,13 @@
- (void)drawableResize:(CGSize)size;
- (void)renderTo:(nonnull CAMetalLayer *)layer;
- (void)waitToRenderTo:(nonnull CAMetalLayer *)layer;
- (void)shutdown;
@end
// The Metal game view base class.
@interface MetalView : PlatformView <CALayerDelegate>
@interface MetalView : UIView <CALayerDelegate>
@property (nonatomic, nonnull, readonly) CAMetalLayer *metalLayer;
@property (nonatomic, getter=isPaused) BOOL paused;
@property (nonatomic, nullable) id<MetalViewDelegate> delegate;
@property (nonatomic) float framerate;
VoidLink/Metal/MetalView.m
+9 -53
View File
@@ -1,4 +1,4 @@
// This is based on the following Apple example:
// This is based on the following Apple example
// https://developer.apple.com/documentation/metal/achieving-smooth-frame-rates-with-a-metal-display-link?language=objc
// https://developer.apple.com/wwdc23/10123/
@@ -29,29 +29,23 @@
}
- (void)initCommon {
#if TARGET_OS_OSX
self.wantsLayer = YES;
self.layerContentsRedrawPolicy = NSViewLayerContentsRedrawDuringViewResize;
#endif
_metalLayer = (CAMetalLayer *)self.layer;
self.layer.delegate = self;
}
- (void)shutdown {
if (_renderThread) {
Log(LOG_I, @"[MetalView] sending renderThread a cancel message");
[_renderThread cancel];
// wait for thread to exist
Log(LOG_I, @"[MetalView] waiting on renderThread to finish");
while (!_renderThread.isFinished) {
Log(LOG_I, @"XXX MetalView waiting on renderThread to finish");
usleep(100);
}
Log(LOG_I, @"[MetalView] renderThread has finished");
_renderThread = nil;
}
}
#if TARGET_OS_IOS || TARGET_OS_TV
+ (Class)layerClass {
return [CAMetalLayer class];
}
@@ -59,29 +53,11 @@
- (void)didMoveToWindow {
[self movedToWindow];
}
#else
- (CALayer *)makeBackingLayer {
return [CAMetalLayer layer];
}
- (void)viewDidMoveToWindow {
[self movedToWindow];
}
#endif // END TARGET_OS_IOS || TARGET_OS_TV
- (void)movedToWindow {
if (!self.window) {
return;
// We have been removed
if (_renderThread) {
[_renderThread cancel];
// wait for thread to exist
while (!_renderThread.isFinished) {
Log(LOG_I, @"XXX MetalView waiting on renderThread to finish");
usleep(100);
}
}
Log(LOG_I, @"[MetalView] movedToWindow(nil): shutting down...");
[self shutdown];
return;
}
@@ -93,21 +69,18 @@
[self.delegate renderTo:self.metalLayer];
}
}
Log(LOG_I, @"XXX Metal renderThread shutting down");
Log(LOG_I, @"[MetalView] renderThread is exiting");
}];
_renderThread.name = @"MetalVideoRenderer";
_renderThread.qualityOfService = NSQualityOfServiceUserInteractive;
[_renderThread start];
Log(LOG_I, @"[MetalView] started renderThread %@", _renderThread);
// Perform any actions that need to know the size and scale of the drawable. When UIKit calls
// didMoveToWindow after the view initialization, this is the first opportunity to notify
// components of the drawable's size.
#if AUTOMATICALLY_RESIZE
#if TARGET_OS_IOS || TARGET_OS_TV
[self resizeDrawable:self.window.screen.nativeScale];
#else
[self resizeDrawable:self.window.screen.backingScaleFactor];
#endif
#else
// Notify the delegate of the default drawable size when the system can calculate it.
CGSize defaultDrawableSize = self.bounds.size;
@@ -123,7 +96,6 @@
// Override all methods that indicate the view's size has changed.
#if TARGET_OS_IOS || TARGET_OS_TV
- (void)setContentScaleFactor:(CGFloat)contentScaleFactor {
[super setContentScaleFactor:contentScaleFactor];
[self resizeDrawable:self.window.screen.nativeScale];
@@ -143,22 +115,6 @@
[super setBounds:bounds];
[self resizeDrawable:self.window.screen.nativeScale];
}
#else
- (void)viewDidChangeBackingProperties {
[super viewDidChangeBackingProperties];
[self resizeDrawable:self.window.screen.backingScaleFactor];
}
- (void)setFrameSize:(NSSize)size {
[super setFrameSize:size];
[self resizeDrawable:self.window.screen.backingScaleFactor];
}
- (void)setBoundsSize:(NSSize)size {
[super setBoundsSize:size];
[self resizeDrawable:self.window.screen.backingScaleFactor];
}
#endif
- (void)resizeDrawable:(CGFloat)scaleFactor {
CGSize newSize = self.bounds.size;
VoidLink/Metal/MetalViewController.h
+2 -9
View File
@@ -1,18 +1,11 @@
#import <Metal/Metal.h>
#import <UIKit/UIKit.h>
#import "FrameQueue.h"
#import "ImGuiRenderer.h"
#import "MetalVideoRenderer.h"
#import "MetalView.h"
#if TARGET_OS_IOS || TARGET_OS_TV
#import <UIKit/UIKit.h>
#define PlatformViewController UIViewController
#else
#import <AppKit/AppKit.h>
#define PlatformViewController NSViewController
#endif
@interface MetalViewController : PlatformViewController <MetalViewDelegate>
@interface MetalViewController : UIViewController <MetalViewDelegate>
@property (nonatomic) CGRect bounds;
VoidLink/Metal/MetalViewController.m
+8 -33
View File
@@ -33,6 +33,7 @@ The implementation of the cross-platform game view controller.
- (void)loadView {
self.view = [[MetalView alloc] initWithFrame:_bounds];
Log(LOG_I, @"[MetalViewController] created MetalView %@", (MetalView *)self.view);
}
- (void)viewDidLoad {
@@ -51,7 +52,7 @@ The implementation of the cross-platform game view controller.
id<MTLDevice> device = MTLCreateSystemDefaultDevice();
if (!device) {
Log(LOG_E, @"Metal isn't supported on this device.");
self.view = [[PlatformView alloc] initWithFrame:self.view.frame];
self.view = [[UIView alloc] initWithFrame:self.view.frame];
return;
}
view.metalLayer.device = device;
@@ -64,12 +65,12 @@ The implementation of the cross-platform game view controller.
Log(LOG_E, @"The renderer couldn't be initialized.");
return;
}
self->_renderer = renderer;
Log(LOG_I, @"[MetalViewController] viewDidLoad, created renderer: %@", renderer);
// Initialize the renderer-dependent view properties.
view.metalLayer.pixelFormat = renderer.colorPixelFormat;
view.metalLayer.maximumDrawableCount = 3;
self->_renderer = renderer;
}
- (void)waitToRenderTo:(nonnull CAMetalLayer *)layer {
@@ -95,46 +96,20 @@ The implementation of the cross-platform game view controller.
[_renderer drawableResize:size];
}
- (void)shutdown {
[_renderer shutdown];
}
- (void)viewDidDisappear:(BOOL)animated {
[super viewDidDisappear:animated];
Log(LOG_I, @"XXX MetalViewController viewDidDisappear");
Log(LOG_I, @"[MetalViewController] viewDidDisappear");
[_metalView shutdown];
[_renderer shutdown];
_renderer = nil;
}
#if TARGET_OS_IOS
/// Hides the Home indicator button automatically.
// Hides the Home indicator button automatically.
- (BOOL)prefersHomeIndicatorAutoHidden {
return YES;
}
#endif
#if TARGET_OS_OSX
/// Makes the view controller the first responder to receive keyboard events.
- (void)viewDidAppear {
[_metalView.window makeFirstResponder:self];
}
/// Receives the keydown events to avoid system beeps.
///
/// The `GameInputKeyboardMouse` class handles keyboard events.
- (void)keyDown:(NSEvent *)event {
// Reference the parameter to avoid an unused parameter warning.
(void)(event);
}
/// Receives the keyup events to avoid system beeps.
///
/// The `GameInputKeyboardMouse` class handles keyboard events.
- (void)keyUp:(NSEvent *)event {
// Reference the parameter to avoid an unused parameter warning.
(void)(event);
}
#endif
@end
VoidLink/Stats/ImGuiPlots.h
+1
View File
@@ -24,6 +24,7 @@ struct PlotDef {
- (instancetype _Nonnull)init NS_UNAVAILABLE;
+ (instancetype _Nonnull)new NS_UNAVAILABLE;
- (void)clearData;
- (void)observeFloat:(int)plotId value:(CFTimeInterval)value;
- (void)observeFloatReturnMetrics:(int)plotId value:(CFTimeInterval)value plotMetrics:(PlotMetrics * _Nullable)plotMetrics;
VoidLink/Stats/ImGuiPlots.mm
+8
View File
@@ -92,4 +92,12 @@
}
}
- (void) clearData {
for (int i = 0; i < PlotCount; i++) {
if (_plots[i].buffer) {
[_plots[i].buffer clear];
}
}
}
@end
VoidLink/Stream/VideoDecoderRenderer.m
+17 -15
View File
@@ -65,7 +65,7 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size,
_displayLayer.videoGravity = AVLayerVideoGravityResize;
[_view.layer addSublayer:_displayLayer];
}
// Ensure the AVSampleBufferDisplayLayer is sized to preserve the aspect ratio
// of the video stream. We used to use AVLayerVideoGravityResizeAspect, but that
// respects the PAR encoded in the SPS which causes our computed video-relative
@@ -77,30 +77,30 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size,
} else {
videoSize = CGSizeMake(_view.bounds.size.width, _view.bounds.size.width / _streamAspectRatio);
}
[CATransaction begin];
[CATransaction setDisableActions:YES];
_displayLayer.position = CGPointMake(CGRectGetMidX(_view.bounds), CGRectGetMidY(_view.bounds));
_displayLayer.bounds = CGRectMake(0, 0, videoSize.width, videoSize.height);
[CATransaction commit];
// Hide the layer until we get an IDR frame. This ensures we
// can see the loading progress label as the stream is starting.
_displayLayer.hidden = YES;
if (_formatDesc != nil) {
CFRelease(_formatDesc);
_formatDesc = nil;
}
if (_formatDescImageBuffer != nil) {
CFRelease(_formatDescImageBuffer);
_formatDescImageBuffer = nil;
}
if (_decompressionSession != nil){
VTDecompressionSessionWaitForAsynchronousFrames(_decompressionSession);
VTDecompressionSessionInvalidate(_decompressionSession);
CFRelease(_decompressionSession);
_decompressionSession = nil;
@@ -122,14 +122,13 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size,
_view = view;
_callbacks = callbacks;
_streamAspectRatio = aspectRatio;
_parameterSetBuffers = [[NSMutableArray alloc] init];
_frameQueue = [FrameQueue sharedInstance];
[_frameQueue start];
_maxRefreshRate = [[UIScreen mainScreen] maximumFramesPerSecond];
_parameterSetBuffers = [[NSMutableArray alloc] init];
DataManager* dataMan = [[DataManager alloc] init];
_frameQueue = [FrameQueue sharedInstance];
[_frameQueue start];
[_frameQueue setHighWaterMark:(int)[[dataMan getSettings].frameQueueSize integerValue]];
[self reinitializeDisplayLayer];
@@ -149,6 +148,9 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size,
self->_videoFormat = videoFormat;
self->_frameRate = frameRate;
// reset plot data in case we've already used it for a previous renderer
[[ImGuiPlots sharedInstance] clearData];
DataManager* dataMan = [[DataManager alloc] init];
if ([[dataMan getSettings].renderingBackend integerValue] == RENDER_AVSB) {
// PACING_MODE_VSYNC:
@@ -221,7 +223,7 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size,
// so this is safe to do right here.
[self->_displayLayer flushAndRemoveImage];
[self reinitializeDisplayLayer];
// Request an IDR frame to initialize the new decoder
LiRequestIdrFrame();
}
@@ -349,8 +351,8 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit);
}
- (void)cleanup
{
[_frameQueue shutdown];
[_frameQueue stop];
if (_renderingBackend == RENDER_AVSB) {
[_displayLink invalidate];
}
@@ -745,7 +747,7 @@ int DrSubmitDecodeUnit(PDECODE_UNIT decodeUnit);
free(data);
return DR_NEED_IDR;
}
// Now we're decoding actual frame data here
CMBlockBufferRef frameBlockBuffer;
CMBlockBufferRef dataBlockBuffer;
VoidLink/Utility/FrameQueue.h
+2 -2
View File
@@ -12,7 +12,7 @@ NS_ASSUME_NONNULL_BEGIN
@property (nonatomic) FloatBuffer *frameDropMetrics;
@property (nonatomic) int highWaterMark;
@property (nonatomic, readonly) int maxCapacity;
@property (atomic) BOOL isStopping;
@property (atomic) BOOL paused;
+ (instancetype)sharedInstance;
@@ -27,8 +27,8 @@ NS_ASSUME_NONNULL_BEGIN
- (CFTimeInterval)estimatedFramerate;
- (int)currentSoftCap;
- (void)waitForEnqueue;
- (void)shutdown;
- (void)start;
- (void)stop;
@end
VoidLink/Utility/FrameQueue.m
+11 -18
View File
@@ -60,7 +60,7 @@
_ptsCorrection = CMTimeMake(0, 90000);
_queueSizeHistory = [[FloatBuffer alloc] initWithCapacity:64];
_lock = OS_UNFAIR_LOCK_INIT;
_isStopping = NO;
_paused = YES; // caller will call start()
// ring buffer
_capacity = _maxCapacity;
@@ -226,7 +226,7 @@
// Allows the render loop to wait if the queue is empty
- (void)waitForEnqueue {
while (!self.isStopping && [self isEmpty]) {
while (!self.paused && [self isEmpty]) {
dispatch_time_t timeout = dispatch_time(DISPATCH_TIME_NOW, (int64_t)(0.1f * NSEC_PER_SEC)); // 100ms
dispatch_semaphore_wait(_frameSemaphore, timeout);
}
@@ -256,7 +256,7 @@
CFTimeInterval deadline = start + timeout;
int round = 0;
if (self.isStopping) {
if (self.paused) {
return nil;
}
@@ -287,15 +287,10 @@
return [self count] == 0;
}
- (void)_clear_unsafe {
_head = _tail = _count = 0;
_frameDropMetrics = [[FloatBuffer alloc] initWithCapacity:512];
}
// The public clear method now safely calls the unsafe version.
- (void)clear {
os_unfair_lock_lock(&_lock);
[self _clear_unsafe];
_head = _tail = _count = 0;
_frameDropMetrics = [[FloatBuffer alloc] initWithCapacity:512];
os_unfair_lock_unlock(&_lock);
}
@@ -329,19 +324,17 @@
return cap;
}
- (void)shutdown {
- (void)stop {
// new frames will no longer be coming in, make sure consumer side is not left waiting
self.isStopping = YES;
Log(LOG_I, @"XXX FrameQueue shutting down");
self.paused = YES;
Log(LOG_I, @"FrameQueue stopped");
dispatch_semaphore_signal(_frameSemaphore);
}
- (void)start {
os_unfair_lock_lock(&_lock);
[self _clear_unsafe]; // Use the version without the lock
self.isStopping = NO;
os_unfair_lock_unlock(&_lock);
Log(LOG_I, @"FrameQueue (re)started");
// (re)start for a new renderer
self.paused = NO;
Log(LOG_I, @"FrameQueue started");
}
// For use with NSLog("%@", franeQueue);
VoidLink/ViewControllers/StreamFrameViewController.m
+1 -1
View File
@@ -663,6 +663,7 @@
if ([_settings.renderingBackend intValue] == RENDER_METAL) {
// Metal view for video
Log(LOG_I, @"StreamFrameViewController creating MetalViewController");
self.metalViewController = [[MetalViewController alloc] initWithFrame:self.view.bounds
framerate:[self->_settings.framerate floatValue]
enableHdr:self->_settings.enableHdr
@@ -745,7 +746,6 @@
_inactivityTimer = nil;
}
if (self.metalViewController) {
[self.metalViewController shutdown];
[self.metalViewController.view removeFromSuperview];
[self.metalViewController removeFromParentViewController];
self.metalViewController = nil;