From a2c4d440dbc0bfa343dbbc0b694d761e6fef3297 Mon Sep 17 00:00:00 2001 From: Acaki Date: Sun, 24 Aug 2025 20:54:51 +0800 Subject: [PATCH] fix: memory leak for Metal renderer when repeatedly connect/disconnect stream or toggling HDR on host --- VoidLink/Metal/MetalVideoRenderer.m | 80 ++++++++++++++++++++++++---- VoidLink/Metal/MetalViewController.m | 13 ++++- 2 files changed, 82 insertions(+), 11 deletions(-) diff --git a/VoidLink/Metal/MetalVideoRenderer.m b/VoidLink/Metal/MetalVideoRenderer.m index 2375ff4f..b860f88c 100644 --- a/VoidLink/Metal/MetalVideoRenderer.m +++ b/VoidLink/Metal/MetalVideoRenderer.m @@ -163,6 +163,11 @@ CFStringRef __currentColorSpace; _renderPassDescriptor.colorAttachments[0].loadAction = MTLLoadActionClear; _renderPassDescriptor.colorAttachments[0].clearColor = MTLClearColorMake(0, 0, 0, 0); _renderPassDescriptor.colorAttachments[0].storeAction = MTLStoreActionStore; + + // Initialize texture array to NULL + for (int i = 0; i < MAX_VIDEO_PLANES; i++) { + _cvMetalTextures[i] = NULL; + } } return self; } @@ -179,20 +184,36 @@ CFStringRef __currentColorSpace; if (_VideoVertexBuffer) { _VideoVertexBuffer = nil; } + + // Clean up pipeline states for (int i = 0; i < MAX_VIDEO_PLANES; i++) { if (_videoPipelineState[i]) { _videoPipelineState[i] = nil; } _videoPipelinePixelFormat[i] = MTLPixelFormatInvalid; } + + // Clean up any remaining Metal textures + for (int i = 0; i < MAX_VIDEO_PLANES; i++) { + if (_cvMetalTextures[i]) { + CFRelease(_cvMetalTextures[i]); + _cvMetalTextures[i] = NULL; + } + } + + // Properly release texture cache + if (_textureCache) { + CFRelease(_textureCache); + _textureCache = NULL; + } + if (_renderPassDescriptor) { _renderPassDescriptor = nil; } - if (_textureCache) { - _textureCache = nil; - } + if (__currentColorSpace) { CFRelease(__currentColorSpace); + __currentColorSpace = NULL; } } @@ -345,6 +366,17 @@ CFStringRef __currentColorSpace; BOOL fullRange = NO; int colorspace = [self getFrameColorspaceAndRange:frame isFullRange:&fullRange]; if (colorspace != _lastColorSpace || fullRange != _lastFullRange) { + // Clean up any pending textures before colorspace change + for (int i = 0; i < MAX_VIDEO_PLANES; i++) { + if (_cvMetalTextures[i]) { + CFRelease(_cvMetalTextures[i]); + _cvMetalTextures[i] = NULL; + } + } + // Flush texture cache to avoid memory accumulation + if (_textureCache) { + CVMetalTextureCacheFlush(_textureCache, 0); + } CGColorSpaceRef newColorSpace = nil; MTLPixelFormat newPixelFormat = layer.pixelFormat; BOOL isHDR = NO; @@ -436,6 +468,7 @@ CFStringRef __currentColorSpace; #endif if (__currentColorSpace) { CFRelease(__currentColorSpace); + __currentColorSpace = NULL; } __currentColorSpace = CGColorSpaceCopyName(newColorSpace); CGColorSpaceRelease(newColorSpace); @@ -443,11 +476,14 @@ CFStringRef __currentColorSpace; // Create the new colorspace parameter buffer for our fragment shader MTLResourceOptions bufferOptions = MTLResourceStorageModeShared; - _CscParamsBuffer = [_device newBufferWithBytes:(void *)¶mBuffer length:sizeof(paramBuffer) options:bufferOptions]; - if (!_CscParamsBuffer) { + id newCscParamsBuffer = [_device newBufferWithBytes:(void *)¶mBuffer length:sizeof(paramBuffer) options:bufferOptions]; + if (!newCscParamsBuffer) { Log(LOG_E, @"Failed to create CSC parameters buffer"); return NO; } + + // Replace old buffer with new one + _CscParamsBuffer = newCscParamsBuffer; _lastColorSpace = colorspace; _lastFullRange = fullRange; @@ -504,11 +540,14 @@ CFStringRef __currentColorSpace; }; MTLResourceOptions bufferOptions = MTLResourceStorageModeShared; - _VideoVertexBuffer = [_device newBufferWithBytes:verts length:sizeof(verts) options:bufferOptions]; - if (!_VideoVertexBuffer) { + id newVideoVertexBuffer = [_device newBufferWithBytes:verts length:sizeof(verts) options:bufferOptions]; + if (!newVideoVertexBuffer) { Log(LOG_E, @"Failed to create video vertex buffer"); return NO; } + + // Replace old buffer with new one + _VideoVertexBuffer = newVideoVertexBuffer; _lastFrameWidth = [frame width]; _lastFrameHeight = [frame height]; @@ -550,7 +589,9 @@ CFStringRef __currentColorSpace; Log(LOG_I, @"Metal frame changed layer's colorspace and/or pixel format"); // Invalidate all pipeline states since pixel format affects all of them for (int i = 0; i < MAX_VIDEO_PLANES; i++) { - _videoPipelineState[i] = nil; + if (_videoPipelineState[i]) { + _videoPipelineState[i] = nil; + } _videoPipelinePixelFormat[i] = MTLPixelFormatInvalid; } } @@ -700,7 +741,7 @@ CFStringRef __currentColorSpace; for (size_t i = 0; i < planes; i++) { if (self->_cvMetalTextures[i]) { CFRelease(self->_cvMetalTextures[i]); - self->_cvMetalTextures[i] = nil; + self->_cvMetalTextures[i] = NULL; } } @@ -737,6 +778,27 @@ CFStringRef __currentColorSpace; for (NSUInteger i = 0; i < MaxFramesInFlight; i++) { dispatch_semaphore_signal(_inFlightSemaphore); } + + // Clean up any pending Metal textures + for (int i = 0; i < MAX_VIDEO_PLANES; i++) { + if (_cvMetalTextures[i]) { + CFRelease(_cvMetalTextures[i]); + _cvMetalTextures[i] = NULL; + } + } + + // Flush texture cache to free memory + if (_textureCache) { + CVMetalTextureCacheFlush(_textureCache, 0); + } + + // Clear pipeline states + for (int i = 0; i < MAX_VIDEO_PLANES; i++) { + if (_videoPipelineState[i]) { + _videoPipelineState[i] = nil; + } + _videoPipelinePixelFormat[i] = MTLPixelFormatInvalid; + } } } diff --git a/VoidLink/Metal/MetalViewController.m b/VoidLink/Metal/MetalViewController.m index 729100a2..9c5395da 100644 --- a/VoidLink/Metal/MetalViewController.m +++ b/VoidLink/Metal/MetalViewController.m @@ -124,10 +124,19 @@ if (_displayLink) { [_displayLink invalidate]; + _displayLink = nil; } - [_renderer shutdown]; - _renderer = nil; + if (_renderer) { + [_renderer shutdown]; + _renderer = nil; + } + + if (_metalView) { + _metalView.delegate = nil; + [_metalView shutdown]; + _metalView = nil; + } } #if TARGET_OS_IOS