toolbox pip entry

VoidLink.xcodeproj/project.pbxproj

@@ -7,6 +7,7 @@
 	objects = {
 
 /* Begin PBXBuildFile section */
+		1829CE3F2E2154DF0074A99A /* SceneDelegate.m in Sources */ = {isa = PBXBuildFile; fileRef = 1829CE3E2E2154DF0074A99A /* SceneDelegate.m */; };
 		18341A042DEB662C0094E5F2 /* FixedTintImageView.m in Sources */ = {isa = PBXBuildFile; fileRef = 18341A032DEB662C0094E5F2 /* FixedTintImageView.m */; };
 		187F4B282E04658A00EECC59 /* WidgetPanelStackView.m in Sources */ = {isa = PBXBuildFile; fileRef = 187F4B272E04658A00EECC59 /* WidgetPanelStackView.m */; };
 		189AD29A2DE7446000E24B88 /* HostCollectionViewController.m in Sources */ = {isa = PBXBuildFile; fileRef = 189AD2992DE7446000E24B88 /* HostCollectionViewController.m */; };
@@ -130,6 +131,8 @@
 /* End PBXContainerItemProxy section */
 
 /* Begin PBXFileReference section */
+		1829CE3D2E2154DF0074A99A /* SceneDelegate.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = SceneDelegate.h; sourceTree = "<group>"; };
+		1829CE3E2E2154DF0074A99A /* SceneDelegate.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = SceneDelegate.m; sourceTree = "<group>"; };
 		18341A022DEB65AB0094E5F2 /* FixedTintImageView.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = FixedTintImageView.h; sourceTree = "<group>"; };
 		18341A032DEB662C0094E5F2 /* FixedTintImageView.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = FixedTintImageView.m; sourceTree = "<group>"; };
 		187F4B232E04633100EECC59 /* WidgetPanelStackView.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = WidgetPanelStackView.h; sourceTree = "<group>"; };
@@ -495,6 +498,8 @@
 				FBD3495F1A004453002D2A60 /* Database */,
 				FB290CFA19B2C406004C83CF /* Supporting Files */,
 				FB290D0219B2C406004C83CF /* AppDelegate.h */,
+				1829CE3D2E2154DF0074A99A /* SceneDelegate.h */,
+				1829CE3E2E2154DF0074A99A /* SceneDelegate.m */,
 				18341A022DEB65AB0094E5F2 /* FixedTintImageView.h */,
 				18341A032DEB662C0094E5F2 /* FixedTintImageView.m */,
 				FB290D0319B2C406004C83CF /* AppDelegate.m */,
@@ -907,6 +912,7 @@
 				84351B102C29ACCB00A77989 /* OSCProfile.m in Sources */,
 				84351B2C2C29AD6100A77989 /* OSCProfilesTableViewController.m in Sources */,
 				98D585701C0ED0E800F6CC00 /* TemporarySettings.m in Sources */,
+				1829CE3F2E2154DF0074A99A /* SceneDelegate.m in Sources */,
 				FB89462819F646E200339C8A /* CryptoManager.m in Sources */,
 				DC1F5A07206436B20037755F /* ConnectionHelper.m in Sources */,
 				FB89462E19F646E200339C8A /* PairManager.m in Sources */,

VoidLink/Input/StreamView.h

@@ -24,6 +24,7 @@
 - (void)toggleStatsOverlay;
 - (void)localMousePointerMode;
 - (void)toggleMouseVisible;
+- (bool)isPipActive;
 
 @end
 
@@ -63,6 +64,7 @@
 - (void)keyboardWillShow:(NSNotification *)notification;
 - (void)keyboardWillHide;
 
+- (bool)isPipActive;
 
 #if !TARGET_OS_TV
 - (void) updateCursorLocation:(CGPoint)location isMouse:(BOOL)isMouse;

VoidLink/Input/StreamView.m

@@ -364,6 +364,10 @@ static const double X1_MOUSE_SPEED_DIVISOR = 2.5;
 #endif
 }
 
+- (bool)isPipActive{
+    return [interactionDelegate isPipActive];
+}
+
 // we'll enable on screen buttons, and disable on screen controllers for absolute touch
 - (bool) isOscEnabled{
     return (settings.touchMode.intValue == RelativeTouch || settings.touchMode.intValue == NativeTouch) && settings.onscreenControls.intValue != OnScreenControlsLevelOff;

VoidLink/Stream/Connection.m

@@ -255,7 +255,9 @@ int ArInit(int audioConfiguration, POPUS_MULTISTREAM_CONFIGURATION opusConfig, v
     SDL_PauseAudioDevice(audioDevice, 0);
     
     // Disable lowering volume of other audio streams (SDL sets AVAudioSessionCategoryOptionDuckOthers by default)
+
     [[AVAudioSession sharedInstance] setCategory:AVAudioSessionCategoryPlayback withOptions:AVAudioSessionCategoryOptionMixWithOthers error:nil];
+ 
     
     return 0;
 }

VoidLink/Stream/VideoDecoderRenderer.m

@@ -20,7 +20,7 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size,
                               int write_seq_header);
 
 @implementation VideoDecoderRenderer {
-    StreamView* _view;
+    UIView* _view;
     id<ConnectionCallbacks> _callbacks;
     float _streamAspectRatio;
     
@@ -39,11 +39,21 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size,
 
 - (void)reinitializeDisplayLayer
 {
-    if (_displayLayer == nil) {
+    CALayer *oldLayer = _displayLayer;
+    bool pipIsActive = [(StreamView* )_view.superview isPipActive];
+    
+    NSLog(@"pipIsActive: %d,    %f", pipIsActive, CACurrentMediaTime());
+        
+    if (!pipIsActive) {
         _displayLayer = [[AVSampleBufferDisplayLayer alloc] init];
-        _displayLayer.backgroundColor = [UIColor blackColor].CGColor;
+        _displayLayer.backgroundColor = [UIColor greenColor].CGColor;
-        [_view.layer addSublayer:_displayLayer];
+        //if(pipIsActive) [_view.layer addSublayer:_displayLayer];
     }
+
+
+    
+    //_displayLayer = [[AVSampleBufferDisplayLayer alloc] init];
+    //_displayLayer.backgroundColor = [UIColor blackColor].CGColor;
     
     // Ensure the AVSampleBufferDisplayLayer is sized to preserve the aspect ratio
     // of the video stream. We used to use AVLayerVideoGravityResizeAspect, but that
@@ -64,13 +74,23 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size,
     // can see the loading progress label as the stream is starting.
     _displayLayer.hidden = YES;
     
+    if(!pipIsActive){
+        if (oldLayer != nil) {
+            // Switch out the old display layer with the new one
+            [_view.layer replaceSublayer:oldLayer with:_displayLayer];
+        }
+        else {
+            [_view.layer addSublayer:_displayLayer];
+        }
+    }
+    
     if (formatDesc != nil) {
         CFRelease(formatDesc);
         formatDesc = nil;
     }
 }
 
-- (id)initWithView:(StreamView*)view callbacks:(id<ConnectionCallbacks>)callbacks streamAspectRatio:(float)aspectRatio useFramePacing:(BOOL)useFramePacing
+- (id)initWithView:(UIView*)view callbacks:(id<ConnectionCallbacks>)callbacks streamAspectRatio:(float)aspectRatio useFramePacing:(BOOL)useFramePacing
 {
     NSLog(@"initializing video decoder %f", CACurrentMediaTime());
     self = [super init];

VoidLink/ViewControllers/StreamFrameViewController.m

@@ -169,7 +169,7 @@
 
     if ([AVPictureInPictureController isPictureInPictureSupported]) {
         if (@available(iOS 15.0, *)) {
-            self.pipContentSource = [[AVPictureInPictureControllerContentSource alloc] initWithSampleBufferDisplayLayer:streamLayer playbackDelegate:self];
+            self.pipContentSource = [[AVPictureInPictureControllerContentSource alloc] initWithSampleBufferDisplayLayer:streamLayer playbackDelegate:(id<AVPictureInPictureSampleBufferPlaybackDelegate>)self];
             self.pipController = [[AVPictureInPictureController alloc] initWithContentSource:self.pipContentSource];
             self.pipController.canStartPictureInPictureAutomaticallyFromInline = YES;
         } else {
@@ -329,20 +329,24 @@
     [super viewDidAppear:animated];
     
     _deviceWindow = self.view.window;
-    UIScreen *currentScreen = self.view.window.windowScene.screen;
+    if (@available(iOS 13.0, *)) {
-    if (UIScreen.screens.count > 1 && [self isAirPlayEnabled] && currentScreen == UIScreen.mainScreen) {
+        UIScreen *currentScreen = self.view.window.windowScene.screen;
-        [SceneDelegate setExternalDisplayRenderView:self->_streamVideoRenderView];
+        if (UIScreen.screens.count > 1 && [self isAirPlayEnabled] && currentScreen == UIScreen.mainScreen) {
-    }
+            [SceneDelegate setExternalDisplayRenderView:self->_streamVideoRenderView];
-    else {
+        }
-        /*
+        else {
-         _settings.externalDisplayMode.intValue:
+            /*
-         0 - stage manager
+             _settings.externalDisplayMode.intValue:
-         1 - airplay
+             0 - stage manager
-         2 - disabled
+             1 - airplay
-         */
+             2 - disabled
-        dispatch_async(dispatch_get_main_queue(), ^{
+             */
-            if(self->_settings.externalDisplayMode.intValue != 2) [self->_streamView insertSubview:self->_streamVideoRenderView atIndex:0];
+            dispatch_async(dispatch_get_main_queue(), ^{
-        });
+                [self->_streamView insertSubview:self->_streamVideoRenderView atIndex:0];
+            });
+        }
+    } else {
+        // Fallback on earlier versions
     }
 
     self->_streamView.originalFrame = self->_streamView.frame;
@@ -620,6 +624,10 @@
     [self->_streamView readyToBringUpSoftKeyboardByToolbox];
 }
 
+- (void)enterPip{
+    [self.pipController startPictureInPicture];
+}
+
 - (void)oscLayoutClosed{
     // Handle the callback
     [self->_streamView disableOnScreenControls]; // add this to get realtime back menu working.
@@ -833,11 +841,16 @@
 
 // This will fire if the user opens control center or gets a low battery message
 - (void)applicationWillResignActive:(NSNotification *)notification {
+    
+    //[self.pipController startPictureInPicture];
+    //sleep(1);
+    
     if (_inactivityTimer != nil) {
         [_inactivityTimer invalidate];
         _inactivityTimer = nil;
     }
     
+    
 #if !TARGET_OS_TV
     // Terminate the stream if the app is inactive for 60 seconds
     Log(LOG_I, @"Starting inactivity termination timer");
@@ -867,33 +880,30 @@
     if (self.pipController && self.pipController.isPictureInPictureActive) {
         [self.pipController stopPictureInPicture];
     }
+    
+    self.pipController = nil;
+    [self setupPiPControllerWithRenderer:self->_streamMan.videoRenderer];
+    //self->_streamMan.videoRenderer.
+}
+
+- (bool)isPipActive{
+    return self.pipController.isPictureInPictureActive;
 }
 
 // This fires when the home button is pressed
 - (void)applicationDidEnterBackground:(UIApplication *)application {
-
-    Log(LOG_I, @"Terminating stream immediately for backgrounding");
-
-    if (_inactivityTimer != nil) {
-        [_inactivityTimer invalidate];
-        _inactivityTimer = nil;
-    }
     
     // [self returnToMainFrame];
     // Don't terminate if pip is active
+    // [self.pipController startPictureInPicture];
+
+    NSLog(@"did enter background, %d, %@, %d", _settings.enablePIP, self.pipController, self.pipController.isPictureInPictureActive);
     if (_settings.enablePIP && self.pipController && self.pipController.isPictureInPictureActive) {
         Log(LOG_I, @"PIP is active, not terminating stream");
         return;
     }
 
 #if !TARGET_OS_TV
-    // Terminate the stream if the app is inactive for 60 seconds
-    Log(LOG_I, @"Starting inactivity termination timer");
-    _inactivityTimer = [NSTimer scheduledTimerWithTimeInterval:60
-                                                      target:self
-                                                    selector:@selector(inactiveTimerExpired:)
-                                                    userInfo:nil
-                                                     repeats:NO];
 #endif
 }
 
@@ -1173,11 +1183,11 @@
 
 - (void) videoContentShown {
     dispatch_async(dispatch_get_main_queue(), ^{
-        [_spinner stopAnimating];
+        [self->_spinner stopAnimating];
         [self.view setBackgroundColor:[UIColor blackColor]];
 
         if (@available(iOS 15.0, *)) {
-            if (_settings.enablePIP) {
+            if (self->_settings.enablePIP) {
                 if (self->_streamMan && self->_streamMan.videoRenderer) {
                     Log(LOG_I, @"Setting up PiP with renderer: %p", self->_streamMan.videoRenderer);
                     [self setupPiPControllerWithRenderer:self->_streamMan.videoRenderer];

VoidLink/ViewControllers/ToolboxViewController.swift

@@ -12,6 +12,7 @@ import UIKit
     @objc optional func openWidgetLayoutTool()
     @objc optional func switchWidgetProfile()
     @objc optional func bringUpSoftKeyboard()
+    @objc optional func enterPip()
 }
 
 
@@ -27,11 +28,12 @@ import UIKit
     private let viewBackgroundColor = UIColor(white: 0.2, alpha: 0.8);
     private let highlightColor = UIColor(white: 0.39, alpha: 0.8);
     private let titleLabel = UILabel()
-    @objc public var specialEntries : NSMutableArray = ["widgetSwitchTool", "widgetLayoutTool", "bringUpSoftKeyboard"]
+    @objc public var specialEntries : NSMutableArray = ["widgetSwitchTool", "widgetLayoutTool", "bringUpSoftKeyboard", "enterPip"]
     private let specialEntryAliasDic : [String:String] = [
         "widgetSwitchTool":SwiftLocalizationHelper.localizedString(forKey: "[ Switch among on-screen widget profiles ]"),
         "widgetLayoutTool":SwiftLocalizationHelper.localizedString(forKey: "[ Open on-screen widget tool ]"),
-        "bringUpSoftKeyboard":SwiftLocalizationHelper.localizedString(forKey: "[ Bring up soft keyboard ]")
+        "bringUpSoftKeyboard":SwiftLocalizationHelper.localizedString(forKey: "[ Bring up soft keyboard ]"),
+        "enterPip":SwiftLocalizationHelper.localizedString(forKey: "[ Enter picture-in-picture mode ]")
     ]
     
     private var viewPinned: Bool = false
@@ -374,6 +376,8 @@ import UIKit
             specialEntryDelegate?.switchWidgetProfile?()
         case "bringUpSoftKeyboard":
             specialEntryDelegate?.bringUpSoftKeyboard?()
+        case "enterPip":
+            specialEntryDelegate?.enterPip?()
         default: break
         }
     }