From 15bc488f715e1f1de507913b93c3b3ebeea9c5cd Mon Sep 17 00:00:00 2001 From: TrueZhuangjia Date: Sat, 14 Sep 2024 12:29:31 +0800 Subject: [PATCH] - Completely refactor relative touch mode: implements quick double tap for dragging, simulates a real note book computer touchpad. - Onscreen controllers, buttons & touchpad mouse able to work independently without blocking each other - Tap exclusion area size upto 1000% --- Limelight/Base.lproj/iPad.storyboard | 4 +- Limelight/Base.lproj/iPhone.storyboard | 4 +- Limelight/Input/CustomTapGestureRecognizer.m | 1 + Limelight/Input/OnScreenControls.m | 33 ++- Limelight/Input/RelativeTouchHandler.m | 268 ++++++++++-------- Limelight/Input/StreamView.h | 2 + Limelight/Input/StreamView.m | 17 +- .../StreamFrameViewController.m | 1 + 8 files changed, 199 insertions(+), 131 deletions(-) diff --git a/Limelight/Base.lproj/iPad.storyboard b/Limelight/Base.lproj/iPad.storyboard index 85c41021..59b26d10 100644 --- a/Limelight/Base.lproj/iPad.storyboard +++ b/Limelight/Base.lproj/iPad.storyboard @@ -534,7 +534,7 @@ ICAgICAgICAgICAgICA - + @@ -940,7 +940,7 @@ ICAgICAgICAgICAgICA - + diff --git a/Limelight/Base.lproj/iPhone.storyboard b/Limelight/Base.lproj/iPhone.storyboard index db950dff..8530760e 100644 --- a/Limelight/Base.lproj/iPhone.storyboard +++ b/Limelight/Base.lproj/iPhone.storyboard @@ -471,7 +471,7 @@ ICAgICAgICAgICAgIA - + @@ -941,7 +941,7 @@ ICAgICAgICAgICAgIA - + diff --git a/Limelight/Input/CustomTapGestureRecognizer.m b/Limelight/Input/CustomTapGestureRecognizer.m index de31fef4..fbc14ea3 100644 --- a/Limelight/Input/CustomTapGestureRecognizer.m +++ b/Limelight/Input/CustomTapGestureRecognizer.m @@ -92,6 +92,7 @@ static CGFloat screenWidthInPoints; - (bool)containOnScreenButtonTaps { NSTimeInterval allFingersTapDownTime = CACurrentMediaTime() - _gestureCapturedTime; //RIGHTCLICK_TAP_DOWN_TIME_THRESHOLD_S has been passed to this recognizer as _tapDownTimeShreshold, we'll decide how to deal with pressed flag of on-screen button views based on tapDownTime bool gotOneButtonPressed = false; + // the tap gestures are all added to the streamFrameTopLayerView, where all the onScreenButtonViews are added. so we can iterate them in this way: for(UIView* view in self.view.subviews){ if ([view isKindOfClass:[OnScreenButtonView class]]) { OnScreenButtonView* buttonView = (OnScreenButtonView*) view; diff --git a/Limelight/Input/OnScreenControls.m b/Limelight/Input/OnScreenControls.m index d5da66b0..4202cb85 100644 --- a/Limelight/Input/OnScreenControls.m +++ b/Limelight/Input/OnScreenControls.m @@ -50,7 +50,7 @@ static NSMutableSet* touchAddrsCapturedByOnScreenControls; BOOL _iPad; CGRect _controlArea; - UIView* _view; + UIView* _view; // the view passed to here, is the streamFrameTopLayerView BOOL _visible; ControllerSupport *_controllerSupport; @@ -933,6 +933,9 @@ static float L3_Y; BOOL updated = false; BOOL stickTouch = false; for (UITouch* touch in touches) { + + bool touchEventCapturedByOsc = false; // this flag will be reset for every touch event in the for-loop + CGPoint touchLocation = [touch locationInView:_view]; if ([_aButton.presentationLayer hitTest:touchLocation]) { @@ -940,24 +943,28 @@ static float L3_Y; _aTouch = touch; [self oscButtonTouchDownVisualEffect:_aButton]; updated = true; + touchEventCapturedByOsc = true; // NSLog(@"Captured OSC A"); } else if ([_bButton.presentationLayer hitTest:touchLocation]) { [_controllerSupport setButtonFlag:_controller flags:B_FLAG]; _bTouch = touch; [self oscButtonTouchDownVisualEffect:_bButton]; updated = true; + touchEventCapturedByOsc = true; // NSLog(@"Captured OSC B"); } else if ([_xButton.presentationLayer hitTest:touchLocation]) { [_controllerSupport setButtonFlag:_controller flags:X_FLAG]; _xTouch = touch; [self oscButtonTouchDownVisualEffect:_xButton]; updated = true; + touchEventCapturedByOsc = true; // NSLog(@"Captured OSC X"); } else if ([_yButton.presentationLayer hitTest:touchLocation]) { [_controllerSupport setButtonFlag:_controller flags:Y_FLAG]; _yTouch = touch; [self oscButtonTouchDownVisualEffect:_yButton]; updated = true; + touchEventCapturedByOsc = true; // NSLog(@"Captured OSC Y"); } else if ([_upButton.presentationLayer hitTest:touchLocation]) { [_controllerSupport setButtonFlag:_controller flags:UP_FLAG]; @@ -965,6 +972,7 @@ static float L3_Y; _upTouch = touch; [self oscButtonTouchDownVisualEffect:_upButton]; updated = true; + touchEventCapturedByOsc = true; // NSLog(@"Captured OSC UP"); } else if ([_downButton.presentationLayer hitTest:touchLocation]) { [_controllerSupport setButtonFlag:_controller flags:DOWN_FLAG]; @@ -972,12 +980,14 @@ static float L3_Y; _downTouch = touch; [self oscButtonTouchDownVisualEffect:_downButton]; updated = true; + touchEventCapturedByOsc = true; } else if ([_leftButton.presentationLayer hitTest:touchLocation]) { [_controllerSupport setButtonFlag:_controller flags:LEFT_FLAG]; _dpadTouch = touch; _leftTouch = touch; [self oscButtonTouchDownVisualEffect:_leftButton]; updated = true; + touchEventCapturedByOsc = true; // NSLog(@"Captured OSC LEFT"); } else if ([_rightButton.presentationLayer hitTest:touchLocation]) { [_controllerSupport setButtonFlag:_controller flags:RIGHT_FLAG]; @@ -985,42 +995,49 @@ static float L3_Y; _rightTouch = touch; [self oscButtonTouchDownVisualEffect:_rightButton]; updated = true; + touchEventCapturedByOsc = true; // NSLog(@"Captured OSC RIGHT"); } else if ([_startButton.presentationLayer hitTest:touchLocation]) { [_controllerSupport setButtonFlag:_controller flags:PLAY_FLAG]; _startTouch = touch; [self oscButtonTouchDownVisualEffect:_startButton]; updated = true; + touchEventCapturedByOsc = true; // NSLog(@"Captured OSC PLAY"); } else if ([_selectButton.presentationLayer hitTest:touchLocation]) { [_controllerSupport setButtonFlag:_controller flags:BACK_FLAG]; _selectTouch = touch; [self oscButtonTouchDownVisualEffect:_selectButton]; updated = true; + touchEventCapturedByOsc = true; // NSLog(@"Captured OSC BACK"); } else if ([_l1Button.presentationLayer hitTest:touchLocation]) { [_controllerSupport setButtonFlag:_controller flags:LB_FLAG]; _l1Touch = touch; [self oscButtonTouchDownVisualEffect:_l1Button]; updated = true; + touchEventCapturedByOsc = true; // NSLog(@"Captured OSC LB"); } else if ([_r1Button.presentationLayer hitTest:touchLocation]) { [_controllerSupport setButtonFlag:_controller flags:RB_FLAG]; _r1Touch = touch; [self oscButtonTouchDownVisualEffect:_r1Button]; updated = true; + touchEventCapturedByOsc = true; // NSLog(@"Captured OSC RB"); } else if ([_l2Button.presentationLayer hitTest:touchLocation]) { [_controllerSupport updateLeftTrigger:_controller left:0xFF]; _l2Touch = touch; [self oscButtonTouchDownVisualEffect:_l2Button]; updated = true; + touchEventCapturedByOsc = true; // NSLog(@"Captured OSC LeftTrigger"); } else if ([_r2Button.presentationLayer hitTest:touchLocation]) { [_controllerSupport updateRightTrigger:_controller right:0xFF]; _r2Touch = touch; [self oscButtonTouchDownVisualEffect:_r2Button]; updated = true; + touchEventCapturedByOsc = true; // NSLog(@"Captured OSC RightTrigger"); } else if ([_l3Button.presentationLayer hitTest:touchLocation]) { if (l3Set) { @@ -1035,6 +1052,7 @@ static float L3_Y; l3Set = !l3Set; _l3Touch = touch; updated = true; + touchEventCapturedByOsc = true; } else if ([_r3Button.presentationLayer hitTest:touchLocation]) { if (r3Set) { [_controllerSupport clearButtonFlag:_controller flags:RS_CLK_FLAG]; @@ -1048,6 +1066,7 @@ static float L3_Y; r3Set = !r3Set; _r3Touch = touch; updated = true; + touchEventCapturedByOsc = true; } else if ([_leftStick.presentationLayer hitTest:touchLocation]) { _leftStick.opacity = 1.0; // make stick opaque while being moved if (l3TouchStart != nil) { @@ -1058,12 +1077,14 @@ static float L3_Y; [_controllerSupport setButtonFlag:_controller flags:LS_CLK_FLAG]; [self oscButtonTouchDownVisualEffect:_leftStick]; updated = true; + touchEventCapturedByOsc = true; // NSLog(@"Captured OSC LS_CLK3"); } } _lsTouch = touch; stickTouch = true; + touchEventCapturedByOsc = true; } else if ([_rightStick.presentationLayer hitTest:touchLocation]) { _rightStick.opacity = 1.0; // make stick opaque while being moved if (r3TouchStart != nil) { @@ -1074,20 +1095,22 @@ static float L3_Y; [_controllerSupport setButtonFlag:_controller flags:RS_CLK_FLAG]; [self oscButtonTouchDownVisualEffect:_rightStick]; updated = true; + touchEventCapturedByOsc = true; // NSLog(@"Captured OSC LS_CLK4"); } } _rsTouch = touch; stickTouch = true; + touchEventCapturedByOsc = true; } if (!updated && !stickTouch && [self isInDeadZone:touch]) { [_deadTouches addObject:touch]; updated = true; + touchEventCapturedByOsc = true; // NSLog(@"blablablab updated true"); } - - // additionally, populate the touchesCapturedByOSButton set, for native touch handler to deal with. - if(updated || stickTouch) [touchAddrsCapturedByOnScreenControls addObject:@((uintptr_t)touch)]; + // additionally, populate the touchesCapturedByOSButton set, for native/relative touch handler to deal with. + if(touchEventCapturedByOsc) [touchAddrsCapturedByOnScreenControls addObject:@((uintptr_t)touch)]; } if (updated) { [_controllerSupport updateFinished:_controller]; @@ -1097,7 +1120,7 @@ static float L3_Y; bool oscTouched = updated || stickTouch; if(oscTouched){ - for (UIGestureRecognizer *gesture in _view.superview.gestureRecognizers) { // we'll iterate the superview(streamframe view, where all the custom gestures are added), instead the streamview, that the osc buttons are pressed + for (UIGestureRecognizer *gesture in _view.gestureRecognizers) { // we'll iterate the streamFrameTopLayerView, which was passed here as _view, where all the custom gestures are added) instead of the streamview, to check if that the osc buttons are pressed if ([gesture isKindOfClass:[CustomTapGestureRecognizer class]]) { // This is a CustomTapGestureRecognizer CustomTapGestureRecognizer *tapGesture = (CustomTapGestureRecognizer *)gesture; diff --git a/Limelight/Input/RelativeTouchHandler.m b/Limelight/Input/RelativeTouchHandler.m index 4900af40..8352986d 100644 --- a/Limelight/Input/RelativeTouchHandler.m +++ b/Limelight/Input/RelativeTouchHandler.m @@ -2,6 +2,7 @@ // RelativeTouchHandler.m // Moonlight // +// Completely refactored by ZWM on 2024.9.13 // Created by Cameron Gutman on 11/1/20. // Copyright © 2020 Moonlight Game Streaming Project. All rights reserved. // @@ -14,26 +15,30 @@ static const int REFERENCE_WIDTH = 1280; static const int REFERENCE_HEIGHT = 720; +static const float QUICK_TAP_TIME_INTERVAL = 0.2; @implementation RelativeTouchHandler { TemporarySettings* currentSettings; - CGPoint touchLocation, originalLocation; - BOOL touchMoved; - BOOL isDragging; - NSTimer* dragTimer; - NSUInteger peakTouchCount; + CGPoint latestMousePointerLocation, initialMousePointerLocation; + CGPoint twoFingerTouchLocation; + NSTimeInterval mousePointerTimestamp; + BOOL mousePointerMoved; + BOOL quickTapDetected; + BOOL mouseLeftButtonHeldDown; + BOOL isInMouseWheelScrollingMode; + UITouch* touchLockedForMouseMove; #if TARGET_OS_TV UIGestureRecognizer* remotePressRecognizer; UIGestureRecognizer* remoteLongPressRecognizer; #endif - UIView* view; + StreamView* streamView; } - (id)initWithView:(StreamView*)view andSettings:(TemporarySettings*)settings { self = [self init]; - self->view = view; + self->streamView = view; self->currentSettings = settings; // replace righclick recoginizing with my CustomTapGestureRecognizer for better experience, higher recoginizing rate. _mouseRightClickTapRecognizer = [[CustomTapGestureRecognizer alloc] initWithTarget:self action:@selector(mouseRightClick)]; @@ -41,9 +46,12 @@ static const int REFERENCE_HEIGHT = 720; _mouseRightClickTapRecognizer.tapDownTimeThreshold = RIGHTCLICK_TAP_DOWN_TIME_THRESHOLD_S; // tap down time in seconds. _mouseRightClickTapRecognizer.delaysTouchesBegan = NO; _mouseRightClickTapRecognizer.delaysTouchesEnded = NO; - [self->view.superview addGestureRecognizer:_mouseRightClickTapRecognizer]; // add all additional gestures to the streamframeview instead of the streamview. - + [self->streamView.streamFrameTopLayerView addGestureRecognizer:_mouseRightClickTapRecognizer]; // add all additional gestures to the streamFrameTopLayerView instead of the streamview. + isInMouseWheelScrollingMode = false; + mousePointerMoved = false; + mouseLeftButtonHeldDown = false; + mousePointerTimestamp = 0; #if TARGET_OS_TV remotePressRecognizer = [[UITapGestureRecognizer alloc] initWithTarget:self action:@selector(remoteButtonPressed:)]; @@ -59,7 +67,8 @@ static const int REFERENCE_HEIGHT = 720; return self; } -- (bool)containOnScreenControllerTaps: (NSSet* )touches{ + +- (bool)containOnScreenControllerTaps:(NSSet* )touches{ for(UITouch* touch in touches){ if([OnScreenControls.touchAddrsCapturedByOnScreenControls containsObject:@((uintptr_t)touch)]) return true; } @@ -69,7 +78,7 @@ static const int REFERENCE_HEIGHT = 720; - (bool)containOnScreenButtonTaps { bool gotOneButtonPressed = false; - for(UIView* view in self->view.superview.subviews){ // iterates all on-screen button views in StreamFrameView + for(UIView* view in self->streamView.superview.subviews){ // iterates all on-screen button views in StreamFrameView if ([view isKindOfClass:[OnScreenButtonView class]]) { OnScreenButtonView* buttonView = (OnScreenButtonView*) view; if(buttonView.pressed){ @@ -81,7 +90,7 @@ static const int REFERENCE_HEIGHT = 720; } - (void)resetAllPressedFlagsForOnscreenButtonViews { - for(UIView* view in self->view.superview.subviews){ // iterates all on-screen button views in StreamFrameView + for(UIView* view in self->streamView.superview.subviews){ // iterates all on-screen button views in StreamFrameView if ([view isKindOfClass:[OnScreenButtonView class]]) { OnScreenButtonView* buttonView = (OnScreenButtonView*) view; buttonView.pressed = false; @@ -105,139 +114,164 @@ static const int REFERENCE_HEIGHT = 720; return hypotf(originalPoint.x - currentPoint.x, originalPoint.y - currentPoint.y) >= 5; } -- (void)onDragStart:(NSTimer*)timer { - if (!touchMoved && !isDragging){ - isDragging = true; - LiSendMouseButtonEvent(BUTTON_ACTION_PRESS, BUTTON_LEFT); - } +- (BOOL)isAdjacentTouches:(CGPoint)currentPoint from:(CGPoint)originalPoint { + // Movements of greater than 5 pixels are considered confirmed + return hypotf(originalPoint.x - currentPoint.x, originalPoint.y - currentPoint.y) <= 100; } + - (void)touchesBegan:(NSSet *)touches withEvent:(UIEvent *)event { - touchMoved = false; - peakTouchCount = [[event allTouches] count]; - if ([[event allTouches] count] == 1) { - UITouch *touch = [[event allTouches] anyObject]; - originalLocation = touchLocation = [touch locationInView:view]; - if (!isDragging) { - dragTimer = [NSTimer scheduledTimerWithTimeInterval:0.650 - target:self - selector:@selector(onDragStart:) - userInfo:nil - repeats:NO]; - } + + if([[event allTouches] count] == 2 && [touches count] != 1 && ![self containOnScreenButtonTaps] && ![self containOnScreenControllerTaps:[event allTouches]]){ + NSLog(@"get in scrolling mode"); + isInMouseWheelScrollingMode = true; + return; // if we got 2 touches on the blank area, it's gonna be a mouse scroll touch, and must prevent UITtouch object for mouse pointer being captured & locked } - else if ([[event allTouches] count] == 2) { - CGPoint firstLocation = [[[[event allTouches] allObjects] objectAtIndex:0] locationInView:view]; - CGPoint secondLocation = [[[[event allTouches] allObjects] objectAtIndex:1] locationInView:view]; - - originalLocation = touchLocation = CGPointMake((firstLocation.x + secondLocation.x) / 2, (firstLocation.y + secondLocation.y) / 2); + + // NSLog(@"touches count in began stage: %llu", (uint64_t)[touches count]); + + UITouch* candidateTouch = nil; + + // the onscreen controllers are implmented by CALayer, which can not intercept UITouch event, touch will penetrate to the streamView level and captured in the touches callback of this touchHandler class. + // the onscreen button are UIViews, they intercept UITouch events, so we don't need to worry about them. + for(UITouch* touch in touches){ + // NSLog(@"candidate touch test: %llu", (uint64_t)touch); + if([OnScreenControls.touchAddrsCapturedByOnScreenControls containsObject:@((uintptr_t)touch)]){ + // NSLog(@"%f controller tap detected", CACurrentMediaTime()); + continue; + } + else candidateTouch = touch; + } + + CGPoint currentTouchLocation = [candidateTouch locationInView:streamView]; + NSTimeInterval clickTimeInterval = CACurrentMediaTime() - mousePointerTimestamp; + // NSLog(@"click interval: %f", clickTimeInterval); + + // quick double tap detection for dragging. simulates a real notebook computer touchpad + if(clickTimeInterval < QUICK_TAP_TIME_INTERVAL && [self isAdjacentTouches:currentTouchLocation from:initialMousePointerLocation] ) { + // NSLog(@"quick click detected"); + quickTapDetected = true; + NSLog(@"quick Tap Detected"); + // LiSendMouseButtonEvent(BUTTON_ACTION_PRESS, BUTTON_LEFT); // do not press down mouse button here, or it wiil easily turn to double click on remote PC + } + + // we must use [event allTouches] to check if touchLockedForMouseMove is captured, because the UITouch object could be captured by upper layer of UIView(in cases like tap gestures), not passed to the touches callbacks in this class, but still available in [event allTouches] + if(candidateTouch != nil && ![[event allTouches] containsObject:touchLockedForMouseMove]){ + touchLockedForMouseMove = candidateTouch; + NSLog(@"%f candidate touch for mouse movement locked, addr: %llu", CACurrentMediaTime(), (uintptr_t)touchLockedForMouseMove); + mousePointerTimestamp = CACurrentMediaTime(); + initialMousePointerLocation = latestMousePointerLocation = [touchLockedForMouseMove locationInView:streamView]; } } - (void)touchesMoved:(NSSet *)touches withEvent:(UIEvent *)event { - if ([[event allTouches] count] == 1) { - UITouch *touch = [[event allTouches] anyObject]; - CGPoint currentLocation = [touch locationInView:view]; - - if (touchLocation.x != currentLocation.x || - touchLocation.y != currentLocation.y) - { - int deltaX = (currentLocation.x - touchLocation.x) * (REFERENCE_WIDTH / view.bounds.size.width) * currentSettings.mousePointerVelocityFactor.floatValue; - int deltaY = (currentLocation.y - touchLocation.y) * (REFERENCE_HEIGHT / view.bounds.size.height) * currentSettings.mousePointerVelocityFactor.floatValue; - - if (deltaX != 0 || deltaY != 0) { - LiSendMouseMoveEvent(deltaX, deltaY); - touchLocation = currentLocation; - - // If we've moved far enough to confirm this wasn't just human/machine error, - // mark it as such. - if ([self isConfirmedMove:touchLocation from:originalLocation]) { - touchMoved = true; - } - } - } - } else if ([[event allTouches] count] == 2) { // mouse wheel scroll & right button click are both triggered by 2 finger gesture, some times cause competing (right click fails & scroll view jumps around). - //I'll deal with this in coming code. + + //NSLog(@"%f, touchesMoved callback, is scrolling: %d, touches count: %d", CACurrentMediaTime(), isInMouseWheelScrollingMode, (uint32_t)[touches count]); + + if(isInMouseWheelScrollingMode){ NSSet* twoTouches = [event allTouches]; - CGPoint firstLocation = [[[twoTouches allObjects] objectAtIndex:0] locationInView:view]; - CGPoint secondLocation = [[[twoTouches allObjects] objectAtIndex:1] locationInView:view]; + CGPoint firstLocation = [[[twoTouches allObjects] objectAtIndex:0] locationInView:streamView]; + CGPoint secondLocation = [[[twoTouches allObjects] objectAtIndex:1] locationInView:streamView]; CGPoint avgLocation = CGPointMake((firstLocation.x + secondLocation.x) / 2, (firstLocation.y + secondLocation.y) / 2); - if ((CACurrentMediaTime() - _mouseRightClickTapRecognizer.gestureCapturedTime > RIGHTCLICK_TAP_DOWN_TIME_THRESHOLD_S) && touchLocation.y != avgLocation.y && ![self containOnScreenButtonTaps] && ![self containOnScreenControllerTaps:twoTouches]) { //prevent sending scrollevent while right click gesture is being recognized. The time threshold is only 150ms, resulting in a barely noticeable delay before the scroll event is activated. + if ((CACurrentMediaTime() - _mouseRightClickTapRecognizer.gestureCapturedTime > RIGHTCLICK_TAP_DOWN_TIME_THRESHOLD_S) && twoFingerTouchLocation.y != avgLocation.y && ![self containOnScreenButtonTaps] && ![self containOnScreenControllerTaps:twoTouches]) { //prevent sending scrollevent while right click gesture is being recognized. The time threshold is only 150ms, resulting in a barely noticeable delay before the scroll event is activated. // and we must exclude onscreen button taps & on-screen controller taps - LiSendHighResScrollEvent((avgLocation.y - touchLocation.y) * 10); + LiSendHighResScrollEvent((avgLocation.y - twoFingerTouchLocation.y) * 10); } + twoFingerTouchLocation = avgLocation; + return; + } - // If we've moved far enough to confirm this wasn't just human/machine error, - // mark it as such. - if ([self isConfirmedMove:firstLocation from:originalLocation]) { - touchMoved = true; + // NSLog(@"%f touchesMoved callback, locked touch: %llu", CACurrentMediaTime(), (uintptr_t)touchLockedForMouseMove); + + if([touches containsObject:touchLockedForMouseMove]){ + mousePointerMoved = true; + if(self->quickTapDetected && !mouseLeftButtonHeldDown) { + NSLog(@"Sending mouse left button down event in touchesMoved callback ..."); + mouseLeftButtonHeldDown = true; + LiSendMouseButtonEvent(BUTTON_ACTION_PRESS, BUTTON_LEFT); // start Dragging... } - touchLocation = avgLocation; + [self sendMouseMoveEvent:touchLockedForMouseMove]; } } - (void)touchesEnded:(NSSet *)touches withEvent:(UIEvent *)event { - [dragTimer invalidate]; - dragTimer = nil; - if (isDragging) { - isDragging = false; - LiSendMouseButtonEvent(BUTTON_ACTION_RELEASE, BUTTON_LEFT); - } else if (!touchMoved) { - /*if (peakTouchCount == 2) { - dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_HIGH, 0), ^{ - Log(LOG_D, @"Sending right mouse button press"); - - LiSendMouseButtonEvent(BUTTON_ACTION_PRESS, BUTTON_RIGHT); - - // Wait 100 ms to simulate a real button press - usleep(100 * 1000); - - LiSendMouseButtonEvent(BUTTON_ACTION_RELEASE, BUTTON_RIGHT); - }); - } else */if (peakTouchCount == 1) { - dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_HIGH, 0), ^{ - if (!self->isDragging){ - Log(LOG_D, @"Sending left mouse button press"); - - LiSendMouseButtonEvent(BUTTON_ACTION_PRESS, BUTTON_LEFT); - - // Wait 100 ms to simulate a real button press - usleep(100 * 1000); - } - self->isDragging = false; - LiSendMouseButtonEvent(BUTTON_ACTION_RELEASE, BUTTON_LEFT); - }); + if(isInMouseWheelScrollingMode){ + isInMouseWheelScrollingMode = false; + return; + } + + if([touches containsObject:touchLockedForMouseMove]){ + if(!mousePointerMoved && !self->quickTapDetected) [self sendLongMouseLeftButtonClickEvent]; + if(self->quickTapDetected){ + // we're in at least the second tap release of the very short time interval after the first tap. + LiSendMouseButtonEvent(BUTTON_ACTION_RELEASE, BUTTON_LEFT); // must release the button first, because the button is likely being held down since the long click turned into a dragging event. + if(!mousePointerMoved) [self sendShortMouseLeftButtonClickEvent]; // if it is a quick tap and the pointer was not moved, we must send another click to simulate double click. + self->quickTapDetected = false; // reset flag + self->mouseLeftButtonHeldDown = false; // reset flag } + touchLockedForMouseMove = nil; + mousePointerMoved = false; } - // We we're moving from 2+ touches to 1. Synchronize the current position - // of the active finger so we don't jump unexpectedly on the next touchesMoved - // callback when finger 1 switches on us. - if ([[event allTouches] count] - [touches count] == 1) { - NSMutableSet *activeSet = [[NSMutableSet alloc] initWithCapacity:[[event allTouches] count]]; - [activeSet unionSet:[event allTouches]]; - [activeSet minusSet:touches]; - touchLocation = [[activeSet anyObject] locationInView:view]; - - // Mark this touch as moved so we don't send a left mouse click if the user - // right clicks without moving their other finger. - touchMoved = true; + if([[event allTouches] count] == [touches count]){ + isInMouseWheelScrollingMode = false; + touchLockedForMouseMove = nil; + mousePointerMoved = false; // need to reset this anyway + [self resetAllPressedFlagsForOnscreenButtonViews]; // reset all pressed flag for on-screen button views after all fingers lifted from screen. } - - if([[event allTouches] count] == [touches count]) [self resetAllPressedFlagsForOnscreenButtonViews]; // reset all pressed flag for on-screen button views after all fingers lifted from screen. } -- (void)touchesCancelled:(NSSet *)touches withEvent:(UIEvent *)event { - [dragTimer invalidate]; - dragTimer = nil; - if (isDragging) { - isDragging = false; - LiSendMouseButtonEvent(BUTTON_ACTION_RELEASE, BUTTON_LEFT); + +- (void)sendMouseMoveEvent:(UITouch* )touch{ + CGPoint currentLocation = [touch locationInView:streamView]; + + if (latestMousePointerLocation.x != currentLocation.x || + latestMousePointerLocation.y != currentLocation.y) + { + int deltaX = (currentLocation.x - latestMousePointerLocation.x) * (REFERENCE_WIDTH / streamView.bounds.size.width) * currentSettings.mousePointerVelocityFactor.floatValue; + int deltaY = (currentLocation.y - latestMousePointerLocation.y) * (REFERENCE_HEIGHT / streamView.bounds.size.height) * currentSettings.mousePointerVelocityFactor.floatValue; + + if (deltaX != 0 || deltaY != 0) { + LiSendMouseMoveEvent(deltaX, deltaY); + latestMousePointerLocation = currentLocation; + + // If we've moved far enough to confirm this wasn't just human/machine error, + // mark it as such. + if ([self isConfirmedMove:latestMousePointerLocation from:initialMousePointerLocation]) { + mousePointerMoved = true; + } + } } - peakTouchCount = 0; } + +// this will turn into a dragging anytime... +- (void)sendLongMouseLeftButtonClickEvent{ + dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_HIGH, 0), ^{ + // if (!self->isDragging){ + Log(LOG_D, @"Sending left mouse button press"); + LiSendMouseButtonEvent(BUTTON_ACTION_PRESS, BUTTON_LEFT); + // Wait 100 ms to simulate a real button press + usleep(QUICK_TAP_TIME_INTERVAL * 1000000); + if(!self->quickTapDetected){ + NSLog(@"Left mouse button release cancelled, keep pressing down, turning into dragging..."); + LiSendMouseButtonEvent(BUTTON_ACTION_RELEASE, BUTTON_LEFT); + } + // do not release the button if we're still dragging, this will prevent the dragging from being interrupted. + }); +} + +- (void)sendShortMouseLeftButtonClickEvent{ + dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_HIGH, 0), ^{ + LiSendMouseButtonEvent(BUTTON_ACTION_PRESS, BUTTON_LEFT); + usleep(50 * 1000); + LiSendMouseButtonEvent(BUTTON_ACTION_RELEASE, BUTTON_LEFT); + }); +} + + + #if TARGET_OS_TV - (void)remoteButtonPressed:(id)sender { dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_HIGH, 0), ^{ diff --git a/Limelight/Input/StreamView.h b/Limelight/Input/StreamView.h index e9dfdd35..420eb868 100644 --- a/Limelight/Input/StreamView.h +++ b/Limelight/Input/StreamView.h @@ -24,6 +24,8 @@ @interface StreamView : UIView #endif +@property (assign, nonatomic) UIView* streamFrameTopLayerView; + - (void) setupStreamView:(ControllerSupport*)controllerSupport interactionDelegate:(id)interactionDelegate config:(StreamConfiguration*)streamConfig diff --git a/Limelight/Input/StreamView.m b/Limelight/Input/StreamView.m index 5ac2bbf1..1aa5e096 100644 --- a/Limelight/Input/StreamView.m +++ b/Limelight/Input/StreamView.m @@ -70,6 +70,7 @@ static const double X1_MOUSE_SPEED_DIVISOR = 2.5; streamFrameTopLayerView:(UIView* )topLayerView{ self->streamFrameTopLayerView = topLayerView; + self.streamFrameTopLayerView = topLayerView; // this will be used as a read-only pointer for other class self->interactionDelegate = interactionDelegate; self->streamAspectRatio = (float)streamConfig.width / (float)streamConfig.height; @@ -130,7 +131,8 @@ static const double X1_MOUSE_SPEED_DIVISOR = 2.5; break; } - onScreenControls = [[OnScreenControls alloc] initWithView:self controllerSup:controllerSupport streamConfig:streamConfig]; // don't delete, this is mandatory + // we'll render on-screen controls on the toplayer too: + onScreenControls = [[OnScreenControls alloc] initWithView:self->streamFrameTopLayerView controllerSup:controllerSupport streamConfig:streamConfig]; // don't delete, this is mandatory /* // here we pass the tap recognizer to the onscreencontrols obj if (settings.touchMode.intValue == RELATIVE_TOUCH){ @@ -337,7 +339,9 @@ static const double X1_MOUSE_SPEED_DIVISOR = 2.5; - (void) reloadOnScreenControlsWith:(ControllerSupport*)controllerSupport andConfig:(StreamConfiguration*)streamConfig { - onScreenControls = [[OnScreenControls alloc] initWithView:self controllerSup:controllerSupport streamConfig:streamConfig]; + + // we'll render on-screen controllers on the toplayer too. + onScreenControls = [[OnScreenControls alloc] initWithView:self->streamFrameTopLayerView controllerSup:controllerSupport streamConfig:streamConfig]; /* // pass mouseRightClickTapRecognizer to onScreenControls obj here: if([self isOscEnabled]){ @@ -568,7 +572,7 @@ static const double X1_MOUSE_SPEED_DIVISOR = 2.5; // Notify of user interaction and start expiration timer [self startInteractionTimer]; - if(settings.touchMode.intValue == REGULAR_NATIVE_TOUCH){ + if(settings.touchMode.intValue == REGULAR_NATIVE_TOUCH || settings.touchMode.intValue == RELATIVE_TOUCH){ [onScreenControls handleTouchDownEvent:touches]; [touchHandler touchesBegan:touches withEvent:event]; } @@ -634,6 +638,9 @@ static const double X1_MOUSE_SPEED_DIVISOR = 2.5; } - (BOOL)handleMouseButtonEvent:(int)buttonAction forTouches:(NSSet *)touches withEvent:(UIEvent *)event { + + + // NSLog(@"mouse click time: %f", CACurrentMediaTime()*1000); #if !TARGET_OS_TV if (@available(iOS 13.4, *)) { UITouch* touch = [touches anyObject]; @@ -729,7 +736,7 @@ static const double X1_MOUSE_SPEED_DIVISOR = 2.5; } hasUserInteracted = YES; - if(settings.touchMode.intValue == REGULAR_NATIVE_TOUCH){ + if(settings.touchMode.intValue == REGULAR_NATIVE_TOUCH || settings.touchMode.intValue == RELATIVE_TOUCH){ [touchHandler touchesMoved:touches withEvent:event]; [onScreenControls handleTouchMovedEvent:touches]; } @@ -802,7 +809,7 @@ static const double X1_MOUSE_SPEED_DIVISOR = 2.5; hasUserInteracted = YES; - if(settings.touchMode.intValue == REGULAR_NATIVE_TOUCH){ + if(settings.touchMode.intValue == REGULAR_NATIVE_TOUCH || settings.touchMode.intValue == RELATIVE_TOUCH){ [touchHandler touchesEnded:touches withEvent:event]; // when touches ended, must call the native touchhandler before onScreenControls, since the NSSet of touches captured by on screen button shall be updated later [onScreenControls handleTouchUpEvent:touches]; } diff --git a/Limelight/ViewControllers/StreamFrameViewController.m b/Limelight/ViewControllers/StreamFrameViewController.m index ed636bd8..3a23de84 100644 --- a/Limelight/ViewControllers/StreamFrameViewController.m +++ b/Limelight/ViewControllers/StreamFrameViewController.m @@ -158,6 +158,7 @@ _controllerSupport = [[ControllerSupport alloc] initWithConfig:self.streamConfig delegate:self]; // reload controllerSupport obj, this is mandatory for OSC reload,especially when the stream view is launched without OSC [_streamView setupStreamView:_controllerSupport interactionDelegate:self config:self.streamConfig streamFrameTopLayerView:self.view]; //reinitiate setupStreamView process. + // we got self.view passed to streamView class as the topLayerView, will be useful in many cases [self->_streamView reloadOnScreenControlsRealtimeWith:(ControllerSupport*)_controllerSupport andConfig:(StreamConfiguration*)_streamConfig]; //reload OSC here. [self->_streamView reloadOnScreenKeyboardButtons]; //reload keyboard buttons here. the keyboard button view will be added to the streamframe view instead streamview, the highest layer, which saves a lot of reengineering