make OSC + native touch works

Limelight/Input/NativeTouchHandler.h

@@ -14,6 +14,9 @@
 NS_ASSUME_NONNULL_BEGIN
 
 @interface NativeTouchHandler : UIResponder
+@property (nonatomic, assign) NSMutableSet<UITouch *>* touchesCapturedByOnScreenButtons;
+
+
 - (id)initWithView:(StreamView* )view andSettings:(TemporarySettings*)settings;
 
 

Limelight/Input/NativeTouchHandler.m

@@ -9,16 +9,20 @@
 #import <Foundation/Foundation.h>
 #import "NativeTouchHandler.h"
 #import "NativeTouchPointer.h"
+#import "OnScreenControls.h"
 #import "StreamView.h"
 #import "DataManager.h"
 
 #include <Limelight.h>
 
 
+
+
 @implementation NativeTouchHandler {
     StreamView* streamView;
     TemporarySettings* currentSettings;
     bool activateCoordSelector;
+    bool notPureNativeTouchMode;
     // Use a Dictionary to store UITouch object's memory address as key, and pointerId as value,字典存放UITouch对象地址和pointerId映射关系
     // pointerId will be generated from a pre-defined pool
     // Use a NSSet store active pointerId,
@@ -34,6 +38,8 @@
     self->streamView = view;
     self->currentSettings = settings;
     self->activateCoordSelector = currentSettings.pointerVelocityModeDivider.floatValue != 1.0;
+    self->notPureNativeTouchMode = !(currentSettings.touchMode.intValue == PURE_NATIVE_TOUCH);
+    //_touchesCapturedByOnScreenButtons = [[NSMutableSet alloc] init];
     
     pointerIdDict = [NSMutableDictionary dictionary];
     pointerIdPool = [NSMutableSet set];
@@ -45,6 +51,7 @@
     [NativeTouchPointer setPointerVelocityDivider:settings.pointerVelocityModeDivider.floatValue];
     [NativeTouchPointer setPointerVelocityFactor:settings.touchPointerVelocityFactor.floatValue];
     [NativeTouchPointer initContextWithView:self->streamView];
+    //_touchesCapturedByOnScreenButtons = [[NSMutableSet alloc] init];
     return self;
 }
 
@@ -94,6 +101,8 @@
 
 - (void)touchesBegan:(NSSet *)touches withEvent:(UIEvent *)event {
     for (UITouch* touch in touches){
+        // continue to the next loop if current touch is already captured by OSC. works only in regular native touch
+        if(notPureNativeTouchMode && [OnScreenControls.touchAddrsCapturedByOnScreenControls containsObject:@((uintptr_t)touch)]) continue;
         [self populatePointerId:touch]; //generate & populate pointerId
         if(activateCoordSelector) [NativeTouchPointer populatePointerObjIntoDict:touch];
         [self sendTouchEvent:touch withTouchtype:LI_TOUCH_EVENT_DOWN];
@@ -101,7 +110,10 @@
 }
 
 - (void)touchesMoved:(NSSet *)touches withEvent:(UIEvent *)event {
+    // NSLog(@"captured by OSB touches: %d", (uint32_t)[OnScreenControls.touchAddrsCapturedByOnScreenControls count]);
     for (UITouch* touch in touches){
+        // continue to the next loop if current touch is already captured by OSC. works only in regular native touch
+        if(notPureNativeTouchMode && [OnScreenControls.touchAddrsCapturedByOnScreenControls containsObject:@((uintptr_t)touch)]) continue;
         if(activateCoordSelector) [NativeTouchPointer updatePointerObjInDict:touch];
         [self sendTouchEvent:touch withTouchtype:LI_TOUCH_EVENT_MOVE];
         [[NativeTouchPointer getPointerObjFromDict:touch] doesNeedResetCoords]; // execute the judging of doesReachBoundary for current pointer instance. (happens after the event is sent to Sunshine service)
@@ -112,6 +124,8 @@
 
 - (void)touchesEnded:(NSSet *)touches withEvent:(UIEvent *)event {
     for (UITouch* touch in touches){
+        // continue to the next loop if current touch is already captured by OSC. works only in regular native touch
+        if(notPureNativeTouchMode && [OnScreenControls.touchAddrsCapturedByOnScreenControls containsObject:@((uintptr_t)touch)]) continue;
         [self sendTouchEvent:touch withTouchtype:LI_TOUCH_EVENT_UP]; //send touch event before remove pointerId
         [self removePointerId:touch]; //then remove pointerId
         if(activateCoordSelector) [NativeTouchPointer removePointerObjFromDict:touch];

Limelight/Input/OnScreenControls.h

@@ -35,6 +35,7 @@ typedef NS_ENUM(NSInteger, OnScreenControlsLevel) {
 
 // @property (nonatomic, assign) CustomTapGestureRecognizer* mouseRightClickTapRecognizer; // this object will be passed to onscreencontrols class for areVirtualControllerTaps flag setting
 @property (nonatomic, assign) bool isLayingOut;
+//@property (nonatomic) NSMutableSet<UITouch* >* touchesCapturedByOnScreenButtons;
 @property CALayer* _aButton;
 @property CALayer* _bButton;
 @property CALayer* _xButton;
@@ -64,6 +65,8 @@ typedef NS_ENUM(NSInteger, OnScreenControlsLevel) {
 
 @property NSMutableArray *OSCButtonLayers;
 
+
++ (NSMutableSet* )touchAddrsCapturedByOnScreenControls;
 - (id) initWithView:(UIView*)view controllerSup:(ControllerSupport*)controllerSupport streamConfig:(StreamConfiguration*)streamConfig;
 - (BOOL) handleTouchDownEvent:(NSSet*)touches;
 - (BOOL) handleTouchUpEvent:(NSSet*)touches;

Limelight/Input/OnScreenControls.m

@@ -19,8 +19,9 @@
 #define UPDATE_BUTTON(x, y) (buttonFlags = \
 (y) ? (buttonFlags | (x)) : (buttonFlags & ~(x)))
 
-@implementation OnScreenControls {
+static NSMutableSet* touchAddrsCapturedByOnScreenControls;
 
+@implementation OnScreenControls {
 
     UITouch* _aTouch;
     UITouch* _bTouch;
@@ -125,6 +126,9 @@ static float L2_Y;
 static float L3_X;
 static float L3_Y;
 
++ (NSMutableSet* )touchAddrsCapturedByOnScreenControls{
+    return touchAddrsCapturedByOnScreenControls;
+}
 
 
 - (id) initWithView:(UIView*)view controllerSup:(ControllerSupport*)controllerSupport streamConfig:(StreamConfiguration*)streamConfig {
@@ -225,6 +229,7 @@ static float L3_Y;
     _leftButton.name = @"leftButton";
     
     _activeCustomOscButtonPositionDict = [[NSMutableDictionary alloc] init];
+    touchAddrsCapturedByOnScreenControls = [[NSMutableSet alloc] init];
     
     return self;
 }
@@ -1037,10 +1042,15 @@ static float L3_Y;
             updated = true;
             // NSLog(@"blablablab updated true");
         }
+        
+        // additionally, populate the touchesCapturedByOSButton set, for native touch handler to deal with.
+        if(updated || stickTouch) [touchAddrsCapturedByOnScreenControls addObject:@((uintptr_t)touch)];
     }
     if (updated) {
         [_controllerSupport updateFinished:_controller];
     }
+    // NSLog(@"captured by OSB touches, OSC Class: %d", (uint32_t)[touchAddrsCapturedByOnScreenControls count]);
+
     
     bool oscTouched = updated || stickTouch;
     if(oscTouched){
@@ -1061,6 +1071,10 @@ static float L3_Y;
     BOOL updated = false;
     BOOL touched = false;
     for (UITouch* touch in touches) {
+        
+        // remove the touch obj from touchesCapturedByOnScreenButtons
+        if([touchAddrsCapturedByOnScreenControls containsObject:@((uintptr_t)touch)]) [touchAddrsCapturedByOnScreenControls removeObject:@((uintptr_t)touch)];
+        
         if (touch == _aTouch) {
             [_controllerSupport clearButtonFlag:_controller flags:A_FLAG];
             _aTouch = nil;

Limelight/Input/StreamView.m

@@ -125,7 +125,7 @@ static const double X1_MOUSE_SPEED_DIVISOR = 2.5;
             break;
     }
     
-    onScreenControls = [[OnScreenControls alloc] initWithView:self controllerSup:controllerSupport streamConfig:streamConfig];
+    onScreenControls = [[OnScreenControls alloc] initWithView:self controllerSup:controllerSupport streamConfig:streamConfig];  // don't delete, this is mandatory
     /*
     // here we pass the tap recognizer to the onscreencontrols obj
     if (settings.touchMode.intValue == RELATIVE_TOUCH){
@@ -137,6 +137,17 @@ static const double X1_MOUSE_SPEED_DIVISOR = 2.5;
     if (settings.touchMode.intValue != RELATIVE_TOUCH && settings.touchMode.intValue != REGULAR_NATIVE_TOUCH ) {
         Log(LOG_I, @"On-screen controls disabled in non-relative touch mode");
         [onScreenControls setLevel:OnScreenControlsLevelOff];
+        
+        //pass touchesCaptureByOnScreenButtons Set to the native touchhandler, this NSSet is init witihin onscreencontrols class, don't do it again in native touch handler class
+        [OnScreenControls.touchAddrsCapturedByOnScreenControls removeAllObjects]; // reset the attribute to nil
+        
+        /*
+        if(settings.touchMode.intValue == REGULAR_NATIVE_TOUCH){
+            NativeTouchHandler* nativeTouchHandler = (NativeTouchHandler* )touchHandler;
+            nativeTouchHandler.touchesCapturedByOnScreenButtons = onScreenControls.touchesCapturedByOnScreenButtons;
+            touchHandler = nativeTouchHandler;
+        }; // pass touchHandler to onScreenControl
+         */
     }
     //else if (level == OnScreenControlsLevelAuto) {
     else if (false) { // level auto cancelled in settings
@@ -507,16 +518,11 @@ static const double X1_MOUSE_SPEED_DIVISOR = 2.5;
     // Notify of user interaction and start expiration timer
     [self startInteractionTimer];
     
-    if (![onScreenControls handleTouchDownEvent:touches]) {
+    if(settings.touchMode.intValue == REGULAR_NATIVE_TOUCH){
-        // We still inform the touch handler even if we're going trigger the
+        [onScreenControls handleTouchDownEvent:touches];
-        // keyboard activation gesture. This is important to ensure the touch
-        // handler has a consistent view of touch events to correctly suppress
-        // activation of one or two finger gestures when a three finger gesture
-        // is triggered.
         [touchHandler touchesBegan:touches withEvent:event];
-        // I refactored keyboard toggle by the CustomTapGestureRecognizer
-        // if ([[event allTouches] count] == keyboardToggleFingers) [self toggleKeyboard];
     }
+    else if(![onScreenControls handleTouchDownEvent:touches]) [touchHandler touchesBegan:touches withEvent:event];
 }
 
 - (UIBarButtonItem *)createButtonWithImageNamed:(NSString *)imageName backgroundColor:(UIColor *)backgroundColor target:(id)target action:(SEL)action keyCode:(NSInteger)keyCode isToggleable:(BOOL)isToggleable {
@@ -646,12 +652,10 @@ static const double X1_MOUSE_SPEED_DIVISOR = 2.5;
         return; //This is a native touch oriented fork, in pure native touch mode, this call back method deals with native touch only.
     }
     
-    
     for (UITouch* touch in touches) {
         if (touch.type == UITouchTypePencil) {
             if ([self sendStylusEvent:touch]) return;
         }
-        
         if (@available(iOS 13.4, *)) {
             UITouch *touch = [touches anyObject];
             if (touch.type == UITouchTypeIndirectPointer) {
@@ -671,13 +675,15 @@ static const double X1_MOUSE_SPEED_DIVISOR = 2.5;
                 return;
             }
         }
-        
 #endif
+    }
+    
     hasUserInteracted = YES;
-        if (![onScreenControls handleTouchMovedEvent:touches]) {
+    if(settings.touchMode.intValue == REGULAR_NATIVE_TOUCH){
         [touchHandler touchesMoved:touches withEvent:event];
+        [onScreenControls handleTouchMovedEvent:touches];
     }
-    }
+    else if(![onScreenControls handleTouchMovedEvent:touches]) [touchHandler touchesMoved:touches withEvent:event];
 }
 
 - (void)pressesBegan:(NSSet<UIPress *> *)presses withEvent:(UIPressesEvent *)event {
@@ -746,9 +752,11 @@ static const double X1_MOUSE_SPEED_DIVISOR = 2.5;
     
     hasUserInteracted = YES;
     
-    if (![onScreenControls handleTouchUpEvent:touches]) {
+    if(settings.touchMode.intValue == REGULAR_NATIVE_TOUCH){
-        [touchHandler touchesEnded:touches withEvent:event];
+        [touchHandler touchesEnded:touches withEvent:event]; // when touches ended, must call the native touchhandler before onScreenControls, since the NSSet of touches captured by on screen button shall be updated later
+        [onScreenControls handleTouchUpEvent:touches];
     }
+    else if(![onScreenControls handleTouchUpEvent:touches]) [touchHandler touchesEnded:touches withEvent:event];
 }
 
 - (void)touchesCancelled:(NSSet *)touches withEvent:(UIEvent *)event {