Simplify border detection in native touch modes

Limelight/Input/NativeTouchHandler.m

@@ -11,7 +11,6 @@
 #import "NativeTouchPointer.h"
 #import "OnScreenControls.h"
 #import "StreamView.h"
-#import "DataManager.h"
 
 #include <Limelight.h>
 
@@ -22,7 +21,7 @@
     StreamView* streamView;
     TemporarySettings* currentSettings;
     bool activateCoordSelector;
-    bool notPureNativeTouchMode;
+    bool touchPointSpawnedAtUpperScreenEdge;
     
     // Use a Dictionary to store UITouch object's memory address as key, and pointerId as value,字典存放UITouch对象地址和pointerId映射关系
     // pointerId will be generated from a pre-defined pool
@@ -31,7 +30,7 @@
     NSMutableSet<NSNumber *> *activePointerIds; //pointerId Set for active touches.
     NSMutableSet<NSNumber *> *pointerIdPool; //pre-defined pool of pointerIds.
     NSMutableSet<NSNumber *> *unassignedPointerIds;
-    NSMutableSet<NSNumber *> *excludedPointerIds; // a NSSet of pointerIds of touches that will not be sent to the remote PC for better swipe gesture handling
+
     CGFloat slideGestureVerticalThreshold;
     CGFloat screenWidthWithThreshold;
     CGFloat EDGE_TOLERANCE;
@@ -42,7 +41,7 @@
     self->streamView = view;
     self->currentSettings = settings;
     self->activateCoordSelector = currentSettings.pointerVelocityModeDivider.floatValue != 1.0;
-    self->notPureNativeTouchMode = !(currentSettings.touchMode.intValue == PURE_NATIVE_TOUCH);
+    // self->notPureNativeTouchMode = !(currentSettings.touchMode.intValue == PURE_NATIVE_TOUCH);
     
     self->pointerIdDict = [NSMutableDictionary dictionary];
     self->pointerIdPool = [NSMutableSet set];
@@ -50,7 +49,8 @@
         [self->pointerIdPool addObject:@(i)];
     }
     self->activePointerIds = [NSMutableSet set];
-    self->excludedPointerIds = [[NSMutableSet alloc] init];
+    // self->excludedPointerIds = [[NSMutableSet alloc] init];
+    self->touchPointSpawnedAtUpperScreenEdge = false;
     
     EDGE_TOLERANCE = 15.0;
     slideGestureVerticalThreshold = CGRectGetHeight([[UIScreen mainScreen] bounds]) * 0.4;
@@ -75,9 +75,12 @@
     [pointerIdDict setObject:@(pointerId) forKey:@(memAddrValue)];
     [activePointerIds addObject:@(pointerId)];
     
-    //check if touch point is spawned on the left or right upper half screen edges, if so we'll populate the excluded pointer id NSSet and not send the touch event to remote PC. this is for better handling in-stream slide gesture
+    //check if touch point is spawned on the left or right upper half screen edges, event to remote PC. this is for better handling in-stream slide gesture
     CGPoint initialPoint = [touch locationInView:self->streamView];
-    if(initialPoint.y < slideGestureVerticalThreshold && (initialPoint.x < EDGE_TOLERANCE || initialPoint.x > screenWidthWithThreshold)) [excludedPointerIds addObject:@(pointerId)];
+    if(initialPoint.y < slideGestureVerticalThreshold && (initialPoint.x < EDGE_TOLERANCE || initialPoint.x > screenWidthWithThreshold)) {
+        self->touchPointSpawnedAtUpperScreenEdge = true;
+        // [excludedPointerIds addObject:@(pointerId)];
+    }
 }
 
 // remove pointerId in touchesEnded or touchesCancelled
@@ -87,7 +90,7 @@
     if(pointerIdObj != nil){
         [activePointerIds removeObject:pointerIdObj];
         [pointerIdDict removeObjectForKey:@(memAddrValue)];
-        if([excludedPointerIds containsObject:pointerIdObj]) [excludedPointerIds removeObject:pointerIdObj]; // remove pointer id from excludedPointerId NSSet
+        // if([excludedPointerIds containsObject:pointerIdObj]) [excludedPointerIds removeObject:pointerIdObj]; // remove pointer id from excludedPointerId NSSet
     }
 }
 
@@ -99,12 +102,10 @@
 
 
 - (void)sendTouchEvent:(UITouch*)touch withTouchtype:(uint8_t)touchType{
-    CGPoint targetCoords;
+    if(touchPointSpawnedAtUpperScreenEdge) return; //  we're done here. this touch event will not be sent to the remote PC. and this must be checked after coord selector finishes populating new relative coords, or the app will crash!
-    uint8_t pointerId = [self retrievePointerIdFromDict:touch];
-    // NSLog(@"excluded count: %d", (uint32_t)[excludedPointerIds count]);
     
-    if([excludedPointerIds containsObject:@(pointerId)]) return; // if the pointerId has been excluded by edge check, we're done here. this touch event will not be sent to the remote PC. and this must be checked after coord selector finishes populating new relative coords, or the app will crash!
+    CGPoint targetCoords;
-
+    // NSLog(@"excluded count: %d", (uint32_t)[excludedPointerIds count]);
     if(activateCoordSelector && touch.phase == UITouchPhaseMoved) targetCoords = [NativeTouchPointer selectCoordsFor:touch]; // coordinates of touch pointer replaced to relative ones here.
     else targetCoords = [touch locationInView:streamView];
 
@@ -112,7 +113,8 @@
     CGSize videoSize = [streamView getVideoAreaSize];
     CGFloat normalizedX = location.x / videoSize.width;
     CGFloat normalizedY = location.y / videoSize.height;
-    
+    uint8_t pointerId = [self retrievePointerIdFromDict:touch];
+
     if([NativeTouchPointer getPointerObjFromDict:touch].needResetCoords){ // access whether the current pointer has reached the boundary, and need a coord reset.
         LiSendTouchEvent(LI_TOUCH_EVENT_UP, pointerId, normalizedX, normalizedY, 0, 0, 0, 0);  //event must sent from the lowest level directy by LiSendTouchEvent to simulate continous dragging to another point on screen
         LiSendTouchEvent(LI_TOUCH_EVENT_DOWN, pointerId, 0.3, 0.4, 0, 0, 0, 0);
@@ -122,7 +124,7 @@
 - (void)touchesBegan:(NSSet *)touches withEvent:(UIEvent *)event {
     for (UITouch* touch in touches){
         // continue to the next loop if current touch is already captured by OSC. works only in regular native touch
-        if(notPureNativeTouchMode && [OnScreenControls.touchAddrsCapturedByOnScreenControls containsObject:@((uintptr_t)touch)]) continue;
+        if([OnScreenControls.touchAddrsCapturedByOnScreenControls containsObject:@((uintptr_t)touch)]) continue;
         [self handleTouchDown:touch]; //generate & populate pointerId
         if(activateCoordSelector) [NativeTouchPointer populatePointerObjIntoDict:touch];
         [self sendTouchEvent:touch withTouchtype:LI_TOUCH_EVENT_DOWN];
@@ -133,7 +135,7 @@
     // NSLog(@"captured by OSB touches: %d", (uint32_t)[OnScreenControls.touchAddrsCapturedByOnScreenControls count]);
     for (UITouch* touch in touches){
         // continue to the next loop if current touch is already captured by OSC. works only in regular native touch
-        if(notPureNativeTouchMode && [OnScreenControls.touchAddrsCapturedByOnScreenControls containsObject:@((uintptr_t)touch)]) continue;
+        if([OnScreenControls.touchAddrsCapturedByOnScreenControls containsObject:@((uintptr_t)touch)]) continue;
         if(activateCoordSelector) [NativeTouchPointer updatePointerObjInDict:touch];
         [self sendTouchEvent:touch withTouchtype:LI_TOUCH_EVENT_MOVE];
         [[NativeTouchPointer getPointerObjFromDict:touch] doesNeedResetCoords]; // execute the judging of doesReachBoundary for current pointer instance. (happens after the event is sent to Sunshine service)
@@ -145,11 +147,12 @@
 - (void)touchesEnded:(NSSet *)touches withEvent:(UIEvent *)event {
     for (UITouch* touch in touches){
         // continue to the next loop if current touch is already captured by OSC. works only in regular native touch
-        if(notPureNativeTouchMode && [OnScreenControls.touchAddrsCapturedByOnScreenControls containsObject:@((uintptr_t)touch)]) continue;
+        if([OnScreenControls.touchAddrsCapturedByOnScreenControls containsObject:@((uintptr_t)touch)]) continue;
         [self sendTouchEvent:touch withTouchtype:LI_TOUCH_EVENT_UP]; //send touch event before remove pointerId
         [self removePointerId:touch]; //then remove pointerId
         if(activateCoordSelector) [NativeTouchPointer removePointerObjFromDict:touch];
     }
+    if(touchPointSpawnedAtUpperScreenEdge && [[event allTouches] count] == [touches count]) touchPointSpawnedAtUpperScreenEdge = false;
     return;
 }
 

Limelight/Input/NativeTouchPointer.m

@@ -110,7 +110,7 @@ static StreamView* _streamView;
 
 + (CGPoint)selectCoordsFor:(UITouch *)touch{
     NativeTouchPointer *pointer = [pointerObjDict objectForKey:@((uintptr_t)touch)];
-    if(pointer == nil) return CGPointMake(0, 0); // THIS will PREVENT CRASH caused by slide gesture!
+    if(pointer == nil) return CGPointMake(0, 0); 
     if((pointer -> initialPoint).x > pointerVelocityDividerLocationByPoints){  //if first contact coords locates on the right side of divider.
         pointer -> useRelativeCoords = true;
         return pointer->latestRelativePoint;

Limelight/Input/PureNativeTouchHandler.h

@@ -0,0 +1,22 @@
+//
+//  NativeTouchHandler.h
+//  Moonlight
+//
+//  Created by ZWM on 2024/6/16.
+//  Copyright © 2024 Moonlight Game Streaming Project. All rights reserved.
+//
+
+
+#import "StreamView.h"
+#import "DataManager.h"
+#include <Limelight.h>
+
+NS_ASSUME_NONNULL_BEGIN
+
+@interface PureNativeTouchHandler : UIResponder
+- (id)initWithView:(StreamView*)view andSettings:(TemporarySettings*)settings;
+
+
+@end
+
+NS_ASSUME_NONNULL_END

Limelight/Input/PureNativeTouchHandler.m

@@ -0,0 +1,148 @@
+//
+//  NativeTouchHandler.m
+//  Moonlight
+//
+//  Created by ZWM on 2024/8/29.
+//  Copyright © 2024 Moonlight Game Streaming Project. All rights reserved.
+//
+
+#import <Foundation/Foundation.h>
+#import "PureNativeTouchHandler.h"
+#import "NativeTouchPointer.h"
+#import "StreamView.h"
+
+#include <Limelight.h>
+
+
+@implementation PureNativeTouchHandler {
+    StreamView* streamView;
+    TemporarySettings* currentSettings;
+    bool activateCoordSelector;
+    bool touchPointSpawnedAtUpperScreenEdge;
+    
+    // Use a Dictionary to store UITouch object's memory address as key, and pointerId as value,字典存放UITouch对象地址和pointerId映射关系
+    // pointerId will be generated from a pre-defined pool
+    // Use a NSSet store active pointerId,
+    NSMutableDictionary *pointerIdDict; //pointerId Dict for active touches.
+    NSMutableSet<NSNumber *> *activePointerIds; //pointerId Set for active touches.
+    NSMutableSet<NSNumber *> *pointerIdPool; //pre-defined pool of pointerIds.
+    NSMutableSet<NSNumber *> *unassignedPointerIds;
+    
+    CGFloat slideGestureVerticalThreshold;
+    CGFloat screenWidthWithThreshold;
+    CGFloat EDGE_TOLERANCE;
+}
+
+
+- (id)initWithView:(StreamView*)view andSettings:(TemporarySettings*)settings{
+    self = [super init];
+    self->streamView = view;
+    self->currentSettings = settings;
+    self->activateCoordSelector = currentSettings.pointerVelocityModeDivider.floatValue != 1.0;
+    
+    pointerIdDict = [NSMutableDictionary dictionary];
+    pointerIdPool = [NSMutableSet set];
+    for (uint8_t i = 0; i <= 10; i++) { //ipadOS supports upto 11 finger touches
+        [pointerIdPool addObject:@(i)];
+    }
+    activePointerIds = [NSMutableSet set];
+    self->touchPointSpawnedAtUpperScreenEdge = false;
+
+    EDGE_TOLERANCE = 15.0;
+    slideGestureVerticalThreshold = CGRectGetHeight([[UIScreen mainScreen] bounds]) * 0.4;
+    screenWidthWithThreshold = CGRectGetWidth([[UIScreen mainScreen] bounds]) - EDGE_TOLERANCE;
+
+    [NativeTouchPointer setPointerVelocityDivider:settings.pointerVelocityModeDivider.floatValue];
+    [NativeTouchPointer setPointerVelocityFactor:settings.touchPointerVelocityFactor.floatValue];
+    [NativeTouchPointer initContextWithView:self->streamView];
+    return self;
+}
+
+
+// generate & populate pointerId into NSDict & NSSet, called in touchesBegan
+- (void)handleTouchDown:(UITouch*)touch{
+    uintptr_t memAddrValue = (uintptr_t)touch;
+    unassignedPointerIds = [pointerIdPool mutableCopy]; //reset unassignedPointerIds
+    [unassignedPointerIds minusSet:activePointerIds];
+    uint8_t pointerId = [[unassignedPointerIds anyObject] unsignedIntValue];
+    [pointerIdDict setObject:@(pointerId) forKey:@(memAddrValue)];
+    [activePointerIds addObject:@(pointerId)];
+    
+    //check if touch point is spawned on the left or right upper half screen edges,
+    CGPoint initialPoint = [touch locationInView:self->streamView];
+    if(initialPoint.y < slideGestureVerticalThreshold && (initialPoint.x < EDGE_TOLERANCE || initialPoint.x > screenWidthWithThreshold)) {
+        self->touchPointSpawnedAtUpperScreenEdge = true;
+    }
+}
+
+// remove pointerId in touchesEnded or touchesCancelled
+- (void)removePointerId:(UITouch*)touch{
+    uintptr_t memAddrValue = (uintptr_t)touch;
+    NSNumber* pointerIdObj = [pointerIdDict objectForKey:@(memAddrValue)];
+    if(pointerIdObj != nil){
+        [activePointerIds removeObject:pointerIdObj];
+        [pointerIdDict removeObjectForKey:@(memAddrValue)];
+    }
+}
+
+// 从字典中获取UITouch事件对应的pointerId
+// called in method of sendTouchEvent
+- (uint8_t) retrievePointerIdFromDict:(UITouch*)touch{
+    return [[pointerIdDict objectForKey:@((uintptr_t)touch)] unsignedIntValue];
+}
+
+
+- (void)sendTouchEvent:(UITouch*)touch withTouchtype:(uint8_t)touchType{
+    if(touchPointSpawnedAtUpperScreenEdge) return; // we're done here. this touch event will not be sent to the remote PC. and this must be checked after coord selector finishes populating new relative coords, or the app will crash!
+
+    CGPoint targetCoords;
+    if(activateCoordSelector && touch.phase == UITouchPhaseMoved) targetCoords = [NativeTouchPointer selectCoordsFor:touch]; // coordinates of touch pointer replaced to relative ones here.
+    else targetCoords = [touch locationInView:streamView];
+    
+    CGPoint location = [streamView adjustCoordinatesForVideoArea:targetCoords];
+    CGSize videoSize = [streamView getVideoAreaSize];
+    CGFloat normalizedX = location.x / videoSize.width;
+    CGFloat normalizedY = location.y / videoSize.height;
+    uint8_t pointerId = [self retrievePointerIdFromDict:touch];
+    
+    if([NativeTouchPointer getPointerObjFromDict:touch].needResetCoords){ // access whether the current pointer has reached the boundary, and need a coord reset.
+        LiSendTouchEvent(LI_TOUCH_EVENT_UP, pointerId, normalizedX, normalizedY, 0, 0, 0, 0);  //event must sent from the lowest level directy by LiSendTouchEvent to simulate continous dragging to another point on screen
+        LiSendTouchEvent(LI_TOUCH_EVENT_DOWN, pointerId, 0.3, 0.4, 0, 0, 0, 0);
+    }else LiSendTouchEvent(touchType, pointerId, normalizedX, normalizedY,(touch.force / touch.maximumPossibleForce) / sin(touch.altitudeAngle),0.0f, 0.0f,[streamView getRotationFromAzimuthAngle:[touch azimuthAngleInView:streamView]]);
+}
+
+- (void)touchesBegan:(NSSet *)touches withEvent:(UIEvent *)event {
+    for (UITouch* touch in touches){
+        [self handleTouchDown:touch]; //generate & populate pointerId
+        if(activateCoordSelector) [NativeTouchPointer populatePointerObjIntoDict:touch];
+        [self sendTouchEvent:touch withTouchtype:LI_TOUCH_EVENT_DOWN];
+    }
+}
+
+- (void)touchesMoved:(NSSet *)touches withEvent:(UIEvent *)event {
+    for (UITouch* touch in touches){
+        if(activateCoordSelector) [NativeTouchPointer updatePointerObjInDict:touch];
+        [self sendTouchEvent:touch withTouchtype:LI_TOUCH_EVENT_MOVE];
+        [[NativeTouchPointer getPointerObjFromDict:touch] doesNeedResetCoords]; // execute the judging of doesReachBoundary for current pointer instance. (happens after the event is sent to Sunshine service)
+    }
+    return;
+}
+
+
+- (void)touchesEnded:(NSSet *)touches withEvent:(UIEvent *)event {
+    for (UITouch* touch in touches){
+        [self sendTouchEvent:touch withTouchtype:LI_TOUCH_EVENT_UP]; //send touch event before remove pointerId
+        [self removePointerId:touch]; //then remove pointerId
+        if(activateCoordSelector) [NativeTouchPointer removePointerObjFromDict:touch];
+    }
+    if(touchPointSpawnedAtUpperScreenEdge && [[event allTouches] count] == [touches count]) touchPointSpawnedAtUpperScreenEdge = false;
+
+    return;
+}
+
+- (void)touchesCancelled:(NSSet *)touches withEvent:(UIEvent *)event {
+    [self touchesEnded:touches withEvent:event];
+}
+
+
+@end

Limelight/Input/StreamView.m

@@ -15,6 +15,7 @@
 #import "OSCProfilesManager.h"
 #import "NativeTouchPointer.h"
 #import "NativeTouchHandler.h"
+#import "PureNativeTouchHandler.h"
 #import "RelativeTouchHandler.h"
 #import "AbsoluteTouchHandler.h"
 #import "KeyboardInputField.h"
@@ -109,10 +110,10 @@ static const double X1_MOUSE_SPEED_DIVISOR = 2.5;
     switch (settings.touchMode.intValue) {
         case REGULAR_NATIVE_TOUCH:
             keyboardToggleRecognizer.immediateTriggering = false;
-            // if(settings.onscreenControls.intValue == OnScreenControlsLevelCustom) keyboardToggleRecognizer.numberOfTouchesRequired = 3; //deprecated: fixing keyboard taps to 3, in order to invoke OSC rebase in stream view by 4-finger tap.
-
-        case PURE_NATIVE_TOUCH:
             self->touchHandler = [[NativeTouchHandler alloc] initWithView:self andSettings:settings];break;
+        case PURE_NATIVE_TOUCH:
+            keyboardToggleRecognizer.immediateTriggering = false;
+            self->touchHandler = [[PureNativeTouchHandler alloc] initWithView:self andSettings:settings];break;
         case RELATIVE_TOUCH:
             self->touchHandler = [[RelativeTouchHandler alloc] initWithView:self andSettings:settings];
             keyboardToggleRecognizer.immediateTriggering = false;

Moonlight.xcodeproj/project.pbxproj

@@ -21,6 +21,7 @@
 		843CC7F02C1E757D00BBC475 /* NativeTouchPointer.m in Sources */ = {isa = PBXBuildFile; fileRef = 843CC7EF2C1E757D00BBC475 /* NativeTouchPointer.m */; };
 		843CC7F62C1E7A1700BBC475 /* NativeTouchHandler.m in Sources */ = {isa = PBXBuildFile; fileRef = 843CC7F52C1E7A1700BBC475 /* NativeTouchHandler.m */; };
 		843E93BE2C41528700B1651F /* OrientationHelper.swift in Sources */ = {isa = PBXBuildFile; fileRef = 843E93BD2C41528700B1651F /* OrientationHelper.swift */; };
+		844CD0EB2C7F8500004861A7 /* PureNativeTouchHandler.m in Sources */ = {isa = PBXBuildFile; fileRef = 844CD0EA2C7F8500004861A7 /* PureNativeTouchHandler.m */; };
 		8454AC662C31907F00F0A302 /* iPad.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 8454AC682C31907F00F0A302 /* iPad.storyboard */; };
 		8454AC6A2C319BDA00F0A302 /* iPhone.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 8454AC6C2C319BDA00F0A302 /* iPhone.storyboard */; };
 		84591A512BE1625E006AAB10 /* CustomEdgeSlideGestureRecognizer.m in Sources */ = {isa = PBXBuildFile; fileRef = 84591A502BE1625E006AAB10 /* CustomEdgeSlideGestureRecognizer.m */; };
@@ -149,6 +150,8 @@
 		843CC7F42C1E778000BBC475 /* NativeTouchHandler.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = NativeTouchHandler.h; sourceTree = "<group>"; };
 		843CC7F52C1E7A1700BBC475 /* NativeTouchHandler.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = NativeTouchHandler.m; sourceTree = "<group>"; };
 		843E93BD2C41528700B1651F /* OrientationHelper.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = OrientationHelper.swift; sourceTree = "<group>"; };
+		844CD0E82C7F847E004861A7 /* PureNativeTouchHandler.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = PureNativeTouchHandler.h; sourceTree = "<group>"; };
+		844CD0EA2C7F8500004861A7 /* PureNativeTouchHandler.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = PureNativeTouchHandler.m; sourceTree = "<group>"; };
 		8454AC672C31907F00F0A302 /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/iPad.storyboard; sourceTree = "<group>"; };
 		8454AC692C31907F00F0A302 /* mul */ = {isa = PBXFileReference; lastKnownFileType = text.json.xcstrings; name = mul; path = mul.lproj/iPad.xcstrings; sourceTree = "<group>"; };
 		8454AC6B2C319BDA00F0A302 /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/iPhone.storyboard; sourceTree = "<group>"; };
@@ -618,6 +621,8 @@
 				840751B22BF49C050096B657 /* CustomTapGestureRecognizer.m */,
 				9819CC12254F107A008A7C8E /* RelativeTouchHandler.h */,
 				843CC7F42C1E778000BBC475 /* NativeTouchHandler.h */,
+				844CD0E82C7F847E004861A7 /* PureNativeTouchHandler.h */,
+				844CD0EA2C7F8500004861A7 /* PureNativeTouchHandler.m */,
 				843CC7F52C1E7A1700BBC475 /* NativeTouchHandler.m */,
 				9819CC13254F107A008A7C8E /* RelativeTouchHandler.m */,
 				9819CC20254F180F008A7C8E /* AbsoluteTouchHandler.h */,
@@ -1050,6 +1055,7 @@
 				9827E7A32514366900F25707 /* HapticContext.m in Sources */,
 				9832D1361BBCD5C50036EF48 /* TemporaryApp.m in Sources */,
 				84E1CDEF2C4F688E00547904 /* CommandManagerViewController.swift in Sources */,
+				844CD0EB2C7F8500004861A7 /* PureNativeTouchHandler.m in Sources */,
 				84351B102C29ACCB00A77989 /* OSCProfile.m in Sources */,
 				84351B2C2C29AD6100A77989 /* OSCProfilesTableViewController.m in Sources */,
 				98D585701C0ED0E800F6CC00 /* TemporarySettings.m in Sources */,