- async native touch passthrough

- touchMoveEvent interval tweak

Limelight/Input/NativeTouchHandler.m

@@ -14,7 +14,16 @@
 
 #include <Limelight.h>
 
+/*
+@interface TouchInfoCarrier : NSObject // 明确继承NSObject
+@property (nonatomic, copy) NSString *name;
+@end
 
+@implementation TouchInfoCarrier {
+    
+}
+@end
+ */
 
 
 @implementation NativeTouchHandler {
@@ -22,6 +31,8 @@
     TemporarySettings* currentSettings;
     bool activateCoordSelector;
     bool touchPointSpawnedAtUpperScreenEdge;
+    CGFloat pointerVelocityDividerLocationByPoints;
+    uint16_t touchMoveEventIntervalUs;
     
     // Use a Dictionary to store UITouch object's memory address as key, and pointerId as value,字典存放UITouch对象地址和pointerId映射关系
     // pointerId will be generated from a pre-defined pool
@@ -30,10 +41,14 @@
     NSMutableSet<NSNumber *> *activePointerIds; //pointerId Set for active touches.
     NSMutableSet<NSNumber *> *pointerIdPool; //pre-defined pool of pointerIds.
     NSMutableSet<NSNumber *> *unassignedPointerIds;
+    
+    NSMutableDictionary *pointerObjDict;
 
     CGFloat slideGestureVerticalThreshold;
     CGFloat screenWidthWithThreshold;
     CGFloat EDGE_TOLERANCE;
+    
+    CGRect streamViewBounds;
 }
 
 - (id)initWithView:(StreamView*)view andSettings:(TemporarySettings*)settings{
@@ -41,7 +56,8 @@
     self->streamView = view;
     self->currentSettings = settings;
     self->activateCoordSelector = currentSettings.pointerVelocityModeDivider.floatValue != 1.0;
-    // self->notPureNativeTouchMode = !(currentSettings.touchMode.intValue == PURE_NATIVE_TOUCH);
+    self->touchMoveEventIntervalUs = currentSettings.touchMoveEventInterval.intValue;
+    self->streamViewBounds = view.bounds;
     
     self->pointerIdDict = [NSMutableDictionary dictionary];
     self->pointerIdPool = [NSMutableSet set];
@@ -52,19 +68,77 @@
     // self->excludedPointerIds = [[NSMutableSet alloc] init];
     self->touchPointSpawnedAtUpperScreenEdge = false;
     
+    
+    
+    self->pointerObjDict = [NSMutableDictionary dictionary];
+    
     EDGE_TOLERANCE = 15.0;
     slideGestureVerticalThreshold = CGRectGetHeight([[UIScreen mainScreen] bounds]) * 0.4;
     screenWidthWithThreshold = CGRectGetWidth([[UIScreen mainScreen] bounds]) - EDGE_TOLERANCE;
 
-    [NativeTouchPointer setPointerVelocityDivider:settings.pointerVelocityModeDivider.floatValue];
-    [NativeTouchPointer setPointerVelocityFactor:settings.touchPointerVelocityFactor.floatValue];
-    [NativeTouchPointer initContextWithView:self->streamView];
+    self->pointerVelocityDividerLocationByPoints = CGRectGetWidth([[UIScreen mainScreen] bounds]) * settings.pointerVelocityModeDivider.floatValue;
+    
+    [NativeTouchPointer initContextWithView:self->streamView andSettings:settings];
     //_touchesCapturedByOnScreenButtons = [[NSMutableSet alloc] init];
     return self;
 }
 
+- (CGSize) getVideoAreaSize {
+    if (self->streamViewBounds.size.width > self->streamViewBounds.size.height * streamView.streamAspectRatio) {
+        return CGSizeMake(self->streamViewBounds.size.height * streamView.streamAspectRatio, self->streamViewBounds.size.height);
+    } else {
+        return CGSizeMake(self->streamViewBounds.size.width, self->streamViewBounds.size.width / streamView.streamAspectRatio);
+    }
+}
 
 
+- (CGPoint) adjustCoordinatesForVideoArea:(CGPoint)point {
+    // These are now relative to the StreamView, however we need to scale them
+    // further to make them relative to the actual video portion.
+    float x = point.x - self->streamViewBounds.origin.x;
+    float y = point.y - self->streamViewBounds.origin.y;
+    
+    // For some reason, we don't seem to always get to the bounds of the window
+    // so we'll subtract 1 pixel if we're to the left/below of the origin and
+    // and add 1 pixel if we're to the right/above. It should be imperceptible
+    // to the user but it will allow activation of gestures that require contact
+    // with the edge of the screen (like Aero Snap).
+    if (x < self->streamViewBounds.size.width / 2) {
+        x--;
+    }
+    else {
+        x++;
+    }
+    if (y < self->streamViewBounds.size.height / 2) {
+        y--;
+    }
+    else {
+        y++;
+    }
+    
+    // This logic mimics what iOS does with AVLayerVideoGravityResizeAspect
+    CGSize videoSize = [self getVideoAreaSize];
+    CGPoint videoOrigin = CGPointMake(self->streamViewBounds.size.width / 2 - videoSize.width / 2,
+                                      self->streamViewBounds.size.height / 2 - videoSize.height / 2);
+    
+    // Confine the cursor to the video region. We don't just discard events outside
+    // the region because we won't always get one exactly when the mouse leaves the region.
+    return CGPointMake(MIN(MAX(x, videoOrigin.x), videoOrigin.x + videoSize.width) - videoOrigin.x,
+                       MIN(MAX(y, videoOrigin.y), videoOrigin.y + videoSize.height) - videoOrigin.y);
+}
+
+
+- (uint16_t)getRotationFromAzimuthAngle:(float)azimuthAngle {
+    // iOS reports azimuth of 0 when the stylus is pointing west, but Moonlight expects
+    // rotation of 0 to mean the stylus is pointing north. Rotate the azimuth angle
+    // clockwise by 90 degrees to convert from iOS to Moonlight rotation conventions.
+    int32_t rotationAngle = (azimuthAngle - M_PI_2) * (180.f / M_PI);
+    if (rotationAngle < 0) {
+        rotationAngle += 360;
+    }
+    return (uint16_t)rotationAngle;
+}
+
 // generate & populate pointerId into NSDict & NSSet, called in touchesBegan
 - (void)handleTouchDown:(UITouch*)touch{
     //populate pointerId
@@ -105,54 +179,63 @@
     if(touchPointSpawnedAtUpperScreenEdge) return; //  we're done here. this touch event will not be sent to the remote PC. and this must be checked after coord selector finishes populating new relative coords, or the app will crash!
     
     CGPoint targetCoords;
+    //NSLog(@"selecting coords: %d", touch.phase == UITouchPhaseMoved);
     // NSLog(@"excluded count: %d", (uint32_t)[excludedPointerIds count]);
-    if(activateCoordSelector && touch.phase == UITouchPhaseMoved) targetCoords = [NativeTouchPointer selectCoordsFor:touch]; // coordinates of touch pointer replaced to relative ones here.
-    else targetCoords = [touch locationInView:streamView];
+    
+    targetCoords = activateCoordSelector ? [self selectCoordsFor:touch] : [touch locationInView:streamView];
 
-    CGPoint location = [streamView adjustCoordinatesForVideoArea:targetCoords];
-    CGSize videoSize = [streamView getVideoAreaSize];
+    CGPoint location = [self adjustCoordinatesForVideoArea:targetCoords];
+    CGSize videoSize = [self getVideoAreaSize];
     CGFloat normalizedX = location.x / videoSize.width;
     CGFloat normalizedY = location.y / videoSize.height;
     uint8_t pointerId = [self retrievePointerIdFromDict:touch];
 
-    if([NativeTouchPointer getPointerObjFromDict:touch].needResetCoords){ // access whether the current pointer has reached the boundary, and need a coord reset.
+    if([self getPointerObjFromDict:touch].needResetCoords){ // access whether the current pointer has reached the boundary, and need a coord reset.
         LiSendTouchEvent(LI_TOUCH_EVENT_UP, pointerId, normalizedX, normalizedY, 0, 0, 0, 0);  //event must sent from the lowest level directy by LiSendTouchEvent to simulate continous dragging to another point on screen
         LiSendTouchEvent(LI_TOUCH_EVENT_DOWN, pointerId, 0.3, 0.4, 0, 0, 0, 0);
-    }else LiSendTouchEvent(touchType, pointerId, normalizedX, normalizedY,(touch.force / touch.maximumPossibleForce) / sin(touch.altitudeAngle),0.0f, 0.0f,[streamView getRotationFromAzimuthAngle:[touch azimuthAngleInView:streamView]]);
+    }else LiSendTouchEvent(touchType, pointerId, normalizedX, normalizedY,(touch.force / touch.maximumPossibleForce) / sin(touch.altitudeAngle),0.0f, 0.0f,[self getRotationFromAzimuthAngle:[touch azimuthAngleInView:streamView]]);
 }
 
 - (void)touchesBegan:(NSSet *)touches withEvent:(UIEvent *)event {
-    for (UITouch* touch in touches){
-        // continue to the next loop if current touch is already captured by OSC. works only in regular native touch
-        if([OnScreenControls.touchAddrsCapturedByOnScreenControls containsObject:@((uintptr_t)touch)]) continue;
-        [self handleTouchDown:touch]; //generate & populate pointerId
-        if(activateCoordSelector) [NativeTouchPointer populatePointerObjIntoDict:touch];
-        [self sendTouchEvent:touch withTouchtype:LI_TOUCH_EVENT_DOWN];
-    }
+    dispatch_async(dispatch_get_global_queue(QOS_CLASS_USER_INTERACTIVE, 0), ^{
+        for (UITouch* touch in touches){
+            // continue to the next loop if current touch is already captured by OSC. works only in regular native touch
+            if([OnScreenControls.touchAddrsCapturedByOnScreenControls containsObject:@((uintptr_t)touch)]) continue;
+            [self handleTouchDown:touch]; //generate & populate pointerId
+            if(self->activateCoordSelector) [self populatePointerObjIntoDict:touch];
+            [self sendTouchEvent:touch withTouchtype:LI_TOUCH_EVENT_DOWN];
+        }
+    });
 }
 
 - (void)touchesMoved:(NSSet *)touches withEvent:(UIEvent *)event {
+    dispatch_async(dispatch_get_global_queue(QOS_CLASS_USER_INTERACTIVE, 0), ^{
     // NSLog(@"captured by OSB touches: %d", (uint32_t)[OnScreenControls.touchAddrsCapturedByOnScreenControls count]);
-    for (UITouch* touch in touches){
-        // continue to the next loop if current touch is already captured by OSC. works only in regular native touch
-        if([OnScreenControls.touchAddrsCapturedByOnScreenControls containsObject:@((uintptr_t)touch)]) continue;
-        if(activateCoordSelector) [NativeTouchPointer updatePointerObjInDict:touch];
-        [self sendTouchEvent:touch withTouchtype:LI_TOUCH_EVENT_MOVE];
-        [[NativeTouchPointer getPointerObjFromDict:touch] doesNeedResetCoords]; // execute the judging of doesReachBoundary for current pointer instance. (happens after the event is sent to Sunshine service)
-    }
+        for (UITouch* touch in touches){
+            // continue to the next loop if current touch is already captured by OSC. works only in regular native touch
+            if([OnScreenControls.touchAddrsCapturedByOnScreenControls containsObject:@((uintptr_t)touch)]) continue;
+            if(self->activateCoordSelector) [self updatePointerObjInDict:touch];
+            [self sendTouchEvent:touch withTouchtype:LI_TOUCH_EVENT_MOVE];
+            [[self getPointerObjFromDict:touch] doesNeedResetCoords]; // execute the judging of doesReachBoundary for current pointer instance. (happens after the event is sent to Sunshine service)
+            usleep(self->touchMoveEventIntervalUs);
+            // NSLog(@"interval: %d", self->touchMoveEventIntervalUs);
+        }
+    });
     return;
 }
 
 
 - (void)touchesEnded:(NSSet *)touches withEvent:(UIEvent *)event {
-    for (UITouch* touch in touches){
-        // continue to the next loop if current touch is already captured by OSC. works only in regular native touch
-        if([OnScreenControls.touchAddrsCapturedByOnScreenControls containsObject:@((uintptr_t)touch)]) continue;
-        [self sendTouchEvent:touch withTouchtype:LI_TOUCH_EVENT_UP]; //send touch event before remove pointerId
-        [self removePointerId:touch]; //then remove pointerId
-        if(activateCoordSelector) [NativeTouchPointer removePointerObjFromDict:touch];
-    }
-    if(touchPointSpawnedAtUpperScreenEdge && [[event allTouches] count] == [touches count]) touchPointSpawnedAtUpperScreenEdge = false;
+    dispatch_async(dispatch_get_global_queue(QOS_CLASS_USER_INTERACTIVE, 0), ^{
+        for (UITouch* touch in touches){
+            // continue to the next loop if current touch is already captured by OSC. works only in regular native touch
+            if([OnScreenControls.touchAddrsCapturedByOnScreenControls containsObject:@((uintptr_t)touch)]) continue;
+            [self sendTouchEvent:touch withTouchtype:LI_TOUCH_EVENT_UP]; //send touch event before remove pointerId
+            [self removePointerId:touch]; //then remove pointerId
+            if(self->activateCoordSelector) [self removePointerObjFromDict:touch];
+        }
+        if(self->touchPointSpawnedAtUpperScreenEdge && [[event allTouches] count] == [touches count]) self->touchPointSpawnedAtUpperScreenEdge = false;
+    });
     return;
 }
 
@@ -161,4 +244,33 @@
 }
 
 
+- (void)populatePointerObjIntoDict:(UITouch*)touch{
+    NativeTouchPointer* pointer = [[NativeTouchPointer alloc] initWithTouch:touch];
+    pointer.useRelativeCoords = pointer.initialPoint.x > pointerVelocityDividerLocationByPoints;
+    [pointerObjDict setObject:pointer forKey:@((uintptr_t)touch)];
+}
+
+- (NativeTouchPointer* )getPointerObjFromDict:(UITouch*)touch{
+    return [pointerObjDict objectForKey:@((uintptr_t)touch)];
+}
+
+- (void)removePointerObjFromDict:(UITouch*)touch{
+    uintptr_t memAddrValue = (uintptr_t)touch;
+    NativeTouchPointer* pointer = [pointerObjDict objectForKey:@(memAddrValue)];
+    if(pointer != nil){
+        [pointerObjDict removeObjectForKey:@(memAddrValue)];
+    }
+}
+
+- (void)updatePointerObjInDict:(UITouch *)touch{
+    [[pointerObjDict objectForKey:@((uintptr_t)touch)] updatePointerCoords:touch];
+}
+
+- (CGPoint)selectCoordsFor:(UITouch *)touch{
+    NativeTouchPointer *pointer = [pointerObjDict objectForKey:@((uintptr_t)touch)];
+    if(pointer == nil) return CGPointMake(0, 0);
+   // NSLog(@"using relative Coords: intialX %f, divider loc %f", pointer.initialPoint.x, pointerVelocityDividerLocationByPoints);
+    return pointer.useRelativeCoords ? pointer.latestRelativePoint : pointer.latestPoint;
+}
+
 @end

Limelight/Input/NativeTouchPointer.h

@@ -13,18 +13,18 @@ NS_ASSUME_NONNULL_BEGIN
 
 @interface NativeTouchPointer : NSObject
 @property (nonatomic, assign) bool needResetCoords;
+@property (nonatomic, assign) CGPoint initialPoint;
+@property (nonatomic, assign) CGPoint latestPoint;
+@property (nonatomic, assign) CGPoint previousPoint;
+@property (nonatomic, assign) CGPoint latestRelativePoint;
+@property (nonatomic, assign) CGPoint previousRelativePoint;
+@property (nonatomic, assign) bool useRelativeCoords;
 
-+ (void)initContextWithView:(StreamView *)view;
-+ (void)setPointerVelocityDivider:(CGFloat)dividerLocation;
-+ (void)setPointerVelocityFactor:(CGFloat)velocityFactor;
 
-+ (NativeTouchPointer* )getPointerObjFromDict:(UITouch*)touch;
++ (void)initContextWithView:(StreamView*)view andSettings:(TemporarySettings*)settings;
+
 - (bool)doesNeedResetCoords;
-
-+ (void)populatePointerObjIntoDict:(UITouch*)touch;
-+ (void)removePointerObjFromDict:(UITouch*)touch;
-+ (void)updatePointerObjInDict:(UITouch *)touch;
-+ (CGPoint)selectCoordsFor:(UITouch *)touch;
+- (void)updatePointerCoords:(UITouch *)touch;
 
 
 - (instancetype)initWithTouch:(UITouch *)touch;

Limelight/Input/NativeTouchPointer.m

@@ -12,38 +12,28 @@
 
 // native touch pointer ojbect that stores & manipulates touch coordinates
 
-static NSMutableDictionary *pointerObjDict;
-
 static CGFloat pointerVelocityFactor;
-static CGFloat pointerVelocityDivider;
-static CGFloat pointerVelocityDividerLocationByPoints;
 static CGFloat streamViewHeight;
 static CGFloat streamViewWidth;
 static CGFloat fixedResetCoordX;
 static CGFloat fixedResetCoordY;
-static StreamView* _streamView;
+static StreamView* streamView;
 
 @implementation NativeTouchPointer{
-    CGPoint initialPoint;
-    CGPoint latestPoint;
-    CGPoint previousPoint;
-    CGPoint latestRelativePoint;
-    CGPoint previousRelativePoint;
-    bool useRelativeCoords;
 }
 
 - (instancetype)initWithTouch:(UITouch *)touch{
     self = [super init];
-    self->initialPoint = [touch locationInView:_streamView];
-    self->latestPoint = self->initialPoint;
-    self->latestRelativePoint = self->initialPoint;
+    _initialPoint = [touch locationInView:streamView];
+    _latestPoint = _initialPoint;
+    _latestRelativePoint = _initialPoint;
     return self;
 }
 
 - (bool)doesNeedResetCoords{
-    bool boundaryReached = (latestRelativePoint.x > streamViewWidth || latestRelativePoint.x < 0.0f ||  latestRelativePoint.y > streamViewHeight || latestRelativePoint.y < 0.0f);
-    bool withinExcludedArea =  (initialPoint.x > streamViewWidth * (0.75) && initialPoint.x < streamViewWidth) && (initialPoint.y > streamViewHeight * (0.8) && initialPoint.y < streamViewHeight);
-    _needResetCoords = (pointerVelocityFactor > 1.0f) && self->useRelativeCoords && boundaryReached && !withinExcludedArea; 
+    bool boundaryReached = (_latestRelativePoint.x > streamViewWidth || _latestRelativePoint.x < 0.0f ||  _latestRelativePoint.y > streamViewHeight || _latestRelativePoint.y < 0.0f);
+    bool withinExcludedArea =  (_initialPoint.x > streamViewWidth * (0.75) && _initialPoint.x < streamViewWidth) && (_initialPoint.y > streamViewHeight * (0.8) && _initialPoint.y < streamViewHeight);
+    _needResetCoords = (pointerVelocityFactor > 1.0f) && _useRelativeCoords && boundaryReached && !withinExcludedArea; 
     // boundary detection & coordinates reset to the specific point for HK:StarTrail(needs a very high pointer velocity)
     // must exclude touch pointer that uses native coords instead of relative ones.
     // also exclude touch pointer created within  the bottom right corner
@@ -52,74 +42,26 @@ static StreamView* _streamView;
 }
 
 - (void)updatePointerCoords:(UITouch *)touch{
-    previousPoint = latestPoint;
-    latestPoint = [touch locationInView:_streamView];
-    previousRelativePoint = latestRelativePoint;
+    _previousPoint = _latestPoint;
+    _latestPoint = [touch locationInView:streamView];
+    _previousRelativePoint = _latestRelativePoint;
     if(_needResetCoords){// boundary detection & coordinates reset to the central screen point for HK:StarTrail(needs a very high pointer velocity); // boundary detection & coordinates reset to specific point for HK:StarTrail(needs a very high pointer velocity)
-        previousRelativePoint.x = fixedResetCoordX;
-        previousRelativePoint.y = fixedResetCoordY;
+        _previousRelativePoint.x = fixedResetCoordX;
+        _previousRelativePoint.y = fixedResetCoordY;
     }
-    latestRelativePoint.x = previousRelativePoint.x + pointerVelocityFactor * (latestPoint.x - previousPoint.x);
-    latestRelativePoint.y = previousRelativePoint.y + pointerVelocityFactor * (latestPoint.y - previousPoint.y);
+    _latestRelativePoint.x = _previousRelativePoint.x + pointerVelocityFactor * (_latestPoint.x - _previousPoint.x);
+    _latestRelativePoint.y = _previousRelativePoint.y + pointerVelocityFactor * (_latestPoint.y - _previousPoint.y);
 }
 
-+ (void)initContextWithView:(StreamView *)view {
-    _streamView = view;
-    streamViewWidth = _streamView.frame.size.width;
++ (void)initContextWithView:(StreamView*)view andSettings:(TemporarySettings*)settings {
+    streamView = view;
+    streamViewWidth = streamView.frame.size.width;
     fixedResetCoordX = streamViewWidth * 0.3;
-    streamViewHeight = _streamView.frame.size.height;
+    streamViewHeight = streamView.frame.size.height;
     fixedResetCoordY = streamViewHeight * 0.4;
+    pointerVelocityFactor = settings.touchPointerVelocityFactor.floatValue;
     NSLog(@"stream wdith %f, stream height %f", streamViewWidth, streamViewHeight);
-    pointerObjDict = [NSMutableDictionary dictionary];
-    pointerVelocityDividerLocationByPoints = CGRectGetWidth([[UIScreen mainScreen] bounds]) * pointerVelocityDivider;
-    NSLog(@"pointerVelocityDivider:  %.2f", pointerVelocityDivider);
     NSLog(@"pointerVelocityFactor:  %.2f", pointerVelocityFactor);
-    NSLog(@"pointerVelocityDividerLocationByPoints:  %.2f", pointerVelocityDividerLocationByPoints);
 }
 
-+ (void)setPointerVelocityDivider:(CGFloat)dividerLocation{
-    pointerVelocityDivider = dividerLocation;
-}
-
-+ (void)setPointerVelocityFactor:(CGFloat)velocityFactor{
-    pointerVelocityFactor = velocityFactor;
-}
-
-
-+ (void)populatePointerObjIntoDict:(UITouch*)touch{
-    [pointerObjDict setObject:[[NativeTouchPointer alloc] initWithTouch:touch] forKey:@((uintptr_t)touch)];
-}
-
-+ (void)removePointerObjFromDict:(UITouch*)touch{
-    uintptr_t memAddrValue = (uintptr_t)touch;
-    NativeTouchPointer* pointer = [pointerObjDict objectForKey:@(memAddrValue)];
-    if(pointer != nil){
-        [pointerObjDict removeObjectForKey:@(memAddrValue)];
-    }
-}
-
-+ (NativeTouchPointer* )getPointerObjFromDict:(UITouch*)touch{
-    return [pointerObjDict objectForKey:@((uintptr_t)touch)];
-}
-
-
-+ (void)updatePointerObjInDict:(UITouch *)touch{
-    [[pointerObjDict objectForKey:@((uintptr_t)touch)] updatePointerCoords:touch];
-}
-
-
-+ (CGPoint)selectCoordsFor:(UITouch *)touch{
-    NativeTouchPointer *pointer = [pointerObjDict objectForKey:@((uintptr_t)touch)];
-    if(pointer == nil) return CGPointMake(0, 0); 
-    if((pointer -> initialPoint).x > pointerVelocityDividerLocationByPoints){  //if first contact coords locates on the right side of divider.
-        pointer -> useRelativeCoords = true;
-        return pointer->latestRelativePoint;
-    }
-    else{
-        pointer -> useRelativeCoords = false;
-        return[touch locationInView:_streamView];
-    }
-}
-
-
 @end

Limelight/Input/OnScreenControls.m

@@ -222,7 +222,7 @@ static float L3_Y;
     DataManager* dataMan = [[DataManager alloc] init];
     Settings* settings = [dataMan retrieveSettings];
     _largerStickLR1 = settings.largerStickLR1;
-    _oscTapExlusionAreaSizeFactor = settings.oscTapExlusionAreaSize.floatValue;
+    _oscTapExlusionAreaSizeFactor = 1.05;
         
     _iPad = ([UIDevice currentDevice].userInterfaceIdiom == UIUserInterfaceIdiomPad);
     _controlArea = CGRectMake(0, 0, _view.frame.size.width, _view.frame.size.height);

Limelight/Input/PureNativeTouchHandler.m

@@ -2,23 +2,37 @@
 //  NativeTouchHandler.m
 //  Moonlight
 //
-//  Created by True砖家 on 2024/8/29.
+//  Created by True砖家 on 2024/6/16.
 //  Copyright © 2024 True砖家 on Bilibili. All rights reserved.
 //
 
 #import <Foundation/Foundation.h>
 #import "PureNativeTouchHandler.h"
 #import "NativeTouchPointer.h"
+#import "OnScreenControls.h"
 #import "StreamView.h"
 
 #include <Limelight.h>
 
+/*
+@interface TouchInfoCarrier : NSObject // 明确继承NSObject
+@property (nonatomic, copy) NSString *name;
+@end
+
+@implementation TouchInfoCarrier {
+    
+}
+@end
+ */
+
 
 @implementation PureNativeTouchHandler {
     StreamView* streamView;
     TemporarySettings* currentSettings;
     bool activateCoordSelector;
     bool touchPointSpawnedAtUpperScreenEdge;
+    CGFloat pointerVelocityDividerLocationByPoints;
+    uint16_t touchMoveEventIntervalUs;
     
     // Use a Dictionary to store UITouch object's memory address as key, and pointerId as value,字典存放UITouch对象地址和pointerId映射关系
     // pointerId will be generated from a pre-defined pool
@@ -28,39 +42,106 @@
     NSMutableSet<NSNumber *> *pointerIdPool; //pre-defined pool of pointerIds.
     NSMutableSet<NSNumber *> *unassignedPointerIds;
     
+    NSMutableDictionary *pointerObjDict;
+
     CGFloat slideGestureVerticalThreshold;
     CGFloat screenWidthWithThreshold;
     CGFloat EDGE_TOLERANCE;
+    
+    CGRect streamViewBounds;
 }
 
-
 - (id)initWithView:(StreamView*)view andSettings:(TemporarySettings*)settings{
     self = [super init];
     self->streamView = view;
     self->currentSettings = settings;
     self->activateCoordSelector = currentSettings.pointerVelocityModeDivider.floatValue != 1.0;
+    self->touchMoveEventIntervalUs = currentSettings.touchMoveEventInterval.intValue;
+    self->streamViewBounds = view.bounds;
     
-    pointerIdDict = [NSMutableDictionary dictionary];
-    pointerIdPool = [NSMutableSet set];
+    self->pointerIdDict = [NSMutableDictionary dictionary];
+    self->pointerIdPool = [NSMutableSet set];
     for (uint8_t i = 0; i <= 10; i++) { //ipadOS supports upto 11 finger touches
-        [pointerIdPool addObject:@(i)];
+        [self->pointerIdPool addObject:@(i)];
     }
-    activePointerIds = [NSMutableSet set];
+    self->activePointerIds = [NSMutableSet set];
+    // self->excludedPointerIds = [[NSMutableSet alloc] init];
     self->touchPointSpawnedAtUpperScreenEdge = false;
-
+    
+    
+    
+    self->pointerObjDict = [NSMutableDictionary dictionary];
+    
     EDGE_TOLERANCE = 15.0;
     slideGestureVerticalThreshold = CGRectGetHeight([[UIScreen mainScreen] bounds]) * 0.4;
     screenWidthWithThreshold = CGRectGetWidth([[UIScreen mainScreen] bounds]) - EDGE_TOLERANCE;
 
-    [NativeTouchPointer setPointerVelocityDivider:settings.pointerVelocityModeDivider.floatValue];
-    [NativeTouchPointer setPointerVelocityFactor:settings.touchPointerVelocityFactor.floatValue];
-    [NativeTouchPointer initContextWithView:self->streamView];
+    self->pointerVelocityDividerLocationByPoints = CGRectGetWidth([[UIScreen mainScreen] bounds]) * settings.pointerVelocityModeDivider.floatValue;
+    
+    [NativeTouchPointer initContextWithView:self->streamView andSettings:settings];
+    //_touchesCapturedByOnScreenButtons = [[NSMutableSet alloc] init];
     return self;
 }
 
+- (CGSize) getVideoAreaSize {
+    if (self->streamViewBounds.size.width > self->streamViewBounds.size.height * streamView.streamAspectRatio) {
+        return CGSizeMake(self->streamViewBounds.size.height * streamView.streamAspectRatio, self->streamViewBounds.size.height);
+    } else {
+        return CGSizeMake(self->streamViewBounds.size.width, self->streamViewBounds.size.width / streamView.streamAspectRatio);
+    }
+}
+
+
+- (CGPoint) adjustCoordinatesForVideoArea:(CGPoint)point {
+    // These are now relative to the StreamView, however we need to scale them
+    // further to make them relative to the actual video portion.
+    float x = point.x - self->streamViewBounds.origin.x;
+    float y = point.y - self->streamViewBounds.origin.y;
+    
+    // For some reason, we don't seem to always get to the bounds of the window
+    // so we'll subtract 1 pixel if we're to the left/below of the origin and
+    // and add 1 pixel if we're to the right/above. It should be imperceptible
+    // to the user but it will allow activation of gestures that require contact
+    // with the edge of the screen (like Aero Snap).
+    if (x < self->streamViewBounds.size.width / 2) {
+        x--;
+    }
+    else {
+        x++;
+    }
+    if (y < self->streamViewBounds.size.height / 2) {
+        y--;
+    }
+    else {
+        y++;
+    }
+    
+    // This logic mimics what iOS does with AVLayerVideoGravityResizeAspect
+    CGSize videoSize = [self getVideoAreaSize];
+    CGPoint videoOrigin = CGPointMake(self->streamViewBounds.size.width / 2 - videoSize.width / 2,
+                                      self->streamViewBounds.size.height / 2 - videoSize.height / 2);
+    
+    // Confine the cursor to the video region. We don't just discard events outside
+    // the region because we won't always get one exactly when the mouse leaves the region.
+    return CGPointMake(MIN(MAX(x, videoOrigin.x), videoOrigin.x + videoSize.width) - videoOrigin.x,
+                       MIN(MAX(y, videoOrigin.y), videoOrigin.y + videoSize.height) - videoOrigin.y);
+}
+
+
+- (uint16_t)getRotationFromAzimuthAngle:(float)azimuthAngle {
+    // iOS reports azimuth of 0 when the stylus is pointing west, but Moonlight expects
+    // rotation of 0 to mean the stylus is pointing north. Rotate the azimuth angle
+    // clockwise by 90 degrees to convert from iOS to Moonlight rotation conventions.
+    int32_t rotationAngle = (azimuthAngle - M_PI_2) * (180.f / M_PI);
+    if (rotationAngle < 0) {
+        rotationAngle += 360;
+    }
+    return (uint16_t)rotationAngle;
+}
 
 // generate & populate pointerId into NSDict & NSSet, called in touchesBegan
 - (void)handleTouchDown:(UITouch*)touch{
+    //populate pointerId
     uintptr_t memAddrValue = (uintptr_t)touch;
     unassignedPointerIds = [pointerIdPool mutableCopy]; //reset unassignedPointerIds
     [unassignedPointerIds minusSet:activePointerIds];
@@ -68,10 +149,11 @@
     [pointerIdDict setObject:@(pointerId) forKey:@(memAddrValue)];
     [activePointerIds addObject:@(pointerId)];
     
-    //check if touch point is spawned on the left or right upper half screen edges,
+    //check if touch point is spawned on the left or right upper half screen edges, event to remote PC. this is for better handling in-stream slide gesture
     CGPoint initialPoint = [touch locationInView:self->streamView];
     if(initialPoint.y < slideGestureVerticalThreshold && (initialPoint.x < EDGE_TOLERANCE || initialPoint.x > screenWidthWithThreshold)) {
         self->touchPointSpawnedAtUpperScreenEdge = true;
+        // [excludedPointerIds addObject:@(pointerId)];
     }
 }
 
@@ -82,6 +164,7 @@
     if(pointerIdObj != nil){
         [activePointerIds removeObject:pointerIdObj];
         [pointerIdDict removeObjectForKey:@(memAddrValue)];
+        // if([excludedPointerIds containsObject:pointerIdObj]) [excludedPointerIds removeObject:pointerIdObj]; // remove pointer id from excludedPointerId NSSet
     }
 }
 
@@ -93,50 +176,60 @@
 
 
 - (void)sendTouchEvent:(UITouch*)touch withTouchtype:(uint8_t)touchType{
-    if(touchPointSpawnedAtUpperScreenEdge) return; // we're done here. this touch event will not be sent to the remote PC. and this must be checked after coord selector finishes populating new relative coords, or the app will crash!
-
-    CGPoint targetCoords;
-    if(activateCoordSelector && touch.phase == UITouchPhaseMoved) targetCoords = [NativeTouchPointer selectCoordsFor:touch]; // coordinates of touch pointer replaced to relative ones here.
-    else targetCoords = [touch locationInView:streamView];
+    if(touchPointSpawnedAtUpperScreenEdge) return; //  we're done here. this touch event will not be sent to the remote PC. and this must be checked after coord selector finishes populating new relative coords, or the app will crash!
     
-    CGPoint location = [streamView adjustCoordinatesForVideoArea:targetCoords];
-    CGSize videoSize = [streamView getVideoAreaSize];
+    CGPoint targetCoords;
+    //NSLog(@"selecting coords: %d", touch.phase == UITouchPhaseMoved);
+    // NSLog(@"excluded count: %d", (uint32_t)[excludedPointerIds count]);
+    
+    targetCoords = activateCoordSelector ? [self selectCoordsFor:touch] : [touch locationInView:streamView];
+
+    CGPoint location = [self adjustCoordinatesForVideoArea:targetCoords];
+    CGSize videoSize = [self getVideoAreaSize];
     CGFloat normalizedX = location.x / videoSize.width;
     CGFloat normalizedY = location.y / videoSize.height;
     uint8_t pointerId = [self retrievePointerIdFromDict:touch];
-    
-    if([NativeTouchPointer getPointerObjFromDict:touch].needResetCoords){ // access whether the current pointer has reached the boundary, and need a coord reset.
+
+    if([self getPointerObjFromDict:touch].needResetCoords){ // access whether the current pointer has reached the boundary, and need a coord reset.
         LiSendTouchEvent(LI_TOUCH_EVENT_UP, pointerId, normalizedX, normalizedY, 0, 0, 0, 0);  //event must sent from the lowest level directy by LiSendTouchEvent to simulate continous dragging to another point on screen
         LiSendTouchEvent(LI_TOUCH_EVENT_DOWN, pointerId, 0.3, 0.4, 0, 0, 0, 0);
-    }else LiSendTouchEvent(touchType, pointerId, normalizedX, normalizedY,(touch.force / touch.maximumPossibleForce) / sin(touch.altitudeAngle),0.0f, 0.0f,[streamView getRotationFromAzimuthAngle:[touch azimuthAngleInView:streamView]]);
+    }else LiSendTouchEvent(touchType, pointerId, normalizedX, normalizedY,(touch.force / touch.maximumPossibleForce) / sin(touch.altitudeAngle),0.0f, 0.0f,[self getRotationFromAzimuthAngle:[touch azimuthAngleInView:streamView]]);
 }
 
 - (void)touchesBegan:(NSSet *)touches withEvent:(UIEvent *)event {
-    for (UITouch* touch in touches){
-        [self handleTouchDown:touch]; //generate & populate pointerId
-        if(activateCoordSelector) [NativeTouchPointer populatePointerObjIntoDict:touch];
-        [self sendTouchEvent:touch withTouchtype:LI_TOUCH_EVENT_DOWN];
-    }
+    dispatch_async(dispatch_get_global_queue(QOS_CLASS_USER_INTERACTIVE, 0), ^{
+        for (UITouch* touch in touches){
+            [self handleTouchDown:touch]; //generate & populate pointerId
+            if(self->activateCoordSelector) [self populatePointerObjIntoDict:touch];
+            [self sendTouchEvent:touch withTouchtype:LI_TOUCH_EVENT_DOWN];
+        }
+    });
 }
 
 - (void)touchesMoved:(NSSet *)touches withEvent:(UIEvent *)event {
-    for (UITouch* touch in touches){
-        if(activateCoordSelector) [NativeTouchPointer updatePointerObjInDict:touch];
-        [self sendTouchEvent:touch withTouchtype:LI_TOUCH_EVENT_MOVE];
-        [[NativeTouchPointer getPointerObjFromDict:touch] doesNeedResetCoords]; // execute the judging of doesReachBoundary for current pointer instance. (happens after the event is sent to Sunshine service)
-    }
+    dispatch_async(dispatch_get_global_queue(QOS_CLASS_USER_INTERACTIVE, 0), ^{
+    // NSLog(@"captured by OSB touches: %d", (uint32_t)[OnScreenControls.touchAddrsCapturedByOnScreenControls count]);
+        for (UITouch* touch in touches){
+            if(self->activateCoordSelector) [self updatePointerObjInDict:touch];
+            [self sendTouchEvent:touch withTouchtype:LI_TOUCH_EVENT_MOVE];
+            [[self getPointerObjFromDict:touch] doesNeedResetCoords]; // execute the judging of doesReachBoundary for current pointer instance. (happens after the event is sent to Sunshine service)
+            usleep(self->touchMoveEventIntervalUs);
+            // NSLog(@"interval: %d", self->touchMoveEventIntervalUs);
+        }
+    });
     return;
 }
 
 
 - (void)touchesEnded:(NSSet *)touches withEvent:(UIEvent *)event {
-    for (UITouch* touch in touches){
-        [self sendTouchEvent:touch withTouchtype:LI_TOUCH_EVENT_UP]; //send touch event before remove pointerId
-        [self removePointerId:touch]; //then remove pointerId
-        if(activateCoordSelector) [NativeTouchPointer removePointerObjFromDict:touch];
-    }
-    if(touchPointSpawnedAtUpperScreenEdge && [[event allTouches] count] == [touches count]) touchPointSpawnedAtUpperScreenEdge = false;
-
+    dispatch_async(dispatch_get_global_queue(QOS_CLASS_USER_INTERACTIVE, 0), ^{
+        for (UITouch* touch in touches){
+            [self sendTouchEvent:touch withTouchtype:LI_TOUCH_EVENT_UP]; //send touch event before remove pointerId
+            [self removePointerId:touch]; //then remove pointerId
+            if(self->activateCoordSelector) [self removePointerObjFromDict:touch];
+        }
+        if(self->touchPointSpawnedAtUpperScreenEdge && [[event allTouches] count] == [touches count]) self->touchPointSpawnedAtUpperScreenEdge = false;
+    });
     return;
 }
 
@@ -145,4 +238,33 @@
 }
 
 
+- (void)populatePointerObjIntoDict:(UITouch*)touch{
+    NativeTouchPointer* pointer = [[NativeTouchPointer alloc] initWithTouch:touch];
+    pointer.useRelativeCoords = pointer.initialPoint.x > pointerVelocityDividerLocationByPoints;
+    [pointerObjDict setObject:pointer forKey:@((uintptr_t)touch)];
+}
+
+- (NativeTouchPointer* )getPointerObjFromDict:(UITouch*)touch{
+    return [pointerObjDict objectForKey:@((uintptr_t)touch)];
+}
+
+- (void)removePointerObjFromDict:(UITouch*)touch{
+    uintptr_t memAddrValue = (uintptr_t)touch;
+    NativeTouchPointer* pointer = [pointerObjDict objectForKey:@(memAddrValue)];
+    if(pointer != nil){
+        [pointerObjDict removeObjectForKey:@(memAddrValue)];
+    }
+}
+
+- (void)updatePointerObjInDict:(UITouch *)touch{
+    [[pointerObjDict objectForKey:@((uintptr_t)touch)] updatePointerCoords:touch];
+}
+
+- (CGPoint)selectCoordsFor:(UITouch *)touch{
+    NativeTouchPointer *pointer = [pointerObjDict objectForKey:@((uintptr_t)touch)];
+    if(pointer == nil) return CGPointMake(0, 0);
+   // NSLog(@"using relative Coords: intialX %f, divider loc %f", pointer.initialPoint.x, pointerVelocityDividerLocationByPoints);
+    return pointer.useRelativeCoords ? pointer.latestRelativePoint : pointer.latestPoint;
+}
+
 @end

Limelight/Input/StreamView.h

@@ -29,6 +29,7 @@
 #endif
 
 @property (assign, nonatomic) UIView* streamFrameTopLayerView;
+@property (assign, nonatomic) CGFloat streamAspectRatio;
 
 - (void) setupStreamView:(ControllerSupport*)controllerSupport
      interactionDelegate:(id<UserInteractionDelegate>)interactionDelegate

Limelight/Input/StreamView.m

@@ -81,6 +81,7 @@ static const double X1_MOUSE_SPEED_DIVISOR = 2.5;
     self.streamFrameTopLayerView = topLayerView; // this will be used as a read-only pointer for other class
     self->interactionDelegate = interactionDelegate;
     self->streamAspectRatio = (float)streamConfig.width / (float)streamConfig.height;
+    self.streamAspectRatio = self->streamAspectRatio;
     
     settings = [[[DataManager alloc] init] getSettings];
     
@@ -598,8 +599,8 @@ static const double X1_MOUSE_SPEED_DIVISOR = 2.5;
     [self startInteractionTimer];
     
     if(settings.touchMode.intValue == REGULAR_NATIVE_TOUCH || settings.touchMode.intValue == RELATIVE_TOUCH){
-        [onScreenControls handleTouchDownEvent:touches];
-        [touchHandler touchesBegan:touches withEvent:event];
+        [self->onScreenControls handleTouchDownEvent:touches];
+        [self->touchHandler touchesBegan:touches withEvent:event];
     }
     else if(![onScreenControls handleTouchDownEvent:touches]) [touchHandler touchesBegan:touches withEvent:event];
 }
@@ -761,11 +762,12 @@ static const double X1_MOUSE_SPEED_DIVISOR = 2.5;
     }
     
     hasUserInteracted = YES;
-    if(settings.touchMode.intValue == REGULAR_NATIVE_TOUCH || settings.touchMode.intValue == RELATIVE_TOUCH){
-        [touchHandler touchesMoved:touches withEvent:event];
-        [onScreenControls handleTouchMovedEvent:touches];
+    
+    if(self->settings.touchMode.intValue == REGULAR_NATIVE_TOUCH || self->settings.touchMode.intValue == RELATIVE_TOUCH){
+        [self->touchHandler touchesMoved:touches withEvent:event];
+        [self->onScreenControls handleTouchMovedEvent:touches];
     }
-    else if(![onScreenControls handleTouchMovedEvent:touches]) [touchHandler touchesMoved:touches withEvent:event];
+    else if(![self->onScreenControls handleTouchMovedEvent:touches]) [self->touchHandler touchesMoved:touches withEvent:event];
 }
 
 - (void) handleKeyCombos:(UIPress*) press{
@@ -858,8 +860,8 @@ static const double X1_MOUSE_SPEED_DIVISOR = 2.5;
     hasUserInteracted = YES;
     
     if(settings.touchMode.intValue == REGULAR_NATIVE_TOUCH || settings.touchMode.intValue == RELATIVE_TOUCH){
-        [touchHandler touchesEnded:touches withEvent:event]; // when touches ended, must call the native touchhandler before onScreenControls, since the NSSet of touches captured by on screen button shall be updated later
-        [onScreenControls handleTouchUpEvent:touches];
+        [self->touchHandler touchesEnded:touches withEvent:event]; // when touches ended, must call the native touchhandler before onScreenControls, since the NSSet of touches captured by on screen button shall be updated later
+        [self->onScreenControls handleTouchUpEvent:touches];
     }
     else if(![onScreenControls handleTouchUpEvent:touches]) [touchHandler touchesEnded:touches withEvent:event];
 }