implements custom on screen keyboard button

Limelight/Input/CommandManager.swift

@@ -165,7 +165,7 @@ import UIKit
         "LEFT_WIN": 0x5B, // VK_LWIN
         "RIGHT_WIN": 0x5C, // VK_RWIN
         "APPS": 0x5D,        // VK_APPS
-
+        
         // macOS Key Codes
         "CMD": 0x37,     // ⌘ Command
         "OPT": 0x3A,      // ⌥ Option
@@ -222,7 +222,7 @@ import UIKit
     ]
     
     private var commands: [RemoteCommand] = []
-
+    
     public weak var viewController: CommandManagerViewController?
     
     private override init() {
@@ -255,7 +255,7 @@ import UIKit
             defaults.set(data, forKey: "savedCommands")
         }
     }
-
+    
     @objc public func createTestKeyMappings() -> [String: Int16] {
         return CommandManager.keyMappings
     }
@@ -278,7 +278,7 @@ import UIKit
         
         let matchedString = (input as NSString).substring(with: match.range(at: 0))
         let keyStrings = matchedString.split(separator: "+").map { String($0) }
-
+        
         guard !keyStrings.isEmpty else {
             print("No key strings found in the matched string")
             return nil
@@ -305,7 +305,7 @@ import UIKit
         }
         
         return validKeyStrings
-
+        
     }
     
     @objc public func addCommand(_ command: RemoteCommand) -> Bool {
@@ -349,11 +349,33 @@ import UIKit
             }
         }
     }
-
+    
     
     private func saveCommands() {
         if let data = try? NSKeyedArchiver.archivedData(withRootObject: commands, requiringSecureCoding: false) {
             UserDefaults.standard.set(data, forKey: "savedCommands")
         }
     }
+    
+    @objc public func sendKeyDownEventWithDelay(keyboardCmdStrings: [String], delay: TimeInterval = 0.05, index: Int = 0) {
+        guard index < keyboardCmdStrings.count else {
+            for keyStr in keyboardCmdStrings {
+                LiSendKeyboardEvent(CommandManager.keyMappings[keyStr]!,Int8(KEY_ACTION_UP), 0)
+            }
+            return
+        } // 如果已经处理完所有键，释放所有按键
+        
+        let keyCode = CommandManager.keyMappings[keyboardCmdStrings[index]]
+        guard let keyCode = keyCode else {
+            print("No mapping found for \(keyboardCmdStrings[index])")
+            sendKeyDownEventWithDelay(keyboardCmdStrings: keyboardCmdStrings, delay: delay, index: index+1) // 跳过当前键，继续下一个
+            return
+        }
+        // 发送当前键的键盘按下事件
+        LiSendKeyboardEvent(keyCode,Int8(KEY_ACTION_DOWN), 0)
+        // 使用 asyncAfter 在指定异步延迟后发送下一个键的键盘按下事件
+        DispatchQueue.main.asyncAfter(deadline: .now() + delay) {
+            self.sendKeyDownEventWithDelay(keyboardCmdStrings: keyboardCmdStrings, delay: delay, index: index+1) // 跳过当前键，继续下一个
+        }
+    }
 }

Limelight/Input/CustomOSC/OSCProfilesManager.m

@@ -200,6 +200,50 @@ static NSMutableDictionary *onScreenKeyViewsDict;
 
 
 - (bool) updateSelectedProfile:(NSMutableArray *) oscButtonLayers {
+    NSMutableArray* buttonStatesEncoded = [self convertOnscreenButtonsToButtonStates:oscButtonLayers];
+    if([self getIndexOfSelectedProfile]==0) return false;
+    OSCProfile *newProfile = [[OSCProfile alloc] initWithName:[self getSelectedProfile].name
+                            buttonStates:buttonStatesEncoded isSelected:YES];        // create a new 'OSCProfile'. Set the array of encoded button states created above to the 'buttonStates' property of the new profile, along with a 'name'. Set 'isSelected' argument to YES which will set this saved profile as the one that will show up in the game stream view
+
+    
+    /* set all saved OSCProfiles 'isSelected' property to NO since the new profile you're adding will be set as the selected profile */
+    NSMutableArray *profiles = [self getAllProfiles];
+    for (OSCProfile *profile in profiles) {
+        profile.isSelected = NO;
+    }
+    [self replaceProfile:[self getSelectedProfile] withProfile:newProfile];
+    return true;
+}
+
+
+- (void) saveProfileWithName:(NSString*)name andButtonLayers:(NSMutableArray *)oscButtonLayers {
+    NSMutableArray* buttonStatesEncoded = [self convertOnscreenButtonsToButtonStates:oscButtonLayers];
+    OSCProfile *newProfile = [[OSCProfile alloc] initWithName:name
+                            buttonStates:buttonStatesEncoded isSelected:YES];        // create a new 'OSCProfile'. Set the array of encoded button states created above to the 'buttonStates' property of the new profile, along with a 'name'. Set 'isSelected' argument to YES which will set this saved profile as the one that will show up in the game stream view
+    /* set all saved OSCProfiles 'isSelected' property to NO since the new profile you're adding will be set as the selected profile */
+    NSMutableArray *profiles = [self getAllProfiles];
+    for (OSCProfile *profile in profiles) {
+        
+        profile.isSelected = NO;
+    }
+    
+    if ([self profileNameAlreadyExist:name]) {  // if a saved profile with the same 'name' already exists in persistent storage then overwrite it and save the change to persistent storage
+        [self replaceProfile:[self OSCProfileWithName:name] withProfile:newProfile];
+    }
+    else {  // otherwise encode then add the new profile to the end of the OSCProfiles array
+        NSData *newProfileEncoded = [NSKeyedArchiver archivedDataWithRootObject:newProfile requiringSecureCoding:YES error:nil];
+        NSMutableArray *profilesEncoded = [self encodedProfilesFromArray:profiles];
+        [profilesEncoded addObject:newProfileEncoded];
+        
+        /* Encode the 'profilesEncoded' array itself, NOT the objects in the 'profilesEncoded' array, all of which are already encoded by this point */
+        NSData *data = [NSKeyedArchiver archivedDataWithRootObject:profilesEncoded requiringSecureCoding:YES error:nil];
+        [[NSUserDefaults standardUserDefaults] setObject:data forKey:@"OSCProfiles"];
+        [[NSUserDefaults standardUserDefaults] synchronize];
+    }
+}
+
+
+- (NSMutableArray* ) convertOnscreenButtonsToButtonStates:(NSMutableArray *) oscButtonLayers {
     /* iterate through each OSC button the user sees on screen, create an 'OnScreenButtonState' object from each button, encode each object, and then add each object to an array */
     /*
     NSSet *validPositionButtonNames = [NSSet setWithObjects:
@@ -223,102 +267,24 @@ static NSMutableDictionary *onScreenKeyViewsDict;
         OnScreenButtonState *buttonState = [[OnScreenButtonState alloc] initWithButtonName:oscButtonLayer.name buttonType:GameControllerButton andPosition:oscButtonLayer.position];
         // add hidden attr here
         buttonState.isHidden = oscButtonLayer.isHidden;
-        
-        /* //-------------------------------------
-        NSLog(@"buttonName to be saved: %@", buttonState.name);
-        if([validPositionButtonNames containsObject:buttonState.name]) buttonState.hasValidPosition = true; // this is for setting up DZ for custom OSC buttons.
-        else buttonState.hasValidPosition = false;
-        //------------------------------------- */
         NSData *buttonStateEncoded = [NSKeyedArchiver archivedDataWithRootObject:buttonState requiringSecureCoding:YES error:nil];
         [buttonStatesEncoded addObject: buttonStateEncoded];
     }
     
-    // save keyboard button as buttonstate
+    // save keyboard button as buttonstate, this is for the new onscreen keyboard buttons:
     [onScreenKeyViewsDict enumerateKeysAndObjectsUsingBlock:^(id key, id value, BOOL *stop) { // this dict is passed from the layout tool VC, as a static obj in this class.
-        OnScreenKeyView *keyView = value;
+        OnScreenKeyboardButtonView *keyView = value;
         OnScreenButtonState *buttonState = [[OnScreenButtonState alloc] initWithButtonName:keyView.keyString buttonType:KeyboardButton andPosition:keyView.frame.origin];
         buttonState.alias = keyView.keyLabel;
         buttonState.timestamp = keyView.timestamp;
         NSData *buttonStateEncoded = [NSKeyedArchiver archivedDataWithRootObject:buttonState requiringSecureCoding:YES error:nil];
         [buttonStatesEncoded addObject: buttonStateEncoded];
     }];
-    
-    
-    
-    if([self getIndexOfSelectedProfile]==0) return false;
-    OSCProfile *newProfile = [[OSCProfile alloc] initWithName:[self getSelectedProfile].name
-                            buttonStates:buttonStatesEncoded isSelected:YES];        // create a new 'OSCProfile'. Set the array of encoded button states created above to the 'buttonStates' property of the new profile, along with a 'name'. Set 'isSelected' argument to YES which will set this saved profile as the one that will show up in the game stream view
-
-    
-    /* set all saved OSCProfiles 'isSelected' property to NO since the new profile you're adding will be set as the selected profile */
-    NSMutableArray *profiles = [self getAllProfiles];
-    for (OSCProfile *profile in profiles) {
-        profile.isSelected = NO;
-    }
-    [self replaceProfile:[self getSelectedProfile] withProfile:newProfile];
-    return true;
+    return buttonStatesEncoded;
 }
 
 
 
-- (void) saveProfileWithName:(NSString*)name andButtonLayers:(NSMutableArray *)buttonLayers {
-    /* iterate through each OSC button the user sees on screen, create an 'OnScreenButtonState' object from each button, encode each object, and then add each object to an array */
-    /*
-    NSSet *validPositionButtonNames = [NSSet setWithObjects:
-        @"l2Button",
-        @"l1Button",
-        @"dPad",
-        @"selectButton",
-        @"leftStickBackground",
-        @"rightStickBackground",
-        @"r2Button",
-        @"r1Button",
-        @"aButton",
-        @"bButton",
-        @"xButton",
-        @"yButton",
-        @"startButton",
-        nil]; */
-    NSMutableArray *buttonStatesEncoded = [[NSMutableArray alloc] init];
-    for (CALayer *buttonLayer in buttonLayers) {
-        OnScreenButtonState *buttonState = [[OnScreenButtonState alloc] initWithButtonName:buttonLayer.name buttonType:GameControllerButton andPosition:buttonLayer.position];
-        // add hidden attr here
-        buttonState.isHidden = buttonLayer.isHidden;
-        /* //-------------------------------------
-        NSLog(@"buttonName to be saved: %@", buttonState.name);
-        if([validPositionButtonNames containsObject:buttonState.name]) buttonState.hasValidPosition = true; // this is for setting up DZ for custom OSC buttons.
-        else buttonState.hasValidPosition = false;
-        //------------------------------------- */
-        NSData *buttonStateEncoded = [NSKeyedArchiver archivedDataWithRootObject:buttonState requiringSecureCoding:YES error:nil];
-        [buttonStatesEncoded addObject: buttonStateEncoded];
-    }
-
-    OSCProfile *newProfile = [[OSCProfile alloc] initWithName:name
-                            buttonStates:buttonStatesEncoded isSelected:YES];        // create a new 'OSCProfile'. Set the array of encoded button states created above to the 'buttonStates' property of the new profile, along with a 'name'. Set 'isSelected' argument to YES which will set this saved profile as the one that will show up in the game stream view
-
-    
-    /* set all saved OSCProfiles 'isSelected' property to NO since the new profile you're adding will be set as the selected profile */
-    NSMutableArray *profiles = [self getAllProfiles];
-    for (OSCProfile *profile in profiles) {
-        
-        profile.isSelected = NO;
-    }
-    
-    if ([self profileNameAlreadyExist:name]) {  // if a saved profile with the same 'name' already exists in persistent storage then overwrite it and save the change to persistent storage
-        [self replaceProfile:[self OSCProfileWithName:name] withProfile:newProfile];
-    }
-    else {  // otherwise encode then add the new profile to the end of the OSCProfiles array
-        NSData *newProfileEncoded = [NSKeyedArchiver archivedDataWithRootObject:newProfile requiringSecureCoding:YES error:nil];
-        NSMutableArray *profilesEncoded = [self encodedProfilesFromArray:profiles];
-        [profilesEncoded addObject:newProfileEncoded];
-        
-        /* Encode the 'profilesEncoded' array itself, NOT the objects in the 'profilesEncoded' array, all of which are already encoded by this point */
-        NSData *data = [NSKeyedArchiver archivedDataWithRootObject:profilesEncoded requiringSecureCoding:YES error:nil];
-        [[NSUserDefaults standardUserDefaults] setObject:data forKey:@"OSCProfiles"];
-        [[NSUserDefaults standardUserDefaults] synchronize];
-    }
-}
-
 #pragma mark - Queries
 
 - (BOOL) profileNameAlreadyExist:(NSString*)name {

Limelight/Input/OnScreenControls.m

@@ -256,7 +256,7 @@ static float L3_Y;
 
 - (void) updateControls {
     if(self._level == OnScreenControlsLevelCustom){
-        // these buttons that has valid coords of positions
+        // mark all OSC buttons that has valid coords of positions
         NSSet *validPositionButtonNames = [NSSet setWithObjects:
             @"l2Button",
             @"l1Button",
@@ -286,9 +286,10 @@ static float L3_Y;
             }
         }
         NSLog(@"_activeCustomOscButtonPositionDict update, active button number: %lu", (unsigned long)[_activeCustomOscButtonPositionDict count]);
-
     }
     
+    
+    // belows are orginal codes:
     switch (self._level) {
         case OnScreenControlsLevelOff:
             [self hideButtons];

Limelight/Input/OnScreenKeyboardButtonView.swift

@@ -9,7 +9,7 @@
 import UIKit
 
 
-@objc class OnScreenKeyView: UIView {
+@objc class OnScreenKeyboardButtonView: UIView {
     @objc static public var editMode: Bool = false
     @objc static public var timestampOfButtonBeingDragged: TimeInterval = 0
     @objc public var keyLabel: String
@@ -42,37 +42,50 @@ import UIKit
     }
     
     @objc public func enableRelocationMode(enabled: Bool){
-        OnScreenKeyView.editMode = enabled
+        OnScreenKeyboardButtonView.editMode = enabled
     }
     
     private func setupView() {
         label.text = self.keyLabel
-        label.font = UIFont.systemFont(ofSize: 22)
-        label.textColor = UIColor(white: 1.0, alpha: 0.8)
-        label.shadowColor = .black
+        //label.font = UIFont.systemFont(ofSize: 28)
         label.translatesAutoresizingMaskIntoConstraints = false
+        label.adjustsFontSizeToFitWidth = true
+        label.minimumScaleFactor = 0.1  // Adjust the scale factor as needed
+        
+        label.textColor = UIColor(white: 1.0, alpha: 0.9)
+        label.textAlignment = .center
+        label.shadowColor = .black
+        label.shadowOffset = CGSize(width: 1, height: 1)
+        label.translatesAutoresizingMaskIntoConstraints = false // enable auto alignment for the label
         
         self.translatesAutoresizingMaskIntoConstraints = true // this is mandatory to prevent unexpected key view location change
         self.layer.borderColor = UIColor.black.cgColor
-        self.layer.borderWidth = 2
-        self.layer.cornerRadius = 8
-        self.backgroundColor = UIColor(white: 0.2, alpha: 0.7)
-        self.layer.shadowRadius = 4
+        self.layer.borderWidth = 0
+        self.layer.cornerRadius = 16
+        self.backgroundColor = UIColor(white: 0.2, alpha: 0.5)
+        self.layer.shadowColor = UIColor.clear.cgColor
+        self.layer.shadowRadius = 8
         self.layer.shadowOpacity = 0.5
         
         addSubview(label)
         
-        NSLayoutConstraint.activate([
-            label.centerXAnchor.constraint(equalTo: centerXAnchor),
-            label.centerYAnchor.constraint(equalTo: centerYAnchor),
-        ])
         
         NSLayoutConstraint.activate([
             //self.widthAnchor.constraint(equalToConstant: UIScreen.main.bounds.width * 0.088),
             //self.heightAnchor.constraint(equalToConstant: UIScreen.main.bounds.height * 0.1),
-            self.widthAnchor.constraint(equalToConstant: 85),
+            self.widthAnchor.constraint(equalToConstant: 75),
             self.heightAnchor.constraint(equalToConstant: 65),
         ])
+        
+        NSLayoutConstraint.activate([
+            label.leadingAnchor.constraint(equalTo: self.leadingAnchor, constant: 5),
+            label.trailingAnchor.constraint(equalTo: self.trailingAnchor, constant: -5),
+            label.centerXAnchor.constraint(equalTo: centerXAnchor),
+            label.centerYAnchor.constraint(equalTo: centerYAnchor),
+            //label.heightAnchor.constraint(equalToConstant: 40)
+        ])
+        
+
     }
     
     // Touch event handling
@@ -80,10 +93,21 @@ import UIKit
         super.touchesBegan(touches, with: event)
         //self.backgroundColor = UIColor(red: 0.0, green: 0.0, blue: 1.0, alpha: 0.7)
         
-        
-        if !OnScreenKeyView.editMode {
+        if !OnScreenKeyboardButtonView.editMode {
             self.backgroundColor = UIColor.systemBlue.withAlphaComponent(0.7)
-            LiSendKeyboardEvent(CommandManager.keyMappings[self.keyString]!,Int8(KEY_ACTION_DOWN), 0)
+            
+            // if the command(keystring contains "+", it's a multi-key command or a quick triggering key, rather than a physical button
+            if(self.keyString.contains("+")){
+                let keyboardCmdStrings = CommandManager.shared.extractKeyStrings(from: self.keyString)!
+                CommandManager.shared.sendKeyDownEventWithDelay(keyboardCmdStrings: keyboardCmdStrings) // send multi-key command
+                DispatchQueue.main.asyncAfter(deadline: .now() + 0.05) { // reset background color immediately 50ms later
+                    self.backgroundColor = self.originalBackgroundColor
+                }
+            }
+            // if there's no "+" in the keystring, treat it as a regular button:
+            else{
+                LiSendKeyboardEvent(CommandManager.keyMappings[self.keyString]!,Int8(KEY_ACTION_DOWN), 0)
+            }
         }
         else {
             if let touch = touches.first {
@@ -96,10 +120,10 @@ import UIKit
     
     override func touchesMoved(_ touches: Set<UITouch>, with event: UIEvent?) {
         super.touchesMoved(touches, with: event)
-        OnScreenKeyView.timestampOfButtonBeingDragged = self.timestamp
+        OnScreenKeyboardButtonView.timestampOfButtonBeingDragged = self.timestamp
         
         // Move the KeyView based on touch movement in relocation mode
-        if OnScreenKeyView.editMode {
+        if OnScreenKeyboardButtonView.editMode {
             if let touch = touches.first {
                 let currentLocation = touch.location(in: superview)
                 let offsetX = currentLocation.x - storedLocation.x
@@ -115,9 +139,9 @@ import UIKit
     
     override func touchesEnded(_ touches: Set<UITouch>, with event: UIEvent?) {
         super.touchesEnded(touches, with: event)
-        if !OnScreenKeyView.editMode {
+        if !OnScreenKeyboardButtonView.editMode && !self.keyString.contains("+") { // if the command(keystring contains "+", it's a multi-key command rather than a single key button
             LiSendKeyboardEvent(CommandManager.keyMappings[self.keyString]!,Int8(KEY_ACTION_UP), 0)
-            return
+            self.backgroundColor = self.originalBackgroundColor
         }
         self.backgroundColor = self.originalBackgroundColor
 

Limelight/Input/StreamView.h

@@ -34,6 +34,9 @@
                           andConfig:(StreamConfiguration*)streamConfig;
 - (void) reloadOnScreenControlsWith:(ControllerSupport*)controllerSupport
                           andConfig:(StreamConfiguration*)streamConfig;
+- (void) clearOnScreenKeyboardButtons;
+- (void) reloadOnScreenKeyboardButtons;
+
 - (CGSize) getVideoAreaSize;
 - (CGPoint) adjustCoordinatesForVideoArea:(CGPoint)point;
 - (uint16_t)getRotationFromAzimuthAngle:(float)azimuthAngle;

Limelight/Input/StreamView.m

@@ -11,6 +11,8 @@
 #import "DataManager.h"
 #import "ControllerSupport.h"
 #import "KeyboardSupport.h"
+#import "Moonlight-Swift.h"
+#import "OSCProfilesManager.h"
 #import "NativeTouchPointer.h"
 #import "NativeTouchHandler.h"
 #import "RelativeTouchHandler.h"
@@ -327,7 +329,6 @@ static const double X1_MOUSE_SPEED_DIVISOR = 2.5;
 - (void) reloadOnScreenControlsWith:(ControllerSupport*)controllerSupport
                          andConfig:(StreamConfiguration*)streamConfig {
     onScreenControls = [[OnScreenControls alloc] initWithView:self controllerSup:controllerSupport streamConfig:streamConfig];
-    
     /*
     // pass mouseRightClickTapRecognizer to onScreenControls obj here:
     if([self isOscEnabled]){
@@ -338,6 +339,45 @@ static const double X1_MOUSE_SPEED_DIVISOR = 2.5;
     [onScreenControls setLevel:(OnScreenControlsLevel)settings.onscreenControls.intValue];
 }
 
+
+- (void) clearOnScreenKeyboardButtons{
+    for (UIView *subview in self.superview.subviews) {
+        // 检查子视图是否是特定类型的实例
+        if ([subview isKindOfClass:[OnScreenKeyboardButtonView class]]) {
+            // 如果是，就添加到将要被移除的数组中
+            [subview removeFromSuperview];
+        }
+    }
+}
+
+- (void) reloadOnScreenKeyboardButtons{
+    
+    NSLog(@"reload on screen keyboard buttons here");
+
+    // remove all keyboard button views first
+    [self clearOnScreenKeyboardButtons];
+    
+    bool customOscEnabled = (settings.touchMode.intValue == RELATIVE_TOUCH || settings.touchMode.intValue == REGULAR_NATIVE_TOUCH) && settings.onscreenControls.intValue == OnScreenControlsLevelCustom;
+    
+    if(!customOscEnabled) return; //returns if we're not in custom osc mode
+    
+    OSCProfilesManager* profilesManager = [OSCProfilesManager sharedManager];
+    OSCProfile *oscProfile = [profilesManager getSelectedProfile]; //returns the currently selected OSCProfile
+
+    if(!OnScreenKeyboardButtonView.editMode){ // in edit mode, keyboard button view will be updated within layoutool view controller.
+        for (NSData *buttonStateEncoded in oscProfile.buttonStates) {
+            OnScreenButtonState* buttonState = [NSKeyedUnarchiver unarchivedObjectOfClass:[OnScreenButtonState class] fromData:buttonStateEncoded error:nil];
+            if(buttonState.buttonType == KeyboardButton){
+                OnScreenKeyboardButtonView* keyView = [[OnScreenKeyboardButtonView alloc] initWithKeyString:buttonState.name keyLabel:buttonState.alias]; //reconstruct keyView
+                keyView.translatesAutoresizingMaskIntoConstraints = NO; // weird but this is mandatory, or you will find no key views added to the right place
+                // Add the KeyView to the view controller's view
+                [self.superview addSubview:keyView]; // add keyboard button to the stream frame view. must add it to the target view before setting location.
+                [keyView setKeyLocationWithXOffset:buttonState.position.x yOffset:buttonState.position.y];
+            }
+        }
+    }
+}
+
 - (OnScreenControlsLevel) getCurrentOscState {
     if (onScreenControls == nil) {
         return OnScreenControlsLevelOff;
@@ -347,6 +387,7 @@ static const double X1_MOUSE_SPEED_DIVISOR = 2.5;
     }
 }
 
+
 - (CGSize) getVideoAreaSize {
     if (self.bounds.size.width > self.bounds.size.height * streamAspectRatio) {
         return CGSizeMake(self.bounds.size.height * streamAspectRatio, self.bounds.size.height);