Add input forwarding to Celeste

TAS.Avalonia/Communication/StudioCommunicationServer.cs

@@ -76,9 +76,6 @@ private void ProcessSendCurrentBindings(byte[] data) {
         Dictionary<int, List<int>> nativeBindings = BinaryFormatterHelper.FromByteArray<Dictionary<int, List<int>>>(data);
         Dictionary<HotkeyID, List<Keys>> bindings =
             nativeBindings.ToDictionary(pair => (HotkeyID) pair.Key, pair => pair.Value.Cast<Keys>().ToList());
-        foreach (var pair in bindings) {
-            Log(pair.ToString());
-        }
 
         OnBindingsUpdated(bindings);
         //

TAS.Avalonia/Controls/EditorControl.axaml.cs

@@ -54,6 +54,7 @@ public EditorControl() {
         var csharpLanguage = _registryOptions.GetLanguageByExtension(".cs");
         _textMateInstallation.SetGrammar(_registryOptions.GetScopeByLanguageId(csharpLanguage.Id));
         editor.TextArea.ActiveInputHandler = new TASInputHandler(editor.TextArea);
+        editor.TextArea.PushStackedInputHandler(new TASStackedInputHandler(editor.TextArea));
         editor.TextArea.Caret.PositionChanged += (_, _) => CaretPosition = editor.TextArea.Caret.Position;
         editor.TextArea.AddHandler(TextInputEvent, HandleActionInput, RoutingStrategies.Tunnel);
         editor.TextArea.TextView.BackgroundRenderers.Add(new TASLineRenderer(editor.TextArea));
@@ -152,6 +153,13 @@ private void Document_TextChanged(object sender, EventArgs args) {
     }
 
     private static void HandleActionInput(TextArea textArea, TextInputEventArgs e) {
+        // Manually cancel the event, because the hotkey got forwarded to Celeste
+        if (TASStackedInputHandler.CancelNextTextInputEvent.IsRunning && TASStackedInputHandler.CancelNextTextInputEvent.Elapsed < TASStackedInputHandler.CancellationTime) {
+            TASStackedInputHandler.CancelNextTextInputEvent.Stop();
+            e.Handled = true;
+            return;
+        }
+
         // We can only handle single characters
         if (e.Text is not { Length: 1 }) return;
 

TAS.Avalonia/Editing/TASStackedInputHandler.cs

@@ -0,0 +1,65 @@
+using Avalonia;
+using Avalonia.Input;
+using AvaloniaEdit.Editing;
+using System.Diagnostics;
+
+namespace TAS.Avalonia.Editing;
+
+public class TASStackedInputHandler : TextAreaStackedInputHandler {
+
+    public TASStackedInputHandler(TextArea textArea) : base(textArea)
+    { }
+
+    // Pretty ugly solution to avoid typing characters when trying to use a hotkey.
+    // Just cancelling the KeyDown event does not work. (see https://github.com/AvaloniaUI/Avalonia/issues/14108)
+    // There is probably some better solution, however this works since the KeyDown event is fired before the TextInput event.
+    // We apply a timeout, since some KeyDown events might not cause a TextInput event to reset it.
+    internal static readonly TimeSpan CancellationTime = TimeSpan.FromMilliseconds(50);
+    internal static readonly Stopwatch CancelNextTextInputEvent = new();
+
+    // This is also horrible because i could not fine a simple way of just checking if a key is down.
+    // Only supports modifier keys, since everything else is even messier.
+    // Using KeyModifiers loses the context if the left or right key was pressed.
+    // That context is needed to properly support input forwarding.
+    private readonly HashSet<Key> pressedModKeys = new();
+
+    public override void OnPreviewKeyDown(KeyEventArgs e) {
+        var app = (Application.Current as App)!;
+
+        // If the key was released outside the window, a KeyUp event wouldn't be raised.
+        // So we need to check if the pressed keys
+        if (!e.KeyModifiers.HasFlag(KeyModifiers.Shift)) {
+            pressedModKeys.Remove(Key.LeftShift);
+            pressedModKeys.Remove(Key.RightShift);
+        }
+        if (!e.KeyModifiers.HasFlag(KeyModifiers.Control)) {
+            pressedModKeys.Remove(Key.LeftCtrl);
+            pressedModKeys.Remove(Key.RightCtrl);
+        }
+        if (!e.KeyModifiers.HasFlag(KeyModifiers.Alt)) {
+            pressedModKeys.Remove(Key.LeftAlt);
+            pressedModKeys.Remove(Key.RightAlt);
+        }
+
+        if (e.Key is Key.LeftShift or Key.RightShift or Key.LeftCtrl or Key.RightCtrl or Key.LeftAlt or Key.RightAlt)
+            pressedModKeys.Add(e.Key);
+
+        if (app.SettingsService.SendInputs) {
+            e.Handled = app.CelesteService.SendKeyEvent(pressedModKeys.Concat(new List<Key> { e.Key }.AsReadOnly()), released: false);
+            if (e.Handled)
+                CancelNextTextInputEvent.Restart();
+        }
+    }
+
+    public override void OnPreviewKeyUp(KeyEventArgs e) {
+        var app = (Application.Current as App)!;
+        if (app.SettingsService.SendInputs) {
+            e.Handled = app.CelesteService.SendKeyEvent(pressedModKeys.Concat(new List<Key> { e.Key }.AsReadOnly()), released: true);
+            if (e.Handled)
+                CancelNextTextInputEvent.Restart();
+        }
+
+        if (e.Key is Key.LeftShift or Key.RightShift or Key.LeftCtrl or Key.RightCtrl or Key.LeftAlt or Key.RightAlt)
+            pressedModKeys.Add(e.Key);
+    }
+}

TAS.Avalonia/Keys.cs

@@ -1,3 +1,5 @@
+using Avalonia.Input;
+
 namespace TAS.Avalonia;
 
 // from WinForms
@@ -391,3 +393,186 @@ public enum Keys {
     /// <summary><para>The computer sleep key.</para></summary>
     Sleep = IMEAccept | A, // 0x0000005F
 }
+
+public static class KeysExtensions {
+    public static Keys ToWinForms(this Key key) => key switch {
+        Key.None => Keys.None,
+        Key.Cancel => Keys.Cancel,
+        Key.Back => Keys.Back,
+        Key.Tab => Keys.Tab,
+        Key.LineFeed => Keys.LineFeed,
+        Key.Clear => Keys.Clear,
+        Key.Return => Keys.Return,
+        Key.Pause => Keys.Pause,
+        Key.CapsLock => Keys.CapsLock,
+        Key.HangulMode => Keys.HangulMode,
+        Key.JunjaMode => Keys.JunjaMode,
+        Key.FinalMode => Keys.FinalMode,
+        Key.KanjiMode => Keys.KanjiMode,
+        Key.Escape => Keys.Escape,
+        Key.ImeConvert => Keys.IMEConvert,
+        Key.ImeNonConvert => Keys.IMENonconvert,
+        Key.ImeAccept => Keys.IMEAccept,
+        Key.ImeModeChange => Keys.IMEModeChange,
+        Key.Space => Keys.Space,
+        Key.PageUp => Keys.PageUp,
+        Key.PageDown => Keys.PageDown,
+        Key.End => Keys.End,
+        Key.Home => Keys.Home,
+        Key.Left => Keys.Left,
+        Key.Up => Keys.Up,
+        Key.Right => Keys.Right,
+        Key.Down => Keys.Down,
+        Key.Select => Keys.Select,
+        Key.Print => Keys.Print,
+        Key.Execute => Keys.Execute,
+        Key.Snapshot => Keys.Snapshot,
+        Key.Insert => Keys.Insert,
+        Key.Delete => Keys.Delete,
+        Key.Help => Keys.Help,
+        Key.D0 => Keys.D0,
+        Key.D1 => Keys.D1,
+        Key.D2 => Keys.D2,
+        Key.D3 => Keys.D3,
+        Key.D4 => Keys.D4,
+        Key.D5 => Keys.D5,
+        Key.D6 => Keys.D6,
+        Key.D7 => Keys.D7,
+        Key.D8 => Keys.D8,
+        Key.D9 => Keys.D9,
+        Key.A => Keys.A,
+        Key.B => Keys.B,
+        Key.C => Keys.C,
+        Key.D => Keys.D,
+        Key.E => Keys.E,
+        Key.F => Keys.F,
+        Key.G => Keys.G,
+        Key.H => Keys.H,
+        Key.I => Keys.I,
+        Key.J => Keys.J,
+        Key.K => Keys.K,
+        Key.L => Keys.L,
+        Key.M => Keys.M,
+        Key.N => Keys.N,
+        Key.O => Keys.O,
+        Key.P => Keys.P,
+        Key.Q => Keys.Q,
+        Key.R => Keys.R,
+        Key.S => Keys.S,
+        Key.T => Keys.T,
+        Key.U => Keys.U,
+        Key.V => Keys.V,
+        Key.W => Keys.W,
+        Key.X => Keys.X,
+        Key.Y => Keys.Y,
+        Key.Z => Keys.Z,
+        Key.LWin => Keys.LWin,
+        Key.RWin => Keys.RWin,
+        Key.Apps => Keys.Apps,
+        Key.Sleep => Keys.Sleep,
+        Key.NumPad0 => Keys.NumPad0,
+        Key.NumPad1 => Keys.NumPad1,
+        Key.NumPad2 => Keys.NumPad2,
+        Key.NumPad3 => Keys.NumPad3,
+        Key.NumPad4 => Keys.NumPad4,
+        Key.NumPad5 => Keys.NumPad5,
+        Key.NumPad6 => Keys.NumPad6,
+        Key.NumPad7 => Keys.NumPad7,
+        Key.NumPad8 => Keys.NumPad8,
+        Key.NumPad9 => Keys.NumPad9,
+        Key.Multiply => Keys.Multiply,
+        Key.Add => Keys.Add,
+        Key.Separator => Keys.Separator,
+        Key.Subtract => Keys.Subtract,
+        Key.Decimal => Keys.Decimal,
+        Key.Divide => Keys.Divide,
+        Key.F1 => Keys.F1,
+        Key.F2 => Keys.F2,
+        Key.F3 => Keys.F3,
+        Key.F4 => Keys.F4,
+        Key.F5 => Keys.F5,
+        Key.F6 => Keys.F6,
+        Key.F7 => Keys.F7,
+        Key.F8 => Keys.F8,
+        Key.F9 => Keys.F9,
+        Key.F10 => Keys.F10,
+        Key.F11 => Keys.F11,
+        Key.F12 => Keys.F12,
+        Key.F13 => Keys.F13,
+        Key.F14 => Keys.F14,
+        Key.F15 => Keys.F15,
+        Key.F16 => Keys.F16,
+        Key.F17 => Keys.F17,
+        Key.F18 => Keys.F18,
+        Key.F19 => Keys.F19,
+        Key.F20 => Keys.F20,
+        Key.F21 => Keys.F21,
+        Key.F22 => Keys.F22,
+        Key.F23 => Keys.F23,
+        Key.F24 => Keys.F24,
+        Key.NumLock => Keys.NumLock,
+        Key.Scroll => Keys.Scroll,
+        Key.LeftShift => Keys.LShiftKey,
+        Key.RightShift => Keys.RShiftKey,
+        Key.LeftCtrl => Keys.LControlKey,
+        Key.RightCtrl => Keys.RControlKey,
+        Key.LeftAlt => Keys.LMenu,
+        Key.RightAlt => Keys.RMenu,
+        Key.BrowserBack => Keys.BrowserBack,
+        Key.BrowserForward => Keys.BrowserForward,
+        Key.BrowserRefresh => Keys.BrowserRefresh,
+        Key.BrowserStop => Keys.BrowserStop,
+        Key.BrowserSearch => Keys.BrowserSearch,
+        Key.BrowserFavorites => Keys.BrowserFavorites,
+        Key.BrowserHome => Keys.BrowserHome,
+        Key.VolumeMute => Keys.VolumeMute,
+        Key.VolumeDown => Keys.VolumeDown,
+        Key.VolumeUp => Keys.VolumeUp,
+        Key.MediaNextTrack => Keys.MediaNextTrack,
+        Key.MediaPreviousTrack => Keys.MediaPreviousTrack,
+        Key.MediaStop => Keys.MediaStop,
+        Key.MediaPlayPause => Keys.MediaPlayPause,
+        Key.LaunchMail => Keys.LaunchMail,
+        Key.SelectMedia => Keys.SelectMedia,
+        Key.LaunchApplication1 => Keys.LaunchApplication1,
+        Key.LaunchApplication2 => Keys.LaunchApplication2,
+        Key.OemSemicolon => Keys.OemSemicolon,
+        Key.OemPlus => Keys.Oemplus,
+        Key.OemComma => Keys.Oemcomma,
+        Key.OemMinus => Keys.OemMinus,
+        Key.OemPeriod => Keys.OemPeriod,
+        Key.OemQuestion => Keys.OemQuestion,
+        Key.OemTilde => Keys.Oemtilde,
+        // Key.AbntC1 => Keys.AbntC1,
+        // Key.AbntC2 => Keys.AbntC2,
+        Key.OemOpenBrackets => Keys.OemOpenBrackets,
+        Key.OemPipe => Keys.OemPipe,
+        Key.OemCloseBrackets => Keys.OemCloseBrackets,
+        Key.OemQuotes => Keys.OemQuotes,
+        Key.Oem8 => Keys.Oem8,
+        Key.OemBackslash => Keys.OemBackslash,
+        // Key.ImeProcessed => Keys.ImeProcessed,
+        // Key.System => Keys.System,
+        // Key.OemAttn => Keys.OemAttn,
+        // Key.OemFinish => Keys.OemFinish,
+        // Key.DbeHiragana => Keys.DbeHiragana,
+        // Key.DbeSbcsChar => Keys.DbeSbcsChar,
+        // Key.DbeDbcsChar => Keys.DbeDbcsChar,
+        // Key.OemBackTab => Keys.OemBackTab,
+        // Key.DbeNoRoman => Keys.DbeNoRoman,
+        Key.CrSel => Keys.Crsel,
+        Key.ExSel => Keys.Exsel,
+        Key.EraseEof => Keys.EraseEof,
+        Key.Play => Keys.Play,
+        // Key.DbeNoCodeInput => Keys.DbeNoCodeInput,
+        Key.NoName => Keys.NoName,
+        // Key.DbeEnterDialogConversionMode => Keys.DbeEnterDialogConversionMode,
+        Key.OemClear => Keys.OemClear,
+        // Key.DeadCharProcessed => Keys.DeadCharProcessed,
+        // Key.FnLeftArrow => Keys.FnLeftArrow,
+        // Key.FnRightArrow => Keys.FnRightArrow,
+        // Key.FnUpArrow => Keys.FnUpArrow,
+        // Key.FnDownArrow => Keys.FnDownArrow,
+        _ => throw new ArgumentOutOfRangeException(nameof(key), key, null)
+    };
+}

TAS.Avalonia/Services/CelesteService.cs

@@ -1,5 +1,5 @@
+using Avalonia.Input;
 using System.Globalization;
-using JetBrains.Annotations;
 using StudioCommunication;
 using TAS.Avalonia.Communication;
 
@@ -21,6 +21,42 @@ public CelesteService() {
     public void WriteWait() => Server.WriteWait();
     public void SendPath(string path) => Server.SendPath(path);
 
+    public bool SendKeyEvent(IEnumerable<Key> keys, bool released) {
+        var winFormsKeys = keys.Select(key => key.ToWinForms()).ToArray();
+        bool pressedAny = false;
+
+        foreach (HotkeyID hotkeyIDs in _bindings.Keys) {
+            List<Keys> bindingKeys = _bindings[hotkeyIDs];
+
+            bool pressed = bindingKeys.Count > 0 && bindingKeys.All(key => winFormsKeys.Contains(key));
+            if (pressed && bindingKeys.Count == 1) {
+                // Don't trigger a hotkey without a modifier if a modifier is pressed
+                if (!bindingKeys.Contains(Keys.LShiftKey) && !bindingKeys.Contains(Keys.RShiftKey) && winFormsKeys.Any(key => key is Keys.LShiftKey or Keys.RShiftKey)) {
+                    pressed = false;
+                }
+                if (!bindingKeys.Contains(Keys.LControlKey) && !bindingKeys.Contains(Keys.RControlKey) && winFormsKeys.Any(key => key is Keys.LControlKey or Keys.RControlKey)) {
+                    pressed = false;
+                }
+                if (!bindingKeys.Contains(Keys.LMenu) && !bindingKeys.Contains(Keys.RMenu) && winFormsKeys.Any(key => key is Keys.LMenu or Keys.RMenu)) {
+                    pressed = false;
+                }
+            }
+
+            if (pressed) {
+                pressedAny = true;
+                // if (hotkeyIDs == HotkeyID.FastForward) {
+                //     fastForwarding = true;
+                // } else if (hotkeyIDs == HotkeyID.SlowForward) {
+                //     slowForwarding = true;
+                // }
+
+                Server.SendHotkeyPressed(hotkeyIDs, released);
+            }
+        }
+
+        return pressedAny;
+    }
+
     public void Play() {
     }
 

TAS.Avalonia/Services/SettingsService.cs

@@ -17,6 +17,10 @@ public string LastOpenFilePath {
     }
     public string LastOpenFileName => LastOpenFilePath == null ? null : Path.GetFileName(LastOpenFilePath);
 
+    public bool SendInputs {
+        get => _settings.SendInputs;
+        set => this.RaiseAndSetIfChanged(ref _settings.SendInputs, value);
+    }
     public bool GameInfoVisible {
         get => _settings.GameInfoVisible;
         set => this.RaiseAndSetIfChanged(ref _settings.GameInfoVisible, value);
@@ -68,6 +72,7 @@ private void OnPropertyChanged(object sender, PropertyChangedEventArgs e) {
     public class Settings {
         public string LastOpenFilePath = "";
 
+        public bool SendInputs = true;
         public bool GameInfoVisible = true;
     }
 }

TAS.Avalonia/TAS.Avalonia.csproj

@@ -2,6 +2,7 @@
   <PropertyGroup>
     <OutputType>WinExe</OutputType>
     <TargetFramework>net7.0</TargetFramework>
+    <LangVersion>latest</LangVersion>
     <Nullable>disable</Nullable>
     <BuiltInComInteropSupport>true</BuiltInComInteropSupport>
     <ApplicationManifest>app.manifest</ApplicationManifest>