From 087b3de434d75c282c82aff6774fa1c96cec7fe6 Mon Sep 17 00:00:00 2001 From: Jacob Fu <141651335+FuJacob@users.noreply.github.com> Date: Wed, 10 Jun 2026 21:19:14 -0700 Subject: [PATCH 1/2] fix(insertion): commit accepts IME-safely so Japanese suggestions land on Tab With a composing input method active (Japanese kana, Chinese pinyin, Korean hangul), the synthetic Unicode keystroke that commits an accepted suggestion is re-absorbed into composition by the IME instead of landing as literal text. The session then desyncs against the live field and the suggestion clears and regenerates, which Japanese users reported as Tab consuming the suggestion word by word while inserting nothing. Detect composing input sources via Text Input Sources (TIS), cached and refreshed on the input-source-changed distributed notification, with the classification kept in a pure helper. While one is active, route the commit through the pasteboard and trigger the paste by AXPress on the host's real Edit > Paste menu item, located language-independently by its Cmd-V key equivalent and cached per process. No key event exists on that path, so the IME can neither swallow nor re-interpret the commit. Synthetic Cmd-V remains as a fallback for hosts without a Paste menu item, posted with a session event source at the annotated session tap so its Command flag survives (the HID tap merges flags with the live hardware state, and during a Tab accept only Tab is physically down; observed on-device as 12/12 pastes traversing the tap chain with nothing landing). The user's clipboard is snapshotted and restored around the paste, and the observer tap now also recognizes synthetic events by identity so a raced suppression token cannot let our own fallback Cmd-V classify as a shortcut mutation and tear down the session it is committing. --- Cotabby.xcodeproj/project.pbxproj | 16 +++ Cotabby/App/Core/CotabbyAppEnvironment.swift | 12 ++ Cotabby/Services/Input/InputMonitor.swift | 6 +- .../Input/KeyboardInputSourceMonitor.swift | 86 ++++++++++++++ .../Suggestion/SuggestionInserter.swift | 110 +++++++++++++++--- Cotabby/Support/AXHelper.swift | 40 +++++++ .../CompositionInputModeClassifier.swift | 48 ++++++++ .../CompositionInputModeClassifierTests.swift | 91 +++++++++++++++ 8 files changed, 392 insertions(+), 17 deletions(-) create mode 100644 Cotabby/Services/Input/KeyboardInputSourceMonitor.swift create mode 100644 Cotabby/Support/CompositionInputModeClassifier.swift create mode 100644 CotabbyTests/CompositionInputModeClassifierTests.swift diff --git a/Cotabby.xcodeproj/project.pbxproj b/Cotabby.xcodeproj/project.pbxproj index 66ce36c6..04888f5a 100644 --- a/Cotabby.xcodeproj/project.pbxproj +++ b/Cotabby.xcodeproj/project.pbxproj @@ -115,6 +115,7 @@ 2872D907299F79A9A69BBFCB /* EmojiPickerPanelController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 764659D09C3F0E8FBD267102 /* EmojiPickerPanelController.swift */; }; 28D217A96946A2005FCBEBFD /* emoji.json in Resources */ = {isa = PBXBuildFile; fileRef = C379D77029D6E88C8C1B9AF7 /* emoji.json */; }; 29ABB5488251FD8089D74F51 /* MidWordContinuationPolicy.swift in Sources */ = {isa = PBXBuildFile; fileRef = 357C18383B047F24A531BDCD /* MidWordContinuationPolicy.swift */; }; + 2A53558D66C96E963B23CA11 /* CompositionInputModeClassifierTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = EC4A3C4BC38793EB11F484F1 /* CompositionInputModeClassifierTests.swift */; }; 2BE029A192E82E795490DC7F /* BrowserDomain.swift in Sources */ = {isa = PBXBuildFile; fileRef = AD8025E4A296845FC53E660D /* BrowserDomain.swift */; }; 2C6159231472A849F15BD0AE /* ScreenFrameReader.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5484C8A04B9C00CF79D589EB /* ScreenFrameReader.swift */; }; 2CCF87FD35BF2C438EEA606D /* SelfCaptureGate.swift in Sources */ = {isa = PBXBuildFile; fileRef = E68BE6A22BA0D42C8DD9868C /* SelfCaptureGate.swift */; }; @@ -132,6 +133,7 @@ 303652F15C0FE55595669D81 /* SpellingDictionaryResourceTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D562A73C7C680F2AA65F9F7F /* SpellingDictionaryResourceTests.swift */; }; 30F3F2B6D13CD583136CD787 /* AXHelper.swift in Sources */ = {isa = PBXBuildFile; fileRef = AC70775535A3428991025AB8 /* AXHelper.swift */; }; 3112A355E61878A6A6D1FDF8 /* EmojiQueryRun.swift in Sources */ = {isa = PBXBuildFile; fileRef = DDF6A4E9CE93FD53C60E67E3 /* EmojiQueryRun.swift */; }; + 3124AD2340D4B58AF48A22F3 /* KeyboardInputSourceMonitor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 534F1297DEF3547D0DE56FB2 /* KeyboardInputSourceMonitor.swift */; }; 314AD8F0FD781CB6DDE4603C /* GhostFontMetrics.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9E6439213F2A273702A6E26B /* GhostFontMetrics.swift */; }; 31515DDD173535C4AC777853 /* MirrorOverlayLayout.swift in Sources */ = {isa = PBXBuildFile; fileRef = 54150A507B03221F137D539B /* MirrorOverlayLayout.swift */; }; 317883210D1D1D5CD654E562 /* ModelFileValidator.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5C4E5869D103865486AAAEEC /* ModelFileValidator.swift */; }; @@ -168,6 +170,7 @@ 3FCEF50FDD9EE01AE3711083 /* AXTreeDumpWriter.swift in Sources */ = {isa = PBXBuildFile; fileRef = B27492B04B627DA53BDAD938 /* AXTreeDumpWriter.swift */; }; 3FF6B7DE34A01C4AB7FA54E3 /* MacroTriggerStateMachine.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1C201A65A6B040F90C528A3B /* MacroTriggerStateMachine.swift */; }; 400E1A5145FC8C5BA2FAED0A /* DeepGeometryWalkThrottle.swift in Sources */ = {isa = PBXBuildFile; fileRef = 684737E62EE6495A71344923 /* DeepGeometryWalkThrottle.swift */; }; + 4086A1B07488C4D3D43D86C9 /* KeyboardInputSourceMonitor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 534F1297DEF3547D0DE56FB2 /* KeyboardInputSourceMonitor.swift */; }; 4134ADBE464D00BB748BD9AE /* GeneralPaneView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 07480CE96ED0EBD94817C6B1 /* GeneralPaneView.swift */; }; 418304869F503EDC6465F8D5 /* SentenceBoundaryClassifier.swift in Sources */ = {isa = PBXBuildFile; fileRef = D4B56C250DDEF3E81F9DCBD7 /* SentenceBoundaryClassifier.swift */; }; 4190F8A76196B16ED94D0A55 /* VisualContextModels.swift in Sources */ = {isa = PBXBuildFile; fileRef = BE97A8169438D593C6C23412 /* VisualContextModels.swift */; }; @@ -281,6 +284,7 @@ 6F2FE689BCA50BEAE80AC6F4 /* ShortcutsPaneView.swift in Sources */ = {isa = PBXBuildFile; fileRef = EB630F9814388203DD1CA2EC /* ShortcutsPaneView.swift */; }; 709F365A846B908D953FA92D /* FoundationModelPromptRenderer.swift in Sources */ = {isa = PBXBuildFile; fileRef = FE7BF162A12703249726F20A /* FoundationModelPromptRenderer.swift */; }; 70D6F9480DA4104AD5669569 /* WelcomePermissionStepView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D5D6C2318E405AA717D1C256 /* WelcomePermissionStepView.swift */; }; + 7179FB0EC6411166CCD79F6B /* CompositionInputModeClassifier.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4F283E5B403F9FEBFB4BA04A /* CompositionInputModeClassifier.swift */; }; 735C2E64CA51F58098B30A0D /* it.txt in Resources */ = {isa = PBXBuildFile; fileRef = 0397F1DACB094A0F6A66BC0E /* it.txt */; }; 74422BB837D6A319D12BF981 /* BaseCompletionPromptRenderer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 85EF79E6144D6C6AD062B569 /* BaseCompletionPromptRenderer.swift */; }; 744B06C2488156B178675615 /* PermissionManager.swift in Sources */ = {isa = PBXBuildFile; fileRef = 85BF316556FDA64CB8AD07B6 /* PermissionManager.swift */; }; @@ -417,6 +421,7 @@ B4D36F5D03E3143CE74582F9 /* AppearancePaneView.swift in Sources */ = {isa = PBXBuildFile; fileRef = AFBE491B3CA04FE9069B7B0F /* AppearancePaneView.swift */; }; B55B160E0534AE23BAC1C3DA /* CotabbyApp.swift in Sources */ = {isa = PBXBuildFile; fileRef = CC1EDFB535AAA2EE0D67828A /* CotabbyApp.swift */; }; B588C09233E69C6EDC69BEDC /* LlamaSuggestionEngine.swift in Sources */ = {isa = PBXBuildFile; fileRef = BE04620C905041680116BE80 /* LlamaSuggestionEngine.swift */; }; + B6346C2A6D8EB02A5BD0E49B /* CompositionInputModeClassifier.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4F283E5B403F9FEBFB4BA04A /* CompositionInputModeClassifier.swift */; }; B65B49F24F59154A7611FD22 /* HomePaneView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D1123AB515110BD0CBA39490 /* HomePaneView.swift */; }; B6652D81162C64248AA4CF0B /* EmojiPickerPanelController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 764659D09C3F0E8FBD267102 /* EmojiPickerPanelController.swift */; }; B6703DAE949C7FB034634424 /* CurrentWordExtractor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 247561C626843957CFB4B632 /* CurrentWordExtractor.swift */; }; @@ -673,8 +678,10 @@ 4B8665A5495891F9E3DDA48B /* de-100k.txt */ = {isa = PBXFileReference; lastKnownFileType = text; path = "de-100k.txt"; sourceTree = ""; }; 4BC92317837813ACA5051177 /* Cotabby Dev.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = "Cotabby Dev.app"; sourceTree = BUILT_PRODUCTS_DIR; }; 4E283DF8948B10268B46811F /* MPL-2.0.txt */ = {isa = PBXFileReference; lastKnownFileType = text; path = "MPL-2.0.txt"; sourceTree = ""; }; + 4F283E5B403F9FEBFB4BA04A /* CompositionInputModeClassifier.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CompositionInputModeClassifier.swift; sourceTree = ""; }; 51020F8CD58338BD643FBF63 /* ModelDownloadManager.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ModelDownloadManager.swift; sourceTree = ""; }; 52BAFA2F989C3C4F7FB892B5 /* MarkerSelectionSynthesizerTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MarkerSelectionSynthesizerTests.swift; sourceTree = ""; }; + 534F1297DEF3547D0DE56FB2 /* KeyboardInputSourceMonitor.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = KeyboardInputSourceMonitor.swift; sourceTree = ""; }; 53CF416511099C6818110F01 /* CompletionRenderModePolicy.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CompletionRenderModePolicy.swift; sourceTree = ""; }; 53E41890930AA80910E461EF /* GhostFontMetricsTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = GhostFontMetricsTests.swift; sourceTree = ""; }; 54150A507B03221F137D539B /* MirrorOverlayLayout.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MirrorOverlayLayout.swift; sourceTree = ""; }; @@ -885,6 +892,7 @@ E7F42112F14026E6253BB865 /* PermissionAndContextModelTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PermissionAndContextModelTests.swift; sourceTree = ""; }; EAAE6B395FAB604DF059280A /* KeyCodeLabels.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = KeyCodeLabels.swift; sourceTree = ""; }; EB630F9814388203DD1CA2EC /* ShortcutsPaneView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ShortcutsPaneView.swift; sourceTree = ""; }; + EC4A3C4BC38793EB11F484F1 /* CompositionInputModeClassifierTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CompositionInputModeClassifierTests.swift; sourceTree = ""; }; EC582636750B78D497119845 /* PerDomainDisableSettingsTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PerDomainDisableSettingsTests.swift; sourceTree = ""; }; ED8672B87CEC72BE3978C6BB /* CotabbyTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = CotabbyTests.xctest; sourceTree = BUILT_PRODUCTS_DIR; }; EE8BB19D8EC9A75CD3458A6B /* EmojiVariantResolverTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = EmojiVariantResolverTests.swift; sourceTree = ""; }; @@ -961,6 +969,7 @@ children = ( B81DD30EB657368AACE9625A /* InputMonitor.swift */, 2D1F9CEBAB0F330F8E7B61D8 /* InputSuppressionController.swift */, + 534F1297DEF3547D0DE56FB2 /* KeyboardInputSourceMonitor.swift */, ); path = Input; sourceTree = ""; @@ -1206,6 +1215,7 @@ EFD89799BB82AF7A92559AEB /* ClipboardContentDistillerTests.swift */, 90B0D133AB77A2503FB08827 /* ClipboardRelevanceFilterTests.swift */, D504BEB224E0C176F5FCFF6E /* CompletionRenderModePolicyTests.swift */, + EC4A3C4BC38793EB11F484F1 /* CompositionInputModeClassifierTests.swift */, 06FF2B0A3094A952A8EBA9B5 /* ConfidenceSuppressionPolicyTests.swift */, 22707E26E2106DF0E826D32D /* ControlTokenMarkersTests.swift */, AF1E065C7FFB697FCEB2FA5C /* CotabbyTestFixtures.swift */, @@ -1413,6 +1423,7 @@ 96495E4147D828C0B1B22765 /* ClipboardContentDistiller.swift */, D3A2AC525DC664DB540D4F19 /* ClipboardRelevanceFilter.swift */, 53CF416511099C6818110F01 /* CompletionRenderModePolicy.swift */, + 4F283E5B403F9FEBFB4BA04A /* CompositionInputModeClassifier.swift */, 1BD71ECC2AE4821B643E0935 /* ConfidenceSuppressionPolicy.swift */, 9CC2D6472ACD377FD73A5801 /* ControlTokenMarkers.swift */, C7B2D34A6F3AC9DFD61350F7 /* CotabbyDebugOptions.swift */, @@ -1700,6 +1711,7 @@ 2314C82FAAA81EB58BFE204D /* ClipboardRelevanceFilter.swift in Sources */, 47EBA122ABE99932326D9E4A /* CompletionRenderMode.swift in Sources */, 53AA9A9D3555A67F8F31DC65 /* CompletionRenderModePolicy.swift in Sources */, + 7179FB0EC6411166CCD79F6B /* CompositionInputModeClassifier.swift in Sources */, 6BE0C8F9D054A2C0D9018001 /* ConfidenceSuppressionPolicy.swift in Sources */, A29594647EA5450A54B36228 /* ContextBuffer.swift in Sources */, 2F1E57A89D1DC775E8880297 /* ContextLivePreviewField.swift in Sources */, @@ -1774,6 +1786,7 @@ ADBCB725707ED11B19C7F08D /* InsertionStrategySelector.swift in Sources */, 1C267B67EA61527B74C9D051 /* KeyCodeLabels.swift in Sources */, BA74281E2DDE659C5CACBF24 /* KeyRecorderView.swift in Sources */, + 4086A1B07488C4D3D43D86C9 /* KeyboardInputSourceMonitor.swift in Sources */, E3CAAEFAAB5BB24CEE16445B /* LLMIOFileHandler.swift in Sources */, CADCC1B825DBE7BFC3135F43 /* LanguageCatalog.swift in Sources */, 47654BDCFD2DE6D4DE85D7FE /* LanguageTagsEditor.swift in Sources */, @@ -1917,6 +1930,7 @@ 157A55EB796BEB7819B90D5D /* ClipboardRelevanceFilter.swift in Sources */, 7C94725B4837DEC9ECF1BC54 /* CompletionRenderMode.swift in Sources */, 3985F0F2B3178DBB945B1064 /* CompletionRenderModePolicy.swift in Sources */, + B6346C2A6D8EB02A5BD0E49B /* CompositionInputModeClassifier.swift in Sources */, 429CE592897D8A952F2916C3 /* ConfidenceSuppressionPolicy.swift in Sources */, 8B2DFC860803C0A7C4D34A36 /* ContextBuffer.swift in Sources */, FB0E2CE46002270A254E5FB3 /* ContextLivePreviewField.swift in Sources */, @@ -1991,6 +2005,7 @@ 9D0F4829D11BCD4DB1290410 /* InsertionStrategySelector.swift in Sources */, F78F594F77C26C233377E71F /* KeyCodeLabels.swift in Sources */, F8E86FA4D6CEEBFA7FB55F8D /* KeyRecorderView.swift in Sources */, + 3124AD2340D4B58AF48A22F3 /* KeyboardInputSourceMonitor.swift in Sources */, 046C133967B32BBF9205EBB1 /* LLMIOFileHandler.swift in Sources */, 0A2DDD946654076675AC0FC6 /* LanguageCatalog.swift in Sources */, 51C069603DA16830868F1628 /* LanguageTagsEditor.swift in Sources */, @@ -2120,6 +2135,7 @@ 8865B95FE84198D70390DF80 /* ClipboardContentDistillerTests.swift in Sources */, BFCA7FAFDAEBF586AB615567 /* ClipboardRelevanceFilterTests.swift in Sources */, 25F91CEF38400FD1ADB6B1AF /* CompletionRenderModePolicyTests.swift in Sources */, + 2A53558D66C96E963B23CA11 /* CompositionInputModeClassifierTests.swift in Sources */, 91D8189EFCD1BA992EA6F038 /* ConfidenceSuppressionPolicyTests.swift in Sources */, 5009AF59DE8D40A45C0A5C2F /* ControlTokenMarkersTests.swift in Sources */, 5E10EFC426217CB7218A5847 /* CotabbyTestFixtures.swift in Sources */, diff --git a/Cotabby/App/Core/CotabbyAppEnvironment.swift b/Cotabby/App/Core/CotabbyAppEnvironment.swift index f045130c..4eecdc9c 100644 --- a/Cotabby/App/Core/CotabbyAppEnvironment.swift +++ b/Cotabby/App/Core/CotabbyAppEnvironment.swift @@ -21,6 +21,10 @@ final class CotabbyAppEnvironment { let suggestionSettings: SuggestionSettingsModel let foundationModelAvailabilityService: FoundationModelAvailabilityService let powerSourceMonitor: PowerSourceMonitor + /// Detects when a composing input method (Japanese kana, Chinese pinyin, Korean hangul, ...) is + /// active so `SuggestionInserter` commits accepted text through an IME-safe path instead of a + /// synthetic keystroke the input method would swallow. See `KeyboardInputSourceMonitor`. + let keyboardInputSourceMonitor: KeyboardInputSourceMonitor let clipboardContextProvider: ClipboardContextProvider let suggestionCoordinator: SuggestionCoordinator let emojiPickerController: EmojiPickerController @@ -49,6 +53,7 @@ final class CotabbyAppEnvironment { let suggestionSettings = SuggestionSettingsModel(configuration: configuration) let foundationModelAvailabilityService = FoundationModelAvailabilityService() let powerSourceMonitor = PowerSourceMonitor() + let keyboardInputSourceMonitor = KeyboardInputSourceMonitor() let suppressionController = InputSuppressionController() let inputMonitor = InputMonitor( permissionProvider: { permissionManager.inputMonitoringGranted }, @@ -112,6 +117,12 @@ final class CotabbyAppEnvironment { // to start sampling, so constructing it eagerly here costs nothing. let systemMetricsStore = SystemMetricsStore() let suggestionInserter = SuggestionInserter(suppressionController: suppressionController) + // Commit accepted text through an IME-safe path (Accessibility / paste) while a composing IME + // is active; a synthetic keystroke would be re-absorbed into composition and the accept would + // silently fail. + suggestionInserter.isComposingIMEActiveProvider = { [weak keyboardInputSourceMonitor] in + keyboardInputSourceMonitor?.isComposingIMEActive ?? false + } let overlayController = OverlayController(suggestionSettings: suggestionSettings) let activationIndicatorController = ActivationIndicatorController() let clipboardContextProvider = ClipboardContextProvider() @@ -255,6 +266,7 @@ final class CotabbyAppEnvironment { self.suggestionSettings = suggestionSettings self.foundationModelAvailabilityService = foundationModelAvailabilityService self.powerSourceMonitor = powerSourceMonitor + self.keyboardInputSourceMonitor = keyboardInputSourceMonitor self.clipboardContextProvider = clipboardContextProvider self.suggestionCoordinator = suggestionCoordinator self.emojiPickerController = emojiPickerController diff --git a/Cotabby/Services/Input/InputMonitor.swift b/Cotabby/Services/Input/InputMonitor.swift index d4e31bf9..4baee1f7 100644 --- a/Cotabby/Services/Input/InputMonitor.swift +++ b/Cotabby/Services/Input/InputMonitor.swift @@ -467,7 +467,11 @@ final class InputMonitor { return Unmanaged.passUnretained(event) case .keyDown: - if suppressionController.consumeIfNeeded() { + // Countdown first (it must consume its token), then identity as the backstop: a real + // keystroke racing in between `registerSyntheticInsertion` and our synthetic event's + // delivery eats the token, and the unsuppressed synthetic Cmd-V of the paste path then + // classifies as `.shortcutMutation` and tears down the very session it was committing. + if suppressionController.consumeIfNeeded() || suppressionController.isSynthetic(event) { onSuppressedSyntheticInput?() return Unmanaged.passUnretained(event) } diff --git a/Cotabby/Services/Input/KeyboardInputSourceMonitor.swift b/Cotabby/Services/Input/KeyboardInputSourceMonitor.swift new file mode 100644 index 00000000..3eb36237 --- /dev/null +++ b/Cotabby/Services/Input/KeyboardInputSourceMonitor.swift @@ -0,0 +1,86 @@ +import Carbon.HIToolbox +import Foundation +import Logging + +/// File overview: +/// Tracks whether the active keyboard input source is a *composing* IME (Japanese kana, Chinese, +/// Korean, Vietnamese, ...). `SuggestionInserter` reads this to pick an IME-safe way to commit an +/// accepted suggestion: a synthetic Unicode keystroke gets re-absorbed into composition by an active +/// IME (the keycode-0 event re-enters the input method instead of landing as literal text), so when +/// this is true the inserter writes through Accessibility / paste instead. The classification rule is +/// the pure `CompositionInputModeClassifier`; this type owns the live read and the change subscription. +/// +/// Out-of-process, Cotabby cannot inspect another app's `NSTextInputContext` or marked range, so the +/// only robust signal is which input source is selected. We read it via Text Input Sources (TIS) and +/// refresh on the distributed `kTISNotifySelectedKeyboardInputSourceChanged` notification, which fires +/// on every input-source AND mode switch (each IME mode is its own selectable source). Caching there +/// keeps `isComposingIMEActive` a synchronous, allocation-free read on the accept path. +@MainActor +final class KeyboardInputSourceMonitor { + /// True while a composing input method is selected. Cached; refreshed on the input-source change + /// notification, so reads at accept time are cheap and (since switching sources fires the + /// notification first) current. + private(set) var isComposingIMEActive = false + + private var observer: NSObjectProtocol? + + init() { + refresh() + // Mirror `PowerSourceMonitor`'s observer pattern: delivered on the main queue, so the + // MainActor-isolated callback runs without an extra hop under the project's main-actor + // default isolation. + observer = DistributedNotificationCenter.default().addObserver( + forName: Self.selectedInputSourceChangedNotification, + object: nil, + queue: .main + ) { [weak self] _ in + self?.handleInputSourceChanged() + } + } + + deinit { + if let observer { + DistributedNotificationCenter.default().removeObserver(observer) + } + } + + /// `kTISNotifySelectedKeyboardInputSourceChanged` bridged to a `Notification.Name`. + private static let selectedInputSourceChangedNotification = Notification.Name( + kTISNotifySelectedKeyboardInputSourceChanged as String + ) + + private func handleInputSourceChanged() { + let wasComposing = isComposingIMEActive + refresh() + if wasComposing != isComposingIMEActive { + CotabbyLogger.app.info("Composing IME active changed to \(self.isComposingIMEActive)") + } + } + + /// Reads the current keyboard input source via TIS and recomputes `isComposingIMEActive`. + func refresh() { + guard let unmanagedSource = TISCopyCurrentKeyboardInputSource() else { + isComposingIMEActive = false + return + } + let source = unmanagedSource.takeRetainedValue() + + let isKeyboardLayout = Self.stringProperty(source, kTISPropertyInputSourceType) + == (kTISTypeKeyboardLayout as String) + let inputModeID = Self.stringProperty(source, kTISPropertyInputModeID) + + isComposingIMEActive = CompositionInputModeClassifier.isComposingInputMode( + isKeyboardLayout: isKeyboardLayout, + inputModeID: inputModeID + ) + } + + /// Reads a CFString-valued TIS property as a Swift `String`. `TISGetInputSourceProperty` returns a + /// non-owning `void *` into the source, so the value is bridged with `takeUnretainedValue()`. + private static func stringProperty(_ source: TISInputSource, _ key: CFString) -> String? { + guard let pointer = TISGetInputSourceProperty(source, key) else { + return nil + } + return Unmanaged.fromOpaque(pointer).takeUnretainedValue() as String + } +} diff --git a/Cotabby/Services/Suggestion/SuggestionInserter.swift b/Cotabby/Services/Suggestion/SuggestionInserter.swift index f6a249b4..1c0af40d 100644 --- a/Cotabby/Services/Suggestion/SuggestionInserter.swift +++ b/Cotabby/Services/Suggestion/SuggestionInserter.swift @@ -16,6 +16,13 @@ final class SuggestionInserter { private(set) var lastErrorMessage: String? + /// Reads whether a composing IME (Japanese kana, Chinese pinyin, Korean hangul, ...) is currently + /// active. Wired to `KeyboardInputSourceMonitor` in `CotabbyAppEnvironment`. When true, `insert(_:)` + /// commits through an IME-safe channel (Accessibility write, then clipboard paste) instead of a + /// synthetic keystroke, which an active input method would otherwise re-absorb into composition so + /// the accept silently fails. Defaults to "no IME" so tests and previews need no wiring. + var isComposingIMEActiveProvider: @MainActor () -> Bool = { false } + /// In-flight state for the opt-in paste path. The user's real clipboard is snapshotted once and /// restored after the paste lands. While a restore is pending, a second paste must NOT re-snapshot /// (the pasteboard then holds OUR completion, which would leak back to the user), so overlapping @@ -23,6 +30,11 @@ final class SuggestionInserter { private var pendingPasteboardRestore: DispatchWorkItem? private var savedClipboardForRestore: [[NSPasteboard.PasteboardType: Data]]? + /// Paste menu items located by `AXHelper.pasteMenuItem(forApplicationPID:)`, cached per app so + /// repeat accepts skip the menu-bar walk. A cached item is validated by its `AXPress` result; + /// a failure (menu rebuilt, app quit) evicts and re-walks once. + private var cachedPasteMenuItems: [pid_t: AXUIElement] = [:] + /// Virtual key code for Delete/Backspace. Posting these at the HID level deletes one UTF-16 unit /// of already-typed text per pair, which is how the picker removes the literal `:query` run. private static let backspaceKeyCode: CGKeyCode = 0x33 @@ -39,9 +51,10 @@ final class SuggestionInserter { } /// How long to leave the completion on the pasteboard before restoring the user's clipboard. Long - /// enough for the host to service Cmd-V, short enough that the user's clipboard is theirs again - /// almost immediately. Catalogued in `docs/POLLING_AND_DELAYS.md`; tune on device. - private static let pasteboardRestoreDelay: TimeInterval = 0.15 + /// enough for the host to service Cmd-V (a busy Chromium page can take a couple hundred ms), + /// short enough that the user's clipboard is theirs again almost immediately. Catalogued in + /// `docs/POLLING_AND_DELAYS.md`; tune on device. + private static let pasteboardRestoreDelay: TimeInterval = 0.3 init(suppressionController: InputSuppressionController) { self.suppressionController = suppressionController @@ -56,6 +69,27 @@ final class SuggestionInserter { return false } + // A composing IME (Japanese kana, Chinese pinyin, Korean hangul, ...) is active. A synthetic + // Unicode keystroke would be re-absorbed into composition by the input method (the placeholder + // keycode-0 event re-enters the IME) instead of landing as literal text, so the accepted + // suggestion never commits and the session desyncs against the live field, which surfaced as + // "Tab regenerates instead of accepting" for Japanese users. Commit via a clipboard paste: + // Cmd-V is a command shortcut the app services directly, so the input method never touches it. + // (An Accessibility `AXSelectedText` write was tried first and rejected: Chromium contenteditable + // accepts the set, reports `.success`, then silently no-ops, so the text never lands and the + // session desyncs exactly as before. Paste actually inserts there.) The clipboard is snapshotted + // and restored around the paste; only the last-resort keystroke below can still be swallowed. + if isComposingIMEActiveProvider() { + if insertViaPaste(normalized) { + lastErrorMessage = nil + CotabbyLogger.suggestion.debug("Inserted \(normalized.count) characters via paste (IME active)") + return true + } + let fallbackMessage = "IME-safe paste failed for \(normalized.count) characters; " + + "falling back to a synthetic keystroke the input method may swallow" + CotabbyLogger.suggestion.warning("\(fallbackMessage)") + } + // Paste path (opt-in): a long or multi-line completion is steadier as a clipboard paste in // apps that mishandle a big synthetic Unicode string. On any failure we fall through to the // reliable keystroke path below, so paste is never worse than the default keystroke insert. @@ -176,20 +210,40 @@ final class SuggestionInserter { } let expectedChangeCount = pasteboard.changeCount - guard let keyDown = CGEvent(keyboardEventSource: nil, virtualKey: Self.vKeyCode, keyDown: true), - let keyUp = CGEvent(keyboardEventSource: nil, virtualKey: Self.vKeyCode, keyDown: false) else { - pendingPasteboardRestore?.cancel() - Self.restorePasteboard(saved, to: pasteboard) - clearPendingPasteboardRestore() - return false + // Preferred trigger: press the host's real Edit > Paste menu item via Accessibility. No key + // event exists, so neither an active IME nor the HID modifier state machine can interfere. + // Synthetic Cmd-V is the fallback, and it has a real failure mode this path avoids: observed + // on-device, Cmd-V posted source-nil at the HID tap had its Command flag stripped (only Tab is + // physically down during an accept), and re-posting at the annotated session tap with a + // session source still never pasted in Chrome. The menu press drives the same paste command + // those key events would have reached, one IPC hop earlier. + if pressPasteMenuItem() { + CotabbyLogger.suggestion.debug("Paste committed via Edit > Paste menu press") + } else { + // Fallback synthetic Cmd-V: session source + annotated session tap so the event's own + // `.maskCommand` flag survives (the HID tap merges flags with live hardware state). The + // suppression filter keeps the physically held accept key from interleaving during the post. + let source = CGEventSource(stateID: .combinedSessionState) + source?.setLocalEventsFilterDuringSuppressionState( + [.permitLocalMouseEvents, .permitSystemDefinedEvents], + state: .eventSuppressionStateSuppressionInterval + ) + guard let keyDown = CGEvent(keyboardEventSource: source, virtualKey: Self.vKeyCode, keyDown: true), + let keyUp = CGEvent(keyboardEventSource: source, virtualKey: Self.vKeyCode, keyDown: false) else { + pendingPasteboardRestore?.cancel() + Self.restorePasteboard(saved, to: pasteboard) + clearPendingPasteboardRestore() + return false + } + keyDown.flags = .maskCommand + keyUp.flags = .maskCommand + suppressionController.markSynthetic(keyDown) + suppressionController.markSynthetic(keyUp) + suppressionController.registerSyntheticInsertion(expectedKeyDownCount: 1) + keyDown.post(tap: .cgAnnotatedSessionEventTap) + keyUp.post(tap: .cgAnnotatedSessionEventTap) + CotabbyLogger.suggestion.debug("Paste committed via synthetic Cmd-V (no Paste menu item found)") } - keyDown.flags = .maskCommand - keyUp.flags = .maskCommand - suppressionController.markSynthetic(keyDown) - suppressionController.markSynthetic(keyUp) - suppressionController.registerSyntheticInsertion(expectedKeyDownCount: 1) - keyDown.post(tap: .cghidEventTap) - keyUp.post(tap: .cghidEventTap) // Give the host time to service Cmd-V, then hand the clipboard back, but only if our completion // is still the thing on it. If the user copied something during the window, `changeCount` @@ -207,6 +261,30 @@ final class SuggestionInserter { return true } + /// Presses the focused app's Edit > Paste menu item via Accessibility. The owning app is resolved + /// from the focused element (not the frontmost app) so accessory panels that hold focus without + /// frontmost status still target the right process. The located item is cached per pid; a cached + /// press that fails (menu rebuilt, app relaunched into the same pid) evicts and re-walks once. + private func pressPasteMenuItem() -> Bool { + guard let focusedElement = AXHelper.focusedElement(), + let application = AXHelper.owningApplication(of: focusedElement) else { + return false + } + let pid = application.processIdentifier + if let cached = cachedPasteMenuItems[pid] { + if AXUIElementPerformAction(cached, kAXPressAction as CFString) == .success { + return true + } + cachedPasteMenuItems[pid] = nil + } + guard let item = AXHelper.pasteMenuItem(forApplicationPID: pid), + AXUIElementPerformAction(item, kAXPressAction as CFString) == .success else { + return false + } + cachedPasteMenuItems[pid] = item + return true + } + /// Clears the in-flight paste-restore bookkeeping once a restore has run (or been abandoned on a /// failure path), so the next paste snapshots the user's real clipboard afresh. private func clearPendingPasteboardRestore() { diff --git a/Cotabby/Support/AXHelper.swift b/Cotabby/Support/AXHelper.swift index a0af6d15..91c70090 100644 --- a/Cotabby/Support/AXHelper.swift +++ b/Cotabby/Support/AXHelper.swift @@ -653,6 +653,46 @@ enum AXHelper { } } + /// Locates the app's "Paste" menu item by its Cmd-V key equivalent rather than by title, so the + /// lookup is language-independent ("Paste" / "貼り付け" / "Coller" depending on the host's + /// localization). The IME-safe insertion path presses this real menu item via `AXPress`: the + /// host runs its paste command without any key event existing, so an active input method can + /// neither swallow nor re-interpret the commit the way it does a synthetic keystroke. + static func pasteMenuItem(forApplicationPID pid: pid_t) -> AXUIElement? { + guard pid > 0 else { return nil } + let appElement = AXUIElementCreateApplication(pid) + AXUIElementSetMessagingTimeout(appElement, pollMessagingTimeout) + guard let menuBarValue = copyAttributeValue(kAXMenuBarAttribute as CFString, on: appElement), + CFGetTypeID(menuBarValue) == AXUIElementGetTypeID() else { + return nil + } + // Same Core Foundation bridging rule as `focusedElement()`. + let menuBar = unsafeBitCast(menuBarValue, to: AXUIElement.self) + + // Menu bar -> top-level items (Apple, app, File, Edit, ...) -> one AXMenu each -> menu items. + // Depth-1 only: Paste always lives directly in a top-level menu, and skipping submenu + // recursion keeps the walk bounded. Early exit on the first Cmd-V item. + for topLevelItem in childElements(of: menuBar) { + for menu in childElements(of: topLevelItem) { + for item in childElements(of: menu) where isCommandVMenuItem(item) { + return item + } + } + } + return nil + } + + /// True for a menu item whose key equivalent is exactly Cmd-V. `kAXMenuItemCmdModifiers` uses the + /// Carbon menu encoding where 0 means "Command alone"; shift/option/control add bits and 8 means + /// the equivalent carries no Command at all, so only an exact 0 matches plain Cmd-V. + private static func isCommandVMenuItem(_ item: AXUIElement) -> Bool { + guard let cmdChar = stringValue(for: kAXMenuItemCmdCharAttribute as CFString, on: item), + cmdChar.uppercased() == "V" else { + return false + } + return intValue(for: kAXMenuItemCmdModifiersAttribute as CFString, on: item) == 0 + } + static func elementIdentity(for element: AXUIElement) -> String { var pid: pid_t = 0 AXUIElementGetPid(element, &pid) diff --git a/Cotabby/Support/CompositionInputModeClassifier.swift b/Cotabby/Support/CompositionInputModeClassifier.swift new file mode 100644 index 00000000..8dc8c41d --- /dev/null +++ b/Cotabby/Support/CompositionInputModeClassifier.swift @@ -0,0 +1,48 @@ +import Foundation + +/// File overview: +/// Pure rule deciding whether the active keyboard input source is a *composing* input method, one +/// that assembles characters from several keystrokes through provisional "marked" text (Japanese +/// kana, Chinese pinyin/zhuyin/cangjie, Korean hangul, Vietnamese Telex, ...). It is intentionally +/// free of Carbon/TIS so it stays trivially testable; `KeyboardInputSourceMonitor` reads the live +/// input source and feeds the already-extracted fields in here. +/// +/// Why this matters: Cotabby commits an accepted suggestion by synthesizing a Unicode keystroke +/// (a key event on virtualKey 0). With a composing IME active, that synthetic keystroke does not +/// land as literal text, the input method re-absorbs it into composition, so the accepted text never +/// arrives, the session desyncs against the live field, and the suggestion regenerates instead of +/// committing. When this rule reports a composing mode, the inserter switches to an IME-safe path +/// (Accessibility write / clipboard paste) that bypasses the input method entirely. +enum CompositionInputModeClassifier { + /// Input mode IDs an IME exposes for typing plain ASCII *directly* (committed per keystroke, no + /// marked text), where the normal keystroke insertion is fine. The system Japanese IMEs share + /// `com.apple.inputmethod.Roman` for their alphanumeric ("英数") mode. + static let nonComposingInputModeIDs: Set = [ + "com.apple.inputmethod.Roman" + ] + + /// Whether the current input source composes through marked text (so committing accepted text + /// needs the IME-safe insertion path). + /// + /// - Parameters: + /// - isKeyboardLayout: the TIS source type is a plain keyboard layout (U.S., Dvorak, British, + /// ...). Layouts commit every keystroke directly and never compose. + /// - inputModeID: `kTISPropertyInputModeID` of the active source when it is an input mode; nil + /// for plain layouts and some method-without-modes IMEs. + /// + /// A plain layout never composes. Any non-layout input *method/mode* is treated as composing, + /// except the shared direct-ASCII mode. Erring toward "composing for any unrecognized non-layout + /// source" is the safe default: a third-party or future IME (ATOK, Sogou, ...) is far likelier to + /// compose through marked text than to commit plain ASCII, and the cost of a false positive is + /// only that we route through the (also-correct) IME-safe insertion path, whereas a false + /// negative reproduces the can't-accept bug. + static func isComposingInputMode(isKeyboardLayout: Bool, inputModeID: String?) -> Bool { + if isKeyboardLayout { + return false + } + if let inputModeID, nonComposingInputModeIDs.contains(inputModeID) { + return false + } + return true + } +} diff --git a/CotabbyTests/CompositionInputModeClassifierTests.swift b/CotabbyTests/CompositionInputModeClassifierTests.swift new file mode 100644 index 00000000..ce85453d --- /dev/null +++ b/CotabbyTests/CompositionInputModeClassifierTests.swift @@ -0,0 +1,91 @@ +import XCTest +@testable import Cotabby + +/// Tests for the pure rule that decides when an accepted suggestion must be committed through the +/// IME-safe insertion path. This classifier is the one piece of IME detection that does not touch +/// Carbon/TIS, so it carries the behavioral contract `KeyboardInputSourceMonitor` relies on. The +/// driving bug: with a composing IME active, the synthetic-keystroke insert is re-absorbed into +/// composition and the accept silently fails, so we detect composing input sources and switch +/// insertion methods. +final class CompositionInputModeClassifierTests: XCTestCase { + /// U.S. / Dvorak / British etc. are keyboard layouts: every keystroke commits directly, so they + /// never need the IME-safe path. + func test_plainKeyboardLayout_isNotComposing() { + XCTAssertFalse( + CompositionInputModeClassifier.isComposingInputMode( + isKeyboardLayout: true, + inputModeID: nil + ) + ) + } + + func test_japaneseHiragana_isComposing() { + XCTAssertTrue( + CompositionInputModeClassifier.isComposingInputMode( + isKeyboardLayout: false, + inputModeID: "com.apple.inputmethod.Japanese.Hiragana" + ) + ) + } + + func test_japaneseKatakana_isComposing() { + XCTAssertTrue( + CompositionInputModeClassifier.isComposingInputMode( + isKeyboardLayout: false, + inputModeID: "com.apple.inputmethod.Japanese.Katakana" + ) + ) + } + + func test_chinesePinyin_isComposing() { + XCTAssertTrue( + CompositionInputModeClassifier.isComposingInputMode( + isKeyboardLayout: false, + inputModeID: "com.apple.inputmethod.SCIM.ITABC" + ) + ) + } + + func test_korean_isComposing() { + XCTAssertTrue( + CompositionInputModeClassifier.isComposingInputMode( + isKeyboardLayout: false, + inputModeID: "com.apple.inputmethod.Korean.2SetKorean" + ) + ) + } + + /// The shared direct-ASCII ("英数") mode of the Japanese IMEs commits per keystroke with no marked + /// text, so the normal keystroke insert works there. + func test_romanDirectMode_isNotComposing() { + XCTAssertFalse( + CompositionInputModeClassifier.isComposingInputMode( + isKeyboardLayout: false, + inputModeID: "com.apple.inputmethod.Roman" + ) + ) + } + + /// A third-party IME (ATOK, Sogou, ...) that is not a plain layout and exposes no recognized + /// direct mode is assumed to compose, the safe default that fixes the reported bug (the reporter + /// uses ATOK). + func test_unknownInputMethod_isComposing() { + XCTAssertTrue( + CompositionInputModeClassifier.isComposingInputMode( + isKeyboardLayout: false, + inputModeID: "com.justsystems.inputmethod.atok33.Japanese" + ) + ) + } + + /// An input method reporting no mode ID (a method-without-modes) that is not a layout still + /// composes. + func test_inputMethodWithoutModeID_isComposing() { + XCTAssertTrue( + CompositionInputModeClassifier.isComposingInputMode( + isKeyboardLayout: false, + inputModeID: nil + ) + ) + } +} From 2ec2fd96a527dead39b14d71ab70c3e7d1647916 Mon Sep 17 00:00:00 2001 From: Jacob Fu <141651335+FuJacob@users.noreply.github.com> Date: Thu, 11 Jun 2026 07:14:55 -0700 Subject: [PATCH 2/2] Address Greptile review on #668 Make KeyboardInputSourceMonitor.refresh() private (the notification path is the only caller), apply the short poll messaging timeout to every element the paste-menu walk messages so a stalled host cannot hold the main actor for the multi-second AX default, and make the missing-modifiers rejection in isCommandVMenuItem an explicit guard instead of relying on nil == 0. --- .../Input/KeyboardInputSourceMonitor.swift | 2 +- Cotabby/Support/AXHelper.swift | 17 +++++++++++++++-- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/Cotabby/Services/Input/KeyboardInputSourceMonitor.swift b/Cotabby/Services/Input/KeyboardInputSourceMonitor.swift index 3eb36237..c4448259 100644 --- a/Cotabby/Services/Input/KeyboardInputSourceMonitor.swift +++ b/Cotabby/Services/Input/KeyboardInputSourceMonitor.swift @@ -58,7 +58,7 @@ final class KeyboardInputSourceMonitor { } /// Reads the current keyboard input source via TIS and recomputes `isComposingIMEActive`. - func refresh() { + private func refresh() { guard let unmanagedSource = TISCopyCurrentKeyboardInputSource() else { isComposingIMEActive = false return diff --git a/Cotabby/Support/AXHelper.swift b/Cotabby/Support/AXHelper.swift index 91c70090..10bf0705 100644 --- a/Cotabby/Support/AXHelper.swift +++ b/Cotabby/Support/AXHelper.swift @@ -672,8 +672,16 @@ enum AXHelper { // Menu bar -> top-level items (Apple, app, File, Edit, ...) -> one AXMenu each -> menu items. // Depth-1 only: Paste always lives directly in a top-level menu, and skipping submenu // recursion keeps the walk bounded. Early exit on the first Cmd-V item. + // + // Every walked element gets the short poll timeout before it is messaged: the walk runs on + // the main actor inside the accept path, and an element handle that never had a timeout set + // would otherwise fall back to the multi-second AX default if the host (a busy Chromium + // process is the common case here) stalls, beachballing typing for the duration. + AXUIElementSetMessagingTimeout(menuBar, pollMessagingTimeout) for topLevelItem in childElements(of: menuBar) { + AXUIElementSetMessagingTimeout(topLevelItem, pollMessagingTimeout) for menu in childElements(of: topLevelItem) { + AXUIElementSetMessagingTimeout(menu, pollMessagingTimeout) for item in childElements(of: menu) where isCommandVMenuItem(item) { return item } @@ -684,13 +692,18 @@ enum AXHelper { /// True for a menu item whose key equivalent is exactly Cmd-V. `kAXMenuItemCmdModifiers` uses the /// Carbon menu encoding where 0 means "Command alone"; shift/option/control add bits and 8 means - /// the equivalent carries no Command at all, so only an exact 0 matches plain Cmd-V. + /// the equivalent carries no Command at all, so only an exact 0 matches plain Cmd-V. A missing + /// modifiers attribute rejects the item explicitly rather than relying on `nil == 0` being false. private static func isCommandVMenuItem(_ item: AXUIElement) -> Bool { + AXUIElementSetMessagingTimeout(item, pollMessagingTimeout) guard let cmdChar = stringValue(for: kAXMenuItemCmdCharAttribute as CFString, on: item), cmdChar.uppercased() == "V" else { return false } - return intValue(for: kAXMenuItemCmdModifiersAttribute as CFString, on: item) == 0 + guard let modifiers = intValue(for: kAXMenuItemCmdModifiersAttribute as CFString, on: item) else { + return false + } + return modifiers == 0 } static func elementIdentity(for element: AXUIElement) -> String {