diff --git a/Cotabby.xcodeproj/project.pbxproj b/Cotabby.xcodeproj/project.pbxproj index 66ce36c6..04888f5a 100644 --- a/Cotabby.xcodeproj/project.pbxproj +++ b/Cotabby.xcodeproj/project.pbxproj @@ -115,6 +115,7 @@ 2872D907299F79A9A69BBFCB /* EmojiPickerPanelController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 764659D09C3F0E8FBD267102 /* EmojiPickerPanelController.swift */; }; 28D217A96946A2005FCBEBFD /* emoji.json in Resources */ = {isa = PBXBuildFile; fileRef = C379D77029D6E88C8C1B9AF7 /* emoji.json */; }; 29ABB5488251FD8089D74F51 /* MidWordContinuationPolicy.swift in Sources */ = {isa = PBXBuildFile; fileRef = 357C18383B047F24A531BDCD /* MidWordContinuationPolicy.swift */; }; + 2A53558D66C96E963B23CA11 /* CompositionInputModeClassifierTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = EC4A3C4BC38793EB11F484F1 /* CompositionInputModeClassifierTests.swift */; }; 2BE029A192E82E795490DC7F /* BrowserDomain.swift in Sources */ = {isa = PBXBuildFile; fileRef = AD8025E4A296845FC53E660D /* BrowserDomain.swift */; }; 2C6159231472A849F15BD0AE /* ScreenFrameReader.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5484C8A04B9C00CF79D589EB /* ScreenFrameReader.swift */; }; 2CCF87FD35BF2C438EEA606D /* SelfCaptureGate.swift in Sources */ = {isa = PBXBuildFile; fileRef = E68BE6A22BA0D42C8DD9868C /* SelfCaptureGate.swift */; }; @@ -132,6 +133,7 @@ 303652F15C0FE55595669D81 /* SpellingDictionaryResourceTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D562A73C7C680F2AA65F9F7F /* SpellingDictionaryResourceTests.swift */; }; 30F3F2B6D13CD583136CD787 /* AXHelper.swift in Sources */ = {isa = PBXBuildFile; fileRef = AC70775535A3428991025AB8 /* AXHelper.swift */; }; 3112A355E61878A6A6D1FDF8 /* EmojiQueryRun.swift in Sources */ = {isa = PBXBuildFile; fileRef = DDF6A4E9CE93FD53C60E67E3 /* EmojiQueryRun.swift */; }; + 3124AD2340D4B58AF48A22F3 /* KeyboardInputSourceMonitor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 534F1297DEF3547D0DE56FB2 /* KeyboardInputSourceMonitor.swift */; }; 314AD8F0FD781CB6DDE4603C /* GhostFontMetrics.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9E6439213F2A273702A6E26B /* GhostFontMetrics.swift */; }; 31515DDD173535C4AC777853 /* MirrorOverlayLayout.swift in Sources */ = {isa = PBXBuildFile; fileRef = 54150A507B03221F137D539B /* MirrorOverlayLayout.swift */; }; 317883210D1D1D5CD654E562 /* ModelFileValidator.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5C4E5869D103865486AAAEEC /* ModelFileValidator.swift */; }; @@ -168,6 +170,7 @@ 3FCEF50FDD9EE01AE3711083 /* AXTreeDumpWriter.swift in Sources */ = {isa = PBXBuildFile; fileRef = B27492B04B627DA53BDAD938 /* AXTreeDumpWriter.swift */; }; 3FF6B7DE34A01C4AB7FA54E3 /* MacroTriggerStateMachine.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1C201A65A6B040F90C528A3B /* MacroTriggerStateMachine.swift */; }; 400E1A5145FC8C5BA2FAED0A /* DeepGeometryWalkThrottle.swift in Sources */ = {isa = PBXBuildFile; fileRef = 684737E62EE6495A71344923 /* DeepGeometryWalkThrottle.swift */; }; + 4086A1B07488C4D3D43D86C9 /* KeyboardInputSourceMonitor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 534F1297DEF3547D0DE56FB2 /* KeyboardInputSourceMonitor.swift */; }; 4134ADBE464D00BB748BD9AE /* GeneralPaneView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 07480CE96ED0EBD94817C6B1 /* GeneralPaneView.swift */; }; 418304869F503EDC6465F8D5 /* SentenceBoundaryClassifier.swift in Sources */ = {isa = PBXBuildFile; fileRef = D4B56C250DDEF3E81F9DCBD7 /* SentenceBoundaryClassifier.swift */; }; 4190F8A76196B16ED94D0A55 /* VisualContextModels.swift in Sources */ = {isa = PBXBuildFile; fileRef = BE97A8169438D593C6C23412 /* VisualContextModels.swift */; }; @@ -281,6 +284,7 @@ 6F2FE689BCA50BEAE80AC6F4 /* ShortcutsPaneView.swift in Sources */ = {isa = PBXBuildFile; fileRef = EB630F9814388203DD1CA2EC /* ShortcutsPaneView.swift */; }; 709F365A846B908D953FA92D /* FoundationModelPromptRenderer.swift in Sources */ = {isa = PBXBuildFile; fileRef = FE7BF162A12703249726F20A /* FoundationModelPromptRenderer.swift */; }; 70D6F9480DA4104AD5669569 /* WelcomePermissionStepView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D5D6C2318E405AA717D1C256 /* WelcomePermissionStepView.swift */; }; + 7179FB0EC6411166CCD79F6B /* CompositionInputModeClassifier.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4F283E5B403F9FEBFB4BA04A /* CompositionInputModeClassifier.swift */; }; 735C2E64CA51F58098B30A0D /* it.txt in Resources */ = {isa = PBXBuildFile; fileRef = 0397F1DACB094A0F6A66BC0E /* it.txt */; }; 74422BB837D6A319D12BF981 /* BaseCompletionPromptRenderer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 85EF79E6144D6C6AD062B569 /* BaseCompletionPromptRenderer.swift */; }; 744B06C2488156B178675615 /* PermissionManager.swift in Sources */ = {isa = PBXBuildFile; fileRef = 85BF316556FDA64CB8AD07B6 /* PermissionManager.swift */; }; @@ -417,6 +421,7 @@ B4D36F5D03E3143CE74582F9 /* AppearancePaneView.swift in Sources */ = {isa = PBXBuildFile; fileRef = AFBE491B3CA04FE9069B7B0F /* AppearancePaneView.swift */; }; B55B160E0534AE23BAC1C3DA /* CotabbyApp.swift in Sources */ = {isa = PBXBuildFile; fileRef = CC1EDFB535AAA2EE0D67828A /* CotabbyApp.swift */; }; B588C09233E69C6EDC69BEDC /* LlamaSuggestionEngine.swift in Sources */ = {isa = PBXBuildFile; fileRef = BE04620C905041680116BE80 /* LlamaSuggestionEngine.swift */; }; + B6346C2A6D8EB02A5BD0E49B /* CompositionInputModeClassifier.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4F283E5B403F9FEBFB4BA04A /* CompositionInputModeClassifier.swift */; }; B65B49F24F59154A7611FD22 /* HomePaneView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D1123AB515110BD0CBA39490 /* HomePaneView.swift */; }; B6652D81162C64248AA4CF0B /* EmojiPickerPanelController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 764659D09C3F0E8FBD267102 /* EmojiPickerPanelController.swift */; }; B6703DAE949C7FB034634424 /* CurrentWordExtractor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 247561C626843957CFB4B632 /* CurrentWordExtractor.swift */; }; @@ -673,8 +678,10 @@ 4B8665A5495891F9E3DDA48B /* de-100k.txt */ = {isa = PBXFileReference; lastKnownFileType = text; path = "de-100k.txt"; sourceTree = ""; }; 4BC92317837813ACA5051177 /* Cotabby Dev.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = "Cotabby Dev.app"; sourceTree = BUILT_PRODUCTS_DIR; }; 4E283DF8948B10268B46811F /* MPL-2.0.txt */ = {isa = PBXFileReference; lastKnownFileType = text; path = "MPL-2.0.txt"; sourceTree = ""; }; + 4F283E5B403F9FEBFB4BA04A /* CompositionInputModeClassifier.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CompositionInputModeClassifier.swift; sourceTree = ""; }; 51020F8CD58338BD643FBF63 /* ModelDownloadManager.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ModelDownloadManager.swift; sourceTree = ""; }; 52BAFA2F989C3C4F7FB892B5 /* MarkerSelectionSynthesizerTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MarkerSelectionSynthesizerTests.swift; sourceTree = ""; }; + 534F1297DEF3547D0DE56FB2 /* KeyboardInputSourceMonitor.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = KeyboardInputSourceMonitor.swift; sourceTree = ""; }; 53CF416511099C6818110F01 /* CompletionRenderModePolicy.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CompletionRenderModePolicy.swift; sourceTree = ""; }; 53E41890930AA80910E461EF /* GhostFontMetricsTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = GhostFontMetricsTests.swift; sourceTree = ""; }; 54150A507B03221F137D539B /* MirrorOverlayLayout.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MirrorOverlayLayout.swift; sourceTree = ""; }; @@ -885,6 +892,7 @@ E7F42112F14026E6253BB865 /* PermissionAndContextModelTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PermissionAndContextModelTests.swift; sourceTree = ""; }; EAAE6B395FAB604DF059280A /* KeyCodeLabels.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = KeyCodeLabels.swift; sourceTree = ""; }; EB630F9814388203DD1CA2EC /* ShortcutsPaneView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ShortcutsPaneView.swift; sourceTree = ""; }; + EC4A3C4BC38793EB11F484F1 /* CompositionInputModeClassifierTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CompositionInputModeClassifierTests.swift; sourceTree = ""; }; EC582636750B78D497119845 /* PerDomainDisableSettingsTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PerDomainDisableSettingsTests.swift; sourceTree = ""; }; ED8672B87CEC72BE3978C6BB /* CotabbyTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = CotabbyTests.xctest; sourceTree = BUILT_PRODUCTS_DIR; }; EE8BB19D8EC9A75CD3458A6B /* EmojiVariantResolverTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = EmojiVariantResolverTests.swift; sourceTree = ""; }; @@ -961,6 +969,7 @@ children = ( B81DD30EB657368AACE9625A /* InputMonitor.swift */, 2D1F9CEBAB0F330F8E7B61D8 /* InputSuppressionController.swift */, + 534F1297DEF3547D0DE56FB2 /* KeyboardInputSourceMonitor.swift */, ); path = Input; sourceTree = ""; @@ -1206,6 +1215,7 @@ EFD89799BB82AF7A92559AEB /* ClipboardContentDistillerTests.swift */, 90B0D133AB77A2503FB08827 /* ClipboardRelevanceFilterTests.swift */, D504BEB224E0C176F5FCFF6E /* CompletionRenderModePolicyTests.swift */, + EC4A3C4BC38793EB11F484F1 /* CompositionInputModeClassifierTests.swift */, 06FF2B0A3094A952A8EBA9B5 /* ConfidenceSuppressionPolicyTests.swift */, 22707E26E2106DF0E826D32D /* ControlTokenMarkersTests.swift */, AF1E065C7FFB697FCEB2FA5C /* CotabbyTestFixtures.swift */, @@ -1413,6 +1423,7 @@ 96495E4147D828C0B1B22765 /* ClipboardContentDistiller.swift */, D3A2AC525DC664DB540D4F19 /* ClipboardRelevanceFilter.swift */, 53CF416511099C6818110F01 /* CompletionRenderModePolicy.swift */, + 4F283E5B403F9FEBFB4BA04A /* CompositionInputModeClassifier.swift */, 1BD71ECC2AE4821B643E0935 /* ConfidenceSuppressionPolicy.swift */, 9CC2D6472ACD377FD73A5801 /* ControlTokenMarkers.swift */, C7B2D34A6F3AC9DFD61350F7 /* CotabbyDebugOptions.swift */, @@ -1700,6 +1711,7 @@ 2314C82FAAA81EB58BFE204D /* ClipboardRelevanceFilter.swift in Sources */, 47EBA122ABE99932326D9E4A /* CompletionRenderMode.swift in Sources */, 53AA9A9D3555A67F8F31DC65 /* CompletionRenderModePolicy.swift in Sources */, + 7179FB0EC6411166CCD79F6B /* CompositionInputModeClassifier.swift in Sources */, 6BE0C8F9D054A2C0D9018001 /* ConfidenceSuppressionPolicy.swift in Sources */, A29594647EA5450A54B36228 /* ContextBuffer.swift in Sources */, 2F1E57A89D1DC775E8880297 /* ContextLivePreviewField.swift in Sources */, @@ -1774,6 +1786,7 @@ ADBCB725707ED11B19C7F08D /* InsertionStrategySelector.swift in Sources */, 1C267B67EA61527B74C9D051 /* KeyCodeLabels.swift in Sources */, BA74281E2DDE659C5CACBF24 /* KeyRecorderView.swift in Sources */, + 4086A1B07488C4D3D43D86C9 /* KeyboardInputSourceMonitor.swift in Sources */, E3CAAEFAAB5BB24CEE16445B /* LLMIOFileHandler.swift in Sources */, CADCC1B825DBE7BFC3135F43 /* LanguageCatalog.swift in Sources */, 47654BDCFD2DE6D4DE85D7FE /* LanguageTagsEditor.swift in Sources */, @@ -1917,6 +1930,7 @@ 157A55EB796BEB7819B90D5D /* ClipboardRelevanceFilter.swift in Sources */, 7C94725B4837DEC9ECF1BC54 /* CompletionRenderMode.swift in Sources */, 3985F0F2B3178DBB945B1064 /* CompletionRenderModePolicy.swift in Sources */, + B6346C2A6D8EB02A5BD0E49B /* CompositionInputModeClassifier.swift in Sources */, 429CE592897D8A952F2916C3 /* ConfidenceSuppressionPolicy.swift in Sources */, 8B2DFC860803C0A7C4D34A36 /* ContextBuffer.swift in Sources */, FB0E2CE46002270A254E5FB3 /* ContextLivePreviewField.swift in Sources */, @@ -1991,6 +2005,7 @@ 9D0F4829D11BCD4DB1290410 /* InsertionStrategySelector.swift in Sources */, F78F594F77C26C233377E71F /* KeyCodeLabels.swift in Sources */, F8E86FA4D6CEEBFA7FB55F8D /* KeyRecorderView.swift in Sources */, + 3124AD2340D4B58AF48A22F3 /* KeyboardInputSourceMonitor.swift in Sources */, 046C133967B32BBF9205EBB1 /* LLMIOFileHandler.swift in Sources */, 0A2DDD946654076675AC0FC6 /* LanguageCatalog.swift in Sources */, 51C069603DA16830868F1628 /* LanguageTagsEditor.swift in Sources */, @@ -2120,6 +2135,7 @@ 8865B95FE84198D70390DF80 /* ClipboardContentDistillerTests.swift in Sources */, BFCA7FAFDAEBF586AB615567 /* ClipboardRelevanceFilterTests.swift in Sources */, 25F91CEF38400FD1ADB6B1AF /* CompletionRenderModePolicyTests.swift in Sources */, + 2A53558D66C96E963B23CA11 /* CompositionInputModeClassifierTests.swift in Sources */, 91D8189EFCD1BA992EA6F038 /* ConfidenceSuppressionPolicyTests.swift in Sources */, 5009AF59DE8D40A45C0A5C2F /* ControlTokenMarkersTests.swift in Sources */, 5E10EFC426217CB7218A5847 /* CotabbyTestFixtures.swift in Sources */, diff --git a/Cotabby/App/Core/CotabbyAppEnvironment.swift b/Cotabby/App/Core/CotabbyAppEnvironment.swift index f045130c..4eecdc9c 100644 --- a/Cotabby/App/Core/CotabbyAppEnvironment.swift +++ b/Cotabby/App/Core/CotabbyAppEnvironment.swift @@ -21,6 +21,10 @@ final class CotabbyAppEnvironment { let suggestionSettings: SuggestionSettingsModel let foundationModelAvailabilityService: FoundationModelAvailabilityService let powerSourceMonitor: PowerSourceMonitor + /// Detects when a composing input method (Japanese kana, Chinese pinyin, Korean hangul, ...) is + /// active so `SuggestionInserter` commits accepted text through an IME-safe path instead of a + /// synthetic keystroke the input method would swallow. See `KeyboardInputSourceMonitor`. + let keyboardInputSourceMonitor: KeyboardInputSourceMonitor let clipboardContextProvider: ClipboardContextProvider let suggestionCoordinator: SuggestionCoordinator let emojiPickerController: EmojiPickerController @@ -49,6 +53,7 @@ final class CotabbyAppEnvironment { let suggestionSettings = SuggestionSettingsModel(configuration: configuration) let foundationModelAvailabilityService = FoundationModelAvailabilityService() let powerSourceMonitor = PowerSourceMonitor() + let keyboardInputSourceMonitor = KeyboardInputSourceMonitor() let suppressionController = InputSuppressionController() let inputMonitor = InputMonitor( permissionProvider: { permissionManager.inputMonitoringGranted }, @@ -112,6 +117,12 @@ final class CotabbyAppEnvironment { // to start sampling, so constructing it eagerly here costs nothing. let systemMetricsStore = SystemMetricsStore() let suggestionInserter = SuggestionInserter(suppressionController: suppressionController) + // Commit accepted text through an IME-safe path (Accessibility / paste) while a composing IME + // is active; a synthetic keystroke would be re-absorbed into composition and the accept would + // silently fail. + suggestionInserter.isComposingIMEActiveProvider = { [weak keyboardInputSourceMonitor] in + keyboardInputSourceMonitor?.isComposingIMEActive ?? false + } let overlayController = OverlayController(suggestionSettings: suggestionSettings) let activationIndicatorController = ActivationIndicatorController() let clipboardContextProvider = ClipboardContextProvider() @@ -255,6 +266,7 @@ final class CotabbyAppEnvironment { self.suggestionSettings = suggestionSettings self.foundationModelAvailabilityService = foundationModelAvailabilityService self.powerSourceMonitor = powerSourceMonitor + self.keyboardInputSourceMonitor = keyboardInputSourceMonitor self.clipboardContextProvider = clipboardContextProvider self.suggestionCoordinator = suggestionCoordinator self.emojiPickerController = emojiPickerController diff --git a/Cotabby/Services/Input/InputMonitor.swift b/Cotabby/Services/Input/InputMonitor.swift index d4e31bf9..4baee1f7 100644 --- a/Cotabby/Services/Input/InputMonitor.swift +++ b/Cotabby/Services/Input/InputMonitor.swift @@ -467,7 +467,11 @@ final class InputMonitor { return Unmanaged.passUnretained(event) case .keyDown: - if suppressionController.consumeIfNeeded() { + // Countdown first (it must consume its token), then identity as the backstop: a real + // keystroke racing in between `registerSyntheticInsertion` and our synthetic event's + // delivery eats the token, and the unsuppressed synthetic Cmd-V of the paste path then + // classifies as `.shortcutMutation` and tears down the very session it was committing. + if suppressionController.consumeIfNeeded() || suppressionController.isSynthetic(event) { onSuppressedSyntheticInput?() return Unmanaged.passUnretained(event) } diff --git a/Cotabby/Services/Input/KeyboardInputSourceMonitor.swift b/Cotabby/Services/Input/KeyboardInputSourceMonitor.swift new file mode 100644 index 00000000..c4448259 --- /dev/null +++ b/Cotabby/Services/Input/KeyboardInputSourceMonitor.swift @@ -0,0 +1,86 @@ +import Carbon.HIToolbox +import Foundation +import Logging + +/// File overview: +/// Tracks whether the active keyboard input source is a *composing* IME (Japanese kana, Chinese, +/// Korean, Vietnamese, ...). `SuggestionInserter` reads this to pick an IME-safe way to commit an +/// accepted suggestion: a synthetic Unicode keystroke gets re-absorbed into composition by an active +/// IME (the keycode-0 event re-enters the input method instead of landing as literal text), so when +/// this is true the inserter writes through Accessibility / paste instead. The classification rule is +/// the pure `CompositionInputModeClassifier`; this type owns the live read and the change subscription. +/// +/// Out-of-process, Cotabby cannot inspect another app's `NSTextInputContext` or marked range, so the +/// only robust signal is which input source is selected. We read it via Text Input Sources (TIS) and +/// refresh on the distributed `kTISNotifySelectedKeyboardInputSourceChanged` notification, which fires +/// on every input-source AND mode switch (each IME mode is its own selectable source). Caching there +/// keeps `isComposingIMEActive` a synchronous, allocation-free read on the accept path. +@MainActor +final class KeyboardInputSourceMonitor { + /// True while a composing input method is selected. Cached; refreshed on the input-source change + /// notification, so reads at accept time are cheap and (since switching sources fires the + /// notification first) current. + private(set) var isComposingIMEActive = false + + private var observer: NSObjectProtocol? + + init() { + refresh() + // Mirror `PowerSourceMonitor`'s observer pattern: delivered on the main queue, so the + // MainActor-isolated callback runs without an extra hop under the project's main-actor + // default isolation. + observer = DistributedNotificationCenter.default().addObserver( + forName: Self.selectedInputSourceChangedNotification, + object: nil, + queue: .main + ) { [weak self] _ in + self?.handleInputSourceChanged() + } + } + + deinit { + if let observer { + DistributedNotificationCenter.default().removeObserver(observer) + } + } + + /// `kTISNotifySelectedKeyboardInputSourceChanged` bridged to a `Notification.Name`. + private static let selectedInputSourceChangedNotification = Notification.Name( + kTISNotifySelectedKeyboardInputSourceChanged as String + ) + + private func handleInputSourceChanged() { + let wasComposing = isComposingIMEActive + refresh() + if wasComposing != isComposingIMEActive { + CotabbyLogger.app.info("Composing IME active changed to \(self.isComposingIMEActive)") + } + } + + /// Reads the current keyboard input source via TIS and recomputes `isComposingIMEActive`. + private func refresh() { + guard let unmanagedSource = TISCopyCurrentKeyboardInputSource() else { + isComposingIMEActive = false + return + } + let source = unmanagedSource.takeRetainedValue() + + let isKeyboardLayout = Self.stringProperty(source, kTISPropertyInputSourceType) + == (kTISTypeKeyboardLayout as String) + let inputModeID = Self.stringProperty(source, kTISPropertyInputModeID) + + isComposingIMEActive = CompositionInputModeClassifier.isComposingInputMode( + isKeyboardLayout: isKeyboardLayout, + inputModeID: inputModeID + ) + } + + /// Reads a CFString-valued TIS property as a Swift `String`. `TISGetInputSourceProperty` returns a + /// non-owning `void *` into the source, so the value is bridged with `takeUnretainedValue()`. + private static func stringProperty(_ source: TISInputSource, _ key: CFString) -> String? { + guard let pointer = TISGetInputSourceProperty(source, key) else { + return nil + } + return Unmanaged.fromOpaque(pointer).takeUnretainedValue() as String + } +} diff --git a/Cotabby/Services/Suggestion/SuggestionInserter.swift b/Cotabby/Services/Suggestion/SuggestionInserter.swift index f6a249b4..1c0af40d 100644 --- a/Cotabby/Services/Suggestion/SuggestionInserter.swift +++ b/Cotabby/Services/Suggestion/SuggestionInserter.swift @@ -16,6 +16,13 @@ final class SuggestionInserter { private(set) var lastErrorMessage: String? + /// Reads whether a composing IME (Japanese kana, Chinese pinyin, Korean hangul, ...) is currently + /// active. Wired to `KeyboardInputSourceMonitor` in `CotabbyAppEnvironment`. When true, `insert(_:)` + /// commits through an IME-safe channel (Accessibility write, then clipboard paste) instead of a + /// synthetic keystroke, which an active input method would otherwise re-absorb into composition so + /// the accept silently fails. Defaults to "no IME" so tests and previews need no wiring. + var isComposingIMEActiveProvider: @MainActor () -> Bool = { false } + /// In-flight state for the opt-in paste path. The user's real clipboard is snapshotted once and /// restored after the paste lands. While a restore is pending, a second paste must NOT re-snapshot /// (the pasteboard then holds OUR completion, which would leak back to the user), so overlapping @@ -23,6 +30,11 @@ final class SuggestionInserter { private var pendingPasteboardRestore: DispatchWorkItem? private var savedClipboardForRestore: [[NSPasteboard.PasteboardType: Data]]? + /// Paste menu items located by `AXHelper.pasteMenuItem(forApplicationPID:)`, cached per app so + /// repeat accepts skip the menu-bar walk. A cached item is validated by its `AXPress` result; + /// a failure (menu rebuilt, app quit) evicts and re-walks once. + private var cachedPasteMenuItems: [pid_t: AXUIElement] = [:] + /// Virtual key code for Delete/Backspace. Posting these at the HID level deletes one UTF-16 unit /// of already-typed text per pair, which is how the picker removes the literal `:query` run. private static let backspaceKeyCode: CGKeyCode = 0x33 @@ -39,9 +51,10 @@ final class SuggestionInserter { } /// How long to leave the completion on the pasteboard before restoring the user's clipboard. Long - /// enough for the host to service Cmd-V, short enough that the user's clipboard is theirs again - /// almost immediately. Catalogued in `docs/POLLING_AND_DELAYS.md`; tune on device. - private static let pasteboardRestoreDelay: TimeInterval = 0.15 + /// enough for the host to service Cmd-V (a busy Chromium page can take a couple hundred ms), + /// short enough that the user's clipboard is theirs again almost immediately. Catalogued in + /// `docs/POLLING_AND_DELAYS.md`; tune on device. + private static let pasteboardRestoreDelay: TimeInterval = 0.3 init(suppressionController: InputSuppressionController) { self.suppressionController = suppressionController @@ -56,6 +69,27 @@ final class SuggestionInserter { return false } + // A composing IME (Japanese kana, Chinese pinyin, Korean hangul, ...) is active. A synthetic + // Unicode keystroke would be re-absorbed into composition by the input method (the placeholder + // keycode-0 event re-enters the IME) instead of landing as literal text, so the accepted + // suggestion never commits and the session desyncs against the live field, which surfaced as + // "Tab regenerates instead of accepting" for Japanese users. Commit via a clipboard paste: + // Cmd-V is a command shortcut the app services directly, so the input method never touches it. + // (An Accessibility `AXSelectedText` write was tried first and rejected: Chromium contenteditable + // accepts the set, reports `.success`, then silently no-ops, so the text never lands and the + // session desyncs exactly as before. Paste actually inserts there.) The clipboard is snapshotted + // and restored around the paste; only the last-resort keystroke below can still be swallowed. + if isComposingIMEActiveProvider() { + if insertViaPaste(normalized) { + lastErrorMessage = nil + CotabbyLogger.suggestion.debug("Inserted \(normalized.count) characters via paste (IME active)") + return true + } + let fallbackMessage = "IME-safe paste failed for \(normalized.count) characters; " + + "falling back to a synthetic keystroke the input method may swallow" + CotabbyLogger.suggestion.warning("\(fallbackMessage)") + } + // Paste path (opt-in): a long or multi-line completion is steadier as a clipboard paste in // apps that mishandle a big synthetic Unicode string. On any failure we fall through to the // reliable keystroke path below, so paste is never worse than the default keystroke insert. @@ -176,20 +210,40 @@ final class SuggestionInserter { } let expectedChangeCount = pasteboard.changeCount - guard let keyDown = CGEvent(keyboardEventSource: nil, virtualKey: Self.vKeyCode, keyDown: true), - let keyUp = CGEvent(keyboardEventSource: nil, virtualKey: Self.vKeyCode, keyDown: false) else { - pendingPasteboardRestore?.cancel() - Self.restorePasteboard(saved, to: pasteboard) - clearPendingPasteboardRestore() - return false + // Preferred trigger: press the host's real Edit > Paste menu item via Accessibility. No key + // event exists, so neither an active IME nor the HID modifier state machine can interfere. + // Synthetic Cmd-V is the fallback, and it has a real failure mode this path avoids: observed + // on-device, Cmd-V posted source-nil at the HID tap had its Command flag stripped (only Tab is + // physically down during an accept), and re-posting at the annotated session tap with a + // session source still never pasted in Chrome. The menu press drives the same paste command + // those key events would have reached, one IPC hop earlier. + if pressPasteMenuItem() { + CotabbyLogger.suggestion.debug("Paste committed via Edit > Paste menu press") + } else { + // Fallback synthetic Cmd-V: session source + annotated session tap so the event's own + // `.maskCommand` flag survives (the HID tap merges flags with live hardware state). The + // suppression filter keeps the physically held accept key from interleaving during the post. + let source = CGEventSource(stateID: .combinedSessionState) + source?.setLocalEventsFilterDuringSuppressionState( + [.permitLocalMouseEvents, .permitSystemDefinedEvents], + state: .eventSuppressionStateSuppressionInterval + ) + guard let keyDown = CGEvent(keyboardEventSource: source, virtualKey: Self.vKeyCode, keyDown: true), + let keyUp = CGEvent(keyboardEventSource: source, virtualKey: Self.vKeyCode, keyDown: false) else { + pendingPasteboardRestore?.cancel() + Self.restorePasteboard(saved, to: pasteboard) + clearPendingPasteboardRestore() + return false + } + keyDown.flags = .maskCommand + keyUp.flags = .maskCommand + suppressionController.markSynthetic(keyDown) + suppressionController.markSynthetic(keyUp) + suppressionController.registerSyntheticInsertion(expectedKeyDownCount: 1) + keyDown.post(tap: .cgAnnotatedSessionEventTap) + keyUp.post(tap: .cgAnnotatedSessionEventTap) + CotabbyLogger.suggestion.debug("Paste committed via synthetic Cmd-V (no Paste menu item found)") } - keyDown.flags = .maskCommand - keyUp.flags = .maskCommand - suppressionController.markSynthetic(keyDown) - suppressionController.markSynthetic(keyUp) - suppressionController.registerSyntheticInsertion(expectedKeyDownCount: 1) - keyDown.post(tap: .cghidEventTap) - keyUp.post(tap: .cghidEventTap) // Give the host time to service Cmd-V, then hand the clipboard back, but only if our completion // is still the thing on it. If the user copied something during the window, `changeCount` @@ -207,6 +261,30 @@ final class SuggestionInserter { return true } + /// Presses the focused app's Edit > Paste menu item via Accessibility. The owning app is resolved + /// from the focused element (not the frontmost app) so accessory panels that hold focus without + /// frontmost status still target the right process. The located item is cached per pid; a cached + /// press that fails (menu rebuilt, app relaunched into the same pid) evicts and re-walks once. + private func pressPasteMenuItem() -> Bool { + guard let focusedElement = AXHelper.focusedElement(), + let application = AXHelper.owningApplication(of: focusedElement) else { + return false + } + let pid = application.processIdentifier + if let cached = cachedPasteMenuItems[pid] { + if AXUIElementPerformAction(cached, kAXPressAction as CFString) == .success { + return true + } + cachedPasteMenuItems[pid] = nil + } + guard let item = AXHelper.pasteMenuItem(forApplicationPID: pid), + AXUIElementPerformAction(item, kAXPressAction as CFString) == .success else { + return false + } + cachedPasteMenuItems[pid] = item + return true + } + /// Clears the in-flight paste-restore bookkeeping once a restore has run (or been abandoned on a /// failure path), so the next paste snapshots the user's real clipboard afresh. private func clearPendingPasteboardRestore() { diff --git a/Cotabby/Support/AXHelper.swift b/Cotabby/Support/AXHelper.swift index a0af6d15..10bf0705 100644 --- a/Cotabby/Support/AXHelper.swift +++ b/Cotabby/Support/AXHelper.swift @@ -653,6 +653,59 @@ enum AXHelper { } } + /// Locates the app's "Paste" menu item by its Cmd-V key equivalent rather than by title, so the + /// lookup is language-independent ("Paste" / "貼り付け" / "Coller" depending on the host's + /// localization). The IME-safe insertion path presses this real menu item via `AXPress`: the + /// host runs its paste command without any key event existing, so an active input method can + /// neither swallow nor re-interpret the commit the way it does a synthetic keystroke. + static func pasteMenuItem(forApplicationPID pid: pid_t) -> AXUIElement? { + guard pid > 0 else { return nil } + let appElement = AXUIElementCreateApplication(pid) + AXUIElementSetMessagingTimeout(appElement, pollMessagingTimeout) + guard let menuBarValue = copyAttributeValue(kAXMenuBarAttribute as CFString, on: appElement), + CFGetTypeID(menuBarValue) == AXUIElementGetTypeID() else { + return nil + } + // Same Core Foundation bridging rule as `focusedElement()`. + let menuBar = unsafeBitCast(menuBarValue, to: AXUIElement.self) + + // Menu bar -> top-level items (Apple, app, File, Edit, ...) -> one AXMenu each -> menu items. + // Depth-1 only: Paste always lives directly in a top-level menu, and skipping submenu + // recursion keeps the walk bounded. Early exit on the first Cmd-V item. + // + // Every walked element gets the short poll timeout before it is messaged: the walk runs on + // the main actor inside the accept path, and an element handle that never had a timeout set + // would otherwise fall back to the multi-second AX default if the host (a busy Chromium + // process is the common case here) stalls, beachballing typing for the duration. + AXUIElementSetMessagingTimeout(menuBar, pollMessagingTimeout) + for topLevelItem in childElements(of: menuBar) { + AXUIElementSetMessagingTimeout(topLevelItem, pollMessagingTimeout) + for menu in childElements(of: topLevelItem) { + AXUIElementSetMessagingTimeout(menu, pollMessagingTimeout) + for item in childElements(of: menu) where isCommandVMenuItem(item) { + return item + } + } + } + return nil + } + + /// True for a menu item whose key equivalent is exactly Cmd-V. `kAXMenuItemCmdModifiers` uses the + /// Carbon menu encoding where 0 means "Command alone"; shift/option/control add bits and 8 means + /// the equivalent carries no Command at all, so only an exact 0 matches plain Cmd-V. A missing + /// modifiers attribute rejects the item explicitly rather than relying on `nil == 0` being false. + private static func isCommandVMenuItem(_ item: AXUIElement) -> Bool { + AXUIElementSetMessagingTimeout(item, pollMessagingTimeout) + guard let cmdChar = stringValue(for: kAXMenuItemCmdCharAttribute as CFString, on: item), + cmdChar.uppercased() == "V" else { + return false + } + guard let modifiers = intValue(for: kAXMenuItemCmdModifiersAttribute as CFString, on: item) else { + return false + } + return modifiers == 0 + } + static func elementIdentity(for element: AXUIElement) -> String { var pid: pid_t = 0 AXUIElementGetPid(element, &pid) diff --git a/Cotabby/Support/CompositionInputModeClassifier.swift b/Cotabby/Support/CompositionInputModeClassifier.swift new file mode 100644 index 00000000..8dc8c41d --- /dev/null +++ b/Cotabby/Support/CompositionInputModeClassifier.swift @@ -0,0 +1,48 @@ +import Foundation + +/// File overview: +/// Pure rule deciding whether the active keyboard input source is a *composing* input method, one +/// that assembles characters from several keystrokes through provisional "marked" text (Japanese +/// kana, Chinese pinyin/zhuyin/cangjie, Korean hangul, Vietnamese Telex, ...). It is intentionally +/// free of Carbon/TIS so it stays trivially testable; `KeyboardInputSourceMonitor` reads the live +/// input source and feeds the already-extracted fields in here. +/// +/// Why this matters: Cotabby commits an accepted suggestion by synthesizing a Unicode keystroke +/// (a key event on virtualKey 0). With a composing IME active, that synthetic keystroke does not +/// land as literal text, the input method re-absorbs it into composition, so the accepted text never +/// arrives, the session desyncs against the live field, and the suggestion regenerates instead of +/// committing. When this rule reports a composing mode, the inserter switches to an IME-safe path +/// (Accessibility write / clipboard paste) that bypasses the input method entirely. +enum CompositionInputModeClassifier { + /// Input mode IDs an IME exposes for typing plain ASCII *directly* (committed per keystroke, no + /// marked text), where the normal keystroke insertion is fine. The system Japanese IMEs share + /// `com.apple.inputmethod.Roman` for their alphanumeric ("英数") mode. + static let nonComposingInputModeIDs: Set = [ + "com.apple.inputmethod.Roman" + ] + + /// Whether the current input source composes through marked text (so committing accepted text + /// needs the IME-safe insertion path). + /// + /// - Parameters: + /// - isKeyboardLayout: the TIS source type is a plain keyboard layout (U.S., Dvorak, British, + /// ...). Layouts commit every keystroke directly and never compose. + /// - inputModeID: `kTISPropertyInputModeID` of the active source when it is an input mode; nil + /// for plain layouts and some method-without-modes IMEs. + /// + /// A plain layout never composes. Any non-layout input *method/mode* is treated as composing, + /// except the shared direct-ASCII mode. Erring toward "composing for any unrecognized non-layout + /// source" is the safe default: a third-party or future IME (ATOK, Sogou, ...) is far likelier to + /// compose through marked text than to commit plain ASCII, and the cost of a false positive is + /// only that we route through the (also-correct) IME-safe insertion path, whereas a false + /// negative reproduces the can't-accept bug. + static func isComposingInputMode(isKeyboardLayout: Bool, inputModeID: String?) -> Bool { + if isKeyboardLayout { + return false + } + if let inputModeID, nonComposingInputModeIDs.contains(inputModeID) { + return false + } + return true + } +} diff --git a/CotabbyTests/CompositionInputModeClassifierTests.swift b/CotabbyTests/CompositionInputModeClassifierTests.swift new file mode 100644 index 00000000..ce85453d --- /dev/null +++ b/CotabbyTests/CompositionInputModeClassifierTests.swift @@ -0,0 +1,91 @@ +import XCTest +@testable import Cotabby + +/// Tests for the pure rule that decides when an accepted suggestion must be committed through the +/// IME-safe insertion path. This classifier is the one piece of IME detection that does not touch +/// Carbon/TIS, so it carries the behavioral contract `KeyboardInputSourceMonitor` relies on. The +/// driving bug: with a composing IME active, the synthetic-keystroke insert is re-absorbed into +/// composition and the accept silently fails, so we detect composing input sources and switch +/// insertion methods. +final class CompositionInputModeClassifierTests: XCTestCase { + /// U.S. / Dvorak / British etc. are keyboard layouts: every keystroke commits directly, so they + /// never need the IME-safe path. + func test_plainKeyboardLayout_isNotComposing() { + XCTAssertFalse( + CompositionInputModeClassifier.isComposingInputMode( + isKeyboardLayout: true, + inputModeID: nil + ) + ) + } + + func test_japaneseHiragana_isComposing() { + XCTAssertTrue( + CompositionInputModeClassifier.isComposingInputMode( + isKeyboardLayout: false, + inputModeID: "com.apple.inputmethod.Japanese.Hiragana" + ) + ) + } + + func test_japaneseKatakana_isComposing() { + XCTAssertTrue( + CompositionInputModeClassifier.isComposingInputMode( + isKeyboardLayout: false, + inputModeID: "com.apple.inputmethod.Japanese.Katakana" + ) + ) + } + + func test_chinesePinyin_isComposing() { + XCTAssertTrue( + CompositionInputModeClassifier.isComposingInputMode( + isKeyboardLayout: false, + inputModeID: "com.apple.inputmethod.SCIM.ITABC" + ) + ) + } + + func test_korean_isComposing() { + XCTAssertTrue( + CompositionInputModeClassifier.isComposingInputMode( + isKeyboardLayout: false, + inputModeID: "com.apple.inputmethod.Korean.2SetKorean" + ) + ) + } + + /// The shared direct-ASCII ("英数") mode of the Japanese IMEs commits per keystroke with no marked + /// text, so the normal keystroke insert works there. + func test_romanDirectMode_isNotComposing() { + XCTAssertFalse( + CompositionInputModeClassifier.isComposingInputMode( + isKeyboardLayout: false, + inputModeID: "com.apple.inputmethod.Roman" + ) + ) + } + + /// A third-party IME (ATOK, Sogou, ...) that is not a plain layout and exposes no recognized + /// direct mode is assumed to compose, the safe default that fixes the reported bug (the reporter + /// uses ATOK). + func test_unknownInputMethod_isComposing() { + XCTAssertTrue( + CompositionInputModeClassifier.isComposingInputMode( + isKeyboardLayout: false, + inputModeID: "com.justsystems.inputmethod.atok33.Japanese" + ) + ) + } + + /// An input method reporting no mode ID (a method-without-modes) that is not a layout still + /// composes. + func test_inputMethodWithoutModeID_isComposing() { + XCTAssertTrue( + CompositionInputModeClassifier.isComposingInputMode( + isKeyboardLayout: false, + inputModeID: nil + ) + ) + } +}