diff --git a/CHANGELOG.md b/CHANGELOG.md index 6097d07..474c6c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ ### JSON Output - feat: expose raw Messages `balloon_bundle_id` in message JSON/RPC payloads so consumers can recognize URL-preview rows without parsing text (#137, thanks @omarshahine). +- fix: coalesce Apple URL-preview split-send rows into one logical message across history, search, and watch, with `url_preview` metadata when the preview row is folded into the text row (#141, thanks @omarshahine). ### Advanced IMCore - fix: defer injected bridge bootstrap until after Messages startup so macOS 26 dyld constructor ordering cannot touch ObjC/Foundation/IMCore before the process is ready (#138, thanks @omarshahine). diff --git a/README.md b/README.md index 3051b1d..beadcec 100644 --- a/README.md +++ b/README.md @@ -159,6 +159,7 @@ Message objects include: - `reply_to_text`, `reply_to_sender` (parent body + handle for threaded replies and non-reaction associations, when the parent row is still in chat.db) +- `balloon_bundle_id`, `url_preview` - `sender`, `sender_name`, `is_from_me`, `text`, `created_at` - `attachments`, `reactions` - `poll` for native Apple Messages poll creation and vote rows diff --git a/Sources/IMsgCore/Message+URLPreview.swift b/Sources/IMsgCore/Message+URLPreview.swift new file mode 100644 index 0000000..eb82f9b --- /dev/null +++ b/Sources/IMsgCore/Message+URLPreview.swift @@ -0,0 +1,49 @@ +import Foundation + +extension Message { + public struct URLPreviewMetadata: Sendable, Equatable { + public let rowID: Int64 + public let guid: String + public let balloonBundleID: String + public let date: Date + + public init(rowID: Int64, guid: String, balloonBundleID: String, date: Date) { + self.rowID = rowID + self.guid = guid + self.balloonBundleID = balloonBundleID + self.date = date + } + } + + public func withURLPreview(_ preview: URLPreviewMetadata) -> Message { + Message( + rowID: rowID, + chatID: chatID, + sender: sender, + text: text, + date: date, + isFromMe: isFromMe, + service: service, + handleID: handleID, + attachmentsCount: attachmentsCount, + guid: guid, + routing: RoutingMetadata( + replyToGUID: replyToGUID, + threadOriginatorGUID: threadOriginatorGUID, + threadOriginatorPart: threadOriginatorPart, + destinationCallerID: destinationCallerID, + replyToText: replyToText, + replyToSender: replyToSender + ), + balloonBundleID: balloonBundleID, + urlPreview: preview, + reaction: ReactionMetadata( + isReaction: isReaction, + reactionType: reactionType, + isReactionAdd: isReactionAdd, + reactedToGUID: reactedToGUID + ), + poll: poll + ) + } +} diff --git a/Sources/IMsgCore/MessageSendStatus.swift b/Sources/IMsgCore/MessageSendStatus.swift new file mode 100644 index 0000000..3b48e93 --- /dev/null +++ b/Sources/IMsgCore/MessageSendStatus.swift @@ -0,0 +1,61 @@ +import Foundation + +public enum MessageSendState: String, Sendable, Equatable { + case pending + case sent + case delivered + case failed +} + +public struct MessageSendStatus: Sendable, Equatable { + public let rowID: Int64 + public let guid: String + public let service: String + public let error: Int + public let dateDelivered: Date? + public let dateRead: Date? + public let isSent: Bool + public let isDelivered: Bool + public let isFinished: Bool + public let isDelayed: Bool + public let isPrepared: Bool + public let isPendingSatelliteSend: Bool + public let wasDowngraded: Bool + + public var state: MessageSendState { + if error != 0 { return .failed } + if isDelivered || dateDelivered != nil { return .delivered } + if isSent { return .sent } + return .pending + } + + public init( + rowID: Int64, + guid: String, + service: String, + error: Int, + dateDelivered: Date?, + dateRead: Date?, + isSent: Bool, + isDelivered: Bool, + isFinished: Bool, + isDelayed: Bool, + isPrepared: Bool, + isPendingSatelliteSend: Bool, + wasDowngraded: Bool + ) { + self.rowID = rowID + self.guid = guid + self.service = service + self.error = error + self.dateDelivered = dateDelivered + self.dateRead = dateRead + self.isSent = isSent + self.isDelivered = isDelivered + self.isFinished = isFinished + self.isDelayed = isDelayed + self.isPrepared = isPrepared + self.isPendingSatelliteSend = isPendingSatelliteSend + self.wasDowngraded = wasDowngraded + } +} diff --git a/Sources/IMsgCore/MessageStore+MessageConstruction.swift b/Sources/IMsgCore/MessageStore+MessageConstruction.swift new file mode 100644 index 0000000..b54f99a --- /dev/null +++ b/Sources/IMsgCore/MessageStore+MessageConstruction.swift @@ -0,0 +1,104 @@ +import Foundation +import SQLite + +extension MessageStore { + func message( + from decoded: DecodedMessageRow, + _ db: Connection, + parentCache: inout ReplyParentCache, + pollOptionCache: inout PollOptionTextCache + ) throws -> Message { + let poll = try enrichedPollEvent( + decoded.poll, + db: db, + cache: &pollOptionCache + ) + let reaction = decodeReaction( + associatedType: decoded.associatedType, + associatedGUID: decoded.associatedGUID, + text: decoded.text + ) + let replyToGUID = routedReplyToGUID(decoded) + let threadOriginatorGUID = + reaction.isReaction || decoded.threadOriginatorGUID.isEmpty + ? nil : decoded.threadOriginatorGUID + let threadOriginatorPart = + reaction.isReaction || decoded.threadOriginatorPart.isEmpty + ? nil : decoded.threadOriginatorPart + let parent = + reaction.isReaction + ? nil + : enrichedReplyContext( + db, + replyToGUID: replyToGUID, + threadOriginatorGUID: threadOriginatorGUID, + cache: &parentCache + ) + + return Message( + rowID: decoded.rowID, + chatID: decoded.chatID, + sender: decoded.sender, + text: decoded.text, + date: decoded.date, + isFromMe: decoded.isFromMe, + service: decoded.service, + handleID: decoded.handleID, + attachmentsCount: decoded.attachments, + guid: decoded.guid, + routing: Message.RoutingMetadata( + replyToGUID: replyToGUID, + threadOriginatorGUID: threadOriginatorGUID, + threadOriginatorPart: threadOriginatorPart, + destinationCallerID: decoded.destinationCallerID.isEmpty + ? nil : decoded.destinationCallerID, + replyToText: parent?.text, + replyToSender: parent?.sender + ), + balloonBundleID: decoded.balloonBundleID.isEmpty ? nil : decoded.balloonBundleID, + reaction: Message.ReactionMetadata( + isReaction: reaction.isReaction, + reactionType: reaction.reactionType, + isReactionAdd: reaction.isReactionAdd, + reactedToGUID: reaction.reactedToGUID + ), + poll: poll + ) + } + + func precedingTextMessageForURLPreview(_ preview: Message, db: Connection) throws -> Message? { + guard isURLPreviewBalloon(preview) else { return nil } + let selection = MessageRowSelection(store: self, includeChatID: true) + let reactionFilter = + schema.hasReactionColumns + ? "AND (m.associated_message_type IS NULL OR m.associated_message_type < 2000 OR m.associated_message_type > 3006)" + : "" + let sql = """ + SELECT \(selection.selectList) + FROM message m + JOIN chat_message_join cmj ON m.ROWID = cmj.message_id + LEFT JOIN handle h ON m.handle_id = h.ROWID + WHERE m.ROWID < ? + AND cmj.chat_id = ? + \(reactionFilter) + ORDER BY m.ROWID DESC + LIMIT 1 + """ + let rows = try db.prepareRowIterator(sql, bindings: [preview.rowID, preview.chatID]) + guard let row = try rows.failableNext() else { return nil } + let decoded = try decodeMessageRow( + row, + columns: selection.columns, + fallbackChatID: preview.chatID + ) + var parentCache: ReplyParentCache = [:] + var pollOptionCache = PollOptionTextCache() + let previous = try message( + from: decoded, + db, + parentCache: &parentCache, + pollOptionCache: &pollOptionCache + ) + return canCoalesceURLPreview(textMessage: previous, previewMessage: preview) ? previous : nil + } +} diff --git a/Sources/IMsgCore/MessageStore+MessageRows.swift b/Sources/IMsgCore/MessageStore+MessageRows.swift new file mode 100644 index 0000000..5e4842b --- /dev/null +++ b/Sources/IMsgCore/MessageStore+MessageRows.swift @@ -0,0 +1,155 @@ +import Foundation + +struct MessageRowColumns { + static let balloonBundleID = "balloon_bundle_id" + static let payloadData = "payload_data" + static let messageSummaryInfo = "message_summary_info" + + let rowID: String + let chatID: String? + let handleID: String + let sender: String + let text: String + let date: String + let isFromMe: String + let service: String + let isAudioMessage: String + let destinationCallerID: String + let guid: String + let associatedGUID: String + let associatedType: String + let attachments: String + let body: String + let threadOriginatorGUID: String + let threadOriginatorPart: String + let replyToGUID: String + let balloonBundleID: String + let payloadData: String + let messageSummaryInfo: String + + static func message(chatID: String?) -> MessageRowColumns { + MessageRowColumns( + rowID: "message_rowid", + chatID: chatID, + handleID: "handle_id", + sender: "sender", + text: "text", + date: "date", + isFromMe: "is_from_me", + service: "service", + isAudioMessage: "is_audio_message", + destinationCallerID: "destination_caller_id", + guid: "guid", + associatedGUID: "associated_guid", + associatedType: "associated_type", + attachments: "attachments", + body: "body", + threadOriginatorGUID: "thread_originator_guid", + threadOriginatorPart: "thread_originator_part", + replyToGUID: "reply_to_guid", + balloonBundleID: MessageRowColumns.balloonBundleID, + payloadData: MessageRowColumns.payloadData, + messageSummaryInfo: MessageRowColumns.messageSummaryInfo + ) + } +} + +struct DecodedMessageRow { + let rowID: Int64 + let chatID: Int64 + let handleID: Int64? + let sender: String + let text: String + let date: Date + let isFromMe: Bool + let service: String + let destinationCallerID: String + let guid: String + let associatedGUID: String + let associatedType: Int? + let attachments: Int + let threadOriginatorGUID: String + let threadOriginatorPart: String + let databaseReplyToGUID: String + let balloonBundleID: String + let poll: MessagePollEvent? +} + +struct PollOptionTextCache { + var optionsByPollGUID: [String: [String: String]] = [:] + var missingPollGUIDs = Set() +} + +struct MessagesAfterBatch { + let messages: [Message] + let maxScannedRowID: Int64 +} + +struct MessageRowSelection { + let selectList: String + let columns: MessageRowColumns + + init(store: MessageStore, includeChatID: Bool) { + let columns = MessageRowColumns.message(chatID: includeChatID ? "chat_id" : nil) + let schema = store.schema + let bodyColumn = schema.hasAttributedBody ? "m.attributedBody" : "NULL" + let guidColumn = schema.hasReactionColumns ? "m.guid" : "NULL" + let associatedGuidColumn = schema.hasReactionColumns ? "m.associated_message_guid" : "NULL" + let associatedTypeColumn = schema.hasReactionColumns ? "m.associated_message_type" : "NULL" + let destinationCallerColumn = + schema.hasDestinationCallerID ? "m.destination_caller_id" : "NULL" + let audioMessageColumn = schema.hasAudioMessageColumn ? "m.is_audio_message" : "0" + let threadOriginatorColumn = + schema.hasThreadOriginatorGUIDColumn ? "m.thread_originator_guid" : "NULL" + let threadOriginatorPartColumn = + schema.hasThreadOriginatorPartColumn ? "m.thread_originator_part" : "NULL" + let replyToColumn = schema.hasReplyToGUIDColumn ? "m.reply_to_guid" : "NULL" + let balloonColumn = schema.hasBalloonBundleIDColumn ? "m.balloon_bundle_id" : "NULL" + let pollCandidatePredicate = Self.pollCandidatePredicate(schema: schema) + let payloadDataColumn = + schema.hasPayloadDataColumn + ? "CASE WHEN \(pollCandidatePredicate) THEN m.payload_data ELSE NULL END" : "NULL" + let summaryInfoColumn = + schema.hasMessageSummaryInfoColumn + ? "CASE WHEN \(pollCandidatePredicate) THEN m.message_summary_info ELSE NULL END" : "NULL" + let chatColumn = includeChatID ? ", cmj.chat_id AS \(columns.chatID!)" : "" + + let selectList = """ + m.ROWID AS \(columns.rowID)\(chatColumn), m.handle_id AS \(columns.handleID), + h.id AS \(columns.sender), IFNULL(m.text, '') AS \(columns.text), + m.date AS \(columns.date), m.is_from_me AS \(columns.isFromMe), + m.service AS \(columns.service), + \(audioMessageColumn) AS \(columns.isAudioMessage), + \(destinationCallerColumn) AS \(columns.destinationCallerID), + \(guidColumn) AS \(columns.guid), \(associatedGuidColumn) AS \(columns.associatedGUID), + \(associatedTypeColumn) AS \(columns.associatedType), + (SELECT COUNT(*) FROM message_attachment_join maj WHERE maj.message_id = m.ROWID) AS \(columns.attachments), + \(bodyColumn) AS \(columns.body), + \(threadOriginatorColumn) AS \(columns.threadOriginatorGUID), + \(threadOriginatorPartColumn) AS \(columns.threadOriginatorPart), + \(replyToColumn) AS \(columns.replyToGUID), + \(balloonColumn) AS \(columns.balloonBundleID), + \(payloadDataColumn) AS \(columns.payloadData), + \(summaryInfoColumn) AS \(columns.messageSummaryInfo) + """ + self.selectList = selectList + self.columns = columns + } + + private static func pollCandidatePredicate(schema: MessageStoreSchema) -> String { + let pollBundle = sqlStringLiteral(MessagePollDecoder.pollsBundleIdentifier) + let pollBalloonPredicate = + schema.hasBalloonBundleIDColumn + ? "(m.balloon_bundle_id = \(pollBundle) OR m.balloon_bundle_id LIKE '%:' || \(pollBundle))" + : "0" + let votePredicate = + schema.hasReactionColumns + ? "m.associated_message_type = \(MessagePollDecoder.voteAssociatedMessageType)" + : "0" + return "(\(pollBalloonPredicate) OR \(votePredicate))" + } + + private static func sqlStringLiteral(_ value: String) -> String { + "'\(value.replacingOccurrences(of: "'", with: "''"))'" + } +} diff --git a/Sources/IMsgCore/MessageStore+Messages.swift b/Sources/IMsgCore/MessageStore+Messages.swift index ac29dbc..8e1f168 100644 --- a/Sources/IMsgCore/MessageStore+Messages.swift +++ b/Sources/IMsgCore/MessageStore+Messages.swift @@ -1,155 +1,6 @@ import Foundation import SQLite -struct MessageRowColumns { - static let balloonBundleID = "balloon_bundle_id" - static let payloadData = "payload_data" - static let messageSummaryInfo = "message_summary_info" - - let rowID: String - let chatID: String? - let handleID: String - let sender: String - let text: String - let date: String - let isFromMe: String - let service: String - let isAudioMessage: String - let destinationCallerID: String - let guid: String - let associatedGUID: String - let associatedType: String - let attachments: String - let body: String - let threadOriginatorGUID: String - let threadOriginatorPart: String - let replyToGUID: String - let balloonBundleID: String - let payloadData: String - let messageSummaryInfo: String - - static func message(chatID: String?) -> MessageRowColumns { - MessageRowColumns( - rowID: "message_rowid", - chatID: chatID, - handleID: "handle_id", - sender: "sender", - text: "text", - date: "date", - isFromMe: "is_from_me", - service: "service", - isAudioMessage: "is_audio_message", - destinationCallerID: "destination_caller_id", - guid: "guid", - associatedGUID: "associated_guid", - associatedType: "associated_type", - attachments: "attachments", - body: "body", - threadOriginatorGUID: "thread_originator_guid", - threadOriginatorPart: "thread_originator_part", - replyToGUID: "reply_to_guid", - balloonBundleID: MessageRowColumns.balloonBundleID, - payloadData: MessageRowColumns.payloadData, - messageSummaryInfo: MessageRowColumns.messageSummaryInfo - ) - } -} - -struct DecodedMessageRow { - let rowID: Int64 - let chatID: Int64 - let handleID: Int64? - let sender: String - let text: String - let date: Date - let isFromMe: Bool - let service: String - let destinationCallerID: String - let guid: String - let associatedGUID: String - let associatedType: Int? - let attachments: Int - let threadOriginatorGUID: String - let threadOriginatorPart: String - let databaseReplyToGUID: String - let balloonBundleID: String - let poll: MessagePollEvent? -} - -struct PollOptionTextCache { - var optionsByPollGUID: [String: [String: String]] = [:] - var missingPollGUIDs = Set() -} - -struct MessageRowSelection { - let selectList: String - let columns: MessageRowColumns - - init(store: MessageStore, includeChatID: Bool) { - let columns = MessageRowColumns.message(chatID: includeChatID ? "chat_id" : nil) - let schema = store.schema - let bodyColumn = schema.hasAttributedBody ? "m.attributedBody" : "NULL" - let guidColumn = schema.hasReactionColumns ? "m.guid" : "NULL" - let associatedGuidColumn = schema.hasReactionColumns ? "m.associated_message_guid" : "NULL" - let associatedTypeColumn = schema.hasReactionColumns ? "m.associated_message_type" : "NULL" - let destinationCallerColumn = - schema.hasDestinationCallerID ? "m.destination_caller_id" : "NULL" - let audioMessageColumn = schema.hasAudioMessageColumn ? "m.is_audio_message" : "0" - let threadOriginatorColumn = - schema.hasThreadOriginatorGUIDColumn ? "m.thread_originator_guid" : "NULL" - let threadOriginatorPartColumn = - schema.hasThreadOriginatorPartColumn ? "m.thread_originator_part" : "NULL" - let replyToColumn = schema.hasReplyToGUIDColumn ? "m.reply_to_guid" : "NULL" - let balloonColumn = schema.hasBalloonBundleIDColumn ? "m.balloon_bundle_id" : "NULL" - let pollCandidatePredicate = Self.pollCandidatePredicate(schema: schema) - let payloadDataColumn = - schema.hasPayloadDataColumn - ? "CASE WHEN \(pollCandidatePredicate) THEN m.payload_data ELSE NULL END" : "NULL" - let summaryInfoColumn = - schema.hasMessageSummaryInfoColumn - ? "CASE WHEN \(pollCandidatePredicate) THEN m.message_summary_info ELSE NULL END" : "NULL" - let chatColumn = includeChatID ? ", cmj.chat_id AS \(columns.chatID!)" : "" - - let selectList = """ - m.ROWID AS \(columns.rowID)\(chatColumn), m.handle_id AS \(columns.handleID), - h.id AS \(columns.sender), IFNULL(m.text, '') AS \(columns.text), - m.date AS \(columns.date), m.is_from_me AS \(columns.isFromMe), - m.service AS \(columns.service), - \(audioMessageColumn) AS \(columns.isAudioMessage), - \(destinationCallerColumn) AS \(columns.destinationCallerID), - \(guidColumn) AS \(columns.guid), \(associatedGuidColumn) AS \(columns.associatedGUID), - \(associatedTypeColumn) AS \(columns.associatedType), - (SELECT COUNT(*) FROM message_attachment_join maj WHERE maj.message_id = m.ROWID) AS \(columns.attachments), - \(bodyColumn) AS \(columns.body), - \(threadOriginatorColumn) AS \(columns.threadOriginatorGUID), - \(threadOriginatorPartColumn) AS \(columns.threadOriginatorPart), - \(replyToColumn) AS \(columns.replyToGUID), - \(balloonColumn) AS \(columns.balloonBundleID), - \(payloadDataColumn) AS \(columns.payloadData), - \(summaryInfoColumn) AS \(columns.messageSummaryInfo) - """ - self.selectList = selectList - self.columns = columns - } - - private static func pollCandidatePredicate(schema: MessageStoreSchema) -> String { - let pollBundle = sqlStringLiteral(MessagePollDecoder.pollsBundleIdentifier) - let pollBalloonPredicate = - schema.hasBalloonBundleIDColumn - ? "(m.balloon_bundle_id = \(pollBundle) OR m.balloon_bundle_id LIKE '%:' || \(pollBundle))" - : "0" - let votePredicate = - schema.hasReactionColumns - ? "m.associated_message_type = \(MessagePollDecoder.voteAssociatedMessageType)" - : "0" - return "(\(pollBalloonPredicate) OR \(votePredicate))" - } - - private static func sqlStringLiteral(_ value: String) -> String { - "'\(value.replacingOccurrences(of: "'", with: "''"))'" - } -} - extension MessageStore { public func maxRowID() throws -> Int64 { return try withConnection { db in @@ -163,69 +14,79 @@ extension MessageStore { } public func messages(chatID: Int64, limit: Int, filter: MessageFilter?) throws -> [Message] { - let query = ChatMessagesQuery( - store: self, - chatID: ChatID(rawValue: chatID), - limit: limit, - filter: filter - ) + guard limit > 0 else { return [] } + var physicalLimit = limit return try withConnection { db in - var messages: [Message] = [] - var parentCache: ReplyParentCache = [:] - var pollOptionCache = PollOptionTextCache() - let rows = try db.prepareRowIterator(query.sql, bindings: query.bindings) - while let row = try rows.failableNext() { - let decoded = try decodeMessageRow( - row, - columns: query.selection.columns, - fallbackChatID: query.fallbackChatID - ) - let poll = try enrichedPollEvent( - decoded.poll, - db: db, - cache: &pollOptionCache - ) - let replyToGUID = routedReplyToGUID(decoded) - let threadOriginatorGUID = - decoded.threadOriginatorGUID.isEmpty ? nil : decoded.threadOriginatorGUID - let threadOriginatorPart = - decoded.threadOriginatorPart.isEmpty ? nil : decoded.threadOriginatorPart - let parent = enrichedReplyContext( - db, - replyToGUID: replyToGUID, - threadOriginatorGUID: threadOriginatorGUID, - cache: &parentCache + while true { + let query = ChatMessagesQuery( + store: self, + chatID: ChatID(rawValue: chatID), + limit: physicalLimit, + filter: filter ) - messages.append( - Message( - rowID: decoded.rowID, - chatID: decoded.chatID, - sender: decoded.sender, - text: decoded.text, - date: decoded.date, - isFromMe: decoded.isFromMe, - service: decoded.service, - handleID: decoded.handleID, - attachmentsCount: decoded.attachments, - guid: decoded.guid, - routing: Message.RoutingMetadata( - replyToGUID: replyToGUID, - threadOriginatorGUID: threadOriginatorGUID, - threadOriginatorPart: threadOriginatorPart, - destinationCallerID: decoded.destinationCallerID.isEmpty - ? nil : decoded.destinationCallerID, - replyToText: parent?.text, - replyToSender: parent?.sender - ), - balloonBundleID: decoded.balloonBundleID.isEmpty ? nil : decoded.balloonBundleID, - poll: poll - )) + var messages: [Message] = [] + var parentCache: ReplyParentCache = [:] + var pollOptionCache = PollOptionTextCache() + let rows = try db.prepareRowIterator(query.sql, bindings: query.bindings) + while let row = try rows.failableNext() { + let decoded = try decodeMessageRow( + row, + columns: query.selection.columns, + fallbackChatID: query.fallbackChatID + ) + messages.append( + try message( + from: decoded, + db, + parentCache: &parentCache, + pollOptionCache: &pollOptionCache + )) + } + var usedFallbackReplacement = false + let coalesced = try coalesceURLPreviewMessages( + messages, + validateExistingCoalescence: { text, preview in + try self.precedingTextMessageForURLPreview(preview, db: db)?.rowID == text.rowID + }, + fallbackForUnmatchedPreview: { preview in + guard let previous = try self.precedingTextMessageForURLPreview(preview, db: db) else { + return nil + } + if let filter, !filter.allows(previous) { + return nil + } + return .replace(previous) + }, + fallbackReplacementUsed: { + usedFallbackReplacement = true + } + ).sorted(by: messageHistoryNewestFirst) + + if messages.count < physicalLimit || (coalesced.count >= limit && !usedFallbackReplacement) + { + return Array(coalesced.prefix(limit)) + } + guard let nextLimit = nextHistoryPhysicalLimit(after: physicalLimit) else { + return Array(coalesced.prefix(limit)) + } + physicalLimit = nextLimit } - return messages } } + private func nextHistoryPhysicalLimit(after current: Int) -> Int? { + guard current > 0, current <= Int.max / 2 else { return nil } + return current * 2 + } + + private func messageHistoryNewestFirst(_ lhs: Message, _ rhs: Message) -> Bool { + if lhs.date == rhs.date { + return lhs.rowID > rhs.rowID + } + return lhs.date > rhs.date + } + public func messagesAfter(afterRowID: Int64, chatID: Int64?, limit: Int) throws -> [Message] { return try messagesAfter( afterRowID: afterRowID, @@ -241,6 +102,31 @@ extension MessageStore { limit: Int, includeReactions: Bool ) throws -> [Message] { + guard limit > 0 else { return [] } + var cursor = afterRowID + while true { + let batch = try messagesAfterBatch( + afterRowID: cursor, + chatID: chatID, + limit: limit, + includeReactions: includeReactions + ) + if !batch.messages.isEmpty { + return batch.messages + } + guard batch.maxScannedRowID > cursor else { + return [] + } + cursor = batch.maxScannedRowID + } + } + + func messagesAfterBatch( + afterRowID: Int64, + chatID: Int64?, + limit: Int, + includeReactions: Bool + ) throws -> MessagesAfterBatch { let query = MessagesAfterQuery( store: self, afterRowID: MessageID(rawValue: afterRowID), @@ -253,7 +139,7 @@ extension MessageStore { var messages: [Message] = [] var parentCache: ReplyParentCache = [:] var pollOptionCache = PollOptionTextCache() - let urlBalloonProvider = "com.apple.messages.URLBalloonProvider" + var maxScannedRowID = afterRowID let rows = try db.prepareRowIterator(query.sql, bindings: query.bindings) while let row = try rows.failableNext() { @@ -262,79 +148,39 @@ extension MessageStore { columns: query.selection.columns, fallbackChatID: query.fallbackChatID ) - let poll = try enrichedPollEvent( - decoded.poll, - db: db, - cache: &pollOptionCache - ) - let balloonBundleID = try stringValue(row, MessageRowColumns.balloonBundleID) - if balloonBundleID == urlBalloonProvider, - shouldSkipURLBalloonDuplicate( - chatID: decoded.chatID, - sender: decoded.sender, - text: decoded.text, - isFromMe: decoded.isFromMe, - date: decoded.date, - rowID: decoded.rowID - ) - { - continue - } - - let reaction = decodeReaction( - associatedType: decoded.associatedType, - associatedGUID: decoded.associatedGUID, - text: decoded.text - ) - let replyToGUID = routedReplyToGUID(decoded) - let threadOriginatorGUID = - reaction.isReaction || decoded.threadOriginatorGUID.isEmpty - ? nil : decoded.threadOriginatorGUID - let threadOriginatorPart = - reaction.isReaction || decoded.threadOriginatorPart.isEmpty - ? nil : decoded.threadOriginatorPart - let parent = - reaction.isReaction - ? nil - : enrichedReplyContext( - db, - replyToGUID: replyToGUID, - threadOriginatorGUID: threadOriginatorGUID, - cache: &parentCache - ) - + maxScannedRowID = max(maxScannedRowID, decoded.rowID) messages.append( - Message( - rowID: decoded.rowID, - chatID: decoded.chatID, - sender: decoded.sender, - text: decoded.text, - date: decoded.date, - isFromMe: decoded.isFromMe, - service: decoded.service, - handleID: decoded.handleID, - attachmentsCount: decoded.attachments, - guid: decoded.guid, - routing: Message.RoutingMetadata( - replyToGUID: replyToGUID, - threadOriginatorGUID: threadOriginatorGUID, - threadOriginatorPart: threadOriginatorPart, - destinationCallerID: decoded.destinationCallerID.isEmpty - ? nil : decoded.destinationCallerID, - replyToText: parent?.text, - replyToSender: parent?.sender - ), - balloonBundleID: decoded.balloonBundleID.isEmpty ? nil : decoded.balloonBundleID, - reaction: Message.ReactionMetadata( - isReaction: reaction.isReaction, - reactionType: reaction.reactionType, - isReactionAdd: reaction.isReactionAdd, - reactedToGUID: reaction.reactedToGUID - ), - poll: poll + try message( + from: decoded, + db, + parentCache: &parentCache, + pollOptionCache: &pollOptionCache )) } - return messages + let coalesced = try coalesceURLPreviewMessages( + messages, + validateExistingCoalescence: { text, preview in + try self.precedingTextMessageForURLPreview(preview, db: db)?.rowID == text.rowID + }, + fallbackForUnmatchedPreview: { preview in + guard try self.precedingTextMessageForURLPreview(preview, db: db) != nil else { + return nil + } + return .suppress + } + ) + let visibleMessages = coalesced.filter { message in + guard isURLPreviewBalloon(message) else { return true } + return !shouldSkipURLBalloonDuplicate( + chatID: message.chatID, + sender: message.sender, + text: message.text, + isFromMe: message.isFromMe, + date: message.date, + rowID: message.rowID + ) + } + return MessagesAfterBatch(messages: visibleMessages, maxScannedRowID: maxScannedRowID) } } diff --git a/Sources/IMsgCore/MessageStore+Search.swift b/Sources/IMsgCore/MessageStore+Search.swift index 9bfd3a2..ac4c5b3 100644 --- a/Sources/IMsgCore/MessageStore+Search.swift +++ b/Sources/IMsgCore/MessageStore+Search.swift @@ -46,67 +46,84 @@ extension MessageStore { public func searchMessages(query text: String, match: String, limit: Int) throws -> [Message] { let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) guard !trimmed.isEmpty else { return [] } + guard limit > 0 else { return [] } let exact = match.lowercased() == "exact" - let query = SearchMessagesQuery( - store: self, - text: trimmed, - exact: exact, - limit: limit - ) + var physicalLimit = limit return try withConnection { db in - var messages: [Message] = [] - var parentCache: ReplyParentCache = [:] - var pollOptionCache = PollOptionTextCache() - let rows = try db.prepareRowIterator(query.sql, bindings: query.bindings) - while let row = try rows.failableNext() { - let decoded = try decodeMessageRow( - row, - columns: query.selection.columns, - fallbackChatID: query.fallbackChatID + while true { + let query = SearchMessagesQuery( + store: self, + text: trimmed, + exact: exact, + limit: physicalLimit ) - let poll = try enrichedPollEvent( - decoded.poll, - db: db, - cache: &pollOptionCache - ) - let replyToGUID = routedReplyToGUID(decoded) - let threadOriginatorGUID = - decoded.threadOriginatorGUID.isEmpty ? nil : decoded.threadOriginatorGUID - let threadOriginatorPart = - decoded.threadOriginatorPart.isEmpty ? nil : decoded.threadOriginatorPart - let parent = enrichedReplyContext( - db, - replyToGUID: replyToGUID, - threadOriginatorGUID: threadOriginatorGUID, - cache: &parentCache - ) - messages.append( - Message( - rowID: decoded.rowID, - chatID: decoded.chatID, - sender: decoded.sender, - text: decoded.text, - date: decoded.date, - isFromMe: decoded.isFromMe, - service: decoded.service, - handleID: decoded.handleID, - attachmentsCount: decoded.attachments, - guid: decoded.guid, - routing: Message.RoutingMetadata( - replyToGUID: replyToGUID, - threadOriginatorGUID: threadOriginatorGUID, - threadOriginatorPart: threadOriginatorPart, - destinationCallerID: decoded.destinationCallerID.isEmpty - ? nil : decoded.destinationCallerID, - replyToText: parent?.text, - replyToSender: parent?.sender - ), - balloonBundleID: decoded.balloonBundleID.isEmpty ? nil : decoded.balloonBundleID, - poll: poll - )) + var messages: [Message] = [] + var parentCache: ReplyParentCache = [:] + var pollOptionCache = PollOptionTextCache() + let rows = try db.prepareRowIterator(query.sql, bindings: query.bindings) + while let row = try rows.failableNext() { + let decoded = try decodeMessageRow( + row, + columns: query.selection.columns, + fallbackChatID: query.fallbackChatID + ) + messages.append( + try message( + from: decoded, + db, + parentCache: &parentCache, + pollOptionCache: &pollOptionCache + )) + } + var usedFallbackReplacement = false + let coalesced = try coalesceURLPreviewMessages( + messages, + validateExistingCoalescence: { text, preview in + try self.precedingTextMessageForURLPreview(preview, db: db)?.rowID == text.rowID + }, + fallbackForUnmatchedPreview: { preview in + guard let previous = try self.precedingTextMessageForURLPreview(preview, db: db) else { + return nil + } + guard self.searchMessage(previous, matches: trimmed, exact: exact) else { + return nil + } + return .replace(previous) + }, + fallbackReplacementUsed: { + usedFallbackReplacement = true + } + ).sorted(by: searchMessagesNewestFirst) + + if messages.count < physicalLimit || (coalesced.count >= limit && !usedFallbackReplacement) + { + return Array(coalesced.prefix(limit)) + } + guard let nextLimit = nextSearchPhysicalLimit(after: physicalLimit) else { + return Array(coalesced.prefix(limit)) + } + physicalLimit = nextLimit } - return messages } } + + private func searchMessage(_ message: Message, matches text: String, exact: Bool) -> Bool { + if exact { + return message.text.caseInsensitiveCompare(text) == .orderedSame + } + return message.text.range(of: text, options: [.caseInsensitive]) != nil + } + + private func nextSearchPhysicalLimit(after current: Int) -> Int? { + guard current > 0, current <= Int.max / 2 else { return nil } + return current * 2 + } + + private func searchMessagesNewestFirst(_ lhs: Message, _ rhs: Message) -> Bool { + if lhs.date == rhs.date { + return lhs.rowID > rhs.rowID + } + return lhs.date > rhs.date + } } diff --git a/Sources/IMsgCore/MessageStore+URLPreviews.swift b/Sources/IMsgCore/MessageStore+URLPreviews.swift new file mode 100644 index 0000000..c51df01 --- /dev/null +++ b/Sources/IMsgCore/MessageStore+URLPreviews.swift @@ -0,0 +1,144 @@ +import Foundation + +enum URLPreviewCoalescingFallback { + case suppress + case replace(Message) +} + +extension MessageStore { + static let urlPreviewBalloonBundleID = "com.apple.messages.URLBalloonProvider" + private static let urlPreviewCoalescingWindow: TimeInterval = 5 + + func coalesceURLPreviewMessages( + _ messages: [Message], + validateExistingCoalescence: ((Message, Message) throws -> Bool)? = nil, + fallbackForUnmatchedPreview: ((Message) throws -> URLPreviewCoalescingFallback?)? = nil, + fallbackReplacementUsed: (() -> Void)? = nil + ) throws -> [Message] { + guard !messages.isEmpty else { return messages } + + let rowOrdered = messages.enumerated().sorted { lhs, rhs in + lhs.element.rowID < rhs.element.rowID + } + var replacements: [Int: Message] = [:] + var suppressed = Set() + + for position in rowOrdered.indices { + let preview = rowOrdered[position] + guard isURLPreviewBalloon(preview.element), !suppressed.contains(preview.offset) else { + continue + } + + if let candidate = previousMessageInSameChat( + rowOrdered, + before: position, + suppressed: suppressed + ) { + let textMessage = replacements[candidate.offset] ?? candidate.element + let isValidExistingMatch = + try validateExistingCoalescence?(textMessage, preview.element) ?? true + if isValidExistingMatch + && canCoalesceURLPreview(textMessage: textMessage, previewMessage: preview.element) + { + replacements[candidate.offset] = textMessage.withURLPreview( + urlPreviewMetadata(from: preview.element) + ) + suppressed.insert(preview.offset) + continue + } + } + + guard let fallback = try fallbackForUnmatchedPreview?(preview.element) else { + continue + } + switch fallback { + case .suppress: + suppressed.insert(preview.offset) + case .replace(let textMessage): + fallbackReplacementUsed?() + replacements[preview.offset] = textMessage.withURLPreview( + urlPreviewMetadata(from: preview.element) + ) + } + } + + var result: [Message] = [] + result.reserveCapacity(messages.count - suppressed.count) + for (index, message) in messages.enumerated() where !suppressed.contains(index) { + result.append(replacements[index] ?? message) + } + return result + } + + func canCoalesceURLPreview(textMessage: Message, previewMessage: Message) -> Bool { + guard isURLPreviewBalloon(previewMessage) else { return false } + guard textMessage.balloonBundleID == nil else { return false } + guard textMessage.chatID == previewMessage.chatID else { return false } + guard textMessage.isFromMe == previewMessage.isFromMe else { return false } + guard textMessage.sender == previewMessage.sender else { return false } + if let textHandle = textMessage.handleID, let previewHandle = previewMessage.handleID, + textHandle != previewHandle + { + return false + } + guard previewMessage.rowID > textMessage.rowID else { return false } + let delta = previewMessage.date.timeIntervalSince(textMessage.date) + guard delta >= 0 && delta <= MessageStore.urlPreviewCoalescingWindow else { + return false + } + return textMessageContainsPreviewURL( + textMessage.text, + previewText: previewMessage.text + ) + } + + func isURLPreviewBalloon(_ message: Message) -> Bool { + message.balloonBundleID == MessageStore.urlPreviewBalloonBundleID + } + + private func previousMessageInSameChat( + _ chronological: [(offset: Int, element: Message)], + before position: Int, + suppressed: Set + ) -> (offset: Int, element: Message)? { + guard position > 0 else { return nil } + let preview = chronological[position].element + for index in stride(from: position - 1, through: 0, by: -1) { + let candidate = chronological[index] + guard !suppressed.contains(candidate.offset) else { continue } + guard !candidate.element.isReaction else { continue } + if candidate.element.chatID == preview.chatID { + return candidate + } + } + return nil + } + + private func textMessageContainsPreviewURL(_ text: String, previewText: String) -> Bool { + let preview = previewText.trimmingCharacters(in: .whitespacesAndNewlines) + guard isLikelyURLPreviewText(preview) else { return false } + let candidates = [ + preview, + preview.trimmingCharacters(in: CharacterSet(charactersIn: "/")), + ] + return candidates.contains { candidate in + !candidate.isEmpty && text.range(of: candidate, options: [.caseInsensitive]) != nil + } + } + + private func isLikelyURLPreviewText(_ text: String) -> Bool { + let lowercased = text.lowercased() + return lowercased.hasPrefix("http://") + || lowercased.hasPrefix("https://") + || lowercased.hasPrefix("www.") + } + + private func urlPreviewMetadata(from message: Message) -> Message.URLPreviewMetadata { + Message.URLPreviewMetadata( + rowID: message.rowID, + guid: message.guid, + balloonBundleID: message.balloonBundleID ?? MessageStore.urlPreviewBalloonBundleID, + date: message.date + ) + } +} diff --git a/Sources/IMsgCore/MessageWatcher.swift b/Sources/IMsgCore/MessageWatcher.swift index ea740ed..79b509d 100644 --- a/Sources/IMsgCore/MessageWatcher.swift +++ b/Sources/IMsgCore/MessageWatcher.swift @@ -236,13 +236,13 @@ private final class WatchState: @unchecked Sendable { private func poll() { if stopped { return } do { - let messages = try store.messagesAfter( + let batch = try store.messagesAfterBatch( afterRowID: cursor, chatID: chatID, limit: configuration.batchLimit, includeReactions: configuration.includeReactions ) - for message in messages { + for message in batch.messages { switch yieldDecision(for: message) { case .yield: break @@ -256,6 +256,9 @@ private final class WatchState: @unchecked Sendable { cursor = message.rowID } } + if batch.maxScannedRowID > cursor { + cursor = batch.maxScannedRowID + } } catch { continuation.finish(throwing: error) } diff --git a/Sources/IMsgCore/Models.swift b/Sources/IMsgCore/Models.swift index 8d6cc76..0c1d41f 100644 --- a/Sources/IMsgCore/Models.swift +++ b/Sources/IMsgCore/Models.swift @@ -322,6 +322,10 @@ public struct Message: Sendable, Equatable { /// Native Messages Polls metadata when the row is a Polls extension balloon /// or a Polls vote update. public let poll: MessagePollEvent? + /// Metadata for an Apple URL preview balloon row that was folded into this + /// message. The message itself still uses the originating text row's id, + /// guid, text, and timestamp. + public let urlPreview: URLPreviewMetadata? // Reaction metadata (populated when message is a reaction event) /// Whether this message is a reaction event (tapback add/remove) @@ -346,6 +350,7 @@ public struct Message: Sendable, Equatable { guid: String = "", routing: RoutingMetadata = RoutingMetadata(), balloonBundleID: String? = nil, + urlPreview: URLPreviewMetadata? = nil, reaction: ReactionMetadata = ReactionMetadata(), poll: MessagePollEvent? = nil ) { @@ -367,6 +372,7 @@ public struct Message: Sendable, Equatable { self.destinationCallerID = routing.destinationCallerID self.balloonBundleID = balloonBundleID self.poll = poll + self.urlPreview = urlPreview self.isReaction = reaction.isReaction self.reactionType = reaction.reactionType self.isReactionAdd = reaction.isReactionAdd @@ -389,6 +395,7 @@ public struct Message: Sendable, Equatable { threadOriginatorPart: String? = nil, destinationCallerID: String? = nil, balloonBundleID: String? = nil, + urlPreview: URLPreviewMetadata? = nil, replyToText: String? = nil, replyToSender: String? = nil, isReaction: Bool = false, @@ -417,6 +424,7 @@ public struct Message: Sendable, Equatable { replyToSender: replyToSender ), balloonBundleID: balloonBundleID, + urlPreview: urlPreview, reaction: ReactionMetadata( isReaction: isReaction, reactionType: reactionType, @@ -426,66 +434,7 @@ public struct Message: Sendable, Equatable { poll: poll ) } -} - -public enum MessageSendState: String, Sendable, Equatable { - case pending - case sent - case delivered - case failed -} -public struct MessageSendStatus: Sendable, Equatable { - public let rowID: Int64 - public let guid: String - public let service: String - public let error: Int - public let dateDelivered: Date? - public let dateRead: Date? - public let isSent: Bool - public let isDelivered: Bool - public let isFinished: Bool - public let isDelayed: Bool - public let isPrepared: Bool - public let isPendingSatelliteSend: Bool - public let wasDowngraded: Bool - - public var state: MessageSendState { - if error != 0 { return .failed } - if isDelivered || dateDelivered != nil { return .delivered } - if isSent { return .sent } - return .pending - } - - public init( - rowID: Int64, - guid: String, - service: String, - error: Int, - dateDelivered: Date?, - dateRead: Date?, - isSent: Bool, - isDelivered: Bool, - isFinished: Bool, - isDelayed: Bool, - isPrepared: Bool, - isPendingSatelliteSend: Bool, - wasDowngraded: Bool - ) { - self.rowID = rowID - self.guid = guid - self.service = service - self.error = error - self.dateDelivered = dateDelivered - self.dateRead = dateRead - self.isSent = isSent - self.isDelivered = isDelivered - self.isFinished = isFinished - self.isDelayed = isDelayed - self.isPrepared = isPrepared - self.isPendingSatelliteSend = isPendingSatelliteSend - self.wasDowngraded = wasDowngraded - } } public struct AttachmentMeta: Sendable, Equatable { diff --git a/Sources/imsg/OutputModels.swift b/Sources/imsg/OutputModels.swift index 7fa71f9..1bdf360 100644 --- a/Sources/imsg/OutputModels.swift +++ b/Sources/imsg/OutputModels.swift @@ -54,6 +54,27 @@ struct ChatPayload: Codable { } struct MessagePayload: Codable { + struct URLPreviewPayload: Codable { + let id: Int64 + let guid: String + let balloonBundleID: String + let createdAt: String + + init(preview: Message.URLPreviewMetadata) { + self.id = preview.rowID + self.guid = preview.guid + self.balloonBundleID = preview.balloonBundleID + self.createdAt = CLIISO8601.format(preview.date) + } + + enum CodingKeys: String, CodingKey { + case id + case guid + case balloonBundleID = "balloon_bundle_id" + case createdAt = "created_at" + } + } + let id: Int64 let chatID: Int64 let guid: String @@ -78,6 +99,7 @@ struct MessagePayload: Codable { /// messages received on a secondary phone number registered with the same Apple ID. let destinationCallerID: String? let balloonBundleID: String? + let urlPreview: URLPreviewPayload? let poll: MessagePollEvent? // Reaction event metadata (populated when this message is a reaction event) @@ -113,6 +135,7 @@ struct MessagePayload: Codable { } self.destinationCallerID = message.destinationCallerID self.balloonBundleID = message.balloonBundleID + self.urlPreview = message.urlPreview.map { URLPreviewPayload(preview: $0) } self.poll = message.poll // Reaction event metadata @@ -149,6 +172,7 @@ struct MessagePayload: Codable { case reactions case destinationCallerID = "destination_caller_id" case balloonBundleID = "balloon_bundle_id" + case urlPreview = "url_preview" case poll case isReaction = "is_reaction" case reactionType = "reaction_type" diff --git a/Tests/IMsgCoreTests/MessageStoreURLPreviewLimitTests.swift b/Tests/IMsgCoreTests/MessageStoreURLPreviewLimitTests.swift new file mode 100644 index 0000000..4e1887f --- /dev/null +++ b/Tests/IMsgCoreTests/MessageStoreURLPreviewLimitTests.swift @@ -0,0 +1,241 @@ +import Foundation +import Testing + +@testable import IMsgCore + +@Test +func messagesByChatFillsLogicalLimitAfterCoalescingPreview() throws { + let db = try makeURLPreviewTestDB() + let now = Date() + try db.run("INSERT INTO handle(ROWID, id) VALUES (1, '+123')") + try insertURLPreviewTestMessage( + db, + rowID: 1, + text: "Dump https://example.com", + guid: "text-guid", + date: now + ) + try insertURLPreviewTestMessage( + db, + rowID: 2, + text: "https://example.com", + guid: "preview-guid", + balloonBundleID: MessageStore.urlPreviewBalloonBundleID, + date: now.addingTimeInterval(1) + ) + try insertURLPreviewTestMessage( + db, + rowID: 3, + text: "older", + guid: "older-guid", + date: now.addingTimeInterval(-1) + ) + + let store = try MessageStore(connection: db, path: ":memory:") + let messages = try store.messages(chatID: 1, limit: 2) + + #expect(messages.map(\.rowID) == [1, 3]) + #expect(messages.first?.urlPreview?.rowID == 2) +} + +@Test +func messagesByChatContinuesPastFallbackReplacementLimitBoundary() throws { + let db = try makeURLPreviewTestDB() + let now = Date() + try db.run("INSERT INTO handle(ROWID, id) VALUES (1, '+123')") + try insertURLPreviewTestMessage( + db, + rowID: 1, + text: "Dump https://example.com", + guid: "text-guid", + date: now + ) + try insertURLPreviewTestMessage( + db, + rowID: 2, + text: "https://example.com", + guid: "preview-guid", + balloonBundleID: MessageStore.urlPreviewBalloonBundleID, + date: now.addingTimeInterval(4) + ) + try insertURLPreviewTestMessage( + db, + rowID: 3, + text: "newer visible", + guid: "newer-guid", + date: now.addingTimeInterval(3) + ) + try insertURLPreviewTestMessage( + db, + rowID: 4, + text: "second visible", + guid: "second-guid", + date: now.addingTimeInterval(2) + ) + + let store = try MessageStore(connection: db, path: ":memory:") + let messages = try store.messages(chatID: 1, limit: 2) + + #expect(messages.map(\.rowID) == [3, 4]) +} + +@Test +func messagesAfterCoalescesRepeatedURLSendsBeforeDedupingStandalonePreviews() throws { + let db = try makeURLPreviewTestDB() + let now = Date() + try db.run("INSERT INTO handle(ROWID, id) VALUES (1, '+123')") + try insertURLPreviewTestMessage( + db, + rowID: 1, + text: "First https://example.com", + guid: "first-text-guid", + date: now + ) + try insertURLPreviewTestMessage( + db, + rowID: 2, + text: "https://example.com", + guid: "first-preview-guid", + balloonBundleID: MessageStore.urlPreviewBalloonBundleID, + date: now.addingTimeInterval(1) + ) + try insertURLPreviewTestMessage( + db, + rowID: 3, + text: "Again https://example.com", + guid: "second-text-guid", + date: now.addingTimeInterval(30) + ) + try insertURLPreviewTestMessage( + db, + rowID: 4, + text: "https://example.com", + guid: "second-preview-guid", + balloonBundleID: MessageStore.urlPreviewBalloonBundleID, + date: now.addingTimeInterval(31) + ) + + let store = try MessageStore(connection: db, path: ":memory:") + let messages = try store.messagesAfter(afterRowID: 0, chatID: 1, limit: 10) + + #expect(messages.map(\.rowID) == [1, 3]) + #expect(messages.map { $0.urlPreview?.rowID } == [2, 4]) +} + +@Test +func searchMessagesFillsLogicalLimitAfterCoalescingPreview() throws { + let db = try makeURLPreviewTestDB() + let now = Date() + try db.run("INSERT INTO handle(ROWID, id) VALUES (1, '+123')") + try insertURLPreviewTestMessage( + db, + rowID: 1, + text: "Dump https://example.com", + guid: "text-guid", + date: now + ) + try insertURLPreviewTestMessage( + db, + rowID: 2, + text: "https://example.com", + guid: "preview-guid", + balloonBundleID: MessageStore.urlPreviewBalloonBundleID, + date: now.addingTimeInterval(1) + ) + try insertURLPreviewTestMessage( + db, + rowID: 3, + text: "older https://example.com", + guid: "older-guid", + date: now.addingTimeInterval(-1) + ) + + let store = try MessageStore(connection: db, path: ":memory:") + let messages = try store.searchMessages(query: "example.com", match: "contains", limit: 2) + + #expect(messages.map(\.rowID) == [1, 3]) + #expect(messages.first?.urlPreview?.rowID == 2) +} + +@Test +func searchMessagesContinuesPastFallbackReplacementLimitBoundary() throws { + let db = try makeURLPreviewTestDB() + let now = Date() + try db.run("INSERT INTO handle(ROWID, id) VALUES (1, '+123')") + try insertURLPreviewTestMessage( + db, + rowID: 1, + chatID: 1, + text: "Dump https://example.com", + guid: "text-guid", + date: now + ) + try insertURLPreviewTestMessage( + db, + rowID: 2, + chatID: 1, + text: "https://example.com", + guid: "preview-guid", + balloonBundleID: MessageStore.urlPreviewBalloonBundleID, + date: now.addingTimeInterval(4) + ) + try insertURLPreviewTestMessage( + db, + rowID: 3, + chatID: 2, + text: "newer https://example.com", + guid: "newer-guid", + date: now.addingTimeInterval(3) + ) + try insertURLPreviewTestMessage( + db, + rowID: 4, + chatID: 3, + text: "second https://example.com", + guid: "second-guid", + date: now.addingTimeInterval(2) + ) + + let store = try MessageStore(connection: db, path: ":memory:") + let messages = try store.searchMessages(query: "example.com", match: "contains", limit: 2) + + #expect(messages.map(\.rowID) == [3, 4]) +} + +@Test +func searchMessagesSortsReplacedPreviewByLogicalMessageDate() throws { + let db = try makeURLPreviewTestDB() + let now = Date() + try db.run("INSERT INTO handle(ROWID, id) VALUES (1, '+123')") + try insertURLPreviewTestMessage( + db, + rowID: 1, + chatID: 1, + text: "Dump https://example.com", + guid: "text-guid", + date: now + ) + try insertURLPreviewTestMessage( + db, + rowID: 2, + chatID: 1, + text: "https://example.com", + guid: "preview-guid", + balloonBundleID: MessageStore.urlPreviewBalloonBundleID, + date: now.addingTimeInterval(2) + ) + try insertURLPreviewTestMessage( + db, + rowID: 3, + chatID: 2, + text: "middle https://example.com", + guid: "middle-guid", + date: now.addingTimeInterval(1) + ) + + let store = try MessageStore(connection: db, path: ":memory:") + let messages = try store.searchMessages(query: "example.com", match: "contains", limit: 2) + + #expect(messages.map(\.rowID) == [3, 1]) + #expect(messages.last?.urlPreview?.rowID == 2) +} diff --git a/Tests/IMsgCoreTests/MessageStoreURLPreviewTests.swift b/Tests/IMsgCoreTests/MessageStoreURLPreviewTests.swift new file mode 100644 index 0000000..75c3898 --- /dev/null +++ b/Tests/IMsgCoreTests/MessageStoreURLPreviewTests.swift @@ -0,0 +1,490 @@ +import Foundation +import SQLite +import Testing + +@testable import IMsgCore + +func makeURLPreviewTestDB() throws -> Connection { + let db = try Connection(.inMemory) + try db.execute( + """ + CREATE TABLE message ( + ROWID INTEGER PRIMARY KEY, + handle_id INTEGER, + text TEXT, + guid TEXT, + associated_message_guid TEXT, + associated_message_type INTEGER, + balloon_bundle_id TEXT, + date INTEGER, + is_from_me INTEGER, + service TEXT + ); + """ + ) + try db.execute("CREATE TABLE handle (ROWID INTEGER PRIMARY KEY, id TEXT);") + try db.execute("CREATE TABLE chat_message_join (chat_id INTEGER, message_id INTEGER);") + try db.execute( + "CREATE TABLE message_attachment_join (message_id INTEGER, attachment_id INTEGER);") + return db +} + +func insertURLPreviewTestMessage( + _ db: Connection, + rowID: Int64, + chatID: Int64 = 1, + handleID: Int64 = 1, + text: String, + guid: String, + associatedMessageGUID: String? = nil, + associatedMessageType: Int? = nil, + balloonBundleID: String? = nil, + date: Date, + isFromMe: Bool = false +) throws { + try db.run( + """ + INSERT INTO message( + ROWID, handle_id, text, guid, associated_message_guid, associated_message_type, + balloon_bundle_id, date, is_from_me, service + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'iMessage') + """, + rowID, + handleID, + text, + guid, + associatedMessageGUID, + associatedMessageType, + balloonBundleID, + TestDatabase.appleEpoch(date), + isFromMe ? 1 : 0 + ) + try db.run( + "INSERT INTO chat_message_join(chat_id, message_id) VALUES (?, ?)", + chatID, + rowID + ) +} + +@Test +func messagesAfterSkipsPreviewOnlyBatchForPublicCursorlessAPI() throws { + let db = try makeURLPreviewTestDB() + let now = Date() + try db.run("INSERT INTO handle(ROWID, id) VALUES (1, '+123')") + try insertURLPreviewTestMessage( + db, + rowID: 1, + text: "Dump https://example.com", + guid: "text-guid", + date: now + ) + try insertURLPreviewTestMessage( + db, + rowID: 2, + text: "https://example.com", + guid: "preview-guid", + balloonBundleID: MessageStore.urlPreviewBalloonBundleID, + date: now.addingTimeInterval(1) + ) + try insertURLPreviewTestMessage( + db, + rowID: 3, + text: "after", + guid: "after-guid", + date: now.addingTimeInterval(2) + ) + + let store = try MessageStore(connection: db, path: ":memory:") + let messages = try store.messagesAfter(afterRowID: 1, chatID: 1, limit: 1) + + #expect(messages.map(\.rowID) == [3]) +} + +@Test +func messagesByChatCoalescesURLPreviewSplitSend() throws { + let db = try makeURLPreviewTestDB() + let now = Date() + try db.run("INSERT INTO handle(ROWID, id) VALUES (1, '+123')") + try insertURLPreviewTestMessage( + db, + rowID: 1, + text: "Dump https://example.com", + guid: "text-guid", + date: now + ) + try insertURLPreviewTestMessage( + db, + rowID: 2, + text: "https://example.com", + guid: "preview-guid", + balloonBundleID: MessageStore.urlPreviewBalloonBundleID, + date: now.addingTimeInterval(1) + ) + + let store = try MessageStore(connection: db, path: ":memory:") + let messages = try store.messages(chatID: 1, limit: 10) + + #expect(messages.map(\.rowID) == [1]) + #expect(messages.first?.guid == "text-guid") + #expect(messages.first?.text == "Dump https://example.com") + #expect(messages.first?.balloonBundleID == nil) + #expect(messages.first?.urlPreview?.rowID == 2) + #expect(messages.first?.urlPreview?.guid == "preview-guid") + #expect(messages.first?.urlPreview?.balloonBundleID == MessageStore.urlPreviewBalloonBundleID) +} + +@Test +func messagesByChatUsesRowOrderForURLPreviewPredecessor() throws { + let db = try makeURLPreviewTestDB() + let now = Date() + try db.run("INSERT INTO handle(ROWID, id) VALUES (1, '+123')") + try insertURLPreviewTestMessage( + db, + rowID: 1, + text: "Dump https://example.com", + guid: "text-guid", + date: now + ) + try insertURLPreviewTestMessage( + db, + rowID: 3, + text: "https://example.com", + guid: "preview-guid", + balloonBundleID: MessageStore.urlPreviewBalloonBundleID, + date: now.addingTimeInterval(1) + ) + try insertURLPreviewTestMessage( + db, + rowID: 4, + text: "timestamp between text and preview, inserted after preview", + guid: "late-row-guid", + date: now.addingTimeInterval(0.5) + ) + + let store = try MessageStore(connection: db, path: ":memory:") + let messages = try store.messages(chatID: 1, limit: 10) + + #expect(messages.map(\.rowID) == [4, 1]) + #expect(messages.last?.urlPreview?.rowID == 3) +} + +@Test +func messagesAfterCoalescesPreviewPastReactionCandidate() throws { + let db = try makeURLPreviewTestDB() + let now = Date() + try db.run("INSERT INTO handle(ROWID, id) VALUES (1, '+123')") + try insertURLPreviewTestMessage( + db, + rowID: 1, + text: "Dump https://example.com", + guid: "text-guid", + date: now + ) + try insertURLPreviewTestMessage( + db, + rowID: 2, + text: "Loved https://example.com", + guid: "reaction-guid", + associatedMessageGUID: "text-guid", + associatedMessageType: 2000, + date: now.addingTimeInterval(0.5) + ) + try insertURLPreviewTestMessage( + db, + rowID: 3, + text: "https://example.com", + guid: "preview-guid", + balloonBundleID: MessageStore.urlPreviewBalloonBundleID, + date: now.addingTimeInterval(1) + ) + + let store = try MessageStore(connection: db, path: ":memory:") + let messages = try store.messagesAfter( + afterRowID: 0, + chatID: 1, + limit: 10, + includeReactions: true + ) + + #expect(messages.map(\.rowID) == [1, 2]) + #expect(messages.first?.urlPreview?.rowID == 3) + #expect(messages.last?.isReaction == true) +} + +@Test +func messagesAfterCoalescesURLPreviewSplitSend() throws { + let db = try makeURLPreviewTestDB() + let now = Date() + try db.run("INSERT INTO handle(ROWID, id) VALUES (1, '+123')") + try insertURLPreviewTestMessage( + db, + rowID: 1, + text: "Dump https://example.com", + guid: "text-guid", + date: now + ) + try insertURLPreviewTestMessage( + db, + rowID: 2, + text: "https://example.com", + guid: "preview-guid", + balloonBundleID: MessageStore.urlPreviewBalloonBundleID, + date: now.addingTimeInterval(1) + ) + + let store = try MessageStore(connection: db, path: ":memory:") + let messages = try store.messagesAfter(afterRowID: 0, chatID: 1, limit: 10) + + #expect(messages.map(\.rowID) == [1]) + #expect(messages.first?.urlPreview?.rowID == 2) +} + +func messagesAfterSuppressesLateURLPreviewWhenTextWasAlreadySeen() throws { + let db = try makeURLPreviewTestDB() + let now = Date() + try db.run("INSERT INTO handle(ROWID, id) VALUES (1, '+123')") + try insertURLPreviewTestMessage( + db, + rowID: 1, + text: "Dump https://example.com", + guid: "text-guid", + date: now + ) + try insertURLPreviewTestMessage( + db, + rowID: 2, + text: "https://example.com", + guid: "preview-guid", + balloonBundleID: MessageStore.urlPreviewBalloonBundleID, + date: now.addingTimeInterval(1) + ) + + let store = try MessageStore(connection: db, path: ":memory:") + let messages = try store.messagesAfter(afterRowID: 1, chatID: 1, limit: 10) + + #expect(messages.isEmpty) +} + +@Test +func messagesAfterBatchAdvancesAcrossSuppressedLateURLPreview() throws { + let db = try makeURLPreviewTestDB() + let now = Date() + try db.run("INSERT INTO handle(ROWID, id) VALUES (1, '+123')") + try insertURLPreviewTestMessage( + db, + rowID: 1, + text: "Dump https://example.com", + guid: "text-guid", + date: now + ) + try insertURLPreviewTestMessage( + db, + rowID: 2, + text: "https://example.com", + guid: "preview-guid", + balloonBundleID: MessageStore.urlPreviewBalloonBundleID, + date: now.addingTimeInterval(1) + ) + try insertURLPreviewTestMessage( + db, + rowID: 3, + text: "after", + guid: "after-guid", + date: now.addingTimeInterval(2) + ) + + let store = try MessageStore(connection: db, path: ":memory:") + let previewBatch = try store.messagesAfterBatch( + afterRowID: 1, + chatID: 1, + limit: 1, + includeReactions: false + ) + #expect(previewBatch.messages.isEmpty) + #expect(previewBatch.maxScannedRowID == 2) + + let nextBatch = try store.messagesAfterBatch( + afterRowID: previewBatch.maxScannedRowID, + chatID: 1, + limit: 1, + includeReactions: false + ) + #expect(nextBatch.messages.map(\.rowID) == [3]) +} + +@Test +func searchMessagesCoalescesURLPreviewSplitSend() throws { + let db = try makeURLPreviewTestDB() + let now = Date() + try db.run("INSERT INTO handle(ROWID, id) VALUES (1, '+123')") + try insertURLPreviewTestMessage( + db, + rowID: 1, + text: "Dump https://example.com", + guid: "text-guid", + date: now + ) + try insertURLPreviewTestMessage( + db, + rowID: 2, + text: "https://example.com", + guid: "preview-guid", + balloonBundleID: MessageStore.urlPreviewBalloonBundleID, + date: now.addingTimeInterval(1) + ) + + let store = try MessageStore(connection: db, path: ":memory:") + let messages = try store.searchMessages(query: "example.com", match: "contains", limit: 10) + + #expect(messages.map(\.rowID) == [1]) + #expect(messages.first?.urlPreview?.guid == "preview-guid") +} + +@Test +func exactSearchDoesNotReplaceMatchingPreviewWithNonmatchingText() throws { + let db = try makeURLPreviewTestDB() + let now = Date() + try db.run("INSERT INTO handle(ROWID, id) VALUES (1, '+123')") + try insertURLPreviewTestMessage( + db, + rowID: 1, + text: "Dump https://example.com", + guid: "text-guid", + date: now + ) + try insertURLPreviewTestMessage( + db, + rowID: 2, + text: "https://example.com", + guid: "preview-guid", + balloonBundleID: MessageStore.urlPreviewBalloonBundleID, + date: now.addingTimeInterval(1) + ) + + let store = try MessageStore(connection: db, path: ":memory:") + let messages = try store.searchMessages(query: "https://example.com", match: "exact", limit: 10) + + #expect(messages.map(\.rowID) == [2]) + #expect(messages.first?.balloonBundleID == MessageStore.urlPreviewBalloonBundleID) +} + +@Test +func historyDateFilterDoesNotReplacePreviewWithFilteredOutText() throws { + let db = try makeURLPreviewTestDB() + let now = Date() + try db.run("INSERT INTO handle(ROWID, id) VALUES (1, '+123')") + try insertURLPreviewTestMessage( + db, + rowID: 1, + text: "Dump https://example.com", + guid: "text-guid", + date: now + ) + try insertURLPreviewTestMessage( + db, + rowID: 2, + text: "https://example.com", + guid: "preview-guid", + balloonBundleID: MessageStore.urlPreviewBalloonBundleID, + date: now.addingTimeInterval(1) + ) + + let store = try MessageStore(connection: db, path: ":memory:") + let filter = MessageFilter(startDate: now.addingTimeInterval(0.5)) + let messages = try store.messages(chatID: 1, limit: 10, filter: filter) + + #expect(messages.map(\.rowID) == [2]) + #expect(messages.first?.balloonBundleID == MessageStore.urlPreviewBalloonBundleID) +} + +@Test +func searchCoalescingRequiresActualPreviousChatRow() throws { + let db = try makeURLPreviewTestDB() + let now = Date() + try db.run("INSERT INTO handle(ROWID, id) VALUES (1, '+123')") + try insertURLPreviewTestMessage( + db, + rowID: 1, + text: "Dump https://example.com", + guid: "text-guid", + date: now + ) + try insertURLPreviewTestMessage( + db, + rowID: 2, + text: "intervening", + guid: "middle-guid", + date: now.addingTimeInterval(0.5) + ) + try insertURLPreviewTestMessage( + db, + rowID: 3, + text: "https://example.com", + guid: "preview-guid", + balloonBundleID: MessageStore.urlPreviewBalloonBundleID, + date: now.addingTimeInterval(1) + ) + + let store = try MessageStore(connection: db, path: ":memory:") + let messages = try store.searchMessages(query: "example.com", match: "contains", limit: 10) + + #expect(messages.map(\.rowID) == [3, 1]) + #expect(messages.first?.balloonBundleID == MessageStore.urlPreviewBalloonBundleID) + #expect(messages.last?.urlPreview == nil) +} + +@Test +func nonURLBalloonMessagesAreNotCoalesced() throws { + let db = try makeURLPreviewTestDB() + let now = Date() + try db.run("INSERT INTO handle(ROWID, id) VALUES (1, '+123')") + try insertURLPreviewTestMessage( + db, + rowID: 1, + text: "Poll https://example.com", + guid: "text-guid", + date: now + ) + try insertURLPreviewTestMessage( + db, + rowID: 2, + text: "https://example.com", + guid: "poll-guid", + balloonBundleID: MessagePollDecoder.pollsBundleIdentifier, + date: now.addingTimeInterval(1) + ) + + let store = try MessageStore(connection: db, path: ":memory:") + let messages = try store.messagesAfter(afterRowID: 0, chatID: 1, limit: 10) + + #expect(messages.map(\.rowID) == [1, 2]) + #expect(messages[1].balloonBundleID == MessagePollDecoder.pollsBundleIdentifier) +} + +@Test +func separateSameSenderTextMessagesAreNotCoalesced() throws { + let db = try makeURLPreviewTestDB() + let now = Date() + try db.run("INSERT INTO handle(ROWID, id) VALUES (1, '+123')") + try insertURLPreviewTestMessage( + db, + rowID: 1, + text: "Dump https://example.com", + guid: "first-guid", + date: now + ) + try insertURLPreviewTestMessage( + db, + rowID: 2, + text: "https://example.com", + guid: "second-guid", + date: now.addingTimeInterval(1) + ) + + let store = try MessageStore(connection: db, path: ":memory:") + let messages = try store.messagesAfter(afterRowID: 0, chatID: 1, limit: 10) + + #expect(messages.map(\.rowID) == [1, 2]) + #expect(messages.allSatisfy { $0.urlPreview == nil }) +} diff --git a/Tests/imsgTests/RPCPayloadsTests.swift b/Tests/imsgTests/RPCPayloadsTests.swift index fe9fb21..3961770 100644 --- a/Tests/imsgTests/RPCPayloadsTests.swift +++ b/Tests/imsgTests/RPCPayloadsTests.swift @@ -114,6 +114,42 @@ func messagePayloadIncludesChatFields() throws { == ReactionType.like.emoji) } +@Test +func messagePayloadIncludesCoalescedURLPreview() throws { + let message = Message( + rowID: 5, + chatID: 10, + sender: "+123", + text: "Dump https://example.com", + date: Date(timeIntervalSince1970: 1), + isFromMe: false, + service: "iMessage", + handleID: nil, + attachmentsCount: 0, + guid: "text-guid", + urlPreview: Message.URLPreviewMetadata( + rowID: 6, + guid: "preview-guid", + balloonBundleID: MessageStore.urlPreviewBalloonBundleID, + date: Date(timeIntervalSince1970: 2) + ) + ) + + let payload = try messagePayload( + message: message, + chatInfo: nil, + participants: [], + attachments: [], + reactions: [] + ) + + let preview = payload["url_preview"] as? [String: Any] + #expect(preview?["id"] as? Int64 == 6) + #expect(preview?["guid"] as? String == "preview-guid") + #expect(preview?["balloon_bundle_id"] as? String == MessageStore.urlPreviewBalloonBundleID) + #expect(preview?["created_at"] as? String == "1970-01-01T00:00:02.000Z") +} + @Test func messagePayloadIncludesPollObject() throws { let poll = MessagePollEvent( diff --git a/docs/json.md b/docs/json.md index 944de19..14130d1 100644 --- a/docs/json.md +++ b/docs/json.md @@ -50,6 +50,7 @@ Returned by `imsg history`, `imsg watch`, and the JSON-RPC `messages.history` an | `reply_to_guid` | string | When set, this message is an inline reply to that GUID. | | `destination_caller_id` | string | Outgoing only — which of your numbers Messages routed through. | | `balloon_bundle_id` | string | Raw Messages `message.balloon_bundle_id`, when present. URL preview rows use `com.apple.messages.URLBalloonProvider`, which lets consumers recognize link-preview payload rows without inferring from message text. | +| `url_preview` | object | Present when imsg folds an Apple URL-preview balloon row into its originating text row. The outer message keeps the text row's `id`, `guid`, `text`, and `created_at`. | | `sender` | string | Raw handle. Empty for some self-sent messages. | | `sender_name` | string | Resolved Contacts name when permission granted. | | `is_from_me` | bool | True for outbound. | @@ -59,6 +60,19 @@ Returned by `imsg history`, `imsg watch`, and the JSON-RPC `messages.history` an | `thread_originator_guid` | string | For inline-reply threads. | | `poll` | object | Present for native Apple Messages Polls creation and vote rows. See below. | +### URL preview coalescing + +Messages may store a link send as two rows: the user's text row and a later `com.apple.messages.URLBalloonProvider` preview row. `history`, `search`, `watch`, `messages.history`, and `watch.subscribe` coalesce those rows into one logical message when the preview immediately follows a same-chat/same-sender text row containing the preview URL. In batch reads the coalesced message includes: + +| Field | Type | Notes | +|-------|------|-------| +| `id` | int | Preview rowid that was folded into the outer text message. | +| `guid` | string | Preview row GUID. | +| `balloon_bundle_id` | string | `com.apple.messages.URLBalloonProvider`. | +| `created_at` | ISO8601 | Preview row timestamp. | + +Live watch calls do not delay the text message waiting for a preview. If the preview row arrives in a later poll after the text row was already emitted, imsg suppresses the preview row so consumers still receive one notification. + ### Reaction extensions Present on `imsg watch --reactions` events: