From 774c88ed973406d75683f4f28b940db4fc946d13 Mon Sep 17 00:00:00 2001 From: jack <212554440+jackjackbits@users.noreply.github.com> Date: Wed, 23 Sep 2026 20:19:59 +0200 Subject: [PATCH] Cap packet payload size per type and bound sync stores by bytes (#1719) The decoder accepted a payload of any type up to the framed-file ceiling, compressed or not, and the stores holding decoded packets evicted by count only. - New PacketPayloadLimits (BitFoundation): per-type decoded payload caps, enforced in BinaryProtocol.decodeCore for compressed and uncompressed payloads. Only fileTransfer and noiseEncrypted keep the framed-file cap; every other type is capped at >= 2x the largest payload current iOS and Android encoders produce. - The unreachable 50,000:1 ratio guard becomes DEFLATE's 1032:1 bound. - Fragment reassembly is limited by the claimed type's frame ceiling, and a reassembled packet of a different type is dropped. - Gossip PacketStores, archive restore and the directed relay spool get byte budgets alongside their count caps. - Broadcast file transfers are validated before being tracked for sync. - Public mesh messages larger than a v1 frame are not delivered. Receive-side only; senders and the wire format are unchanged. Also: a re-delivered fragment replaces its index's bytes in the assembly size projection instead of counting twice, and a test-harness race in GossipSyncManagerTests.concurrentPacketIntakeAndSyncRequest is fixed. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../Services/BLE/BLEDirectedRelaySpool.swift | 83 ++++-- .../Services/BLE/BLEFileTransferHandler.swift | 65 +++-- .../BLE/BLEFragmentAssemblyBuffer.swift | 24 +- bitchat/Services/BLE/BLEFragmentHandler.swift | 11 +- .../BLE/BLEPublicMessageHandler.swift | 12 + bitchat/Services/TransportConfig.swift | 8 + bitchat/Sync/GossipSyncManager.swift | 69 +++-- bitchatTests/GossipSyncManagerTests.swift | 89 +++++- .../PacketPayloadCapInteropTests.swift | 257 ++++++++++++++++++ .../Services/BLEDirectedRelaySpoolTests.swift | 91 ++++++- .../BLEFileTransferHandlerTests.swift | 41 ++- .../BLEFragmentAssemblyBufferTests.swift | 80 +++++- .../Services/BLEFragmentHandlerTests.swift | 110 +++++++- .../BLEPublicMessageHandlerTests.swift | 22 ++ .../BitFoundation/BinaryProtocol.swift | 23 +- .../BitFoundation/PacketPayloadLimits.swift | 106 ++++++++ .../BinaryProtocolTests.swift | 8 +- .../PacketPayloadLimitsTests.swift | 164 +++++++++++ 18 files changed, 1164 insertions(+), 99 deletions(-) create mode 100644 bitchatTests/Protocols/PacketPayloadCapInteropTests.swift create mode 100644 localPackages/BitFoundation/Sources/BitFoundation/PacketPayloadLimits.swift create mode 100644 localPackages/BitFoundation/Tests/BitFoundationTests/PacketPayloadLimitsTests.swift diff --git a/bitchat/Services/BLE/BLEDirectedRelaySpool.swift b/bitchat/Services/BLE/BLEDirectedRelaySpool.swift index 9f890eb6..d6149a3f 100644 --- a/bitchat/Services/BLE/BLEDirectedRelaySpool.swift +++ b/bitchat/Services/BLE/BLEDirectedRelaySpool.swift @@ -6,22 +6,53 @@ struct BLEDirectedRelaySpoolEntry { let packet: BitchatPacket } +/// Directed packets held at a relay while no onward link exists. +/// +/// Bounded by count and by total payload bytes, evicting oldest-first: a +/// spooled noiseEncrypted packet keeps the framed-file decompression cap, so +/// a ~1 KB compressed frame can decode to ~1 MB, and a window-only bound let +/// a stream of such frames pin gigabytes before the first entry expired. struct BLEDirectedRelaySpool { + private struct Key: Hashable { + let recipient: PeerID + // periphery:ignore - read only via the synthesized Hashable + // conformance (dictionary-key identity), which the indexer + // cannot attribute. + let messageID: String + } + private struct StoredPacket { let packet: BitchatPacket let enqueuedAt: Date } - private var packetsByRecipient: [PeerID: [String: StoredPacket]] = [:] + private let capacity: Int + private let byteBudget: Int + private var packets: [Key: StoredPacket] = [:] + /// Enqueue order, oldest first: the eviction order, and the drain order. + private var order: [Key] = [] + /// Sum of spooled payload sizes, maintained on every insert/remove. + private var payloadBytes = 0 + + init( + capacity: Int = TransportConfig.bleDirectedSpoolCapacity, + byteBudget: Int = TransportConfig.bleDirectedSpoolByteBudget + ) { + self.capacity = capacity + self.byteBudget = byteBudget + } var isEmpty: Bool { - packetsByRecipient.isEmpty + packets.isEmpty } var count: Int { - packetsByRecipient.values.reduce(0) { $0 + $1.count } + packets.count } + /// Spools `packet` unless it is already spooled for `recipient` or is + /// larger than the whole byte budget, then evicts oldest-first until + /// both the count and the byte budget hold. @discardableResult mutating func enqueue( packet: BitchatPacket, @@ -29,43 +60,55 @@ struct BLEDirectedRelaySpool { messageID: String, enqueuedAt: Date ) -> Bool { - var packets = packetsByRecipient[recipient] ?? [:] - guard packets[messageID] == nil else { + let key = Key(recipient: recipient, messageID: messageID) + guard packets[key] == nil, packet.payload.count <= byteBudget else { return false } - packets[messageID] = StoredPacket(packet: packet, enqueuedAt: enqueuedAt) - packetsByRecipient[recipient] = packets + packets[key] = StoredPacket(packet: packet, enqueuedAt: enqueuedAt) + order.append(key) + payloadBytes += packet.payload.count + while order.count > capacity || payloadBytes > byteBudget { + let victim = order.removeFirst() + if let evicted = packets.removeValue(forKey: victim) { + payloadBytes -= evicted.packet.payload.count + } + } return true } mutating func drainUnexpired(now: Date, window: TimeInterval) -> [BLEDirectedRelaySpoolEntry] { var entries: [BLEDirectedRelaySpoolEntry] = [] - for (recipient, packets) in packetsByRecipient { - for stored in packets.values where now.timeIntervalSince(stored.enqueuedAt) <= window { - entries.append(BLEDirectedRelaySpoolEntry(recipient: recipient, packet: stored.packet)) - } + for key in order { + guard let stored = packets[key], + now.timeIntervalSince(stored.enqueuedAt) <= window else { continue } + entries.append(BLEDirectedRelaySpoolEntry(recipient: key.recipient, packet: stored.packet)) } - packetsByRecipient.removeAll() + removeAll() return entries } mutating func pruneExpired(now: Date, window: TimeInterval) { - guard !packetsByRecipient.isEmpty else { return } + guard !packets.isEmpty else { return } - var pruned: [PeerID: [String: StoredPacket]] = [:] - for (recipient, packets) in packetsByRecipient { - let freshPackets = packets.filter { now.timeIntervalSince($0.value.enqueuedAt) <= window } - if !freshPackets.isEmpty { - pruned[recipient] = freshPackets + var freshOrder: [Key] = [] + for key in order { + guard let stored = packets[key] else { continue } + if now.timeIntervalSince(stored.enqueuedAt) <= window { + freshOrder.append(key) + } else { + packets.removeValue(forKey: key) + payloadBytes -= stored.packet.payload.count } } - packetsByRecipient = pruned + order = freshOrder } mutating func removeAll() { - packetsByRecipient.removeAll() + packets.removeAll() + order.removeAll() + payloadBytes = 0 } } diff --git a/bitchat/Services/BLE/BLEFileTransferHandler.swift b/bitchat/Services/BLE/BLEFileTransferHandler.swift index f451411c..56700117 100644 --- a/bitchat/Services/BLE/BLEFileTransferHandler.swift +++ b/bitchat/Services/BLE/BLEFileTransferHandler.swift @@ -152,12 +152,21 @@ final class BLEFileTransferHandler { return true } + // Once authenticated, a local decode/quota/save failure is not proof + // that downstream nodes should be denied the valid signed packet, so + // every exit below still relays. + guard let acceptance = validatedFile(packet.payload, from: peerID) else { + return true + } + + // Track only validated payloads: the gossip store holds and re-serves + // what it is given, so a malformed or oversized file must not enter it. if deliveryPlan.shouldTrackForSync { env.trackPacketSeen(packet) } - _ = storeIncomingPayload( - packet.payload, + _ = storeIncomingFile( + acceptance, from: peerID, senderNickname: senderNickname, timestamp: Date(timeIntervalSince1970: Double(packet.timestamp) / 1000), @@ -165,8 +174,6 @@ final class BLEFileTransferHandler { usesDurableReceipts: false, env: env ) - // Once authenticated, a local decode/quota/save failure is not proof - // that downstream nodes should be denied the valid signed packet. return true } @@ -178,6 +185,9 @@ final class BLEFileTransferHandler { @discardableResult func handlePrivatePayload(_ payload: Data, from peerID: PeerID, timestamp: Date) -> Bool { let env = environment + guard let acceptance = validatedFile(payload, from: peerID) else { + return false + } let peers = env.peersSnapshot() let senderNickname = BLEPeerSenderDisplayName.resolveKnownPeer( peerID: peerID, @@ -187,8 +197,8 @@ final class BLEFileTransferHandler { allowConnectedUnverified: true ) ?? BLEPeerSenderDisplayName.anonymousNickname(for: peerID) - return storeIncomingPayload( - payload, + return storeIncomingFile( + acceptance, from: peerID, senderNickname: senderNickname, timestamp: timestamp, @@ -202,8 +212,26 @@ final class BLEFileTransferHandler { ) } - private func storeIncomingPayload( - _ payload: Data, + /// Decodes and validates a file payload (size, MIME allow-list, magic + /// bytes), logging the reason for any rejection. + private func validatedFile(_ payload: Data, from peerID: PeerID) -> BLEIncomingFileAcceptance? { + switch BLEIncomingFileValidator.validate(payload: payload) { + case .success(let acceptance): + return acceptance + case .failure(.malformedPayload): + SecureLogger.error("โŒ Failed to decode file transfer payload", category: .session) + case .failure(.payloadTooLarge(let bytes)): + SecureLogger.warning("๐Ÿšซ Dropping file transfer exceeding size cap (\(bytes) bytes)", category: .security) + case .failure(.unsupportedMime(let mimeType, let bytes)): + SecureLogger.warning("๐Ÿšซ MIME REJECT: '\(mimeType ?? "")' not supported. Size=\(bytes)b from \(peerID.id.prefix(8))...", category: .security) + case .failure(.magicMismatch(let mime, let bytes, let prefixHex)): + SecureLogger.warning("๐Ÿšซ MAGIC REJECT: MIME='\(mime)' size=\(bytes)b prefix=[\(prefixHex)] from \(peerID.id.prefix(8))...", category: .security) + } + return nil + } + + private func storeIncomingFile( + _ acceptance: BLEIncomingFileAcceptance, from peerID: PeerID, senderNickname: String, timestamp: Date, @@ -213,25 +241,8 @@ final class BLEFileTransferHandler { ) -> Bool { let localPeerID = env.localPeerID() - let filePacket: BitchatFilePacket - let mime: MimeType - switch BLEIncomingFileValidator.validate(payload: payload) { - case .success(let acceptance): - filePacket = acceptance.filePacket - mime = acceptance.mime - case .failure(.malformedPayload): - SecureLogger.error("โŒ Failed to decode file transfer payload", category: .session) - return false - case .failure(.payloadTooLarge(let bytes)): - SecureLogger.warning("๐Ÿšซ Dropping file transfer exceeding size cap (\(bytes) bytes)", category: .security) - return false - case .failure(.unsupportedMime(let mimeType, let bytes)): - SecureLogger.warning("๐Ÿšซ MIME REJECT: '\(mimeType ?? "")' not supported. Size=\(bytes)b from \(peerID.id.prefix(8))...", category: .security) - return false - case .failure(.magicMismatch(let mime, let bytes, let prefixHex)): - SecureLogger.warning("๐Ÿšซ MAGIC REJECT: MIME='\(mime)' size=\(bytes)b prefix=[\(prefixHex)] from \(peerID.id.prefix(8))...", category: .security) - return false - } + let filePacket = acceptance.filePacket + let mime = acceptance.mime if isPrivate, env.isPrivateMediaSenderBlocked(peerID) { SecureLogger.debug( diff --git a/bitchat/Services/BLE/BLEFragmentAssemblyBuffer.swift b/bitchat/Services/BLE/BLEFragmentAssemblyBuffer.swift index 31656cb4..1fcdee81 100644 --- a/bitchat/Services/BLE/BLEFragmentAssemblyBuffer.swift +++ b/bitchat/Services/BLE/BLEFragmentAssemblyBuffer.swift @@ -97,9 +97,14 @@ struct BLEFragmentAssemblyBuffer { ) -> AppendResult { let started = startAssemblyIfNeeded(for: header, maxInFlightAssemblies: maxInFlightAssemblies, now: now) - let currentSize = fragmentsByKey[header.key]?.values.reduce(0) { $0 + $1.count } ?? 0 + let held = fragmentsByKey[header.key] + let currentSize = held?.values.reduce(0) { $0 + $1.count } ?? 0 let limit = Self.assemblyLimit(for: header.originalType) - let projectedSize = currentSize + header.fragmentData.count + // A re-delivered index replaces what it held rather than adding to + // it: fragments bypass dedup and sync re-serves held indexes, so a + // duplicate near the cap must not tip a valid assembly over it. + let replacedSize = held?[header.index]?.count ?? 0 + let projectedSize = currentSize - replacedSize + header.fragmentData.count guard projectedSize <= limit else { fragmentsByKey.removeValue(forKey: header.key) @@ -200,15 +205,12 @@ struct BLEFragmentAssemblyBuffer { } } + /// The largest frame a packet of the claimed type could decode from: + /// framed-file scale only for media (a large noiseEncrypted packet can be + /// an E2E-encrypted private file, validated after decrypt), link or + /// message scale for everything else. The handler drops a reassembled + /// packet whose type differs from the claim. private static func assemblyLimit(for originalType: UInt8) -> Int { - if originalType == MessageType.fileTransfer.rawValue - || originalType == MessageType.noiseEncrypted.rawValue { - // Allow headroom for TLV metadata and binary framing overhead. - // A large noiseEncrypted packet can be an E2E-encrypted private - // file; its authenticated plaintext is validated after decrypt. - return FileTransferLimits.maxFramedFileBytes - } - - return FileTransferLimits.maxPayloadBytes + PacketPayloadLimits.maxFrameBytes(forType: originalType) } } diff --git a/bitchat/Services/BLE/BLEFragmentHandler.swift b/bitchat/Services/BLE/BLEFragmentHandler.swift index 9d913151..9ead6b44 100644 --- a/bitchat/Services/BLE/BLEFragmentHandler.swift +++ b/bitchat/Services/BLE/BLEFragmentHandler.swift @@ -61,9 +61,16 @@ final class BLEFragmentHandler { // Decode the original packet bytes we reassembled, so flags/compression are preserved if var originalPacket = BinaryProtocol.decode(reassembled) { - // Reassembled packet validation + // Reassembled packet validation. Both platforms' fragmenters copy + // the inner type into every fragment, and the assembly was sized + // for that claim, so a packet of any other type is forged. let innerSender = PeerID(hexData: originalPacket.senderID) - if !env.isAcceptedIngressPayload(originalPacket, innerSender) { + if originalPacket.type != completedHeader.originalType { + SecureLogger.warning( + "๐Ÿšซ Reassembled packet id=\(completedHeader.idLogString) is type \(originalPacket.type), fragments claimed \(completedHeader.originalType)", + category: .security + ) + } else if !env.isAcceptedIngressPayload(originalPacket, innerSender) { // Cleanup below } else { SecureLogger.debug("โœ… Reassembled packet id=\(completedHeader.idLogString) type=\(originalPacket.type) bytes=\(reassembled.count)", category: .session) diff --git a/bitchat/Services/BLE/BLEPublicMessageHandler.swift b/bitchat/Services/BLE/BLEPublicMessageHandler.swift index 7ff786be..0be228c9 100644 --- a/bitchat/Services/BLE/BLEPublicMessageHandler.swift +++ b/bitchat/Services/BLE/BLEPublicMessageHandler.swift @@ -40,6 +40,8 @@ struct BLEPublicMessageHandlerEnvironment { /// policy, sender display-name resolution, gossip tracking, payload decoding, /// and UI delivery. final class BLEPublicMessageHandler { + private static let maxDeliveredPayloadBytes = Int(UInt16.max) + private let environment: BLEPublicMessageHandlerEnvironment init(environment: BLEPublicMessageHandlerEnvironment) { @@ -110,6 +112,16 @@ final class BLEPublicMessageHandler { env.trackPacketSeen(packet) } + // Both platforms send public messages as v1 frames, whose payload + // tops out at 65,535 bytes. The decode cap leaves headroom above + // that for relaying, but the timeline keeps messages by count and + // its length check counts characters (a combining-mark run is one), + // so nothing larger than a v1 frame is delivered. + guard packet.payload.count <= Self.maxDeliveredPayloadBytes else { + SecureLogger.warning("๐Ÿšซ Dropping \(packet.payload.count)-byte public message from \(peerID.id.prefix(8))โ€ฆ: larger than a v1 frame", category: .security) + return + } + guard let content = String(data: packet.payload, encoding: .utf8) else { SecureLogger.error("โŒ Failed to decode message payload as UTF-8", category: .session) return diff --git a/bitchat/Services/TransportConfig.swift b/bitchat/Services/TransportConfig.swift index 764910da..af21a6dc 100644 --- a/bitchat/Services/TransportConfig.swift +++ b/bitchat/Services/TransportConfig.swift @@ -319,6 +319,14 @@ enum TransportConfig { // on each maintenance flush until the window lapses; a longer window lets // brief link gaps (walking between rooms, reconnect churn) heal themselves. static let bleDirectedSpoolWindowSeconds: TimeInterval = 60.0 + // The spool evicts oldest-first past either bound. Only noiseEncrypted + // and noiseHandshake packets are spooled, and relayed ones arrive as + // single frames (larger sends travel as fragments, which are never + // spooled), so ordinary entries are a few hundred bytes. The byte budget + // still fits one packet at the framed-file decompression cap (~1.13 MiB) + // plus hundreds of ordinary DMs. + static let bleDirectedSpoolCapacity: Int = 512 + static let bleDirectedSpoolByteBudget: Int = 2 * 1024 * 1024 // Log/UI debounce windows // Shorter debounce so UI reacts faster while still suppressing duplicate callbacks diff --git a/bitchat/Sync/GossipSyncManager.swift b/bitchat/Sync/GossipSyncManager.swift index 01deee8a..c54140f0 100644 --- a/bitchat/Sync/GossipSyncManager.swift +++ b/bitchat/Sync/GossipSyncManager.swift @@ -14,18 +14,27 @@ final class GossipSyncManager { private struct PacketStore { private(set) var packets: [String: BitchatPacket] = [:] private(set) var order: [String] = [] + /// Sum of retained payload sizes, maintained on every insert/remove. + private var payloadBytes = 0 - mutating func insert(idHex: String, packet: BitchatPacket, capacity: Int) { - guard capacity > 0 else { return } - if packets[idHex] != nil { + /// Evicts oldest-first until both the count and the byte budget hold. + /// The count cap alone let a few max-size packets per slot pin + /// hundreds of MB; the budget bounds memory whatever the sizes. + mutating func insert(idHex: String, packet: BitchatPacket, capacity: Int, byteBudget: Int) { + guard capacity > 0, packet.payload.count <= byteBudget else { return } + if let existing = packets[idHex] { + payloadBytes += packet.payload.count - existing.payload.count packets[idHex] = packet - return + } else { + packets[idHex] = packet + order.append(idHex) + payloadBytes += packet.payload.count } - packets[idHex] = packet - order.append(idHex) - while order.count > capacity { + while order.count > capacity || payloadBytes > byteBudget { let victim = order.removeFirst() - packets.removeValue(forKey: victim) + if let evicted = packets.removeValue(forKey: victim) { + payloadBytes -= evicted.payload.count + } } } @@ -42,6 +51,7 @@ final class GossipSyncManager { guard let packet = packets[key] else { continue } if shouldRemove(packet) { packets.removeValue(forKey: key) + payloadBytes -= packet.payload.count } else { nextOrder.append(key) } @@ -74,6 +84,14 @@ final class GossipSyncManager { var fragmentCapacity: Int = 600 var fileTransferCapacity: Int = 200 var groupMessageCapacity: Int = 200 + // Payload byte budgets on top of the count caps. Each sits well above + // what the count cap holds in ordinary traffic (1000 chat messages, + // 600 fragments of ~500 B), so it only binds when packets are + // oversized โ€” e.g. 200 file transfers at the 1.1 MB frame cap. + var messageByteBudget: Int = 8 * 1024 * 1024 + var fragmentByteBudget: Int = 1024 * 1024 + var fileTransferByteBudget: Int = 32 * 1024 * 1024 + var groupMessageByteBudget: Int = 4 * 1024 * 1024 var fragmentSyncIntervalSeconds: TimeInterval = 30.0 var fileTransferSyncIntervalSeconds: TimeInterval = 60.0 var messageSyncIntervalSeconds: TimeInterval = 15.0 @@ -273,25 +291,25 @@ final class GossipSyncManager { guard isBroadcastRecipient else { return } guard isPacketFresh(packet) else { return } let idHex = PacketIdUtil.computeId(packet).hexEncodedString() - messages.insert(idHex: idHex, packet: packet, capacity: max(1, config.seenCapacity)) + messages.insert(idHex: idHex, packet: packet, capacity: max(1, config.seenCapacity), byteBudget: config.messageByteBudget) archiveDirty = true case .fragment: guard isBroadcastRecipient else { return } guard isPacketFresh(packet) else { return } let idHex = PacketIdUtil.computeId(packet).hexEncodedString() - fragments.insert(idHex: idHex, packet: packet, capacity: max(1, config.fragmentCapacity)) + fragments.insert(idHex: idHex, packet: packet, capacity: max(1, config.fragmentCapacity), byteBudget: config.fragmentByteBudget) case .fileTransfer: guard isBroadcastRecipient else { return } guard isPacketFresh(packet) else { return } let idHex = PacketIdUtil.computeId(packet).hexEncodedString() - fileTransfers.insert(idHex: idHex, packet: packet, capacity: max(1, config.fileTransferCapacity)) + fileTransfers.insert(idHex: idHex, packet: packet, capacity: max(1, config.fileTransferCapacity), byteBudget: config.fileTransferByteBudget) case .groupMessage: // Opaque ciphertext to non-members; carried and served like any // other broadcast so members get backfill from any relay. guard isBroadcastRecipient else { return } guard isPacketFresh(packet) else { return } let idHex = PacketIdUtil.computeId(packet).hexEncodedString() - groupMessages.insert(idHex: idHex, packet: packet, capacity: max(1, config.groupMessageCapacity)) + groupMessages.insert(idHex: idHex, packet: packet, capacity: max(1, config.groupMessageCapacity), byteBudget: config.groupMessageByteBudget) case .prekeyBundle: // Callers only feed verified bundles here (own bundles at send // time, peers' after signature verification), so gossip never @@ -632,25 +650,36 @@ final class GossipSyncManager { /// anything that aged out while the app was dead โ€” or that is dated in /// the future, which an older build could have archived from a sync /// reply. Any drop rewrites the file so it is purged from disk too. + /// + /// Bounded like live intake: entries are stored oldest-first, so walk + /// them newest-first and stop once the count or byte budget is spent. An + /// oversized archive then costs at most one budget of decode work at + /// launch instead of re-inflating everything it holds. private func restoreArchivedMessages() { guard let archive else { return } - var restored = 0 + let capacity = max(1, config.seenCapacity) + var kept: [(idHex: String, packet: BitchatPacket)] = [] + var keptBytes = 0 var droppedAny = false - for data in archive.load() { + for data in archive.load().reversed() { + guard kept.count < capacity else { break } guard let packet = BitchatPacket.from(data), packet.type == MessageType.message.rawValue, isPacketFresh(packet) else { droppedAny = true continue } - let idHex = PacketIdUtil.computeId(packet).hexEncodedString() - messages.insert(idHex: idHex, packet: packet, capacity: max(1, config.seenCapacity)) - restored += 1 + guard keptBytes + packet.payload.count <= config.messageByteBudget else { break } + keptBytes += packet.payload.count + kept.append((PacketIdUtil.computeId(packet).hexEncodedString(), packet)) } - if restored > 0 { - SecureLogger.debug("Restored \(restored) archived public message(s) for gossip sync", category: .sync) + for entry in kept.reversed() { + messages.insert(idHex: entry.idHex, packet: entry.packet, capacity: capacity, byteBudget: config.messageByteBudget) } - if restored > 0 || droppedAny { + if !kept.isEmpty { + SecureLogger.debug("Restored \(kept.count) archived public message(s) for gossip sync", category: .sync) + } + if !kept.isEmpty || droppedAny { archiveDirty = true } } diff --git a/bitchatTests/GossipSyncManagerTests.swift b/bitchatTests/GossipSyncManagerTests.swift index 7714c806..41f9f801 100644 --- a/bitchatTests/GossipSyncManagerTests.swift +++ b/bitchatTests/GossipSyncManagerTests.swift @@ -979,6 +979,89 @@ struct GossipSyncManagerTests { #expect(!FileManager.default.fileExists(atPath: poisonOnlyURL.path)) } + // MARK: - Byte budgets + + /// Three 100-byte packets from distinct senders, oldest first. + private func makeBudgetPackets(type: MessageType) throws -> [BitchatPacket] { + let nowMs = UInt64(Date().timeIntervalSince1970 * 1000) + return try ["00000000000000a1", "00000000000000a2", "00000000000000a3"].enumerated().map { index, hex in + BitchatPacket( + type: type.rawValue, + senderID: try #require(Data(hexString: hex)), + recipientID: nil, + timestamp: nowMs + UInt64(index), + payload: Data(repeating: UInt8(index), count: 100), + signature: nil, + ttl: 1 + ) + } + } + + @Test func messageStoreEvictsOldestPastItsByteBudget() throws { + var config = GossipSyncManager.Config() + config.messageByteBudget = 250 + let manager = GossipSyncManager(myPeerID: myPeerID, config: config, requestSyncManager: RequestSyncManager()) + + let packets = try makeBudgetPackets(type: .message) + packets.forEach(manager.onPublicPacketSeen) + + // 300 bytes against a 250-byte budget: only the oldest goes, even + // though the 1000-packet count cap is nowhere near. + #expect(manager._messageCount(for: PeerID(hexData: packets[0].senderID)) == 0) + #expect(manager._messageCount(for: PeerID(hexData: packets[1].senderID)) == 1) + #expect(manager._messageCount(for: PeerID(hexData: packets[2].senderID)) == 1) + } + + @Test func fragmentFileAndGroupStoresEvictOldestPastTheirByteBudgets() async throws { + var config = GossipSyncManager.Config() + config.fragmentByteBudget = 250 + config.fileTransferByteBudget = 250 + config.groupMessageByteBudget = 250 + config.messageSyncIntervalSeconds = 0 + config.fragmentSyncIntervalSeconds = 0 + config.fileTransferSyncIntervalSeconds = 0 + config.prekeyBundleSyncIntervalSeconds = 0 + let manager = GossipSyncManager(myPeerID: myPeerID, config: config, requestSyncManager: RequestSyncManager()) + let delegate = RecordingDelegate() + manager.delegate = delegate + + var expectedIDs: Set = [] + for type in [MessageType.fragment, .fileTransfer, .groupMessage] { + let packets = try makeBudgetPackets(type: type) + packets.forEach(manager.onPublicPacketSeen) + expectedIDs.formUnion(packets.dropFirst().map { PacketIdUtil.computeId($0) }) + } + + let request = RequestSyncPacket(p: 4, m: 1, data: Data(), types: [.fragment, .fileTransfer, .groupMessage]) + manager.handleRequestSync(from: PeerID(str: "FFFFFFFFFFFFFFFF"), request: request) + + try await TestHelpers.waitFor({ delegate.packets.count >= expectedIDs.count }, timeout: TestConstants.settleTimeout) + // Barrier: flush the sync queue so an evicted packet served late would be visible. + manager._performMaintenanceSynchronously(now: Date()) + #expect(Set(delegate.packets.map { PacketIdUtil.computeId($0) }) == expectedIDs) + #expect(delegate.packets.count == expectedIDs.count) + } + + @Test func archiveRestoreStopsAtTheByteBudgetKeepingTheNewest() throws { + let fileURL = FileManager.default.temporaryDirectory + .appendingPathComponent("gossip-archive-\(UUID().uuidString).json") + defer { try? FileManager.default.removeItem(at: fileURL) } + + let packets = try makeBudgetPackets(type: .message) + let archive = GossipMessageArchive(fileURL: fileURL) + archive.save(try packets.map { try #require($0.toBinaryData(padding: false)) }) + + var config = GossipSyncManager.Config() + config.messageByteBudget = 250 + let manager = GossipSyncManager(myPeerID: myPeerID, config: config, requestSyncManager: RequestSyncManager(), archive: archive) + // Barrier: the restore is queued on the sync queue at init. + manager._performMaintenanceSynchronously(now: Date()) + + #expect(manager._messageCount(for: PeerID(hexData: packets[0].senderID)) == 0) + #expect(manager._messageCount(for: PeerID(hexData: packets[1].senderID)) == 1) + #expect(manager._messageCount(for: PeerID(hexData: packets[2].senderID)) == 1) + } + /// Clearing the mesh timeline must leave nothing behind on disk: a /// relaunch that restored the archive would undo the clear. @Test func removeAllPublicMessagesErasesTheArchiveOnDisk() async throws { @@ -1028,11 +1111,15 @@ private final class RecordingDelegate: GossipSyncManager.Delegate { private let lock = NSLock() func sendPacket(_ packet: BitchatPacket) { + // Run the hook before recording: a waiter that sees `lastPacket` + // then knows the hook already ran. Recording first let + // concurrentPacketIntakeAndSyncRequest's waitFor win the race and + // close its confirmation before the hook confirmed it. + onSend?() lock.lock() lastPacket = packet packets.append(packet) lock.unlock() - onSend?() } func sendPacket(to peerID: PeerID, packet: BitchatPacket) { diff --git a/bitchatTests/Protocols/PacketPayloadCapInteropTests.swift b/bitchatTests/Protocols/PacketPayloadCapInteropTests.swift new file mode 100644 index 00000000..d55256be --- /dev/null +++ b/bitchatTests/Protocols/PacketPayloadCapInteropTests.swift @@ -0,0 +1,257 @@ +// +// PacketPayloadCapInteropTests.swift +// bitchatTests +// +// This is free and unencumbered software released into the public domain. +// For more information, see +// + +import BitFoundation +import Foundation +import Testing +@testable import bitchat + +/// Receive-side interop for the per-type payload caps in `BinaryProtocol`: +/// the largest payload each shipping encoder produces (iOS, and Android for +/// the types it sends) still decodes, while the same type inflated from ~1 KB +/// on air to ~1 MB, or reassembled uncompressed from a fragment stream, does +/// not. +struct PacketPayloadCapInteropTests { + private let senderID = Data(repeating: 0x5A, count: 8) + private let signature = Data(repeating: 0xAB, count: 64) + + @Test func longestPublicMessagesStillDecode() throws { + // Both platforms send public messages as v1 frames (iOS + // BLEService.sendMessage, Android MeshCore.sendMessage), so these + // composer-sized messages are as large as the type legitimately gets. + let cjk = String(repeating: "ๆผขๅญ—ไปฎๅไบคใ˜ใ‚Šๆ–‡", count: 2_000) + let ascii = String(repeating: "x", count: InputValidator.Limits.maxMessageLength) + #expect(cjk.count == 16_000) + for content in [cjk, ascii] { + let packet = makePacket(type: .message, payload: Data(content.utf8)) + #expect(try roundTripIsCompressed(packet)) + } + + #expect(BinaryProtocol.decode(try inflatedFrame(type: .message)) == nil) + } + + @Test func androidAndIOSSizedFragmentsStillDecode() throws { + // Every byte value once, then zeros: the inner packet stays + // uncompressed (its bytes are 100% unique) while the zero-filled + // chunks compress, so the fragment frames take the inflate path. + var inner = Data((0...255).map { UInt8($0) }) + inner.append(Data(count: 4_000)) + let packet = makePacket(type: .fileTransfer, payload: inner, version: 2) + + // 469 B: Android's MAX_FRAGMENT_SIZE and the iOS default chunk. + // 482 B: the largest iOS chunk, from a 524-byte BLE link limit. + // 500 B: the chunk both platforms used before mid-2025. + let chunkSizes = [ + TransportConfig.bleDefaultFragmentSize, + BLEOutboundPacketPolicy.fragmentChunkSize(forLinkLimit: 524), + 500 + ] + for chunkSize in chunkSizes { + let request = BLEOutboundFragmentTransferRequest( + packet: packet, + pad: false, + maxChunk: chunkSize, + directedPeer: nil, + transferId: nil + ) + let plan = try #require(BLEOutboundFragmentPlanner.makePlan( + for: request, + defaultChunkSize: TransportConfig.bleDefaultFragmentSize, + bleMaxMTU: 512 + )) + #expect(plan.fragmentPackets.map(\.payload.count).max() == 13 + chunkSize) + + var compressedFragments = 0 + for fragment in plan.fragmentPackets { + if try roundTripIsCompressed(fragment) { + compressedFragments += 1 + } + } + #expect(compressedFragments > 0) + } + + // Fragments are link-scale: 2 KiB is four times the largest any + // sender produces, so a few dozen bytes on air cannot stand for it. + let oversized = makePacket(type: .fragment, payload: Data(repeating: 0x41, count: 2 * 1_024)) + let oversizedFrame = try #require(oversized.toBinaryData(padding: false)) + #expect(oversizedFrame[BinaryProtocol.Offsets.flags] & BinaryProtocol.Flags.isCompressed != 0) + #expect(BinaryProtocol.decode(oversizedFrame) == nil) + #expect(BinaryProtocol.decode(try inflatedFrame(type: .fragment)) == nil) + } + + @Test func largestControlAndEnvelopePayloadsStillDecode() throws { + let cases: [(type: MessageType, payload: Data, compresses: Bool)] = [ + (.announce, try maxAnnounce(), true), + (.boardPost, maxBoardPost(), true), + (.nostrCarrier, try maxNostrCarrier(), true), + (.requestSync, maxRequestSync(), false), + (.courierEnvelope, try maxCourierEnvelope(), false), + (.groupMessage, try maxGroupMessage(), false), + (.prekeyBundle, try maxPrekeyBundle(), false), + (.voiceFrame, try maxVoiceFrame(), false) + // announceV2's largest payload is checked in BitFoundation's + // PacketPayloadLimitsTests: nothing in the app emits it yet, and a + // reference here would make it look used to the dead-code scan. + ] + for testCase in cases { + let packet = makePacket(type: testCase.type, payload: testCase.payload) + let compressed = try roundTripIsCompressed(packet) + #expect(compressed || !testCase.compresses, "\(testCase.type.description) should take the inflate path") + + #expect( + BinaryProtocol.decode(try inflatedFrame(type: testCase.type)) == nil, + "\(testCase.type.description) inflated to ~1 MB must be rejected" + ) + } + } + + @Test func largeFileTransfersStillDecode() throws { + // A 1 MiB image whose bytes compress well: the largest legitimate + // inflation a broadcast file can ask for. + var content = Data([0xFF, 0xD8, 0xFF, 0xE0]) + content.append(Data(count: FileTransferLimits.maxPayloadBytes - content.count)) + let file = BitchatFilePacket(fileName: "photo.jpg", fileSize: nil, mimeType: "image/jpeg", content: content) + let payload = try #require(file.encode()) + let packet = makePacket(type: .fileTransfer, payload: payload, version: 2) + #expect(try roundTripIsCompressed(packet)) + + // The same bytes under a non-media type are what the caps stop. + let asGroupMessage = makePacket(type: .groupMessage, payload: payload, version: 2) + #expect(BinaryProtocol.decode(try #require(asGroupMessage.toBinaryData())) == nil) + } + + // MARK: - Legitimate maxima, built with the real encoders + + private func maxAnnounce() throws -> Data { + try #require(AnnouncementPacket( + nickname: String(repeating: "n", count: 255), + noisePublicKey: Data(repeating: 0x21, count: 32), + signingPublicKey: Data(repeating: 0x22, count: 32), + directNeighbors: (0..<10).map { Data(repeating: UInt8($0), count: 8) }, + capabilities: PeerCapabilities(rawValue: .max), + bridgeGeohash: "9q8yyk8yuv12" + ).encode()) + } + + private func maxBoardPost() -> Data { + BoardWire.post(BoardPostPacket( + postID: Data(repeating: 0x31, count: BoardWireConstants.postIDLength), + geohash: "9q8yyk8yuv12", + content: String(repeating: "x", count: BoardWireConstants.contentMaxBytes), + authorSigningKey: Data(repeating: 0x32, count: BoardWireConstants.signingKeyLength), + authorNickname: String(repeating: "n", count: BoardWireConstants.nicknameMaxBytes), + createdAt: 1_740_000_000_000, + expiresAt: 1_740_000_000_000 + BoardWireConstants.maxLifetimeMs, + flags: BoardPostPacket.urgentFlag, + signature: signature + )).encode() + } + + private func maxNostrCarrier() throws -> Data { + let prefix = Data(#"{"content":""#.utf8) + let suffix = Data(#""}"#.utf8) + var json = prefix + json.append(Data(repeating: 0x61, count: NostrCarrierPacket.maxEventJSONBytes - prefix.count - suffix.count)) + json.append(suffix) + let carrier = try #require(NostrCarrierPacket(direction: .fromGateway, geohash: "9q8yyk8yuv12", eventJSON: json)) + return try #require(carrier.encode()) + } + + private func maxRequestSync() -> Data { + let fragmentIDs = (0.. Data { + try #require(CourierEnvelope( + recipientTag: Data(repeating: 0x41, count: CourierEnvelope.tagLength), + expiry: 1_740_000_000_000, + ciphertext: randomBytes(CourierEnvelope.maxCiphertextBytes), + copies: CourierEnvelope.maxCopies, + prekeyID: 7 + ).encode()) + } + + private func maxGroupMessage() throws -> Data { + try GroupCrypto.sealMessage( + content: String(repeating: "g", count: InputValidator.Limits.maxMessageLength), + messageID: UUID().uuidString, + senderNickname: String(repeating: "n", count: 50), + senderSigningKey: Data(repeating: 0x51, count: 32), + timestampMs: 1_740_000_000_000, + groupID: Data(repeating: 0x52, count: BitchatGroup.groupIDLength), + epoch: 1, + key: Data(repeating: 0x53, count: BitchatGroup.keyLength), + sign: { [signature] _ in signature } + ) + } + + private func maxPrekeyBundle() throws -> Data { + try #require(PrekeyBundle( + noiseStaticPublicKey: Data(repeating: 0x61, count: PrekeyBundle.keyLength), + prekeys: (0.. Data { + // 11-byte header + 2-byte length + frame = the 210-byte burst budget. + let frame = randomBytes(TransportConfig.pttMaxBurstContentBytes - 13) + let burst = try #require(VoiceBurstPacket(burstID: Data(repeating: 0x71, count: 8), seq: 1, kind: .frames([frame]))) + return burst.encode() + } + + // MARK: - Helpers + + private func makePacket(type: MessageType, payload: Data, version: UInt8 = 1) -> BitchatPacket { + BitchatPacket( + type: type.rawValue, + senderID: senderID, + recipientID: nil, + timestamp: 1_740_000_000_000, + payload: payload, + signature: signature, + ttl: TransportConfig.messageTTLDefault, + version: version + ) + } + + /// Encodes as on the wire (padded), decodes, and checks the payload + /// survives; returns whether the frame went out compressed. + private func roundTripIsCompressed(_ packet: BitchatPacket) throws -> Bool { + let frame = try #require(packet.toBinaryData()) + let decoded = try #require(BinaryProtocol.decode(frame), "type \(packet.type) failed to decode") + #expect(decoded.payload == packet.payload) + return frame[BinaryProtocol.Offsets.flags] & BinaryProtocol.Flags.isCompressed != 0 + } + + /// ~1 MB of one byte in a v2 frame: about 1 KB on air, and accepted by + /// the old frame-wide cap for every type. + private func inflatedFrame(type: MessageType) throws -> Data { + let packet = makePacket(type: type, payload: Data(repeating: 0x41, count: 1_024 * 1_024), version: 2) + let frame = try #require(packet.toBinaryData(padding: false)) + #expect(frame.count < 4_096) + return frame + } + + private func randomBytes(_ count: Int) -> Data { + Data((0..= 1) + } + private func makePacket(payload: [UInt8]) -> BitchatPacket { + makePacket(payload: Data(payload)) + } + + private func makePacket(payload: Data) -> BitchatPacket { BitchatPacket( type: MessageType.noiseEncrypted.rawValue, senderID: Data(hexString: "8877665544332211") ?? Data(), recipientID: Data(hexString: "1122334455667788"), timestamp: 1234, - payload: Data(payload), + payload: payload, signature: nil, ttl: TransportConfig.messageTTLDefault ) diff --git a/bitchatTests/Services/BLEFileTransferHandlerTests.swift b/bitchatTests/Services/BLEFileTransferHandlerTests.swift index 835bcdba..ffe828c4 100644 --- a/bitchatTests/Services/BLEFileTransferHandlerTests.swift +++ b/bitchatTests/Services/BLEFileTransferHandlerTests.swift @@ -953,7 +953,7 @@ struct BLEFileTransferHandlerTests { } @Test - func malformedPayloadIsTrackedForSyncButDropped() { + func malformedPayloadIsRelayedButNeverTrackedForSync() { let recorder = Recorder() recorder.peers = [remotePeerID: makePeerInfo(remotePeerID, nickname: "Alice", isVerified: true, signingPublicKey: sampleSigningKey)] recorder.signatureVerifies = true @@ -971,8 +971,9 @@ struct BLEFileTransferHandlerTests { // Local decode failures are not proof of forgery; the packet stays relayable. #expect(handler.handle(packet, from: remotePeerID)) - // Sync tracking happens before payload validation, matching the original order. - #expect(recorder.trackedPackets.count == 1) + // But the gossip store re-serves whatever it tracks, so validation + // runs first and an undecodable payload never enters it. + #expect(recorder.trackedPackets.isEmpty) #expect(recorder.quotaReservations.isEmpty) #expect(recorder.saveCalls.isEmpty) #expect(recorder.deliveredMessages.isEmpty) @@ -988,7 +989,39 @@ struct BLEFileTransferHandlerTests { #expect(handler.handle(packet, from: remotePeerID)) - #expect(recorder.trackedPackets.count == 1) + #expect(recorder.trackedPackets.isEmpty) + #expect(recorder.quotaReservations.isEmpty) + #expect(recorder.saveCalls.isEmpty) + #expect(recorder.deliveredMessages.isEmpty) + } + + @Test + func oversizedBroadcastFileIsRelayedButNeverTrackedForSync() { + let recorder = Recorder() + recorder.peers = [remotePeerID: makePeerInfo(remotePeerID, nickname: "Alice", isVerified: true, signingPublicKey: sampleSigningKey)] + recorder.signatureVerifies = true + let handler = makeHandler(recorder: recorder) + // A CONTENT TLV one byte over the file cap: it fits the framed-file + // ceiling at decode, so only payload validation can stop it. + let oversizedCount = FileTransferLimits.maxPayloadBytes + 1 + var length = UInt32(oversizedCount).bigEndian + var payload = Data([0x04]) + withUnsafeBytes(of: &length) { payload.append(contentsOf: $0) } + payload.append(Data(repeating: 0x41, count: oversizedCount)) + let packet = BitchatPacket( + type: MessageType.fileTransfer.rawValue, + senderID: Data(hexString: remotePeerID.id) ?? Data(), + recipientID: nil, + timestamp: 900_000, + payload: payload, + signature: Data(repeating: 0x5A, count: 64), + ttl: TransportConfig.messageTTLDefault, + version: 2 + ) + + #expect(handler.handle(packet, from: remotePeerID)) + + #expect(recorder.trackedPackets.isEmpty) #expect(recorder.quotaReservations.isEmpty) #expect(recorder.saveCalls.isEmpty) #expect(recorder.deliveredMessages.isEmpty) diff --git a/bitchatTests/Services/BLEFragmentAssemblyBufferTests.swift b/bitchatTests/Services/BLEFragmentAssemblyBufferTests.swift index ed4da0a1..5299c722 100644 --- a/bitchatTests/Services/BLEFragmentAssemblyBufferTests.swift +++ b/bitchatTests/Services/BLEFragmentAssemblyBufferTests.swift @@ -84,12 +84,13 @@ struct BLEFragmentAssemblyBufferTests { func appendOversizedAssemblyDropsPartialState() throws { var buffer = BLEFragmentAssemblyBuffer() let fragmentID = Data(repeating: 0x05, count: 8) + let limit = PacketPayloadLimits.maxFrameBytes(forType: MessageType.message.rawValue) let first = try #require(BLEFragmentHeader(packet: makeFragmentPacket( fragmentID: fragmentID, index: 0, total: 2, originalType: MessageType.message.rawValue, - fragmentData: Data(repeating: 0x01, count: FileTransferLimits.maxPayloadBytes) + fragmentData: Data(repeating: 0x01, count: limit) ))) let oversized = try #require(BLEFragmentHeader(packet: makeFragmentPacket( fragmentID: fragmentID, @@ -102,9 +103,9 @@ struct BLEFragmentAssemblyBufferTests { _ = buffer.append(first, maxInFlightAssemblies: 8) let result = buffer.append(oversized, maxInFlightAssemblies: 8) - if case let .oversized(_, projectedSize, limit, started) = result { - #expect(projectedSize == FileTransferLimits.maxPayloadBytes + 1) - #expect(limit == FileTransferLimits.maxPayloadBytes) + if case let .oversized(_, projectedSize, reportedLimit, started) = result { + #expect(projectedSize == limit + 1) + #expect(reportedLimit == limit) #expect(!started) } else { Issue.record("Expected oversized fragment assembly to be evicted") @@ -117,6 +118,77 @@ struct BLEFragmentAssemblyBufferTests { } } + @Test + func redeliveredIndexNearTheLimitDoesNotEvictTheAssembly() throws { + // Fragments bypass dedup and sync re-serves held indexes, so a + // duplicate must replace its index's bytes, not count them twice. + var buffer = BLEFragmentAssemblyBuffer() + let fragmentID = Data(repeating: 0x06, count: 8) + let limit = PacketPayloadLimits.maxFrameBytes(forType: MessageType.message.rawValue) + func fragment(_ index: Int, bytes: Int) throws -> BLEFragmentHeader { + try #require(BLEFragmentHeader(packet: makeFragmentPacket( + fragmentID: fragmentID, + index: index, + total: 3, + originalType: MessageType.message.rawValue, + fragmentData: Data(repeating: UInt8(index + 1), count: bytes) + ))) + } + let large = try fragment(0, bytes: limit - 2) + + _ = buffer.append(large, maxInFlightAssemblies: 8) + _ = buffer.append(try fragment(1, bytes: 1), maxInFlightAssemblies: 8) + guard case .stored = buffer.append(large, maxInFlightAssemblies: 8) else { + Issue.record("Expected a re-delivered index to keep the assembly") + return + } + + if case let .complete(_, data, _) = buffer.append(try fragment(2, bytes: 1), maxInFlightAssemblies: 8) { + #expect(data.count == limit) + } else { + Issue.record("Expected the assembly to complete at its limit") + } + } + + @Test + func assemblyIsHeldToTheFrameItsClaimedTypeCanFill() throws { + // Not a flat 1 MiB for every type: an assembly claiming a + // link-scale or message-scale type is cut off just past the largest + // frame that type could still decode from. + let claims: [(type: MessageType, id: UInt8)] = [(.fragment, 0x30), (.announce, 0x40), (.message, 0x50)] + for claim in claims { + var buffer = BLEFragmentAssemblyBuffer() + let limit = PacketPayloadLimits.maxFrameBytes(forType: claim.type.rawValue) + #expect(limit < FileTransferLimits.maxPayloadBytes / 4) + + let atLimit = try #require(BLEFragmentHeader(packet: makeFragmentPacket( + fragmentID: Data(repeating: claim.id, count: 8), + index: 0, + total: 1, + originalType: claim.type.rawValue, + fragmentData: Data(repeating: 0x01, count: limit) + ))) + if case let .complete(_, data, _) = buffer.append(atLimit, maxInFlightAssemblies: 8) { + #expect(data.count == limit) + } else { + Issue.record("Expected a \(claim.type.description) assembly at its frame ceiling to complete") + } + + let pastLimit = try #require(BLEFragmentHeader(packet: makeFragmentPacket( + fragmentID: Data(repeating: claim.id + 1, count: 8), + index: 0, + total: 1, + originalType: claim.type.rawValue, + fragmentData: Data(repeating: 0x01, count: limit + 1) + ))) + if case let .oversized(_, _, reportedLimit, _) = buffer.append(pastLimit, maxInFlightAssemblies: 8) { + #expect(reportedLimit == limit) + } else { + Issue.record("Expected a \(claim.type.description) assembly past its frame ceiling to be evicted") + } + } + } + @Test func encryptedPrivateFileAssemblyGetsFramedFileHeadroom() throws { var buffer = BLEFragmentAssemblyBuffer() diff --git a/bitchatTests/Services/BLEFragmentHandlerTests.swift b/bitchatTests/Services/BLEFragmentHandlerTests.swift index e48a74ed..a71326c8 100644 --- a/bitchatTests/Services/BLEFragmentHandlerTests.swift +++ b/bitchatTests/Services/BLEFragmentHandlerTests.swift @@ -1,4 +1,4 @@ -import BitFoundation +@testable import BitFoundation import Foundation import Testing @testable import bitchat @@ -172,6 +172,77 @@ struct BLEFragmentHandlerTests { #expect(recorder.reinjectedPackets.isEmpty) } + @Test + func reassembledPacketPastItsTypeCapNeverReachesThePipeline() throws { + // A large uncompressed inner frame streamed as compressed fragments + // (each zero-filled chunk is a few dozen bytes on air) through the + // real fragmenter and assembly buffer. One byte past the type's cap still fits the + // assembly, whose ceiling adds framing, so decode must stop it. + for type in [MessageType.message, .announce] { + let cap = PacketPayloadLimits.maxPayloadBytes(forType: type.rawValue) + var buffer = BLEFragmentAssemblyBuffer() + let recorder = Recorder() + recorder.appendResult = { header in + buffer.append(header, maxInFlightAssemblies: 8) + } + let handler = makeHandler(recorder: recorder) + + for payloadSize in [cap, cap + 1] { + let inner = BitchatPacket( + type: type.rawValue, + senderID: Data(hexString: "99AABBCCDDEEFF00") ?? Data(), + recipientID: nil, + timestamp: 900_000, + payload: uncompressiblePrefixThenZeros(count: payloadSize), + signature: nil, + ttl: 5, + version: payloadSize > Int(UInt16.max) ? 2 : 1 + ) + for fragment in try fragmentsAsReceived(inner) { + handler.handle(fragment, from: remotePeerID) + } + } + + #expect( + recorder.reinjectedPackets.map(\.packet.payload.count) == [cap], + "\(type.description) past its cap must not be reassembled into the pipeline" + ) + } + } + + @Test + func reassembledPacketOfAnotherTypeThanItsFragmentsClaimIsDropped() throws { + let innerPacket = BitchatPacket( + type: MessageType.message.rawValue, + senderID: Data(hexString: "99AABBCCDDEEFF00") ?? Data(), + recipientID: nil, + timestamp: 900_000, + payload: Data("hello".utf8), + signature: nil, + ttl: 5 + ) + let reassembled = try #require(innerPacket.toBinaryData()) + + let recorder = Recorder() + recorder.appendResult = { header in + .complete(header: header, reassembledData: reassembled, started: false) + } + let handler = makeHandler(recorder: recorder) + // A file-transfer claim buys a framed-file-sized assembly; the + // packet it produced must be a file transfer. + let packet = makeFragmentPacket( + sender: remotePeerID, + index: 1, + total: 2, + originalType: MessageType.fileTransfer.rawValue + ) + + handler.handle(packet, from: remotePeerID) + + #expect(recorder.ingressChecks.isEmpty) + #expect(recorder.reinjectedPackets.isEmpty) + } + @Test func undecodableReassembledDataIsDropped() { let recorder = Recorder() @@ -200,6 +271,43 @@ struct BLEFragmentHandlerTests { #expect(recorder.reinjectedPackets.isEmpty) } + /// Every byte value once, then zeros: the encoder leaves the inner + /// payload uncompressed (100% byte diversity), while the zero chunks + /// compress inside their fragments. + private func uncompressiblePrefixThenZeros(count: Int) -> Data { + var payload = Data((0...255).map { UInt8($0) }) + payload.append(Data(count: count - payload.count)) + return payload + } + + /// Fragments `packet` with the real planner, then puts each fragment + /// through the wire codec as a receiver would. + private func fragmentsAsReceived(_ packet: BitchatPacket) throws -> [BitchatPacket] { + let request = BLEOutboundFragmentTransferRequest( + packet: packet, + pad: false, + maxChunk: nil, + directedPeer: nil, + transferId: nil + ) + let plan = try #require(BLEOutboundFragmentPlanner.makePlan( + for: request, + defaultChunkSize: TransportConfig.bleDefaultFragmentSize, + bleMaxMTU: 512 + )) + var compressedFragments = 0 + let received = try plan.fragmentPackets.map { fragment in + let frame = try #require(fragment.toBinaryData()) + if frame[BinaryProtocol.Offsets.flags] & BinaryProtocol.Flags.isCompressed != 0 { + compressedFragments += 1 + } + return try #require(BinaryProtocol.decode(frame)) + } + // All but the first (the diverse bytes) and a short tail. + #expect(compressedFragments >= received.count - 2) + return received + } + private func makeFragmentPacket( sender: PeerID, index: UInt16, diff --git a/bitchatTests/Services/BLEPublicMessageHandlerTests.swift b/bitchatTests/Services/BLEPublicMessageHandlerTests.swift index 4e9b027b..c910a384 100644 --- a/bitchatTests/Services/BLEPublicMessageHandlerTests.swift +++ b/bitchatTests/Services/BLEPublicMessageHandlerTests.swift @@ -238,6 +238,28 @@ struct BLEPublicMessageHandlerTests { #expect(recorder.deliveries.isEmpty) } + @Test + func payloadLargerThanAV1FrameIsTrackedForSyncButNotDelivered() { + let now = Date(timeIntervalSince1970: 1_000) + let recorder = Recorder() + recorder.peers = [remotePeerID: makePeerInfo(remotePeerID, nickname: "Alice", isVerified: true)] + recorder.signedName = "SignedAlice" + let handler = makeHandler(recorder: recorder, now: now) + // One character on screen, 65,537 bytes: combining marks pass any + // character-count check, so only a byte bound keeps them out of the + // count-capped timeline. + let combining = "a" + String(repeating: "\u{0301}", count: 32_768) + #expect(combining.count == 1) + let largest = String(repeating: "x", count: Int(UInt16.max)) + + handler.handle(makeMessagePacket(sender: remotePeerID, content: combining, timestamp: timestamp(now)), from: remotePeerID) + handler.handle(makeMessagePacket(sender: remotePeerID, content: largest, timestamp: timestamp(now)), from: remotePeerID) + + // Sync tracking still follows the decode cap; only delivery is bounded. + #expect(recorder.trackedPackets.count == 2) + #expect(recorder.deliveries.map(\.content) == [largest]) + } + @Test func selfSyncReplayResolvesOriginalMessageID() { let now = Date(timeIntervalSince1970: 1_000) diff --git a/localPackages/BitFoundation/Sources/BitFoundation/BinaryProtocol.swift b/localPackages/BitFoundation/Sources/BitFoundation/BinaryProtocol.swift index e326eed2..8590db7d 100644 --- a/localPackages/BitFoundation/Sources/BitFoundation/BinaryProtocol.swift +++ b/localPackages/BitFoundation/Sources/BitFoundation/BinaryProtocol.swift @@ -46,7 +46,8 @@ /// ## Compression Strategy /// - Automatic compression for payloads > 256 bytes /// - zlib compression for broad compatibility on Apple platforms -/// - Original size stored for decompression +/// - Original size stored for decompression; decoded payloads, compressed or +/// not, are capped per message type (see `PacketPayloadLimits`) /// - Flag bit indicates compressed payload /// /// ## Flag Bits @@ -355,6 +356,10 @@ public struct BinaryProtocol { } // Payload: payloadLength is exactly the payload size (+ compression preamble if compressed) + // Per-type ceiling on the decoded payload, compressed or not, + // checked before any allocation: see PacketPayloadLimits for why + // one frame-wide cap is not enough. + let maxPayloadSize = PacketPayloadLimits.maxPayloadBytes(forType: type) let payload: Data if isCompressed { guard payloadLength >= lengthFieldBytes else { return nil } @@ -366,13 +371,15 @@ public struct BinaryProtocol { guard let rawSize = read16() else { return nil } originalSize = Int(rawSize) } - guard originalSize >= 0 && originalSize <= FileTransferLimits.maxFramedFileBytes else { return nil } + guard originalSize >= 0 && originalSize <= maxPayloadSize else { + SecureLogger.debug("๐Ÿšซ Compressed payload for type \(type) declares \(originalSize) bytes (cap \(maxPayloadSize))", category: .security) + return nil + } let compressedSize = payloadLength - lengthFieldBytes guard compressedSize > 0, let compressed = readData(compressedSize) else { return nil } - let compressionRatio = Double(originalSize) / Double(compressedSize) - guard compressionRatio <= 50_000.0 else { - SecureLogger.warning("๐Ÿšซ Suspicious compression ratio: \(String(format: "%.0f", compressionRatio)):1", category: .security) + guard originalSize <= compressedSize * PacketPayloadLimits.maxDeflateRatio else { + SecureLogger.debug("๐Ÿšซ Impossible compression ratio: \(originalSize) bytes from \(compressedSize)", category: .security) return nil } @@ -380,6 +387,12 @@ public struct BinaryProtocol { decompressed.count == originalSize else { return nil } payload = decompressed } else { + // Uncompressed payloads get the same ceiling, or a large frame + // could arrive as small compressed fragments and be reassembled. + guard payloadLength <= maxPayloadSize else { + SecureLogger.debug("๐Ÿšซ Payload for type \(type) is \(payloadLength) bytes (cap \(maxPayloadSize))", category: .security) + return nil + } guard let rawPayload = readData(payloadLength) else { return nil } payload = rawPayload } diff --git a/localPackages/BitFoundation/Sources/BitFoundation/PacketPayloadLimits.swift b/localPackages/BitFoundation/Sources/BitFoundation/PacketPayloadLimits.swift new file mode 100644 index 00000000..a464e0bc --- /dev/null +++ b/localPackages/BitFoundation/Sources/BitFoundation/PacketPayloadLimits.swift @@ -0,0 +1,106 @@ +// +// PacketPayloadLimits.swift +// BitFoundation +// +// This is free and unencumbered software released into the public domain. +// For more information, see +// + +/// Receive-side ceilings on how large a packet's payload may be, per type. +/// +/// A compressed frame declares its original size, and raw DEFLATE lets ~1 KB +/// on air stand for ~1 MB decoded; fragment reassembly lets a large +/// uncompressed frame arrive as a stream of small, compressed fragments. +/// Every accepted packet is then held at its decoded size by stores that +/// evict by count (gossip sync, the public-message archive, the mesh +/// timeline), relayed, and re-served by sync, so a single frame-wide ceiling +/// turns airtime into memory on every node. `BinaryProtocol` enforces these +/// caps on every decoded payload, compressed or not, and fragment reassembly +/// bounds an assembly by the frame its claimed type can fill. Only the types +/// that carry media keep the framed-file ceiling; every other type is capped +/// at no less than twice the largest payload any shipping iOS or Android +/// encoder produces for it. Receive-side only: senders are unchanged. +/// +/// | Type | Largest legitimate payload | Cap | +/// |------------------------------|--------------------------------------------------------------|--------------------| +/// | fileTransfer, noiseEncrypted | 1 MiB file + name/MIME TLVs (+ Noise framing for private) | maxFramedFileBytes | +/// | message | 65,535 B: iOS and Android both send public messages as v1 | 128 KiB | +/// | | frames (16-bit length); 60,000-char composer limit | | +/// | groupMessage | 65,535 B: v1 frame (iOS only) | 128 KiB | +/// | courierEnvelope | 16 KiB ciphertext + 44 B of TLVs (iOS only) | 64 KiB | +/// | nostrCarrier | 16 KiB event JSON + 22 B of TLVs (iOS only) | 64 KiB | +/// | requestSync | ~2.1 KB: 1,024 B GCS filter (both platforms) + 1,019 B | 8 KiB | +/// | | fragment-ID filter (iOS) | | +/// | announce | ~680 B: nickname, keys, 10 neighbours, capabilities, cell | 4 KiB | +/// | fragment | โ‰ค 513 B: 13 B header + chunk (Android 469 B, 500 B before | 1,280 B | +/// | | Aug 2025; iOS 469 B default, โ‰ค 482 B from the link limit; | | +/// | | iOS 500 B before Jul 2025). Kept tight: every fragment also | | +/// | | lands in the reassembly buffer and the gossip fragment store | | +/// | boardPost | ~750 B: 512 B content cap + IDs, key, signature (iOS only) | 4 KiB | +/// | prekeyBundle | ~400 B: 8 prekeys + key + signature (iOS only) | 4 KiB | +/// | announceV2 | ~340 B: 64 B tag block + capabilities + cell (iOS only) | 4 KiB | +/// | voiceFrame | ~210 B burst budget (both platforms) | 4 KiB | +/// | noiseHandshake | 96 B (XX with empty payloads, both platforms) | 4 KiB | +/// | leave | โ‰ค 255 B: empty on iOS, nickname from Android Wi-Fi Aware | 4 KiB | +/// | ping, pong | 9 B (iOS only) | 4 KiB | +/// | unknown / future types | a v1 frame (65,535 B) stays relayable | 128 KiB | +/// +/// A future type that needs more than a v1 frame must be added here before it +/// ships, or current receivers will drop it at decode. +public enum PacketPayloadLimits { + /// Raw DEFLATE's hard expansion ceiling: every 258-byte match costs at + /// least a 1-bit length code plus a 1-bit distance code. A declared + /// original size beyond this multiple of the compressed bytes cannot be + /// produced by any stream, so it is rejected before the output buffer is + /// allocated. + static let maxDeflateRatio = 1032 + + private static let fragmentBytes = 1_280 + private static let controlBytes = 4 * 1024 + private static let requestSyncBytes = 8 * 1024 + private static let envelopeBytes = 64 * 1024 + private static let twiceV1FrameBytes = 128 * 1024 + + /// Everything a frame can carry besides its payload: the v2 header, the + /// sender and recipient, the widest route the decoder accepts (255 hops), + /// the compressed-size field, the signature, and the most padding + /// `MessagePadding` adds (iOS fragments the padded frame). + private static let maxFrameOverheadBytes = BinaryProtocol.v2HeaderSize + + BinaryProtocol.senderIDSize + + BinaryProtocol.recipientIDSize + + 1 + Int(UInt8.max) * BinaryProtocol.senderIDSize + + 4 + + BinaryProtocol.signatureSize + + Int(UInt8.max) + + /// Largest decoded payload accepted for a packet of `type`, whether it + /// arrived compressed or not. + static func maxPayloadBytes(forType type: UInt8) -> Int { + guard let messageType = MessageType(rawValue: type) else { + return twiceV1FrameBytes + } + switch messageType { + case .fileTransfer, .noiseEncrypted: + return FileTransferLimits.maxFramedFileBytes + case .message, .groupMessage: + return twiceV1FrameBytes + case .courierEnvelope, .nostrCarrier: + return envelopeBytes + case .requestSync: + return requestSyncBytes + case .fragment: + return fragmentBytes + case .announce, .announceV2, .leave, .boardPost, .prekeyBundle, + .voiceFrame, .noiseHandshake, .ping, .pong: + return controlBytes + } + } + + /// Largest encoded frame, padding included, that can still decode as a + /// packet of `type`. Fragment reassembly holds an assembly to this for + /// the type its fragments claim, so it never buffers more than the + /// reassembled packet would be allowed to decode to. + public static func maxFrameBytes(forType type: UInt8) -> Int { + maxPayloadBytes(forType: type) + maxFrameOverheadBytes + } +} diff --git a/localPackages/BitFoundation/Tests/BitFoundationTests/BinaryProtocolTests.swift b/localPackages/BitFoundation/Tests/BitFoundationTests/BinaryProtocolTests.swift index ddf7c8fc..cc984657 100644 --- a/localPackages/BitFoundation/Tests/BitFoundationTests/BinaryProtocolTests.swift +++ b/localPackages/BitFoundation/Tests/BitFoundationTests/BinaryProtocolTests.swift @@ -296,9 +296,11 @@ struct BinaryProtocolTests { func payloadCompression() throws { let repeatedString = String(repeating: "This is a test message. ", count: 200) let largePayload = Data(repeatedString.utf8) - - let packet = TestHelpers.createTestPacket(payload: largePayload) - + + // A public message: the helper's default type (announce) is capped at + // 4 KiB decompressed, below this 4.8 KB payload. + let packet = TestHelpers.createTestPacket(type: MessageType.message.rawValue, payload: largePayload) + // Encode (should compress) let encodedData = try #require(BinaryProtocol.encode(packet), "Failed to encode packet with large payload") diff --git a/localPackages/BitFoundation/Tests/BitFoundationTests/PacketPayloadLimitsTests.swift b/localPackages/BitFoundation/Tests/BitFoundationTests/PacketPayloadLimitsTests.swift new file mode 100644 index 00000000..0d83001e --- /dev/null +++ b/localPackages/BitFoundation/Tests/BitFoundationTests/PacketPayloadLimitsTests.swift @@ -0,0 +1,164 @@ +// +// PacketPayloadLimitsTests.swift +// BitFoundationTests +// +// This is free and unencumbered software released into the public domain. +// For more information, see +// + +import Foundation +import Testing +@testable import BitFoundation + +/// Per-type payload caps at decode: a payload may reach its type's cap and no +/// further, whether it inflates from a few bytes on the wire or arrives +/// uncompressed (as a reassembled fragment stream does). +struct PacketPayloadLimitsTests { + + private static let cappedTypes: [UInt8] = [ + MessageType.message.rawValue, + MessageType.groupMessage.rawValue, + MessageType.fragment.rawValue, + MessageType.announce.rawValue, + MessageType.requestSync.rawValue, + MessageType.courierEnvelope.rawValue, + MessageType.nostrCarrier.rawValue, + MessageType.boardPost.rawValue, + MessageType.noiseHandshake.rawValue, + 0x7F // unassigned: future types keep a v1 frame's worth + ] + + @Test(arguments: cappedTypes) + func compressedPayloadInflatesToItsTypeCapAndNoFurther(type: UInt8) throws { + let cap = PacketPayloadLimits.maxPayloadBytes(forType: type) + #expect(cap < FileTransferLimits.maxFramedFileBytes) + // v1 frames carry a 16-bit original size, so caps below that are + // exercised on v1 (the default for everything but media) too. + let version: UInt8 = cap + 1 <= Int(UInt16.max) ? 1 : 2 + + let atCap = try encodeCompressed(type: type, payloadSize: cap, version: version) + #expect(BinaryProtocol.decode(atCap)?.payload.count == cap) + + let pastCap = try encodeCompressed(type: type, payloadSize: cap + 1, version: version) + #expect(BinaryProtocol.decode(pastCap) == nil) + } + + @Test(arguments: cappedTypes) + func uncompressedPayloadIsCappedLikeACompressedOne(type: UInt8) throws { + let cap = PacketPayloadLimits.maxPayloadBytes(forType: type) + let version: UInt8 = cap + 1 <= Int(UInt16.max) ? 1 : 2 + + let atCap = try encodeUncompressed(type: type, payloadSize: cap, version: version) + #expect(BinaryProtocol.decode(atCap)?.payload.count == cap) + + let pastCap = try encodeUncompressed(type: type, payloadSize: cap + 1, version: version) + #expect(BinaryProtocol.decode(pastCap) == nil) + } + + @Test func onlyMediaTypesReachTheFramedFileCeiling() throws { + let size = FileTransferLimits.maxFramedFileBytes + for mediaType in [MessageType.fileTransfer, .noiseEncrypted] { + let frame = try encodeCompressed(type: mediaType.rawValue, payloadSize: size, version: 2) + #expect(BinaryProtocol.decode(frame)?.payload.count == size) + let raw = try encodeUncompressed(type: mediaType.rawValue, payloadSize: size, version: 2) + #expect(BinaryProtocol.decode(raw)?.payload.count == size) + } + + // The same ~1 KB on air as a public message would have pinned ~1.1 MB. + let message = try encodeCompressed(type: MessageType.message.rawValue, payloadSize: size, version: 2) + #expect(BinaryProtocol.decode(message) == nil) + } + + @Test func frameCeilingCoversTheLargestFrameThatStillDecodes() throws { + // An uncompressed payload at its cap with everything optional at its + // widest (recipient, a 255-hop route, signature), padded as iOS pads + // before fragmenting. Reassembly must never cut off such a frame. + for type in Self.cappedTypes + [MessageType.fileTransfer.rawValue] { + let cap = PacketPayloadLimits.maxPayloadBytes(forType: type) + let packet = BitchatPacket( + type: type, + senderID: Data(repeating: 0x11, count: BinaryProtocol.senderIDSize), + recipientID: Data(repeating: 0x22, count: BinaryProtocol.recipientIDSize), + timestamp: 1_740_000_000_000, + payload: diversePayload(count: cap), + signature: Data(repeating: 0x33, count: BinaryProtocol.signatureSize), + ttl: 7, + version: 2, + route: (0..<255).map { Data(repeating: UInt8($0), count: BinaryProtocol.senderIDSize) } + ) + let raw = try #require(BinaryProtocol.encode(packet, padding: true)) + #expect(raw[BinaryProtocol.Offsets.flags] & BinaryProtocol.Flags.isCompressed == 0) + #expect(BinaryProtocol.decode(raw)?.payload.count == cap) + #expect(raw.count <= PacketPayloadLimits.maxFrameBytes(forType: type)) + } + } + + /// AnnounceV2 isn't emitted by the app yet, so its largest payload is + /// checked here rather than in the app's interop tests. + @Test func largestAnnounceV2PayloadDecodesWithHeadroom() throws { + let payload = try #require(AnnounceV2Packet( + epoch: 1, + tagBlock: Data(repeating: 0x5A, count: AnnounceV2Packet.tagBlockLength), + capabilities: PeerCapabilities(rawValue: .max), + bridgeGeohash: "9q8yyk8yuv12" + ).encode()) + #expect(payload.count * 2 <= PacketPayloadLimits.maxPayloadBytes(forType: MessageType.announceV2.rawValue)) + + let packet = BitchatPacket( + type: MessageType.announceV2.rawValue, + senderID: Data(repeating: 0x11, count: BinaryProtocol.senderIDSize), + recipientID: nil, + timestamp: 1_740_000_000_000, + payload: payload, + signature: Data(repeating: 0x33, count: BinaryProtocol.signatureSize), + ttl: 7, + version: 1 + ) + let frame = try #require(BinaryProtocol.encode(packet, padding: true)) + #expect(BinaryProtocol.decode(frame)?.payload == payload) + } + + /// Every byte value once, then zeros: 100% byte diversity, so the encoder + /// leaves it uncompressed, while the zero run would still compress well + /// inside fragments (the reassembly bypass shape). + private func diversePayload(count: Int) -> Data { + var payload = Data((0...255).map { UInt8($0) }.prefix(count)) + payload.append(Data(count: count - payload.count)) + return payload + } + + private func encodeUncompressed(type: UInt8, payloadSize: Int, version: UInt8) throws -> Data { + let packet = BitchatPacket( + type: type, + senderID: Data(repeating: 0x11, count: BinaryProtocol.senderIDSize), + recipientID: nil, + timestamp: 1_740_000_000_000, + payload: diversePayload(count: payloadSize), + signature: nil, + ttl: 7, + version: version + ) + let frame = try #require(BinaryProtocol.encode(packet, padding: false)) + #expect(frame[BinaryProtocol.Offsets.flags] & BinaryProtocol.Flags.isCompressed == 0) + return frame + } + + /// Encodes a maximally compressible payload and checks that the frame + /// really went out compressed, so decode exercises the inflate path. + private func encodeCompressed(type: UInt8, payloadSize: Int, version: UInt8) throws -> Data { + let packet = BitchatPacket( + type: type, + senderID: Data(repeating: 0x11, count: BinaryProtocol.senderIDSize), + recipientID: nil, + timestamp: 1_740_000_000_000, + payload: Data(repeating: 0x41, count: payloadSize), + signature: nil, + ttl: 7, + version: version + ) + let frame = try #require(BinaryProtocol.encode(packet, padding: false)) + #expect(frame.count < payloadSize / 10) + #expect(frame[BinaryProtocol.Offsets.flags] & BinaryProtocol.Flags.isCompressed != 0) + return frame + } +}