From 938619fb5d60697136a1fc207e0863acd41aca8b Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Sun, 25 Oct 2026 23:16:02 +0100 Subject: [PATCH 1/5] Port the Android spam filter to iOS Hides public mesh and geohash messages from senders who post too fast, repeat themselves, copy each other, flood links or post repeated-character junk, with the same rules and defaults as bitchat-android's spam package. Hidden messages never reach the timeline and trigger no mention notification or haptic. Private and own messages are never judged. Settings get a spam filter toggle, on by default; the panic wipe resets it and forgets the filter's history. An upper bound on the alignment score and per-message comparison caps keep the cross-user check cheap on the main actor. --- bitchat/Localizable.xcstrings | 558 +++++++++++++++++ bitchat/Services/Spam/LinkCounter.swift | 24 + .../Services/Spam/RepetitionDetector.swift | 249 ++++++++ bitchat/Services/Spam/SpamFilter.swift | 327 ++++++++++ bitchat/Services/Spam/SpamFilterConfig.swift | 79 +++ .../Services/Spam/SpamFilterSettings.swift | 41 ++ .../Services/Spam/TokenBucketSchedule.swift | 93 +++ .../ChatPublicConversationCoordinator.swift | 16 +- bitchat/ViewModels/ChatViewModel.swift | 6 + bitchat/ViewModels/GeoPresenceTracker.swift | 12 + bitchat/Views/AppInfoView.swift | 24 + .../ChatNostrCoordinatorContextTests.swift | 1 + ...cConversationCoordinatorContextTests.swift | 84 +++ bitchatTests/ChatViewModelTests.swift | 3 + .../PerformanceBaselineTests.swift | 18 +- bitchatTests/SpamFilterTests.swift | 584 ++++++++++++++++++ 16 files changed, 2117 insertions(+), 2 deletions(-) create mode 100644 bitchat/Services/Spam/LinkCounter.swift create mode 100644 bitchat/Services/Spam/RepetitionDetector.swift create mode 100644 bitchat/Services/Spam/SpamFilter.swift create mode 100644 bitchat/Services/Spam/SpamFilterConfig.swift create mode 100644 bitchat/Services/Spam/SpamFilterSettings.swift create mode 100644 bitchat/Services/Spam/TokenBucketSchedule.swift create mode 100644 bitchatTests/SpamFilterTests.swift diff --git a/bitchat/Localizable.xcstrings b/bitchat/Localizable.xcstrings index 601d5b6a..4ccc88f6 100644 --- a/bitchat/Localizable.xcstrings +++ b/bitchat/Localizable.xcstrings @@ -31798,6 +31798,564 @@ } } }, + "app_info.settings.public_chats.title" : { + "comment" : "Section header (uppercase) for settings about public chats, such as the spam filter", + "extractionState" : "manual", + "localizations" : { + "ar" : { + "stringUnit" : { + "state" : "translated", + "value" : "الدردشات العامة" + } + }, + "bn" : { + "stringUnit" : { + "state" : "translated", + "value" : "পাবলিক চ্যাট" + } + }, + "de" : { + "stringUnit" : { + "state" : "translated", + "value" : "ÖFFENTLICHE CHATS" + } + }, + "en" : { + "stringUnit" : { + "state" : "translated", + "value" : "PUBLIC CHATS" + } + }, + "es" : { + "stringUnit" : { + "state" : "translated", + "value" : "CHATS PÚBLICOS" + } + }, + "fa" : { + "stringUnit" : { + "state" : "translated", + "value" : "گفتگوهای عمومی" + } + }, + "fil" : { + "stringUnit" : { + "state" : "translated", + "value" : "MGA PUBLIC NA CHAT" + } + }, + "fr" : { + "stringUnit" : { + "state" : "translated", + "value" : "DISCUSSIONS PUBLIQUES" + } + }, + "he" : { + "stringUnit" : { + "state" : "translated", + "value" : "צ'אטים ציבוריים" + } + }, + "hi" : { + "stringUnit" : { + "state" : "translated", + "value" : "सार्वजनिक चैट" + } + }, + "id" : { + "stringUnit" : { + "state" : "translated", + "value" : "OBROLAN PUBLIK" + } + }, + "it" : { + "stringUnit" : { + "state" : "translated", + "value" : "CHAT PUBBLICHE" + } + }, + "ja" : { + "stringUnit" : { + "state" : "translated", + "value" : "公開チャット" + } + }, + "ko" : { + "stringUnit" : { + "state" : "translated", + "value" : "공개 채팅" + } + }, + "ms" : { + "stringUnit" : { + "state" : "translated", + "value" : "SEMBANG AWAM" + } + }, + "ne" : { + "stringUnit" : { + "state" : "translated", + "value" : "सार्वजनिक च्याट" + } + }, + "nl" : { + "stringUnit" : { + "state" : "translated", + "value" : "OPENBARE CHATS" + } + }, + "pl" : { + "stringUnit" : { + "state" : "translated", + "value" : "CZATY PUBLICZNE" + } + }, + "pt" : { + "stringUnit" : { + "state" : "translated", + "value" : "CONVERSAS PÚBLICAS" + } + }, + "pt-BR" : { + "stringUnit" : { + "state" : "translated", + "value" : "CHATS PÚBLICOS" + } + }, + "ru" : { + "stringUnit" : { + "state" : "translated", + "value" : "ПУБЛИЧНЫЕ ЧАТЫ" + } + }, + "sv" : { + "stringUnit" : { + "state" : "translated", + "value" : "OFFENTLIGA CHATTAR" + } + }, + "ta" : { + "stringUnit" : { + "state" : "translated", + "value" : "பொது அரட்டைகள்" + } + }, + "th" : { + "stringUnit" : { + "state" : "translated", + "value" : "แชทสาธารณะ" + } + }, + "tr" : { + "stringUnit" : { + "state" : "translated", + "value" : "HERKESE AÇIK SOHBETLER" + } + }, + "uk" : { + "stringUnit" : { + "state" : "translated", + "value" : "ПУБЛІЧНІ ЧАТИ" + } + }, + "ur" : { + "stringUnit" : { + "state" : "translated", + "value" : "عوامی چیٹس" + } + }, + "vi" : { + "stringUnit" : { + "state" : "translated", + "value" : "TRÒ CHUYỆN CÔNG KHAI" + } + }, + "zh-Hans" : { + "stringUnit" : { + "state" : "translated", + "value" : "公开聊天" + } + }, + "zh-Hant" : { + "stringUnit" : { + "state" : "translated", + "value" : "公開聊天" + } + } + } + }, + "app_info.settings.spam_filter.title" : { + "comment" : "Title of the setting that hides spam in public mesh and location channels", + "extractionState" : "manual", + "localizations" : { + "ar" : { + "stringUnit" : { + "state" : "translated", + "value" : "مرشّح الرسائل المزعجة" + } + }, + "bn" : { + "stringUnit" : { + "state" : "translated", + "value" : "স্প্যাম ফিল্টার" + } + }, + "de" : { + "stringUnit" : { + "state" : "translated", + "value" : "spamfilter" + } + }, + "en" : { + "stringUnit" : { + "state" : "translated", + "value" : "spam filter" + } + }, + "es" : { + "stringUnit" : { + "state" : "translated", + "value" : "filtro de spam" + } + }, + "fa" : { + "stringUnit" : { + "state" : "translated", + "value" : "فیلتر هرزنامه" + } + }, + "fil" : { + "stringUnit" : { + "state" : "translated", + "value" : "spam filter" + } + }, + "fr" : { + "stringUnit" : { + "state" : "translated", + "value" : "filtre anti-spam" + } + }, + "he" : { + "stringUnit" : { + "state" : "translated", + "value" : "מסנן ספאם" + } + }, + "hi" : { + "stringUnit" : { + "state" : "translated", + "value" : "स्पैम फ़िल्टर" + } + }, + "id" : { + "stringUnit" : { + "state" : "translated", + "value" : "filter spam" + } + }, + "it" : { + "stringUnit" : { + "state" : "translated", + "value" : "filtro antispam" + } + }, + "ja" : { + "stringUnit" : { + "state" : "translated", + "value" : "スパムフィルター" + } + }, + "ko" : { + "stringUnit" : { + "state" : "translated", + "value" : "스팸 필터" + } + }, + "ms" : { + "stringUnit" : { + "state" : "translated", + "value" : "penapis spam" + } + }, + "ne" : { + "stringUnit" : { + "state" : "translated", + "value" : "स्प्याम फिल्टर" + } + }, + "nl" : { + "stringUnit" : { + "state" : "translated", + "value" : "spamfilter" + } + }, + "pl" : { + "stringUnit" : { + "state" : "translated", + "value" : "filtr spamu" + } + }, + "pt" : { + "stringUnit" : { + "state" : "translated", + "value" : "filtro de spam" + } + }, + "pt-BR" : { + "stringUnit" : { + "state" : "translated", + "value" : "filtro de spam" + } + }, + "ru" : { + "stringUnit" : { + "state" : "translated", + "value" : "фильтр спама" + } + }, + "sv" : { + "stringUnit" : { + "state" : "translated", + "value" : "spamfilter" + } + }, + "ta" : { + "stringUnit" : { + "state" : "translated", + "value" : "ஸ்பேம் வடிகட்டி" + } + }, + "th" : { + "stringUnit" : { + "state" : "translated", + "value" : "ตัวกรองสแปม" + } + }, + "tr" : { + "stringUnit" : { + "state" : "translated", + "value" : "spam filtresi" + } + }, + "uk" : { + "stringUnit" : { + "state" : "translated", + "value" : "фільтр спаму" + } + }, + "ur" : { + "stringUnit" : { + "state" : "translated", + "value" : "اسپام فلٹر" + } + }, + "vi" : { + "stringUnit" : { + "state" : "translated", + "value" : "bộ lọc spam" + } + }, + "zh-Hans" : { + "stringUnit" : { + "state" : "translated", + "value" : "垃圾消息过滤" + } + }, + "zh-Hant" : { + "stringUnit" : { + "state" : "translated", + "value" : "垃圾訊息過濾" + } + } + } + }, + "app_info.settings.spam_filter.subtitle" : { + "comment" : "Subtitle explaining what the spam filter hides and that private messages are never filtered", + "extractionState" : "manual", + "localizations" : { + "ar" : { + "stringUnit" : { + "state" : "translated", + "value" : "يخفي الرسائل العامة من الأشخاص الذين ينشرون بسرعة كبيرة أو يكررون أنفسهم أو يغرقون الدردشة بالروابط. لا تتم تصفية الرسائل الخاصة أبدًا. مفعّل افتراضيًا." + } + }, + "bn" : { + "stringUnit" : { + "state" : "translated", + "value" : "যারা খুব দ্রুত পোস্ট করে, একই কথা বারবার বলে বা লিংকে ভরিয়ে দেয়, তাদের পাবলিক বার্তা লুকায়। ব্যক্তিগত বার্তা কখনো ফিল্টার হয় না। ডিফল্টভাবে চালু।" + } + }, + "de" : { + "stringUnit" : { + "state" : "translated", + "value" : "blendet öffentliche nachrichten von leuten aus, die zu schnell posten, sich wiederholen oder links fluten. private nachrichten werden nie gefiltert. standardmäßig an." + } + }, + "en" : { + "stringUnit" : { + "state" : "translated", + "value" : "hides public messages from people who post too fast, repeat themselves, or flood links. private messages are never filtered. on by default." + } + }, + "es" : { + "stringUnit" : { + "state" : "translated", + "value" : "oculta mensajes públicos de quienes publican demasiado rápido, se repiten o inundan el chat de enlaces. los mensajes privados nunca se filtran. activado por defecto." + } + }, + "fa" : { + "stringUnit" : { + "state" : "translated", + "value" : "پیام‌های عمومی افرادی را که خیلی سریع پست می‌کنند، حرفشان را تکرار می‌کنند یا گفتگو را با پیوند پر می‌کنند پنهان می‌کند. پیام‌های خصوصی هرگز فیلتر نمی‌شوند. به‌طور پیش‌فرض روشن است." + } + }, + "fil" : { + "stringUnit" : { + "state" : "translated", + "value" : "itinatago ang mga public na mensahe mula sa mga taong masyadong mabilis mag-post, paulit-ulit, o nagbabaha ng link. hindi kailanman sinasala ang mga pribadong mensahe. naka-on bilang default." + } + }, + "fr" : { + "stringUnit" : { + "state" : "translated", + "value" : "masque les messages publics des personnes qui publient trop vite, se répètent ou inondent le chat de liens. les messages privés ne sont jamais filtrés. activé par défaut." + } + }, + "he" : { + "stringUnit" : { + "state" : "translated", + "value" : "מסתיר הודעות ציבוריות של אנשים שמפרסמים מהר מדי, חוזרים על עצמם או מציפים בקישורים. הודעות פרטיות אף פעם לא מסוננות. פעיל כברירת מחדל." + } + }, + "hi" : { + "stringUnit" : { + "state" : "translated", + "value" : "बहुत तेज़ी से पोस्ट करने वालों, बार-बार एक ही बात दोहराने वालों या लिंक की बाढ़ लाने वालों के सार्वजनिक संदेश छुपाता है। निजी संदेश कभी फ़िल्टर नहीं होते। डिफ़ॉल्ट रूप से चालू।" + } + }, + "id" : { + "stringUnit" : { + "state" : "translated", + "value" : "menyembunyikan pesan publik dari orang yang memposting terlalu cepat, mengulang-ulang, atau membanjiri tautan. pesan pribadi tidak pernah difilter. aktif secara default." + } + }, + "it" : { + "stringUnit" : { + "state" : "translated", + "value" : "nasconde i messaggi pubblici di chi scrive troppo in fretta, si ripete o inonda la chat di link. i messaggi privati non vengono mai filtrati. attivo per impostazione predefinita." + } + }, + "ja" : { + "stringUnit" : { + "state" : "translated", + "value" : "投稿が速すぎる人、同じ内容を繰り返す人、リンクを大量に送る人の公開メッセージを非表示にします。プライベートメッセージがフィルタされることはありません。デフォルトでオン。" + } + }, + "ko" : { + "stringUnit" : { + "state" : "translated", + "value" : "너무 빠르게 올리거나, 같은 말을 반복하거나, 링크를 도배하는 사람의 공개 메시지를 숨깁니다. 비공개 메시지는 절대 필터링되지 않습니다. 기본값은 켜짐입니다." + } + }, + "ms" : { + "stringUnit" : { + "state" : "translated", + "value" : "menyembunyikan mesej awam daripada orang yang menghantar terlalu laju, mengulang-ulang, atau membanjiri pautan. mesej peribadi tidak pernah ditapis. dihidupkan secara lalai." + } + }, + "ne" : { + "stringUnit" : { + "state" : "translated", + "value" : "धेरै छिटो पोस्ट गर्ने, एउटै कुरा दोहोर्‍याउने वा लिङ्कले भरिदिने मानिसहरूका सार्वजनिक सन्देश लुकाउँछ। निजी सन्देश कहिल्यै फिल्टर हुँदैनन्। पूर्वनिर्धारित रूपमा अन।" + } + }, + "nl" : { + "stringUnit" : { + "state" : "translated", + "value" : "verbergt openbare berichten van mensen die te snel posten, zichzelf herhalen of links spammen. privéberichten worden nooit gefilterd. standaard aan." + } + }, + "pl" : { + "stringUnit" : { + "state" : "translated", + "value" : "ukrywa publiczne wiadomości od osób, które piszą za szybko, powtarzają się lub zalewają czat linkami. wiadomości prywatne nigdy nie są filtrowane. domyślnie włączony." + } + }, + "pt" : { + "stringUnit" : { + "state" : "translated", + "value" : "oculta mensagens públicas de quem publica depressa demais, se repete ou inunda a conversa de ligações. as mensagens privadas nunca são filtradas. ativado por predefinição." + } + }, + "pt-BR" : { + "stringUnit" : { + "state" : "translated", + "value" : "oculta mensagens públicas de quem posta rápido demais, se repete ou inunda o chat de links. mensagens privadas nunca são filtradas. ativado por padrão." + } + }, + "ru" : { + "stringUnit" : { + "state" : "translated", + "value" : "скрывает публичные сообщения тех, кто пишет слишком часто, повторяется или заваливает чат ссылками. личные сообщения никогда не фильтруются. включено по умолчанию." + } + }, + "sv" : { + "stringUnit" : { + "state" : "translated", + "value" : "döljer offentliga meddelanden från personer som skriver för snabbt, upprepar sig eller översvämmar chatten med länkar. privata meddelanden filtreras aldrig. på som standard." + } + }, + "ta" : { + "stringUnit" : { + "state" : "translated", + "value" : "மிக வேகமாகப் பதிவிடுபவர்கள், திரும்பத் திரும்ப ஒரே செய்தியை அனுப்புபவர்கள் அல்லது இணைப்புகளால் நிரப்புபவர்களின் பொதுச் செய்திகளை மறைக்கிறது. தனிப்பட்ட செய்திகள் ஒருபோதும் வடிகட்டப்படாது. இயல்பாக இயக்கத்தில் உள்ளது." + } + }, + "th" : { + "stringUnit" : { + "state" : "translated", + "value" : "ซ่อนข้อความสาธารณะจากคนที่โพสต์เร็วเกินไป พูดซ้ำ หรือส่งลิงก์รัว ๆ ข้อความส่วนตัวจะไม่ถูกกรองเลย เปิดเป็นค่าเริ่มต้น" + } + }, + "tr" : { + "stringUnit" : { + "state" : "translated", + "value" : "çok hızlı yazan, kendini tekrarlayan veya bağlantı yağdıran kişilerin herkese açık mesajlarını gizler. özel mesajlar asla filtrelenmez. varsayılan olarak açık." + } + }, + "uk" : { + "stringUnit" : { + "state" : "translated", + "value" : "приховує публічні повідомлення тих, хто пише надто часто, повторюється або засипає чат посиланнями. особисті повідомлення ніколи не фільтруються. увімкнено за замовчуванням." + } + }, + "ur" : { + "stringUnit" : { + "state" : "translated", + "value" : "بہت تیزی سے پوسٹ کرنے والوں، بار بار ایک ہی بات دہرانے والوں یا لنکس کی بھرمار کرنے والوں کے عوامی پیغامات چھپاتا ہے۔ نجی پیغامات کبھی فلٹر نہیں ہوتے۔ پہلے سے آن۔" + } + }, + "vi" : { + "stringUnit" : { + "state" : "translated", + "value" : "ẩn tin nhắn công khai của những người đăng quá nhanh, lặp lại chính mình hoặc spam liên kết. tin nhắn riêng không bao giờ bị lọc. bật theo mặc định." + } + }, + "zh-Hans" : { + "stringUnit" : { + "state" : "translated", + "value" : "隐藏发言太快、重复刷屏或大量发链接的人的公开消息。私信永远不会被过滤。默认开启。" + } + }, + "zh-Hant" : { + "stringUnit" : { + "state" : "translated", + "value" : "隱藏發言太快、重複洗版或大量貼連結的人的公開訊息。私訊永遠不會被過濾。預設開啟。" + } + } + } + }, "app_info.tab.info" : { "extractionState" : "manual", "localizations" : { diff --git a/bitchat/Services/Spam/LinkCounter.swift b/bitchat/Services/Spam/LinkCounter.swift new file mode 100644 index 00000000..bbb52f06 --- /dev/null +++ b/bitchat/Services/Spam/LinkCounter.swift @@ -0,0 +1,24 @@ +// +// LinkCounter.swift +// bitchat +// +// This is free and unencumbered software released into the public domain. +// For more information, see +// + +import Foundation + +/// Counts links: scheme URLs, `www.` hosts and bare domains such as +/// `example.com/path`. Email addresses and abbreviations like `e.g.` don't +/// count. Same pattern as Android, so both apps rate-limit the same messages. +enum LinkCounter { + private static let link: NSRegularExpression? = try? NSRegularExpression( + pattern: #"(?]+|www\.[^\s<>]+|[a-z0-9\-]+(?:\.[a-z0-9\-]+)*\.[a-z]{2,}(?:/[^\s<>]*)?)(?![\w@])"#, + options: [.caseInsensitive] + ) + + static func count(_ text: String) -> Int { + guard let link else { return 0 } + return link.numberOfMatches(in: text, range: NSRange(text.startIndex..., in: text)) + } +} diff --git a/bitchat/Services/Spam/RepetitionDetector.swift b/bitchat/Services/Spam/RepetitionDetector.swift new file mode 100644 index 00000000..35bbd575 --- /dev/null +++ b/bitchat/Services/Spam/RepetitionDetector.swift @@ -0,0 +1,249 @@ +// +// RepetitionDetector.swift +// bitchat +// +// This is free and unencumbered software released into the public domain. +// For more information, see +// + +import Foundation + +/// Character autocorrelation measures for repeated-content spam. +/// +/// Text is compared as Unicode scalars after `normalize`, so case, spacing, +/// look-alike letters, accents and invisible characters don't hide a repeat. +enum RepetitionDetector { + + /// Folds a message to what a reader sees, for comparison only (never for + /// display): NFKD turns fullwidth and math-styled letters into plain ones + /// and splits off accents, then combining marks and invisible characters + /// are dropped, whitespace runs become one space and the result is + /// lowercased and capped to `maxLength` scalars. + static func normalize(_ text: String, maxLength: Int) -> [Int] { + // Disguises add at most a few scalars per visible character; bound the work up front. + let scalars = text.unicodeScalars + let head = scalars.count > maxLength * 4 + ? String(String.UnicodeScalarView(scalars.prefix(maxLength * 4))) + : text + var out: [Int] = [] + out.reserveCapacity(min(maxLength, head.unicodeScalars.count)) + var pendingSpace = false + for scalar in head.decomposedStringWithCompatibilityMapping.unicodeScalars { + if out.count >= maxLength { break } + if isMark(scalar) || isInvisible(scalar) { continue } + if isWhitespace(scalar) { + pendingSpace = !out.isEmpty + continue + } + if pendingSpace && out.count < maxLength - 1 { out.append(0x20) } + pendingSpace = false + out.append(lowercase(scalar)) + } + return out + } + + private static func isWhitespace(_ scalar: Unicode.Scalar) -> Bool { + scalar.properties.isWhitespace + } + + /// Zero-width and direction controls, soft hyphens and other non-printing characters. + private static func isInvisible(_ scalar: Unicode.Scalar) -> Bool { + switch scalar.properties.generalCategory { + case .format: return true + case .control: return !isWhitespace(scalar) + default: return false + } + } + + private static func isMark(_ scalar: Unicode.Scalar) -> Bool { + switch scalar.properties.generalCategory { + case .nonspacingMark, .spacingMark, .enclosingMark: return true + default: return false + } + } + + /// Simple one-to-one lowercase mapping; multi-scalar mappings keep the original. + private static func lowercase(_ scalar: Unicode.Scalar) -> Int { + let mapped = scalar.properties.lowercaseMapping.unicodeScalars + guard mapped.count == 1, let first = mapped.first else { return Int(scalar.value) } + return Int(first.value) + } + + /// Highest autocorrelation of `text` with itself over shifts 1...n/2: the + /// fraction of overlapping positions where `text[i] == text[i + lag]`. A + /// pattern repeated back to back ("aaaa", "buy now buy now") scores 1.0; + /// ordinary prose stays well below 0.3. + static func selfSimilarity(_ text: [Int]) -> Double { + let n = text.count + guard n >= 2 else { return 0 } + var best = 0.0 + for lag in 1...(n / 2) { + let overlap = n - lag + var matches = 0 + for i in 0.. Double { + align(a, b, maxPeaks: maxPeaks, minRun: minRun).score + } + + /// Cross-correlation of two texts that tolerates small edits. + /// + /// The first peak is the classic correlation maximum, the shift where the + /// most characters match. An inserted or deleted character moves the rest + /// of the text onto a neighboring shift, so up to `maxPeaks - 1` further + /// peaks add matches from other shifts. Those count only runs of at least + /// `minRun` consecutive characters, so coincidental single letters don't + /// add up, and no character of either text is used twice. The score is the + /// matched characters over the longer length: 1.0 for identical texts, and + /// a short text never matches a long one. + static func align(_ a: [Int], _ b: [Int], maxPeaks: Int = 3, minRun: Int = 3) -> Alignment { + guard !a.isEmpty, !b.isEmpty else { return Alignment(peaks: [], matches: 0, score: 0) } + var usedA = [Bool](repeating: false, count: a.count) + var usedB = [Bool](repeating: false, count: b.count) + var peaks: [Peak] = [] + var total = 0 + while peaks.count < maxPeaks { + let run = peaks.isEmpty ? 1 : minRun + var bestGain = 0 + var bestShift = 0 + for shift in -(b.count - 1).. bestGain { + bestGain = gain + bestShift = shift + } + } + if bestGain < run { break } + let from = max(0, -bestShift) + let to = min(b.count, a.count - bestShift) + _ = runMatches(a, b, shift: bestShift, from: from, to: to, + usedA: &usedA, usedB: &usedB, minRun: run, mark: true) + peaks.append(Peak(shift: bestShift, matches: bestGain)) + total += bestGain + } + return Alignment(peaks: peaks, matches: total, score: Double(total) / Double(max(a.count, b.count))) + } + + /// Unused matches at `shift` that sit in runs of at least `minRun`; `mark` claims them. + private static func runMatches( + _ a: [Int], + _ b: [Int], + shift: Int, + from: Int, + to: Int, + usedA: inout [Bool], + usedB: inout [Bool], + minRun: Int, + mark: Bool + ) -> Int { + var gain = 0 + var runStart = -1 + for j in from...to { + let hit = j < to && !usedB[j] && !usedA[j + shift] && a[j + shift] == b[j] + if hit { + if runStart < 0 { runStart = j } + } else if runStart >= 0 { + let length = j - runStart + if length >= minRun { + gain += length + if mark { + for k in runStart.. [Int: [Int]] { + var positions: [Int: [Int]] = [:] + for (index, point) in text.enumerated() { positions[point, default: []].append(index) } + return positions + } + + /// An upper bound on `align(a, b, maxPeaks:).matches`, at a fraction of its cost. + /// + /// Every peak sits on a different shift (a claimed shift has nothing left + /// to claim) and gains at most the matches on its diagonal, so the + /// `maxPeaks` fullest diagonals bound the total. Counting every diagonal + /// takes one pass over the character pairs that match, instead of one pass + /// over the overlap per shift and peak. Pairs whose bound falls short of + /// the near-copy threshold can skip `align` without changing its verdict. + static func alignmentUpperBound(_ a: [Int], _ b: [Int], positionsInB: [Int: [Int]], maxPeaks: Int) -> Int { + guard !a.isEmpty, !b.isEmpty else { return 0 } + // Diagonal `shift` (a[j + shift] == b[j]) is stored at shift + b.count - 1. + var diagonals = [Int](repeating: 0, count: a.count + b.count - 1) + let offset = b.count - 1 + diagonals.withUnsafeMutableBufferPointer { counts in + for (i, point) in a.enumerated() { + guard let matches = positionsInB[point] else { continue } + for j in matches { counts[i - j + offset] += 1 } + } + } + var top = [Int](repeating: 0, count: max(1, maxPeaks)) + for count in diagonals where count > top[top.count - 1] { + var k = top.count - 1 + while k > 0 && top[k - 1] < count { + top[k] = top[k - 1] + k -= 1 + } + top[k] = count + } + return min(top.reduce(0, +), min(a.count, b.count)) + } + + /// Distinct character trigrams as sorted hashes, for a cheap pre-check before `align`. + static func trigrams(_ text: [Int]) -> [Int] { + guard text.count >= 3 else { return [] } + let hashes = (0..<(text.count - 2)).map { i in (text[i] * 31 + text[i + 1]) * 31 + text[i + 2] } + return Array(Set(hashes)).sorted() + } + + /// Shared trigrams over the larger trigram set. Near-copies share most of + /// theirs, unrelated texts few, so pairs below a low bar can skip the + /// quadratic `align`. Texts too short to have trigrams always pass. + static func trigramOverlap(_ a: [Int], _ b: [Int]) -> Double { + guard !a.isEmpty, !b.isEmpty else { return 1 } + var i = 0 + var j = 0 + var shared = 0 + while i < a.count && j < b.count { + if a[i] == b[j] { + shared += 1 + i += 1 + j += 1 + } else if a[i] < b[j] { + i += 1 + } else { + j += 1 + } + } + return Double(shared) / Double(max(a.count, b.count)) + } +} diff --git a/bitchat/Services/Spam/SpamFilter.swift b/bitchat/Services/Spam/SpamFilter.swift new file mode 100644 index 00000000..69799ebe --- /dev/null +++ b/bitchat/Services/Spam/SpamFilter.swift @@ -0,0 +1,327 @@ +// +// SpamFilter.swift +// bitchat +// +// This is free and unencumbered software released into the public domain. +// For more information, see +// + +import BitFoundation +import Foundation + +enum SpamVerdict: Equatable { + case allowed + case messageRate + case characterRate + case linkRate + case repeatedMessage + case repeatedAcrossUsers + case repeatedCharacters + + var isSpam: Bool { self != .allowed } +} + +/// Decides whether an incoming public message should be hidden. +/// +/// Each sender gets three token buckets (messages, characters, links; see +/// `SpamFilterConfig`) plus three repetition checks: a message that repeats +/// itself internally, a near-copy of the sender's own recent history, and a +/// near-copy of what *other* senders posted recently. The last one catches spam +/// spread over many throwaway identities, which per-sender limits cannot see. +/// Hidden messages never spend tokens, but they do enter the repetition history +/// so a spammer cannot cycle the same text. +/// +/// Verdicts are remembered by message ID, so the same message arriving over +/// several links or relays is judged once and counted once. +/// +/// Mirrors bitchat-android's `spam` package, with the same defaults. +final class SpamFilter: @unchecked Sendable { + + /// Process-wide filter shared by the mesh and geohash intake, starting + /// from the user's saved choice. + static let shared: SpamFilter = { + let filter = SpamFilter() + filter.enabled = SpamFilterSettings.isEnabled(in: .standard) + return filter + }() + + /// A message prepared for comparison: its leading characters and their trigrams. + private struct Fingerprint { + let time: TimeInterval + let text: [Int] + let trigrams: [Int] + let positions: [Int: [Int]] + } + + private struct SharedEntry { + let senderKey: String + let fingerprint: Fingerprint + } + + private struct SenderState { + var rate: TokenBucketSchedule + var history: [Fingerprint] = [] + var lastActive: UInt64 + } + + private let lock = NSLock() + private let clock: () -> Date + private var _enabled = true + private var _config: SpamFilterConfig + private var senders: [String: SenderState] = [:] + private var activity: UInt64 = 0 + /// Recent texts from every sender, for the cross-user repeat check. + private var recentTexts: [SharedEntry] = [] + private var verdicts: [String: SpamVerdict] = [:] + private var verdictOrder: [String] = [] + private var verdictHead = 0 + + init(config: SpamFilterConfig = SpamFilterConfig(), clock: @escaping () -> Date = Date.init) { + self._config = config + self.clock = clock + } + + var enabled: Bool { + get { lock.withLock { _enabled } } + set { lock.withLock { _enabled = newValue } } + } + + /// Changing the limits forgets every sender, so new budgets start full. + var config: SpamFilterConfig { + get { lock.withLock { _config } } + set { + lock.withLock { + _config = newValue + clearLocked() + } + } + } + + /// Returns true when the message must be kept out of the chat. + /// `senderKey` must identify the author stably within a transport. + func shouldHide(_ message: BitchatMessage, senderKey: String) -> Bool { + check(message, senderKey: senderKey).isSpam + } + + func check(_ message: BitchatMessage, senderKey: String) -> SpamVerdict { + check( + senderKey: senderKey, + messageID: message.id, + content: message.content, + timestamp: message.timestamp + ) + } + + func check( + senderKey: String, + messageID: String, + content: String, + timestamp: Date, + isText: Bool = true + ) -> SpamVerdict { + lock.lock() + defer { lock.unlock() } + guard _enabled else { return .allowed } + if let verdict = verdicts[messageID] { return verdict } + + let config = _config + let now = clock().timeIntervalSince1970 + let oldestTrusted = now - config.maxBackdate + let time = min(max(timestamp.timeIntervalSince1970, oldestTrusted), now) + var state = senders[senderKey] ?? SenderState( + rate: TokenBucketSchedule(buckets: [config.messageBucket, config.characterBucket, config.linkBucket]), + lastActive: 0 + ) + activity += 1 + state.lastActive = activity + state.rate.prune(before: oldestTrusted) + + let repetition = config.repetition + let text = isText ? RepetitionDetector.normalize(content, maxLength: repetition.maxAnalyzedLength) : nil + let fingerprint = text.map { fingerprint($0, time: time, config: repetition) } + + let verdict: SpamVerdict + if let text, let fingerprint { + if hasRepeatedCharacters(text, config: repetition) { + verdict = .repeatedCharacters + } else if repeatsHistory(state, length: text.count, message: fingerprint, config: repetition) { + verdict = .repeatedMessage + } else if repeatsOtherSenders(senderKey, length: text.count, message: fingerprint, config: repetition) { + verdict = .repeatedAcrossUsers + } else { + verdict = admitRate(&state, content: content, time: time, isText: isText) + } + state.history.append(fingerprint) + if state.history.count > repetition.historySize { + state.history.removeFirst(state.history.count - repetition.historySize) + } + if text.count >= repetition.crossUserMinLength { + recentTexts.append(SharedEntry(senderKey: senderKey, fingerprint: fingerprint)) + if recentTexts.count > repetition.crossUserHistorySize { + recentTexts.removeFirst(recentTexts.count - repetition.crossUserHistorySize) + } + } + } else { + verdict = admitRate(&state, content: content, time: time, isText: isText) + } + + senders[senderKey] = state + evictSendersIfNeeded(limit: config.maxTrackedSenders) + remember(verdict, for: messageID, limit: config.maxRememberedVerdicts) + return verdict + } + + /// Whether a message already checked was hidden, without judging or + /// counting it again. Lets mention notifications and haptics, which run + /// after intake, skip what intake dropped. + func wasHidden(messageID: String) -> Bool { + lock.withLock { _enabled && verdicts[messageID]?.isSpam == true } + } + + /// Forgets all senders, e.g. when the filter is toggled or the app is wiped. + func reset() { + lock.withLock { clearLocked() } + } + + /// Key for geohash messages, by the sender's per-geohash Nostr pubkey. + /// Matches the public intake's sender key, so both geohash paths share + /// one budget per sender. + static func nostrSenderKey(pubkeyHex: String) -> String { + "nostr:" + pubkeyHex.lowercased() + } + + // MARK: - Checks + + private func clearLocked() { + senders.removeAll() + recentTexts.removeAll() + verdicts.removeAll() + verdictOrder.removeAll() + verdictHead = 0 + activity = 0 + } + + private func hasRepeatedCharacters(_ text: [Int], config: RepetitionConfig) -> Bool { + text.count >= config.minRepeatedCharactersLength && + RepetitionDetector.selfSimilarity(text) >= config.repeatedCharactersThreshold + } + + private func fingerprint(_ text: [Int], time: TimeInterval, config: RepetitionConfig) -> Fingerprint { + let compared = text.count > config.maxComparedLength ? Array(text.prefix(config.maxComparedLength)) : text + return Fingerprint( + time: time, + text: compared, + trigrams: RepetitionDetector.trigrams(compared), + positions: RepetitionDetector.positions(compared) + ) + } + + private func repeatsHistory(_ state: SenderState, length: Int, message: Fingerprint, config: RepetitionConfig) -> Bool { + guard length >= config.minRepeatedMessageLength else { return false } + // At most `historySize` comparisons, so no budget is needed. + var budget = ComparisonBudget(comparisons: .max, alignments: .max) + let copies = state.history.filter { isNearCopy($0, message, config: config, budget: &budget) }.count + return copies > config.allowedRepeats + } + + /// Scans newest first, so the comparison budget falls on the oldest texts + /// if a burst of look-alikes ever exhausts it. + private func repeatsOtherSenders(_ senderKey: String, length: Int, message: Fingerprint, config: RepetitionConfig) -> Bool { + guard length >= config.crossUserMinLength else { return false } + var copies = 0 + var budget = ComparisonBudget( + comparisons: config.crossUserMaxComparisons, + alignments: config.crossUserMaxAlignments + ) + for entry in recentTexts.reversed() where entry.senderKey != senderKey { + guard !budget.isSpent else { break } + guard isNearCopy(entry.fingerprint, message, config: config, budget: &budget) else { continue } + copies += 1 + if copies > config.crossUserAllowedRepeats { return true } + } + return false + } + + /// Work left for comparing one message: pairs that get past the trigram + /// check, and of those, pairs that need the full alignment. + private struct ComparisonBudget { + var comparisons: Int + var alignments: Int + + var isSpent: Bool { comparisons <= 0 || alignments <= 0 } + } + + /// The cheap checks (time, length, shared trigrams) cost nothing from + /// `budget`; the diagonal bound spends a comparison and `align` an alignment. + private func isNearCopy( + _ earlier: Fingerprint, + _ message: Fingerprint, + config: RepetitionConfig, + budget: inout ComparisonBudget + ) -> Bool { + let longer = max(earlier.text.count, message.text.count) + guard abs(earlier.time - message.time) <= config.historyWindow, + lengthsCanMatch(earlier.text.count, message.text.count, config: config), + RepetitionDetector.trigramOverlap(earlier.trigrams, message.trigrams) >= config.minSharedTrigrams + else { return false } + budget.comparisons -= 1 + guard Double(RepetitionDetector.alignmentUpperBound( + message.text, + earlier.text, + positionsInB: earlier.positions, + maxPeaks: config.maxAlignmentPeaks + )) / Double(longer) >= config.repeatedMessageSimilarity + else { return false } + budget.alignments -= 1 + return RepetitionDetector.crossSimilarity( + earlier.text, + message.text, + maxPeaks: config.maxAlignmentPeaks, + minRun: config.minPeakRun + ) >= config.repeatedMessageSimilarity + } + + // Cross-similarity is bounded by shorter/longer, so skip pairs that can never reach it. + private func lengthsCanMatch(_ a: Int, _ b: Int, config: RepetitionConfig) -> Bool { + guard max(a, b) > 0 else { return false } + return Double(min(a, b)) / Double(max(a, b)) >= config.repeatedMessageSimilarity + } + + private func admitRate(_ state: inout SenderState, content: String, time: TimeInterval, isText: Bool) -> SpamVerdict { + let characters = isText ? Double(content.unicodeScalars.count) : 0 + let links = isText ? Double(LinkCounter.count(content)) : 0 + switch state.rate.tryAdmit(at: time, costs: [1, characters, links]) { + case nil: return .allowed + case 0: return .messageRate + case 1: return .characterRate + default: return .linkRate + } + } + + // MARK: - Bookkeeping + + /// Drops the least recently active senders once over `limit`, in one pass + /// down to 3/4 of it so the sort runs rarely. + private func evictSendersIfNeeded(limit: Int) { + guard senders.count > limit else { return } + let keep = max(1, (limit * 3) / 4) + let evicted = senders + .sorted { $0.value.lastActive < $1.value.lastActive } + .prefix(senders.count - keep) + for (key, _) in evicted { senders.removeValue(forKey: key) } + } + + private func remember(_ verdict: SpamVerdict, for messageID: String, limit: Int) { + verdicts[messageID] = verdict + verdictOrder.append(messageID) + while verdictOrder.count - verdictHead > limit { + verdicts.removeValue(forKey: verdictOrder[verdictHead]) + verdictHead += 1 + } + // Compact once the consumed prefix dominates, keeping appends amortized O(1). + if verdictHead > limit { + verdictOrder.removeFirst(verdictHead) + verdictHead = 0 + } + } +} diff --git a/bitchat/Services/Spam/SpamFilterConfig.swift b/bitchat/Services/Spam/SpamFilterConfig.swift new file mode 100644 index 00000000..e89086fc --- /dev/null +++ b/bitchat/Services/Spam/SpamFilterConfig.swift @@ -0,0 +1,79 @@ +// +// SpamFilterConfig.swift +// bitchat +// +// This is free and unencumbered software released into the public domain. +// For more information, see +// + +import Foundation + +/// Tunables for `SpamFilter`. These are code-level knobs only; the user can +/// just turn the filter on or off. +struct SpamFilterConfig: Equatable { + /// Messages per sender: a burst of 6, then one more every 10 seconds. + var messageBucket = TokenBucketConfig(capacity: 6, refillPerSecond: 1.0 / 10.0) + /// Characters per sender: 1200 per minute, refilled continuously (20 per second). + var characterBucket = TokenBucketConfig(capacity: 1200, refillPerSecond: 1200.0 / 60.0) + /// Links per sender: 2 per minute, refilled continuously (one every 30 seconds). + var linkBucket = TokenBucketConfig(capacity: 2, refillPerSecond: 2.0 / 60.0) + var repetition = RepetitionConfig() + /// How far in the past a message timestamp is trusted. Relay backfill + /// (geohash subscribes to the last hour) and gossip sync deliver history in + /// bursts, so buckets run on message time; older or future timestamps are + /// clamped into `[now - maxBackdate, now]`. + var maxBackdate: TimeInterval = 60 * 60 + /// Least recently active senders beyond this are forgotten. + var maxTrackedSenders = 1_000 + /// Verdicts remembered by message ID so multi-path duplicates are judged once. + var maxRememberedVerdicts = 4_000 +} + +struct TokenBucketConfig: Equatable { + var capacity: Double + var refillPerSecond: Double +} + +struct RepetitionConfig: Equatable { + /// Messages shorter than this (in characters) are never checked for repeated characters. + var minRepeatedCharactersLength = 32 + /// Hide when some shift of the message matches itself at this fraction of positions. + var repeatedCharactersThreshold = 0.85 + /// Messages shorter than this are never compared with the sender's history. + var minRepeatedMessageLength = 10 + /// Two messages count as the same when their best alignment matches this fraction. + var repeatedMessageSimilarity = 0.8 + /// How many near-copies of one text a sender may post within `historyWindow`. + var allowedRepeats = 1 + /// How many near-copies of a text *other* senders may already have posted + /// within `historyWindow` before it is hidden. Catches spam spread over many + /// identities. + var crossUserAllowedRepeats = 2 + /// Messages shorter than this are never compared across senders; short replies coincide. + var crossUserMinLength = 16 + /// Recent messages from all senders kept for the cross-user comparison. + var crossUserHistorySize = 200 + /// Comparisons one message may run against other senders' texts once + /// the trigram check passes, and full alignments among those. Ordinary + /// chat rarely needs either (shared trigrams and the diagonal bound rule + /// most pairs out), but texts built from a tiny alphabet or vocabulary + /// pass both, and the caps keep a flood of them from stalling the main + /// actor. A real campaign's copies are recent, and the scan runs newest + /// first. Alignments must exceed `crossUserAllowedRepeats`, or no copy + /// could ever be confirmed. + var crossUserMaxComparisons = 64 + var crossUserMaxAlignments = 8 + /// Recent messages per sender kept for comparison. + var historySize = 10 + var historyWindow: TimeInterval = 10 * 60 + /// Longer messages are analyzed on their first N characters only, bounding CPU per message. + var maxAnalyzedLength = 1_000 + /// Messages are compared with others on their first N characters only. + var maxComparedLength = 280 + /// Correlation peaks combined per comparison; extra peaks absorb inserted or deleted text. + var maxAlignmentPeaks = 3 + /// Extra peaks only count runs of at least this many matching characters. + var minPeakRun = 3 + /// Pairs sharing fewer trigrams than this fraction skip the full comparison. + var minSharedTrigrams = 0.25 +} diff --git a/bitchat/Services/Spam/SpamFilterSettings.swift b/bitchat/Services/Spam/SpamFilterSettings.swift new file mode 100644 index 00000000..16ef15d0 --- /dev/null +++ b/bitchat/Services/Spam/SpamFilterSettings.swift @@ -0,0 +1,41 @@ +// +// SpamFilterSettings.swift +// bitchat +// +// This is free and unencumbered software released into the public domain. +// For more information, see +// + +import Foundation + +/// The user's on/off choice for `SpamFilter.shared`, on by default. +enum SpamFilterSettings { + private static let enabledKey = "spamFilter.enabled" + + static var isEnabled: Bool { + get { isEnabled(in: .standard) } + set { setEnabled(newValue, in: .standard, filter: .shared) } + } + + /// Store-injecting forms, so tests can assert the default and both settings + /// without touching the shared preferences other tests read. + static func isEnabled(in defaults: UserDefaults) -> Bool { + defaults.object(forKey: enabledKey) as? Bool ?? true + } + + static func setEnabled(_ enabled: Bool, in defaults: UserDefaults, filter: SpamFilter) { + guard isEnabled(in: defaults) != enabled || filter.enabled != enabled else { return } + defaults.set(enabled, forKey: enabledKey) + filter.enabled = enabled + // Start from a clean slate so budgets don't carry over from before the toggle. + filter.reset() + } + + /// Panic-wipe hook: restores the default and forgets the recent message + /// text and sender keys the filter keeps for its repeat checks. + static func reset(in defaults: UserDefaults = .standard, filter: SpamFilter = .shared) { + defaults.removeObject(forKey: enabledKey) + filter.enabled = isEnabled(in: defaults) + filter.reset() + } +} diff --git a/bitchat/Services/Spam/TokenBucketSchedule.swift b/bitchat/Services/Spam/TokenBucketSchedule.swift new file mode 100644 index 00000000..1f0d9df0 --- /dev/null +++ b/bitchat/Services/Spam/TokenBucketSchedule.swift @@ -0,0 +1,93 @@ +// +// TokenBucketSchedule.swift +// bitchat +// +// This is free and unencumbered software released into the public domain. +// For more information, see +// + +import Foundation + +/// A set of token buckets for one sender, evaluated on message time instead of +/// arrival order. +/// +/// Relay backfill and gossip sync hand over a sender's recent history in one +/// burst, often newest first, so a plain arrival-time bucket would hide an +/// ordinary conversation. Instead each admitted message stays on a timeline, +/// and a candidate is admitted only if replaying every bucket over the timeline +/// with the candidate inserted never runs dry. For live, in-order traffic this +/// behaves exactly like a classic token bucket. Rejected messages cost nothing. +/// +/// Entries older than the backdate window are folded into a checkpoint, and +/// nothing can be inserted before the checkpoint, so the replay stays short +/// and exact. +struct TokenBucketSchedule { + private struct Entry { + let time: TimeInterval + let costs: [Double] + } + + private static let epsilon = 1e-9 + + private let buckets: [TokenBucketConfig] + private var entries: [Entry] = [] + private var checkpoint: TimeInterval? + private var checkpointTokens: [Double] + + init(buckets: [TokenBucketConfig]) { + self.buckets = buckets + self.checkpointTokens = buckets.map(\.capacity) + } + + /// Admits and records the message when every bucket can pay `costs` at `time`. + /// Returns nil on success, otherwise the index of the first bucket that ran dry. + mutating func tryAdmit(at time: TimeInterval, costs: [Double]) -> Int? { + precondition(costs.count == buckets.count) + let time = checkpoint.map { max($0, time) } ?? time + var insertAt = entries.count + while insertAt > 0 && entries[insertAt - 1].time > time { insertAt -= 1 } + let candidate = Entry(time: time, costs: costs) + + var tokens = checkpointTokens + var last = checkpoint + for i in 0...entries.count { + let entry: Entry + if i < insertAt { + entry = entries[i] + } else if i == insertAt { + entry = candidate + } else { + entry = entries[i - 1] + } + refill(&tokens, from: last, to: entry.time) + last = entry.time + for b in buckets.indices { + tokens[b] -= entry.costs[b] + if tokens[b] < -Self.epsilon { return b } + } + } + entries.insert(candidate, at: insertAt) + return nil + } + + /// Folds entries before `cutoff` into the checkpoint. + mutating func prune(before cutoff: TimeInterval) { + var folded = 0 + while folded < entries.count && entries[folded].time < cutoff { + let entry = entries[folded] + refill(&checkpointTokens, from: checkpoint, to: entry.time) + for b in buckets.indices { checkpointTokens[b] -= entry.costs[b] } + checkpoint = entry.time + folded += 1 + } + if folded > 0 { entries.removeFirst(folded) } + } + + private func refill(_ tokens: inout [Double], from: TimeInterval?, to: TimeInterval) { + guard let from, to > from else { return } + let seconds = to - from + for b in buckets.indices { + tokens[b] = min(buckets[b].capacity, tokens[b] + seconds * buckets[b].refillPerSecond) + } + } +} diff --git a/bitchat/ViewModels/ChatPublicConversationCoordinator.swift b/bitchat/ViewModels/ChatPublicConversationCoordinator.swift index ef9eb989..3efa6fbe 100644 --- a/bitchat/ViewModels/ChatPublicConversationCoordinator.swift +++ b/bitchat/ViewModels/ChatPublicConversationCoordinator.swift @@ -81,6 +81,8 @@ protocol ChatPublicConversationContext: AnyObject { /// `powBits` is the validated NIP-13 difficulty of the source Nostr event /// (0 for mesh messages); sufficient PoW relaxes the per-sender bucket. func allowPublicMessage(senderKey: String, contentKey: String, powBits: Int) -> Bool + /// Hides public spam (floods, repeats, link bursts); see `SpamFilter`. + var spamFilter: SpamFilter { get } /// Buffers a visible-channel message for the batched (~80 ms) pipeline /// flush, which commits it to `conversationID` in the store. func enqueuePublicMessage(_ message: BitchatMessage, to conversationID: ConversationID) @@ -490,6 +492,17 @@ final class ChatPublicConversationCoordinator: PublicMessagePipelineDelegate { if archivedEchoKeys.contains(key) { return } } + // Local notices carry no sender ID and my own gossip or relay echoes + // are not spam, so neither is judged. Duplicates of a judged message + // get its first verdict without being counted again. + if shouldRateLimit, !isOwnMessage(finalMessage) { + let senderKey = normalizedSenderKey(for: finalMessage) + if context.spamFilter.shouldHide(finalMessage, senderKey: senderKey) { + SecureLogger.debug("🧹 Hiding public spam \(finalMessage.id.prefix(8))…", category: .session) + return + } + } + let channelMatches: Bool = { switch context.activeChannel { case .mesh: return !isGeo || isSystem @@ -521,7 +534,7 @@ final class ChatPublicConversationCoordinator: PublicMessagePipelineDelegate { } let isMentioned = message.mentions?.contains { myTokens.contains($0.normalizedNickname) } ?? false - if isMentioned && !isOwnMessage(message) { + if isMentioned && !isOwnMessage(message) && !context.spamFilter.wasHidden(messageID: message.id) { SecureLogger.info("🔔 Mention from \(message.sender)", category: .session) context.notifyMention(from: message.sender, message: message.content) } @@ -530,6 +543,7 @@ final class ChatPublicConversationCoordinator: PublicMessagePipelineDelegate { func sendHapticFeedback(for message: BitchatMessage) { #if os(iOS) guard UIApplication.shared.applicationState == .active else { return } + guard !context.spamFilter.wasHidden(messageID: message.id) else { return } var tokens: [String] = [context.nickname] switch context.activeChannel { diff --git a/bitchat/ViewModels/ChatViewModel.swift b/bitchat/ViewModels/ChatViewModel.swift index 36742d29..c2557185 100644 --- a/bitchat/ViewModels/ChatViewModel.swift +++ b/bitchat/ViewModels/ChatViewModel.swift @@ -148,6 +148,10 @@ final class ChatViewModel: ObservableObject, BitchatDelegate, SynchronousMessage contentRefillPerSec: TransportConfig.uiContentRateBucketRefillPerSec ) + /// Hides public spam on the mesh and geohash timelines. Under test each + /// view model gets its own, so parallel tests don't share sender history. + let spamFilter: SpamFilter = TestEnvironment.isRunningTests ? SpamFilter() : .shared + // MARK: - Published Properties /// Read-only derived view of the ACTIVE public channel's conversation in @@ -1648,6 +1652,8 @@ final class ChatViewModel: ObservableObject, BitchatDelegate, SynchronousMessage MeshSightingsTracker.shared.clear() MeshEchoSettings.reset() NotificationPrivacySettings.reset() + // The spam filter keeps recent message text and sender keys for its repeat checks + SpamFilterSettings.reset(filter: spamFilter) ComposerDraftStore.reset() // A hand-added relay names an operator someone chose to route through, // which is the kind of trace a wipe should not leave behind. diff --git a/bitchat/ViewModels/GeoPresenceTracker.swift b/bitchat/ViewModels/GeoPresenceTracker.swift index bbcf0969..7ed926c6 100644 --- a/bitchat/ViewModels/GeoPresenceTracker.swift +++ b/bitchat/ViewModels/GeoPresenceTracker.swift @@ -18,6 +18,8 @@ protocol GeoPresenceContext: AnyObject { func deriveNostrIdentity(forGeohash geohash: String) throws -> NostrIdentity func isNostrBlocked(pubkeyHexLowercased: String) -> Bool func parseMentions(from content: String) -> [String] + /// Hides public spam; shared with the open channel's intake. + var spamFilter: SpamFilter { get } func recordGeoParticipant(pubkeyHex: String, geohash: String, seenAt: Date) func geoParticipantCount(for geohash: String) -> Int @@ -129,6 +131,16 @@ final class GeoPresenceTracker { my.publicKeyHex.lowercased() == event.pubkey.lowercased() { return } + // Spam neither hints at a lively nearby chat nor notifies. Same + // event ID as the open channel's copy, so it is judged only once. + if context.spamFilter.check( + senderKey: SpamFilter.nostrSenderKey(pubkeyHex: event.pubkey), + messageID: event.id, + content: content, + timestamp: min(Date(timeIntervalSince1970: TimeInterval(event.created_at)), Date()) + ).isSpam { + return + } // Non-empty content on a sampled event means an actual chat message // (presence events are empty) — feed the nearby-conversation hint. diff --git a/bitchat/Views/AppInfoView.swift b/bitchat/Views/AppInfoView.swift index cb99fbac..d077a26e 100644 --- a/bitchat/Views/AppInfoView.swift +++ b/bitchat/Views/AppInfoView.swift @@ -22,6 +22,7 @@ struct AppInfoView: View { @State private var liveVoiceEnabled = PTTSettings.liveVoiceEnabled @State private var locationNotesEnabled = LocationNotesSettings.enabled @State private var hideMessagePreviews = NotificationPrivacySettings.hideMessagePreviews + @State private var spamFilterEnabled = SpamFilterSettings.isEnabled @State private var customRelays = NostrRelaySettings.customRelays() @State private var relayInput = "" @State private var relayError: String? @@ -114,6 +115,10 @@ struct AppInfoView: View { static let toggleOn: LocalizedStringKey = "common.toggle.on" static let toggleOff: LocalizedStringKey = "common.toggle.off" + static let publicChatsTitle = String(localized: "app_info.settings.public_chats.title", defaultValue: "PUBLIC CHATS", comment: "Section header (uppercase) for settings about public chats, such as the spam filter") + static let spamFilterTitle = String(localized: "app_info.settings.spam_filter.title", defaultValue: "spam filter", comment: "Title of the setting that hides spam in public mesh and location channels") + static let spamFilterSubtitle = String(localized: "app_info.settings.spam_filter.subtitle", defaultValue: "hides public messages from people who post too fast, repeat themselves, or flood links. private messages are never filtered. on by default.", comment: "Subtitle explaining what the spam filter hides and that private messages are never filtered") + static let privacyTitle = String(localized: "app_info.settings.privacy.title", defaultValue: "PRIVACY", comment: "Section header (uppercase) for privacy settings such as hiding notification previews") static let hidePreviewsTitle = String(localized: "app_info.settings.hide_previews.title", defaultValue: "hide message previews", comment: "Title of the setting that keeps message text, sender names, and geohashes out of lock-screen notifications") static let hidePreviewsSubtitle = String(localized: "app_info.settings.hide_previews.subtitle", defaultValue: "notifications say that something arrived without showing the message, who sent it, or which location channel it came from. anyone holding your locked phone learns nothing from the lock screen. on by default.", comment: "Subtitle explaining what hiding notification message previews does") @@ -523,6 +528,25 @@ struct AppInfoView: View { } } + // Public chats: what the mesh and location channels show. + VStack(alignment: .leading, spacing: 12) { + SectionHeader(verbatim: Strings.Settings.publicChatsTitle) + + settingsCard { + settingToggle( + title: Text(verbatim: Strings.Settings.spamFilterTitle), + subtitle: Text(verbatim: Strings.Settings.spamFilterSubtitle), + isOn: Binding( + get: { spamFilterEnabled }, + set: { newValue in + spamFilterEnabled = newValue + SpamFilterSettings.isEnabled = newValue + } + ) + ) + } + } + // Privacy: what a locked, seized, or borrowed phone gives away // without being unlocked. VStack(alignment: .leading, spacing: 12) { diff --git a/bitchatTests/ChatNostrCoordinatorContextTests.swift b/bitchatTests/ChatNostrCoordinatorContextTests.swift index ffe22d9f..8813ba26 100644 --- a/bitchatTests/ChatNostrCoordinatorContextTests.swift +++ b/bitchatTests/ChatNostrCoordinatorContextTests.swift @@ -74,6 +74,7 @@ private final class MockChatNostrContext: ChatNostrContext { func checkForMentions(_ message: BitchatMessage) { mentionCheckedMessageIDs.append(message.id) } func sendHapticFeedback(for message: BitchatMessage) { hapticMessageIDs.append(message.id) } func parseMentions(from content: String) -> [String] { [] } + let spamFilter = SpamFilter() // Inbound private (geohash DM) payloads var selectedPrivateChatPeer: PeerID? diff --git a/bitchatTests/ChatPublicConversationCoordinatorContextTests.swift b/bitchatTests/ChatPublicConversationCoordinatorContextTests.swift index f53b7c9b..d71c049b 100644 --- a/bitchatTests/ChatPublicConversationCoordinatorContextTests.swift +++ b/bitchatTests/ChatPublicConversationCoordinatorContextTests.swift @@ -199,6 +199,8 @@ private final class MockChatPublicConversationContext: ChatPublicConversationCon return rateLimitAllowed } + let spamFilter = SpamFilter() + func enqueuePublicMessage(_ message: BitchatMessage, to conversationID: ConversationID) { enqueuedMessages.append((message.id, conversationID)) } @@ -668,4 +670,86 @@ struct ChatPublicConversationCoordinatorContextTests { #expect(context.mentionNotifications.map(\.message) == ["hi @me", "yo @me"]) } + + // MARK: - Spam filter + + @Test @MainActor + func handlePublicMessage_floodFromOneSender_hidesSpamAndItsMention() async { + let context = MockChatPublicConversationContext() + let coordinator = ChatPublicConversationCoordinator(context: context) + let words = ["apple", "bridge", "candle", "dolphin", "engine", "forest", "garden"] + let messages = words.enumerated().map { i, word in + BitchatMessage( + id: "flood-\(i)", + sender: "flooder", + content: "@me \(word) \(i)", + timestamp: Date(), + isRelay: false, + senderPeerID: PeerID(str: "aabbccddeeff0011"), + mentions: ["me"] + ) + } + + for message in messages { + coordinator.handlePublicMessage(message) + coordinator.checkForMentions(message) + } + + // A burst of 6, then the 7th is over the message budget. + #expect(context.enqueuedMessageIDs == (0..<6).map { "flood-\($0)" }) + #expect(context.mentionNotifications.count == 6) + + // A duplicate of the hidden message keeps its verdict. + coordinator.handlePublicMessage(messages[6]) + #expect(context.enqueuedMessageIDs.count == 6) + } + + @Test @MainActor + func handlePublicMessage_sameTextFromManyGeohashIdentities_hidesTheFourth() async { + let context = MockChatPublicConversationContext() + let coordinator = ChatPublicConversationCoordinator(context: context) + context.activeChannel = .location(GeohashChannel(level: .city, geohash: "u4pruy")) + context.currentGeohash = "u4pruy" + + for i in 0..<5 { + let pubkey = String(repeating: String(format: "%02x", i + 1), count: 32) + coordinator.handlePublicMessage(makePublicMessage( + id: "swarm-\(i)", + content: "Free sats for everyone at freesats dot example", + senderPeerID: PeerID(nostr: pubkey) + )) + } + + #expect(context.enqueuedMessageIDs == ["swarm-0", "swarm-1", "swarm-2"]) + } + + @Test @MainActor + func handlePublicMessage_ownEchoesAndLocalNotices_areNotJudged() async { + let context = MockChatPublicConversationContext() + let coordinator = ChatPublicConversationCoordinator(context: context) + + // Gossip replays of my own text, stamped with my peer ID. + for i in 0..<10 { + coordinator.handlePublicMessage(makePublicMessage( + id: "own-\(i)", + sender: "me", + content: "my own message number \(i)", + senderPeerID: context.myPeerID + )) + } + #expect(context.enqueuedMessageIDs.count == 10) + #expect(!context.spamFilter.wasHidden(messageID: "own-9")) + } + + @Test @MainActor + func handlePublicMessage_filterOff_letsFloodThrough() async { + let context = MockChatPublicConversationContext() + let coordinator = ChatPublicConversationCoordinator(context: context) + context.spamFilter.enabled = false + + for i in 0..<10 { + coordinator.handlePublicMessage(makePublicMessage(id: "off-\(i)", content: String(repeating: "A", count: 40) + "\(i)")) + } + #expect(context.enqueuedMessageIDs.count == 10) + } } diff --git a/bitchatTests/ChatViewModelTests.swift b/bitchatTests/ChatViewModelTests.swift index 9306c606..1a661dbe 100644 --- a/bitchatTests/ChatViewModelTests.swift +++ b/bitchatTests/ChatViewModelTests.swift @@ -791,6 +791,9 @@ struct ChatViewModelRateLimitingTests { @Test @MainActor func handlePublicMessage_rateLimitsBurstBySender() async { let (viewModel, _) = makeTestableViewModel() + // "rate-msg-N" are near-copies the spam filter would hide; this + // measures the per-sender limiter alone. + viewModel.spamFilter.enabled = false let senderID = PeerID(str: "1122334455667788") let now = Date() diff --git a/bitchatTests/Performance/PerformanceBaselineTests.swift b/bitchatTests/Performance/PerformanceBaselineTests.swift index 6eda1e09..5f981071 100644 --- a/bitchatTests/Performance/PerformanceBaselineTests.swift +++ b/bitchatTests/Performance/PerformanceBaselineTests.swift @@ -428,7 +428,9 @@ final class PerformanceBaselineTests: XCTestCase { return InboundPublic( peerID: PeerID(str: String(format: "%016x", 0xC0DE_0000 + sender)), nickname: "perfpeer\(sender)", - content: "public pipeline message \(i) from sender \(sender)", + // Distinct sentences: numbered copies of one template are + // near-copies across senders, which the spam filter hides. + content: Self.perfSentence(i), timestamp: base.addingTimeInterval(Double(i)), messageID: "perf-pub-\(i)" ) @@ -467,6 +469,19 @@ final class PerformanceBaselineTests: XCTestCase { reportThroughput("pipeline.publicIngest", samples: samples, operations: messageCount, unit: "messages") } + /// A deterministic sentence of unrelated words, different for every `seed`. + private static func perfSentence(_ seed: Int) -> String { + let words = ["market", "river", "lantern", "bicycle", "orange", "window", "garden", "quiet", "station", + "morning", "paper", "copper", "valley", "thunder", "pocket", "silver", "harbor", "candle", + "meadow", "button", "ladder", "violet", "engine", "basket", "cloud", "pepper", "forest", + "mirror", "saddle", "tunnel", "anchor", "biscuit", "canyon", "dragon", "ember", "falcon"] + var state = UInt64(seed) &* 0x9E37_79B9_7F4A_7C15 &+ 1 + return (0..<7).map { _ in + state = state &* 6364136223846793005 &+ 1442695040888963407 + return words[Int((state >> 33) % UInt64(words.count))] + }.joined(separator: " ") + } + // MARK: - 7. ConversationStore append (to-be architecture) /// Core op of the new single-source-of-truth `ConversationStore` @@ -681,6 +696,7 @@ private final class PerfNostrContext: ChatNostrContext { func parseMentions(from content: String) -> [String] { MessageFormattingEngine.extractMentions(from: content) } + let spamFilter = SpamFilter() var selectedPrivateChatPeer: PeerID? var nostrKeyMapping: [PeerID: String] = [:] diff --git a/bitchatTests/SpamFilterTests.swift b/bitchatTests/SpamFilterTests.swift new file mode 100644 index 00000000..68771d1e --- /dev/null +++ b/bitchatTests/SpamFilterTests.swift @@ -0,0 +1,584 @@ +// +// SpamFilterTests.swift +// bitchatTests +// +// This is free and unencumbered software released into the public domain. +// For more information, see +// + +import BitFoundation +import Foundation +import Testing +@testable import bitchat + +/// Ported from bitchat-android's `SpamFilterTest`, so both apps hide the same messages. +@Suite struct SpamFilterTests { + + /// Deterministic clock and word source shared by one test. + private final class Harness { + var now = Date(timeIntervalSince1970: 1_000_000) + var nextID = 0 + var rng = SplitMix64(seed: 7) + lazy var filter = SpamFilter(clock: { [unowned self] in self.now }) + + func send( + _ content: String? = nil, + sender: String = "alice", + timestamp: Date? = nil, + id: String? = nil + ) -> SpamVerdict { + defer { nextID += 1 } + return filter.check( + senderKey: sender, + messageID: id ?? "m\(nextID)", + content: content ?? prose(), + timestamp: timestamp ?? now + ) + } + + /// Unrelated synthetic words, so only the rule under test can fire. + func prose(_ length: Int = 40) -> String { + var text = "" + while text.count < length { + if !text.isEmpty { text.append(" ") } + for _ in 0.. Date { + now.addingTimeInterval(-seconds) + } + } + + // MARK: - Message bucket + + @Test func sixthMessageInABurstIsShownAndSeventhIsHidden() { + let h = Harness() + for _ in 0..<6 { #expect(h.send() == .allowed) } + #expect(h.send() == .messageRate) + } + + @Test func messageBudgetRefillsOneTokenEveryTenSeconds() { + let h = Harness() + for _ in 0..<6 { _ = h.send() } + h.advance(seconds: 9) + #expect(h.send() == .messageRate) + h.advance(seconds: 1) + #expect(h.send() == .allowed) + #expect(h.send() == .messageRate) + } + + @Test func hiddenMessagesDoNotDrainTheBudgetOfAFloodingSender() { + let h = Harness() + var shown = 0 + for _ in 0..<120 { + if h.send() == .allowed { shown += 1 } + h.advance(seconds: 1) + } + // A burst of 6, then one every 10 seconds for the remaining two minutes. + #expect(shown == 6 + 11) + } + + // MARK: - Character bucket + + @Test func characterBudgetAllowsTwelveHundredCharactersPerMinute() { + let h = Harness() + #expect(h.send(h.prose(600)) == .allowed) + #expect(h.send(h.prose(600)) == .allowed) + #expect(h.send(h.prose(20)) == .characterRate) + h.advance(seconds: 2) + #expect(h.send(h.prose(20)) == .allowed) + } + + @Test func messageLongerThanTheCharacterBudgetIsHidden() { + let h = Harness() + #expect(h.send(h.prose(1201)) == .characterRate) + } + + @Test func charactersAreCountedAsScalars() { + let h = Harness() + let sentence = (0..<1200).map { i -> String in + i % 3 == 0 ? "😀" : String(Character(Unicode.Scalar(UInt8(97 + Int.random(in: 0..<26, using: &h.rng))))) + }.joined() + #expect(h.send(sentence) == .allowed) + } + + // MARK: - Link bucket + + @Test func thirdLinkWithinAMinuteIsHidden() { + let h = Harness() + #expect(h.send("see https://example.org/a for details") == .allowed) + #expect(h.send("mirror at www.example.net too") == .allowed) + #expect(h.send("and example.com/promo as well") == .linkRate) + #expect(h.send("no link in this one at all") == .allowed) + h.advance(seconds: 30) + #expect(h.send("ok one more https://example.org/b") == .allowed) + } + + @Test func messageWithThreeLinksIsHiddenEvenForAFreshSender() { + let h = Harness() + #expect(h.send("https://alpha.example/one and bravo.example plus www.charlie.example/three") == .linkRate) + } + + @Test func linkCounterIgnoresEmailAddressesAndAbbreviations() { + #expect(LinkCounter.count("mail me at someone@example.com e.g. today") == 0) + #expect(LinkCounter.count("pi is 3.14, version v1.2.3") == 0) + #expect(LinkCounter.count("https://a.example/x, www.b.example and c.example.") == 3) + } + + // MARK: - Repeated messages + + @Test func thirdCopyOfAMessageWithinTenMinutesIsHidden() { + let h = Harness() + let text = "anyone near the north gate tonight?" + #expect(h.send(text) == .allowed) + h.advance(seconds: 60) + #expect(h.send(text) == .allowed) + h.advance(seconds: 60) + #expect(h.send(text) == .repeatedMessage) + } + + @Test func nearCopiesWithAChangingSuffixCountAsRepeats() { + let h = Harness() + #expect(h.send("FREE SATS at the link below 101") == .allowed) + h.advance(seconds: 20) + #expect(h.send("free sats at the link below 202") == .allowed) + h.advance(seconds: 20) + #expect(h.send("free sats at the link below 303") == .repeatedMessage) + } + + @Test func repeatsSpreadBeyondTheHistoryWindowAreAllowed() { + let h = Harness() + let text = "good morning everyone, coffee anyone?" + _ = h.send(text) + h.advance(seconds: 11 * 60) + _ = h.send(text) + h.advance(seconds: 11 * 60) + #expect(h.send(text) == .allowed) + } + + @Test func shortRepliesMayRepeatFreely() { + let h = Harness() + for _ in 0..<5 { + #expect(h.send("lol yes") == .allowed) + h.advance(seconds: 10) + } + } + + // MARK: - Repeated characters + + @Test func messageRepeatingACharacterIsHidden() { + let h = Harness() + #expect(h.send(String(repeating: "A", count: 40)) == .repeatedCharacters) + #expect(h.send("hey " + String(repeating: "!", count: 60)) == .repeatedCharacters) + } + + @Test func messageRepeatingAPhraseIsHidden() { + let h = Harness() + #expect(h.send(String(repeating: "buy now ", count: 6)) == .repeatedCharacters) + } + + @Test func ordinaryLongSentenceIsNotMistakenForRepetition() { + let h = Harness() + let text = "The market by the old bridge opens at nine, bring cash because the card reader is flaky." + #expect(RepetitionDetector.selfSimilarity(RepetitionDetector.normalize(text, maxLength: 1000)) < 0.3) + #expect(h.send(text) == .allowed) + } + + // MARK: - Senders, duplicates, timestamps + + @Test func sendersHaveIndependentBudgets() { + let h = Harness() + for _ in 0..<6 { _ = h.send(sender: "alice") } + #expect(h.send(sender: "alice") == .messageRate) + #expect(h.send(sender: "bob") == .allowed) + } + + @Test func sameMessageIDArrivingTwiceIsJudgedOnce() { + let h = Harness() + for _ in 0..<5 { _ = h.send() } + #expect(h.send(id: "dup") == .allowed) + #expect(h.send(id: "dup") == .allowed) + #expect(h.send() == .messageRate) + } + + @Test func backfilledHistoryDeliveredNewestFirstIsShown() { + let h = Harness() + // Ten messages, one per minute over the last ten minutes, arriving in one burst. + let verdicts = (0..<10).map { minutesAgo in h.send(timestamp: h.ago(Double(minutesAgo) * 60)) } + #expect(verdicts.allSatisfy { $0 == .allowed }) + } + + @Test func backfilledBurstIsStillRateLimitedOnItsOwnTimestamps() { + let h = Harness() + let start = h.ago(5 * 60) + let verdicts = (0..<10).map { h.send(timestamp: start.addingTimeInterval(Double($0))) } + #expect(verdicts.filter { $0 == .allowed }.count == 6) + } + + @Test func backfillCannotBeSqueezedInBetweenAlreadyShownMessages() { + let h = Harness() + for _ in 0..<6 { _ = h.send(timestamp: h.ago(60)) } + // A message claiming to be from just before the burst would overdraw the bucket. + #expect(h.send(timestamp: h.ago(61)) == .messageRate) + } + + @Test func timestampsOlderThanTheBackdateWindowAreClamped() { + let h = Harness() + // Spreading fake timestamps over days must not buy extra budget. + let verdicts = (0..<10).map { h.send(timestamp: h.ago(Double(10 - $0) * 86_400)) } + #expect(verdicts.filter { $0 == .allowed }.count == 6) + } + + @Test func futureTimestampsAreClampedToNow() { + let h = Harness() + let verdicts = (0..<10).map { h.send(timestamp: h.now.addingTimeInterval(Double($0) * 60)) } + #expect(verdicts.filter { $0 == .allowed }.count == 6) + } + + // MARK: - Settings + + @Test func disabledFilterAllowsEverything() { + let h = Harness() + h.filter.enabled = false + for _ in 0..<20 { #expect(h.send(String(repeating: "A", count: 40)) == .allowed) } + } + + @Test func limitsAreConfigurable() { + let h = Harness() + var config = SpamFilterConfig() + config.messageBucket = TokenBucketConfig(capacity: 2, refillPerSecond: 1) + let strict = SpamFilter(config: config, clock: { h.now }) + #expect(strict.check(senderKey: "a", messageID: "1", content: h.prose(), timestamp: h.now) == .allowed) + #expect(strict.check(senderKey: "a", messageID: "2", content: h.prose(), timestamp: h.now) == .allowed) + #expect(strict.check(senderKey: "a", messageID: "3", content: h.prose(), timestamp: h.now) == .messageRate) + h.advance(seconds: 1) + #expect(strict.check(senderKey: "a", messageID: "4", content: h.prose(), timestamp: h.now) == .allowed) + } + + @Test func spamIsHidden() { + let h = Harness() + let messages = (1...7).map { + BitchatMessage(id: "h\($0)", sender: "flooder", content: "hi \($0)", timestamp: h.now, isRelay: false) + } + #expect(messages.map { h.filter.shouldHide($0, senderKey: "flooder") } == [false, false, false, false, false, false, true]) + } + + @Test func changingLimitsStartsEverySenderWithAFullBudget() { + let h = Harness() + for _ in 0..<6 { _ = h.send() } + #expect(h.send() == .messageRate) + + var config = SpamFilterConfig() + config.messageBucket.capacity = 10 + h.filter.config = config + + for _ in 0..<10 { #expect(h.send() == .allowed) } + #expect(h.send() == .messageRate) + } + + @Test func resetForgetsEverySender() { + let h = Harness() + for _ in 0..<6 { _ = h.send() } + #expect(h.send() == .messageRate) + h.filter.reset() + #expect(h.send() == .allowed) + } + + @Test func forgetsLeastRecentlyActiveSendersBeyondTheLimit() { + let h = Harness() + var config = SpamFilterConfig() + config.maxTrackedSenders = 4 + let filter = SpamFilter(config: config, clock: { h.now }) + for i in 0..<6 { + _ = filter.check(senderKey: "alice", messageID: "a\(i)", content: h.prose(), timestamp: h.now) + } + #expect(filter.check(senderKey: "alice", messageID: "a6", content: h.prose(), timestamp: h.now) == .messageRate) + // Five other senders push alice out; she comes back with a full budget. + for i in 0..<5 { + _ = filter.check(senderKey: "user\(i)", messageID: "u\(i)", content: h.prose(), timestamp: h.now) + } + #expect(filter.check(senderKey: "alice", messageID: "a7", content: h.prose(), timestamp: h.now) == .allowed) + } + + // MARK: - Across users + + @Test func sameTextFromManyThrowawayIdentitiesIsHiddenAfterThreeCopies() { + let h = Harness() + let text = "Free sats for everyone at freesats dot example" + let verdicts = (1...6).map { h.send(text, sender: "npub\($0)") } + #expect(verdicts == [.allowed, .allowed, .allowed, .repeatedAcrossUsers, .repeatedAcrossUsers, .repeatedAcrossUsers]) + } + + @Test func nearCopiesWithSmallChangesFromDifferentIdentitiesAreHidden() { + let h = Harness() + // Each variant is one small edit (a suffix, an insertion, a substitution) off the first. + let variants = [ + "Free sats for everyone, claim yours now", + "free sats for everyone, claim yours now 1", + "Free sats for everyone!! claim yours now", + "free sats 4 everyone, claim yours now", + "free sats for everyone, claim yours now 55" + ] + let verdicts = variants.enumerated().map { h.send($1, sender: "npub\($0)") } + #expect(verdicts == [.allowed, .allowed, .allowed, .allowed, .repeatedAcrossUsers]) + } + + @Test func shortCommonRepliesFromManyUsersAreNotCompared() { + let h = Harness() + for i in 0..<8 { #expect(h.send("good morning!", sender: "user\(i)") == .allowed) } + } + + @Test func differentMessagesOnTheSameTopicFromManyUsersAreAllowed() { + let h = Harness() + let messages = [ + "is the bakery on the corner still open?", + "the bakery closes at eight I think", + "bakery was still open when I walked by", + "anyone know if the bakery has bread left?", + "the corner bakery is open until eight" + ] + for (i, text) in messages.enumerated() { #expect(h.send(text, sender: "user\(i)") == .allowed) } + } + + @Test func copiesFromOtherUsersOutsideTheWindowDoNotCount() { + let h = Harness() + let text = "anyone up for chess at the tables later?" + for i in 0..<3 { + _ = h.send(text, sender: "user\(i)") + h.advance(seconds: 4 * 60) + } + // Only the copy from 4 minutes ago and the one from 8 minutes ago are in the window. + #expect(h.send(text, sender: "late") == .allowed) + } + + @Test func aSendersOwnCopiesDoNotCountTowardTheCrossUserLimit() { + let h = Harness() + let text = "anyone near the north gate tonight?" + _ = h.send(text, sender: "alice") + h.advance(seconds: 60) + _ = h.send(text, sender: "alice") + #expect(h.send(text, sender: "bob") == .allowed) + #expect(h.send(text, sender: "carol") == .repeatedAcrossUsers) + } + + // MARK: - Alignment + + @Test func bestAlignmentReportsWhereTheTextsLineUp() { + let a = RepetitionDetector.normalize("xx hello world", maxLength: 1000) + let b = RepetitionDetector.normalize("hello world", maxLength: 1000) + let alignment = RepetitionDetector.align(a, b) + #expect(alignment.peaks.first?.shift == 3) + #expect(alignment.matches == 11) + } + + @Test func anInsertedCharacterDoesNotBreakTheMatch() { + let a = RepetitionDetector.normalize("free sats for everyone, claim yours now", maxLength: 1000) + let b = RepetitionDetector.normalize("free sats for everyone!!, claim yours now", maxLength: 1000) + #expect(RepetitionDetector.align(a, b, maxPeaks: 1).score < 0.8) + #expect(RepetitionDetector.align(a, b).score > 0.9) + } + + @Test func unrelatedSentencesStayFarBelowTheNearCopyThreshold() { + let a = RepetitionDetector.normalize("the bakery closes at eight I think", maxLength: 1000) + let b = RepetitionDetector.normalize("the corner bakery is open until eight", maxLength: 1000) + #expect(RepetitionDetector.align(a, b).score < 0.6) + } + + @Test func alignmentUpperBoundNeverUnderestimates() { + var rng = SplitMix64(seed: 11) + for alphabet in [2, 3, 5, 26] { + for _ in 0..<60 { + func text() -> [Int] { + (0..= exact) + } + } + } + + @Test func recentCopiesAreCaughtBehindAFloodOfLookAlikes() { + let now = Date(timeIntervalSince1970: 1_000_000) + var config = SpamFilterConfig() + config.repetition.crossUserMaxComparisons = 8 + // Enough to confirm the copies that hide the next one, and no more. + config.repetition.crossUserMaxAlignments = config.repetition.crossUserAllowedRepeats + 1 + let filter = SpamFilter(config: config, clock: { now }) + var rng = SplitMix64(seed: 3) + // Texts over a two-letter alphabet pass every cheap check against each + // other, so they spend the comparison budget without being near-copies. + for i in 0..<30 { + let junk = String((0..<120).map { _ in Bool.random(using: &rng) ? "a" : "b" }) + _ = filter.check(senderKey: "junk\(i)", messageID: "j\(i)", content: junk, timestamp: now) + } + let text = "Free sats for everyone at freesats dot example" + let verdicts = (1...4).map { + filter.check(senderKey: "npub\($0)", messageID: "c\($0)", content: text, timestamp: now) + } + #expect(verdicts == [.allowed, .allowed, .allowed, .repeatedAcrossUsers]) + } + + // MARK: - Disguised copies + + private static let disguiseBase = "Free sats for everyone, claim yours now" + + /// The same text a reader sees, written so that a plain comparison would miss it. + private static let disguises: [(String, String)] = { + let base = disguiseBase + func mapScalars(_ transform: (Unicode.Scalar) -> String) -> String { + base.unicodeScalars.map(transform).joined() + } + let invisibles = ["\u{200C}", "\u{200D}", "\u{2060}", "\u{00AD}", "\u{200E}"] + return [ + ("zero-width spaces", mapScalars { "\($0)\u{200B}" }), + ("mixed invisible characters", base.unicodeScalars.enumerated().map { "\($1)" + invisibles[$0 % 5] }.joined()), + ("an accent on every letter", mapScalars { $0.properties.isAlphabetic ? "\($0)\u{0301}" : "\($0)" }), + ("math bold letters", mapScalars { c in + switch c.value { + case 0x61...0x7A: return String(Unicode.Scalar(0x1D41A + c.value - 0x61)!) + case 0x41...0x5A: return String(Unicode.Scalar(0x1D400 + c.value - 0x41)!) + default: return "\(c)" + } + }), + ("fullwidth letters", mapScalars { c in + (0x21...0x7E).contains(c.value) ? String(Unicode.Scalar(c.value + 0xFEE0)!) : "\(c)" + }), + ("non-breaking spaces", base.replacingOccurrences(of: " ", with: "\u{00A0}")) + ] + }() + + @Test func normalizationFoldsDisguisesToTheTextAReaderSees() { + let plain = RepetitionDetector.normalize(Self.disguiseBase, maxLength: 1000) + for (label, text) in Self.disguises { + #expect(RepetitionDetector.normalize(text, maxLength: 1000) == plain, "\(label)") + } + } + + @Test func disguisedCopiesFromOtherUsersAreStillCaught() { + let now = Date(timeIntervalSince1970: 1_000_000) + for (label, text) in Self.disguises { + let fresh = SpamFilter(clock: { now }) + for i in 0..<3 { + _ = fresh.check(senderKey: "npub\(i)", messageID: "\(label)-\(i)", content: Self.disguiseBase, timestamp: now) + } + #expect(fresh.check(senderKey: "npub9", messageID: "\(label)-x", content: text, timestamp: now) == .repeatedAcrossUsers, "\(label)") + } + } + + @Test func disguisedCopiesFromTheSameUserAreStillCaught() { + let now = Date(timeIntervalSince1970: 1_000_000) + for (label, text) in Self.disguises { + let fresh = SpamFilter(clock: { now }) + _ = fresh.check(senderKey: "alice", messageID: "\(label)-1", content: Self.disguiseBase, timestamp: now) + _ = fresh.check(senderKey: "alice", messageID: "\(label)-2", content: Self.disguiseBase, timestamp: now) + #expect(fresh.check(senderKey: "alice", messageID: "\(label)-3", content: text, timestamp: now) == .repeatedMessage, "\(label)") + } + } + + @Test func invisibleCharactersCannotBreakUpARepeatedCharacterRun() { + let h = Harness() + #expect(h.send(String(repeating: "A\u{200B}", count: 40)) == .repeatedCharacters) + } + + @Test func ordinaryTextWithAccentsAndOtherScriptsIsStillAllowed() { + let h = Harness() + let texts = [ + "Tiếng Việt có dấu, chúng tôi đang ở quán cà phê gần chợ", + "क्या कोई पास में चाय की दुकान जानता है? मैं स्टेशन के पास हूँ", + "Café au lait, crème brûlée, s'il vous plaît", + "家族と一緒に駅の近くのカフェにいます、誰か来ますか" + ] + for (i, text) in texts.enumerated() { #expect(h.send(text, sender: "user\(i)") == .allowed, "\(text)") } + } + + @Test func differentSentencesInAnotherScriptAreNotMistakenForCopies() { + let h = Harness() + #expect(h.send("क्या कोई पास में चाय की दुकान जानता है?", sender: "a") == .allowed) + #expect(h.send("मैं स्टेशन के पास हूँ, बारिश हो रही है", sender: "b") == .allowed) + #expect(h.send("आज बाज़ार कितने बजे बंद होगा?", sender: "c") == .allowed) + #expect(h.send("क्या किसी के पास छाता है जो मैं ले सकूँ?", sender: "d") == .allowed) + } +} + +@Suite struct SpamFilterSettingsTests { + + private static let words = ["apple", "bridge", "candle", "dolphin", "engine", "forest", "garden", "harbor"] + + private func makeDefaults() -> UserDefaults { + let suite = "SpamFilterSettingsTests-\(UUID().uuidString)" + let defaults = UserDefaults(suiteName: suite)! + defaults.removePersistentDomain(forName: suite) + return defaults + } + + @Test func filterIsOnByDefault() { + #expect(SpamFilterSettings.isEnabled(in: makeDefaults())) + } + + @Test func turningTheFilterOffPersistsAndAppliesImmediately() { + let defaults = makeDefaults() + let filter = SpamFilter() + SpamFilterSettings.setEnabled(false, in: defaults, filter: filter) + #expect(!SpamFilterSettings.isEnabled(in: defaults)) + #expect(!filter.enabled) + } + + @Test func togglingStartsEverySenderWithAFullBudget() { + let defaults = makeDefaults() + let now = Date() + let filter = SpamFilter(clock: { now }) + let verdicts = (0..<7).map { + filter.check(senderKey: "a", messageID: "\($0)", content: Self.words[$0], timestamp: now) + } + #expect(verdicts.last == .messageRate) + + SpamFilterSettings.setEnabled(false, in: defaults, filter: filter) + SpamFilterSettings.setEnabled(true, in: defaults, filter: filter) + + #expect(filter.check(senderKey: "a", messageID: "7", content: Self.words[7], timestamp: now) == .allowed) + } + + @Test func panicResetRestoresTheDefaultAndForgetsHistory() { + let defaults = makeDefaults() + let now = Date() + let filter = SpamFilter(clock: { now }) + for i in 0..<7 { + _ = filter.check(senderKey: "a", messageID: "\(i)", content: Self.words[i], timestamp: now) + } + #expect(filter.wasHidden(messageID: "6")) + SpamFilterSettings.setEnabled(false, in: defaults, filter: filter) + + SpamFilterSettings.reset(in: defaults, filter: filter) + + #expect(SpamFilterSettings.isEnabled(in: defaults)) + #expect(filter.enabled) + #expect(!filter.wasHidden(messageID: "6")) + } +} + +/// Small seeded generator so the synthetic prose is the same on every run. +private struct SplitMix64: RandomNumberGenerator { + private var state: UInt64 + + init(seed: UInt64) { state = seed } + + mutating func next() -> UInt64 { + state &+= 0x9E37_79B9_7F4A_7C15 + var z = state + z = (z ^ (z >> 30)) &* 0xBF58_476D_1CE4_E5B9 + z = (z ^ (z >> 27)) &* 0x94D0_49BB_1331_11EB + return z ^ (z >> 31) + } +} From a8b14d5c43f3437d44a1fd2a12d307564d9f1c5e Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Sun, 25 Oct 2026 23:22:44 +0100 Subject: [PATCH 2/5] Judge received mesh attachments by message rate only Public images, voice notes and files arrive as "[image] ", so the text checks read the filename: photo1.jpg, photo2.jpg, photo3.jpg looked like repeats and plain names like beach.jpg counted as links. Rows the timeline renders as media (a mesh sender, a media prefix and one safe filename) now only spend a message token. The same line in a geohash or bridged message stays text. --- bitchat/Services/Spam/SpamFilter.swift | 20 ++++++++++++++- bitchatTests/SpamFilterTests.swift | 34 ++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/bitchat/Services/Spam/SpamFilter.swift b/bitchat/Services/Spam/SpamFilter.swift index 69799ebe..db3a32fa 100644 --- a/bitchat/Services/Spam/SpamFilter.swift +++ b/bitchat/Services/Spam/SpamFilter.swift @@ -108,10 +108,28 @@ final class SpamFilter: @unchecked Sendable { senderKey: senderKey, messageID: message.id, content: message.content, - timestamp: message.timestamp + timestamp: message.timestamp, + isText: !Self.isMeshAttachment(message) ) } + /// A received image, voice note or file, which the timeline shows as + /// `[image] `. Its filename is not text someone wrote, so only + /// the message rate applies. Like the timeline's media detection, only + /// mesh senders can attach files: the same line in a geohash or bridged + /// message stays text, so the prefix can't dodge the text checks there. + static func isMeshAttachment(_ message: BitchatMessage) -> Bool { + if let peerID = message.senderPeerID, peerID.isGeoChat || peerID.isGeoDM || peerID.isBridge { + return false + } + let categories: [MimeType.Category] = [.image, .audio, .file] + return categories.contains { category in + guard message.content.hasPrefix(category.messagePrefix) else { return false } + let filename = String(message.content.dropFirst(category.messagePrefix.count)) + return BitchatMessage.isSafeMediaFilename(filename) + } + } + func check( senderKey: String, messageID: String, diff --git a/bitchatTests/SpamFilterTests.swift b/bitchatTests/SpamFilterTests.swift index 68771d1e..32e96252 100644 --- a/bitchatTests/SpamFilterTests.swift +++ b/bitchatTests/SpamFilterTests.swift @@ -308,6 +308,40 @@ import Testing #expect(filter.check(senderKey: "alice", messageID: "a7", content: h.prose(), timestamp: h.now) == .allowed) } + // MARK: - Attachments + + private func attachment(_ content: String, id: String, from peerID: PeerID, at now: Date) -> BitchatMessage { + BitchatMessage(id: id, sender: "alice", content: content, timestamp: now, isRelay: false, senderPeerID: peerID) + } + + @Test func meshAttachmentFilenamesAreNotJudgedAsText() { + let h = Harness() + let peer = PeerID(str: "aabbccddeeff0011") + // Numbered names would be near-copies, plain names would count as links. + let names = ["[image] photo1.jpg", "[image] photo2.jpg", "[image] photo3.jpg", "[voice] beach.m4a", "[file] notes.pdf"] + for (i, name) in names.enumerated() { + #expect(h.filter.check(attachment(name, id: "a\(i)", from: peer, at: h.now), senderKey: "mesh:a") == .allowed, "\(name)") + } + } + + @Test func meshAttachmentFloodIsStillRateLimited() { + let h = Harness() + let peer = PeerID(str: "aabbccddeeff0011") + let verdicts = (0..<7).map { + h.filter.check(attachment("[image] img_\($0).jpg", id: "f\($0)", from: peer, at: h.now), senderKey: "mesh:a") + } + #expect(verdicts.last == .messageRate) + #expect(verdicts.dropLast().allSatisfy { $0 == .allowed }) + } + + @Test func attachmentPrefixInAGeohashMessageIsStillText() { + let h = Harness() + let geo = PeerID(nostr: String(repeating: "ab", count: 32)) + let message = attachment("[image] " + String(repeating: "A", count: 80), id: "g", from: geo, at: h.now) + #expect(!SpamFilter.isMeshAttachment(message)) + #expect(h.filter.check(message, senderKey: "nostr:ab") == .repeatedCharacters) + } + // MARK: - Across users @Test func sameTextFromManyThrowawayIdentitiesIsHiddenAfterThreeCopies() { From aa6335f9b420781369f2f54874ae2512f78d7850 Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Sun, 25 Oct 2026 23:45:56 +0100 Subject: [PATCH 3/5] Fix spam filter build on Xcode 26.6 CI's Swift compiler gave up type-checking the trigram hash expression and resolved the fingerprint closure, whose local shadowed the method it called, against SwiftUI's Gesture.map. Split the hash into typed steps, rename the helper to makeFingerprint, and give the dense test closures explicit types. --- .../Services/Spam/RepetitionDetector.swift | 8 ++- bitchat/Services/Spam/SpamFilter.swift | 5 +- bitchatTests/SpamFilterTests.swift | 53 +++++++++++++------ 3 files changed, 45 insertions(+), 21 deletions(-) diff --git a/bitchat/Services/Spam/RepetitionDetector.swift b/bitchat/Services/Spam/RepetitionDetector.swift index 35bbd575..0b45086d 100644 --- a/bitchat/Services/Spam/RepetitionDetector.swift +++ b/bitchat/Services/Spam/RepetitionDetector.swift @@ -221,8 +221,12 @@ enum RepetitionDetector { /// Distinct character trigrams as sorted hashes, for a cheap pre-check before `align`. static func trigrams(_ text: [Int]) -> [Int] { guard text.count >= 3 else { return [] } - let hashes = (0..<(text.count - 2)).map { i in (text[i] * 31 + text[i + 1]) * 31 + text[i + 2] } - return Array(Set(hashes)).sorted() + var hashes = Set() + for i in 0..<(text.count - 2) { + let pair: Int = text[i] * 31 + text[i + 1] + hashes.insert(pair * 31 + text[i + 2]) + } + return hashes.sorted() } /// Shared trigrams over the larger trigram set. Near-copies share most of diff --git a/bitchat/Services/Spam/SpamFilter.swift b/bitchat/Services/Spam/SpamFilter.swift index db3a32fa..6ba4f85a 100644 --- a/bitchat/Services/Spam/SpamFilter.swift +++ b/bitchat/Services/Spam/SpamFilter.swift @@ -156,7 +156,8 @@ final class SpamFilter: @unchecked Sendable { let repetition = config.repetition let text = isText ? RepetitionDetector.normalize(content, maxLength: repetition.maxAnalyzedLength) : nil - let fingerprint = text.map { fingerprint($0, time: time, config: repetition) } + var fingerprint: Fingerprint? + if let text { fingerprint = makeFingerprint(text, time: time, config: repetition) } let verdict: SpamVerdict if let text, let fingerprint { @@ -224,7 +225,7 @@ final class SpamFilter: @unchecked Sendable { RepetitionDetector.selfSimilarity(text) >= config.repeatedCharactersThreshold } - private func fingerprint(_ text: [Int], time: TimeInterval, config: RepetitionConfig) -> Fingerprint { + private func makeFingerprint(_ text: [Int], time: TimeInterval, config: RepetitionConfig) -> Fingerprint { let compared = text.count > config.maxComparedLength ? Array(text.prefix(config.maxComparedLength)) : text return Fingerprint( time: time, diff --git a/bitchatTests/SpamFilterTests.swift b/bitchatTests/SpamFilterTests.swift index 32e96252..e8c3efbc 100644 --- a/bitchatTests/SpamFilterTests.swift +++ b/bitchatTests/SpamFilterTests.swift @@ -42,12 +42,17 @@ import Testing while text.count < length { if !text.isEmpty { text.append(" ") } for _ in 0.. Character { + let offset: Int = Int.random(in: 0..<26, using: &rng) + return Character(Unicode.Scalar(UInt8(97 + offset))) + } + func advance(seconds: TimeInterval) { now = now.addingTimeInterval(seconds) } @@ -105,7 +110,7 @@ import Testing @Test func charactersAreCountedAsScalars() { let h = Harness() let sentence = (0..<1200).map { i -> String in - i % 3 == 0 ? "😀" : String(Character(Unicode.Scalar(UInt8(97 + Int.random(in: 0..<26, using: &h.rng))))) + i % 3 == 0 ? "😀" : String(h.randomLetter()) }.joined() #expect(h.send(sentence) == .allowed) } @@ -431,7 +436,8 @@ import Testing for alphabet in [2, 3, 5, 26] { for _ in 0..<60 { func text() -> [Int] { - (0.. Int in 97 + Int.random(in: 0.. String in "\(c)\u{200B}" } + var mixed = "" + for (i, c) in base.unicodeScalars.enumerated() { + mixed += String(c) + invisibles[i % invisibles.count] + } + let accented: String = mapScalars { (c: Unicode.Scalar) -> String in + c.properties.isAlphabetic ? "\(c)\u{0301}" : "\(c)" + } + let bold: String = mapScalars { (c: Unicode.Scalar) -> String in + switch c.value { + case 0x61...0x7A: return String(Unicode.Scalar(0x1D41A + c.value - 0x61)!) + case 0x41...0x5A: return String(Unicode.Scalar(0x1D400 + c.value - 0x41)!) + default: return "\(c)" + } + } + let fullwidth: String = mapScalars { (c: Unicode.Scalar) -> String in + guard (0x21...0x7E).contains(c.value) else { return "\(c)" } + return String(Unicode.Scalar(c.value + 0xFEE0)!) + } + let nonBreaking: String = base.replacingOccurrences(of: " ", with: "\u{00A0}") return [ - ("zero-width spaces", mapScalars { "\($0)\u{200B}" }), - ("mixed invisible characters", base.unicodeScalars.enumerated().map { "\($1)" + invisibles[$0 % 5] }.joined()), - ("an accent on every letter", mapScalars { $0.properties.isAlphabetic ? "\($0)\u{0301}" : "\($0)" }), - ("math bold letters", mapScalars { c in - switch c.value { - case 0x61...0x7A: return String(Unicode.Scalar(0x1D41A + c.value - 0x61)!) - case 0x41...0x5A: return String(Unicode.Scalar(0x1D400 + c.value - 0x41)!) - default: return "\(c)" - } - }), - ("fullwidth letters", mapScalars { c in - (0x21...0x7E).contains(c.value) ? String(Unicode.Scalar(c.value + 0xFEE0)!) : "\(c)" - }), - ("non-breaking spaces", base.replacingOccurrences(of: " ", with: "\u{00A0}")) + ("zero-width spaces", zeroWidth), + ("mixed invisible characters", mixed), + ("an accent on every letter", accented), + ("math bold letters", bold), + ("fullwidth letters", fullwidth), + ("non-breaking spaces", nonBreaking) ] }() From a797867e471c17f79b9124a18220198a0ed69c47 Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Sun, 25 Oct 2026 23:51:44 +0100 Subject: [PATCH 4/5] Drop the unused per-peak match count Periphery flagged RepetitionDetector.Peak.matches as assigned but never read; only the alignment's total is used. --- bitchat/Services/Spam/RepetitionDetector.swift | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bitchat/Services/Spam/RepetitionDetector.swift b/bitchat/Services/Spam/RepetitionDetector.swift index 0b45086d..d71606c9 100644 --- a/bitchat/Services/Spam/RepetitionDetector.swift +++ b/bitchat/Services/Spam/RepetitionDetector.swift @@ -89,7 +89,6 @@ enum RepetitionDetector { /// One diagonal of the cross-correlation: `b[j]` lined up under `a[j + shift]`. struct Peak: Equatable { let shift: Int - let matches: Int } /// How two texts line up: the peaks used and the share of the longer text they cover. @@ -139,7 +138,7 @@ enum RepetitionDetector { let to = min(b.count, a.count - bestShift) _ = runMatches(a, b, shift: bestShift, from: from, to: to, usedA: &usedA, usedB: &usedB, minRun: run, mark: true) - peaks.append(Peak(shift: bestShift, matches: bestGain)) + peaks.append(Peak(shift: bestShift)) total += bestGain } return Alignment(peaks: peaks, matches: total, score: Double(total) / Double(max(a.count, b.count))) From 0ae6905d1a30965b8b0012b75ab56cc4d181e007 Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Mon, 26 Oct 2026 00:15:18 +0100 Subject: [PATCH 5/5] Cut the spam filter's per-message cost for the perf floor pipeline.publicIngest fell to 138 msg/s on CI (floor 2400). Its benchmark sentences drew 7 words from one 36-word list, so 31% of pairs shared enough trigrams for full comparisons; natural chat lines share that much in 0.6% of pairs. The benchmark now uses made-up words, which behave like chat between different people. The filter itself also got cheaper, with identical verdicts: - an inverted trigram index replaces 200 trigram merges per message - the repeated-character check gives up on a shift once it can't reach the threshold - the diagonal bound indexes the message once, lazily, not every stored text - live, in-order messages reuse the bucket state after the newest entry instead of replaying the sender's timeline - trigram hashing, sorting and the hot loops read buffers directly Locally the ingest benchmark goes from 2580 to 8400 msg/s (13600 with the filter off). --- .../Services/Spam/RepetitionDetector.swift | 72 +++++++++++++++-- bitchat/Services/Spam/SpamFilter.swift | 78 +++++++++++++++---- .../Services/Spam/TokenBucketSchedule.swift | 22 ++++++ .../PerformanceBaselineTests.swift | 22 ++++-- bitchatTests/SpamFilterTests.swift | 54 +++++++++++++ 5 files changed, 218 insertions(+), 30 deletions(-) diff --git a/bitchat/Services/Spam/RepetitionDetector.swift b/bitchat/Services/Spam/RepetitionDetector.swift index d71606c9..69005df0 100644 --- a/bitchat/Services/Spam/RepetitionDetector.swift +++ b/bitchat/Services/Spam/RepetitionDetector.swift @@ -86,6 +86,36 @@ enum RepetitionDetector { return best } + /// Whether `selfSimilarity(text) >= threshold`, giving up on each shift as + /// soon as its mismatches rule it out. Ordinary text fails within a few + /// characters per shift, where `selfSimilarity` scans every one. + static func selfSimilarity(_ text: [Int], reaches threshold: Double) -> Bool { + let n = text.count + guard n >= 2 else { return threshold <= 0 } + // Runs for every incoming message, so it reads the buffer directly. + return text.withUnsafeBufferPointer { text in + for lag in 1...(n / 2) { + let overlap = n - lag + var mismatches = 0 + var possible = true + var i = 0 + while i < overlap { + if text[i] != text[i + lag] { + mismatches += 1 + // The best this shift can still score, compared as the score itself is. + if Double(overlap - mismatches) / Double(overlap) < threshold { + possible = false + break + } + } + i += 1 + } + if possible { return true } + } + return false + } + } + /// One diagonal of the cross-correlation: `b[j]` lined up under `a[j + shift]`. struct Peak: Equatable { let shift: Int @@ -220,12 +250,44 @@ enum RepetitionDetector { /// Distinct character trigrams as sorted hashes, for a cheap pre-check before `align`. static func trigrams(_ text: [Int]) -> [Int] { guard text.count >= 3 else { return [] } - var hashes = Set() - for i in 0..<(text.count - 2) { - let pair: Int = text[i] * 31 + text[i + 1] - hashes.insert(pair * 31 + text[i + 2]) + let count = text.count - 2 + var hashes = [Int](unsafeUninitializedCapacity: count) { hashes, initialized in + text.withUnsafeBufferPointer { text in + for i in 0..= 0 && hashes[j] > value { + hashes[j + 1] = hashes[j] + j -= 1 + } + hashes[j + 1] = value + } + } + } else { + hashes.sort() + } + // Drop repeats in place; sorting brought them together. + let unique: Int = hashes.withUnsafeMutableBufferPointer { hashes in + var unique = 0 + for i in 0..= repetition.crossUserMinLength { - recentTexts.append(SharedEntry(senderKey: senderKey, fingerprint: fingerprint)) - if recentTexts.count > repetition.crossUserHistorySize { - recentTexts.removeFirst(recentTexts.count - repetition.crossUserHistorySize) - } + rememberRecentText(SharedEntry(senderKey: senderKey, fingerprint: fingerprint), limit: repetition.crossUserHistorySize) } } else { verdict = admitRate(&state, content: content, time: time, isText: isText) @@ -214,6 +216,8 @@ final class SpamFilter: @unchecked Sendable { private func clearLocked() { senders.removeAll() recentTexts.removeAll() + recentTextsStart = 0 + recentTrigrams.removeAll() verdicts.removeAll() verdictOrder.removeAll() verdictHead = 0 @@ -222,7 +226,7 @@ final class SpamFilter: @unchecked Sendable { private func hasRepeatedCharacters(_ text: [Int], config: RepetitionConfig) -> Bool { text.count >= config.minRepeatedCharactersLength && - RepetitionDetector.selfSimilarity(text) >= config.repeatedCharactersThreshold + RepetitionDetector.selfSimilarity(text, reaches: config.repeatedCharactersThreshold) } private func makeFingerprint(_ text: [Int], time: TimeInterval, config: RepetitionConfig) -> Fingerprint { @@ -230,8 +234,7 @@ final class SpamFilter: @unchecked Sendable { return Fingerprint( time: time, text: compared, - trigrams: RepetitionDetector.trigrams(compared), - positions: RepetitionDetector.positions(compared) + trigrams: RepetitionDetector.trigrams(compared) ) } @@ -239,7 +242,10 @@ final class SpamFilter: @unchecked Sendable { guard length >= config.minRepeatedMessageLength else { return false } // At most `historySize` comparisons, so no budget is needed. var budget = ComparisonBudget(comparisons: .max, alignments: .max) - let copies = state.history.filter { isNearCopy($0, message, config: config, budget: &budget) }.count + let copies = state.history.filter { earlier in + let overlap = RepetitionDetector.trigramOverlap(earlier.trigrams, message.trigrams) + return isNearCopy(earlier, message, sharedTrigrams: overlap, config: config, budget: &budget) + }.count return copies > config.allowedRepeats } @@ -252,42 +258,80 @@ final class SpamFilter: @unchecked Sendable { comparisons: config.crossUserMaxComparisons, alignments: config.crossUserMaxAlignments ) - for entry in recentTexts.reversed() where entry.senderKey != senderKey { + // Shared distinct trigrams per recent text (by position), from the + // index: the same counts `trigramOverlap` would find, without visiting + // unrelated texts. + var shared = [Int](repeating: 0, count: recentTexts.count) + shared.withUnsafeMutableBufferPointer { shared in + for trigram in message.trigrams { + guard let sequences = recentTrigrams[trigram] else { continue } + for sequence in sequences { shared[sequence - recentTextsStart] += 1 } + } + } + // Newest first. A zero threshold admits texts that share nothing. + for index in shared.indices.reversed() where shared[index] > 0 || config.minSharedTrigrams <= 0 { + let entry = recentTexts[index] + guard entry.senderKey != senderKey else { continue } guard !budget.isSpent else { break } - guard isNearCopy(entry.fingerprint, message, config: config, budget: &budget) else { continue } + let larger = max(entry.fingerprint.trigrams.count, message.trigrams.count) + let overlap = larger == 0 ? 1 : Double(shared[index]) / Double(larger) + guard isNearCopy(entry.fingerprint, message, sharedTrigrams: overlap, config: config, budget: &budget) else { continue } copies += 1 if copies > config.crossUserAllowedRepeats { return true } } return false } + private func rememberRecentText(_ entry: SharedEntry, limit: Int) { + let sequence = recentTextsStart + recentTexts.count + recentTexts.append(entry) + for trigram in entry.fingerprint.trigrams { recentTrigrams[trigram, default: []].append(sequence) } + while recentTexts.count > limit { + let oldest = recentTexts.removeFirst() + // The oldest text holds the smallest sequence number in each of its lists. + for trigram in oldest.fingerprint.trigrams { + guard var sequences = recentTrigrams[trigram] else { continue } + sequences.removeFirst() + recentTrigrams[trigram] = sequences.isEmpty ? nil : sequences + } + recentTextsStart += 1 + } + } + /// Work left for comparing one message: pairs that get past the trigram /// check, and of those, pairs that need the full alignment. private struct ComparisonBudget { var comparisons: Int var alignments: Int + /// The message's character positions, built when a bound first needs them. + var positions: [Int: [Int]]? var isSpent: Bool { comparisons <= 0 || alignments <= 0 } } - /// The cheap checks (time, length, shared trigrams) cost nothing from - /// `budget`; the diagonal bound spends a comparison and `align` an alignment. + /// The cheap checks (time, length, `sharedTrigrams`, their + /// `trigramOverlap`) cost nothing from `budget`; the diagonal bound spends + /// a comparison and `align` an alignment. private func isNearCopy( _ earlier: Fingerprint, _ message: Fingerprint, + sharedTrigrams: Double, config: RepetitionConfig, budget: inout ComparisonBudget ) -> Bool { let longer = max(earlier.text.count, message.text.count) guard abs(earlier.time - message.time) <= config.historyWindow, lengthsCanMatch(earlier.text.count, message.text.count, config: config), - RepetitionDetector.trigramOverlap(earlier.trigrams, message.trigrams) >= config.minSharedTrigrams + sharedTrigrams >= config.minSharedTrigrams else { return false } budget.comparisons -= 1 + let positions = budget.positions ?? RepetitionDetector.positions(message.text) + budget.positions = positions + // The bound is symmetric, so it indexes the message, not each earlier text. guard Double(RepetitionDetector.alignmentUpperBound( - message.text, earlier.text, - positionsInB: earlier.positions, + message.text, + positionsInB: positions, maxPeaks: config.maxAlignmentPeaks )) / Double(longer) >= config.repeatedMessageSimilarity else { return false } diff --git a/bitchat/Services/Spam/TokenBucketSchedule.swift b/bitchat/Services/Spam/TokenBucketSchedule.swift index 1f0d9df0..7b6555f6 100644 --- a/bitchat/Services/Spam/TokenBucketSchedule.swift +++ b/bitchat/Services/Spam/TokenBucketSchedule.swift @@ -33,10 +33,15 @@ struct TokenBucketSchedule { private var entries: [Entry] = [] private var checkpoint: TimeInterval? private var checkpointTokens: [Double] + /// Tokens left after the newest entry, as a replay would compute them, so + /// an in-order message (live traffic) is checked without a replay. + private var tailTokens: [Double] + private var tailTime: TimeInterval? init(buckets: [TokenBucketConfig]) { self.buckets = buckets self.checkpointTokens = buckets.map(\.capacity) + self.tailTokens = checkpointTokens } /// Admits and records the message when every bucket can pay `costs` at `time`. @@ -48,6 +53,19 @@ struct TokenBucketSchedule { while insertAt > 0 && entries[insertAt - 1].time > time { insertAt -= 1 } let candidate = Entry(time: time, costs: costs) + if insertAt == entries.count { + var tokens = tailTokens + refill(&tokens, from: tailTime, to: time) + for b in buckets.indices { + tokens[b] -= costs[b] + if tokens[b] < -Self.epsilon { return b } + } + entries.append(candidate) + tailTokens = tokens + tailTime = time + return nil + } + var tokens = checkpointTokens var last = checkpoint for i in 0...entries.count { @@ -67,6 +85,10 @@ struct TokenBucketSchedule { } } entries.insert(candidate, at: insertAt) + // The replay ran through the newest entry; folding old entries into the + // checkpoint later repeats the same steps, so this stays exact. + tailTokens = tokens + tailTime = last return nil } diff --git a/bitchatTests/Performance/PerformanceBaselineTests.swift b/bitchatTests/Performance/PerformanceBaselineTests.swift index 5f981071..00c273f7 100644 --- a/bitchatTests/Performance/PerformanceBaselineTests.swift +++ b/bitchatTests/Performance/PerformanceBaselineTests.swift @@ -469,17 +469,23 @@ final class PerformanceBaselineTests: XCTestCase { reportThroughput("pipeline.publicIngest", samples: samples, operations: messageCount, unit: "messages") } - /// A deterministic sentence of unrelated words, different for every `seed`. + /// A deterministic sentence of made-up words, different for every `seed`. + /// Like real chat between different people, two of them rarely share + /// enough trigrams to need the spam filter's full comparison; words drawn + /// from one small list would, which measures the filter's worst case + /// rather than ordinary ingest. private static func perfSentence(_ seed: Int) -> String { - let words = ["market", "river", "lantern", "bicycle", "orange", "window", "garden", "quiet", "station", - "morning", "paper", "copper", "valley", "thunder", "pocket", "silver", "harbor", "candle", - "meadow", "button", "ladder", "violet", "engine", "basket", "cloud", "pepper", "forest", - "mirror", "saddle", "tunnel", "anchor", "biscuit", "canyon", "dragon", "ember", "falcon"] var state = UInt64(seed) &* 0x9E37_79B9_7F4A_7C15 &+ 1 - return (0..<7).map { _ in + func next(_ bound: Int) -> Int { state = state &* 6364136223846793005 &+ 1442695040888963407 - return words[Int((state >> 33) % UInt64(words.count))] - }.joined(separator: " ") + return Int((state >> 33) % UInt64(bound)) + } + let words: [String] = (0..<7).map { _ in + let length = 2 + next(6) + let letters: [Character] = (0.. Int? { + let timeline = (admitted + [(time, costs)]).enumerated().sorted { a, b in + a.element.0 != b.element.0 ? a.element.0 < b.element.0 : a.offset < b.offset + }.map(\.element) + var tokens = configs.map(\.capacity) + var last: TimeInterval? + for (t, c) in timeline { + if let last, t > last { + for b in configs.indices { tokens[b] = min(configs[b].capacity, tokens[b] + (t - last) * configs[b].refillPerSecond) } + } + last = t + for b in configs.indices { + tokens[b] -= c[b] + if tokens[b] < -1e-9 { return b } + } + } + return nil + } + var rng = SplitMix64(seed: 9) + for _ in 0..<40 { + var schedule = TokenBucketSchedule(buckets: configs) + var admitted: [(TimeInterval, [Double])] = [] + var clock: TimeInterval = 0 + for _ in 0..<40 { + clock += Double(Int.random(in: 0...6, using: &rng)) + // Mostly live traffic, sometimes a backfilled message from earlier. + let time = Bool.random(using: &rng) && Int.random(in: 0..<4, using: &rng) == 0 + ? max(0, clock - Double(Int.random(in: 1...20, using: &rng))) + : clock + let costs: [Double] = [1, Double(Int.random(in: 1...30, using: &rng))] + let expected = replayAdmits(admitted, time, costs) + #expect(schedule.tryAdmit(at: time, costs: costs) == expected) + if expected == nil { admitted.append((time, costs)) } + } + } + } + + @Test func earlyExitSelfSimilarityAgreesWithTheFullScore() { + var rng = SplitMix64(seed: 5) + for alphabet in [1, 2, 3, 26] { + for _ in 0..<80 { + let length: Int = Int.random(in: 0...90, using: &rng) + let text: [Int] = (0.. Int in 97 + Int.random(in: 0..= threshold + #expect(RepetitionDetector.selfSimilarity(text, reaches: threshold) == full) + } + } + } + } + @Test func recentCopiesAreCaughtBehindAFloodOfLookAlikes() { let now = Date(timeIntervalSince1970: 1_000_000) var config = SpamFilterConfig()