From 81a17f6bbb933fe03bca12ffe3c51244b3b452da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Deluan=20Quint=C3=A3o?= Date: Sat, 25 Apr 2026 20:27:38 -0400 Subject: [PATCH] =?UTF-8?q?fix(search):=20normalization=20for=20non-NFKD?= =?UTF-8?q?=20Unicode=20letters=20(=C3=B8,=20=C3=A6,=20=C5=93,=20=C3=9F)?= =?UTF-8?q?=20(#5413)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(search): transliterate non-ASCII letters symmetrically in FTS5 path Songs and artists with letters like ø, æ, œ, ß were unsearchable. The query path in server/subsonic/searching.go transliterates with sanitize.Accents (Øystein → Oystein), but the FTS5 tokenizer's remove_diacritics 2 only strips NFKD-decomposable marks — atomic letters with built-in strokes/ligatures survive tokenization, so the query side and index side disagreed. Apply sanitize.Accents on both sides: - normalizeForFTS now also emits an ASCII-transliterated form for each word, so search_normalized contains the variant the query produces. - buildFTS5Query transliterates the unquoted portion of the input so every caller (Subsonic, REST fullTextFilter) gets the same handling. Quoted phrases stay as typed, preserving phrase matches against the original title/artist columns. Existing libraries pick up the fix as records are re-scanned; users can trigger a manual full rescan to refresh older entries. * fix(search): cache transliteration and add ß/quoted-phrase test coverage Address review feedback: call sanitize.Accents once per word and reuse the result for both the punct-stripped and accent-only paths. Add missing test entries for ß→ss transliteration and quoted Unicode phrase preservation. --------- Co-authored-by: Claude --- persistence/sql_search_fts.go | 43 +++++++++++++++++++++--------- persistence/sql_search_fts_test.go | 18 +++++++++++-- 2 files changed, 46 insertions(+), 15 deletions(-) diff --git a/persistence/sql_search_fts.go b/persistence/sql_search_fts.go index 1d4116b5d..e9b961d91 100644 --- a/persistence/sql_search_fts.go +++ b/persistence/sql_search_fts.go @@ -8,6 +8,7 @@ import ( "unicode/utf8" . "github.com/Masterminds/squirrel" + "github.com/deluan/sanitize" "github.com/navidrome/navidrome/log" "github.com/navidrome/navidrome/model" ) @@ -44,24 +45,33 @@ var fts5Operators = regexp.MustCompile(`(?i)\b(AND|OR|NOT|NEAR)\b`) // fts5LeadingStar matches a * at the start of a token. FTS5 only supports * at the end (prefix queries). var fts5LeadingStar = regexp.MustCompile(`(^|[\s])\*+`) -// normalizeForFTS takes multiple strings, strips non-letter/non-number characters from each word, -// and returns a space-separated string of words that changed after stripping (deduplicated). -// This is used at index time to create concatenated forms: "R.E.M." → "REM", "AC/DC" → "ACDC". +// normalizeForFTS takes multiple strings and returns a space-separated, deduplicated list of +// alternative searchable forms for each word: punctuation-stripped (R.E.M. → REM, AC/DC → ACDC) +// and ASCII-transliterated (Bjørk → Bjork, œuvre → oeuvre). The transliterated form is needed +// because FTS5's `unicode61 remove_diacritics 2` only handles NFKD-decomposable diacritics — +// atomic letters like ø/æ/œ/ß survive tokenization, so the query side and index side disagree +// without an explicit transliterated entry here. func normalizeForFTS(values ...string) string { seen := make(map[string]struct{}) var result []string + add := func(orig, variant string) { + if variant == "" || variant == orig { + return + } + lower := strings.ToLower(variant) + if _, ok := seen[lower]; ok { + return + } + seen[lower] = struct{}{} + result = append(result, variant) + } for _, v := range values { for _, word := range strings.Fields(v) { - stripped := fts5PunctStrip.ReplaceAllString(word, "") - if stripped == "" || stripped == word { - continue - } - lower := strings.ToLower(stripped) - if _, ok := seen[lower]; ok { - continue - } - seen[lower] = struct{}{} - result = append(result, stripped) + transliterated := sanitize.Accents(word) + // Concatenated ASCII form: R.E.M. → REM, AC/DC → ACDC, St-Étienne → StEtienne. + add(word, fts5PunctStrip.ReplaceAllString(transliterated, "")) + // Accent-only transliteration for words without name-punctuation (Bjørk → Bjork). + add(word, transliterated) } } return strings.Join(result, " ") @@ -158,6 +168,13 @@ func buildFTS5Query(userInput string) string { result = result[:start] + fmt.Sprintf("\x00PHRASE%d\x00", len(phrases)-1) + result[end+1:] } + // Transliterate non-ASCII letters in the unquoted portion (ø→o, æ→ae, œ→oe, ß→ss, …) + // so the query matches the ASCII variants emitted by normalizeForFTS at index time. + // FTS5's own `remove_diacritics 2` only strips NFKD-decomposable marks, so without + // this step queries for words containing these letters can miss. Quoted phrases are + // left untouched so they continue to match the original text in title/artist columns. + result = sanitize.Accents(result) + // Neutralize FTS5 operators by lowercasing them (FTS5 operators are case-sensitive: // AND, OR, NOT, NEAR are operators, but and, or, not, near are plain tokens) result = fts5Operators.ReplaceAllStringFunc(result, strings.ToLower) diff --git a/persistence/sql_search_fts_test.go b/persistence/sql_search_fts_test.go index d0e26c8e3..b54e5856a 100644 --- a/persistence/sql_search_fts_test.go +++ b/persistence/sql_search_fts_test.go @@ -37,7 +37,13 @@ var _ = DescribeTable("buildFTS5Query", Entry("handles punctuated word mixed with plain words", "best of a-ha", `best* AND of* AND ("a ha" OR aha*)`), Entry("handles contraction followed by plain words", "you've got", `("you ve" OR youve*) AND got*`), Entry("strips miscellaneous punctuation", "rock & roll, vol. 2", "rock* AND roll* AND vol* AND 2*"), - Entry("preserves unicode characters with diacritics", "Björk début", "Björk* AND début*"), + Entry("transliterates NFKD-decomposable diacritics", "Björk début", "Bjork* AND debut*"), + Entry("transliterates ø to o", "Øystein", "Oystein*"), + Entry("transliterates œ ligature to oe", "œuvre", "oeuvre*"), + Entry("transliterates æ ligature to ae", "Brennæ", "Brennae*"), + Entry("transliterates mixed unicode words", "Mø Sigur Rós", "Mo* AND Sigur* AND Ros*"), + Entry("transliterates ß to ss", "Straße", "Strasse*"), + Entry("preserves quoted unicode phrase verbatim", `"Björk"`, `"Björk"`), Entry("collapses dotted abbreviation into phrase", "R.E.M.", `"R E M"`), Entry("collapses abbreviation without trailing dot", "R.E.M", `"R E M"`), Entry("collapses abbreviation mixed with words", "best of R.E.M.", `best* AND of* AND "R E M"`), @@ -75,11 +81,19 @@ var _ = DescribeTable("normalizeForFTS", Entry("strips dots and concatenates", "REM", "R.E.M."), Entry("strips slash", "ACDC", "AC/DC"), Entry("strips hyphen", "Aha", "A-ha"), - Entry("skips unchanged words", "", "The Beatles"), + Entry("skips unchanged ASCII words", "", "The Beatles"), Entry("handles mixed input", "REM", "R.E.M.", "Automatic for the People"), Entry("deduplicates", "REM", "R.E.M.", "R.E.M."), Entry("strips apostrophe from word", "N", "Guns N' Roses"), Entry("handles multiple values with punctuation", "REM ACDC", "R.E.M.", "AC/DC"), + Entry("transliterates ø to o", "Bjork", "Bjørk"), + Entry("transliterates Ø to O", "Oystein", "Øystein"), + Entry("transliterates œ ligature to oe", "oeuvre", "œuvre"), + Entry("transliterates Latin diacritics", "cafe", "café"), + Entry("transliterates only the non-ASCII words", "Mo Ros", "Mø Rós"), + Entry("combines punctuation strip and transliteration", "StEtienne St-Etienne", "St-Étienne"), + Entry("deduplicates against punctuation form", "Cafe", "Café", "Cafe"), + Entry("transliterates ß to ss", "Strasse", "Straße"), ) var _ = DescribeTable("containsCJK",