diff --git a/persistence/sql_search_fts.go b/persistence/sql_search_fts.go index 999c904af..fce77afbb 100644 --- a/persistence/sql_search_fts.go +++ b/persistence/sql_search_fts.go @@ -105,13 +105,15 @@ func isDottedAbbreviation(w string, subTokens []string) bool { } // buildFTS5Query preprocesses user input into a safe FTS5 MATCH expression. +// Plain tokens are emitted as (token OR token*) so bm25 ranks exact-token hits above prefix-only matches. // It preserves quoted phrases and * prefix wildcards, neutralizes FTS5 operators // (by lowercasing them, since FTS5 operators are case-sensitive) and strips // special characters to prevent query injection. -func buildFTS5Query(userInput string) string { +// The second return reports whether tokenization degraded the query (see ftsQueryDegraded). +func buildFTS5Query(userInput string) (string, bool) { q := strings.TrimSpace(userInput) if q == "" || q == `""` { - return "" + return "", false } var phrases []string @@ -151,25 +153,38 @@ func buildFTS5Query(userInput string) string { result = fts5LeadingStar.ReplaceAllString(result, "$1") tokens := strings.Fields(result) - // Append * to plain tokens for prefix matching (e.g., "love" → "love*"). - // Skip tokens that are already wildcarded or are quoted phrase placeholders. + // Two forms per token: a plain prefix form (love*) used only to evaluate query + // degradation, and the final (love OR love*) form. The OR adds no matches + // (exact ⊂ prefix) but gives bm25 a high-IDF exact-term hit, ranking rows that + // contain the literal word above prefix-only matches. Placeholders and + // user-supplied wildcards pass through untouched in both forms. + prefixTokens := make([]string, len(tokens)) + wrappedTokens := make([]string, len(tokens)) for i, t := range tokens { if strings.HasPrefix(t, "\x00") || strings.HasSuffix(t, "*") { + prefixTokens[i], wrappedTokens[i] = t, t continue } - tokens[i] = t + "*" + prefixTokens[i] = t + "*" + wrappedTokens[i] = "(" + t + " OR " + t + "*)" } // Use explicit AND between tokens — FTS5's implicit AND (space-separated) - // doesn't work correctly with parenthesized OR groups from processPunctuatedWords. - result = strings.Join(tokens, " AND ") + // doesn't work correctly with parenthesized OR groups. The prefix form is + // space-joined instead: it only feeds ftsQueryDegraded, which would count a + // literal "AND" as a long token and never flag all-short-token queries. + prefixQuery := strings.Join(prefixTokens, " ") + result = strings.Join(wrappedTokens, " AND ") for i, phrase := range phrases { placeholder := fmt.Sprintf("\x00PHRASE%d\x00", i) + prefixQuery = strings.ReplaceAll(prefixQuery, placeholder, phrase) result = strings.ReplaceAll(result, placeholder, phrase) } - return result + // Degradation is evaluated on the prefix form: ftsQueryDegraded treats + // leading-( tokens as punctuated-word groups and would never flag wrapped ones. + return result, ftsQueryDegraded(userInput, prefixQuery) } // ftsColumn pairs an FTS5 column name with its BM25 relevance weight. @@ -209,7 +224,10 @@ var ftsColumnDefs = map[string][]ftsColumn{ "artist": { {"name", 10.0}, {"sort_artist_name", 1.0}, - {"search_normalized", 1.0}, + // Same weight as name: for artists this column is purely the name in + // alternate spelling (unlike media_file/album, where it mixes + // title/album/artist variants and full weight would distort ranking). + {"search_normalized", 10.0}, }, } @@ -338,8 +356,8 @@ func ftsQueryDegraded(original, ftsQuery string) bool { // tokenization stripped significant content from the query (e.g., "1+" → "1*"). // Returns nil when the query produces no searchable tokens at all. func newFTSSearch(tableName, query string) searchStrategy { - q := buildFTS5Query(query) - if q == "" || ftsQueryDegraded(query, q) { + q, degraded := buildFTS5Query(query) + if q == "" || degraded { // Fallback: try LIKE search with the raw query cleaned := strings.TrimSpace(strings.ReplaceAll(query, `"`, "")) if cleaned != "" { diff --git a/persistence/sql_search_fts_test.go b/persistence/sql_search_fts_test.go index 6c975c601..d0b26e8d5 100644 --- a/persistence/sql_search_fts_test.go +++ b/persistence/sql_search_fts_test.go @@ -12,44 +12,45 @@ import ( var _ = DescribeTable("buildFTS5Query", func(input, expected string) { - Expect(buildFTS5Query(input)).To(Equal(expected)) + q, _ := buildFTS5Query(input) + Expect(q).To(Equal(expected)) }, Entry("returns empty string for empty input", "", ""), Entry("returns empty string for whitespace-only input", " ", ""), - Entry("appends * to a single word for prefix matching", "beatles", "beatles*"), - Entry("appends * to each word for prefix matching", "abbey road", "abbey* AND road*"), - Entry("preserves quoted phrases without appending *", `"the beatles"`, `"the beatles"`), - Entry("does not double-append * to existing prefix wildcard", "beat*", "beat*"), - Entry("strips FTS5 operators and appends * to lowercased words", "AND OR NOT NEAR", "and* AND or* AND not* AND near*"), - Entry("strips special FTS5 syntax characters and appends *", "test^col:val", "test* AND col* AND val*"), - Entry("handles mixed phrases and words", `"the beatles" abbey`, `"the beatles" AND abbey*`), - Entry("handles prefix with multiple words", "beat* abbey", "beat* AND abbey*"), - Entry("collapses multiple spaces", "abbey road", "abbey* AND road*"), - Entry("strips leading * from tokens and appends trailing *", "*livia", "livia*"), - Entry("strips leading * and preserves existing trailing *", "*livia oliv*", "livia* AND oliv*"), + Entry("wraps a single word as exact OR prefix", "beatles", "(beatles OR beatles*)"), + Entry("wraps each word as exact OR prefix", "abbey road", "(abbey OR abbey*) AND (road OR road*)"), + Entry("preserves quoted phrases without wrapping", `"the beatles"`, `"the beatles"`), + Entry("does not wrap user-supplied prefix wildcard", "beat*", "beat*"), + Entry("strips FTS5 operators and wraps lowercased words", "AND OR NOT NEAR", "(and OR and*) AND (or OR or*) AND (not OR not*) AND (near OR near*)"), + Entry("strips special FTS5 syntax characters and wraps", "test^col:val", "(test OR test*) AND (col OR col*) AND (val OR val*)"), + Entry("handles mixed phrases and words", `"the beatles" abbey`, `"the beatles" AND (abbey OR abbey*)`), + Entry("handles prefix with multiple words", "beat* abbey", "beat* AND (abbey OR abbey*)"), + Entry("collapses multiple spaces", "abbey road", "(abbey OR abbey*) AND (road OR road*)"), + Entry("strips leading * from tokens and wraps", "*livia", "(livia OR livia*)"), + Entry("strips leading * and preserves existing trailing *", "*livia oliv*", "(livia OR livia*) AND oliv*"), Entry("strips standalone *", "*", ""), - Entry("strips apostrophe from input", "Guns N' Roses", "Guns* AND N* AND Roses*"), + Entry("strips apostrophe from input", "Guns N' Roses", "(Guns OR Guns*) AND (N OR N*) AND (Roses OR Roses*)"), Entry("converts slashed word to phrase+concat OR", "AC/DC", `("AC DC" OR ACDC*)`), Entry("converts hyphenated word to phrase+concat OR", "a-ha", `("a ha" OR aha*)`), Entry("converts partial hyphenated word to phrase+concat OR", "a-h", `("a h" OR ah*)`), Entry("converts hyphenated name to phrase+concat OR", "Jay-Z", `("Jay Z" OR JayZ*)`), Entry("converts contraction to phrase+concat OR", "it's", `("it s" OR its*)`), - Entry("handles punctuated word mixed with plain words", "best of a-ha", `best* AND of* AND ("a ha" OR aha*)`), - Entry("handles contraction followed by plain words", "you've got", `("you ve" OR youve*) AND got*`), - Entry("strips miscellaneous punctuation", "rock & roll, vol. 2", "rock* AND roll* AND vol* AND 2*"), - Entry("transliterates NFKD-decomposable diacritics", "Björk début", "Bjork* AND debut*"), - Entry("transliterates ø to o", "Øystein", "Oystein*"), - Entry("transliterates œ ligature to oe", "œuvre", "oeuvre*"), - Entry("transliterates æ ligature to ae", "Brennæ", "Brennae*"), - Entry("transliterates mixed unicode words", "Mø Sigur Rós", "Mo* AND Sigur* AND Ros*"), - Entry("transliterates ß to ss", "Straße", "Strasse*"), + Entry("handles punctuated word mixed with plain words", "best of a-ha", `(best OR best*) AND (of OR of*) AND ("a ha" OR aha*)`), + Entry("handles contraction followed by plain words", "you've got", `("you ve" OR youve*) AND (got OR got*)`), + Entry("strips miscellaneous punctuation", "rock & roll, vol. 2", "(rock OR rock*) AND (roll OR roll*) AND (vol OR vol*) AND (2 OR 2*)"), + Entry("transliterates NFKD-decomposable diacritics", "Björk début", "(Bjork OR Bjork*) AND (debut OR debut*)"), + Entry("transliterates ø to o", "Øystein", "(Oystein OR Oystein*)"), + Entry("transliterates œ ligature to oe", "œuvre", "(oeuvre OR oeuvre*)"), + Entry("transliterates æ ligature to ae", "Brennæ", "(Brennae OR Brennae*)"), + Entry("transliterates mixed unicode words", "Mø Sigur Rós", "(Mo OR Mo*) AND (Sigur OR Sigur*) AND (Ros OR Ros*)"), + Entry("transliterates ß to ss", "Straße", "(Strasse OR Strasse*)"), Entry("preserves quoted unicode phrase verbatim", `"Björk"`, `"Björk"`), Entry("collapses dotted abbreviation into phrase", "R.E.M.", `"R E M"`), Entry("collapses abbreviation without trailing dot", "R.E.M", `"R E M"`), - Entry("collapses abbreviation mixed with words", "best of R.E.M.", `best* AND of* AND "R E M"`), + Entry("collapses abbreviation mixed with words", "best of R.E.M.", `(best OR best*) AND (of OR of*) AND "R E M"`), Entry("collapses two-letter abbreviation", "U.K.", `"U K"`), - Entry("does not collapse single letter surrounded by words", "I am fine", "I* AND am* AND fine*"), - Entry("does not collapse single standalone letter", "A test", "A* AND test*"), + Entry("does not collapse single letter surrounded by words", "I am fine", "(I OR I*) AND (am OR am*) AND (fine OR fine*)"), + Entry("does not collapse single standalone letter", "A test", "(A OR A*) AND (test OR test*)"), Entry("preserves quoted phrase with punctuation verbatim", `"ac/dc"`, `"ac/dc"`), Entry("preserves quoted abbreviation verbatim", `"R.E.M."`, `"R.E.M."`), Entry("returns empty string for punctuation-only input", "!!!!!!!", ""), @@ -57,6 +58,20 @@ var _ = DescribeTable("buildFTS5Query", Entry("returns empty string for empty quoted phrase", `""`, ""), ) +var _ = DescribeTable("buildFTS5Query degraded flag", + func(input string, expected bool) { + _, degraded := buildFTS5Query(input) + Expect(degraded).To(Equal(expected)) + }, + Entry("plain words are not degraded", "beatles", false), + Entry("special chars stripped leaving short token is degraded", "1+", true), + Entry("multiple short tokens are degraded", "1+ 2+", true), + Entry("short tokens mixed with a long word are not degraded", "1+ beatles", false), + Entry("quoted short-token phrase is degraded", `"1+"`, true), + Entry("punctuated-name group is not degraded", "AC/DC", false), + Entry("empty input is not degraded", "", false), +) + var _ = DescribeTable("ftsQueryDegraded", func(original, ftsQuery string, expected bool) { Expect(ftsQueryDegraded(original, ftsQuery)).To(Equal(expected)) @@ -143,7 +158,7 @@ var _ = Describe("ftsColumnDefs helpers", func() { It("returns weight CSV for artist", func() { Expect(ftsBM25Weights).To(HaveKeyWithValue("artist", - "10.0, 1.0, 1.0", + "10.0, 1.0, 10.0", )) }) @@ -237,18 +252,18 @@ var _ = Describe("newFTSSearch", func() { Expect(fts.rankExpr).To(Equal("unknown_table_fts.rank")) }) - It("wraps query with column filter for known tables", func() { + It("wraps query with column filter", func() { strategy := newFTSSearch("artist", "Beatles") fts, ok := strategy.(*ftsSearch) Expect(ok).To(BeTrue()) - Expect(fts.matchExpr).To(Equal("{name sort_artist_name search_normalized} : (Beatles*)")) + Expect(fts.matchExpr).To(Equal("{name sort_artist_name search_normalized} : ((Beatles OR Beatles*))")) }) It("passes query without column filter for unknown tables", func() { strategy := newFTSSearch("unknown_table", "test") fts, ok := strategy.(*ftsSearch) Expect(ok).To(BeTrue()) - Expect(fts.matchExpr).To(Equal("test*")) + Expect(fts.matchExpr).To(Equal("(test OR test*)")) }) It("preserves phrase queries inside column filter", func() { @@ -425,4 +440,38 @@ var _ = Describe("FTS5 Integration Search", func() { Expect(results).ToNot(BeEmpty(), "Max=0 should mean no limit, not LIMIT 0") }) }) + + Describe("Exact-match ranking", func() { + BeforeEach(func() { + // Registered before the inserts so a mid-loop failure cannot leak corpus rows. + DeferCleanup(func() { + // library_artist rows are removed by the artist_id ON DELETE CASCADE FK. + _, err := GetDBXBuilder().NewQuery("DELETE FROM artist WHERE id LIKE 'fts-rank-%'").Execute() + Expect(err).ToNot(HaveOccurred()) + }) + // Corpus has no competing exact-word names ("Mo X"): exact-vs-exact order depends + // on corpus statistics; the guaranteed property is exact > prefix. + for _, a := range []model.Artist{ + {ID: "fts-rank-1", Name: "MØ", OrderArtistName: "mø"}, + {ID: "fts-rank-2", Name: "Modest Mouse", OrderArtistName: "modest mouse"}, + {ID: "fts-rank-3", Name: "Morrissey", OrderArtistName: "morrissey"}, + } { + Expect(createArtistWithLibrary(arr, &a, 1)).To(Succeed()) + } + }) + + It("ranks the exact transliterated match first for 'MO'", func() { + results, err := arr.Search("MO", model.QueryOptions{Max: 10}) + Expect(err).ToNot(HaveOccurred()) + Expect(results).To(HaveLen(3)) + Expect(results[0].Name).To(Equal("MØ"), "exact match via search_normalized must outrank prefix matches") + }) + + It("ranks the exact match first for the accented query 'MØ'", func() { + results, err := arr.Search("MØ", model.QueryOptions{Max: 10}) + Expect(err).ToNot(HaveOccurred()) + Expect(results).ToNot(BeEmpty()) + Expect(results[0].Name).To(Equal("MØ")) + }) + }) })