mirror of
https://github.com/navidrome/navidrome.git
synced 2026-03-04 06:35:52 +00:00
* build: add sqlite_fts5 build tag to enable FTS5 support
* feat: add SearchBackend config option (default: fts)
* feat: add buildFTS5Query for safe FTS5 query preprocessing
* feat: add FTS5 search backend with config toggle, refactor legacy search
- Add searchExprFunc type and getSearchExpr() for backend selection
- Rename fullTextExpr to legacySearchExpr
- Add ftsSearchExpr using FTS5 MATCH subquery
- Update fullTextFilter in sql_restful.go to use configured backend
* feat: add FTS5 migration with virtual tables, triggers, and search_participants
Creates FTS5 virtual tables for media_file, album, and artist with
unicode61 tokenizer and diacritic folding. Adds search_participants
column, populates from JSON, and sets up INSERT/UPDATE/DELETE triggers.
* feat: populate search_participants in PostMapArgs for FTS5 indexing
* test: add FTS5 search integration tests
* fix: exclude FTS5 virtual tables from e2e DB restore
The restoreDB function iterates all tables in sqlite_master and
runs DELETE + INSERT to reset state. FTS5 contentless virtual tables
cannot be directly deleted from. Since triggers handle FTS5 sync
automatically, simply skip tables matching *_fts and *_fts_* patterns.
* build: add compile-time guard for sqlite_fts5 build tag
Same pattern as netgo: compilation fails with a clear error if
the sqlite_fts5 build tag is missing.
* build: add sqlite_fts5 tag to reflex dev server config
* build: extract GO_BUILD_TAGS variable in Makefile to avoid duplication
* fix: strip leading * from FTS5 queries to prevent "unknown special query" error
* feat: auto-append prefix wildcard to FTS5 search tokens for broader matching
Every plain search token now gets a trailing * appended (e.g., "love" becomes
"love*"), so searching for "love" also matches "lovelace", "lovely", etc.
Quoted phrases are preserved as exact matches without wildcards. Results are
ordered alphabetically by name/title, so shorter exact matches naturally
appear first.
* fix: clarify comments about FTS5 operator neutralization
The comments said "strip" but the code lowercases operators to
neutralize them (FTS5 operators are case-sensitive). Updated comments
to accurately describe the behavior.
* fix: use fmt.Sprintf for FTS5 phrase placeholders
The previous encoding used rune('0'+index) which silently breaks with
10+ quoted phrases. Use fmt.Sprintf for arbitrary index support.
* fix: validate and normalize SearchBackend config option
Normalize the value to lowercase and fall back to "fts" with a log
warning for unrecognized values. This prevents silent misconfiguration
from typos like "FTS", "Legacy", or "fts5".
* refactor: improve documentation for build tags and FTS5 requirements
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor: convert FTS5 query and search backend normalization tests to DescribeTable format
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: add sqlite_fts5 build tag to golangci configuration
Signed-off-by: Deluan <deluan@navidrome.org>
* feat: add UISearchDebounceMs configuration option and update related components
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: fall back to legacy search when SearchFullString is enabled
FTS5 is token-based and cannot match substrings within words, so
getSearchExpr now returns legacySearchExpr when SearchFullString
is true, regardless of SearchBackend setting.
* fix: add sqlite_fts5 build tag to CI pipeline and Dockerfile
* fix: add WHEN clauses to FTS5 AFTER UPDATE triggers
Added WHEN clauses to the media_file_fts_au, album_fts_au, and
artist_fts_au triggers so they only fire when FTS-indexed columns
actually change. Previously, every row update (e.g., play count, rating,
starred status) triggered an unnecessary delete+insert cycle in the FTS
shadow tables. The WHEN clauses use IS NOT for NULL-safe comparison of
each indexed column, avoiding FTS index churn for non-indexed updates.
* feat: add SearchBackend configuration option to data and insights components
Signed-off-by: Deluan <deluan@navidrome.org>
* fix: enhance input sanitization for FTS5 by stripping additional punctuation and special characters
Signed-off-by: Deluan <deluan@navidrome.org>
* feat: add search_normalized column for punctuated name search (R.E.M., AC/DC)
Add index-time normalization and query-time single-letter collapsing to
fix FTS5 search for punctuated names. A new search_normalized column
stores concatenated forms of punctuated words (e.g., "R.E.M." → "REM",
"AC/DC" → "ACDC") and is indexed in FTS5 tables. At query time, runs of
consecutive single letters (from dot-stripping) are collapsed into OR
expressions like ("R E M" OR REM*) to match both the original tokens and
the normalized form. This enables searching by "R.E.M.", "REM", "AC/DC",
"ACDC", "A-ha", or "Aha" and finding the correct results.
* refactor: simplify isSingleUnicodeLetter to avoid []rune allocation
Use utf8.DecodeRuneInString to check for a single Unicode letter
instead of converting the entire string to a []rune slice.
* feat: define ftsSearchColumns for flexible FTS5 search column inclusion
Signed-off-by: Deluan <deluan@navidrome.org>
* feat: update collapseSingleLetterRuns to return quoted phrases for abbreviations
Signed-off-by: Deluan <deluan@navidrome.org>
* feat: implement extractPunctuatedWords to handle artist/album names with embedded punctuation
Signed-off-by: Deluan <deluan@navidrome.org>
* feat: implement extractPunctuatedWords to handle artist/album names with embedded punctuation
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor: punctuated word handling to improve processing of artist/album names
Signed-off-by: Deluan <deluan@navidrome.org>
* feat: add CJK support for search queries with LIKE filters
Signed-off-by: Deluan <deluan@navidrome.org>
* feat: enhance FTS5 search by adding album version support and CJK handling
Signed-off-by: Deluan <deluan@navidrome.org>
* refactor: search configuration to use structured options
Signed-off-by: Deluan <deluan@navidrome.org>
* feat: enhance search functionality to support punctuation-only queries and update related tests
Signed-off-by: Deluan <deluan@navidrome.org>
---------
Signed-off-by: Deluan <deluan@navidrome.org>
296 lines
11 KiB
Go
296 lines
11 KiB
Go
package persistence
|
|
|
|
import (
|
|
"context"
|
|
"path/filepath"
|
|
"testing"
|
|
|
|
_ "github.com/mattn/go-sqlite3"
|
|
"github.com/navidrome/navidrome/conf"
|
|
"github.com/navidrome/navidrome/db"
|
|
"github.com/navidrome/navidrome/log"
|
|
"github.com/navidrome/navidrome/model"
|
|
"github.com/navidrome/navidrome/model/request"
|
|
"github.com/navidrome/navidrome/tests"
|
|
"github.com/navidrome/navidrome/utils/gg"
|
|
. "github.com/onsi/ginkgo/v2"
|
|
. "github.com/onsi/gomega"
|
|
"github.com/pocketbase/dbx"
|
|
)
|
|
|
|
func TestPersistence(t *testing.T) {
|
|
tests.Init(t, true)
|
|
|
|
//os.Remove("./test-123.db")
|
|
//conf.Server.DbPath = "./test-123.db"
|
|
conf.Server.DbPath = "file::memory:?cache=shared&_foreign_keys=on"
|
|
defer db.Init(context.Background())()
|
|
log.SetLevel(log.LevelFatal)
|
|
RegisterFailHandler(Fail)
|
|
RunSpecs(t, "Persistence Suite")
|
|
}
|
|
|
|
func mf(mf model.MediaFile) model.MediaFile {
|
|
mf.Tags = model.Tags{}
|
|
mf.LibraryID = 1
|
|
mf.LibraryPath = "music" // Default folder
|
|
mf.LibraryName = "Music Library"
|
|
mf.Participants = model.Participants{
|
|
model.RoleArtist: model.ParticipantList{
|
|
model.Participant{Artist: model.Artist{ID: mf.ArtistID, Name: mf.Artist}},
|
|
},
|
|
}
|
|
if mf.Lyrics == "" {
|
|
mf.Lyrics = "[]"
|
|
}
|
|
return mf
|
|
}
|
|
|
|
func al(al model.Album) model.Album {
|
|
al.LibraryID = 1
|
|
al.LibraryPath = "music"
|
|
al.LibraryName = "Music Library"
|
|
al.Discs = model.Discs{}
|
|
al.Tags = model.Tags{}
|
|
al.Participants = model.Participants{}
|
|
return al
|
|
}
|
|
|
|
func alWithTags(a model.Album, tags model.Tags) model.Album {
|
|
a = al(a)
|
|
a.Tags = tags
|
|
return a
|
|
}
|
|
|
|
var (
|
|
artistKraftwerk = model.Artist{ID: "2", Name: "Kraftwerk", OrderArtistName: "kraftwerk"}
|
|
artistBeatles = model.Artist{ID: "3", Name: "The Beatles", OrderArtistName: "beatles"}
|
|
artistCJK = model.Artist{ID: "4", Name: "シートベルツ", SortArtistName: "Seatbelts", OrderArtistName: "seatbelts"}
|
|
artistPunctuation = model.Artist{ID: "5", Name: "The Roots", OrderArtistName: "roots"}
|
|
testArtists = model.Artists{
|
|
artistKraftwerk,
|
|
artistBeatles,
|
|
artistCJK,
|
|
artistPunctuation,
|
|
}
|
|
)
|
|
|
|
var (
|
|
albumSgtPeppers = al(model.Album{ID: "101", Name: "Sgt Peppers", AlbumArtist: "The Beatles", OrderAlbumName: "sgt peppers", AlbumArtistID: "3", EmbedArtPath: p("/beatles/1/sgt/a day.mp3"), SongCount: 1, MaxYear: 1967})
|
|
albumAbbeyRoad = al(model.Album{ID: "102", Name: "Abbey Road", AlbumArtist: "The Beatles", OrderAlbumName: "abbey road", AlbumArtistID: "3", EmbedArtPath: p("/beatles/1/come together.mp3"), SongCount: 1, MaxYear: 1969})
|
|
albumRadioactivity = al(model.Album{ID: "103", Name: "Radioactivity", AlbumArtist: "Kraftwerk", OrderAlbumName: "radioactivity", AlbumArtistID: "2", EmbedArtPath: p("/kraft/radio/radio.mp3"), SongCount: 2})
|
|
albumMultiDisc = al(model.Album{ID: "104", Name: "Multi Disc Album", AlbumArtist: "Test Artist", OrderAlbumName: "multi disc album", AlbumArtistID: "1", EmbedArtPath: p("/test/multi/disc1/track1.mp3"), SongCount: 4})
|
|
albumCJK = al(model.Album{ID: "105", Name: "COWBOY BEBOP", AlbumArtist: "シートベルツ", OrderAlbumName: "cowboy bebop", AlbumArtistID: "4", EmbedArtPath: p("/seatbelts/cowboy-bebop/track1.mp3"), SongCount: 1})
|
|
albumWithVersion = alWithTags(model.Album{ID: "106", Name: "Abbey Road", AlbumArtist: "The Beatles", OrderAlbumName: "abbey road", AlbumArtistID: "3", EmbedArtPath: p("/beatles/2/come together.mp3"), SongCount: 1, MaxYear: 2019},
|
|
model.Tags{model.TagAlbumVersion: {"Deluxe Edition"}})
|
|
albumPunctuation = al(model.Album{ID: "107", Name: "Things Fall Apart", AlbumArtist: "The Roots", OrderAlbumName: "things fall apart", AlbumArtistID: "5", EmbedArtPath: p("/roots/things/track1.mp3"), SongCount: 1})
|
|
testAlbums = model.Albums{
|
|
albumSgtPeppers,
|
|
albumAbbeyRoad,
|
|
albumRadioactivity,
|
|
albumMultiDisc,
|
|
albumCJK,
|
|
albumWithVersion,
|
|
albumPunctuation,
|
|
}
|
|
)
|
|
|
|
var (
|
|
songDayInALife = mf(model.MediaFile{ID: "1001", Title: "A Day In A Life", ArtistID: "3", Artist: "The Beatles", AlbumID: "101", Album: "Sgt Peppers", Path: p("/beatles/1/sgt/a day.mp3")})
|
|
songComeTogether = mf(model.MediaFile{ID: "1002", Title: "Come Together", ArtistID: "3", Artist: "The Beatles", AlbumID: "102", Album: "Abbey Road", Path: p("/beatles/1/come together.mp3")})
|
|
songRadioactivity = mf(model.MediaFile{ID: "1003", Title: "Radioactivity", ArtistID: "2", Artist: "Kraftwerk", AlbumID: "103", Album: "Radioactivity", Path: p("/kraft/radio/radio.mp3")})
|
|
songAntenna = mf(model.MediaFile{ID: "1004", Title: "Antenna", ArtistID: "2", Artist: "Kraftwerk",
|
|
AlbumID: "103",
|
|
Path: p("/kraft/radio/antenna.mp3"),
|
|
RGAlbumGain: gg.P(1.0), RGAlbumPeak: gg.P(2.0), RGTrackGain: gg.P(3.0), RGTrackPeak: gg.P(4.0),
|
|
})
|
|
songAntennaWithLyrics = mf(model.MediaFile{
|
|
ID: "1005",
|
|
Title: "Antenna",
|
|
ArtistID: "2",
|
|
Artist: "Kraftwerk",
|
|
AlbumID: "103",
|
|
Lyrics: `[{"lang":"xxx","line":[{"value":"This is a set of lyrics"}],"synced":false}]`,
|
|
})
|
|
songAntenna2 = mf(model.MediaFile{ID: "1006", Title: "Antenna", ArtistID: "2", Artist: "Kraftwerk", AlbumID: "103"})
|
|
// Multi-disc album tracks (intentionally out of order to test sorting)
|
|
songDisc2Track11 = mf(model.MediaFile{ID: "2001", Title: "Disc 2 Track 11", ArtistID: "1", Artist: "Test Artist", AlbumID: "104", Album: "Multi Disc Album", DiscNumber: 2, TrackNumber: 11, Path: p("/test/multi/disc2/track11.mp3"), OrderAlbumName: "multi disc album", OrderArtistName: "test artist"})
|
|
songDisc1Track01 = mf(model.MediaFile{ID: "2002", Title: "Disc 1 Track 1", ArtistID: "1", Artist: "Test Artist", AlbumID: "104", Album: "Multi Disc Album", DiscNumber: 1, TrackNumber: 1, Path: p("/test/multi/disc1/track1.mp3"), OrderAlbumName: "multi disc album", OrderArtistName: "test artist"})
|
|
songDisc2Track01 = mf(model.MediaFile{ID: "2003", Title: "Disc 2 Track 1", ArtistID: "1", Artist: "Test Artist", AlbumID: "104", Album: "Multi Disc Album", DiscNumber: 2, TrackNumber: 1, Path: p("/test/multi/disc2/track1.mp3"), OrderAlbumName: "multi disc album", OrderArtistName: "test artist"})
|
|
songDisc1Track02 = mf(model.MediaFile{ID: "2004", Title: "Disc 1 Track 2", ArtistID: "1", Artist: "Test Artist", AlbumID: "104", Album: "Multi Disc Album", DiscNumber: 1, TrackNumber: 2, Path: p("/test/multi/disc1/track2.mp3"), OrderAlbumName: "multi disc album", OrderArtistName: "test artist"})
|
|
songCJK = mf(model.MediaFile{ID: "3001", Title: "プラチナ・ジェット", ArtistID: "4", Artist: "シートベルツ", AlbumID: "105", Album: "COWBOY BEBOP", Path: p("/seatbelts/cowboy-bebop/track1.mp3")})
|
|
songVersioned = mf(model.MediaFile{ID: "3002", Title: "Come Together", ArtistID: "3", Artist: "The Beatles", AlbumID: "106", Album: "Abbey Road", Path: p("/beatles/2/come together.mp3")})
|
|
songPunctuation = mf(model.MediaFile{ID: "3003", Title: "!!!!!!!", ArtistID: "5", Artist: "The Roots", AlbumID: "107", Album: "Things Fall Apart", Path: p("/roots/things/track1.mp3")})
|
|
testSongs = model.MediaFiles{
|
|
songDayInALife,
|
|
songComeTogether,
|
|
songRadioactivity,
|
|
songAntenna,
|
|
songAntennaWithLyrics,
|
|
songAntenna2,
|
|
songDisc2Track11,
|
|
songDisc1Track01,
|
|
songDisc2Track01,
|
|
songDisc1Track02,
|
|
songCJK,
|
|
songVersioned,
|
|
songPunctuation,
|
|
}
|
|
)
|
|
|
|
var (
|
|
radioWithoutHomePage = model.Radio{ID: "1235", StreamUrl: "https://example.com:8000/1/stream.mp3", HomePageUrl: "", Name: "No Homepage"}
|
|
radioWithHomePage = model.Radio{ID: "5010", StreamUrl: "https://example.com/stream.mp3", Name: "Example Radio", HomePageUrl: "https://example.com"}
|
|
testRadios = model.Radios{radioWithoutHomePage, radioWithHomePage}
|
|
)
|
|
|
|
var (
|
|
plsBest model.Playlist
|
|
plsCool model.Playlist
|
|
testPlaylists []*model.Playlist
|
|
)
|
|
|
|
var (
|
|
adminUser = model.User{ID: "userid", UserName: "userid", Name: "admin", Email: "admin@email.com", IsAdmin: true}
|
|
regularUser = model.User{ID: "2222", UserName: "regular-user", Name: "Regular User", Email: "regular@example.com"}
|
|
thirdUser = model.User{ID: "3333", UserName: "third-user", Name: "Third User", Email: "third@example.com"}
|
|
testUsers = model.Users{adminUser, regularUser, thirdUser}
|
|
)
|
|
|
|
func p(path string) string {
|
|
return filepath.FromSlash(path)
|
|
}
|
|
|
|
// Initialize test DB
|
|
// TODO Load this data setup from file(s)
|
|
var _ = BeforeSuite(func() {
|
|
conn := GetDBXBuilder()
|
|
ctx := log.NewContext(context.TODO())
|
|
ctx = request.WithUser(ctx, adminUser)
|
|
|
|
ur := NewUserRepository(ctx, conn)
|
|
for i := range testUsers {
|
|
err := ur.Put(&testUsers[i])
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
}
|
|
|
|
// Associate users with library 1 (default test library)
|
|
for i := range testUsers {
|
|
err := ur.SetUserLibraries(testUsers[i].ID, []int{1})
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
}
|
|
|
|
alr := NewAlbumRepository(ctx, conn).(*albumRepository)
|
|
for i := range testAlbums {
|
|
a := testAlbums[i]
|
|
err := alr.Put(&a)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
}
|
|
|
|
arr := NewArtistRepository(ctx, conn)
|
|
for i := range testArtists {
|
|
a := testArtists[i]
|
|
err := arr.Put(&a)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
}
|
|
|
|
// Associate artists with library 1 (default test library)
|
|
lr := NewLibraryRepository(ctx, conn)
|
|
for i := range testArtists {
|
|
err := lr.AddArtist(1, testArtists[i].ID)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
}
|
|
|
|
mr := NewMediaFileRepository(ctx, conn)
|
|
for i := range testSongs {
|
|
err := mr.Put(&testSongs[i])
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
}
|
|
|
|
rar := NewRadioRepository(ctx, conn)
|
|
for i := range testRadios {
|
|
r := testRadios[i]
|
|
err := rar.Put(&r)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
}
|
|
|
|
plsBest = model.Playlist{
|
|
Name: "Best",
|
|
Comment: "No Comments",
|
|
OwnerID: "userid",
|
|
OwnerName: "userid",
|
|
Public: true,
|
|
SongCount: 2,
|
|
}
|
|
plsBest.AddMediaFilesByID([]string{"1001", "1003"})
|
|
plsCool = model.Playlist{Name: "Cool", OwnerID: "userid", OwnerName: "userid"}
|
|
plsCool.AddMediaFilesByID([]string{"1004"})
|
|
testPlaylists = []*model.Playlist{&plsBest, &plsCool}
|
|
|
|
pr := NewPlaylistRepository(ctx, conn)
|
|
for i := range testPlaylists {
|
|
err := pr.Put(testPlaylists[i])
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
}
|
|
|
|
// Prepare annotations
|
|
if err := arr.SetStar(true, artistBeatles.ID); err != nil {
|
|
panic(err)
|
|
}
|
|
ar, err := arr.Get(artistBeatles.ID)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
if ar == nil {
|
|
panic("artist not found after SetStar")
|
|
}
|
|
artistBeatles.Starred = true
|
|
artistBeatles.StarredAt = ar.StarredAt
|
|
testArtists[1] = artistBeatles
|
|
|
|
if err := alr.SetStar(true, albumRadioactivity.ID); err != nil {
|
|
panic(err)
|
|
}
|
|
al, err := alr.Get(albumRadioactivity.ID)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
if al == nil {
|
|
panic("album not found after SetStar")
|
|
}
|
|
albumRadioactivity.Starred = true
|
|
albumRadioactivity.StarredAt = al.StarredAt
|
|
testAlbums[2] = albumRadioactivity
|
|
|
|
if err := mr.SetStar(true, songComeTogether.ID); err != nil {
|
|
panic(err)
|
|
}
|
|
mf, err := mr.Get(songComeTogether.ID)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
songComeTogether.Starred = true
|
|
songComeTogether.StarredAt = mf.StarredAt
|
|
testSongs[1] = songComeTogether
|
|
})
|
|
|
|
func GetDBXBuilder() *dbx.DB {
|
|
return dbx.NewFromDB(db.Db(), db.Dialect)
|
|
}
|