diff --git a/db/db.go b/db/db.go index 0945d1a00..ac795cd5c 100644 --- a/db/db.go +++ b/db/db.go @@ -6,7 +6,9 @@ import ( "embed" "fmt" "runtime" + "strings" + "github.com/maruel/natural" "github.com/mattn/go-sqlite3" "github.com/navidrome/navidrome/conf" _ "github.com/navidrome/navidrome/db/migrations" @@ -31,7 +33,12 @@ func Db() *sql.DB { return singleton.GetInstance(func() *sql.DB { sql.Register(Driver, &sqlite3.SQLiteDriver{ ConnectHook: func(conn *sqlite3.SQLiteConn) error { - return conn.RegisterFunc("SEEDEDRAND", hasher.HashFunc(), false) + if err := conn.RegisterFunc("SEEDEDRAND", hasher.HashFunc(), false); err != nil { + return err + } + return conn.RegisterCollation("NATURALSORT", func(a, b string) int { + return natural.Compare(strings.ToLower(a), strings.ToLower(b)) + }) }, }) Path = conf.Server.DbPath diff --git a/db/migrations/20260216200000_add_natural_sort_collation_indexes.sql b/db/migrations/20260216200000_add_natural_sort_collation_indexes.sql new file mode 100644 index 000000000..a92234ba6 --- /dev/null +++ b/db/migrations/20260216200000_add_natural_sort_collation_indexes.sql @@ -0,0 +1,152 @@ +-- +goose Up + +-- Change order_*/sort_* column collation from NOCASE to NATURALSORT. +-- This way bare ORDER BY on these columns automatically uses natural sorting, +-- without needing explicit COLLATE NATURALSORT in every query. +PRAGMA writable_schema = ON; +UPDATE sqlite_master +SET sql = replace(sql, 'collate NOCASE', 'collate NATURALSORT') +WHERE type = 'table' AND name IN ('artist', 'album', 'media_file', 'playlist', 'radio'); +PRAGMA writable_schema = OFF; + +-- Recreate indexes on order_* and sort expression fields to use NATURALSORT collation. +-- This enables natural number ordering (e.g., "Album 2" before "Album 10"). + +-- Artist indexes +drop index if exists artist_order_artist_name; +create index artist_order_artist_name + on artist (order_artist_name collate NATURALSORT); + +drop index if exists artist_sort_name; +create index artist_sort_name + on artist (coalesce(nullif(sort_artist_name,''),order_artist_name) collate NATURALSORT); + +-- Album indexes +drop index if exists album_order_album_name; +create index album_order_album_name + on album (order_album_name collate NATURALSORT); + +drop index if exists album_order_album_artist_name; +create index album_order_album_artist_name + on album (order_album_artist_name collate NATURALSORT); + +drop index if exists album_alphabetical_by_artist; +create index album_alphabetical_by_artist + on album (compilation, order_album_artist_name collate NATURALSORT, order_album_name collate NATURALSORT); + +drop index if exists album_sort_name; +create index album_sort_name + on album (coalesce(nullif(sort_album_name,''),order_album_name) collate NATURALSORT); + +drop index if exists album_sort_album_artist_name; +create index album_sort_album_artist_name + on album (coalesce(nullif(sort_album_artist_name,''),order_album_artist_name) collate NATURALSORT); + +-- Media file indexes +drop index if exists media_file_order_title; +create index media_file_order_title + on media_file (order_title collate NATURALSORT); + +drop index if exists media_file_order_album_name; +create index media_file_order_album_name + on media_file (order_album_name collate NATURALSORT); + +drop index if exists media_file_order_artist_name; +create index media_file_order_artist_name + on media_file (order_artist_name collate NATURALSORT); + +drop index if exists media_file_sort_title; +create index media_file_sort_title + on media_file (coalesce(nullif(sort_title,''),order_title) collate NATURALSORT); + +drop index if exists media_file_sort_artist_name; +create index media_file_sort_artist_name + on media_file (coalesce(nullif(sort_artist_name,''),order_artist_name) collate NATURALSORT); + +drop index if exists media_file_sort_album_name; +create index media_file_sort_album_name + on media_file (coalesce(nullif(sort_album_name,''),order_album_name) collate NATURALSORT); + +-- Playlist and radio indexes: recreate to match new NATURALSORT column collation +drop index if exists playlist_name; +create index playlist_name + on playlist (name collate NATURALSORT); + +drop index if exists radio_name; +create index radio_name + on radio (name collate NATURALSORT); + +-- +goose Down + +-- Restore NOCASE column collation +PRAGMA writable_schema = ON; +UPDATE sqlite_master +SET sql = replace(sql, 'collate NATURALSORT', 'collate NOCASE') +WHERE type = 'table' AND name IN ('artist', 'album', 'media_file', 'playlist', 'radio'); +PRAGMA writable_schema = OFF; + +-- Restore NOCASE collation indexes + +-- Artist indexes +drop index if exists artist_order_artist_name; +create index artist_order_artist_name + on artist (order_artist_name); + +drop index if exists artist_sort_name; +create index artist_sort_name + on artist (coalesce(nullif(sort_artist_name,''),order_artist_name) collate NOCASE); + +-- Album indexes +drop index if exists album_order_album_name; +create index album_order_album_name + on album (order_album_name); + +drop index if exists album_order_album_artist_name; +create index album_order_album_artist_name + on album (order_album_artist_name); + +drop index if exists album_alphabetical_by_artist; +create index album_alphabetical_by_artist + on album (compilation, order_album_artist_name, order_album_name); + +drop index if exists album_sort_name; +create index album_sort_name + on album (coalesce(nullif(sort_album_name,''),order_album_name) collate NOCASE); + +drop index if exists album_sort_album_artist_name; +create index album_sort_album_artist_name + on album (coalesce(nullif(sort_album_artist_name,''),order_album_artist_name) collate NOCASE); + +-- Media file indexes +drop index if exists media_file_order_title; +create index media_file_order_title + on media_file (order_title); + +drop index if exists media_file_order_album_name; +create index media_file_order_album_name + on media_file (order_album_name); + +drop index if exists media_file_order_artist_name; +create index media_file_order_artist_name + on media_file (order_artist_name); + +drop index if exists media_file_sort_title; +create index media_file_sort_title + on media_file (coalesce(nullif(sort_title,''),order_title) collate NOCASE); + +drop index if exists media_file_sort_artist_name; +create index media_file_sort_artist_name + on media_file (coalesce(nullif(sort_artist_name,''),order_artist_name) collate NOCASE); + +drop index if exists media_file_sort_album_name; +create index media_file_sort_album_name + on media_file (coalesce(nullif(sort_album_name,''),order_album_name) collate NOCASE); + +-- Restore playlist and radio indexes +drop index if exists playlist_name; +create index playlist_name + on playlist (name); + +drop index if exists radio_name; +create index radio_name + on radio (name); diff --git a/persistence/collation_test.go b/persistence/collation_test.go index bb1276577..61a28c106 100644 --- a/persistence/collation_test.go +++ b/persistence/collation_test.go @@ -17,45 +17,45 @@ import ( var _ = Describe("Collation", func() { conn := db.Db() DescribeTable("Column collation", - func(table, column string) { - Expect(checkCollation(conn, table, column)).To(Succeed()) + func(table, column, expectedCollation string) { + Expect(checkCollation(conn, table, column, expectedCollation)).To(Succeed()) }, - Entry("artist.order_artist_name", "artist", "order_artist_name"), - Entry("artist.sort_artist_name", "artist", "sort_artist_name"), - Entry("album.order_album_name", "album", "order_album_name"), - Entry("album.order_album_artist_name", "album", "order_album_artist_name"), - Entry("album.sort_album_name", "album", "sort_album_name"), - Entry("album.sort_album_artist_name", "album", "sort_album_artist_name"), - Entry("media_file.order_title", "media_file", "order_title"), - Entry("media_file.order_album_name", "media_file", "order_album_name"), - Entry("media_file.order_artist_name", "media_file", "order_artist_name"), - Entry("media_file.sort_title", "media_file", "sort_title"), - Entry("media_file.sort_album_name", "media_file", "sort_album_name"), - Entry("media_file.sort_artist_name", "media_file", "sort_artist_name"), - Entry("playlist.name", "playlist", "name"), - Entry("radio.name", "radio", "name"), - Entry("user.name", "user", "name"), + Entry("artist.order_artist_name", "artist", "order_artist_name", "NATURALSORT"), + Entry("artist.sort_artist_name", "artist", "sort_artist_name", "NATURALSORT"), + Entry("album.order_album_name", "album", "order_album_name", "NATURALSORT"), + Entry("album.order_album_artist_name", "album", "order_album_artist_name", "NATURALSORT"), + Entry("album.sort_album_name", "album", "sort_album_name", "NATURALSORT"), + Entry("album.sort_album_artist_name", "album", "sort_album_artist_name", "NATURALSORT"), + Entry("media_file.order_title", "media_file", "order_title", "NATURALSORT"), + Entry("media_file.order_album_name", "media_file", "order_album_name", "NATURALSORT"), + Entry("media_file.order_artist_name", "media_file", "order_artist_name", "NATURALSORT"), + Entry("media_file.sort_title", "media_file", "sort_title", "NATURALSORT"), + Entry("media_file.sort_album_name", "media_file", "sort_album_name", "NATURALSORT"), + Entry("media_file.sort_artist_name", "media_file", "sort_artist_name", "NATURALSORT"), + Entry("playlist.name", "playlist", "name", "NATURALSORT"), + Entry("radio.name", "radio", "name", "NATURALSORT"), + Entry("user.name", "user", "name", "NOCASE"), ) DescribeTable("Index collation", func(table, column string) { Expect(checkIndexUsage(conn, table, column)).To(Succeed()) }, - Entry("artist.order_artist_name", "artist", "order_artist_name collate nocase"), - Entry("artist.sort_artist_name", "artist", "coalesce(nullif(sort_artist_name,''),order_artist_name) collate nocase"), - Entry("album.order_album_name", "album", "order_album_name collate nocase"), - Entry("album.order_album_artist_name", "album", "order_album_artist_name collate nocase"), - Entry("album.sort_album_name", "album", "coalesce(nullif(sort_album_name,''),order_album_name) collate nocase"), - Entry("album.sort_album_artist_name", "album", "coalesce(nullif(sort_album_artist_name,''),order_album_artist_name) collate nocase"), - Entry("media_file.order_title", "media_file", "order_title collate nocase"), - Entry("media_file.order_album_name", "media_file", "order_album_name collate nocase"), - Entry("media_file.order_artist_name", "media_file", "order_artist_name collate nocase"), - Entry("media_file.sort_title", "media_file", "coalesce(nullif(sort_title,''),order_title) collate nocase"), - Entry("media_file.sort_album_name", "media_file", "coalesce(nullif(sort_album_name,''),order_album_name) collate nocase"), - Entry("media_file.sort_artist_name", "media_file", "coalesce(nullif(sort_artist_name,''),order_artist_name) collate nocase"), + Entry("artist.order_artist_name", "artist", "order_artist_name collate NATURALSORT"), + Entry("artist.sort_artist_name", "artist", "coalesce(nullif(sort_artist_name,''),order_artist_name) collate NATURALSORT"), + Entry("album.order_album_name", "album", "order_album_name collate NATURALSORT"), + Entry("album.order_album_artist_name", "album", "order_album_artist_name collate NATURALSORT"), + Entry("album.sort_album_name", "album", "coalesce(nullif(sort_album_name,''),order_album_name) collate NATURALSORT"), + Entry("album.sort_album_artist_name", "album", "coalesce(nullif(sort_album_artist_name,''),order_album_artist_name) collate NATURALSORT"), + Entry("media_file.order_title", "media_file", "order_title collate NATURALSORT"), + Entry("media_file.order_album_name", "media_file", "order_album_name collate NATURALSORT"), + Entry("media_file.order_artist_name", "media_file", "order_artist_name collate NATURALSORT"), + Entry("media_file.sort_title", "media_file", "coalesce(nullif(sort_title,''),order_title) collate NATURALSORT"), + Entry("media_file.sort_album_name", "media_file", "coalesce(nullif(sort_album_name,''),order_album_name) collate NATURALSORT"), + Entry("media_file.sort_artist_name", "media_file", "coalesce(nullif(sort_artist_name,''),order_artist_name) collate NATURALSORT"), Entry("media_file.path", "media_file", "path collate nocase"), - Entry("playlist.name", "playlist", "name collate nocase"), - Entry("radio.name", "radio", "name collate nocase"), + Entry("playlist.name", "playlist", "name collate NATURALSORT"), + Entry("radio.name", "radio", "name collate NATURALSORT"), Entry("user.user_name", "user", "user_name collate nocase"), ) }) @@ -91,7 +91,7 @@ order by %[2]s`, table, column)) return errors.New("no rows returned") } -func checkCollation(conn *sql.DB, table string, column string) error { +func checkCollation(conn *sql.DB, table, column, expectedCollation string) error { rows, err := conn.Query(fmt.Sprintf("SELECT sql FROM sqlite_master WHERE type='table' AND tbl_name='%s'", table)) if err != nil { return err @@ -113,12 +113,12 @@ func checkCollation(conn *sql.DB, table string, column string) error { if !re.MatchString(res) { return fmt.Errorf("column '%s' not found in table '%s'", column, table) } - re = regexp.MustCompile(fmt.Sprintf(`(?i)\b%s\b.*collate\s+NOCASE`, column)) + re = regexp.MustCompile(fmt.Sprintf(`(?i)\b%s\b.*collate\s+%s`, column, expectedCollation)) if re.MatchString(res) { return nil } } else { return fmt.Errorf("table '%s' not found", table) } - return fmt.Errorf("column '%s' in table '%s' does not have NOCASE collation", column, table) + return fmt.Errorf("column '%s' in table '%s' does not have %s collation", column, table, expectedCollation) } diff --git a/persistence/helpers.go b/persistence/helpers.go index fd6a9a4cd..3f58b3480 100644 --- a/persistence/helpers.go +++ b/persistence/helpers.go @@ -82,11 +82,11 @@ func (e existsCond) ToSql() (string, []any, error) { var sortOrderRegex = regexp.MustCompile(`order_([a-z_]+)`) -// Convert the order_* columns to an expression using sort_* columns. Example: -// sort_album_name -> (coalesce(nullif(sort_album_name,”),order_album_name) collate nocase) +// mapSortOrder converts order_* columns to an expression using sort_* columns with NATURALSORT collation. Example: +// order_album_name -> (coalesce(nullif(sort_album_name,”),order_album_name) collate NATURALSORT) // It finds order column names anywhere in the substring func mapSortOrder(tableName, order string) string { order = strings.ToLower(order) - repl := fmt.Sprintf("(coalesce(nullif(%[1]s.sort_$1,''),%[1]s.order_$1) collate nocase)", tableName) + repl := fmt.Sprintf("(coalesce(nullif(%[1]s.sort_$1,''),%[1]s.order_$1) collate NATURALSORT)", tableName) return sortOrderRegex.ReplaceAllString(order, repl) } diff --git a/persistence/helpers_test.go b/persistence/helpers_test.go index 85893ef55..dfb7c3f21 100644 --- a/persistence/helpers_test.go +++ b/persistence/helpers_test.go @@ -94,13 +94,13 @@ var _ = Describe("Helpers", func() { sort := "ORDER_ALBUM_NAME asc" mapped := mapSortOrder("album", sort) Expect(mapped).To(Equal(`(coalesce(nullif(album.sort_album_name,''),album.order_album_name)` + - ` collate nocase) asc`)) + ` collate NATURALSORT) asc`)) }) It("changes multiple order columns to sort expressions", func() { sort := "compilation, order_title asc, order_album_artist_name desc, year desc" mapped := mapSortOrder("album", sort) - Expect(mapped).To(Equal(`compilation, (coalesce(nullif(album.sort_title,''),album.order_title) collate nocase) asc,` + - ` (coalesce(nullif(album.sort_album_artist_name,''),album.order_album_artist_name) collate nocase) desc, year desc`)) + Expect(mapped).To(Equal(`compilation, (coalesce(nullif(album.sort_title,''),album.order_title) collate NATURALSORT) asc,` + + ` (coalesce(nullif(album.sort_album_artist_name,''),album.order_album_artist_name) collate NATURALSORT) desc, year desc`)) }) }) }) diff --git a/persistence/sql_base_repository.go b/persistence/sql_base_repository.go index fd263d37b..64968670c 100644 --- a/persistence/sql_base_repository.go +++ b/persistence/sql_base_repository.go @@ -71,8 +71,8 @@ func (r *sqlRepository) registerModel(instance any, filters map[string]filterFun // // If PreferSortTags is enabled, it will map the order fields to the corresponding sort expression, // which gives precedence to sort tags. -// Ex: order_title => (coalesce(nullif(sort_title,”),order_title) collate nocase) -// To avoid performance issues, indexes should be created for these sort expressions +// Ex: order_title => (coalesce(nullif(sort_title,""), order_title) collate NATURALSORT) +// To avoid performance issues, indexes should be created for these sort expressions. // // NOTE: if an individual item has spaces, it should be wrapped in parentheses. For example, // you should write "(lyrics != '[]')". This prevents the item being split unexpectedly.