diff --git a/db/db.go b/db/db.go index 0945d1a00..e69661697 100644 --- a/db/db.go +++ b/db/db.go @@ -13,6 +13,7 @@ import ( "github.com/navidrome/navidrome/log" "github.com/navidrome/navidrome/utils/hasher" "github.com/navidrome/navidrome/utils/singleton" + "github.com/navidrome/navidrome/utils/str" "github.com/pressly/goose/v3" ) @@ -31,7 +32,10 @@ func Db() *sql.DB { return singleton.GetInstance(func() *sql.DB { sql.Register(Driver, &sqlite3.SQLiteDriver{ ConnectHook: func(conn *sqlite3.SQLiteConn) error { - return conn.RegisterFunc("SEEDEDRAND", hasher.HashFunc(), false) + if err := conn.RegisterFunc("SEEDEDRAND", hasher.HashFunc(), false); err != nil { + return err + } + return conn.RegisterCollation("NATURALSORT", str.NaturalSortCompare) }, }) Path = conf.Server.DbPath diff --git a/db/migrations/20260216200000_add_natural_sort_collation_indexes.sql b/db/migrations/20260216200000_add_natural_sort_collation_indexes.sql new file mode 100644 index 000000000..24bcaf405 --- /dev/null +++ b/db/migrations/20260216200000_add_natural_sort_collation_indexes.sql @@ -0,0 +1,118 @@ +-- +goose Up + +-- Recreate indexes on order_* and sort expression fields to use NATURALSORT collation. +-- This enables natural number ordering (e.g., "Album 2" before "Album 10"). + +-- Artist indexes +drop index if exists artist_order_artist_name; +create index artist_order_artist_name + on artist (order_artist_name collate NATURALSORT); + +drop index if exists artist_sort_name; +create index artist_sort_name + on artist (coalesce(nullif(sort_artist_name,''),order_artist_name) collate NATURALSORT); + +-- Album indexes +drop index if exists album_order_album_name; +create index album_order_album_name + on album (order_album_name collate NATURALSORT); + +drop index if exists album_order_album_artist_name; +create index album_order_album_artist_name + on album (order_album_artist_name collate NATURALSORT); + +drop index if exists album_alphabetical_by_artist; +create index album_alphabetical_by_artist + on album (compilation, order_album_artist_name collate NATURALSORT, order_album_name collate NATURALSORT); + +drop index if exists album_sort_name; +create index album_sort_name + on album (coalesce(nullif(sort_album_name,''),order_album_name) collate NATURALSORT); + +drop index if exists album_sort_album_artist_name; +create index album_sort_album_artist_name + on album (coalesce(nullif(sort_album_artist_name,''),order_album_artist_name) collate NATURALSORT); + +-- Media file indexes +drop index if exists media_file_order_title; +create index media_file_order_title + on media_file (order_title collate NATURALSORT); + +drop index if exists media_file_order_album_name; +create index media_file_order_album_name + on media_file (order_album_name collate NATURALSORT); + +drop index if exists media_file_order_artist_name; +create index media_file_order_artist_name + on media_file (order_artist_name collate NATURALSORT); + +drop index if exists media_file_sort_title; +create index media_file_sort_title + on media_file (coalesce(nullif(sort_title,''),order_title) collate NATURALSORT); + +drop index if exists media_file_sort_artist_name; +create index media_file_sort_artist_name + on media_file (coalesce(nullif(sort_artist_name,''),order_artist_name) collate NATURALSORT); + +drop index if exists media_file_sort_album_name; +create index media_file_sort_album_name + on media_file (coalesce(nullif(sort_album_name,''),order_album_name) collate NATURALSORT); + +-- +goose Down + +-- Restore NOCASE collation indexes + +-- Artist indexes +drop index if exists artist_order_artist_name; +create index artist_order_artist_name + on artist (order_artist_name); + +drop index if exists artist_sort_name; +create index artist_sort_name + on artist (coalesce(nullif(sort_artist_name,''),order_artist_name) collate NOCASE); + +-- Album indexes +drop index if exists album_order_album_name; +create index album_order_album_name + on album (order_album_name); + +drop index if exists album_order_album_artist_name; +create index album_order_album_artist_name + on album (order_album_artist_name); + +drop index if exists album_alphabetical_by_artist; +create index album_alphabetical_by_artist + on album (compilation, order_album_artist_name, order_album_name); + +drop index if exists album_sort_name; +create index album_sort_name + on album (coalesce(nullif(sort_album_name,''),order_album_name) collate NOCASE); + +drop index if exists album_sort_album_artist_name; +create index album_sort_album_artist_name + on album (coalesce(nullif(sort_album_artist_name,''),order_album_artist_name) collate NOCASE); + +-- Media file indexes +drop index if exists media_file_order_title; +create index media_file_order_title + on media_file (order_title); + +drop index if exists media_file_order_album_name; +create index media_file_order_album_name + on media_file (order_album_name); + +drop index if exists media_file_order_artist_name; +create index media_file_order_artist_name + on media_file (order_artist_name); + +drop index if exists media_file_sort_title; +create index media_file_sort_title + on media_file (coalesce(nullif(sort_title,''),order_title) collate NOCASE); + +drop index if exists media_file_sort_artist_name; +create index media_file_sort_artist_name + on media_file (coalesce(nullif(sort_artist_name,''),order_artist_name) collate NOCASE); + +drop index if exists media_file_sort_album_name; +create index media_file_sort_album_name + on media_file (coalesce(nullif(sort_album_name,''),order_album_name) collate NOCASE); diff --git a/persistence/collation_test.go b/persistence/collation_test.go index bb1276577..24fb5ed13 100644 --- a/persistence/collation_test.go +++ b/persistence/collation_test.go @@ -41,18 +41,18 @@ var _ = Describe("Collation", func() { func(table, column string) { Expect(checkIndexUsage(conn, table, column)).To(Succeed()) }, - Entry("artist.order_artist_name", "artist", "order_artist_name collate nocase"), - Entry("artist.sort_artist_name", "artist", "coalesce(nullif(sort_artist_name,''),order_artist_name) collate nocase"), - Entry("album.order_album_name", "album", "order_album_name collate nocase"), - Entry("album.order_album_artist_name", "album", "order_album_artist_name collate nocase"), - Entry("album.sort_album_name", "album", "coalesce(nullif(sort_album_name,''),order_album_name) collate nocase"), - Entry("album.sort_album_artist_name", "album", "coalesce(nullif(sort_album_artist_name,''),order_album_artist_name) collate nocase"), - Entry("media_file.order_title", "media_file", "order_title collate nocase"), - Entry("media_file.order_album_name", "media_file", "order_album_name collate nocase"), - Entry("media_file.order_artist_name", "media_file", "order_artist_name collate nocase"), - Entry("media_file.sort_title", "media_file", "coalesce(nullif(sort_title,''),order_title) collate nocase"), - Entry("media_file.sort_album_name", "media_file", "coalesce(nullif(sort_album_name,''),order_album_name) collate nocase"), - Entry("media_file.sort_artist_name", "media_file", "coalesce(nullif(sort_artist_name,''),order_artist_name) collate nocase"), + Entry("artist.order_artist_name", "artist", "order_artist_name collate NATURALSORT"), + Entry("artist.sort_artist_name", "artist", "coalesce(nullif(sort_artist_name,''),order_artist_name) collate NATURALSORT"), + Entry("album.order_album_name", "album", "order_album_name collate NATURALSORT"), + Entry("album.order_album_artist_name", "album", "order_album_artist_name collate NATURALSORT"), + Entry("album.sort_album_name", "album", "coalesce(nullif(sort_album_name,''),order_album_name) collate NATURALSORT"), + Entry("album.sort_album_artist_name", "album", "coalesce(nullif(sort_album_artist_name,''),order_album_artist_name) collate NATURALSORT"), + Entry("media_file.order_title", "media_file", "order_title collate NATURALSORT"), + Entry("media_file.order_album_name", "media_file", "order_album_name collate NATURALSORT"), + Entry("media_file.order_artist_name", "media_file", "order_artist_name collate NATURALSORT"), + Entry("media_file.sort_title", "media_file", "coalesce(nullif(sort_title,''),order_title) collate NATURALSORT"), + Entry("media_file.sort_album_name", "media_file", "coalesce(nullif(sort_album_name,''),order_album_name) collate NATURALSORT"), + Entry("media_file.sort_artist_name", "media_file", "coalesce(nullif(sort_artist_name,''),order_artist_name) collate NATURALSORT"), Entry("media_file.path", "media_file", "path collate nocase"), Entry("playlist.name", "playlist", "name collate nocase"), Entry("radio.name", "radio", "name collate nocase"), diff --git a/persistence/helpers.go b/persistence/helpers.go index fd6a9a4cd..f96dc127b 100644 --- a/persistence/helpers.go +++ b/persistence/helpers.go @@ -82,11 +82,19 @@ func (e existsCond) ToSql() (string, []any, error) { var sortOrderRegex = regexp.MustCompile(`order_([a-z_]+)`) -// Convert the order_* columns to an expression using sort_* columns. Example: -// sort_album_name -> (coalesce(nullif(sort_album_name,”),order_album_name) collate nocase) +// mapSortOrder converts order_* columns to an expression using sort_* columns with NATURALSORT collation. Example: +// order_album_name -> (coalesce(nullif(sort_album_name,”),order_album_name) collate NATURALSORT) // It finds order column names anywhere in the substring func mapSortOrder(tableName, order string) string { order = strings.ToLower(order) - repl := fmt.Sprintf("(coalesce(nullif(%[1]s.sort_$1,''),%[1]s.order_$1) collate nocase)", tableName) + repl := fmt.Sprintf("(coalesce(nullif(%[1]s.sort_$1,''),%[1]s.order_$1) collate NATURALSORT)", tableName) return sortOrderRegex.ReplaceAllString(order, repl) } + +// mapNaturalSortCollation wraps bare order_* column references with NATURALSORT collation. Example: +// order_album_name -> (order_album_name collate NATURALSORT) +// It finds order column names anywhere in the substring +func mapNaturalSortCollation(order string) string { + order = strings.ToLower(order) + return sortOrderRegex.ReplaceAllString(order, "(order_$1 collate NATURALSORT)") +} diff --git a/persistence/helpers_test.go b/persistence/helpers_test.go index 85893ef55..a1bd30651 100644 --- a/persistence/helpers_test.go +++ b/persistence/helpers_test.go @@ -94,13 +94,23 @@ var _ = Describe("Helpers", func() { sort := "ORDER_ALBUM_NAME asc" mapped := mapSortOrder("album", sort) Expect(mapped).To(Equal(`(coalesce(nullif(album.sort_album_name,''),album.order_album_name)` + - ` collate nocase) asc`)) + ` collate NATURALSORT) asc`)) }) It("changes multiple order columns to sort expressions", func() { sort := "compilation, order_title asc, order_album_artist_name desc, year desc" mapped := mapSortOrder("album", sort) - Expect(mapped).To(Equal(`compilation, (coalesce(nullif(album.sort_title,''),album.order_title) collate nocase) asc,` + - ` (coalesce(nullif(album.sort_album_artist_name,''),album.order_album_artist_name) collate nocase) desc, year desc`)) + Expect(mapped).To(Equal(`compilation, (coalesce(nullif(album.sort_title,''),album.order_title) collate NATURALSORT) asc,` + + ` (coalesce(nullif(album.sort_album_artist_name,''),album.order_album_artist_name) collate NATURALSORT) desc, year desc`)) + }) + It("wraps bare order columns with NATURALSORT collation", func() { + sort := "order_album_name, order_album_artist_name" + mapped := mapNaturalSortCollation(sort) + Expect(mapped).To(Equal(`(order_album_name collate NATURALSORT), (order_album_artist_name collate NATURALSORT)`)) + }) + It("does not change non-order columns in mapNaturalSortCollation", func() { + sort := "compilation, year desc" + mapped := mapNaturalSortCollation(sort) + Expect(mapped).To(Equal(sort)) }) }) }) diff --git a/persistence/sql_base_repository.go b/persistence/sql_base_repository.go index fd263d37b..e5ef0e6f6 100644 --- a/persistence/sql_base_repository.go +++ b/persistence/sql_base_repository.go @@ -71,9 +71,12 @@ func (r *sqlRepository) registerModel(instance any, filters map[string]filterFun // // If PreferSortTags is enabled, it will map the order fields to the corresponding sort expression, // which gives precedence to sort tags. -// Ex: order_title => (coalesce(nullif(sort_title,”),order_title) collate nocase) +// Ex: order_title => (coalesce(nullif(sort_title,"),order_title) collate NATURALSORT) // To avoid performance issues, indexes should be created for these sort expressions // +// All order_* fields are wrapped with NATURALSORT collation to enable natural number ordering +// (e.g., "Album 2" sorts before "Album 10"). +// // NOTE: if an individual item has spaces, it should be wrapped in parentheses. For example, // you should write "(lyrics != '[]')". This prevents the item being split unexpectedly. // Without parentheses, "lyrics != '[]'" would be mapped as simply "lyrics" @@ -82,11 +85,13 @@ func (r *sqlRepository) setSortMappings(mappings map[string]string, tableName .. if len(tableName) > 0 { tn = tableName[0] } - if conf.Server.PreferSortTags { - for k, v := range mappings { + for k, v := range mappings { + if conf.Server.PreferSortTags { v = mapSortOrder(tn, v) - mappings[k] = v + } else { + v = mapNaturalSortCollation(v) } + mappings[k] = v } r.sortMappings = mappings } diff --git a/utils/str/natural_sort.go b/utils/str/natural_sort.go new file mode 100644 index 000000000..0a52bfc57 --- /dev/null +++ b/utils/str/natural_sort.go @@ -0,0 +1,142 @@ +package str + +import ( + "strings" +) + +// NaturalSortCompare compares two strings using natural sort ordering, +// where embedded numeric sequences are compared as numbers rather than +// lexicographically. For example, "track2" < "track10" (unlike lexicographic +// ordering which gives "track10" < "track2"). The comparison is also +// case-insensitive. +func NaturalSortCompare(a, b string) int { + ia, ib := 0, 0 + for ia < len(a) && ib < len(b) { + ca := a[ia] + cb := b[ib] + + // If both characters are digits, compare the full numeric sequences + if isDigit(ca) && isDigit(cb) { + result := compareNumericChunks(a, b, &ia, &ib) + if result != 0 { + return result + } + continue + } + + // Case-insensitive character comparison + la := toLower(ca) + lb := toLower(cb) + if la != lb { + if la < lb { + return -1 + } + return 1 + } + + ia++ + ib++ + } + + // The shorter string comes first if all else is equal + return len(a) - len(b) +} + +// compareNumericChunks compares two numeric sequences starting at positions +// ia and ib in strings a and b. It advances the position indices past the +// numeric sequences. Numbers are compared by value, with leading zeros +// used as a tiebreaker (fewer leading zeros comes first). +func compareNumericChunks(a, b string, ia, ib *int) int { + // Skip leading zeros and count them + zerosA := 0 + for *ia < len(a) && a[*ia] == '0' { + zerosA++ + *ia++ + } + zerosB := 0 + for *ib < len(b) && b[*ib] == '0' { + zerosB++ + *ib++ + } + + // Find the extent of the remaining digits + startA := *ia + for *ia < len(a) && isDigit(a[*ia]) { + *ia++ + } + startB := *ib + for *ib < len(b) && isDigit(b[*ib]) { + *ib++ + } + + lenA := *ia - startA + lenB := *ib - startB + + // More significant digits means a larger number + if lenA != lenB { + return lenA - lenB + } + + // Same number of significant digits - compare digit by digit + for i := 0; i < lenA; i++ { + if a[startA+i] != b[startB+i] { + if a[startA+i] < b[startB+i] { + return -1 + } + return 1 + } + } + + // Same numeric value - fewer leading zeros comes first + if zerosA != zerosB { + return zerosA - zerosB + } + + return 0 +} + +func isDigit(c byte) bool { + return c >= '0' && c <= '9' +} + +func toLower(c byte) byte { + if c >= 'A' && c <= 'Z' { + return c + ('a' - 'A') + } + return c +} + +// NaturalSortKey transforms a string into a key that, when compared +// lexicographically with NOCASE collation, produces natural sort order. +// Numeric sequences are zero-padded to a fixed width so that lexicographic +// comparison yields numeric ordering. +func NaturalSortKey(s string) string { + const padWidth = 20 // Enough for uint64 max + + var b strings.Builder + b.Grow(len(s) + padWidth) // Pre-allocate a reasonable size + + i := 0 + for i < len(s) { + if isDigit(s[i]) { + // Find the full numeric sequence + start := i + for i < len(s) && isDigit(s[i]) { + i++ + } + numStr := s[start:i] + + // Pad the number with leading zeros to padWidth + if len(numStr) < padWidth { + for j := 0; j < padWidth-len(numStr); j++ { + b.WriteByte('0') + } + } + b.WriteString(numStr) + } else { + b.WriteByte(s[i]) + i++ + } + } + return b.String() +} diff --git a/utils/str/natural_sort_test.go b/utils/str/natural_sort_test.go new file mode 100644 index 000000000..fab0d7232 --- /dev/null +++ b/utils/str/natural_sort_test.go @@ -0,0 +1,150 @@ +package str_test + +import ( + "testing" + + "github.com/navidrome/navidrome/utils/str" +) + +func TestNaturalSortCompare(t *testing.T) { + tests := []struct { + name string + a, b string + want int // -1, 0, or 1 + }{ + // Basic string comparison + {"equal strings", "abc", "abc", 0}, + {"less than", "abc", "abd", -1}, + {"greater than", "abd", "abc", 1}, + + // Case insensitive + {"case insensitive equal", "ABC", "abc", 0}, + {"case insensitive less", "ABC", "abd", -1}, + {"case insensitive mixed", "aBc", "AbC", 0}, + + // Numeric ordering + {"single digit order", "track1", "track2", -1}, + {"natural sort 2 vs 10", "track2", "track10", -1}, + {"natural sort 9 vs 10", "track9", "track10", -1}, + {"natural sort 10 vs 2", "track10", "track2", 1}, + {"natural sort 1 vs 100", "track1", "track100", -1}, + + // Album numbering (main use case from issue) + {"album numbers 1 vs 2", "Bravo Hits 1", "Bravo Hits 2", -1}, + {"album numbers 2 vs 10", "Bravo Hits 2", "Bravo Hits 10", -1}, + {"album numbers 10 vs 100", "Bravo Hits 10", "Bravo Hits 100", -1}, + {"album numbers 9 vs 10", "Bravo Hits 9", "Bravo Hits 10", -1}, + {"album numbers equal", "Bravo Hits 10", "Bravo Hits 10", 0}, + {"album numbers 99 vs 100", "Bravo Hits 99", "Bravo Hits 100", -1}, + {"album numbers 100 vs 101", "Bravo Hits 100", "Bravo Hits 101", -1}, + + // Pure numeric strings + {"pure number 1 vs 2", "1", "2", -1}, + {"pure number 2 vs 10", "2", "10", -1}, + {"pure number 10 vs 9", "10", "9", 1}, + {"pure number equal", "42", "42", 0}, + + // Leading zeros (same numeric value, but more leading zeros sorts later) + {"leading zeros same value", "01", "1", 1}, + {"leading zeros 02 vs 1", "02", "1", 1}, + {"leading zeros 01 vs 10", "01", "10", -1}, + + // Multiple numeric sequences + {"multi number 1.2 vs 1.10", "file1.2", "file1.10", -1}, + {"multi number 2.1 vs 10.1", "file2.1", "file10.1", -1}, + + // Empty strings + {"both empty", "", "", 0}, + {"empty vs non-empty", "", "a", -1}, + {"non-empty vs empty", "a", "", 1}, + + // Strings with only numbers at different positions + {"number prefix vs alpha prefix", "1abc", "abc", -1}, + {"alpha prefix vs number prefix", "abc", "1abc", 1}, + + // Edge cases + {"same prefix different length", "abc", "abcd", -1}, + {"numbers at beginning", "10abc", "9abc", 1}, + {"numbers at end", "abc10", "abc9", 1}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := str.NaturalSortCompare(tt.a, tt.b) + // Normalize to -1, 0, 1 for comparison + gotNorm := normalize(got) + if gotNorm != tt.want { + t.Errorf("NaturalSortCompare(%q, %q) = %d (normalized: %d), want %d", tt.a, tt.b, got, gotNorm, tt.want) + } + }) + } +} + +func TestNaturalSortCompare_Symmetry(t *testing.T) { + pairs := [][2]string{ + {"track2", "track10"}, + {"abc", "abd"}, + {"Bravo Hits 9", "Bravo Hits 10"}, + {"1", "2"}, + {"file1.2", "file1.10"}, + } + + for _, pair := range pairs { + a, b := pair[0], pair[1] + ab := str.NaturalSortCompare(a, b) + ba := str.NaturalSortCompare(b, a) + if normalize(ab) != -normalize(ba) { + t.Errorf("Symmetry violated: Compare(%q,%q)=%d but Compare(%q,%q)=%d", a, b, ab, b, a, ba) + } + } +} + +func TestNaturalSortKey(t *testing.T) { + // Test that sorting by NaturalSortKey produces natural order + inputs := []string{ + "Bravo Hits 1", + "Bravo Hits 10", + "Bravo Hits 100", + "Bravo Hits 2", + "Bravo Hits 20", + "Bravo Hits 3", + "Bravo Hits 9", + } + expected := []string{ + "Bravo Hits 1", + "Bravo Hits 2", + "Bravo Hits 3", + "Bravo Hits 9", + "Bravo Hits 10", + "Bravo Hits 20", + "Bravo Hits 100", + } + + // Get the keys and verify they sort correctly + keys := make([]string, len(inputs)) + for i, s := range inputs { + keys[i] = str.NaturalSortKey(s) + } + + // Verify the expected order produces keys in ascending order + expectedKeys := make([]string, len(expected)) + for i, s := range expected { + expectedKeys[i] = str.NaturalSortKey(s) + } + for i := 1; i < len(expectedKeys); i++ { + if expectedKeys[i] <= expectedKeys[i-1] { + t.Errorf("NaturalSortKey order violation: key(%q) = %q should be > key(%q) = %q", + expected[i], expectedKeys[i], expected[i-1], expectedKeys[i-1]) + } + } +} + +func normalize(v int) int { + if v < 0 { + return -1 + } + if v > 0 { + return 1 + } + return 0 +}