nik/navidrome
1
0
Fork 0
mirror of https://github.com/navidrome/navidrome.git synced 2026-08-01 07:21:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: update collation handling for natural sorting in SQL queries

Browse Source 
Signed-off-by: Deluan <deluan@navidrome.org>
This commit is contained in:
Deluan 2026-02-17 10:37:08 -05:00
parent 24ab04581a
commit 90d6cd5f47
6 changed files with 46 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
db/db.go
View File

@ -6,6 +6,7 @@ import (
"embed"
"fmt"
"runtime"
"strings"
"github.com/maruel/natural"
"github.com/mattn/go-sqlite3"
@ -35,7 +36,9 @@ func Db() *sql.DB {
if err := conn.RegisterFunc("SEEDEDRAND", hasher.HashFunc(), false); err != nil {
return err
}
return conn.RegisterCollation("NATURALSORT", natural.Compare)
return conn.RegisterCollation("NATURALSORT", func(a, b string) int {
return natural.Compare(strings.ToLower(a), strings.ToLower(b))
})
},
})
Path = conf.Server.DbPath

16
db/migrations/20260216200000_add_natural_sort_collation_indexes.sql
View File

@ -1,5 +1,14 @@
-- +goose Up
-- Change order_*/sort_* column collation from NOCASE to NATURALSORT.
-- This way bare ORDER BY on these columns automatically uses natural sorting,
-- without needing explicit COLLATE NATURALSORT in every query.
PRAGMA writable_schema = ON;
UPDATE sqlite_master
SET sql = replace(sql, 'collate NOCASE', 'collate NATURALSORT')
WHERE type = 'table' AND name IN ('artist', 'album', 'media_file');
PRAGMA writable_schema = OFF;
-- Recreate indexes on order_* and sort expression fields to use NATURALSORT collation.
-- This enables natural number ordering (e.g., "Album 2" before "Album 10").
@ -60,6 +69,13 @@ create index media_file_sort_album_name
-- +goose Down
-- Restore NOCASE column collation
PRAGMA writable_schema = ON;
UPDATE sqlite_master
SET sql = replace(sql, 'collate NATURALSORT', 'collate NOCASE')
WHERE type = 'table' AND name IN ('artist', 'album', 'media_file');
PRAGMA writable_schema = OFF;
-- Restore NOCASE collation indexes
-- Artist indexes

40
persistence/collation_test.go
View File

@ -17,24 +17,24 @@ import (
var _ = Describe("Collation", func() {
conn := db.Db()
DescribeTable("Column collation",
func(table, column string) {
Expect(checkCollation(conn, table, column)).To(Succeed())
func(table, column, expectedCollation string) {
Expect(checkCollation(conn, table, column, expectedCollation)).To(Succeed())
},
Entry("artist.order_artist_name", "artist", "order_artist_name"),
Entry("artist.sort_artist_name", "artist", "sort_artist_name"),
Entry("album.order_album_name", "album", "order_album_name"),
Entry("album.order_album_artist_name", "album", "order_album_artist_name"),
Entry("album.sort_album_name", "album", "sort_album_name"),
Entry("album.sort_album_artist_name", "album", "sort_album_artist_name"),
Entry("media_file.order_title", "media_file", "order_title"),
Entry("media_file.order_album_name", "media_file", "order_album_name"),
Entry("media_file.order_artist_name", "media_file", "order_artist_name"),
Entry("media_file.sort_title", "media_file", "sort_title"),
Entry("media_file.sort_album_name", "media_file", "sort_album_name"),
Entry("media_file.sort_artist_name", "media_file", "sort_artist_name"),
Entry("playlist.name", "playlist", "name"),
Entry("radio.name", "radio", "name"),
Entry("user.name", "user", "name"),
Entry("artist.order_artist_name", "artist", "order_artist_name", "NATURALSORT"),
Entry("artist.sort_artist_name", "artist", "sort_artist_name", "NATURALSORT"),
Entry("album.order_album_name", "album", "order_album_name", "NATURALSORT"),
Entry("album.order_album_artist_name", "album", "order_album_artist_name", "NATURALSORT"),
Entry("album.sort_album_name", "album", "sort_album_name", "NATURALSORT"),
Entry("album.sort_album_artist_name", "album", "sort_album_artist_name", "NATURALSORT"),
Entry("media_file.order_title", "media_file", "order_title", "NATURALSORT"),
Entry("media_file.order_album_name", "media_file", "order_album_name", "NATURALSORT"),
Entry("media_file.order_artist_name", "media_file", "order_artist_name", "NATURALSORT"),
Entry("media_file.sort_title", "media_file", "sort_title", "NATURALSORT"),
Entry("media_file.sort_album_name", "media_file", "sort_album_name", "NATURALSORT"),
Entry("media_file.sort_artist_name", "media_file", "sort_artist_name", "NATURALSORT"),
Entry("playlist.name", "playlist", "name", "NOCASE"),
Entry("radio.name", "radio", "name", "NOCASE"),
Entry("user.name", "user", "name", "NOCASE"),
)
DescribeTable("Index collation",
@ -91,7 +91,7 @@ order by %[2]s`, table, column))
return errors.New("no rows returned")
}
func checkCollation(conn *sql.DB, table string, column string) error {
func checkCollation(conn *sql.DB, table, column, expectedCollation string) error {
rows, err := conn.Query(fmt.Sprintf("SELECT sql FROM sqlite_master WHERE type='table' AND tbl_name='%s'", table))
if err != nil {
return err
@ -113,12 +113,12 @@ func checkCollation(conn *sql.DB, table string, column string) error {
if !re.MatchString(res) {
return fmt.Errorf("column '%s' not found in table '%s'", column, table)
}
re = regexp.MustCompile(fmt.Sprintf(`(?i)\b%s\b.*collate\s+NOCASE`, column))
re = regexp.MustCompile(fmt.Sprintf(`(?i)\b%s\b.*collate\s+%s`, column, expectedCollation))
if re.MatchString(res) {
return nil
}
} else {
return fmt.Errorf("table '%s' not found", table)
}
return fmt.Errorf("column '%s' in table '%s' does not have NOCASE collation", column, table)
return fmt.Errorf("column '%s' in table '%s' does not have %s collation", column, table, expectedCollation)
}

8
persistence/helpers.go
View File

@ -90,11 +90,3 @@ func mapSortOrder(tableName, order string) string {
repl := fmt.Sprintf("(coalesce(nullif(%[1]s.sort_$1,''),%[1]s.order_$1) collate NATURALSORT)", tableName)
return sortOrderRegex.ReplaceAllString(order, repl)
}
// mapNaturalSortCollation wraps bare order_* column references with NATURALSORT collation. Example:
// order_album_name -> (order_album_name collate NATURALSORT)
// It finds order column names anywhere in the substring
func mapNaturalSortCollation(order string) string {
order = strings.ToLower(order)
return sortOrderRegex.ReplaceAllString(order, "(order_$1 collate NATURALSORT)")
}

10
persistence/helpers_test.go
View File

@ -102,15 +102,5 @@ var _ = Describe("Helpers", func() {
Expect(mapped).To(Equal(`compilation, (coalesce(nullif(album.sort_title,''),album.order_title) collate NATURALSORT) asc,` +
` (coalesce(nullif(album.sort_album_artist_name,''),album.order_album_artist_name) collate NATURALSORT) desc, year desc`))
})
It("wraps bare order columns with NATURALSORT collation", func() {
sort := "order_album_name, order_album_artist_name"
mapped := mapNaturalSortCollation(sort)
Expect(mapped).To(Equal(`(order_album_name collate NATURALSORT), (order_album_artist_name collate NATURALSORT)`))
})
It("does not change non-order columns in mapNaturalSortCollation", func() {
sort := "compilation, year desc"
mapped := mapNaturalSortCollation(sort)
Expect(mapped).To(Equal(sort))
})
})
})

14
persistence/sql_base_repository.go
View File

@ -72,10 +72,10 @@ func (r *sqlRepository) registerModel(instance any, filters map[string]filterFun
// If PreferSortTags is enabled, it will map the order fields to the corresponding sort expression,
// which gives precedence to sort tags.
// Ex: order_title => (coalesce(nullif(sort_title,""), order_title) collate NATURALSORT)
// To avoid performance issues, indexes should be created for these sort expressions
// To avoid performance issues, indexes should be created for these sort expressions.
//
// All order_* fields are wrapped with NATURALSORT collation to enable natural number ordering
// (e.g., "Album 2" sorts before "Album 10").
// When PreferSortTags is off, bare order_* columns automatically use their column-defined NATURALSORT
// collation, so no query-time wrapping is needed.
//
// NOTE: if an individual item has spaces, it should be wrapped in parentheses. For example,
// you should write "(lyrics != '[]')". This prevents the item being split unexpectedly.
@ -85,13 +85,11 @@ func (r *sqlRepository) setSortMappings(mappings map[string]string, tableName ..
if len(tableName) > 0 {
tn = tableName[0]
}
for k, v := range mappings {
if conf.Server.PreferSortTags {
if conf.Server.PreferSortTags {
for k, v := range mappings {
v = mapSortOrder(tn, v)
} else {
v = mapNaturalSortCollation(v)
mappings[k] = v
}
mappings[k] = v
}
r.sortMappings = mappings
}