mirror of
https://github.com/navidrome/navidrome.git
synced 2026-08-01 07:21:17 +00:00
feat(db): migrate all ids to canonical 128-bit base62 encoding
This commit is contained in:
Deluan
parent
e8183fdfe0
commit
bf8f669fcf
154
db/migrations/20260720015443_uniform_canonical_ids.go
Normal file
154
db/migrations/20260720015443_uniform_canonical_ids.go
Normal file
@ -0,0 +1,154 @@
|
||||
package migrations
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/pressly/goose/v3"
|
||||
)
|
||||
|
||||
func init() {
|
||||
goose.AddMigrationContext(upUniformCanonicalIds, downUniformCanonicalIds)
|
||||
}
|
||||
|
||||
// idColumns is the exhaustive list of Navidrome-id-bearing columns; anything absent is deliberately exempt.
|
||||
var idColumns = []struct{ table, col string }{
|
||||
{"media_file", "id"}, {"media_file", "pid"}, {"media_file", "artist_id"},
|
||||
{"media_file", "album_id"}, {"media_file", "album_artist_id"}, {"media_file", "folder_id"},
|
||||
{"album", "id"}, {"album", "album_artist_id"},
|
||||
{"artist", "id"},
|
||||
{"folder", "id"}, {"folder", "parent_id"},
|
||||
{"tag", "id"},
|
||||
{"library_artist", "artist_id"},
|
||||
{"user", "id"},
|
||||
{"user_props", "user_id"},
|
||||
{"playlist", "id"}, {"playlist", "owner_id"},
|
||||
{"playlist_tracks", "playlist_id"}, {"playlist_tracks", "media_file_id"},
|
||||
{"playlist_fields", "playlist_id"},
|
||||
{"annotation", "user_id"}, {"annotation", "item_id"},
|
||||
{"bookmark", "user_id"}, {"bookmark", "item_id"},
|
||||
{"player", "id"}, {"player", "user_id"}, {"player", "transcoding_id"},
|
||||
{"transcoding", "id"},
|
||||
{"radio", "id"},
|
||||
{"share", "user_id"},
|
||||
{"scrobble_buffer", "id"}, {"scrobble_buffer", "user_id"}, {"scrobble_buffer", "media_file_id"},
|
||||
{"playqueue", "id"}, {"playqueue", "user_id"},
|
||||
}
|
||||
|
||||
func upUniformCanonicalIds(ctx context.Context, tx *sql.Tx) error {
|
||||
if err := buildIDMap(ctx, tx); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, tc := range idColumns {
|
||||
if err := applyIDMap(ctx, tx, tc.table, tc.col); err != nil {
|
||||
return fmt.Errorf("canonicalizing %s.%s: %w", tc.table, tc.col, err)
|
||||
}
|
||||
}
|
||||
if err := rewriteListColumn(ctx, tx, "playqueue", "items"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := rewriteListColumn(ctx, tx, "share", "resource_ids"); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := tx.ExecContext(ctx, "DROP TABLE _id_map")
|
||||
return err
|
||||
}
|
||||
|
||||
// buildIDMap stages old->new pairs for every id that changes, indexed for the update joins.
|
||||
func buildIDMap(ctx context.Context, tx *sql.Tx) error {
|
||||
_, err := tx.ExecContext(ctx,
|
||||
"CREATE TEMP TABLE _id_map (old_id TEXT PRIMARY KEY, new_id TEXT NOT NULL) WITHOUT ROWID")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ins, err := tx.PrepareContext(ctx, "INSERT OR IGNORE INTO _id_map (old_id, new_id) VALUES (?, ?)")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer ins.Close()
|
||||
for _, tc := range idColumns {
|
||||
if err := collectColumn(ctx, tx, ins, tc.table, tc.col); err != nil {
|
||||
return fmt.Errorf("collecting %s.%s: %w", tc.table, tc.col, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func collectColumn(ctx context.Context, tx *sql.Tx, ins *sql.Stmt, table, col string) error {
|
||||
rows, err := tx.QueryContext(ctx, fmt.Sprintf(
|
||||
"SELECT DISTINCT %[2]s FROM %[1]s WHERE %[2]s IS NOT NULL", table, col))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var old string
|
||||
if err := rows.Scan(&old); err != nil {
|
||||
return err
|
||||
}
|
||||
if newID := canonicalID(old); newID != old {
|
||||
if _, err := ins.ExecContext(ctx, old, newID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return rows.Err()
|
||||
}
|
||||
|
||||
func applyIDMap(ctx context.Context, tx *sql.Tx, table, col string) error {
|
||||
_, err := tx.ExecContext(ctx, fmt.Sprintf(
|
||||
`UPDATE %[1]s SET %[2]s = (SELECT new_id FROM _id_map WHERE old_id = %[1]s.%[2]s)
|
||||
WHERE %[2]s IN (SELECT old_id FROM _id_map)`, table, col))
|
||||
return err
|
||||
}
|
||||
|
||||
// rewriteListColumn canonicalizes comma-separated id lists element-wise.
|
||||
func rewriteListColumn(ctx context.Context, tx *sql.Tx, table, col string) error {
|
||||
rows, err := tx.QueryContext(ctx, fmt.Sprintf(
|
||||
"SELECT rowid, %[2]s FROM %[1]s WHERE ifnull(%[2]s, '') <> ''", table, col))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
type change struct {
|
||||
rowid int64
|
||||
val string
|
||||
}
|
||||
var changes []change
|
||||
for rows.Next() {
|
||||
var rowid int64
|
||||
var val string
|
||||
if err := rows.Scan(&rowid, &val); err != nil {
|
||||
_ = rows.Close()
|
||||
return err
|
||||
}
|
||||
parts := strings.Split(val, ",")
|
||||
changed := false
|
||||
for i, p := range parts {
|
||||
if n := canonicalID(p); n != p {
|
||||
parts[i] = n
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
if changed {
|
||||
changes = append(changes, change{rowid, strings.Join(parts, ",")})
|
||||
}
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
_ = rows.Close()
|
||||
return err
|
||||
}
|
||||
_ = rows.Close()
|
||||
for _, c := range changes {
|
||||
if _, err := tx.ExecContext(ctx, fmt.Sprintf(
|
||||
"UPDATE %s SET %s = ? WHERE rowid = ?", table, col), c.val, c.rowid); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func downUniformCanonicalIds(ctx context.Context, tx *sql.Tx) error {
|
||||
return nil // irreversible data migration
|
||||
}
|
||||
107
db/migrations/uniform_canonical_ids_test.go
Normal file
107
db/migrations/uniform_canonical_ids_test.go
Normal file
@ -0,0 +1,107 @@
|
||||
package migrations
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
|
||||
_ "github.com/mattn/go-sqlite3"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
var _ = Describe("upUniformCanonicalIds", func() {
|
||||
var db *sql.DB
|
||||
var tx *sql.Tx
|
||||
ctx := context.Background()
|
||||
|
||||
const (
|
||||
hashID = "5cLJPkLA5DK2BADhoeotPk" // canonical, kept
|
||||
randOld = "zzzzzzzzzzzzzzzzzzzzzz" // overflows -> remapped
|
||||
randNew = "3LyqmwQBm5IRqlVjNYASwb"
|
||||
legacyOld = "e3b7fc2ae9447bbec37a13bf916e3cf6" // 32-hex -> re-encoded
|
||||
legacyNew = "6VHl3uR4kss6sUPKA8Cwnk"
|
||||
uuidOld = "f47ac10b-58cc-4372-a567-0e02b2c3d479" // uuid -> re-encoded
|
||||
uuidNew = "7rke2SAWaicSeSYzkhww6R"
|
||||
shareID = "aB3xY9kQz1" // exempt family
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
var err error
|
||||
db, err = sql.Open("sqlite3", "file::memory:")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
db.SetMaxOpenConns(1) // non-shared :memory: — every new conn is a fresh empty DB
|
||||
DeferCleanup(func() { _ = db.Close() })
|
||||
|
||||
// Minimal fixture: every table/column idColumns and the list rewrites touch.
|
||||
_, err = db.Exec(`
|
||||
CREATE TABLE media_file (id text, pid text, artist_id text, album_id text, album_artist_id text, folder_id text, mbz_recording_id text);
|
||||
CREATE TABLE album (id text, album_artist_id text);
|
||||
CREATE TABLE artist (id text);
|
||||
CREATE TABLE folder (id text, parent_id text);
|
||||
CREATE TABLE tag (id text);
|
||||
CREATE TABLE library_artist (artist_id text);
|
||||
CREATE TABLE user (id text);
|
||||
CREATE TABLE user_props (user_id text);
|
||||
CREATE TABLE playlist (id text, owner_id text);
|
||||
CREATE TABLE playlist_tracks (playlist_id text, media_file_id text);
|
||||
CREATE TABLE playlist_fields (playlist_id text);
|
||||
CREATE TABLE annotation (user_id text, item_id text, item_type text);
|
||||
CREATE TABLE bookmark (user_id text, item_id text);
|
||||
CREATE TABLE player (id text, user_id text, transcoding_id text);
|
||||
CREATE TABLE transcoding (id text);
|
||||
CREATE TABLE radio (id text);
|
||||
CREATE TABLE share (id text, user_id text, resource_ids text, contents text);
|
||||
CREATE TABLE scrobble_buffer (id text, user_id text, media_file_id text);
|
||||
CREATE TABLE playqueue (id text, user_id text, items text);
|
||||
`)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
seed := func(query string, args ...any) {
|
||||
_, err := db.Exec(query, args...)
|
||||
ExpectWithOffset(1, err).ToNot(HaveOccurred())
|
||||
}
|
||||
// media_file: legacy id/pid, hash artist, legacy album, mbz uuid must stay untouched
|
||||
seed(`INSERT INTO media_file VALUES (?, ?, ?, ?, '', '', ?)`, legacyOld, legacyOld, hashID, legacyOld, uuidOld)
|
||||
seed(`INSERT INTO album VALUES (?, ?)`, legacyOld, hashID)
|
||||
seed(`INSERT INTO user VALUES (?)`, randOld)
|
||||
seed(`INSERT INTO playlist VALUES (?, ?)`, uuidOld, randOld) // uuid id, random owner
|
||||
seed(`INSERT INTO annotation VALUES (?, ?, 'media_file')`, randOld, legacyOld)
|
||||
seed(`INSERT INTO playqueue VALUES (?, ?, ?)`, randOld, randOld, legacyOld+","+hashID)
|
||||
seed(`INSERT INTO share VALUES (?, ?, ?, 'Album Foo...')`, shareID, randOld, legacyOld+","+uuidOld)
|
||||
|
||||
tx, err = db.Begin()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(upUniformCanonicalIds(ctx, tx)).To(Succeed())
|
||||
Expect(tx.Commit()).To(Succeed())
|
||||
})
|
||||
|
||||
get := func(query string) string {
|
||||
var s string
|
||||
ExpectWithOffset(1, db.QueryRow(query).Scan(&s)).To(Succeed())
|
||||
return s
|
||||
}
|
||||
|
||||
It("canonicalizes every id family and keeps references consistent", func() {
|
||||
Expect(get(`SELECT id FROM media_file`)).To(Equal(legacyNew))
|
||||
Expect(get(`SELECT pid FROM media_file`)).To(Equal(legacyNew))
|
||||
Expect(get(`SELECT artist_id FROM media_file`)).To(Equal(hashID))
|
||||
Expect(get(`SELECT album_id FROM media_file`)).To(Equal(legacyNew))
|
||||
Expect(get(`SELECT id FROM album`)).To(Equal(legacyNew))
|
||||
Expect(get(`SELECT id FROM user`)).To(Equal(randNew))
|
||||
Expect(get(`SELECT id FROM playlist`)).To(Equal(uuidNew))
|
||||
Expect(get(`SELECT owner_id FROM playlist`)).To(Equal(randNew))
|
||||
Expect(get(`SELECT item_id FROM annotation`)).To(Equal(legacyNew))
|
||||
Expect(get(`SELECT user_id FROM annotation`)).To(Equal(randNew))
|
||||
})
|
||||
|
||||
It("rewrites list columns element-wise", func() {
|
||||
Expect(get(`SELECT items FROM playqueue`)).To(Equal(legacyNew + "," + hashID))
|
||||
Expect(get(`SELECT resource_ids FROM share`)).To(Equal(legacyNew + "," + uuidNew))
|
||||
})
|
||||
|
||||
It("leaves exempt columns alone", func() {
|
||||
Expect(get(`SELECT id FROM share`)).To(Equal(shareID))
|
||||
Expect(get(`SELECT contents FROM share`)).To(Equal("Album Foo..."))
|
||||
Expect(get(`SELECT mbz_recording_id FROM media_file`)).To(Equal(uuidOld))
|
||||
})
|
||||
})
|
||||
Loading…
x
Reference in New Issue
Block a user