mirror of
https://github.com/navidrome/navidrome.git
synced 2026-08-01 07:21:17 +00:00
9ce51cf575
Backfill enumerated every album, artist, playlist and radio via GetAll and mapped out just the ID. GetAll materializes full entities (library joins, participant/stats/tags JSON, annotation, artwork hydration), so on a large library it loaded tens of thousands of heavy structs only to read one field each — spiking transient RSS to ~1GB during the one-time upgrade backfill, a memory risk on small NAS/Pi hardware. Add GetAllIDs to the album, artist, playlist and radio repositories: it reuses each repo's base row-set filter (library visibility, artist content join, playlist userFilter) but projects only id, skipping the heavy columns and post-processing. A per-repo parity test asserts GetAllIDs returns exactly the same id set as GetAll. Verified on a 727MB / 29k-artist production DB copy: peak RSS during backfill dropped from ~1012MB to ~89MB, file descriptors flat, same 36,138 items enqueued.
103 lines
3.4 KiB
Go
103 lines
3.4 KiB
Go
package artwork
|
|
|
|
import (
|
|
"context"
|
|
"crypto/md5"
|
|
"encoding/hex"
|
|
"fmt"
|
|
"time"
|
|
|
|
"github.com/navidrome/navidrome/conf"
|
|
"github.com/navidrome/navidrome/consts"
|
|
"github.com/navidrome/navidrome/core/auth"
|
|
"github.com/navidrome/navidrome/log"
|
|
"github.com/navidrome/navidrome/model"
|
|
)
|
|
|
|
// FingerprintPropertyKey is the model.PropertyRepository key Backfill compares against
|
|
// to detect artwork-affecting config changes across restarts.
|
|
const FingerprintPropertyKey = "artwork.fingerprint"
|
|
|
|
// staleAbsentAge is how old an absent resolution must be before the recheck job retries it.
|
|
const staleAbsentAge = 24 * time.Hour
|
|
|
|
// staleAbsentKinds are the item kinds eligible for the periodic stale-absent recheck.
|
|
var staleAbsentKinds = []string{"ar", "al", "pl", "ra"}
|
|
|
|
// Fingerprint summarizes the config knobs that affect artwork resolution outcomes; a
|
|
// change means previously resolved (or absent) state may no longer be correct.
|
|
func Fingerprint() string {
|
|
raw := fmt.Sprintf("%s|%s|%s|%s|%t|%t|%s",
|
|
conf.Server.CoverArtPriority, conf.Server.ArtistArtPriority, conf.Server.ArtistImageFolder,
|
|
conf.Server.Agents, conf.Server.EnableExternalServices, conf.Server.EnableM3UExternalAlbumArt, consts.Version)
|
|
sum := md5.Sum([]byte(raw)) //nolint:gosec // fingerprint, not security-sensitive
|
|
return hex.EncodeToString(sum[:])
|
|
}
|
|
|
|
// Backfill enqueues artwork resolution for every entity when the config fingerprint changed
|
|
// (or was never stored), artists first so those pages resolve before the larger backlog.
|
|
func Backfill(ctx context.Context, ds model.DataStore) (bool, error) {
|
|
ctx = auth.WithAdminUser(ctx, ds)
|
|
current := Fingerprint()
|
|
props := ds.Property(ctx)
|
|
stored, err := props.DefaultGet(FingerprintPropertyKey, "")
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
if stored == current {
|
|
return false, nil
|
|
}
|
|
|
|
// Artists first: few entities, most external-dependent, so they get queue headstart.
|
|
kinds := []struct {
|
|
kind string
|
|
fetch func() ([]string, error)
|
|
}{
|
|
{"ar", func() ([]string, error) { return ds.Artist(ctx).GetAllIDs() }},
|
|
{"al", func() ([]string, error) { return ds.Album(ctx).GetAllIDs() }},
|
|
{"pl", func() ([]string, error) { return ds.Playlist(ctx).GetAllIDs() }},
|
|
{"ra", func() ([]string, error) { return ds.Radio(ctx).GetAllIDs() }},
|
|
}
|
|
for _, k := range kinds {
|
|
ids, err := k.fetch()
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
if err := enqueueBackfillKind(ctx, ds, k.kind, ids); err != nil {
|
|
return false, err
|
|
}
|
|
}
|
|
|
|
if err := props.Put(FingerprintPropertyKey, current); err != nil {
|
|
return false, err
|
|
}
|
|
log.Info(ctx, "Artwork: config fingerprint changed, backfill enqueued")
|
|
return true, nil
|
|
}
|
|
|
|
func enqueueBackfillKind(ctx context.Context, ds model.DataStore, kind string, ids []string) error {
|
|
if len(ids) == 0 {
|
|
return nil
|
|
}
|
|
items := make([]model.ArtworkQueueItem, len(ids))
|
|
for i, id := range ids {
|
|
items[i] = model.ArtworkQueueItem{
|
|
ItemKind: kind, ItemID: id, ImageType: model.ImageTypePrimary, Priority: model.ArtworkPriorityBackfill,
|
|
}
|
|
}
|
|
return ds.ArtworkQueue(ctx).Enqueue(items...)
|
|
}
|
|
|
|
// EnqueueStaleAbsentAll requeues absent-state entries older than staleAbsentAge, across
|
|
// every artwork-bearing kind, for the periodic recheck job.
|
|
func EnqueueStaleAbsentAll(ctx context.Context, ds model.DataStore) error {
|
|
cutoff := time.Now().Add(-staleAbsentAge)
|
|
queue := ds.ArtworkQueue(ctx)
|
|
for _, kind := range staleAbsentKinds {
|
|
if _, err := queue.EnqueueStaleAbsent(kind, cutoff); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|