From ad38cd1d584e83e347372cbf3a98ae3068b12a65 Mon Sep 17 00:00:00 2001 From: Deluan Date: Wed, 22 Jul 2026 10:40:34 -0400 Subject: [PATCH] feat(artwork): artwork backfill, fingerprint re-resolution and scheduled jobs --- cmd/root.go | 54 +++++++++- consts/consts.go | 4 + core/artwork/housekeeping.go | 120 +++++++++++++++++++++ core/artwork/housekeeping_test.go | 174 ++++++++++++++++++++++++++++++ 4 files changed, 349 insertions(+), 3 deletions(-) create mode 100644 core/artwork/housekeeping.go create mode 100644 core/artwork/housekeeping_test.go diff --git a/cmd/root.go b/cmd/root.go index 2b64ab850..4df5c1cb8 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -11,6 +11,7 @@ import ( "github.com/go-chi/chi/v5/middleware" "github.com/navidrome/navidrome/conf" "github.com/navidrome/navidrome/consts" + "github.com/navidrome/navidrome/core/artwork" "github.com/navidrome/navidrome/db" "github.com/navidrome/navidrome/log" "github.com/navidrome/navidrome/model" @@ -88,7 +89,9 @@ func runNavidrome(ctx context.Context) { g.Go(startInsightsCollector(ctx)) g.Go(scheduleDBAnalyzer(ctx)) g.Go(startPluginManager(ctx)) - g.Go(startArtworkWorker(ctx)) + artworkWorker := CreateArtworkWorker() + g.Go(startArtworkWorker(ctx, artworkWorker)) + g.Go(scheduleArtworkHousekeeping(ctx, artworkWorker)) g.Go(runInitialScan(ctx)) if conf.Server.Scanner.Enabled { g.Go(startScanWatcher(ctx)) @@ -347,10 +350,55 @@ func startPlaybackServer(ctx context.Context) func() error { // startArtworkWorker starts the background artwork acquisition worker. It always // runs; the queue is simply empty until something enqueues work into it. -func startArtworkWorker(ctx context.Context) func() error { +func startArtworkWorker(ctx context.Context, worker *artwork.Worker) func() error { return func() error { log.Info(ctx, "Starting artwork worker") - return CreateArtworkWorker().Run(ctx) + return worker.Run(ctx) + } +} + +// scheduleArtworkHousekeeping runs the startup fingerprint backfill and registers the +// recurring stale-absent recheck and prune jobs. Scan-triggered prune lands in a later phase. +func scheduleArtworkHousekeeping(ctx context.Context, worker *artwork.Worker) func() error { + return func() error { + ds := CreateDataStore() + schedulerInstance := scheduler.GetInstance() + + if _, err := schedulerInstance.Add(consts.ArtworkStaleAbsentRecheckSchedule, func() { + if err := artwork.EnqueueStaleAbsentAll(ctx, ds); err != nil { + log.Error(ctx, "Error enqueueing stale artwork rechecks", err) + } + }); err != nil { + log.Error(ctx, "Error scheduling artwork stale-absent recheck", err) + } + + if _, err := schedulerInstance.Add(consts.ArtworkPruneSchedule, func() { + if err := worker.RunPrune(ctx); err != nil { + log.Error(ctx, "Error running artwork prune", err) + } + }); err != nil { + log.Error(ctx, "Error scheduling artwork prune", err) + } + + backfilled, err := artwork.Backfill(ctx, ds) + if err != nil { + log.Error(ctx, "Error running artwork backfill", err) + return nil + } + if !backfilled { + return nil + } + log.Info(ctx, "Artwork backfill enqueued, scheduling a follow-up prune") + timer := time.NewTimer(consts.ArtworkPostBackfillPruneDelay) + defer timer.Stop() + select { + case <-timer.C: + if err := worker.RunPrune(ctx); err != nil { + log.Error(ctx, "Error running post-backfill artwork prune", err) + } + case <-ctx.Done(): + } + return nil } } diff --git a/consts/consts.go b/consts/consts.go index f453ac125..0a244f55f 100644 --- a/consts/consts.go +++ b/consts/consts.go @@ -35,6 +35,10 @@ const ( DBAnalyzeCheckSchedule = "@every 30m" DBAnalyzeMaxAge = 24 * time.Hour + ArtworkStaleAbsentRecheckSchedule = "@every 1h" + ArtworkPruneSchedule = "@daily" + ArtworkPostBackfillPruneDelay = 10 * time.Minute + // DefaultEncryptionKey This is the encryption key used if none is specified in the `PasswordEncryptionKey` option // Never ever change this! Or it will break all Navidrome installations that don't set the config option DefaultEncryptionKey = "just for obfuscation" diff --git a/core/artwork/housekeeping.go b/core/artwork/housekeeping.go new file mode 100644 index 000000000..6f1fa20a3 --- /dev/null +++ b/core/artwork/housekeeping.go @@ -0,0 +1,120 @@ +package artwork + +import ( + "context" + "crypto/md5" + "encoding/hex" + "fmt" + "time" + + "github.com/navidrome/navidrome/conf" + "github.com/navidrome/navidrome/consts" + "github.com/navidrome/navidrome/log" + "github.com/navidrome/navidrome/model" +) + +// FingerprintPropertyKey is the model.PropertyRepository key Backfill compares against +// to detect artwork-affecting config changes across restarts. +const FingerprintPropertyKey = "artwork.fingerprint" + +// staleAbsentAge is how old an absent resolution must be before the recheck job retries it. +const staleAbsentAge = 24 * time.Hour + +// staleAbsentKinds are the item kinds eligible for the periodic stale-absent recheck. +var staleAbsentKinds = []string{"ar", "al", "pl", "ra"} + +// Fingerprint summarizes the config knobs that affect artwork resolution outcomes; a +// change means previously resolved (or absent) state may no longer be correct. +func Fingerprint() string { + raw := fmt.Sprintf("%s|%s|%s|%t|%s", + conf.Server.CoverArtPriority, conf.Server.ArtistArtPriority, + conf.Server.Agents, conf.Server.EnableExternalServices, consts.Version) + sum := md5.Sum([]byte(raw)) //nolint:gosec // fingerprint, not security-sensitive + return hex.EncodeToString(sum[:]) +} + +// Backfill enqueues artwork resolution for every entity when the config fingerprint changed +// (or was never stored), artists first so those pages resolve before the larger backlog. +func Backfill(ctx context.Context, ds model.DataStore) (bool, error) { + current := Fingerprint() + props := ds.Property(ctx) + stored, err := props.DefaultGet(FingerprintPropertyKey, "") + if err != nil { + return false, err + } + if stored == current { + return false, nil + } + + artists, err := ds.Artist(ctx).GetAll() + if err != nil { + return false, err + } + if err := enqueueBackfillKind(ctx, ds, "ar", idsOf(artists, func(a model.Artist) string { return a.ID })); err != nil { + return false, err + } + + albums, err := ds.Album(ctx).GetAll() + if err != nil { + return false, err + } + if err := enqueueBackfillKind(ctx, ds, "al", idsOf(albums, func(a model.Album) string { return a.ID })); err != nil { + return false, err + } + + playlists, err := ds.Playlist(ctx).GetAll() + if err != nil { + return false, err + } + if err := enqueueBackfillKind(ctx, ds, "pl", idsOf(playlists, func(p model.Playlist) string { return p.ID })); err != nil { + return false, err + } + + radios, err := ds.Radio(ctx).GetAll() + if err != nil { + return false, err + } + if err := enqueueBackfillKind(ctx, ds, "ra", idsOf(radios, func(r model.Radio) string { return r.ID })); err != nil { + return false, err + } + + if err := props.Put(FingerprintPropertyKey, current); err != nil { + return false, err + } + log.Info(ctx, "Artwork: config fingerprint changed, backfill enqueued") + return true, nil +} + +func enqueueBackfillKind(ctx context.Context, ds model.DataStore, kind string, ids []string) error { + if len(ids) == 0 { + return nil + } + items := make([]model.ArtworkQueueItem, len(ids)) + for i, id := range ids { + items[i] = model.ArtworkQueueItem{ + ItemKind: kind, ItemID: id, ImageType: model.ImageTypePrimary, Priority: model.ArtworkPriorityBackfill, + } + } + return ds.ArtworkQueue(ctx).Enqueue(items...) +} + +func idsOf[T any](items []T, id func(T) string) []string { + ids := make([]string, len(items)) + for i, it := range items { + ids[i] = id(it) + } + return ids +} + +// EnqueueStaleAbsentAll requeues absent-state entries older than staleAbsentAge, across +// every artwork-bearing kind, for the periodic recheck job. +func EnqueueStaleAbsentAll(ctx context.Context, ds model.DataStore) error { + cutoff := time.Now().Add(-staleAbsentAge) + queue := ds.ArtworkQueue(ctx) + for _, kind := range staleAbsentKinds { + if _, err := queue.EnqueueStaleAbsent(kind, cutoff); err != nil { + return err + } + } + return nil +} diff --git a/core/artwork/housekeeping_test.go b/core/artwork/housekeeping_test.go new file mode 100644 index 000000000..8272514b0 --- /dev/null +++ b/core/artwork/housekeeping_test.go @@ -0,0 +1,174 @@ +package artwork + +import ( + "context" + "time" + + "github.com/navidrome/navidrome/conf" + "github.com/navidrome/navidrome/conf/configtest" + "github.com/navidrome/navidrome/model" + "github.com/navidrome/navidrome/tests" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// orderTrackingQueueRepo records the item kind of each Enqueue call, so tests can +// assert phase ordering (artists-first) that same-priority timestamps can't guarantee. +type orderTrackingQueueRepo struct { + *tests.MockArtworkQueueRepo + callKinds []string +} + +func (o *orderTrackingQueueRepo) Enqueue(items ...model.ArtworkQueueItem) error { + if len(items) > 0 { + o.callKinds = append(o.callKinds, items[0].ItemKind) + } + return o.MockArtworkQueueRepo.Enqueue(items...) +} + +var _ = Describe("Housekeeping", func() { + var ( + ctx context.Context + ds *tests.MockDataStore + queueRepo *orderTrackingQueueRepo + propRepo *tests.MockedPropertyRepo + ) + + BeforeEach(func() { + DeferCleanup(configtest.SetupConfig()) + ctx = context.Background() + conf.Server.CoverArtPriority = "embedded, folder" + conf.Server.ArtistArtPriority = "artist.jpg" + conf.Server.Agents = "spotify" + conf.Server.EnableExternalServices = true + + queueRepo = &orderTrackingQueueRepo{MockArtworkQueueRepo: tests.CreateMockArtworkQueueRepo()} + propRepo = &tests.MockedPropertyRepo{} + ds = &tests.MockDataStore{MockedArtworkQueue: queueRepo, MockedProperty: propRepo} + }) + + seedEntities := func() { + artistRepo := tests.CreateMockArtistRepo() + artistRepo.SetData(model.Artists{{ID: "ar1"}, {ID: "ar2"}}) + ds.MockedArtist = artistRepo + + albumRepo := tests.CreateMockAlbumRepo() + albumRepo.SetData(model.Albums{{ID: "al1"}}) + ds.MockedAlbum = albumRepo + + playlistRepo := tests.CreateMockPlaylistRepo() + playlistRepo.SetData(model.Playlists{{ID: "pl1"}}) + ds.MockedPlaylist = playlistRepo + + radioRepo := tests.CreateMockedRadioRepo() + radioRepo.All = model.Radios{{ID: "ra1"}} + ds.MockedRadio = radioRepo + } + + Describe("Fingerprint", func() { + It("changes when a fingerprint-affecting config value changes", func() { + f1 := Fingerprint() + conf.Server.CoverArtPriority = "folder, embedded" + f2 := Fingerprint() + Expect(f1).NotTo(Equal(f2)) + }) + }) + + Describe("Backfill", func() { + It("enqueues nothing and returns false when the stored fingerprint matches", func() { + seedEntities() + Expect(propRepo.Put(FingerprintPropertyKey, Fingerprint())).To(Succeed()) + + did, err := Backfill(ctx, ds) + Expect(err).ToNot(HaveOccurred()) + Expect(did).To(BeFalse()) + + count, err := queueRepo.Count() + Expect(err).ToNot(HaveOccurred()) + Expect(count).To(BeZero()) + }) + + It("runs the backfill when no fingerprint was ever stored", func() { + seedEntities() + + did, err := Backfill(ctx, ds) + Expect(err).ToNot(HaveOccurred()) + Expect(did).To(BeTrue()) + + count, err := queueRepo.Count() + Expect(err).ToNot(HaveOccurred()) + Expect(count).To(Equal(int64(5))) // 2 artists + 1 album + 1 playlist + 1 radio + + stored, err := propRepo.Get(FingerprintPropertyKey) + Expect(err).ToNot(HaveOccurred()) + Expect(stored).To(Equal(Fingerprint())) + }) + + It("enqueues artists before albums/playlists/radios, all at Backfill priority", func() { + seedEntities() + Expect(propRepo.Put(FingerprintPropertyKey, "stale-fingerprint")).To(Succeed()) + + did, err := Backfill(ctx, ds) + Expect(err).ToNot(HaveOccurred()) + Expect(did).To(BeTrue()) + + Expect(queueRepo.callKinds).ToNot(BeEmpty()) + artistCallIdx := -1 + for i, k := range queueRepo.callKinds { + if k == "ar" { + artistCallIdx = i + break + } + } + Expect(artistCallIdx).To(Equal(0), "artists must be the first Enqueue call") + for i, k := range queueRepo.callKinds { + if k != "ar" { + Expect(i).To(BeNumerically(">", artistCallIdx)) + } + } + + for _, it := range queueRepo.Data { + Expect(it.Priority).To(Equal(model.ArtworkPriorityBackfill)) + Expect(it.ItemKind).To(BeElementOf("ar", "al", "pl", "ra")) + } + }) + }) + + Describe("EnqueueStaleAbsentAll", func() { + var artRepo *tests.MockArtworkRepo + + BeforeEach(func() { + artRepo = tests.CreateMockArtworkRepo() + ds.MockedArtwork = artRepo + queueRepo.ItemArtworkSource = artRepo + }) + + It("enqueues only absent entries older than the recheck window, across all kinds", func() { + old := time.Now().Add(-48 * time.Hour) + recent := time.Now().Add(-time.Hour) + + artRepo.ItemData["ar-stale"] = model.ItemArtwork{ItemKind: "ar", ItemID: "ar1", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: old} + artRepo.ItemData["al-stale"] = model.ItemArtwork{ItemKind: "al", ItemID: "al1", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: old} + artRepo.ItemData["pl-stale"] = model.ItemArtwork{ItemKind: "pl", ItemID: "pl1", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: old} + artRepo.ItemData["ra-stale"] = model.ItemArtwork{ItemKind: "ra", ItemID: "ra1", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: old} + // Not stale: too recent. + artRepo.ItemData["ar-recent"] = model.ItemArtwork{ItemKind: "ar", ItemID: "ar2", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: recent} + // Not absent: has a resolved hash. + artRepo.ItemData["al-resolved"] = model.ItemArtwork{ItemKind: "al", ItemID: "al2", ImageType: model.ImageTypePrimary, Hash: "somehash", AttemptedAt: old} + + err := EnqueueStaleAbsentAll(ctx, ds) + Expect(err).ToNot(HaveOccurred()) + + Expect(queueRepo.Data).To(HaveLen(4)) + for _, it := range queueRepo.Data { + Expect(it.Priority).To(Equal(model.ArtworkPriorityRecheck)) + } + Expect(findQueued(queueRepo.MockArtworkQueueRepo, "ar", "ar1")).ToNot(BeNil()) + Expect(findQueued(queueRepo.MockArtworkQueueRepo, "al", "al1")).ToNot(BeNil()) + Expect(findQueued(queueRepo.MockArtworkQueueRepo, "pl", "pl1")).ToNot(BeNil()) + Expect(findQueued(queueRepo.MockArtworkQueueRepo, "ra", "ra1")).ToNot(BeNil()) + Expect(findQueued(queueRepo.MockArtworkQueueRepo, "ar", "ar2")).To(BeNil()) + Expect(findQueued(queueRepo.MockArtworkQueueRepo, "al", "al2")).To(BeNil()) + }) + }) +})