feat(artwork): artwork backfill, fingerprint re-resolution and scheduled jobs

cmd/root.go

@@ -11,6 +11,7 @@ import (
 	"github.com/go-chi/chi/v5/middleware"
 	"github.com/navidrome/navidrome/conf"
 	"github.com/navidrome/navidrome/consts"
+	"github.com/navidrome/navidrome/core/artwork"
 	"github.com/navidrome/navidrome/db"
 	"github.com/navidrome/navidrome/log"
 	"github.com/navidrome/navidrome/model"
@@ -88,7 +89,9 @@ func runNavidrome(ctx context.Context) {
 	g.Go(startInsightsCollector(ctx))
 	g.Go(scheduleDBAnalyzer(ctx))
 	g.Go(startPluginManager(ctx))
-	g.Go(startArtworkWorker(ctx))
+	artworkWorker := CreateArtworkWorker()
+	g.Go(startArtworkWorker(ctx, artworkWorker))
+	g.Go(scheduleArtworkHousekeeping(ctx, artworkWorker))
 	g.Go(runInitialScan(ctx))
 	if conf.Server.Scanner.Enabled {
 		g.Go(startScanWatcher(ctx))
@@ -347,10 +350,55 @@ func startPlaybackServer(ctx context.Context) func() error {
 
 // startArtworkWorker starts the background artwork acquisition worker. It always
 // runs; the queue is simply empty until something enqueues work into it.
-func startArtworkWorker(ctx context.Context) func() error {
+func startArtworkWorker(ctx context.Context, worker *artwork.Worker) func() error {
 	return func() error {
 		log.Info(ctx, "Starting artwork worker")
-		return CreateArtworkWorker().Run(ctx)
+		return worker.Run(ctx)
+	}
+}
+
+// scheduleArtworkHousekeeping runs the startup fingerprint backfill and registers the
+// recurring stale-absent recheck and prune jobs. Scan-triggered prune lands in a later phase.
+func scheduleArtworkHousekeeping(ctx context.Context, worker *artwork.Worker) func() error {
+	return func() error {
+		ds := CreateDataStore()
+		schedulerInstance := scheduler.GetInstance()
+
+		if _, err := schedulerInstance.Add(consts.ArtworkStaleAbsentRecheckSchedule, func() {
+			if err := artwork.EnqueueStaleAbsentAll(ctx, ds); err != nil {
+				log.Error(ctx, "Error enqueueing stale artwork rechecks", err)
+			}
+		}); err != nil {
+			log.Error(ctx, "Error scheduling artwork stale-absent recheck", err)
+		}
+
+		if _, err := schedulerInstance.Add(consts.ArtworkPruneSchedule, func() {
+			if err := worker.RunPrune(ctx); err != nil {
+				log.Error(ctx, "Error running artwork prune", err)
+			}
+		}); err != nil {
+			log.Error(ctx, "Error scheduling artwork prune", err)
+		}
+
+		backfilled, err := artwork.Backfill(ctx, ds)
+		if err != nil {
+			log.Error(ctx, "Error running artwork backfill", err)
+			return nil
+		}
+		if !backfilled {
+			return nil
+		}
+		log.Info(ctx, "Artwork backfill enqueued, scheduling a follow-up prune")
+		timer := time.NewTimer(consts.ArtworkPostBackfillPruneDelay)
+		defer timer.Stop()
+		select {
+		case <-timer.C:
+			if err := worker.RunPrune(ctx); err != nil {
+				log.Error(ctx, "Error running post-backfill artwork prune", err)
+			}
+		case <-ctx.Done():
+		}
+		return nil
 	}
 }
 

consts/consts.go

@@ -35,6 +35,10 @@ const (
 	DBAnalyzeCheckSchedule = "@every 30m"
 	DBAnalyzeMaxAge        = 24 * time.Hour
 
+	ArtworkStaleAbsentRecheckSchedule = "@every 1h"
+	ArtworkPruneSchedule              = "@daily"
+	ArtworkPostBackfillPruneDelay     = 10 * time.Minute
+
 	// DefaultEncryptionKey This is the encryption key used if none is specified in the `PasswordEncryptionKey` option
 	// Never ever change this! Or it will break all Navidrome installations that don't set the config option
 	DefaultEncryptionKey  = "just for obfuscation"

core/artwork/housekeeping.go

@@ -0,0 +1,120 @@
+package artwork
+
+import (
+	"context"
+	"crypto/md5"
+	"encoding/hex"
+	"fmt"
+	"time"
+
+	"github.com/navidrome/navidrome/conf"
+	"github.com/navidrome/navidrome/consts"
+	"github.com/navidrome/navidrome/log"
+	"github.com/navidrome/navidrome/model"
+)
+
+// FingerprintPropertyKey is the model.PropertyRepository key Backfill compares against
+// to detect artwork-affecting config changes across restarts.
+const FingerprintPropertyKey = "artwork.fingerprint"
+
+// staleAbsentAge is how old an absent resolution must be before the recheck job retries it.
+const staleAbsentAge = 24 * time.Hour
+
+// staleAbsentKinds are the item kinds eligible for the periodic stale-absent recheck.
+var staleAbsentKinds = []string{"ar", "al", "pl", "ra"}
+
+// Fingerprint summarizes the config knobs that affect artwork resolution outcomes; a
+// change means previously resolved (or absent) state may no longer be correct.
+func Fingerprint() string {
+	raw := fmt.Sprintf("%s|%s|%s|%t|%s",
+		conf.Server.CoverArtPriority, conf.Server.ArtistArtPriority,
+		conf.Server.Agents, conf.Server.EnableExternalServices, consts.Version)
+	sum := md5.Sum([]byte(raw)) //nolint:gosec // fingerprint, not security-sensitive
+	return hex.EncodeToString(sum[:])
+}
+
+// Backfill enqueues artwork resolution for every entity when the config fingerprint changed
+// (or was never stored), artists first so those pages resolve before the larger backlog.
+func Backfill(ctx context.Context, ds model.DataStore) (bool, error) {
+	current := Fingerprint()
+	props := ds.Property(ctx)
+	stored, err := props.DefaultGet(FingerprintPropertyKey, "")
+	if err != nil {
+		return false, err
+	}
+	if stored == current {
+		return false, nil
+	}
+
+	artists, err := ds.Artist(ctx).GetAll()
+	if err != nil {
+		return false, err
+	}
+	if err := enqueueBackfillKind(ctx, ds, "ar", idsOf(artists, func(a model.Artist) string { return a.ID })); err != nil {
+		return false, err
+	}
+
+	albums, err := ds.Album(ctx).GetAll()
+	if err != nil {
+		return false, err
+	}
+	if err := enqueueBackfillKind(ctx, ds, "al", idsOf(albums, func(a model.Album) string { return a.ID })); err != nil {
+		return false, err
+	}
+
+	playlists, err := ds.Playlist(ctx).GetAll()
+	if err != nil {
+		return false, err
+	}
+	if err := enqueueBackfillKind(ctx, ds, "pl", idsOf(playlists, func(p model.Playlist) string { return p.ID })); err != nil {
+		return false, err
+	}
+
+	radios, err := ds.Radio(ctx).GetAll()
+	if err != nil {
+		return false, err
+	}
+	if err := enqueueBackfillKind(ctx, ds, "ra", idsOf(radios, func(r model.Radio) string { return r.ID })); err != nil {
+		return false, err
+	}
+
+	if err := props.Put(FingerprintPropertyKey, current); err != nil {
+		return false, err
+	}
+	log.Info(ctx, "Artwork: config fingerprint changed, backfill enqueued")
+	return true, nil
+}
+
+func enqueueBackfillKind(ctx context.Context, ds model.DataStore, kind string, ids []string) error {
+	if len(ids) == 0 {
+		return nil
+	}
+	items := make([]model.ArtworkQueueItem, len(ids))
+	for i, id := range ids {
+		items[i] = model.ArtworkQueueItem{
+			ItemKind: kind, ItemID: id, ImageType: model.ImageTypePrimary, Priority: model.ArtworkPriorityBackfill,
+		}
+	}
+	return ds.ArtworkQueue(ctx).Enqueue(items...)
+}
+
+func idsOf[T any](items []T, id func(T) string) []string {
+	ids := make([]string, len(items))
+	for i, it := range items {
+		ids[i] = id(it)
+	}
+	return ids
+}
+
+// EnqueueStaleAbsentAll requeues absent-state entries older than staleAbsentAge, across
+// every artwork-bearing kind, for the periodic recheck job.
+func EnqueueStaleAbsentAll(ctx context.Context, ds model.DataStore) error {
+	cutoff := time.Now().Add(-staleAbsentAge)
+	queue := ds.ArtworkQueue(ctx)
+	for _, kind := range staleAbsentKinds {
+		if _, err := queue.EnqueueStaleAbsent(kind, cutoff); err != nil {
+			return err
+		}
+	}
+	return nil
+}

core/artwork/housekeeping_test.go

@@ -0,0 +1,174 @@
+package artwork
+
+import (
+	"context"
+	"time"
+
+	"github.com/navidrome/navidrome/conf"
+	"github.com/navidrome/navidrome/conf/configtest"
+	"github.com/navidrome/navidrome/model"
+	"github.com/navidrome/navidrome/tests"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+// orderTrackingQueueRepo records the item kind of each Enqueue call, so tests can
+// assert phase ordering (artists-first) that same-priority timestamps can't guarantee.
+type orderTrackingQueueRepo struct {
+	*tests.MockArtworkQueueRepo
+	callKinds []string
+}
+
+func (o *orderTrackingQueueRepo) Enqueue(items ...model.ArtworkQueueItem) error {
+	if len(items) > 0 {
+		o.callKinds = append(o.callKinds, items[0].ItemKind)
+	}
+	return o.MockArtworkQueueRepo.Enqueue(items...)
+}
+
+var _ = Describe("Housekeeping", func() {
+	var (
+		ctx       context.Context
+		ds        *tests.MockDataStore
+		queueRepo *orderTrackingQueueRepo
+		propRepo  *tests.MockedPropertyRepo
+	)
+
+	BeforeEach(func() {
+		DeferCleanup(configtest.SetupConfig())
+		ctx = context.Background()
+		conf.Server.CoverArtPriority = "embedded, folder"
+		conf.Server.ArtistArtPriority = "artist.jpg"
+		conf.Server.Agents = "spotify"
+		conf.Server.EnableExternalServices = true
+
+		queueRepo = &orderTrackingQueueRepo{MockArtworkQueueRepo: tests.CreateMockArtworkQueueRepo()}
+		propRepo = &tests.MockedPropertyRepo{}
+		ds = &tests.MockDataStore{MockedArtworkQueue: queueRepo, MockedProperty: propRepo}
+	})
+
+	seedEntities := func() {
+		artistRepo := tests.CreateMockArtistRepo()
+		artistRepo.SetData(model.Artists{{ID: "ar1"}, {ID: "ar2"}})
+		ds.MockedArtist = artistRepo
+
+		albumRepo := tests.CreateMockAlbumRepo()
+		albumRepo.SetData(model.Albums{{ID: "al1"}})
+		ds.MockedAlbum = albumRepo
+
+		playlistRepo := tests.CreateMockPlaylistRepo()
+		playlistRepo.SetData(model.Playlists{{ID: "pl1"}})
+		ds.MockedPlaylist = playlistRepo
+
+		radioRepo := tests.CreateMockedRadioRepo()
+		radioRepo.All = model.Radios{{ID: "ra1"}}
+		ds.MockedRadio = radioRepo
+	}
+
+	Describe("Fingerprint", func() {
+		It("changes when a fingerprint-affecting config value changes", func() {
+			f1 := Fingerprint()
+			conf.Server.CoverArtPriority = "folder, embedded"
+			f2 := Fingerprint()
+			Expect(f1).NotTo(Equal(f2))
+		})
+	})
+
+	Describe("Backfill", func() {
+		It("enqueues nothing and returns false when the stored fingerprint matches", func() {
+			seedEntities()
+			Expect(propRepo.Put(FingerprintPropertyKey, Fingerprint())).To(Succeed())
+
+			did, err := Backfill(ctx, ds)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(did).To(BeFalse())
+
+			count, err := queueRepo.Count()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(count).To(BeZero())
+		})
+
+		It("runs the backfill when no fingerprint was ever stored", func() {
+			seedEntities()
+
+			did, err := Backfill(ctx, ds)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(did).To(BeTrue())
+
+			count, err := queueRepo.Count()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(count).To(Equal(int64(5))) // 2 artists + 1 album + 1 playlist + 1 radio
+
+			stored, err := propRepo.Get(FingerprintPropertyKey)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(stored).To(Equal(Fingerprint()))
+		})
+
+		It("enqueues artists before albums/playlists/radios, all at Backfill priority", func() {
+			seedEntities()
+			Expect(propRepo.Put(FingerprintPropertyKey, "stale-fingerprint")).To(Succeed())
+
+			did, err := Backfill(ctx, ds)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(did).To(BeTrue())
+
+			Expect(queueRepo.callKinds).ToNot(BeEmpty())
+			artistCallIdx := -1
+			for i, k := range queueRepo.callKinds {
+				if k == "ar" {
+					artistCallIdx = i
+					break
+				}
+			}
+			Expect(artistCallIdx).To(Equal(0), "artists must be the first Enqueue call")
+			for i, k := range queueRepo.callKinds {
+				if k != "ar" {
+					Expect(i).To(BeNumerically(">", artistCallIdx))
+				}
+			}
+
+			for _, it := range queueRepo.Data {
+				Expect(it.Priority).To(Equal(model.ArtworkPriorityBackfill))
+				Expect(it.ItemKind).To(BeElementOf("ar", "al", "pl", "ra"))
+			}
+		})
+	})
+
+	Describe("EnqueueStaleAbsentAll", func() {
+		var artRepo *tests.MockArtworkRepo
+
+		BeforeEach(func() {
+			artRepo = tests.CreateMockArtworkRepo()
+			ds.MockedArtwork = artRepo
+			queueRepo.ItemArtworkSource = artRepo
+		})
+
+		It("enqueues only absent entries older than the recheck window, across all kinds", func() {
+			old := time.Now().Add(-48 * time.Hour)
+			recent := time.Now().Add(-time.Hour)
+
+			artRepo.ItemData["ar-stale"] = model.ItemArtwork{ItemKind: "ar", ItemID: "ar1", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: old}
+			artRepo.ItemData["al-stale"] = model.ItemArtwork{ItemKind: "al", ItemID: "al1", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: old}
+			artRepo.ItemData["pl-stale"] = model.ItemArtwork{ItemKind: "pl", ItemID: "pl1", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: old}
+			artRepo.ItemData["ra-stale"] = model.ItemArtwork{ItemKind: "ra", ItemID: "ra1", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: old}
+			// Not stale: too recent.
+			artRepo.ItemData["ar-recent"] = model.ItemArtwork{ItemKind: "ar", ItemID: "ar2", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: recent}
+			// Not absent: has a resolved hash.
+			artRepo.ItemData["al-resolved"] = model.ItemArtwork{ItemKind: "al", ItemID: "al2", ImageType: model.ImageTypePrimary, Hash: "somehash", AttemptedAt: old}
+
+			err := EnqueueStaleAbsentAll(ctx, ds)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(queueRepo.Data).To(HaveLen(4))
+			for _, it := range queueRepo.Data {
+				Expect(it.Priority).To(Equal(model.ArtworkPriorityRecheck))
+			}
+			Expect(findQueued(queueRepo.MockArtworkQueueRepo, "ar", "ar1")).ToNot(BeNil())
+			Expect(findQueued(queueRepo.MockArtworkQueueRepo, "al", "al1")).ToNot(BeNil())
+			Expect(findQueued(queueRepo.MockArtworkQueueRepo, "pl", "pl1")).ToNot(BeNil())
+			Expect(findQueued(queueRepo.MockArtworkQueueRepo, "ra", "ra1")).ToNot(BeNil())
+			Expect(findQueued(queueRepo.MockArtworkQueueRepo, "ar", "ar2")).To(BeNil())
+			Expect(findQueued(queueRepo.MockArtworkQueueRepo, "al", "al2")).To(BeNil())
+		})
+	})
+})