From af0a6615b293cad1526e53144aaa32df06801b89 Mon Sep 17 00:00:00 2001 From: Deluan Date: Mon, 23 Mar 2026 20:06:28 -0400 Subject: [PATCH] feat(scanner): evaluate smart playlists in background after import Wire SmartPlaylistEvaluator into the scanner pipeline. After importing a .nsp file, the scanner enqueues the playlist for background evaluation. The evaluator processes the queue asynchronously, populating tracks and updating song_count/duration/size without blocking the scan. Fixes #4539 --- cmd/wire_gen.go | 12 ++++++++---- core/wire_providers.go | 1 + scanner/controller.go | 8 +++++--- scanner/controller_test.go | 2 +- scanner/phase_4_playlists.go | 6 +++++- scanner/phase_4_playlists_test.go | 2 +- scanner/scanner.go | 3 ++- scanner/scanner_benchmark_test.go | 2 +- scanner/scanner_multilibrary_test.go | 2 +- scanner/scanner_selective_test.go | 2 +- scanner/scanner_test.go | 2 +- server/e2e/e2e_suite_test.go | 4 ++-- server/e2e/subsonic_multilibrary_test.go | 2 +- 13 files changed, 30 insertions(+), 18 deletions(-) diff --git a/cmd/wire_gen.go b/cmd/wire_gen.go index 5b9fd648f..94ecec945 100644 --- a/cmd/wire_gen.go +++ b/cmd/wire_gen.go @@ -75,7 +75,8 @@ func CreateNativeAPIRouter(ctx context.Context) *nativeapi.Router { provider := external.NewProvider(dataStore, agentsAgents) artworkArtwork := artwork.NewArtwork(dataStore, fileCache, fFmpeg, provider) cacheWarmer := artwork.NewCacheWarmer(artworkArtwork, fileCache) - modelScanner := scanner.New(ctx, dataStore, cacheWarmer, broker, playlistsPlaylists, metricsMetrics) + smartPlaylistEvaluator := playlists.NewSmartPlaylistEvaluator(dataStore) + modelScanner := scanner.New(ctx, dataStore, cacheWarmer, broker, playlistsPlaylists, smartPlaylistEvaluator, metricsMetrics) watcher := scanner.GetWatcher(dataStore, modelScanner) library := core.NewLibrary(dataStore, modelScanner, watcher, broker, manager) user := core.NewUser(dataStore, manager) @@ -103,7 +104,8 @@ func CreateSubsonicAPIRouter(ctx context.Context) *subsonic.Router { cacheWarmer := artwork.NewCacheWarmer(artworkArtwork, fileCache) imageUploadService := core.NewImageUploadService() playlistsPlaylists := playlists.NewPlaylists(dataStore, imageUploadService) - modelScanner := scanner.New(ctx, dataStore, cacheWarmer, broker, playlistsPlaylists, metricsMetrics) + smartPlaylistEvaluator := playlists.NewSmartPlaylistEvaluator(dataStore) + modelScanner := scanner.New(ctx, dataStore, cacheWarmer, broker, playlistsPlaylists, smartPlaylistEvaluator, metricsMetrics) playTracker := scrobbler.GetPlayTracker(dataStore, broker, manager) playbackServer := playback.GetInstance(dataStore) lyricsLyrics := lyrics.NewLyrics(manager) @@ -173,7 +175,8 @@ func CreateScanner(ctx context.Context) model.Scanner { cacheWarmer := artwork.NewCacheWarmer(artworkArtwork, fileCache) imageUploadService := core.NewImageUploadService() playlistsPlaylists := playlists.NewPlaylists(dataStore, imageUploadService) - modelScanner := scanner.New(ctx, dataStore, cacheWarmer, broker, playlistsPlaylists, metricsMetrics) + smartPlaylistEvaluator := playlists.NewSmartPlaylistEvaluator(dataStore) + modelScanner := scanner.New(ctx, dataStore, cacheWarmer, broker, playlistsPlaylists, smartPlaylistEvaluator, metricsMetrics) return modelScanner } @@ -191,7 +194,8 @@ func CreateScanWatcher(ctx context.Context) scanner.Watcher { cacheWarmer := artwork.NewCacheWarmer(artworkArtwork, fileCache) imageUploadService := core.NewImageUploadService() playlistsPlaylists := playlists.NewPlaylists(dataStore, imageUploadService) - modelScanner := scanner.New(ctx, dataStore, cacheWarmer, broker, playlistsPlaylists, metricsMetrics) + smartPlaylistEvaluator := playlists.NewSmartPlaylistEvaluator(dataStore) + modelScanner := scanner.New(ctx, dataStore, cacheWarmer, broker, playlistsPlaylists, smartPlaylistEvaluator, metricsMetrics) watcher := scanner.GetWatcher(dataStore, modelScanner) return watcher } diff --git a/core/wire_providers.go b/core/wire_providers.go index 276d9556a..14443de6a 100644 --- a/core/wire_providers.go +++ b/core/wire_providers.go @@ -20,6 +20,7 @@ var Set = wire.NewSet( NewPlayers, NewShare, playlists.NewPlaylists, + playlists.NewSmartPlaylistEvaluator, NewLibrary, NewUser, NewMaintenance, diff --git a/scanner/controller.go b/scanner/controller.go index 94248ffd0..69b020f6f 100644 --- a/scanner/controller.go +++ b/scanner/controller.go @@ -27,13 +27,14 @@ var ( ) func New(rootCtx context.Context, ds model.DataStore, cw artwork.CacheWarmer, broker events.Broker, - pls playlists.Playlists, m metrics.Metrics) model.Scanner { + pls playlists.Playlists, spe playlists.SmartPlaylistEvaluator, m metrics.Metrics) model.Scanner { c := &controller{ rootCtx: rootCtx, ds: ds, cw: cw, broker: broker, pls: pls, + spe: spe, metrics: m, devExternalScanner: conf.Server.DevExternalScanner, } @@ -47,7 +48,7 @@ func (s *controller) getScanner() scanner { if s.devExternalScanner { return &scannerExternal{} } - return &scannerImpl{ds: s.ds, cw: s.cw, pls: s.pls} + return &scannerImpl{ds: s.ds, cw: s.cw, pls: s.pls, spe: s.spe} } // CallScan starts an in-process scan of specific library/folder pairs. @@ -64,7 +65,7 @@ func CallScan(ctx context.Context, ds model.DataStore, pls playlists.Playlists, progress := make(chan *ProgressInfo, 100) go func() { defer close(progress) - scanner := &scannerImpl{ds: ds, cw: artwork.NoopCacheWarmer(), pls: pls} + scanner := &scannerImpl{ds: ds, cw: artwork.NoopCacheWarmer(), pls: pls, spe: playlists.NoopSmartPlaylistEvaluator()} scanner.scanFolders(ctx, fullScan, targets, progress) }() return progress, nil @@ -99,6 +100,7 @@ type controller struct { broker events.Broker metrics metrics.Metrics pls playlists.Playlists + spe playlists.SmartPlaylistEvaluator limiter *rate.Sometimes devExternalScanner bool count atomic.Uint32 diff --git a/scanner/controller_test.go b/scanner/controller_test.go index d60d432b4..ad71dc278 100644 --- a/scanner/controller_test.go +++ b/scanner/controller_test.go @@ -32,7 +32,7 @@ var _ = Describe("Controller", func() { DeferCleanup(configtest.SetupConfig()) ds = &tests.MockDataStore{RealDS: persistence.New(db.Db())} ds.MockedProperty = &tests.MockedPropertyRepo{} - ctrl = scanner.New(ctx, ds, artwork.NoopCacheWarmer(), events.NoopBroker(), playlists.NewPlaylists(ds, core.NewImageUploadService()), metrics.NewNoopInstance()) + ctrl = scanner.New(ctx, ds, artwork.NoopCacheWarmer(), events.NoopBroker(), playlists.NewPlaylists(ds, core.NewImageUploadService()), playlists.NoopSmartPlaylistEvaluator(), metrics.NewNoopInstance()) }) It("includes last scan error", func() { diff --git a/scanner/phase_4_playlists.go b/scanner/phase_4_playlists.go index ab5f77ae0..e3d5f16d8 100644 --- a/scanner/phase_4_playlists.go +++ b/scanner/phase_4_playlists.go @@ -23,16 +23,19 @@ type phasePlaylists struct { ds model.DataStore pls playlists.Playlists cw artwork.CacheWarmer + spe playlists.SmartPlaylistEvaluator refreshed atomic.Uint32 } -func createPhasePlaylists(ctx context.Context, scanState *scanState, ds model.DataStore, pls playlists.Playlists, cw artwork.CacheWarmer) *phasePlaylists { +func createPhasePlaylists(ctx context.Context, scanState *scanState, ds model.DataStore, + pls playlists.Playlists, cw artwork.CacheWarmer, spe playlists.SmartPlaylistEvaluator) *phasePlaylists { return &phasePlaylists{ ctx: ctx, scanState: scanState, ds: ds, pls: pls, cw: cw, + spe: spe, } } @@ -105,6 +108,7 @@ func (p *phasePlaylists) processPlaylistsInFolder(folder *model.Folder) (*model. continue } if pls.IsSmartPlaylist() { + p.spe.Enqueue(pls.ID) log.Debug("Scanner: Imported smart playlist", "name", pls.Name, "lastUpdated", pls.UpdatedAt, "path", pls.Path, "elapsed", time.Since(started)) } else { log.Debug("Scanner: Imported playlist", "name", pls.Name, "lastUpdated", pls.UpdatedAt, "path", pls.Path, "numTracks", len(pls.Tracks), "elapsed", time.Since(started)) diff --git a/scanner/phase_4_playlists_test.go b/scanner/phase_4_playlists_test.go index 0b50d39cb..923771c48 100644 --- a/scanner/phase_4_playlists_test.go +++ b/scanner/phase_4_playlists_test.go @@ -42,7 +42,7 @@ var _ = Describe("phasePlaylists", func() { pls = &mockPlaylists{} cw = artwork.NoopCacheWarmer() state = &scanState{} - phase = createPhasePlaylists(ctx, state, ds, pls, cw) + phase = createPhasePlaylists(ctx, state, ds, pls, cw, playlists.NoopSmartPlaylistEvaluator()) }) Describe("description", func() { diff --git a/scanner/scanner.go b/scanner/scanner.go index 871b0c696..b7675afc8 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -24,6 +24,7 @@ type scannerImpl struct { ds model.DataStore cw artwork.CacheWarmer pls playlists.Playlists + spe playlists.SmartPlaylistEvaluator } // scanState holds the state of an in-progress scan, to be passed to the various phases @@ -148,7 +149,7 @@ func (s *scannerImpl) scanFolders(ctx context.Context, fullScan bool, targets [] runPhase[*model.Album](ctx, 3, createPhaseRefreshAlbums(ctx, &state, s.ds)), // Phase 4: Import/update playlists - runPhase[*model.Folder](ctx, 4, createPhasePlaylists(ctx, &state, s.ds, s.pls, s.cw)), + runPhase[*model.Folder](ctx, 4, createPhasePlaylists(ctx, &state, s.ds, s.pls, s.cw, s.spe)), ), // Final Steps (cannot be parallelized): diff --git a/scanner/scanner_benchmark_test.go b/scanner/scanner_benchmark_test.go index 8f0dcd340..60b7ace13 100644 --- a/scanner/scanner_benchmark_test.go +++ b/scanner/scanner_benchmark_test.go @@ -41,7 +41,7 @@ func BenchmarkScan(b *testing.B) { ds := persistence.New(db.Db()) conf.Server.DevExternalScanner = false s := scanner.New(context.Background(), ds, artwork.NoopCacheWarmer(), events.NoopBroker(), - playlists.NewPlaylists(ds, core.NewImageUploadService()), metrics.NewNoopInstance()) + playlists.NewPlaylists(ds, core.NewImageUploadService()), playlists.NoopSmartPlaylistEvaluator(), metrics.NewNoopInstance()) fs := storagetest.FakeFS{} storagetest.Register("fake", &fs) diff --git a/scanner/scanner_multilibrary_test.go b/scanner/scanner_multilibrary_test.go index 856015239..ca7ed1e6c 100644 --- a/scanner/scanner_multilibrary_test.go +++ b/scanner/scanner_multilibrary_test.go @@ -78,7 +78,7 @@ var _ = Describe("Scanner - Multi-Library", Ordered, func() { Expect(ds.User(ctx).Put(&adminUser)).To(Succeed()) s = scanner.New(ctx, ds, artwork.NoopCacheWarmer(), events.NoopBroker(), - playlists.NewPlaylists(ds, core.NewImageUploadService()), metrics.NewNoopInstance()) + playlists.NewPlaylists(ds, core.NewImageUploadService()), playlists.NoopSmartPlaylistEvaluator(), metrics.NewNoopInstance()) // Create two test libraries (let DB auto-assign IDs) lib1 = model.Library{Name: "Rock Collection", Path: "rock:///music"} diff --git a/scanner/scanner_selective_test.go b/scanner/scanner_selective_test.go index 594b74e38..1b97b3376 100644 --- a/scanner/scanner_selective_test.go +++ b/scanner/scanner_selective_test.go @@ -64,7 +64,7 @@ var _ = Describe("ScanFolders", Ordered, func() { Expect(ds.User(ctx).Put(&adminUser)).To(Succeed()) s = scanner.New(ctx, ds, artwork.NoopCacheWarmer(), events.NoopBroker(), - playlists.NewPlaylists(ds, core.NewImageUploadService()), metrics.NewNoopInstance()) + playlists.NewPlaylists(ds, core.NewImageUploadService()), playlists.NoopSmartPlaylistEvaluator(), metrics.NewNoopInstance()) lib = model.Library{ID: 1, Name: "Fake Library", Path: "fake:///music"} Expect(ds.Library(ctx).Put(&lib)).To(Succeed()) diff --git a/scanner/scanner_test.go b/scanner/scanner_test.go index 922d21e62..24b545e44 100644 --- a/scanner/scanner_test.go +++ b/scanner/scanner_test.go @@ -85,7 +85,7 @@ var _ = Describe("Scanner", Ordered, func() { Expect(ds.User(ctx).Put(&adminUser)).To(Succeed()) s = scanner.New(ctx, ds, artwork.NoopCacheWarmer(), events.NoopBroker(), - playlists.NewPlaylists(ds, core.NewImageUploadService()), metrics.NewNoopInstance()) + playlists.NewPlaylists(ds, core.NewImageUploadService()), playlists.NoopSmartPlaylistEvaluator(), metrics.NewNoopInstance()) lib = model.Library{ID: 1, Name: "Fake Library", Path: "fake:///music"} Expect(ds.Library(ctx).Put(&lib)).To(Succeed()) diff --git a/server/e2e/e2e_suite_test.go b/server/e2e/e2e_suite_test.go index 262a5ed36..91981a9de 100644 --- a/server/e2e/e2e_suite_test.go +++ b/server/e2e/e2e_suite_test.go @@ -453,7 +453,7 @@ var _ = BeforeSuite(func() { buildTestFS() s := scanner.New(ctx, initDS, artwork.NoopCacheWarmer(), events.NoopBroker(), - playlists.NewPlaylists(initDS, core.NewImageUploadService()), metrics.NewNoopInstance()) + playlists.NewPlaylists(initDS, core.NewImageUploadService()), playlists.NoopSmartPlaylistEvaluator(), metrics.NewNoopInstance()) _, err = s.ScanAll(ctx, true) Expect(err).ToNot(HaveOccurred()) @@ -490,7 +490,7 @@ func setupTestDB() { streamerSpy = &spyStreamer{} decider := stream.NewTranscodeDecider(ds, noopFFmpeg{}) s := scanner.New(ctx, ds, artwork.NoopCacheWarmer(), events.NoopBroker(), - playlists.NewPlaylists(ds, core.NewImageUploadService()), metrics.NewNoopInstance()) + playlists.NewPlaylists(ds, core.NewImageUploadService()), playlists.NoopSmartPlaylistEvaluator(), metrics.NewNoopInstance()) router = subsonic.New( ds, noopArtwork{}, diff --git a/server/e2e/subsonic_multilibrary_test.go b/server/e2e/subsonic_multilibrary_test.go index a837da124..88f4d4cd3 100644 --- a/server/e2e/subsonic_multilibrary_test.go +++ b/server/e2e/subsonic_multilibrary_test.go @@ -54,7 +54,7 @@ var _ = Describe("Multi-Library Support", Ordered, func() { // Run incremental scan to import lib2 content (lib1 files unchanged → skipped) s := scanner.New(ctx, ds, artwork.NoopCacheWarmer(), events.NoopBroker(), - playlists.NewPlaylists(ds, core.NewImageUploadService()), metrics.NewNoopInstance()) + playlists.NewPlaylists(ds, core.NewImageUploadService()), playlists.NoopSmartPlaylistEvaluator(), metrics.NewNoopInstance()) _, err = s.ScanAll(ctx, false) Expect(err).ToNot(HaveOccurred())