feat(scanner): add library stats to DB (#4229)

* Combine library stats migrations

* test: verify full library stats

* Fix total_songs calculation

* Fix library stats migration

* fix(scanner): log elapsed time and number of libraries updated during scan

Signed-off-by: Deluan <deluan@navidrome.org>

* fix(scanner): refresh library stats conditionally, only if changes were detected

Signed-off-by: Deluan <deluan@navidrome.org>

* fix(scanner): refresh library stats conditionally, only if changes were detected

Signed-off-by: Deluan <deluan@navidrome.org>

* fix(scanner): update queries to exclude missing entries in library stats

Signed-off-by: Deluan <deluan@navidrome.org>

---------

Signed-off-by: Deluan <deluan@navidrome.org>

db/migrations/20250701010103_add_library_stats.go

@@ -0,0 +1,48 @@
+package migrations
+
+import (
+	"context"
+	"database/sql"
+
+	"github.com/pressly/goose/v3"
+)
+
+func init() {
+	goose.AddMigrationContext(upAddLibraryStats, downAddLibraryStats)
+}
+
+func upAddLibraryStats(ctx context.Context, tx *sql.Tx) error {
+	_, err := tx.ExecContext(ctx, `
+alter table library add column total_songs integer default 0 not null;
+alter table library add column total_albums integer default 0 not null;
+alter table library add column total_artists integer default 0 not null;
+alter table library add column total_folders integer default 0 not null;
+    alter table library add column total_files integer default 0 not null;
+    alter table library add column total_missing_files integer default 0 not null;
+    alter table library add column total_size integer default 0 not null;
+update library set
+    total_songs = (
+        select count(*) from media_file where library_id = library.id and missing = 0
+    ),
+    total_albums = (select count(*) from album where library_id = library.id and missing = 0),
+    total_artists = (
+        select count(*) from library_artist la 
+        join artist a on la.artist_id = a.id 
+        where la.library_id = library.id and a.missing = 0
+    ),
+    total_folders = (select count(*) from folder where library_id = library.id and missing = 0),
+    total_files = (
+        select ifnull(sum(num_audio_files + num_playlists + json_array_length(image_files)),0)
+        from folder where library_id = library.id and missing = 0
+    ),
+    total_missing_files = (
+        select count(*) from media_file where library_id = library.id and missing = 1
+    ),
+    total_size = (select ifnull(sum(size),0) from album where library_id = library.id and missing = 0);
+`)
+	return err
+}
+
+func downAddLibraryStats(ctx context.Context, tx *sql.Tx) error {
+	return nil
+}

model/library.go

@@ -14,6 +14,14 @@ type Library struct {
 	FullScanInProgress bool
 	UpdatedAt          time.Time
 	CreatedAt          time.Time
+
+	TotalSongs        int
+	TotalAlbums       int
+	TotalArtists      int
+	TotalFolders      int
+	TotalFiles        int
+	TotalMissingFiles int
+	TotalSize         int64
 }
 
 type Libraries []Library
@@ -32,4 +40,5 @@ type LibraryRepository interface {
 	ScanBegin(id int, fullScan bool) error
 	ScanEnd(id int) error
 	ScanInProgress() (bool, error)
+	RefreshStats(id int) error
 }

persistence/library_repository.go

@@ -9,6 +9,7 @@ import (
 	"github.com/navidrome/navidrome/conf"
 	"github.com/navidrome/navidrome/log"
 	"github.com/navidrome/navidrome/model"
+	"github.com/navidrome/navidrome/utils/chain"
 	"github.com/pocketbase/dbx"
 )
 
@@ -146,6 +147,53 @@ func (r *libraryRepository) ScanInProgress() (bool, error) {
 	return count > 0, err
 }
 
+func (r *libraryRepository) RefreshStats(id int) error {
+	var songsRes, albumsRes, artistsRes, foldersRes, filesRes, missingRes struct{ Count int64 }
+	var sizeRes struct{ Sum int64 }
+
+	err := chain.RunParallel(
+		func() error {
+			return r.queryOne(Select("count(*) as count").From("media_file").Where(Eq{"library_id": id, "missing": false}), &songsRes)
+		},
+		func() error {
+			return r.queryOne(Select("count(*) as count").From("album").Where(Eq{"library_id": id, "missing": false}), &albumsRes)
+		},
+		func() error {
+			return r.queryOne(Select("count(*) as count").From("library_artist la").
+				Join("artist a on la.artist_id = a.id").
+				Where(Eq{"la.library_id": id, "a.missing": false}), &artistsRes)
+		},
+		func() error {
+			return r.queryOne(Select("count(*) as count").From("folder").Where(Eq{"library_id": id, "missing": false}), &foldersRes)
+		},
+		func() error {
+			return r.queryOne(Select("ifnull(sum(num_audio_files + num_playlists + json_array_length(image_files)),0) as count").From("folder").Where(Eq{"library_id": id, "missing": false}), &filesRes)
+		},
+		func() error {
+			return r.queryOne(Select("count(*) as count").From("media_file").Where(Eq{"library_id": id, "missing": true}), &missingRes)
+		},
+		func() error {
+			return r.queryOne(Select("ifnull(sum(size),0) as sum").From("album").Where(Eq{"library_id": id, "missing": false}), &sizeRes)
+		},
+	)()
+	if err != nil {
+		return err
+	}
+
+	sq := Update(r.tableName).
+		Set("total_songs", songsRes.Count).
+		Set("total_albums", albumsRes.Count).
+		Set("total_artists", artistsRes.Count).
+		Set("total_folders", foldersRes.Count).
+		Set("total_files", filesRes.Count).
+		Set("total_missing_files", missingRes.Count).
+		Set("total_size", sizeRes.Sum).
+		Set("updated_at", time.Now()).
+		Where(Eq{"id": id})
+	_, err = r.executeSQL(sq)
+	return err
+}
+
 func (r *libraryRepository) GetAll(ops ...model.QueryOptions) (model.Libraries, error) {
 	sq := r.newSelect(ops...).Columns("*")
 	res := model.Libraries{}

persistence/library_repository_test.go

@@ -0,0 +1,52 @@
+package persistence
+
+import (
+	"context"
+
+	"github.com/navidrome/navidrome/log"
+	"github.com/navidrome/navidrome/model"
+	"github.com/navidrome/navidrome/model/request"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"github.com/pocketbase/dbx"
+)
+
+var _ = Describe("LibraryRepository", func() {
+	var repo model.LibraryRepository
+	var ctx context.Context
+	var conn *dbx.DB
+
+	BeforeEach(func() {
+		ctx = request.WithUser(log.NewContext(context.TODO()), model.User{ID: "userid"})
+		conn = GetDBXBuilder()
+		repo = NewLibraryRepository(ctx, conn)
+	})
+
+	It("refreshes stats", func() {
+		libBefore, err := repo.Get(1)
+		Expect(err).ToNot(HaveOccurred())
+		Expect(repo.RefreshStats(1)).To(Succeed())
+		libAfter, err := repo.Get(1)
+		Expect(err).ToNot(HaveOccurred())
+		Expect(libAfter.UpdatedAt).To(BeTemporally(">", libBefore.UpdatedAt))
+
+		var songsRes, albumsRes, artistsRes, foldersRes, filesRes, missingRes struct{ Count int64 }
+		var sizeRes struct{ Sum int64 }
+
+		Expect(conn.NewQuery("select count(*) as count from media_file where library_id = {:id} and missing = 0").Bind(dbx.Params{"id": 1}).One(&songsRes)).To(Succeed())
+		Expect(conn.NewQuery("select count(*) as count from album where library_id = {:id} and missing = 0").Bind(dbx.Params{"id": 1}).One(&albumsRes)).To(Succeed())
+		Expect(conn.NewQuery("select count(*) as count from library_artist la join artist a on la.artist_id = a.id where la.library_id = {:id} and a.missing = 0").Bind(dbx.Params{"id": 1}).One(&artistsRes)).To(Succeed())
+		Expect(conn.NewQuery("select count(*) as count from folder where library_id = {:id} and missing = 0").Bind(dbx.Params{"id": 1}).One(&foldersRes)).To(Succeed())
+		Expect(conn.NewQuery("select ifnull(sum(num_audio_files + num_playlists + json_array_length(image_files)),0) as count from folder where library_id = {:id} and missing = 0").Bind(dbx.Params{"id": 1}).One(&filesRes)).To(Succeed())
+		Expect(conn.NewQuery("select count(*) as count from media_file where library_id = {:id} and missing = 1").Bind(dbx.Params{"id": 1}).One(&missingRes)).To(Succeed())
+		Expect(conn.NewQuery("select ifnull(sum(size),0) as sum from album where library_id = {:id} and missing = 0").Bind(dbx.Params{"id": 1}).One(&sizeRes)).To(Succeed())
+
+		Expect(libAfter.TotalSongs).To(Equal(int(songsRes.Count)))
+		Expect(libAfter.TotalAlbums).To(Equal(int(albumsRes.Count)))
+		Expect(libAfter.TotalArtists).To(Equal(int(artistsRes.Count)))
+		Expect(libAfter.TotalFolders).To(Equal(int(foldersRes.Count)))
+		Expect(libAfter.TotalFiles).To(Equal(int(filesRes.Count)))
+		Expect(libAfter.TotalMissingFiles).To(Equal(int(missingRes.Count)))
+		Expect(libAfter.TotalSize).To(Equal(sizeRes.Sum))
+	})
+})

scanner/controller.go

@@ -7,7 +7,6 @@ import (
 	"sync/atomic"
 	"time"
 
-	"github.com/Masterminds/squirrel"
 	"github.com/navidrome/navidrome/conf"
 	"github.com/navidrome/navidrome/consts"
 	"github.com/navidrome/navidrome/core"
@@ -178,20 +177,14 @@ func (s *controller) Status(ctx context.Context) (*StatusInfo, error) {
 }
 
 func (s *controller) getCounters(ctx context.Context) (int64, int64, error) {
-	count, err := s.ds.MediaFile(ctx).CountAll()
+	libs, err := s.ds.Library(ctx).GetAll()
 	if err != nil {
-		return 0, 0, fmt.Errorf("media file count: %w", err)
+		return 0, 0, fmt.Errorf("library count: %w", err)
 	}
-	folderCount, err := s.ds.Folder(ctx).CountAll(
-		model.QueryOptions{
-			Filters: squirrel.And{
-				squirrel.Gt{"num_audio_files": 0},
-				squirrel.Eq{"missing": false},
-			},
-		},
-	)
-	if err != nil {
-		return 0, 0, fmt.Errorf("folder count: %w", err)
+	var count, folderCount int64
+	for _, l := range libs {
+		count += int64(l.TotalSongs)
+		folderCount += int64(l.TotalFolders)
 	}
 	return count, folderCount, nil
 }

scanner/scanner.go

@@ -100,7 +100,7 @@ func (s *scannerImpl) scanAll(ctx context.Context, fullScan bool, progress chan<
 		s.runRefreshStats(ctx, &state),
 
 		// Update last_scan_completed_at for all libraries
-		s.runUpdateLibraries(ctx, libs),
+		s.runUpdateLibraries(ctx, libs, &state),
 
 		// Optimize DB
 		s.runOptimize(ctx),
@@ -175,8 +175,9 @@ func (s *scannerImpl) runOptimize(ctx context.Context) func() error {
 	}
 }
 
-func (s *scannerImpl) runUpdateLibraries(ctx context.Context, libs model.Libraries) func() error {
+func (s *scannerImpl) runUpdateLibraries(ctx context.Context, libs model.Libraries, state *scanState) func() error {
 	return func() error {
+		start := time.Now()
 		return s.ds.WithTx(func(tx model.DataStore) error {
 			for _, lib := range libs {
 				err := tx.Library(ctx).ScanEnd(lib.ID)
@@ -194,7 +195,17 @@ func (s *scannerImpl) runUpdateLibraries(ctx context.Context, libs model.Librari
 					log.Error(ctx, "Scanner: Error updating album PID conf", err)
 					return fmt.Errorf("updating album PID conf: %w", err)
 				}
+				if state.changesDetected.Load() {
+					log.Debug(ctx, "Scanner: Refreshing library stats", "lib", lib.Name)
+					if err := tx.Library(ctx).RefreshStats(lib.ID); err != nil {
+						log.Error(ctx, "Scanner: Error refreshing library stats", "lib", lib.Name, err)
+						return fmt.Errorf("refreshing library stats: %w", err)
+					}
+				} else {
+					log.Debug(ctx, "Scanner: No changes detected, skipping library stats refresh", "lib", lib.Name)
+				}
 			}
+			log.Debug(ctx, "Scanner: Updated libraries after scan", "elapsed", time.Since(start), "numLibraries", len(libs))
 			return nil
 		}, "scanner: update libraries")
 	}

tests/mock_library_repo.go

@@ -35,4 +35,8 @@ func (m *MockLibraryRepo) GetPath(id int) (string, error) {
 	return "", model.ErrNotFound
 }
 
+func (m *MockLibraryRepo) RefreshStats(id int) error {
+	return nil
+}
+
 var _ model.LibraryRepository = &MockLibraryRepo{}