feat(gc): enhance garbage collection to support selective library purging

Signed-off-by: Deluan <deluan@navidrome.org>
This commit is contained in:
Deluan 2025-11-13 15:27:20 -05:00
parent de773b9bcd
commit 67c876539e
7 changed files with 46 additions and 32 deletions

View File

@ -14,7 +14,7 @@ import (
)
var _ = Describe("Maintenance", func() {
var ds *extendedDataStore
var ds *tests.MockDataStore
var mfRepo *extendedMediaFileRepo
var service Maintenance
var ctx context.Context
@ -42,7 +42,7 @@ var _ = Describe("Maintenance", func() {
Expect(err).ToNot(HaveOccurred())
Expect(mfRepo.deleteMissingCalled).To(BeTrue())
Expect(mfRepo.deletedIDs).To(Equal([]string{"mf1", "mf2"}))
Expect(ds.gcCalled).To(BeTrue(), "GC should be called after deletion")
Expect(ds.GCCalled).To(BeTrue(), "GC should be called after deletion")
})
It("triggers artist stats refresh and album refresh after deletion", func() {
@ -97,7 +97,7 @@ var _ = Describe("Maintenance", func() {
})
// Set GC to return error
ds.gcError = errors.New("gc failed")
ds.GCError = errors.New("gc failed")
err := service.DeleteMissingFiles(ctx, []string{"mf1"})
@ -143,7 +143,7 @@ var _ = Describe("Maintenance", func() {
err := service.DeleteAllMissingFiles(ctx)
Expect(err).ToNot(HaveOccurred())
Expect(ds.gcCalled).To(BeTrue(), "GC should be called after deletion")
Expect(ds.GCCalled).To(BeTrue(), "GC should be called after deletion")
})
It("returns error if deletion fails", func() {
@ -253,11 +253,8 @@ var _ = Describe("Maintenance", func() {
})
// Test helper to create a mock DataStore with controllable behavior
func createTestDataStore() *extendedDataStore {
// Create extended datastore with GC tracking
ds := &extendedDataStore{
MockDataStore: &tests.MockDataStore{},
}
func createTestDataStore() *tests.MockDataStore {
ds := &tests.MockDataStore{}
// Create extended album repo with Put tracking
albumRepo := &extendedAlbumRepo{
@ -365,18 +362,3 @@ func (m *extendedArtistRepo) IsRefreshStatsCalled() bool {
defer m.mu.RUnlock()
return m.refreshStatsCalled
}
// Extension of MockDataStore to track GC calls
type extendedDataStore struct {
*tests.MockDataStore
gcCalled bool
gcError error
}
func (ds *extendedDataStore) GC(ctx context.Context) error {
ds.gcCalled = true
if ds.gcError != nil {
return ds.gcError
}
return ds.MockDataStore.GC(ctx)
}

View File

@ -43,5 +43,5 @@ type DataStore interface {
WithTx(block func(tx DataStore) error, scope ...string) error
WithTxImmediate(block func(tx DataStore) error, scope ...string) error
GC(ctx context.Context) error
GC(ctx context.Context, libraryIDs ...int) error
}

View File

@ -337,8 +337,12 @@ on conflict (user_id, item_id, item_type) do update
return r.executeSQL(query)
}
func (r *albumRepository) purgeEmpty() error {
func (r *albumRepository) purgeEmpty(libraryIDs ...int) error {
del := Delete(r.tableName).Where("id not in (select distinct(album_id) from media_file)")
// If libraryIDs are specified, only purge albums from those libraries
if len(libraryIDs) > 0 {
del = del.Where(Eq{"library_id": libraryIDs})
}
c, err := r.executeSQL(del)
if err != nil {
return fmt.Errorf("purging empty albums: %w", err)

View File

@ -191,7 +191,7 @@ func (r folderRepository) GetTouchedWithPlaylists() (model.FolderCursor, error)
}, nil
}
func (r folderRepository) purgeEmpty() error {
func (r folderRepository) purgeEmpty(libraryIDs ...int) error {
sq := Delete(r.tableName).Where(And{
Eq{"num_audio_files": 0},
Eq{"num_playlists": 0},
@ -199,6 +199,10 @@ func (r folderRepository) purgeEmpty() error {
ConcatExpr("id not in (select parent_id from folder)"),
ConcatExpr("id not in (select folder_id from media_file)"),
})
// If libraryIDs are specified, only purge folders from those libraries
if len(libraryIDs) > 0 {
sq = sq.Where(Eq{"library_id": libraryIDs})
}
c, err := r.executeSQL(sq)
if err != nil {
return fmt.Errorf("purging empty folders: %w", err)

View File

@ -157,7 +157,7 @@ func (s *SQLStore) WithTxImmediate(block func(tx model.DataStore) error, scope .
}, scope...)
}
func (s *SQLStore) GC(ctx context.Context) error {
func (s *SQLStore) GC(ctx context.Context, libraryIDs ...int) error {
trace := func(ctx context.Context, msg string, f func() error) func() error {
return func() error {
start := time.Now()
@ -167,11 +167,17 @@ func (s *SQLStore) GC(ctx context.Context) error {
}
}
// If libraryIDs are provided, scope operations to those libraries where possible
scoped := len(libraryIDs) > 0
if scoped {
log.Debug(ctx, "GC: Running selective garbage collection", "libraryIDs", libraryIDs)
}
err := run.Sequentially(
trace(ctx, "purge empty albums", func() error { return s.Album(ctx).(*albumRepository).purgeEmpty() }),
trace(ctx, "purge empty albums", func() error { return s.Album(ctx).(*albumRepository).purgeEmpty(libraryIDs...) }),
trace(ctx, "purge empty artists", func() error { return s.Artist(ctx).(*artistRepository).purgeEmpty() }),
trace(ctx, "mark missing artists", func() error { return s.Artist(ctx).(*artistRepository).markMissing() }),
trace(ctx, "purge empty folders", func() error { return s.Folder(ctx).(*folderRepository).purgeEmpty() }),
trace(ctx, "purge empty folders", func() error { return s.Folder(ctx).(*folderRepository).purgeEmpty(libraryIDs...) }),
trace(ctx, "clean album annotations", func() error { return s.Album(ctx).(*albumRepository).cleanAnnotations() }),
trace(ctx, "clean artist annotations", func() error { return s.Artist(ctx).(*artistRepository).cleanAnnotations() }),
trace(ctx, "clean media file annotations", func() error { return s.MediaFile(ctx).(*mediaFileRepository).cleanAnnotations() }),

View File

@ -3,6 +3,8 @@ package scanner
import (
"context"
"fmt"
"maps"
"slices"
"sync/atomic"
"time"
@ -229,7 +231,15 @@ func (s *scannerImpl) runGC(ctx context.Context, state *scanState) func() error
return s.ds.WithTx(func(tx model.DataStore) error {
if state.changesDetected.Load() {
start := time.Now()
err := tx.GC(ctx)
// For selective scans, extract library IDs to scope GC operations
var libraryIDs []int
if state.isSelectiveScan() {
libraryIDs = slices.Collect(maps.Keys(state.targets))
log.Debug(ctx, "Scanner: Running selective GC", "libraryIDs", libraryIDs)
}
err := tx.GC(ctx, libraryIDs...)
if err != nil {
log.Error(ctx, "Scanner: Error running GC", err)
return fmt.Errorf("running GC: %w", err)

View File

@ -28,6 +28,10 @@ type MockDataStore struct {
MockedRadio model.RadioRepository
scrobbleBufferMu sync.Mutex
repoMu sync.Mutex
// GC tracking
GCCalled bool
GCError error
}
func (db *MockDataStore) Library(ctx context.Context) model.LibraryRepository {
@ -258,6 +262,10 @@ func (db *MockDataStore) Resource(ctx context.Context, m any) model.ResourceRepo
}
}
func (db *MockDataStore) GC(context.Context) error {
func (db *MockDataStore) GC(context.Context, ...int) error {
db.GCCalled = true
if db.GCError != nil {
return db.GCError
}
return nil
}