mirror of
https://github.com/navidrome/navidrome.git
synced 2026-01-03 06:15:22 +00:00
* feat: Add selective folder scanning capability Implement targeted scanning of specific library/folder pairs without full recursion. This enables efficient rescanning of individual folders when changes are detected, significantly reducing scan time for large libraries. Key changes: - Add ScanTarget struct and ScanFolders API to Scanner interface - Implement CLI flag --targets for specifying libraryID:folderPath pairs - Add FolderRepository.GetByPaths() for batch folder info retrieval - Create loadSpecificFolders() for non-recursive directory loading - Scope GC operations to affected libraries only (with TODO for full impl) - Add comprehensive tests for selective scanning behavior The selective scan: - Only processes specified folders (no subdirectory recursion) - Maintains library isolation - Runs full maintenance pipeline scoped to affected libraries - Supports both full and quick scan modes Examples: navidrome scan --targets "1:Music/Rock,1:Music/Jazz" navidrome scan --full --targets "2:Classical" * feat(folder): replace GetByPaths with GetFolderUpdateInfo for improved folder updates retrieval Signed-off-by: Deluan <deluan@navidrome.org> * test: update parseTargets test to handle folder names with spaces Signed-off-by: Deluan <deluan@navidrome.org> * refactor(folder): remove unused LibraryPath struct and update GC logging message Signed-off-by: Deluan <deluan@navidrome.org> * refactor(folder): enhance external scanner to support target-specific scanning Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): simplify scanner methods Signed-off-by: Deluan <deluan@navidrome.org> * feat(watcher): implement folder scanning notifications with deduplication Signed-off-by: Deluan <deluan@navidrome.org> * refactor(watcher): add resolveFolderPath function for testability Signed-off-by: Deluan <deluan@navidrome.org> * feat(watcher): implement path ignoring based on .ndignore patterns Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): implement IgnoreChecker for managing .ndignore patterns Signed-off-by: Deluan <deluan@navidrome.org> * refactor(ignore_checker): rename scanner to lineScanner for clarity Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): enhance ScanTarget struct with String method for better target representation Signed-off-by: Deluan <deluan@navidrome.org> * fix(scanner): validate library ID to prevent negative values Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): simplify GC method by removing library ID parameter Signed-off-by: Deluan <deluan@navidrome.org> * feat(scanner): update folder scanning to include all descendants of specified folders Signed-off-by: Deluan <deluan@navidrome.org> * feat(subsonic): allow selective scan in the /startScan endpoint Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): update CallScan to handle specific library/folder pairs Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): streamline scanning logic by removing scanAll method Signed-off-by: Deluan <deluan@navidrome.org> * test: enhance mockScanner for thread safety and improve test reliability Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): move scanner.ScanTarget to model.ScanTarget Signed-off-by: Deluan <deluan@navidrome.org> * refactor: move scanner types to model,implement MockScanner Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): update scanner interface and implementations to use model.Scanner Signed-off-by: Deluan <deluan@navidrome.org> * refactor(folder_repository): normalize target path handling by using filepath.Clean Signed-off-by: Deluan <deluan@navidrome.org> * test(folder_repository): add comprehensive tests for folder retrieval and child exclusion Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): simplify selective scan logic using slice.Filter Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): streamline phase folder and album creation by removing unnecessary library parameter Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): move initialization logic from phase_1 to the scanner itself Signed-off-by: Deluan <deluan@navidrome.org> * refactor(tests): rename selective scan test file to scanner_selective_test.go Signed-off-by: Deluan <deluan@navidrome.org> * feat(configuration): add DevSelectiveWatcher configuration option Signed-off-by: Deluan <deluan@navidrome.org> * feat(watcher): enhance .ndignore handling for folder deletions and file changes Signed-off-by: Deluan <deluan@navidrome.org> * docs(scanner): comments Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): enhance walkDirTree to support target folder scanning Signed-off-by: Deluan <deluan@navidrome.org> * fix(scanner, watcher): handle errors when pushing ignore patterns for folders Signed-off-by: Deluan <deluan@navidrome.org> * Update scanner/phase_1_folders.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * refactor(scanner): replace parseTargets function with direct call to scanner.ParseTargets Signed-off-by: Deluan <deluan@navidrome.org> * test(scanner): add tests for ScanBegin and ScanEnd functionality Signed-off-by: Deluan <deluan@navidrome.org> * fix(library): update PRAGMA optimize to check table sizes without ANALYZE Signed-off-by: Deluan <deluan@navidrome.org> * test(scanner): refactor tests Signed-off-by: Deluan <deluan@navidrome.org> * feat(ui): add selective scan options and update translations Signed-off-by: Deluan <deluan@navidrome.org> * feat(ui): add quick and full scan options for individual libraries Signed-off-by: Deluan <deluan@navidrome.org> * feat(ui): add Scan buttonsto the LibraryList Signed-off-by: Deluan <deluan@navidrome.org> * feat(scan): update scanning parameters from 'path' to 'target' for selective scans. * refactor(scan): move ParseTargets function to model package * test(scan): suppress unused return value from SetUserLibraries in tests * feat(gc): enhance garbage collection to support selective library purging Signed-off-by: Deluan <deluan@navidrome.org> * fix(scanner): prevent race condition when scanning deleted folders When the watcher detects changes in a folder that gets deleted before the scanner runs (due to the 10-second delay), the scanner was prematurely removing these folders from the tracking map, preventing them from being marked as missing. The issue occurred because `newFolderEntry` was calling `popLastUpdate` before verifying the folder actually exists on the filesystem. Changes: - Move fs.Stat check before newFolderEntry creation in loadDir to ensure deleted folders remain in lastUpdates for finalize() to handle - Add early existence check in walkDirTree to skip non-existent target folders with a warning log - Add unit test verifying non-existent folders aren't removed from lastUpdates prematurely - Add integration test for deleted folder scenario with ScanFolders Fixes the issue where deleting entire folders (e.g., /music/AC_DC) wouldn't mark tracks as missing when using selective folder scanning. * refactor(scan): streamline folder entry creation and update handling Signed-off-by: Deluan <deluan@navidrome.org> * feat(scan): add '@Recycle' (QNAP) to ignored directories list Signed-off-by: Deluan <deluan@navidrome.org> * fix(log): improve thread safety in logging level management * test(scan): move unit tests for ParseTargets function Signed-off-by: Deluan <deluan@navidrome.org> * review Signed-off-by: Deluan <deluan@navidrome.org> --------- Signed-off-by: Deluan <deluan@navidrome.org> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: deluan <deluan.quintao@mechanical-orchard.com>
200 lines
6.6 KiB
Go
200 lines
6.6 KiB
Go
package persistence
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"reflect"
|
|
"time"
|
|
|
|
"github.com/navidrome/navidrome/db"
|
|
"github.com/navidrome/navidrome/log"
|
|
"github.com/navidrome/navidrome/model"
|
|
"github.com/navidrome/navidrome/utils/run"
|
|
"github.com/pocketbase/dbx"
|
|
)
|
|
|
|
type SQLStore struct {
|
|
db dbx.Builder
|
|
}
|
|
|
|
func New(conn *sql.DB) model.DataStore {
|
|
return &SQLStore{db: dbx.NewFromDB(conn, db.Driver)}
|
|
}
|
|
|
|
func (s *SQLStore) Album(ctx context.Context) model.AlbumRepository {
|
|
return NewAlbumRepository(ctx, s.getDBXBuilder())
|
|
}
|
|
|
|
func (s *SQLStore) Artist(ctx context.Context) model.ArtistRepository {
|
|
return NewArtistRepository(ctx, s.getDBXBuilder())
|
|
}
|
|
|
|
func (s *SQLStore) MediaFile(ctx context.Context) model.MediaFileRepository {
|
|
return NewMediaFileRepository(ctx, s.getDBXBuilder())
|
|
}
|
|
|
|
func (s *SQLStore) Library(ctx context.Context) model.LibraryRepository {
|
|
return NewLibraryRepository(ctx, s.getDBXBuilder())
|
|
}
|
|
|
|
func (s *SQLStore) Folder(ctx context.Context) model.FolderRepository {
|
|
return newFolderRepository(ctx, s.getDBXBuilder())
|
|
}
|
|
|
|
func (s *SQLStore) Genre(ctx context.Context) model.GenreRepository {
|
|
return NewGenreRepository(ctx, s.getDBXBuilder())
|
|
}
|
|
|
|
func (s *SQLStore) Tag(ctx context.Context) model.TagRepository {
|
|
return NewTagRepository(ctx, s.getDBXBuilder())
|
|
}
|
|
|
|
func (s *SQLStore) PlayQueue(ctx context.Context) model.PlayQueueRepository {
|
|
return NewPlayQueueRepository(ctx, s.getDBXBuilder())
|
|
}
|
|
|
|
func (s *SQLStore) Playlist(ctx context.Context) model.PlaylistRepository {
|
|
return NewPlaylistRepository(ctx, s.getDBXBuilder())
|
|
}
|
|
|
|
func (s *SQLStore) Property(ctx context.Context) model.PropertyRepository {
|
|
return NewPropertyRepository(ctx, s.getDBXBuilder())
|
|
}
|
|
|
|
func (s *SQLStore) Radio(ctx context.Context) model.RadioRepository {
|
|
return NewRadioRepository(ctx, s.getDBXBuilder())
|
|
}
|
|
|
|
func (s *SQLStore) UserProps(ctx context.Context) model.UserPropsRepository {
|
|
return NewUserPropsRepository(ctx, s.getDBXBuilder())
|
|
}
|
|
|
|
func (s *SQLStore) Share(ctx context.Context) model.ShareRepository {
|
|
return NewShareRepository(ctx, s.getDBXBuilder())
|
|
}
|
|
|
|
func (s *SQLStore) User(ctx context.Context) model.UserRepository {
|
|
return NewUserRepository(ctx, s.getDBXBuilder())
|
|
}
|
|
|
|
func (s *SQLStore) Transcoding(ctx context.Context) model.TranscodingRepository {
|
|
return NewTranscodingRepository(ctx, s.getDBXBuilder())
|
|
}
|
|
|
|
func (s *SQLStore) Player(ctx context.Context) model.PlayerRepository {
|
|
return NewPlayerRepository(ctx, s.getDBXBuilder())
|
|
}
|
|
|
|
func (s *SQLStore) ScrobbleBuffer(ctx context.Context) model.ScrobbleBufferRepository {
|
|
return NewScrobbleBufferRepository(ctx, s.getDBXBuilder())
|
|
}
|
|
|
|
func (s *SQLStore) Resource(ctx context.Context, m interface{}) model.ResourceRepository {
|
|
switch m.(type) {
|
|
case model.User:
|
|
return s.User(ctx).(model.ResourceRepository)
|
|
case model.Transcoding:
|
|
return s.Transcoding(ctx).(model.ResourceRepository)
|
|
case model.Player:
|
|
return s.Player(ctx).(model.ResourceRepository)
|
|
case model.Artist:
|
|
return s.Artist(ctx).(model.ResourceRepository)
|
|
case model.Album:
|
|
return s.Album(ctx).(model.ResourceRepository)
|
|
case model.MediaFile:
|
|
return s.MediaFile(ctx).(model.ResourceRepository)
|
|
case model.Genre:
|
|
return s.Genre(ctx).(model.ResourceRepository)
|
|
case model.Playlist:
|
|
return s.Playlist(ctx).(model.ResourceRepository)
|
|
case model.Radio:
|
|
return s.Radio(ctx).(model.ResourceRepository)
|
|
case model.Share:
|
|
return s.Share(ctx).(model.ResourceRepository)
|
|
case model.Tag:
|
|
return s.Tag(ctx).(model.ResourceRepository)
|
|
}
|
|
log.Error("Resource not implemented", "model", reflect.TypeOf(m).Name())
|
|
return nil
|
|
}
|
|
|
|
func (s *SQLStore) WithTx(block func(tx model.DataStore) error, scope ...string) error {
|
|
var msg string
|
|
if len(scope) > 0 {
|
|
msg = scope[0]
|
|
}
|
|
start := time.Now()
|
|
conn, inTx := s.db.(*dbx.DB)
|
|
if !inTx {
|
|
log.Trace("Nested Transaction started", "scope", msg)
|
|
conn = dbx.NewFromDB(db.Db(), db.Driver)
|
|
} else {
|
|
log.Trace("Transaction started", "scope", msg)
|
|
}
|
|
return conn.Transactional(func(tx *dbx.Tx) error {
|
|
newDb := &SQLStore{db: tx}
|
|
err := block(newDb)
|
|
if !inTx {
|
|
log.Trace("Nested Transaction finished", "scope", msg, "elapsed", time.Since(start), err)
|
|
} else {
|
|
log.Trace("Transaction finished", "scope", msg, "elapsed", time.Since(start), err)
|
|
}
|
|
return err
|
|
})
|
|
}
|
|
|
|
func (s *SQLStore) WithTxImmediate(block func(tx model.DataStore) error, scope ...string) error {
|
|
ctx := context.Background()
|
|
return s.WithTx(func(tx model.DataStore) error {
|
|
// Workaround to force the transaction to be upgraded to immediate mode to avoid deadlocks
|
|
// See https://berthub.eu/articles/posts/a-brief-post-on-sqlite3-database-locked-despite-timeout/
|
|
_ = tx.Property(ctx).Put("tmp_lock_flag", "")
|
|
defer func() {
|
|
_ = tx.Property(ctx).Delete("tmp_lock_flag")
|
|
}()
|
|
|
|
return block(tx)
|
|
}, scope...)
|
|
}
|
|
|
|
func (s *SQLStore) GC(ctx context.Context, libraryIDs ...int) error {
|
|
trace := func(ctx context.Context, msg string, f func() error) func() error {
|
|
return func() error {
|
|
start := time.Now()
|
|
err := f()
|
|
log.Debug(ctx, "GC: "+msg, "elapsed", time.Since(start), err)
|
|
return err
|
|
}
|
|
}
|
|
|
|
// If libraryIDs are provided, scope operations to those libraries where possible
|
|
scoped := len(libraryIDs) > 0
|
|
if scoped {
|
|
log.Debug(ctx, "GC: Running selective garbage collection", "libraryIDs", libraryIDs)
|
|
}
|
|
|
|
err := run.Sequentially(
|
|
trace(ctx, "purge empty albums", func() error { return s.Album(ctx).(*albumRepository).purgeEmpty(libraryIDs...) }),
|
|
trace(ctx, "purge empty artists", func() error { return s.Artist(ctx).(*artistRepository).purgeEmpty() }),
|
|
trace(ctx, "mark missing artists", func() error { return s.Artist(ctx).(*artistRepository).markMissing() }),
|
|
trace(ctx, "purge empty folders", func() error { return s.Folder(ctx).(*folderRepository).purgeEmpty(libraryIDs...) }),
|
|
trace(ctx, "clean album annotations", func() error { return s.Album(ctx).(*albumRepository).cleanAnnotations() }),
|
|
trace(ctx, "clean artist annotations", func() error { return s.Artist(ctx).(*artistRepository).cleanAnnotations() }),
|
|
trace(ctx, "clean media file annotations", func() error { return s.MediaFile(ctx).(*mediaFileRepository).cleanAnnotations() }),
|
|
trace(ctx, "clean media file bookmarks", func() error { return s.MediaFile(ctx).(*mediaFileRepository).cleanBookmarks() }),
|
|
trace(ctx, "purge non used tags", func() error { return s.Tag(ctx).(*tagRepository).purgeUnused() }),
|
|
trace(ctx, "remove orphan playlist tracks", func() error { return s.Playlist(ctx).(*playlistRepository).removeOrphans() }),
|
|
)
|
|
if err != nil {
|
|
log.Error(ctx, "Error tidying up database", err)
|
|
}
|
|
return err
|
|
}
|
|
|
|
func (s *SQLStore) getDBXBuilder() dbx.Builder {
|
|
if s.db == nil {
|
|
return dbx.NewFromDB(db.Db(), db.Driver)
|
|
}
|
|
return s.db
|
|
}
|