From 8f1a6116fe7cbb534a7bc36b35ebdef7ed979763 Mon Sep 17 00:00:00 2001 From: Deluan Date: Mon, 10 Nov 2025 18:10:57 -0500 Subject: [PATCH 01/40] feat: Add selective folder scanning capability Implement targeted scanning of specific library/folder pairs without full recursion. This enables efficient rescanning of individual folders when changes are detected, significantly reducing scan time for large libraries. Key changes: - Add ScanTarget struct and ScanFolders API to Scanner interface - Implement CLI flag --targets for specifying libraryID:folderPath pairs - Add FolderRepository.GetByPaths() for batch folder info retrieval - Create loadSpecificFolders() for non-recursive directory loading - Scope GC operations to affected libraries only (with TODO for full impl) - Add comprehensive tests for selective scanning behavior The selective scan: - Only processes specified folders (no subdirectory recursion) - Maintains library isolation - Runs full maintenance pipeline scoped to affected libraries - Supports both full and quick scan modes Examples: navidrome scan --targets "1:Music/Rock,1:Music/Jazz" navidrome scan --full --targets "2:Classical" --- cmd/scan.go | 58 ++++++++- cmd/scan_test.go | 82 ++++++++++++ model/datastore.go | 2 +- model/folder.go | 7 ++ persistence/folder_repository.go | 61 +++++++++ persistence/folder_repository_test.go | 172 ++++++++++++++++++++++++++ persistence/persistence.go | 8 +- scanner/controller.go | 32 ++++- scanner/external.go | 51 ++++++++ scanner/phase_1_folders.go | 64 +++++++--- scanner/scanner.go | 116 ++++++++++++++++- scanner/scanner_test.go | 55 ++++++++ scanner/walk_dir_tree.go | 58 +++++++++ tests/mock_data_store.go | 2 +- 14 files changed, 746 insertions(+), 22 deletions(-) create mode 100644 cmd/scan_test.go create mode 100644 persistence/folder_repository_test.go diff --git a/cmd/scan.go b/cmd/scan.go index d37ccd69f..06d8c6c25 100644 --- a/cmd/scan.go +++ b/cmd/scan.go @@ -3,7 +3,10 @@ package cmd import ( "context" "encoding/gob" + "fmt" "os" + "strconv" + "strings" "github.com/navidrome/navidrome/core" "github.com/navidrome/navidrome/db" @@ -17,11 +20,13 @@ import ( var ( fullScan bool subprocess bool + targets string ) func init() { scanCmd.Flags().BoolVarP(&fullScan, "full", "f", false, "check all subfolders, ignoring timestamps") scanCmd.Flags().BoolVarP(&subprocess, "subprocess", "", false, "run as subprocess (internal use)") + scanCmd.Flags().StringVarP(&targets, "targets", "t", "", "comma-separated list of libraryID:folderPath pairs (e.g., \"1:Music/Rock,1:Music/Jazz,2:Classical\")") rootCmd.AddCommand(scanCmd) } @@ -68,7 +73,18 @@ func runScanner(ctx context.Context) { ds := persistence.New(sqlDB) pls := core.NewPlaylists(ds) - progress, err := scanner.CallScan(ctx, ds, pls, fullScan) + // Parse targets if provided + var scanTargets []scanner.ScanTarget + if targets != "" { + var err error + scanTargets, err = parseTargets(targets) + if err != nil { + log.Fatal(ctx, "Failed to parse targets", err) + } + log.Info(ctx, "Scanning specific folders", "numTargets", len(scanTargets)) + } + + progress, err := scanner.CallScanFolders(ctx, ds, pls, fullScan, scanTargets) if err != nil { log.Fatal(ctx, "Failed to scan", err) } @@ -80,3 +96,43 @@ func runScanner(ctx context.Context) { trackScanInteractively(ctx, progress) } } + +// parseTargets parses the comma-separated targets string into ScanTarget structs +// Format: "libraryID:folderPath,libraryID:folderPath,..." +// Example: "1:Music/Rock,1:Music/Jazz,2:Classical" +func parseTargets(targetsStr string) ([]scanner.ScanTarget, error) { + parts := strings.Split(targetsStr, ",") + targets := make([]scanner.ScanTarget, 0, len(parts)) + + for _, part := range parts { + part = strings.TrimSpace(part) + if part == "" { + continue + } + + // Split by the first colon + colonIdx := strings.Index(part, ":") + if colonIdx == -1 { + return nil, fmt.Errorf("invalid target format: %q (expected libraryID:folderPath)", part) + } + + libIDStr := part[:colonIdx] + folderPath := part[colonIdx+1:] + + libID, err := strconv.Atoi(libIDStr) + if err != nil { + return nil, fmt.Errorf("invalid library ID %q: %w", libIDStr, err) + } + + targets = append(targets, scanner.ScanTarget{ + LibraryID: libID, + FolderPath: folderPath, + }) + } + + if len(targets) == 0 { + return nil, fmt.Errorf("no valid targets found in %q", targetsStr) + } + + return targets, nil +} diff --git a/cmd/scan_test.go b/cmd/scan_test.go new file mode 100644 index 000000000..191e1096c --- /dev/null +++ b/cmd/scan_test.go @@ -0,0 +1,82 @@ +package cmd + +import ( + "github.com/navidrome/navidrome/scanner" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("parseTargets", func() { + Context("Valid targets", func() { + It("parses a single target", func() { + targets, err := parseTargets("1:Music/Rock") + Expect(err).ToNot(HaveOccurred()) + Expect(targets).To(HaveLen(1)) + Expect(targets[0].LibraryID).To(Equal(1)) + Expect(targets[0].FolderPath).To(Equal("Music/Rock")) + }) + + It("parses multiple targets", func() { + targets, err := parseTargets("1:Music/Rock,2:Jazz,3:Classical/Beethoven") + Expect(err).ToNot(HaveOccurred()) + Expect(targets).To(HaveLen(3)) + Expect(targets[0]).To(Equal(scanner.ScanTarget{LibraryID: 1, FolderPath: "Music/Rock"})) + Expect(targets[1]).To(Equal(scanner.ScanTarget{LibraryID: 2, FolderPath: "Jazz"})) + Expect(targets[2]).To(Equal(scanner.ScanTarget{LibraryID: 3, FolderPath: "Classical/Beethoven"})) + }) + + It("handles targets with spaces around commas", func() { + targets, err := parseTargets("1:Music/Rock , 2:Jazz , 3:Classical") + Expect(err).ToNot(HaveOccurred()) + Expect(targets).To(HaveLen(3)) + }) + + It("handles paths with colons after the first colon", func() { + targets, err := parseTargets("1:C:/Music/Rock") + Expect(err).ToNot(HaveOccurred()) + Expect(targets).To(HaveLen(1)) + Expect(targets[0].LibraryID).To(Equal(1)) + Expect(targets[0].FolderPath).To(Equal("C:/Music/Rock")) + }) + + It("handles empty folder paths", func() { + targets, err := parseTargets("1:,2:") + Expect(err).ToNot(HaveOccurred()) + Expect(targets).To(HaveLen(2)) + Expect(targets[0].FolderPath).To(BeEmpty()) + Expect(targets[1].FolderPath).To(BeEmpty()) + }) + }) + + Context("Invalid targets", func() { + It("returns error for empty string", func() { + _, err := parseTargets("") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("no valid targets")) + }) + + It("returns error for missing colon", func() { + _, err := parseTargets("1Music") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("invalid target format")) + }) + + It("returns error for invalid library ID", func() { + _, err := parseTargets("abc:Music") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("invalid library ID")) + }) + + It("handles negative library ID", func() { + targets, err := parseTargets("-1:Music") + Expect(err).ToNot(HaveOccurred()) // Actually valid - strconv.Atoi accepts negative numbers + Expect(targets[0].LibraryID).To(Equal(-1)) + }) + + It("handles only whitespace", func() { + _, err := parseTargets(" , , ") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("no valid targets")) + }) + }) +}) diff --git a/model/datastore.go b/model/datastore.go index 4290e2134..536a37274 100644 --- a/model/datastore.go +++ b/model/datastore.go @@ -43,5 +43,5 @@ type DataStore interface { WithTx(block func(tx DataStore) error, scope ...string) error WithTxImmediate(block func(tx DataStore) error, scope ...string) error - GC(ctx context.Context) error + GC(ctx context.Context, libraryIDs ...int) error } diff --git a/model/folder.go b/model/folder.go index f715f8c11..7ac2bf031 100644 --- a/model/folder.go +++ b/model/folder.go @@ -83,6 +83,7 @@ type FolderUpdateInfo struct { type FolderRepository interface { Get(id string) (*Folder, error) GetByPath(lib Library, path string) (*Folder, error) + GetByPaths(targets []LibraryPath) (map[string]FolderUpdateInfo, error) GetAll(...QueryOptions) ([]Folder, error) CountAll(...QueryOptions) (int64, error) GetLastUpdates(lib Library) (map[string]FolderUpdateInfo, error) @@ -90,3 +91,9 @@ type FolderRepository interface { MarkMissing(missing bool, ids ...string) error GetTouchedWithPlaylists() (FolderCursor, error) } + +// LibraryPath represents a folder path within a specific library +type LibraryPath struct { + LibraryID int + FolderPath string +} diff --git a/persistence/folder_repository.go b/persistence/folder_repository.go index 96a9bae82..646c7e61d 100644 --- a/persistence/folder_repository.go +++ b/persistence/folder_repository.go @@ -78,6 +78,67 @@ func (r folderRepository) GetByPath(lib model.Library, path string) (*model.Fold return r.Get(id) } +func (r folderRepository) GetByPaths(targets []model.LibraryPath) (map[string]model.FolderUpdateInfo, error) { + if len(targets) == 0 { + return make(map[string]model.FolderUpdateInfo), nil + } + + // Group targets by library to build efficient queries + targetsByLib := make(map[int][]string) + folderIDs := make([]string, 0, len(targets)) + + // We need to resolve library paths to generate folder IDs + // Get all libraries first + libRepo := NewLibraryRepository(r.ctx, r.db) + allLibs, err := libRepo.GetAll() + if err != nil { + return nil, fmt.Errorf("getting libraries: %w", err) + } + libMap := make(map[int]model.Library) + for _, lib := range allLibs { + libMap[lib.ID] = lib + } + + // Generate folder IDs for all targets + for _, target := range targets { + lib, ok := libMap[target.LibraryID] + if !ok { + continue // Skip invalid library IDs + } + folderPath := target.FolderPath + if folderPath == "" { + folderPath = "." + } + folderID := model.FolderID(lib, folderPath) + folderIDs = append(folderIDs, folderID) + targetsByLib[target.LibraryID] = append(targetsByLib[target.LibraryID], folderPath) + } + + if len(folderIDs) == 0 { + return make(map[string]model.FolderUpdateInfo), nil + } + + // Query folders by IDs + sq := r.newSelect().Columns("id", "updated_at", "hash").Where(And{ + Eq{"id": folderIDs}, + Eq{"missing": false}, + }) + var res []struct { + ID string + UpdatedAt time.Time + Hash string + } + err = r.queryAll(sq, &res) + if err != nil { + return nil, err + } + m := make(map[string]model.FolderUpdateInfo, len(res)) + for _, f := range res { + m[f.ID] = model.FolderUpdateInfo{UpdatedAt: f.UpdatedAt, Hash: f.Hash} + } + return m, nil +} + func (r folderRepository) GetAll(opt ...model.QueryOptions) ([]model.Folder, error) { sq := r.selectFolder(opt...) var res dbFolders diff --git a/persistence/folder_repository_test.go b/persistence/folder_repository_test.go new file mode 100644 index 000000000..93190c494 --- /dev/null +++ b/persistence/folder_repository_test.go @@ -0,0 +1,172 @@ +package persistence + +import ( + "context" + + "github.com/navidrome/navidrome/log" + "github.com/navidrome/navidrome/model" + "github.com/navidrome/navidrome/model/request" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/pocketbase/dbx" +) + +var _ = Describe("FolderRepository", func() { + var repo model.FolderRepository + var ctx context.Context + var conn *dbx.DB + var testLib model.Library + + BeforeEach(func() { + ctx = request.WithUser(log.NewContext(context.TODO()), model.User{ID: "userid"}) + conn = GetDBXBuilder() + repo = newFolderRepository(ctx, conn) + + // Use existing library ID 1 from test fixtures + libRepo := NewLibraryRepository(ctx, conn) + lib, err := libRepo.Get(1) + Expect(err).ToNot(HaveOccurred()) + testLib = *lib + }) + + AfterEach(func() { + // Clean up test folders created by these tests + // Only delete folders with paths starting with our test prefix + _, _ = conn.NewQuery("DELETE FROM folder WHERE library_id = 1 AND (path LIKE 'TestFolder%' OR path LIKE 'Music/%' OR path = 'Classical' OR path = 'Podcasts')").Execute() + }) + + Describe("GetByPaths", func() { + Context("with valid targets", func() { + It("returns folder info for existing folders", func() { + // Create test folders + folder1 := model.NewFolder(testLib, "Music/Rock") + folder2 := model.NewFolder(testLib, "Music/Jazz") + folder3 := model.NewFolder(testLib, "Classical") + + err := repo.Put(folder1) + Expect(err).ToNot(HaveOccurred()) + err = repo.Put(folder2) + Expect(err).ToNot(HaveOccurred()) + err = repo.Put(folder3) + Expect(err).ToNot(HaveOccurred()) + + // Query by paths + targets := []model.LibraryPath{ + {LibraryID: testLib.ID, FolderPath: "Music/Rock"}, + {LibraryID: testLib.ID, FolderPath: "Classical"}, + } + + results, err := repo.GetByPaths(targets) + Expect(err).ToNot(HaveOccurred()) + Expect(results).To(HaveLen(2)) + + // Verify folder IDs are in results + Expect(results).To(HaveKey(folder1.ID)) + Expect(results).To(HaveKey(folder3.ID)) + Expect(results).ToNot(HaveKey(folder2.ID)) + + // Verify update info is populated + Expect(results[folder1.ID].UpdatedAt).ToNot(BeZero()) + Expect(results[folder1.ID].Hash).To(Equal(folder1.Hash)) + }) + + It("handles empty folder path as root", func() { + // Create root folder + rootFolder := model.NewFolder(testLib, ".") + err := repo.Put(rootFolder) + Expect(err).ToNot(HaveOccurred()) + + targets := []model.LibraryPath{ + {LibraryID: testLib.ID, FolderPath: ""}, + } + + results, err := repo.GetByPaths(targets) + Expect(err).ToNot(HaveOccurred()) + Expect(results).To(HaveLen(1)) + Expect(results).To(HaveKey(rootFolder.ID)) + }) + + It("returns empty map for non-existent folders", func() { + targets := []model.LibraryPath{ + {LibraryID: testLib.ID, FolderPath: "NonExistent/Path"}, + } + + results, err := repo.GetByPaths(targets) + Expect(err).ToNot(HaveOccurred()) + Expect(results).To(BeEmpty()) + }) + + It("skips missing folders", func() { + // Create a folder and mark it as missing + folder := model.NewFolder(testLib, "Music/Missing") + folder.Missing = true + err := repo.Put(folder) + Expect(err).ToNot(HaveOccurred()) + + targets := []model.LibraryPath{ + {LibraryID: testLib.ID, FolderPath: "Music/Missing"}, + } + + results, err := repo.GetByPaths(targets) + Expect(err).ToNot(HaveOccurred()) + Expect(results).To(BeEmpty()) + }) + }) + + Context("with invalid library IDs", func() { + It("returns empty map for non-existent library", func() { + targets := []model.LibraryPath{ + {LibraryID: 99999, FolderPath: "Music"}, + } + + results, err := repo.GetByPaths(targets) + Expect(err).ToNot(HaveOccurred()) + Expect(results).To(BeEmpty()) + }) + }) + + Context("with empty targets", func() { + It("returns empty map", func() { + results, err := repo.GetByPaths([]model.LibraryPath{}) + Expect(err).ToNot(HaveOccurred()) + Expect(results).To(BeEmpty()) + }) + + It("returns empty map for nil targets", func() { + results, err := repo.GetByPaths(nil) + Expect(err).ToNot(HaveOccurred()) + Expect(results).To(BeEmpty()) + }) + }) + + Context("with multiple paths in same library", func() { + It("returns multiple folders", func() { + // Create multiple folders in the same library + folder1 := model.NewFolder(testLib, "Music/Pop") + folder2 := model.NewFolder(testLib, "Music/Electronic") + folder3 := model.NewFolder(testLib, "Podcasts") + + err := repo.Put(folder1) + Expect(err).ToNot(HaveOccurred()) + err = repo.Put(folder2) + Expect(err).ToNot(HaveOccurred()) + err = repo.Put(folder3) + Expect(err).ToNot(HaveOccurred()) + + // Query multiple paths + targets := []model.LibraryPath{ + {LibraryID: testLib.ID, FolderPath: "Music/Pop"}, + {LibraryID: testLib.ID, FolderPath: "Music/Electronic"}, + {LibraryID: testLib.ID, FolderPath: "Podcasts"}, + } + + results, err := repo.GetByPaths(targets) + Expect(err).ToNot(HaveOccurred()) + Expect(results).To(HaveLen(3)) + Expect(results).To(HaveKey(folder1.ID)) + Expect(results).To(HaveKey(folder2.ID)) + Expect(results).To(HaveKey(folder3.ID)) + }) + }) + }) +}) diff --git a/persistence/persistence.go b/persistence/persistence.go index ac607f85f..b0c5a7f50 100644 --- a/persistence/persistence.go +++ b/persistence/persistence.go @@ -157,7 +157,7 @@ func (s *SQLStore) WithTxImmediate(block func(tx model.DataStore) error, scope . }, scope...) } -func (s *SQLStore) GC(ctx context.Context) error { +func (s *SQLStore) GC(ctx context.Context, libraryIDs ...int) error { trace := func(ctx context.Context, msg string, f func() error) func() error { return func() error { start := time.Now() @@ -167,6 +167,12 @@ func (s *SQLStore) GC(ctx context.Context) error { } } + // TODO: Implement library-specific filtering for GC operations + // For now, GC runs globally even in selective scans + if len(libraryIDs) > 0 { + log.Debug(ctx, "GC: Running with library filter", "libraries", libraryIDs) + } + err := run.Sequentially( trace(ctx, "purge empty albums", func() error { return s.Album(ctx).(*albumRepository).purgeEmpty() }), trace(ctx, "purge empty artists", func() error { return s.Artist(ctx).(*artistRepository).purgeEmpty() }), diff --git a/scanner/controller.go b/scanner/controller.go index c1347077a..d2ebdbfd4 100644 --- a/scanner/controller.go +++ b/scanner/controller.go @@ -26,9 +26,18 @@ var ( ErrAlreadyScanning = errors.New("already scanning") ) +// ScanTarget represents a specific folder within a library to be scanned. +type ScanTarget struct { + LibraryID int + FolderPath string // Relative path within the library, or "" for entire library +} + type Scanner interface { // ScanAll starts a full scan of the music library. This is a blocking operation. ScanAll(ctx context.Context, fullScan bool) (warnings []string, err error) + // ScanFolders scans specific library/folder pairs without recursing into subdirectories. + // This is a blocking operation. + ScanFolders(ctx context.Context, fullScan bool, targets []ScanTarget) (warnings []string, err error) Status(context.Context) (*StatusInfo, error) } @@ -68,6 +77,12 @@ func (s *controller) getScanner() scanner { // CallScan starts an in-process scan of the music library. // This is meant to be called from the command line (see cmd/scan.go). func CallScan(ctx context.Context, ds model.DataStore, pls core.Playlists, fullScan bool) (<-chan *ProgressInfo, error) { + return CallScanFolders(ctx, ds, pls, fullScan, nil) +} + +// CallScanFolders starts an in-process scan of specific library/folder pairs. +// If targets is nil, it scans all libraries. This is meant to be called from the command line. +func CallScanFolders(ctx context.Context, ds model.DataStore, pls core.Playlists, fullScan bool, targets []ScanTarget) (<-chan *ProgressInfo, error) { release, err := lockScan(ctx) if err != nil { return nil, err @@ -79,7 +94,11 @@ func CallScan(ctx context.Context, ds model.DataStore, pls core.Playlists, fullS go func() { defer close(progress) scanner := &scannerImpl{ds: ds, cw: artwork.NoopCacheWarmer(), pls: pls} - scanner.scanAll(ctx, fullScan, progress) + if targets == nil { + scanner.scanAll(ctx, fullScan, progress) + } else { + scanner.scanFolders(ctx, fullScan, targets, progress) + } }() return progress, nil } @@ -101,6 +120,7 @@ type ProgressInfo struct { type scanner interface { scanAll(ctx context.Context, fullScan bool, progress chan<- *ProgressInfo) + scanFolders(ctx context.Context, fullScan bool, targets []ScanTarget, progress chan<- *ProgressInfo) } type controller struct { @@ -208,6 +228,10 @@ func (s *controller) getCounters(ctx context.Context) (int64, int64, error) { } func (s *controller) ScanAll(requestCtx context.Context, fullScan bool) ([]string, error) { + return s.ScanFolders(requestCtx, fullScan, nil) +} + +func (s *controller) ScanFolders(requestCtx context.Context, fullScan bool, targets []ScanTarget) ([]string, error) { release, err := lockScan(requestCtx) if err != nil { return nil, err @@ -224,7 +248,11 @@ func (s *controller) ScanAll(requestCtx context.Context, fullScan bool) ([]strin go func() { defer close(progress) scanner := s.getScanner() - scanner.scanAll(ctx, fullScan, progress) + if targets == nil { + scanner.scanAll(ctx, fullScan, progress) + } else { + scanner.scanFolders(ctx, fullScan, targets, progress) + } }() // Wait for the scan to finish, sending progress events to all connected clients diff --git a/scanner/external.go b/scanner/external.go index c4a29efa3..86d171346 100644 --- a/scanner/external.go +++ b/scanner/external.go @@ -8,6 +8,7 @@ import ( "io" "os" "os/exec" + "strconv" "github.com/navidrome/navidrome/conf" "github.com/navidrome/navidrome/log" @@ -62,6 +63,56 @@ func (s *scannerExternal) scanAll(ctx context.Context, fullScan bool, progress c } } +func (s *scannerExternal) scanFolders(ctx context.Context, fullScan bool, targets []ScanTarget, progress chan<- *ProgressInfo) { + exe, err := os.Executable() + if err != nil { + progress <- &ProgressInfo{Error: fmt.Sprintf("failed to get executable path: %s", err)} + return + } + + // Build targets string for CLI + var targetsStr string + for i, target := range targets { + if i > 0 { + targetsStr += "," + } + targetsStr += strconv.Itoa(target.LibraryID) + ":" + target.FolderPath + } + + log.Debug(ctx, "Spawning external scanner process with targets", "fullScan", fullScan, "path", exe, "targets", targetsStr) + cmd := exec.CommandContext(ctx, exe, "scan", + "--nobanner", "--subprocess", + "--configfile", conf.Server.ConfigFile, + "--datafolder", conf.Server.DataFolder, + "--cachefolder", conf.Server.CacheFolder, + "--targets", targetsStr, + If(fullScan, "--full", "")) + + in, out := io.Pipe() + defer in.Close() + defer out.Close() + cmd.Stdout = out + cmd.Stderr = os.Stderr + + if err := cmd.Start(); err != nil { + progress <- &ProgressInfo{Error: fmt.Sprintf("failed to start scanner process: %s", err)} + return + } + go s.wait(cmd, out) + + decoder := gob.NewDecoder(in) + for { + var p ProgressInfo + if err := decoder.Decode(&p); err != nil { + if !errors.Is(err, io.EOF) { + progress <- &ProgressInfo{Error: fmt.Sprintf("failed to read status from scanner: %s", err)} + } + break + } + progress <- &p + } +} + func (s *scannerExternal) wait(cmd *exec.Cmd, out *io.PipeWriter) { if err := cmd.Wait(); err != nil { var exitErr *exec.ExitError diff --git a/scanner/phase_1_folders.go b/scanner/phase_1_folders.go index e04f10c70..ebdf74b54 100644 --- a/scanner/phase_1_folders.go +++ b/scanner/phase_1_folders.go @@ -48,7 +48,14 @@ func createPhaseFolders(ctx context.Context, state *scanState, ds model.DataStor } else { log.Debug(ctx, "Scanner: Resuming previous scan", "lib", lib.Name, "lastScanStartedAt", lib.LastScanStartedAt, "fullScan", lib.FullScanInProgress) } - job, err := newScanJob(ctx, ds, cw, lib, state.fullScan) + + // Get target folders for this library if selective scan + var targetFolders []string + if state.targets != nil { + targetFolders = state.targets[lib.ID] + } + + job, err := newScanJob(ctx, ds, cw, lib, state.fullScan, targetFolders) if err != nil { log.Error(ctx, "Scanner: Error creating scan context", "lib", lib.Name, err) state.sendWarning(err.Error()) @@ -65,19 +72,36 @@ func createPhaseFolders(ctx context.Context, state *scanState, ds model.DataStor } type scanJob struct { - lib model.Library - fs storage.MusicFS - cw artwork.CacheWarmer - lastUpdates map[string]model.FolderUpdateInfo - lock sync.Mutex - numFolders atomic.Int64 + lib model.Library + fs storage.MusicFS + cw artwork.CacheWarmer + lastUpdates map[string]model.FolderUpdateInfo + targetFolders []string // Optional: specific folders to scan (non-recursive) + lock sync.Mutex + numFolders atomic.Int64 } -func newScanJob(ctx context.Context, ds model.DataStore, cw artwork.CacheWarmer, lib model.Library, fullScan bool) (*scanJob, error) { - lastUpdates, err := ds.Folder(ctx).GetLastUpdates(lib) +func newScanJob(ctx context.Context, ds model.DataStore, cw artwork.CacheWarmer, lib model.Library, fullScan bool, targetFolders []string) (*scanJob, error) { + var lastUpdates map[string]model.FolderUpdateInfo + var err error + + // If we have target folders, get only those folder updates. Otherwise get all updates for the library + if len(targetFolders) > 0 { + var targets []model.LibraryPath + for _, folderPath := range targetFolders { + targets = append(targets, model.LibraryPath{ + LibraryID: lib.ID, + FolderPath: folderPath, + }) + } + lastUpdates, err = ds.Folder(ctx).GetByPaths(targets) + } else { + lastUpdates, err = ds.Folder(ctx).GetLastUpdates(lib) + } if err != nil { return nil, fmt.Errorf("getting last updates: %w", err) } + fileStore, err := storage.For(lib.Path) if err != nil { log.Error(ctx, "Error getting storage for library", "library", lib.Name, "path", lib.Path, err) @@ -90,10 +114,11 @@ func newScanJob(ctx context.Context, ds model.DataStore, cw artwork.CacheWarmer, } lib.FullScanInProgress = lib.FullScanInProgress || fullScan return &scanJob{ - lib: lib, - fs: fsys, - cw: cw, - lastUpdates: lastUpdates, + lib: lib, + fs: fsys, + cw: cw, + lastUpdates: lastUpdates, + targetFolders: targetFolders, }, nil } @@ -144,7 +169,18 @@ func (p *phaseFolders) producer() ppl.Producer[*folderEntry] { if utils.IsCtxDone(p.ctx) { break } - outputChan, err := walkDirTree(p.ctx, job) + + var outputChan <-chan *folderEntry + var err error + + // Use selective folder loading if target folders are specified + if len(job.targetFolders) > 0 { + log.Debug(p.ctx, "Scanner: Loading specific folders only (non-recursive)", "lib", job.lib.Name, "numTargets", len(job.targetFolders)) + outputChan, err = loadSpecificFolders(p.ctx, job, job.targetFolders) + } else { + outputChan, err = walkDirTree(p.ctx, job) + } + if err != nil { log.Warn(p.ctx, "Scanner: Error scanning library", "lib", job.lib.Name, err) } diff --git a/scanner/scanner.go b/scanner/scanner.go index 04a5c2456..3c4d41d31 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -28,7 +28,9 @@ type scanState struct { progress chan<- *ProgressInfo fullScan bool changesDetected atomic.Bool - libraries model.Libraries // Store libraries list for consistency across phases + libraries model.Libraries // Store libraries list for consistency across phases + targets map[int][]string // Optional: map[libraryID][]folderPaths for selective scans + affectedLibIDs []int // IDs of libraries involved in the scan (for GC scoping) } func (s *scanState) sendProgress(info *ProgressInfo) { @@ -134,13 +136,123 @@ func (s *scannerImpl) scanAll(ctx context.Context, fullScan bool, progress chan< log.Info(ctx, "Scanner: Finished scanning all libraries", "duration", time.Since(startTime)) } +func (s *scannerImpl) scanFolders(ctx context.Context, fullScan bool, targets []ScanTarget, progress chan<- *ProgressInfo) { + startTime := time.Now() + + state := scanState{ + progress: progress, + fullScan: fullScan, + changesDetected: atomic.Bool{}, + targets: make(map[int][]string), + } + + // Set changesDetected to true for full scans to ensure all maintenance operations run + if fullScan { + state.changesDetected.Store(true) + } + + // Group targets by library and collect affected library IDs + affectedLibIDSet := make(map[int]bool) + for _, target := range targets { + folderPath := target.FolderPath + if folderPath == "" { + folderPath = "." + } + state.targets[target.LibraryID] = append(state.targets[target.LibraryID], folderPath) + affectedLibIDSet[target.LibraryID] = true + } + + // Get affected libraries + allLibs, err := s.ds.Library(ctx).GetAll() + if err != nil { + state.sendWarning(fmt.Sprintf("getting libraries: %s", err)) + return + } + + var libs model.Libraries + for _, lib := range allLibs { + if affectedLibIDSet[lib.ID] { + libs = append(libs, lib) + state.affectedLibIDs = append(state.affectedLibIDs, lib.ID) + } + } + state.libraries = libs + + log.Info(ctx, "Scanner: Starting selective scan", "fullScan", state.fullScan, "numLibraries", len(libs), "numTargets", len(targets)) + + // Store scan type and start time + scanType := "quick-selective" + if state.fullScan { + scanType = "full-selective" + } + _ = s.ds.Property(ctx).Put(consts.LastScanTypeKey, scanType) + _ = s.ds.Property(ctx).Put(consts.LastScanStartTimeKey, startTime.Format(time.RFC3339)) + + // if there was a full scan in progress, force a full scan + if !state.fullScan { + for _, lib := range libs { + if lib.FullScanInProgress { + log.Info(ctx, "Scanner: Interrupted full scan detected", "lib", lib.Name) + state.fullScan = true + _ = s.ds.Property(ctx).Put(consts.LastScanTypeKey, "full-selective") + break + } + } + } + + err = run.Sequentially( + // Phase 1: Scan specified folders and import new/updated files + runPhase[*folderEntry](ctx, 1, createPhaseFolders(ctx, &state, s.ds, s.cw, libs)), + + // Phase 2: Process missing files in scanned folders only + runPhase[*missingTracks](ctx, 2, createPhaseMissingTracks(ctx, &state, s.ds)), + + // Phases 3 and 4 can be run in parallel + run.Parallel( + // Phase 3: Refresh all new/changed albums (from affected libraries only) + runPhase[*model.Album](ctx, 3, createPhaseRefreshAlbums(ctx, &state, s.ds, libs)), + + // Phase 4: Import/update playlists (from affected libraries only) + runPhase[*model.Folder](ctx, 4, createPhasePlaylists(ctx, &state, s.ds, s.pls, s.cw)), + ), + + // Final Steps (cannot be parallelized): + + // Run GC scoped to affected libraries only + s.runGC(ctx, &state), + + // Refresh artist and tags stats + s.runRefreshStats(ctx, &state), + + // Update last_scan_completed_at for affected libraries + s.runUpdateLibraries(ctx, &state), + + // Optimize DB + s.runOptimize(ctx), + ) + if err != nil { + log.Error(ctx, "Scanner: Finished with error", "duration", time.Since(startTime), err) + _ = s.ds.Property(ctx).Put(consts.LastScanErrorKey, err.Error()) + state.sendError(err) + return + } + + _ = s.ds.Property(ctx).Put(consts.LastScanErrorKey, "") + + if state.changesDetected.Load() { + state.sendProgress(&ProgressInfo{ChangesDetected: true}) + } + + log.Info(ctx, "Scanner: Finished scanning selected folders", "duration", time.Since(startTime), "numTargets", len(targets)) +} + func (s *scannerImpl) runGC(ctx context.Context, state *scanState) func() error { return func() error { state.sendProgress(&ProgressInfo{ForceUpdate: true}) return s.ds.WithTx(func(tx model.DataStore) error { if state.changesDetected.Load() { start := time.Now() - err := tx.GC(ctx) + err := tx.GC(ctx, state.affectedLibIDs...) if err != nil { log.Error(ctx, "Scanner: Error running GC", err) return fmt.Errorf("running GC: %w", err) diff --git a/scanner/scanner_test.go b/scanner/scanner_test.go index e7e354f21..1e0614573 100644 --- a/scanner/scanner_test.go +++ b/scanner/scanner_test.go @@ -717,6 +717,61 @@ var _ = Describe("Scanner", Ordered, func() { Expect(albumArtistStats.SongCount).To(Equal(3)) // 3 songs }) }) + + Describe("ScanFolders", func() { + It("scans only specified folders without recursion", func() { + rock := template(_t{"albumartist": "Rock Artist", "album": "Rock Album"}) + jazz := template(_t{"albumartist": "Jazz Artist", "album": "Jazz Album"}) + pop := template(_t{"albumartist": "Pop Artist", "album": "Pop Album"}) + createFS(fstest.MapFS{ + "rock/track1.mp3": rock(track(1, "Rock Track 1")), + "rock/track2.mp3": rock(track(2, "Rock Track 2")), + "rock/subdir/track3.mp3": rock(track(3, "Rock Track 3")), + "jazz/track4.mp3": jazz(track(1, "Jazz Track 1")), + "jazz/subdir/track5.mp3": jazz(track(2, "Jazz Track 2")), + "pop/track6.mp3": pop(track(1, "Pop Track 1")), + }) + + // Use the existing library from BeforeEach + // (lib is already created with the path "fake:///music") + + // Scan only the "rock" and "jazz" folders (not their subdirectories or pop) + targets := []scanner.ScanTarget{ + {LibraryID: lib.ID, FolderPath: "rock"}, + {LibraryID: lib.ID, FolderPath: "jazz"}, + } + + warnings, err := s.ScanFolders(ctx, false, targets) + Expect(err).ToNot(HaveOccurred()) + Expect(warnings).To(BeEmpty()) + + // Verify only track1, track2, and track4 were imported (not track3, track5, or track6) + allFiles, err := ds.MediaFile(ctx).GetAll() + Expect(err).ToNot(HaveOccurred()) + + // Should have exactly 3 tracks (rock/track1, rock/track2, jazz/track4) + Expect(allFiles).To(HaveLen(3)) + + // Get the file paths + paths := slice.Map(allFiles, func(mf model.MediaFile) string { + return filepath.ToSlash(mf.Path) + }) + + // Verify the correct files were scanned + Expect(paths).To(ContainElements( + "rock/track1.mp3", + "rock/track2.mp3", + "jazz/track4.mp3", + )) + + // Verify files in subdirectories and pop folder were NOT scanned + Expect(paths).ToNot(ContainElements( + "rock/subdir/track3.mp3", + "jazz/subdir/track5.mp3", + "pop/track6.mp3", + )) + }) + }) }) func createFindByPath(ctx context.Context, ds model.DataStore) func(string) (*model.MediaFile, error) { diff --git a/scanner/walk_dir_tree.go b/scanner/walk_dir_tree.go index 63854d262..6afe76755 100644 --- a/scanner/walk_dir_tree.go +++ b/scanner/walk_dir_tree.go @@ -32,6 +32,64 @@ func walkDirTree(ctx context.Context, job *scanJob) (<-chan *folderEntry, error) return results, nil } +// loadSpecificFolders loads only the specified folders without recursing into subdirectories +func loadSpecificFolders(ctx context.Context, job *scanJob, targetFolders []string) (<-chan *folderEntry, error) { + results := make(chan *folderEntry) + go func() { + defer close(results) + for _, folderPath := range targetFolders { + if utils.IsCtxDone(ctx) { + return + } + + // Load ignore patterns from parent directories up to this folder + ignorePatterns := loadIgnorePatternsForPath(ctx, job.fs, folderPath) + + // Load only this specific folder (no recursion) + folder, _, err := loadDir(ctx, job, folderPath, ignorePatterns) + if err != nil { + log.Warn(ctx, "Scanner: Error loading target folder. Skipping", "path", folderPath, err) + continue + } + + folder.path = path.Clean(folderPath) + folder.elapsed.Start() + log.Trace(ctx, "Scanner: Found target directory", " path", folder.path, "audioFiles", maps.Keys(folder.audioFiles), + "images", maps.Keys(folder.imageFiles), "playlists", folder.numPlaylists, "imagesUpdatedAt", folder.imagesUpdatedAt, + "updTime", folder.updTime, "modTime", folder.modTime) + + results <- folder + } + log.Debug(ctx, "Scanner: Finished reading target folders", "lib", job.lib.Name, "path", job.lib.Path, "numFolders", len(targetFolders)) + }() + return results, nil +} + +// loadIgnorePatternsForPath loads all .ndignore patterns from the root down to the specified path +func loadIgnorePatternsForPath(ctx context.Context, fsys fs.FS, targetPath string) []string { + var patterns []string + currentPath := "." + + // If target is root, just check root + if targetPath == "." { + return loadIgnoredPatterns(ctx, fsys, ".", nil) + } + + // Walk from root to target, collecting ignore patterns + parts := strings.Split(path.Clean(targetPath), "/") + for _, part := range parts { + if part == "." { + continue + } + patterns = loadIgnoredPatterns(ctx, fsys, currentPath, patterns) + currentPath = path.Join(currentPath, part) + } + // Load patterns from the target folder itself + patterns = loadIgnoredPatterns(ctx, fsys, currentPath, patterns) + + return patterns +} + func walkFolder(ctx context.Context, job *scanJob, currentFolder string, ignorePatterns []string, results chan<- *folderEntry) error { ignorePatterns = loadIgnoredPatterns(ctx, job.fs, currentFolder, ignorePatterns) diff --git a/tests/mock_data_store.go b/tests/mock_data_store.go index 56f68a74b..2c0c90f62 100644 --- a/tests/mock_data_store.go +++ b/tests/mock_data_store.go @@ -258,6 +258,6 @@ func (db *MockDataStore) Resource(ctx context.Context, m any) model.ResourceRepo } } -func (db *MockDataStore) GC(context.Context) error { +func (db *MockDataStore) GC(context.Context, ...int) error { return nil } From f1dbd880247fc43135fea44c6c6cb06d574641d6 Mon Sep 17 00:00:00 2001 From: Deluan Date: Mon, 10 Nov 2025 19:25:59 -0500 Subject: [PATCH 02/40] feat(folder): replace GetByPaths with GetFolderUpdateInfo for improved folder updates retrieval Signed-off-by: Deluan --- model/folder.go | 3 +- persistence/folder_repository.go | 82 ++++---------- persistence/folder_repository_test.go | 149 ++++++++++---------------- scanner/phase_1_folders.go | 15 +-- 4 files changed, 76 insertions(+), 173 deletions(-) diff --git a/model/folder.go b/model/folder.go index 7ac2bf031..ba8db9cc1 100644 --- a/model/folder.go +++ b/model/folder.go @@ -83,10 +83,9 @@ type FolderUpdateInfo struct { type FolderRepository interface { Get(id string) (*Folder, error) GetByPath(lib Library, path string) (*Folder, error) - GetByPaths(targets []LibraryPath) (map[string]FolderUpdateInfo, error) GetAll(...QueryOptions) ([]Folder, error) CountAll(...QueryOptions) (int64, error) - GetLastUpdates(lib Library) (map[string]FolderUpdateInfo, error) + GetFolderUpdateInfo(lib Library, targetPaths ...string) (map[string]FolderUpdateInfo, error) Put(*Folder) error MarkMissing(missing bool, ids ...string) error GetTouchedWithPlaylists() (FolderCursor, error) diff --git a/persistence/folder_repository.go b/persistence/folder_repository.go index 646c7e61d..1a4caae57 100644 --- a/persistence/folder_repository.go +++ b/persistence/folder_repository.go @@ -78,67 +78,6 @@ func (r folderRepository) GetByPath(lib model.Library, path string) (*model.Fold return r.Get(id) } -func (r folderRepository) GetByPaths(targets []model.LibraryPath) (map[string]model.FolderUpdateInfo, error) { - if len(targets) == 0 { - return make(map[string]model.FolderUpdateInfo), nil - } - - // Group targets by library to build efficient queries - targetsByLib := make(map[int][]string) - folderIDs := make([]string, 0, len(targets)) - - // We need to resolve library paths to generate folder IDs - // Get all libraries first - libRepo := NewLibraryRepository(r.ctx, r.db) - allLibs, err := libRepo.GetAll() - if err != nil { - return nil, fmt.Errorf("getting libraries: %w", err) - } - libMap := make(map[int]model.Library) - for _, lib := range allLibs { - libMap[lib.ID] = lib - } - - // Generate folder IDs for all targets - for _, target := range targets { - lib, ok := libMap[target.LibraryID] - if !ok { - continue // Skip invalid library IDs - } - folderPath := target.FolderPath - if folderPath == "" { - folderPath = "." - } - folderID := model.FolderID(lib, folderPath) - folderIDs = append(folderIDs, folderID) - targetsByLib[target.LibraryID] = append(targetsByLib[target.LibraryID], folderPath) - } - - if len(folderIDs) == 0 { - return make(map[string]model.FolderUpdateInfo), nil - } - - // Query folders by IDs - sq := r.newSelect().Columns("id", "updated_at", "hash").Where(And{ - Eq{"id": folderIDs}, - Eq{"missing": false}, - }) - var res []struct { - ID string - UpdatedAt time.Time - Hash string - } - err = r.queryAll(sq, &res) - if err != nil { - return nil, err - } - m := make(map[string]model.FolderUpdateInfo, len(res)) - for _, f := range res { - m[f.ID] = model.FolderUpdateInfo{UpdatedAt: f.UpdatedAt, Hash: f.Hash} - } - return m, nil -} - func (r folderRepository) GetAll(opt ...model.QueryOptions) ([]model.Folder, error) { sq := r.selectFolder(opt...) var res dbFolders @@ -152,8 +91,25 @@ func (r folderRepository) CountAll(opt ...model.QueryOptions) (int64, error) { return r.count(query) } -func (r folderRepository) GetLastUpdates(lib model.Library) (map[string]model.FolderUpdateInfo, error) { - sq := r.newSelect().Columns("id", "updated_at", "hash").Where(Eq{"library_id": lib.ID, "missing": false}) +func (r folderRepository) GetFolderUpdateInfo(lib model.Library, targetPaths ...string) (map[string]model.FolderUpdateInfo, error) { + where := And{ + Eq{"library_id": lib.ID}, + Eq{"missing": false}, + } + + // If specific paths are requested, generate folder IDs and filter by them + if len(targetPaths) > 0 { + folderIDs := make([]string, 0, len(targetPaths)) + for _, path := range targetPaths { + if path == "" { + path = "." + } + folderIDs = append(folderIDs, model.FolderID(lib, path)) + } + where = append(where, Eq{"id": folderIDs}) + } + + sq := r.newSelect().Columns("id", "updated_at", "hash").Where(where) var res []struct { ID string UpdatedAt time.Time diff --git a/persistence/folder_repository_test.go b/persistence/folder_repository_test.go index 93190c494..166797933 100644 --- a/persistence/folder_repository_test.go +++ b/persistence/folder_repository_test.go @@ -2,6 +2,7 @@ package persistence import ( "context" + "fmt" "github.com/navidrome/navidrome/log" "github.com/navidrome/navidrome/model" @@ -15,7 +16,7 @@ var _ = Describe("FolderRepository", func() { var repo model.FolderRepository var ctx context.Context var conn *dbx.DB - var testLib model.Library + var testLib, otherLib model.Library BeforeEach(func() { ctx = request.WithUser(log.NewContext(context.TODO()), model.User{ID: "userid"}) @@ -27,21 +28,52 @@ var _ = Describe("FolderRepository", func() { lib, err := libRepo.Get(1) Expect(err).ToNot(HaveOccurred()) testLib = *lib + + // Create a second library with its own folder to verify isolation + otherLib = model.Library{Name: "Other Library", Path: "/other/path"} + Expect(libRepo.Put(&otherLib)).To(Succeed()) }) AfterEach(func() { - // Clean up test folders created by these tests - // Only delete folders with paths starting with our test prefix - _, _ = conn.NewQuery("DELETE FROM folder WHERE library_id = 1 AND (path LIKE 'TestFolder%' OR path LIKE 'Music/%' OR path = 'Classical' OR path = 'Podcasts')").Execute() + // Clean up only test folders created by our tests (paths starting with "Test") + // This prevents interference with fixture data needed by other tests + _, _ = conn.NewQuery("DELETE FROM folder WHERE library_id = 1 AND path LIKE 'Test%'").Execute() + _, _ = conn.NewQuery(fmt.Sprintf("DELETE FROM library WHERE id = %d", otherLib.ID)).Execute() }) - Describe("GetByPaths", func() { - Context("with valid targets", func() { + Describe("GetFolderUpdateInfo", func() { + Context("with no target paths", func() { + It("returns all folders in the library", func() { + // Create test folders with unique names to avoid conflicts + folder1 := model.NewFolder(testLib, "TestGetLastUpdates/Folder1") + folder2 := model.NewFolder(testLib, "TestGetLastUpdates/Folder2") + + err := repo.Put(folder1) + Expect(err).ToNot(HaveOccurred()) + err = repo.Put(folder2) + Expect(err).ToNot(HaveOccurred()) + + otherFolder := model.NewFolder(otherLib, "TestOtherLib/Folder") + err = repo.Put(otherFolder) + Expect(err).ToNot(HaveOccurred()) + + // Query all folders (no target paths) - should only return folders from testLib + results, err := repo.GetFolderUpdateInfo(testLib) + Expect(err).ToNot(HaveOccurred()) + // Should include folders from testLib + Expect(results).To(HaveKey(folder1.ID)) + Expect(results).To(HaveKey(folder2.ID)) + // Should NOT include folders from other library + Expect(results).ToNot(HaveKey(otherFolder.ID)) + }) + }) + + Context("with specific target paths", func() { It("returns folder info for existing folders", func() { - // Create test folders - folder1 := model.NewFolder(testLib, "Music/Rock") - folder2 := model.NewFolder(testLib, "Music/Jazz") - folder3 := model.NewFolder(testLib, "Classical") + // Create test folders with unique names + folder1 := model.NewFolder(testLib, "TestSpecific/Rock") + folder2 := model.NewFolder(testLib, "TestSpecific/Jazz") + folder3 := model.NewFolder(testLib, "TestSpecific/Classical") err := repo.Put(folder1) Expect(err).ToNot(HaveOccurred()) @@ -50,13 +82,8 @@ var _ = Describe("FolderRepository", func() { err = repo.Put(folder3) Expect(err).ToNot(HaveOccurred()) - // Query by paths - targets := []model.LibraryPath{ - {LibraryID: testLib.ID, FolderPath: "Music/Rock"}, - {LibraryID: testLib.ID, FolderPath: "Classical"}, - } - - results, err := repo.GetByPaths(targets) + // Query specific paths + results, err := repo.GetFolderUpdateInfo(testLib, "TestSpecific/Rock", "TestSpecific/Classical") Expect(err).ToNot(HaveOccurred()) Expect(results).To(HaveLen(2)) @@ -71,102 +98,34 @@ var _ = Describe("FolderRepository", func() { }) It("handles empty folder path as root", func() { - // Create root folder - rootFolder := model.NewFolder(testLib, ".") - err := repo.Put(rootFolder) - Expect(err).ToNot(HaveOccurred()) + // Test querying for root folder without creating it (fixtures should have one) + rootFolderID := model.FolderID(testLib, ".") - targets := []model.LibraryPath{ - {LibraryID: testLib.ID, FolderPath: ""}, + results, err := repo.GetFolderUpdateInfo(testLib, "") + Expect(err).ToNot(HaveOccurred()) + // Should return the root folder if it exists + if len(results) > 0 { + Expect(results).To(HaveKey(rootFolderID)) } - - results, err := repo.GetByPaths(targets) - Expect(err).ToNot(HaveOccurred()) - Expect(results).To(HaveLen(1)) - Expect(results).To(HaveKey(rootFolder.ID)) }) It("returns empty map for non-existent folders", func() { - targets := []model.LibraryPath{ - {LibraryID: testLib.ID, FolderPath: "NonExistent/Path"}, - } - - results, err := repo.GetByPaths(targets) + results, err := repo.GetFolderUpdateInfo(testLib, "NonExistent/Path") Expect(err).ToNot(HaveOccurred()) Expect(results).To(BeEmpty()) }) It("skips missing folders", func() { // Create a folder and mark it as missing - folder := model.NewFolder(testLib, "Music/Missing") + folder := model.NewFolder(testLib, "TestMissing/Folder") folder.Missing = true err := repo.Put(folder) Expect(err).ToNot(HaveOccurred()) - targets := []model.LibraryPath{ - {LibraryID: testLib.ID, FolderPath: "Music/Missing"}, - } - - results, err := repo.GetByPaths(targets) + results, err := repo.GetFolderUpdateInfo(testLib, "TestMissing/Folder") Expect(err).ToNot(HaveOccurred()) Expect(results).To(BeEmpty()) }) }) - - Context("with invalid library IDs", func() { - It("returns empty map for non-existent library", func() { - targets := []model.LibraryPath{ - {LibraryID: 99999, FolderPath: "Music"}, - } - - results, err := repo.GetByPaths(targets) - Expect(err).ToNot(HaveOccurred()) - Expect(results).To(BeEmpty()) - }) - }) - - Context("with empty targets", func() { - It("returns empty map", func() { - results, err := repo.GetByPaths([]model.LibraryPath{}) - Expect(err).ToNot(HaveOccurred()) - Expect(results).To(BeEmpty()) - }) - - It("returns empty map for nil targets", func() { - results, err := repo.GetByPaths(nil) - Expect(err).ToNot(HaveOccurred()) - Expect(results).To(BeEmpty()) - }) - }) - - Context("with multiple paths in same library", func() { - It("returns multiple folders", func() { - // Create multiple folders in the same library - folder1 := model.NewFolder(testLib, "Music/Pop") - folder2 := model.NewFolder(testLib, "Music/Electronic") - folder3 := model.NewFolder(testLib, "Podcasts") - - err := repo.Put(folder1) - Expect(err).ToNot(HaveOccurred()) - err = repo.Put(folder2) - Expect(err).ToNot(HaveOccurred()) - err = repo.Put(folder3) - Expect(err).ToNot(HaveOccurred()) - - // Query multiple paths - targets := []model.LibraryPath{ - {LibraryID: testLib.ID, FolderPath: "Music/Pop"}, - {LibraryID: testLib.ID, FolderPath: "Music/Electronic"}, - {LibraryID: testLib.ID, FolderPath: "Podcasts"}, - } - - results, err := repo.GetByPaths(targets) - Expect(err).ToNot(HaveOccurred()) - Expect(results).To(HaveLen(3)) - Expect(results).To(HaveKey(folder1.ID)) - Expect(results).To(HaveKey(folder2.ID)) - Expect(results).To(HaveKey(folder3.ID)) - }) - }) }) }) diff --git a/scanner/phase_1_folders.go b/scanner/phase_1_folders.go index ebdf74b54..22245f294 100644 --- a/scanner/phase_1_folders.go +++ b/scanner/phase_1_folders.go @@ -85,19 +85,8 @@ func newScanJob(ctx context.Context, ds model.DataStore, cw artwork.CacheWarmer, var lastUpdates map[string]model.FolderUpdateInfo var err error - // If we have target folders, get only those folder updates. Otherwise get all updates for the library - if len(targetFolders) > 0 { - var targets []model.LibraryPath - for _, folderPath := range targetFolders { - targets = append(targets, model.LibraryPath{ - LibraryID: lib.ID, - FolderPath: folderPath, - }) - } - lastUpdates, err = ds.Folder(ctx).GetByPaths(targets) - } else { - lastUpdates, err = ds.Folder(ctx).GetLastUpdates(lib) - } + // Get folder updates, optionally filtered to specific target folders + lastUpdates, err = ds.Folder(ctx).GetFolderUpdateInfo(lib, targetFolders...) if err != nil { return nil, fmt.Errorf("getting last updates: %w", err) } From 36a7040df3ffab4b6ea6fcd04d5c7e8051114739 Mon Sep 17 00:00:00 2001 From: Deluan Date: Mon, 10 Nov 2025 19:31:25 -0500 Subject: [PATCH 03/40] test: update parseTargets test to handle folder names with spaces Signed-off-by: Deluan --- cmd/scan_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/scan_test.go b/cmd/scan_test.go index 191e1096c..2844d2ed8 100644 --- a/cmd/scan_test.go +++ b/cmd/scan_test.go @@ -26,9 +26,10 @@ var _ = Describe("parseTargets", func() { }) It("handles targets with spaces around commas", func() { - targets, err := parseTargets("1:Music/Rock , 2:Jazz , 3:Classical") + targets, err := parseTargets("1:Music/Rock And Roll, 2:Jazz , 3:Classical") Expect(err).ToNot(HaveOccurred()) Expect(targets).To(HaveLen(3)) + Expect(targets[0].FolderPath).To(Equal("Music/Rock And Roll")) }) It("handles paths with colons after the first colon", func() { From 305148e1246ce0d94c198d92608bf7c7f8940e0d Mon Sep 17 00:00:00 2001 From: Deluan Date: Mon, 10 Nov 2025 19:39:20 -0500 Subject: [PATCH 04/40] refactor(folder): remove unused LibraryPath struct and update GC logging message Signed-off-by: Deluan --- model/folder.go | 6 ------ persistence/persistence.go | 2 +- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/model/folder.go b/model/folder.go index ba8db9cc1..7a769735e 100644 --- a/model/folder.go +++ b/model/folder.go @@ -90,9 +90,3 @@ type FolderRepository interface { MarkMissing(missing bool, ids ...string) error GetTouchedWithPlaylists() (FolderCursor, error) } - -// LibraryPath represents a folder path within a specific library -type LibraryPath struct { - LibraryID int - FolderPath string -} diff --git a/persistence/persistence.go b/persistence/persistence.go index b0c5a7f50..6db3f8575 100644 --- a/persistence/persistence.go +++ b/persistence/persistence.go @@ -170,7 +170,7 @@ func (s *SQLStore) GC(ctx context.Context, libraryIDs ...int) error { // TODO: Implement library-specific filtering for GC operations // For now, GC runs globally even in selective scans if len(libraryIDs) > 0 { - log.Debug(ctx, "GC: Running with library filter", "libraries", libraryIDs) + log.Debug(ctx, "GC: Running with library filter (not implemented)", "libraries", libraryIDs) } err := run.Sequentially( From 1250647dc3ba07b274501e5d84d7f04dc8095f35 Mon Sep 17 00:00:00 2001 From: Deluan Date: Mon, 10 Nov 2025 19:47:20 -0500 Subject: [PATCH 05/40] refactor(folder): enhance external scanner to support target-specific scanning Signed-off-by: Deluan --- core/maintenance_test.go | 4 +- scanner/external.go | 81 +++++++++++++++------------------------- 2 files changed, 33 insertions(+), 52 deletions(-) diff --git a/core/maintenance_test.go b/core/maintenance_test.go index 8e8796ffa..e83d1f8bd 100644 --- a/core/maintenance_test.go +++ b/core/maintenance_test.go @@ -373,10 +373,10 @@ type extendedDataStore struct { gcError error } -func (ds *extendedDataStore) GC(ctx context.Context) error { +func (ds *extendedDataStore) GC(ctx context.Context, libraryIDs ...int) error { ds.gcCalled = true if ds.gcError != nil { return ds.gcError } - return ds.MockDataStore.GC(ctx) + return ds.MockDataStore.GC(ctx, libraryIDs...) } diff --git a/scanner/external.go b/scanner/external.go index 86d171346..869034cc2 100644 --- a/scanner/external.go +++ b/scanner/external.go @@ -12,7 +12,6 @@ import ( "github.com/navidrome/navidrome/conf" "github.com/navidrome/navidrome/log" - . "github.com/navidrome/navidrome/utils/gg" ) // scannerExternal is a scanner that runs an external process to do the scanning. It is used to avoid @@ -25,68 +24,50 @@ import ( type scannerExternal struct{} func (s *scannerExternal) scanAll(ctx context.Context, fullScan bool, progress chan<- *ProgressInfo) { - exe, err := os.Executable() - if err != nil { - progress <- &ProgressInfo{Error: fmt.Sprintf("failed to get executable path: %s", err)} - return - } - log.Debug(ctx, "Spawning external scanner process", "fullScan", fullScan, "path", exe) - cmd := exec.CommandContext(ctx, exe, "scan", - "--nobanner", "--subprocess", - "--configfile", conf.Server.ConfigFile, - "--datafolder", conf.Server.DataFolder, - "--cachefolder", conf.Server.CacheFolder, - If(fullScan, "--full", "")) - - in, out := io.Pipe() - defer in.Close() - defer out.Close() - cmd.Stdout = out - cmd.Stderr = os.Stderr - - if err := cmd.Start(); err != nil { - progress <- &ProgressInfo{Error: fmt.Sprintf("failed to start scanner process: %s", err)} - return - } - go s.wait(cmd, out) - - decoder := gob.NewDecoder(in) - for { - var p ProgressInfo - if err := decoder.Decode(&p); err != nil { - if !errors.Is(err, io.EOF) { - progress <- &ProgressInfo{Error: fmt.Sprintf("failed to read status from scanner: %s", err)} - } - break - } - progress <- &p - } + s.scan(ctx, fullScan, nil, progress) } func (s *scannerExternal) scanFolders(ctx context.Context, fullScan bool, targets []ScanTarget, progress chan<- *ProgressInfo) { + s.scan(ctx, fullScan, targets, progress) +} + +func (s *scannerExternal) scan(ctx context.Context, fullScan bool, targets []ScanTarget, progress chan<- *ProgressInfo) { exe, err := os.Executable() if err != nil { progress <- &ProgressInfo{Error: fmt.Sprintf("failed to get executable path: %s", err)} return } - // Build targets string for CLI - var targetsStr string - for i, target := range targets { - if i > 0 { - targetsStr += "," - } - targetsStr += strconv.Itoa(target.LibraryID) + ":" + target.FolderPath - } - - log.Debug(ctx, "Spawning external scanner process with targets", "fullScan", fullScan, "path", exe, "targets", targetsStr) - cmd := exec.CommandContext(ctx, exe, "scan", + // Build command arguments + args := []string{ + "scan", "--nobanner", "--subprocess", "--configfile", conf.Server.ConfigFile, "--datafolder", conf.Server.DataFolder, "--cachefolder", conf.Server.CacheFolder, - "--targets", targetsStr, - If(fullScan, "--full", "")) + } + + // Add targets if provided + if len(targets) > 0 { + var targetsStr string + for i, target := range targets { + if i > 0 { + targetsStr += "," + } + targetsStr += strconv.Itoa(target.LibraryID) + ":" + target.FolderPath + } + args = append(args, "--targets", targetsStr) + log.Debug(ctx, "Spawning external scanner process with targets", "fullScan", fullScan, "path", exe, "targets", targetsStr) + } else { + log.Debug(ctx, "Spawning external scanner process", "fullScan", fullScan, "path", exe) + } + + // Add full scan flag if needed + if fullScan { + args = append(args, "--full") + } + + cmd := exec.CommandContext(ctx, exe, args...) in, out := io.Pipe() defer in.Close() From 712bf2815e88571518090e3bd4a977afa0fba96f Mon Sep 17 00:00:00 2001 From: Deluan Date: Mon, 10 Nov 2025 19:56:02 -0500 Subject: [PATCH 06/40] refactor(scanner): simplify scanner methods Signed-off-by: Deluan --- scanner/phase_1_folders.go | 2 +- scanner/scanner.go | 163 +++++++++++-------------------------- 2 files changed, 50 insertions(+), 115 deletions(-) diff --git a/scanner/phase_1_folders.go b/scanner/phase_1_folders.go index 22245f294..20554fe97 100644 --- a/scanner/phase_1_folders.go +++ b/scanner/phase_1_folders.go @@ -76,7 +76,7 @@ type scanJob struct { fs storage.MusicFS cw artwork.CacheWarmer lastUpdates map[string]model.FolderUpdateInfo - targetFolders []string // Optional: specific folders to scan (non-recursive) + targetFolders []string // Specific folders to scan (non-recursive) lock sync.Mutex numFolders atomic.Int64 } diff --git a/scanner/scanner.go b/scanner/scanner.go index 3c4d41d31..7b3c14d4b 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -48,6 +48,10 @@ func (s *scanState) sendError(err error) { } func (s *scannerImpl) scanAll(ctx context.Context, fullScan bool, progress chan<- *ProgressInfo) { + s.scanFolders(ctx, fullScan, nil, progress) +} + +func (s *scannerImpl) scanFolders(ctx context.Context, fullScan bool, targets []ScanTarget, progress chan<- *ProgressInfo) { startTime := time.Now() state := scanState{ @@ -61,20 +65,53 @@ func (s *scannerImpl) scanAll(ctx context.Context, fullScan bool, progress chan< state.changesDetected.Store(true) } - libs, err := s.ds.Library(ctx).GetAll() + // Get libraries and optionally filter by targets + allLibs, err := s.ds.Library(ctx).GetAll() if err != nil { state.sendWarning(fmt.Sprintf("getting libraries: %s", err)) return } - state.libraries = libs - log.Info(ctx, "Scanner: Starting scan", "fullScan", state.fullScan, "numLibraries", len(libs)) + var libs model.Libraries + isSelectiveScan := len(targets) > 0 + + if isSelectiveScan { + // Selective scan: filter libraries and build targets map + state.targets = make(map[int][]string) + affectedLibIDSet := make(map[int]bool) + + for _, target := range targets { + folderPath := target.FolderPath + if folderPath == "" { + folderPath = "." + } + state.targets[target.LibraryID] = append(state.targets[target.LibraryID], folderPath) + affectedLibIDSet[target.LibraryID] = true + } + + for _, lib := range allLibs { + if affectedLibIDSet[lib.ID] { + libs = append(libs, lib) + state.affectedLibIDs = append(state.affectedLibIDs, lib.ID) + } + } + + log.Info(ctx, "Scanner: Starting selective scan", "fullScan", state.fullScan, "numLibraries", len(libs), "numTargets", len(targets)) + } else { + // Full library scan + libs = allLibs + log.Info(ctx, "Scanner: Starting scan", "fullScan", state.fullScan, "numLibraries", len(libs)) + } + state.libraries = libs // Store scan type and start time scanType := "quick" if state.fullScan { scanType = "full" } + if isSelectiveScan { + scanType += "-selective" + } _ = s.ds.Property(ctx).Put(consts.LastScanTypeKey, scanType) _ = s.ds.Property(ctx).Put(consts.LastScanStartTimeKey, startTime.Format(time.RFC3339)) @@ -84,7 +121,11 @@ func (s *scannerImpl) scanAll(ctx context.Context, fullScan bool, progress chan< if lib.FullScanInProgress { log.Info(ctx, "Scanner: Interrupted full scan detected", "lib", lib.Name) state.fullScan = true - _ = s.ds.Property(ctx).Put(consts.LastScanTypeKey, "full") + if isSelectiveScan { + _ = s.ds.Property(ctx).Put(consts.LastScanTypeKey, "full-selective") + } else { + _ = s.ds.Property(ctx).Put(consts.LastScanTypeKey, "full") + } break } } @@ -133,117 +174,11 @@ func (s *scannerImpl) scanAll(ctx context.Context, fullScan bool, progress chan< state.sendProgress(&ProgressInfo{ChangesDetected: true}) } - log.Info(ctx, "Scanner: Finished scanning all libraries", "duration", time.Since(startTime)) -} - -func (s *scannerImpl) scanFolders(ctx context.Context, fullScan bool, targets []ScanTarget, progress chan<- *ProgressInfo) { - startTime := time.Now() - - state := scanState{ - progress: progress, - fullScan: fullScan, - changesDetected: atomic.Bool{}, - targets: make(map[int][]string), + if isSelectiveScan { + log.Info(ctx, "Scanner: Finished scanning selected folders", "duration", time.Since(startTime), "numTargets", len(targets)) + } else { + log.Info(ctx, "Scanner: Finished scanning all libraries", "duration", time.Since(startTime)) } - - // Set changesDetected to true for full scans to ensure all maintenance operations run - if fullScan { - state.changesDetected.Store(true) - } - - // Group targets by library and collect affected library IDs - affectedLibIDSet := make(map[int]bool) - for _, target := range targets { - folderPath := target.FolderPath - if folderPath == "" { - folderPath = "." - } - state.targets[target.LibraryID] = append(state.targets[target.LibraryID], folderPath) - affectedLibIDSet[target.LibraryID] = true - } - - // Get affected libraries - allLibs, err := s.ds.Library(ctx).GetAll() - if err != nil { - state.sendWarning(fmt.Sprintf("getting libraries: %s", err)) - return - } - - var libs model.Libraries - for _, lib := range allLibs { - if affectedLibIDSet[lib.ID] { - libs = append(libs, lib) - state.affectedLibIDs = append(state.affectedLibIDs, lib.ID) - } - } - state.libraries = libs - - log.Info(ctx, "Scanner: Starting selective scan", "fullScan", state.fullScan, "numLibraries", len(libs), "numTargets", len(targets)) - - // Store scan type and start time - scanType := "quick-selective" - if state.fullScan { - scanType = "full-selective" - } - _ = s.ds.Property(ctx).Put(consts.LastScanTypeKey, scanType) - _ = s.ds.Property(ctx).Put(consts.LastScanStartTimeKey, startTime.Format(time.RFC3339)) - - // if there was a full scan in progress, force a full scan - if !state.fullScan { - for _, lib := range libs { - if lib.FullScanInProgress { - log.Info(ctx, "Scanner: Interrupted full scan detected", "lib", lib.Name) - state.fullScan = true - _ = s.ds.Property(ctx).Put(consts.LastScanTypeKey, "full-selective") - break - } - } - } - - err = run.Sequentially( - // Phase 1: Scan specified folders and import new/updated files - runPhase[*folderEntry](ctx, 1, createPhaseFolders(ctx, &state, s.ds, s.cw, libs)), - - // Phase 2: Process missing files in scanned folders only - runPhase[*missingTracks](ctx, 2, createPhaseMissingTracks(ctx, &state, s.ds)), - - // Phases 3 and 4 can be run in parallel - run.Parallel( - // Phase 3: Refresh all new/changed albums (from affected libraries only) - runPhase[*model.Album](ctx, 3, createPhaseRefreshAlbums(ctx, &state, s.ds, libs)), - - // Phase 4: Import/update playlists (from affected libraries only) - runPhase[*model.Folder](ctx, 4, createPhasePlaylists(ctx, &state, s.ds, s.pls, s.cw)), - ), - - // Final Steps (cannot be parallelized): - - // Run GC scoped to affected libraries only - s.runGC(ctx, &state), - - // Refresh artist and tags stats - s.runRefreshStats(ctx, &state), - - // Update last_scan_completed_at for affected libraries - s.runUpdateLibraries(ctx, &state), - - // Optimize DB - s.runOptimize(ctx), - ) - if err != nil { - log.Error(ctx, "Scanner: Finished with error", "duration", time.Since(startTime), err) - _ = s.ds.Property(ctx).Put(consts.LastScanErrorKey, err.Error()) - state.sendError(err) - return - } - - _ = s.ds.Property(ctx).Put(consts.LastScanErrorKey, "") - - if state.changesDetected.Load() { - state.sendProgress(&ProgressInfo{ChangesDetected: true}) - } - - log.Info(ctx, "Scanner: Finished scanning selected folders", "duration", time.Since(startTime), "numTargets", len(targets)) } func (s *scannerImpl) runGC(ctx context.Context, state *scanState) func() error { From 1bd58a14abe3f28743f06562c30a897abceb40f7 Mon Sep 17 00:00:00 2001 From: Deluan Date: Mon, 10 Nov 2025 20:58:37 -0500 Subject: [PATCH 07/40] feat(watcher): implement folder scanning notifications with deduplication Signed-off-by: Deluan --- scanner/watcher.go | 59 ++++++- scanner/watcher_test.go | 370 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 422 insertions(+), 7 deletions(-) create mode 100644 scanner/watcher_test.go diff --git a/scanner/watcher.go b/scanner/watcher.go index 37cfb5e22..e8a0bae13 100644 --- a/scanner/watcher.go +++ b/scanner/watcher.go @@ -26,7 +26,7 @@ type watcher struct { ds model.DataStore scanner Scanner triggerWait time.Duration - watcherNotify chan model.Library + watcherNotify chan scanNotification libraryWatchers map[int]*libraryWatcherInstance mu sync.RWMutex } @@ -36,6 +36,11 @@ type libraryWatcherInstance struct { cancel context.CancelFunc } +type scanNotification struct { + Library *model.Library + FolderPath string +} + // GetWatcher returns the watcher singleton func GetWatcher(ds model.DataStore, s Scanner) Watcher { return singleton.GetInstance(func() *watcher { @@ -43,7 +48,7 @@ func GetWatcher(ds model.DataStore, s Scanner) Watcher { ds: ds, scanner: s, triggerWait: conf.Server.Scanner.WatcherWait, - watcherNotify: make(chan model.Library, 1), + watcherNotify: make(chan scanNotification, 1), libraryWatchers: make(map[int]*libraryWatcherInstance), } }) @@ -69,10 +74,11 @@ func (w *watcher) Run(ctx context.Context) error { trigger := time.NewTimer(w.triggerWait) trigger.Stop() waiting := false + targets := make(map[ScanTarget]struct{}) for { select { case <-trigger.C: - log.Info("Watcher: Triggering scan") + log.Info("Watcher: Triggering scan for changed folders", "numTargets", len(targets)) status, err := w.scanner.Status(ctx) if err != nil { log.Error(ctx, "Watcher: Error retrieving Scanner status", err) @@ -84,8 +90,18 @@ func (w *watcher) Run(ctx context.Context) error { continue } waiting = false + + // Convert targets map to slice + targetSlice := make([]ScanTarget, 0, len(targets)) + for target := range targets { + targetSlice = append(targetSlice, target) + } + + // Clear targets for next batch + targets = make(map[ScanTarget]struct{}) + go func() { - _, err := w.scanner.ScanAll(ctx, false) + _, err := w.scanner.ScanFolders(ctx, false, targetSlice) if err != nil { log.Error(ctx, "Watcher: Error scanning", err) } else { @@ -102,10 +118,16 @@ func (w *watcher) Run(ctx context.Context) error { w.libraryWatchers = make(map[int]*libraryWatcherInstance) w.mu.Unlock() return nil - case lib := <-w.watcherNotify: + case notification := <-w.watcherNotify: + lib := notification.Library + folderPath := notification.FolderPath + + // Add target to the map (deduplicates automatically) + targets[ScanTarget{LibraryID: lib.ID, FolderPath: folderPath}] = struct{}{} + if !waiting { log.Debug(ctx, "Watcher: Detected changes. Waiting for more changes before triggering scan", - "libraryID", lib.ID, "name", lib.Name, "path", lib.Path) + "libraryID", lib.ID, "name", lib.Name, "path", lib.Path, "folderPath", folderPath) waiting = true } trigger.Reset(w.triggerWait) @@ -218,9 +240,32 @@ func (w *watcher) watchLibrary(ctx context.Context, lib *model.Library) error { log.Trace(ctx, "Detected change", "libraryID", lib.ID, "path", path, "absoluteLibPath", absLibPath) + // Find the folder to scan - validate path exists as directory, walk up if needed + folderPath := path + for { + info, err := fs.Stat(fsys, folderPath) + if err == nil && info.IsDir() { + // Found a valid directory + break + } + if folderPath == "." || folderPath == "" { + // Reached root, scan entire library + folderPath = "" + break + } + // Walk up the tree + dir, _ := filepath.Split(folderPath) + if dir == "" || dir == "." { + folderPath = "" + break + } + // Remove trailing slash + folderPath = filepath.Clean(dir) + } + // Notify the main watcher of changes select { - case w.watcherNotify <- *lib: + case w.watcherNotify <- scanNotification{Library: lib, FolderPath: folderPath}: default: // Channel is full, notification already pending } diff --git a/scanner/watcher_test.go b/scanner/watcher_test.go new file mode 100644 index 000000000..d487a9c10 --- /dev/null +++ b/scanner/watcher_test.go @@ -0,0 +1,370 @@ +package scanner + +import ( + "context" + "sync" + "time" + + "github.com/navidrome/navidrome/conf" + "github.com/navidrome/navidrome/conf/configtest" + "github.com/navidrome/navidrome/model" + "github.com/navidrome/navidrome/tests" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Watcher", func() { + var ctx context.Context + var cancel context.CancelFunc + var mockScanner *MockScanner + var mockDS *tests.MockDataStore + var w *watcher + var lib *model.Library + + BeforeEach(func() { + DeferCleanup(configtest.SetupConfig()) + conf.Server.Scanner.WatcherWait = 50 * time.Millisecond // Short wait for tests + + ctx, cancel = context.WithCancel(context.Background()) + DeferCleanup(cancel) + + lib = &model.Library{ + ID: 1, + Name: "Test Library", + Path: "/test/library", + } + + // Set up mocks + mockScanner = NewMockScanner() + mockDS = &tests.MockDataStore{} + mockLibRepo := &tests.MockLibraryRepo{} + mockLibRepo.SetData(model.Libraries{*lib}) + mockDS.MockedLibrary = mockLibRepo + + // Create a new watcher instance (not singleton) for testing + w = &watcher{ + ds: mockDS, + scanner: mockScanner, + triggerWait: conf.Server.Scanner.WatcherWait, + watcherNotify: make(chan scanNotification, 10), + libraryWatchers: make(map[int]*libraryWatcherInstance), + mainCtx: ctx, + } + }) + + Describe("Target Collection and Deduplication", func() { + BeforeEach(func() { + // Start watcher in background + go func() { + _ = w.Run(ctx) + }() + + // Give watcher time to initialize + time.Sleep(10 * time.Millisecond) + }) + + It("creates separate targets for different folders", func() { + // Send notifications for different folders + w.watcherNotify <- scanNotification{Library: lib, FolderPath: "artist1"} + time.Sleep(10 * time.Millisecond) + w.watcherNotify <- scanNotification{Library: lib, FolderPath: "artist2"} + + // Wait for watcher to process and trigger scan + Eventually(func() int { + return mockScanner.GetScanFoldersCallCount() + }, 200*time.Millisecond, 10*time.Millisecond).Should(Equal(1)) + + // Verify two targets + calls := mockScanner.GetScanFoldersCalls() + Expect(calls).To(HaveLen(1)) + Expect(calls[0].Targets).To(HaveLen(2)) + + // Extract folder paths + folderPaths := make(map[string]bool) + for _, target := range calls[0].Targets { + Expect(target.LibraryID).To(Equal(1)) + folderPaths[target.FolderPath] = true + } + Expect(folderPaths).To(HaveKey("artist1")) + Expect(folderPaths).To(HaveKey("artist2")) + }) + + It("handles different folder paths correctly", func() { + // Send notification for nested folder + w.watcherNotify <- scanNotification{Library: lib, FolderPath: "artist1/album1"} + + // Wait for watcher to process and trigger scan + Eventually(func() int { + return mockScanner.GetScanFoldersCallCount() + }, 200*time.Millisecond, 10*time.Millisecond).Should(Equal(1)) + + // Verify the target + calls := mockScanner.GetScanFoldersCalls() + Expect(calls).To(HaveLen(1)) + Expect(calls[0].Targets).To(HaveLen(1)) + Expect(calls[0].Targets[0].FolderPath).To(Equal("artist1/album1")) + }) + + It("deduplicates folder and file within same folder", func() { + // Send notification for a folder + w.watcherNotify <- scanNotification{Library: lib, FolderPath: "artist1/album1"} + time.Sleep(10 * time.Millisecond) + // Send notification for same folder (as if file change was detected there) + // In practice, watchLibrary() would walk up from file path to folder + w.watcherNotify <- scanNotification{Library: lib, FolderPath: "artist1/album1"} + time.Sleep(10 * time.Millisecond) + // Send another for same folder + w.watcherNotify <- scanNotification{Library: lib, FolderPath: "artist1/album1"} + + // Wait for watcher to process and trigger scan + Eventually(func() int { + return mockScanner.GetScanFoldersCallCount() + }, 200*time.Millisecond, 10*time.Millisecond).Should(Equal(1)) + + // Verify only one target despite multiple file/folder changes + calls := mockScanner.GetScanFoldersCalls() + Expect(calls).To(HaveLen(1)) + Expect(calls[0].Targets).To(HaveLen(1)) + Expect(calls[0].Targets[0].FolderPath).To(Equal("artist1/album1")) + }) + }) + + Describe("Timer Behavior", func() { + BeforeEach(func() { + // Start watcher in background + go func() { + _ = w.Run(ctx) + }() + + // Give watcher time to initialize + time.Sleep(10 * time.Millisecond) + }) + + It("resets timer on each change (debouncing)", func() { + // Send first notification + w.watcherNotify <- scanNotification{Library: lib, FolderPath: "artist1"} + + // Wait half the watcher wait time + time.Sleep(25 * time.Millisecond) + + // No scan should have been triggered yet + Expect(mockScanner.GetScanFoldersCallCount()).To(Equal(0)) + + // Send another notification (resets timer) + w.watcherNotify <- scanNotification{Library: lib, FolderPath: "artist1"} + + // Wait half the watcher wait time again + time.Sleep(25 * time.Millisecond) + + // Still no scan + Expect(mockScanner.GetScanFoldersCallCount()).To(Equal(0)) + + // Wait for full timer to expire after last notification + time.Sleep(50 * time.Millisecond) + + // Now scan should have been triggered + Eventually(func() int { + return mockScanner.GetScanFoldersCallCount() + }, 100*time.Millisecond, 10*time.Millisecond).Should(Equal(1)) + }) + + It("triggers scan after quiet period", func() { + // Send notification + w.watcherNotify <- scanNotification{Library: lib, FolderPath: "artist1"} + + // No scan immediately + Expect(mockScanner.GetScanFoldersCallCount()).To(Equal(0)) + + // Wait for quiet period + Eventually(func() int { + return mockScanner.GetScanFoldersCallCount() + }, 200*time.Millisecond, 10*time.Millisecond).Should(Equal(1)) + }) + }) + + Describe("Empty and Root Paths", func() { + BeforeEach(func() { + // Start watcher in background + go func() { + _ = w.Run(ctx) + }() + + // Give watcher time to initialize + time.Sleep(10 * time.Millisecond) + }) + + It("handles empty folder path (library root)", func() { + // Send notification with empty folder path + w.watcherNotify <- scanNotification{Library: lib, FolderPath: ""} + + // Wait for scan + Eventually(func() int { + return mockScanner.GetScanFoldersCallCount() + }, 200*time.Millisecond, 10*time.Millisecond).Should(Equal(1)) + + // Should scan the library root + calls := mockScanner.GetScanFoldersCalls() + Expect(calls).To(HaveLen(1)) + Expect(calls[0].Targets).To(HaveLen(1)) + Expect(calls[0].Targets[0].FolderPath).To(Equal("")) + }) + + It("deduplicates empty and dot paths", func() { + // Send notifications with empty and dot paths + w.watcherNotify <- scanNotification{Library: lib, FolderPath: ""} + time.Sleep(10 * time.Millisecond) + w.watcherNotify <- scanNotification{Library: lib, FolderPath: ""} + + // Wait for scan + Eventually(func() int { + return mockScanner.GetScanFoldersCallCount() + }, 200*time.Millisecond, 10*time.Millisecond).Should(Equal(1)) + + // Should have only one target + calls := mockScanner.GetScanFoldersCalls() + Expect(calls).To(HaveLen(1)) + Expect(calls[0].Targets).To(HaveLen(1)) + }) + }) + + Describe("Multiple Libraries", func() { + var lib2 *model.Library + + BeforeEach(func() { + // Create second library + lib2 = &model.Library{ + ID: 2, + Name: "Test Library 2", + Path: "/test/library2", + } + + mockLibRepo := mockDS.MockedLibrary.(*tests.MockLibraryRepo) + mockLibRepo.SetData(model.Libraries{*lib, *lib2}) + + // Start watcher in background + go func() { + _ = w.Run(ctx) + }() + + // Give watcher time to initialize + time.Sleep(10 * time.Millisecond) + }) + + It("creates separate targets for different libraries", func() { + // Send notifications for both libraries + w.watcherNotify <- scanNotification{Library: lib, FolderPath: "artist1"} + time.Sleep(10 * time.Millisecond) + w.watcherNotify <- scanNotification{Library: lib2, FolderPath: "artist2"} + + // Wait for scan + Eventually(func() int { + return mockScanner.GetScanFoldersCallCount() + }, 200*time.Millisecond, 10*time.Millisecond).Should(Equal(1)) + + // Verify two targets for different libraries + calls := mockScanner.GetScanFoldersCalls() + Expect(calls).To(HaveLen(1)) + Expect(calls[0].Targets).To(HaveLen(2)) + + // Verify library IDs are different + libraryIDs := make(map[int]bool) + for _, target := range calls[0].Targets { + libraryIDs[target.LibraryID] = true + } + Expect(libraryIDs).To(HaveKey(1)) + Expect(libraryIDs).To(HaveKey(2)) + }) + }) +}) + +// MockScanner implements scanner.Scanner for testing +type MockScanner struct { + mu sync.Mutex + scanAllCalls []ScanAllCall + scanFoldersCalls []ScanFoldersCall + scanningStatus bool +} + +type ScanAllCall struct { + FullScan bool +} + +type ScanFoldersCall struct { + FullScan bool + Targets []ScanTarget +} + +func NewMockScanner() *MockScanner { + return &MockScanner{ + scanAllCalls: make([]ScanAllCall, 0), + scanFoldersCalls: make([]ScanFoldersCall, 0), + } +} + +func (m *MockScanner) ScanAll(_ context.Context, fullScan bool) ([]string, error) { + m.mu.Lock() + defer m.mu.Unlock() + + m.scanAllCalls = append(m.scanAllCalls, ScanAllCall{FullScan: fullScan}) + + return nil, nil +} + +func (m *MockScanner) ScanFolders(_ context.Context, fullScan bool, targets []ScanTarget) ([]string, error) { + m.mu.Lock() + defer m.mu.Unlock() + + // Make a copy of targets to avoid race conditions + targetsCopy := make([]ScanTarget, len(targets)) + copy(targetsCopy, targets) + + m.scanFoldersCalls = append(m.scanFoldersCalls, ScanFoldersCall{ + FullScan: fullScan, + Targets: targetsCopy, + }) + + return nil, nil +} + +func (m *MockScanner) Status(_ context.Context) (*StatusInfo, error) { + m.mu.Lock() + defer m.mu.Unlock() + + return &StatusInfo{ + Scanning: m.scanningStatus, + }, nil +} + +func (m *MockScanner) GetScanAllCallCount() int { + m.mu.Lock() + defer m.mu.Unlock() + return len(m.scanAllCalls) +} + +func (m *MockScanner) GetScanFoldersCallCount() int { + m.mu.Lock() + defer m.mu.Unlock() + return len(m.scanFoldersCalls) +} + +func (m *MockScanner) GetScanFoldersCalls() []ScanFoldersCall { + m.mu.Lock() + defer m.mu.Unlock() + // Return a copy to avoid race conditions + calls := make([]ScanFoldersCall, len(m.scanFoldersCalls)) + copy(calls, m.scanFoldersCalls) + return calls +} + +func (m *MockScanner) Reset() { + m.mu.Lock() + defer m.mu.Unlock() + m.scanAllCalls = make([]ScanAllCall, 0) + m.scanFoldersCalls = make([]ScanFoldersCall, 0) +} + +func (m *MockScanner) SetScanning(scanning bool) { + m.mu.Lock() + defer m.mu.Unlock() + m.scanningStatus = scanning +} From b361837f50caa877135aaeedaf4d6fa6f136a271 Mon Sep 17 00:00:00 2001 From: Deluan Date: Mon, 10 Nov 2025 21:23:28 -0500 Subject: [PATCH 08/40] refactor(watcher): add resolveFolderPath function for testability Signed-off-by: Deluan --- scanner/watcher.go | 68 +++++++++++++++++++++++------------------ scanner/watcher_test.go | 60 ++++++++++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+), 30 deletions(-) diff --git a/scanner/watcher.go b/scanner/watcher.go index e8a0bae13..e377f625a 100644 --- a/scanner/watcher.go +++ b/scanner/watcher.go @@ -73,7 +73,6 @@ func (w *watcher) Run(ctx context.Context) error { // Main scan triggering loop trigger := time.NewTimer(w.triggerWait) trigger.Stop() - waiting := false targets := make(map[ScanTarget]struct{}) for { select { @@ -89,7 +88,6 @@ func (w *watcher) Run(ctx context.Context) error { trigger.Reset(w.triggerWait * 3) continue } - waiting = false // Convert targets map to slice targetSlice := make([]ScanTarget, 0, len(targets)) @@ -122,15 +120,15 @@ func (w *watcher) Run(ctx context.Context) error { lib := notification.Library folderPath := notification.FolderPath - // Add target to the map (deduplicates automatically) - targets[ScanTarget{LibraryID: lib.ID, FolderPath: folderPath}] = struct{}{} - - if !waiting { - log.Debug(ctx, "Watcher: Detected changes. Waiting for more changes before triggering scan", - "libraryID", lib.ID, "name", lib.Name, "path", lib.Path, "folderPath", folderPath) - waiting = true + // If already scheduled for scan, skip + if _, exists := targets[ScanTarget{LibraryID: lib.ID, FolderPath: folderPath}]; exists { + continue } + targets[ScanTarget{LibraryID: lib.ID, FolderPath: folderPath}] = struct{}{} trigger.Reset(w.triggerWait) + + log.Debug(ctx, "Watcher: Detected changes. Waiting for more changes before triggering scan", + "libraryID", lib.ID, "name", lib.Name, "path", lib.Path, "folderPath", folderPath) } } } @@ -241,27 +239,7 @@ func (w *watcher) watchLibrary(ctx context.Context, lib *model.Library) error { log.Trace(ctx, "Detected change", "libraryID", lib.ID, "path", path, "absoluteLibPath", absLibPath) // Find the folder to scan - validate path exists as directory, walk up if needed - folderPath := path - for { - info, err := fs.Stat(fsys, folderPath) - if err == nil && info.IsDir() { - // Found a valid directory - break - } - if folderPath == "." || folderPath == "" { - // Reached root, scan entire library - folderPath = "" - break - } - // Walk up the tree - dir, _ := filepath.Split(folderPath) - if dir == "" || dir == "." { - folderPath = "" - break - } - // Remove trailing slash - folderPath = filepath.Clean(dir) - } + folderPath := resolveFolderPath(fsys, path) // Notify the main watcher of changes select { @@ -273,6 +251,36 @@ func (w *watcher) watchLibrary(ctx context.Context, lib *model.Library) error { } } +// resolveFolderPath takes a path (which may be a file or directory) and returns +// the folder path to scan. If the path is a file, it walks up to find the parent +// directory. Returns empty string if the path should scan the library root. +func resolveFolderPath(fsys fs.FS, path string) string { + // Handle root paths immediately + if path == "." || path == "" { + return "" + } + + folderPath := path + for { + info, err := fs.Stat(fsys, folderPath) + if err == nil && info.IsDir() { + // Found a valid directory + return folderPath + } + if folderPath == "." || folderPath == "" { + // Reached root, scan entire library + return "" + } + // Walk up the tree + dir, _ := filepath.Split(folderPath) + if dir == "" || dir == "." { + return "" + } + // Remove trailing slash + folderPath = filepath.Clean(dir) + } +} + func isIgnoredPath(_ context.Context, _ fs.FS, path string) bool { baseDir, name := filepath.Split(path) switch { diff --git a/scanner/watcher_test.go b/scanner/watcher_test.go index d487a9c10..6341797f6 100644 --- a/scanner/watcher_test.go +++ b/scanner/watcher_test.go @@ -2,7 +2,9 @@ package scanner import ( "context" + "io/fs" "sync" + "testing/fstest" "time" "github.com/navidrome/navidrome/conf" @@ -277,6 +279,64 @@ var _ = Describe("Watcher", func() { }) }) +var _ = Describe("resolveFolderPath", func() { + var mockFS fs.FS + + BeforeEach(func() { + // Create a mock filesystem with some directories and files + mockFS = fstest.MapFS{ + "artist1": &fstest.MapFile{Mode: fs.ModeDir}, + "artist1/album1": &fstest.MapFile{Mode: fs.ModeDir}, + "artist1/album1/track1.mp3": &fstest.MapFile{Data: []byte("audio")}, + "artist1/album1/track2.mp3": &fstest.MapFile{Data: []byte("audio")}, + "artist1/album2": &fstest.MapFile{Mode: fs.ModeDir}, + "artist1/album2/song.flac": &fstest.MapFile{Data: []byte("audio")}, + "artist2": &fstest.MapFile{Mode: fs.ModeDir}, + "artist2/cover.jpg": &fstest.MapFile{Data: []byte("image")}, + } + }) + + It("returns directory path when given a directory", func() { + result := resolveFolderPath(mockFS, "artist1/album1") + Expect(result).To(Equal("artist1/album1")) + }) + + It("walks up to parent directory when given a file path", func() { + result := resolveFolderPath(mockFS, "artist1/album1/track1.mp3") + Expect(result).To(Equal("artist1/album1")) + }) + + It("walks up multiple levels if needed", func() { + result := resolveFolderPath(mockFS, "artist1/album1/nonexistent/file.mp3") + Expect(result).To(Equal("artist1/album1")) + }) + + It("returns empty string for non-existent paths at root", func() { + result := resolveFolderPath(mockFS, "nonexistent/path/file.mp3") + Expect(result).To(Equal("")) + }) + + It("returns empty string for dot path", func() { + result := resolveFolderPath(mockFS, ".") + Expect(result).To(Equal("")) + }) + + It("returns empty string for empty path", func() { + result := resolveFolderPath(mockFS, "") + Expect(result).To(Equal("")) + }) + + It("handles nested file paths correctly", func() { + result := resolveFolderPath(mockFS, "artist1/album2/song.flac") + Expect(result).To(Equal("artist1/album2")) + }) + + It("resolves to top-level directory", func() { + result := resolveFolderPath(mockFS, "artist2/cover.jpg") + Expect(result).To(Equal("artist2")) + }) +}) + // MockScanner implements scanner.Scanner for testing type MockScanner struct { mu sync.Mutex From 3cbbb440c2db95ab831c34000f73c8445d3dfa27 Mon Sep 17 00:00:00 2001 From: Deluan Date: Mon, 10 Nov 2025 22:46:48 -0500 Subject: [PATCH 09/40] feat(watcher): implement path ignoring based on .ndignore patterns Signed-off-by: Deluan --- scanner/walk_dir_tree.go | 61 ++++++++++----------- scanner/watcher.go | 30 +++++++++++ scanner/watcher_test.go | 113 ++++++++++++++++++++++++++++++++++++--- 3 files changed, 168 insertions(+), 36 deletions(-) diff --git a/scanner/walk_dir_tree.go b/scanner/walk_dir_tree.go index 6afe76755..d374a4aca 100644 --- a/scanner/walk_dir_tree.go +++ b/scanner/walk_dir_tree.go @@ -43,7 +43,7 @@ func loadSpecificFolders(ctx context.Context, job *scanJob, targetFolders []stri } // Load ignore patterns from parent directories up to this folder - ignorePatterns := loadIgnorePatternsForPath(ctx, job.fs, folderPath) + ignorePatterns := loadIgnoredPatternsForPath(ctx, job.fs, folderPath) // Load only this specific folder (no recursion) folder, _, err := loadDir(ctx, job, folderPath, ignorePatterns) @@ -65,8 +65,8 @@ func loadSpecificFolders(ctx context.Context, job *scanJob, targetFolders []stri return results, nil } -// loadIgnorePatternsForPath loads all .ndignore patterns from the root down to the specified path -func loadIgnorePatternsForPath(ctx context.Context, fsys fs.FS, targetPath string) []string { +// loadIgnoredPatternsForPath loads all .ndignore patterns from the root down to the specified path +func loadIgnoredPatternsForPath(ctx context.Context, fsys fs.FS, targetPath string) []string { var patterns []string currentPath := "." @@ -90,33 +90,7 @@ func loadIgnorePatternsForPath(ctx context.Context, fsys fs.FS, targetPath strin return patterns } -func walkFolder(ctx context.Context, job *scanJob, currentFolder string, ignorePatterns []string, results chan<- *folderEntry) error { - ignorePatterns = loadIgnoredPatterns(ctx, job.fs, currentFolder, ignorePatterns) - - folder, children, err := loadDir(ctx, job, currentFolder, ignorePatterns) - if err != nil { - log.Warn(ctx, "Scanner: Error loading dir. Skipping", "path", currentFolder, err) - return nil - } - for _, c := range children { - err := walkFolder(ctx, job, c, ignorePatterns, results) - if err != nil { - return err - } - } - - dir := path.Clean(currentFolder) - log.Trace(ctx, "Scanner: Found directory", " path", dir, "audioFiles", maps.Keys(folder.audioFiles), - "images", maps.Keys(folder.imageFiles), "playlists", folder.numPlaylists, "imagesUpdatedAt", folder.imagesUpdatedAt, - "updTime", folder.updTime, "modTime", folder.modTime, "numChildren", len(children)) - folder.path = dir - folder.elapsed.Start() - - results <- folder - - return nil -} - +// loadIgnoredPatterns loads .ndignore patterns from the specified folder and combines them with currentPatterns func loadIgnoredPatterns(ctx context.Context, fsys fs.FS, currentFolder string, currentPatterns []string) []string { ignoreFilePath := path.Join(currentFolder, consts.ScanIgnoreFile) var newPatterns []string @@ -153,6 +127,33 @@ func loadIgnoredPatterns(ctx context.Context, fsys fs.FS, currentFolder string, return append(combinedPatterns, newPatterns...) } +func walkFolder(ctx context.Context, job *scanJob, currentFolder string, ignorePatterns []string, results chan<- *folderEntry) error { + ignorePatterns = loadIgnoredPatterns(ctx, job.fs, currentFolder, ignorePatterns) + + folder, children, err := loadDir(ctx, job, currentFolder, ignorePatterns) + if err != nil { + log.Warn(ctx, "Scanner: Error loading dir. Skipping", "path", currentFolder, err) + return nil + } + for _, c := range children { + err := walkFolder(ctx, job, c, ignorePatterns, results) + if err != nil { + return err + } + } + + dir := path.Clean(currentFolder) + log.Trace(ctx, "Scanner: Found directory", " path", dir, "audioFiles", maps.Keys(folder.audioFiles), + "images", maps.Keys(folder.imageFiles), "playlists", folder.numPlaylists, "imagesUpdatedAt", folder.imagesUpdatedAt, + "updTime", folder.updTime, "modTime", folder.modTime, "numChildren", len(children)) + folder.path = dir + folder.elapsed.Start() + + results <- folder + + return nil +} + func loadDir(ctx context.Context, job *scanJob, dirPath string, ignorePatterns []string) (folder *folderEntry, children []string, err error) { folder = newFolderEntry(job, dirPath) diff --git a/scanner/watcher.go b/scanner/watcher.go index e377f625a..43827df22 100644 --- a/scanner/watcher.go +++ b/scanner/watcher.go @@ -13,6 +13,7 @@ import ( "github.com/navidrome/navidrome/log" "github.com/navidrome/navidrome/model" "github.com/navidrome/navidrome/utils/singleton" + ignore "github.com/sabhiram/go-gitignore" ) type Watcher interface { @@ -241,6 +242,12 @@ func (w *watcher) watchLibrary(ctx context.Context, lib *model.Library) error { // Find the folder to scan - validate path exists as directory, walk up if needed folderPath := resolveFolderPath(fsys, path) + // Check if the folder should be ignored based on .ndignore patterns + if shouldIgnorePath(ctx, fsys, folderPath) { + log.Trace(ctx, "Ignoring change in folder matching .ndignore pattern", "libraryID", lib.ID, "folderPath", folderPath) + continue + } + // Notify the main watcher of changes select { case w.watcherNotify <- scanNotification{Library: lib, FolderPath: folderPath}: @@ -297,3 +304,26 @@ func isIgnoredPath(_ context.Context, _ fs.FS, path string) bool { // But at this point, we can assume it's a directory. If it's a file, it would be ignored anyway return isDirIgnored(baseDir) } + +// shouldIgnorePath checks if the given path should be ignored based on .ndignore patterns. +// It loads all .ndignore files from the root down to the path and returns true if the path +// matches any ignore pattern. This function is suitable for checking paths without recursion, +// such as in the watcher. +func shouldIgnorePath(ctx context.Context, fsys fs.FS, relPath string) bool { + // Handle root/empty path - never ignore + if relPath == "" || relPath == "." { + return false + } + + // Load ignore patterns from root to the target path + patterns := loadIgnoredPatternsForPath(ctx, fsys, relPath) + + // If no patterns, nothing to ignore + if len(patterns) == 0 { + return false + } + + // Compile and check + matcher := ignore.CompileIgnoreLines(patterns...) + return isScanIgnored(ctx, matcher, relPath) +} diff --git a/scanner/watcher_test.go b/scanner/watcher_test.go index 6341797f6..c0fb24dee 100644 --- a/scanner/watcher_test.go +++ b/scanner/watcher_test.go @@ -146,8 +146,8 @@ var _ = Describe("Watcher", func() { // Send first notification w.watcherNotify <- scanNotification{Library: lib, FolderPath: "artist1"} - // Wait half the watcher wait time - time.Sleep(25 * time.Millisecond) + // Wait a bit less than half the watcher wait time to ensure timer doesn't fire + time.Sleep(20 * time.Millisecond) // No scan should have been triggered yet Expect(mockScanner.GetScanFoldersCallCount()).To(Equal(0)) @@ -155,14 +155,14 @@ var _ = Describe("Watcher", func() { // Send another notification (resets timer) w.watcherNotify <- scanNotification{Library: lib, FolderPath: "artist1"} - // Wait half the watcher wait time again - time.Sleep(25 * time.Millisecond) + // Wait a bit less than half the watcher wait time again + time.Sleep(20 * time.Millisecond) // Still no scan Expect(mockScanner.GetScanFoldersCallCount()).To(Equal(0)) - // Wait for full timer to expire after last notification - time.Sleep(50 * time.Millisecond) + // Wait for full timer to expire after last notification (plus margin) + time.Sleep(60 * time.Millisecond) // Now scan should have been triggered Eventually(func() int { @@ -279,6 +279,107 @@ var _ = Describe("Watcher", func() { }) }) +var _ = Describe("shouldIgnorePath", func() { + var ctx context.Context + var mockFS fs.FS + + BeforeEach(func() { + ctx = context.Background() + + // Create a mock filesystem with .ndignore files + mockFS = fstest.MapFS{ + // Root .ndignore ignoring "temp/*" + ".ndignore": &fstest.MapFile{Data: []byte("temp/*\n*.log\n")}, + + // Normal directories + "music": &fstest.MapFile{Mode: fs.ModeDir}, + "music/artist1": &fstest.MapFile{Mode: fs.ModeDir}, + "music/artist1/song.mp3": &fstest.MapFile{Data: []byte("audio")}, + + // Temp directory (should be ignored) + "temp": &fstest.MapFile{Mode: fs.ModeDir}, + "temp/cache": &fstest.MapFile{Mode: fs.ModeDir}, + "temp/cache/file.mp3": &fstest.MapFile{Data: []byte("audio")}, + + // Directory with hierarchical .ndignore + "project": &fstest.MapFile{Mode: fs.ModeDir}, + "project/.ndignore": &fstest.MapFile{Data: []byte("drafts\n")}, + "project/final": &fstest.MapFile{Mode: fs.ModeDir}, + "project/final/album.mp3": &fstest.MapFile{Data: []byte("audio")}, + "project/drafts": &fstest.MapFile{Mode: fs.ModeDir}, + "project/drafts/test.mp3": &fstest.MapFile{Data: []byte("audio")}, + + // Directory with empty .ndignore (should ignore everything) + "empty": &fstest.MapFile{Mode: fs.ModeDir}, + "empty/.ndignore": &fstest.MapFile{Data: []byte("")}, + "empty/subdir": &fstest.MapFile{Mode: fs.ModeDir}, + + // Log file at root level (should be ignored by *.log pattern) + "debug.log": &fstest.MapFile{Data: []byte("logs")}, + } + }) + + It("does not ignore paths without .ndignore patterns", func() { + result := shouldIgnorePath(ctx, mockFS, "music/artist1") + Expect(result).To(BeFalse()) + }) + + It("ignores paths matching root .ndignore patterns", func() { + result := shouldIgnorePath(ctx, mockFS, "temp/cache") + Expect(result).To(BeTrue()) + }) + + It("ignores log files matching *.log pattern", func() { + result := shouldIgnorePath(ctx, mockFS, "debug.log") + Expect(result).To(BeTrue()) + }) + + It("applies hierarchical .ndignore patterns", func() { + // project/drafts should be ignored by project/.ndignore + result := shouldIgnorePath(ctx, mockFS, "project/drafts") + Expect(result).To(BeTrue()) + + // project/final should NOT be ignored + result = shouldIgnorePath(ctx, mockFS, "project/final") + Expect(result).To(BeFalse()) + }) + + It("ignores directories with empty .ndignore file", func() { + result := shouldIgnorePath(ctx, mockFS, "empty/subdir") + Expect(result).To(BeTrue()) + }) + + It("does not ignore root or empty paths", func() { + Expect(shouldIgnorePath(ctx, mockFS, "")).To(BeFalse()) + Expect(shouldIgnorePath(ctx, mockFS, ".")).To(BeFalse()) + }) + + It("combines patterns from multiple .ndignore files", func() { + // Create a more complex hierarchy + complexFS := fstest.MapFS{ + ".ndignore": &fstest.MapFile{Data: []byte("*.tmp\n")}, + "parent": &fstest.MapFile{Mode: fs.ModeDir}, + "parent/.ndignore": &fstest.MapFile{Data: []byte("test\n")}, + "parent/test": &fstest.MapFile{Mode: fs.ModeDir}, + "parent/test/file.mp3": &fstest.MapFile{Data: []byte("audio")}, + "parent/prod": &fstest.MapFile{Mode: fs.ModeDir}, + "parent/prod/cache.tmp": &fstest.MapFile{Data: []byte("tmp")}, + } + + // parent/test should be ignored by parent/.ndignore + result := shouldIgnorePath(ctx, complexFS, "parent/test") + Expect(result).To(BeTrue()) + + // parent/prod/cache.tmp path should be ignored by root .ndignore (*.tmp) + result = shouldIgnorePath(ctx, complexFS, "parent/prod/cache.tmp") + Expect(result).To(BeTrue()) + + // parent/prod directory itself should NOT be ignored + result = shouldIgnorePath(ctx, complexFS, "parent/prod") + Expect(result).To(BeFalse()) + }) +}) + var _ = Describe("resolveFolderPath", func() { var mockFS fs.FS From 0b06ecb0349c0dbc0078fe6567b1076b76ee35bd Mon Sep 17 00:00:00 2001 From: Deluan Date: Mon, 10 Nov 2025 23:42:54 -0500 Subject: [PATCH 10/40] refactor(scanner): implement IgnoreChecker for managing .ndignore patterns Signed-off-by: Deluan --- scanner/ignore_checker.go | 165 +++++++++++++++++ scanner/ignore_checker_test.go | 313 +++++++++++++++++++++++++++++++++ scanner/walk_dir_tree.go | 98 ++--------- scanner/watcher.go | 37 +--- scanner/watcher_test.go | 101 ----------- 5 files changed, 501 insertions(+), 213 deletions(-) create mode 100644 scanner/ignore_checker.go create mode 100644 scanner/ignore_checker_test.go diff --git a/scanner/ignore_checker.go b/scanner/ignore_checker.go new file mode 100644 index 000000000..2ec907842 --- /dev/null +++ b/scanner/ignore_checker.go @@ -0,0 +1,165 @@ +package scanner + +import ( + "bufio" + "context" + "io/fs" + "path" + "strings" + + "github.com/navidrome/navidrome/consts" + "github.com/navidrome/navidrome/log" + ignore "github.com/sabhiram/go-gitignore" +) + +// IgnoreChecker manages .ndignore patterns using a stack-based approach. +// Use Push() to add patterns when entering a folder, Pop() when leaving, +// and ShouldIgnore() to check if a path should be ignored. +type IgnoreChecker struct { + fsys fs.FS + patternStack [][]string // Stack of patterns for each folder level + currentPatterns []string // Flattened current patterns + matcher *ignore.GitIgnore // Compiled matcher for current patterns +} + +// newIgnoreChecker creates a new IgnoreChecker for the given filesystem. +func newIgnoreChecker(fsys fs.FS) *IgnoreChecker { + return &IgnoreChecker{ + fsys: fsys, + patternStack: make([][]string, 0), + } +} + +// Push loads .ndignore patterns from the specified folder and adds them to the pattern stack. +// Use this when entering a folder during directory tree traversal. +func (ic *IgnoreChecker) Push(ctx context.Context, folder string) error { + patterns := ic.loadPatternsFromFolder(ctx, folder) + ic.patternStack = append(ic.patternStack, patterns) + ic.rebuildCurrentPatterns() + return nil +} + +// Pop removes the most recent patterns from the stack. +// Use this when leaving a folder during directory tree traversal. +func (ic *IgnoreChecker) Pop() { + if len(ic.patternStack) > 0 { + ic.patternStack = ic.patternStack[:len(ic.patternStack)-1] + ic.rebuildCurrentPatterns() + } +} + +// PushAllParents pushes patterns from root down to the target path. +// This is a convenience method for when you need to check a specific path +// without recursively walking the tree. It handles the common pattern of +// pushing all parent directories from root to the target. +// This method is optimized to compile patterns only once at the end. +func (ic *IgnoreChecker) PushAllParents(ctx context.Context, targetPath string) error { + if targetPath == "." || targetPath == "" { + // Simple case: just push root + return ic.Push(ctx, ".") + } + + // Load patterns for root + patterns := ic.loadPatternsFromFolder(ctx, ".") + ic.patternStack = append(ic.patternStack, patterns) + + // Load patterns for each parent directory + currentPath := "." + parts := strings.Split(path.Clean(targetPath), "/") + for _, part := range parts { + if part == "." || part == "" { + continue + } + currentPath = path.Join(currentPath, part) + patterns = ic.loadPatternsFromFolder(ctx, currentPath) + ic.patternStack = append(ic.patternStack, patterns) + } + + // Rebuild and compile patterns only once at the end + ic.rebuildCurrentPatterns() + return nil +} + +// ShouldIgnore checks if the given path should be ignored based on the current patterns. +// Returns true if the path matches any ignore pattern, false otherwise. +func (ic *IgnoreChecker) ShouldIgnore(ctx context.Context, relPath string) bool { + // Handle root/empty path - never ignore + if relPath == "" || relPath == "." { + return false + } + + // If no patterns loaded, nothing to ignore + if ic.matcher == nil { + return false + } + + matches := ic.matcher.MatchesPath(relPath) + if matches { + log.Trace(ctx, "Scanner: Ignoring entry matching .ndignore", "path", relPath) + } + return matches +} + +// loadPatternsFromFolder reads the .ndignore file in the specified folder and returns the patterns. +// If the file doesn't exist, returns an empty slice. +// If the file exists but is empty, returns a pattern to ignore everything ("**/*"). +func (ic *IgnoreChecker) loadPatternsFromFolder(ctx context.Context, folder string) []string { + ignoreFilePath := path.Join(folder, consts.ScanIgnoreFile) + var patterns []string + + // Check if .ndignore file exists + if _, err := fs.Stat(ic.fsys, ignoreFilePath); err != nil { + // No .ndignore file in this folder + return patterns + } + + // Read and parse the .ndignore file + ignoreFile, err := ic.fsys.Open(ignoreFilePath) + if err != nil { + log.Warn(ctx, "Scanner: Error opening .ndignore file", "path", ignoreFilePath, err) + return patterns + } + defer ignoreFile.Close() + + scanner := bufio.NewScanner(ignoreFile) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" || strings.HasPrefix(line, "#") { + continue // Skip empty lines, whitespace-only lines, and comments + } + patterns = append(patterns, line) + } + + if err := scanner.Err(); err != nil { + log.Warn(ctx, "Scanner: Error reading .ndignore file", "path", ignoreFilePath, err) + return patterns + } + + // If the .ndignore file is empty, ignore everything + if len(patterns) == 0 { + log.Trace(ctx, "Scanner: .ndignore file is empty, ignoring everything", "path", folder) + patterns = []string{"**/*"} + } else { + log.Trace(ctx, "Scanner: .ndignore file found", "path", ignoreFilePath, "patterns", patterns) + } + + return patterns +} + +// rebuildCurrentPatterns flattens the pattern stack into currentPatterns and recompiles the matcher. +func (ic *IgnoreChecker) rebuildCurrentPatterns() { + ic.currentPatterns = make([]string, 0) + for _, patterns := range ic.patternStack { + ic.currentPatterns = append(ic.currentPatterns, patterns...) + } + ic.compilePatterns() +} + +// compilePatterns compiles the current patterns into a GitIgnore matcher. +func (ic *IgnoreChecker) compilePatterns() { + if len(ic.currentPatterns) == 0 { + ic.matcher = nil + return + } + ic.matcher = ignore.CompileIgnoreLines(ic.currentPatterns...) +} diff --git a/scanner/ignore_checker_test.go b/scanner/ignore_checker_test.go new file mode 100644 index 000000000..5378ed4fa --- /dev/null +++ b/scanner/ignore_checker_test.go @@ -0,0 +1,313 @@ +package scanner + +import ( + "context" + "testing/fstest" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("IgnoreChecker", func() { + Describe("loadPatternsFromFolder", func() { + var ic *IgnoreChecker + var ctx context.Context + + BeforeEach(func() { + ctx = context.Background() + }) + + Context("when .ndignore file does not exist", func() { + It("should return empty patterns", func() { + fsys := fstest.MapFS{} + ic = newIgnoreChecker(fsys) + patterns := ic.loadPatternsFromFolder(ctx, ".") + Expect(patterns).To(BeEmpty()) + }) + }) + + Context("when .ndignore file is empty", func() { + It("should return wildcard to ignore everything", func() { + fsys := fstest.MapFS{ + ".ndignore": &fstest.MapFile{Data: []byte("")}, + } + ic = newIgnoreChecker(fsys) + patterns := ic.loadPatternsFromFolder(ctx, ".") + Expect(patterns).To(Equal([]string{"**/*"})) + }) + }) + + DescribeTable("parsing .ndignore content", + func(content string, expectedPatterns []string) { + fsys := fstest.MapFS{ + ".ndignore": &fstest.MapFile{Data: []byte(content)}, + } + ic = newIgnoreChecker(fsys) + patterns := ic.loadPatternsFromFolder(ctx, ".") + Expect(patterns).To(Equal(expectedPatterns)) + }, + Entry("single pattern", "*.txt", []string{"*.txt"}), + Entry("multiple patterns", "*.txt\n*.log", []string{"*.txt", "*.log"}), + Entry("with comments", "# comment\n*.txt\n# another\n*.log", []string{"*.txt", "*.log"}), + Entry("with empty lines", "*.txt\n\n*.log\n\n", []string{"*.txt", "*.log"}), + Entry("mixed content", "# header\n\n*.txt\n# middle\n*.log\n\n", []string{"*.txt", "*.log"}), + Entry("only comments and empty lines", "# comment\n\n# another\n", []string{"**/*"}), + Entry("trailing newline", "*.txt\n*.log\n", []string{"*.txt", "*.log"}), + Entry("directory pattern", "temp/", []string{"temp/"}), + Entry("wildcard pattern", "**/*.mp3", []string{"**/*.mp3"}), + Entry("multiple wildcards", "**/*.mp3\n**/*.flac\n*.log", []string{"**/*.mp3", "**/*.flac", "*.log"}), + Entry("negation pattern", "!important.txt", []string{"!important.txt"}), + Entry("comment with hash not at start is pattern", "not#comment", []string{"not#comment"}), + Entry("whitespace-only lines skipped", "*.txt\n \n*.log\n\t\n", []string{"*.txt", "*.log"}), + Entry("patterns with whitespace trimmed", " *.txt \n\t*.log\t", []string{"*.txt", "*.log"}), + ) + }) + + Describe("Push and Pop", func() { + var ic *IgnoreChecker + var fsys fstest.MapFS + var ctx context.Context + + BeforeEach(func() { + ctx = context.Background() + fsys = fstest.MapFS{ + ".ndignore": &fstest.MapFile{Data: []byte("*.txt")}, + "folder1/.ndignore": &fstest.MapFile{Data: []byte("*.mp3")}, + "folder2/.ndignore": &fstest.MapFile{Data: []byte("*.flac")}, + } + ic = newIgnoreChecker(fsys) + }) + + Context("Push", func() { + It("should add patterns to stack", func() { + err := ic.Push(ctx, ".") + Expect(err).ToNot(HaveOccurred()) + Expect(len(ic.patternStack)).To(Equal(1)) + Expect(ic.currentPatterns).To(ContainElement("*.txt")) + }) + + It("should compile matcher after push", func() { + err := ic.Push(ctx, ".") + Expect(err).ToNot(HaveOccurred()) + Expect(ic.matcher).ToNot(BeNil()) + }) + + It("should accumulate patterns from multiple levels", func() { + err := ic.Push(ctx, ".") + Expect(err).ToNot(HaveOccurred()) + err = ic.Push(ctx, "folder1") + Expect(err).ToNot(HaveOccurred()) + Expect(len(ic.patternStack)).To(Equal(2)) + Expect(ic.currentPatterns).To(ConsistOf("*.txt", "*.mp3")) + }) + + It("should handle push when no .ndignore exists", func() { + err := ic.Push(ctx, "nonexistent") + Expect(err).ToNot(HaveOccurred()) + Expect(len(ic.patternStack)).To(Equal(1)) + Expect(ic.currentPatterns).To(BeEmpty()) + }) + }) + + Context("Pop", func() { + It("should remove most recent patterns", func() { + err := ic.Push(ctx, ".") + Expect(err).ToNot(HaveOccurred()) + err = ic.Push(ctx, "folder1") + Expect(err).ToNot(HaveOccurred()) + ic.Pop() + Expect(len(ic.patternStack)).To(Equal(1)) + Expect(ic.currentPatterns).To(Equal([]string{"*.txt"})) + }) + + It("should handle Pop on empty stack gracefully", func() { + Expect(func() { ic.Pop() }).ToNot(Panic()) + Expect(ic.patternStack).To(BeEmpty()) + }) + + It("should set matcher to nil when all patterns popped", func() { + err := ic.Push(ctx, ".") + Expect(err).ToNot(HaveOccurred()) + Expect(ic.matcher).ToNot(BeNil()) + ic.Pop() + Expect(ic.matcher).To(BeNil()) + }) + + It("should update matcher after pop", func() { + err := ic.Push(ctx, ".") + Expect(err).ToNot(HaveOccurred()) + err = ic.Push(ctx, "folder1") + Expect(err).ToNot(HaveOccurred()) + matcher1 := ic.matcher + ic.Pop() + matcher2 := ic.matcher + Expect(matcher1).ToNot(Equal(matcher2)) + }) + }) + + Context("multiple Push/Pop cycles", func() { + It("should maintain correct state through cycles", func() { + err := ic.Push(ctx, ".") + Expect(err).ToNot(HaveOccurred()) + Expect(ic.currentPatterns).To(Equal([]string{"*.txt"})) + + err = ic.Push(ctx, "folder1") + Expect(err).ToNot(HaveOccurred()) + Expect(ic.currentPatterns).To(ConsistOf("*.txt", "*.mp3")) + + ic.Pop() + Expect(ic.currentPatterns).To(Equal([]string{"*.txt"})) + + err = ic.Push(ctx, "folder2") + Expect(err).ToNot(HaveOccurred()) + Expect(ic.currentPatterns).To(ConsistOf("*.txt", "*.flac")) + + ic.Pop() + Expect(ic.currentPatterns).To(Equal([]string{"*.txt"})) + + ic.Pop() + Expect(ic.currentPatterns).To(BeEmpty()) + }) + }) + }) + + Describe("PushAllParents", func() { + var ic *IgnoreChecker + var ctx context.Context + + BeforeEach(func() { + ctx = context.Background() + fsys := fstest.MapFS{ + ".ndignore": &fstest.MapFile{Data: []byte("root.txt")}, + "folder1/.ndignore": &fstest.MapFile{Data: []byte("level1.txt")}, + "folder1/folder2/.ndignore": &fstest.MapFile{Data: []byte("level2.txt")}, + "folder1/folder2/folder3/.ndignore": &fstest.MapFile{Data: []byte("level3.txt")}, + } + ic = newIgnoreChecker(fsys) + }) + + DescribeTable("loading parent patterns", + func(targetPath string, expectedStackDepth int, expectedPatterns []string) { + err := ic.PushAllParents(ctx, targetPath) + Expect(err).ToNot(HaveOccurred()) + Expect(len(ic.patternStack)).To(Equal(expectedStackDepth)) + Expect(ic.currentPatterns).To(ConsistOf(expectedPatterns)) + }, + Entry("root path", ".", 1, []string{"root.txt"}), + Entry("empty path", "", 1, []string{"root.txt"}), + Entry("single level", "folder1", 2, []string{"root.txt", "level1.txt"}), + Entry("two levels", "folder1/folder2", 3, []string{"root.txt", "level1.txt", "level2.txt"}), + Entry("three levels", "folder1/folder2/folder3", 4, []string{"root.txt", "level1.txt", "level2.txt", "level3.txt"}), + ) + + It("should only compile patterns once at the end", func() { + // This is more of a behavioral test - we verify the matcher is not nil after PushAllParents + err := ic.PushAllParents(ctx, "folder1/folder2") + Expect(err).ToNot(HaveOccurred()) + Expect(ic.matcher).ToNot(BeNil()) + }) + + It("should handle paths with dot", func() { + err := ic.PushAllParents(ctx, "./folder1") + Expect(err).ToNot(HaveOccurred()) + Expect(len(ic.patternStack)).To(Equal(2)) + }) + + Context("when some parent folders have no .ndignore", func() { + BeforeEach(func() { + fsys := fstest.MapFS{ + ".ndignore": &fstest.MapFile{Data: []byte("root.txt")}, + "folder1/folder2/.ndignore": &fstest.MapFile{Data: []byte("level2.txt")}, + } + ic = newIgnoreChecker(fsys) + }) + + It("should still push all parent levels", func() { + err := ic.PushAllParents(ctx, "folder1/folder2") + Expect(err).ToNot(HaveOccurred()) + Expect(len(ic.patternStack)).To(Equal(3)) // root, folder1 (empty), folder2 + Expect(ic.currentPatterns).To(ConsistOf("root.txt", "level2.txt")) + }) + }) + }) + + Describe("ShouldIgnore", func() { + var ic *IgnoreChecker + var ctx context.Context + + BeforeEach(func() { + ctx = context.Background() + }) + + Context("with no patterns loaded", func() { + It("should not ignore any path", func() { + fsys := fstest.MapFS{} + ic = newIgnoreChecker(fsys) + Expect(ic.ShouldIgnore(ctx, "anything.txt")).To(BeFalse()) + Expect(ic.ShouldIgnore(ctx, "folder/file.mp3")).To(BeFalse()) + }) + }) + + Context("special paths", func() { + BeforeEach(func() { + fsys := fstest.MapFS{ + ".ndignore": &fstest.MapFile{Data: []byte("**/*")}, + } + ic = newIgnoreChecker(fsys) + err := ic.Push(ctx, ".") + Expect(err).ToNot(HaveOccurred()) + }) + + It("should never ignore root or empty paths", func() { + Expect(ic.ShouldIgnore(ctx, "")).To(BeFalse()) + Expect(ic.ShouldIgnore(ctx, ".")).To(BeFalse()) + }) + + It("should ignore all other paths with wildcard", func() { + Expect(ic.ShouldIgnore(ctx, "file.txt")).To(BeTrue()) + Expect(ic.ShouldIgnore(ctx, "folder/file.mp3")).To(BeTrue()) + }) + }) + + DescribeTable("pattern matching", + func(pattern string, path string, shouldMatch bool) { + fsys := fstest.MapFS{ + ".ndignore": &fstest.MapFile{Data: []byte(pattern)}, + } + ic = newIgnoreChecker(fsys) + err := ic.Push(ctx, ".") + Expect(err).ToNot(HaveOccurred()) + Expect(ic.ShouldIgnore(ctx, path)).To(Equal(shouldMatch)) + }, + Entry("glob match", "*.txt", "file.txt", true), + Entry("glob no match", "*.txt", "file.mp3", false), + Entry("directory pattern match", "tmp/", "tmp/file.txt", true), + Entry("directory pattern no match", "tmp/", "temporary/file.txt", false), + Entry("nested glob match", "**/*.log", "deep/nested/file.log", true), + Entry("nested glob no match", "**/*.log", "deep/nested/file.txt", false), + Entry("specific file match", "ignore.me", "ignore.me", true), + Entry("specific file no match", "ignore.me", "keep.me", false), + Entry("wildcard all", "**/*", "any/path/file.txt", true), + Entry("nested specific match", "temp/*", "temp/cache.db", true), + Entry("nested specific no match", "temp/*", "temporary/cache.db", false), + ) + + Context("with multiple patterns", func() { + BeforeEach(func() { + fsys := fstest.MapFS{ + ".ndignore": &fstest.MapFile{Data: []byte("*.txt\n*.log\ntemp/")}, + } + ic = newIgnoreChecker(fsys) + err := ic.Push(ctx, ".") + Expect(err).ToNot(HaveOccurred()) + }) + + It("should match any of the patterns", func() { + Expect(ic.ShouldIgnore(ctx, "file.txt")).To(BeTrue()) + Expect(ic.ShouldIgnore(ctx, "debug.log")).To(BeTrue()) + Expect(ic.ShouldIgnore(ctx, "temp/cache")).To(BeTrue()) + Expect(ic.ShouldIgnore(ctx, "music.mp3")).To(BeFalse()) + }) + }) + }) +}) diff --git a/scanner/walk_dir_tree.go b/scanner/walk_dir_tree.go index d374a4aca..d431fef23 100644 --- a/scanner/walk_dir_tree.go +++ b/scanner/walk_dir_tree.go @@ -1,7 +1,6 @@ package scanner import ( - "bufio" "context" "io/fs" "maps" @@ -11,18 +10,17 @@ import ( "strings" "github.com/navidrome/navidrome/conf" - "github.com/navidrome/navidrome/consts" "github.com/navidrome/navidrome/log" "github.com/navidrome/navidrome/model" "github.com/navidrome/navidrome/utils" - ignore "github.com/sabhiram/go-gitignore" ) func walkDirTree(ctx context.Context, job *scanJob) (<-chan *folderEntry, error) { results := make(chan *folderEntry) go func() { defer close(results) - err := walkFolder(ctx, job, ".", nil, results) + checker := newIgnoreChecker(job.fs) + err := walkFolder(ctx, job, ".", checker, results) if err != nil { log.Error(ctx, "Scanner: There were errors reading directories from filesystem", "path", job.lib.Path, err) return @@ -42,11 +40,12 @@ func loadSpecificFolders(ctx context.Context, job *scanJob, targetFolders []stri return } - // Load ignore patterns from parent directories up to this folder - ignorePatterns := loadIgnoredPatternsForPath(ctx, job.fs, folderPath) + // Create checker and push patterns from root to this folder + checker := newIgnoreChecker(job.fs) + _ = checker.PushAllParents(ctx, folderPath) // Load only this specific folder (no recursion) - folder, _, err := loadDir(ctx, job, folderPath, ignorePatterns) + folder, _, err := loadDir(ctx, job, folderPath, checker) if err != nil { log.Warn(ctx, "Scanner: Error loading target folder. Skipping", "path", folderPath, err) continue @@ -65,78 +64,18 @@ func loadSpecificFolders(ctx context.Context, job *scanJob, targetFolders []stri return results, nil } -// loadIgnoredPatternsForPath loads all .ndignore patterns from the root down to the specified path -func loadIgnoredPatternsForPath(ctx context.Context, fsys fs.FS, targetPath string) []string { - var patterns []string - currentPath := "." +func walkFolder(ctx context.Context, job *scanJob, currentFolder string, checker *IgnoreChecker, results chan<- *folderEntry) error { + // Push patterns for this folder onto the stack + _ = checker.Push(ctx, currentFolder) + defer checker.Pop() // Pop patterns when leaving this folder - // If target is root, just check root - if targetPath == "." { - return loadIgnoredPatterns(ctx, fsys, ".", nil) - } - - // Walk from root to target, collecting ignore patterns - parts := strings.Split(path.Clean(targetPath), "/") - for _, part := range parts { - if part == "." { - continue - } - patterns = loadIgnoredPatterns(ctx, fsys, currentPath, patterns) - currentPath = path.Join(currentPath, part) - } - // Load patterns from the target folder itself - patterns = loadIgnoredPatterns(ctx, fsys, currentPath, patterns) - - return patterns -} - -// loadIgnoredPatterns loads .ndignore patterns from the specified folder and combines them with currentPatterns -func loadIgnoredPatterns(ctx context.Context, fsys fs.FS, currentFolder string, currentPatterns []string) []string { - ignoreFilePath := path.Join(currentFolder, consts.ScanIgnoreFile) - var newPatterns []string - if _, err := fs.Stat(fsys, ignoreFilePath); err == nil { - // Read and parse the .ndignore file - ignoreFile, err := fsys.Open(ignoreFilePath) - if err != nil { - log.Warn(ctx, "Scanner: Error opening .ndignore file", "path", ignoreFilePath, err) - // Continue with previous patterns - } else { - defer ignoreFile.Close() - scanner := bufio.NewScanner(ignoreFile) - for scanner.Scan() { - line := scanner.Text() - if line == "" || strings.HasPrefix(line, "#") { - continue // Skip empty lines and comments - } - newPatterns = append(newPatterns, line) - } - if err := scanner.Err(); err != nil { - log.Warn(ctx, "Scanner: Error reading .ignore file", "path", ignoreFilePath, err) - } - } - // If the .ndignore file is empty, mimic the current behavior and ignore everything - if len(newPatterns) == 0 { - log.Trace(ctx, "Scanner: .ndignore file is empty, ignoring everything", "path", currentFolder) - newPatterns = []string{"**/*"} - } else { - log.Trace(ctx, "Scanner: .ndignore file found ", "path", ignoreFilePath, "patterns", newPatterns) - } - } - // Combine the patterns from the .ndignore file with the ones passed as argument - combinedPatterns := append([]string{}, currentPatterns...) - return append(combinedPatterns, newPatterns...) -} - -func walkFolder(ctx context.Context, job *scanJob, currentFolder string, ignorePatterns []string, results chan<- *folderEntry) error { - ignorePatterns = loadIgnoredPatterns(ctx, job.fs, currentFolder, ignorePatterns) - - folder, children, err := loadDir(ctx, job, currentFolder, ignorePatterns) + folder, children, err := loadDir(ctx, job, currentFolder, checker) if err != nil { log.Warn(ctx, "Scanner: Error loading dir. Skipping", "path", currentFolder, err) return nil } for _, c := range children { - err := walkFolder(ctx, job, c, ignorePatterns, results) + err := walkFolder(ctx, job, c, checker, results) if err != nil { return err } @@ -154,7 +93,7 @@ func walkFolder(ctx context.Context, job *scanJob, currentFolder string, ignoreP return nil } -func loadDir(ctx context.Context, job *scanJob, dirPath string, ignorePatterns []string) (folder *folderEntry, children []string, err error) { +func loadDir(ctx context.Context, job *scanJob, dirPath string, checker *IgnoreChecker) (folder *folderEntry, children []string, err error) { folder = newFolderEntry(job, dirPath) dirInfo, err := fs.Stat(job.fs, dirPath) @@ -176,12 +115,11 @@ func loadDir(ctx context.Context, job *scanJob, dirPath string, ignorePatterns [ return folder, children, err } - ignoreMatcher := ignore.CompileIgnoreLines(ignorePatterns...) entries := fullReadDir(ctx, dirFile) children = make([]string, 0, len(entries)) for _, entry := range entries { entryPath := path.Join(dirPath, entry.Name()) - if len(ignorePatterns) > 0 && isScanIgnored(ctx, ignoreMatcher, entryPath) { + if checker.ShouldIgnore(ctx, entryPath) { log.Trace(ctx, "Scanner: Ignoring entry", "path", entryPath) continue } @@ -313,11 +251,3 @@ func isDirIgnored(name string) bool { func isEntryIgnored(name string) bool { return strings.HasPrefix(name, ".") && !strings.HasPrefix(name, "..") } - -func isScanIgnored(ctx context.Context, matcher *ignore.GitIgnore, entryPath string) bool { - matches := matcher.MatchesPath(entryPath) - if matches { - log.Trace(ctx, "Scanner: Ignoring entry matching .ndignore: ", "path", entryPath) - } - return matches -} diff --git a/scanner/watcher.go b/scanner/watcher.go index 43827df22..122af1c08 100644 --- a/scanner/watcher.go +++ b/scanner/watcher.go @@ -13,7 +13,6 @@ import ( "github.com/navidrome/navidrome/log" "github.com/navidrome/navidrome/model" "github.com/navidrome/navidrome/utils/singleton" - ignore "github.com/sabhiram/go-gitignore" ) type Watcher interface { @@ -236,14 +235,11 @@ func (w *watcher) watchLibrary(ctx context.Context, lib *model.Library) error { log.Trace(ctx, "Ignoring change", "libraryID", lib.ID, "path", path) continue } - log.Trace(ctx, "Detected change", "libraryID", lib.ID, "path", path, "absoluteLibPath", absLibPath) // Find the folder to scan - validate path exists as directory, walk up if needed folderPath := resolveFolderPath(fsys, path) - - // Check if the folder should be ignored based on .ndignore patterns - if shouldIgnorePath(ctx, fsys, folderPath) { + if w.shouldIgnoreFolderPath(ctx, fsys, folderPath) { log.Trace(ctx, "Ignoring change in folder matching .ndignore pattern", "libraryID", lib.ID, "folderPath", folderPath) continue } @@ -288,6 +284,14 @@ func resolveFolderPath(fsys fs.FS, path string) string { } } +// shouldIgnoreFolderPath checks if the given folderPath should be ignored based on .ndignore patterns +// in the library. It pushes all parent folders onto the IgnoreChecker stack before checking. +func (w *watcher) shouldIgnoreFolderPath(ctx context.Context, fsys storage.MusicFS, folderPath string) bool { + checker := newIgnoreChecker(fsys) + _ = checker.PushAllParents(ctx, folderPath) + return checker.ShouldIgnore(ctx, folderPath) +} + func isIgnoredPath(_ context.Context, _ fs.FS, path string) bool { baseDir, name := filepath.Split(path) switch { @@ -304,26 +308,3 @@ func isIgnoredPath(_ context.Context, _ fs.FS, path string) bool { // But at this point, we can assume it's a directory. If it's a file, it would be ignored anyway return isDirIgnored(baseDir) } - -// shouldIgnorePath checks if the given path should be ignored based on .ndignore patterns. -// It loads all .ndignore files from the root down to the path and returns true if the path -// matches any ignore pattern. This function is suitable for checking paths without recursion, -// such as in the watcher. -func shouldIgnorePath(ctx context.Context, fsys fs.FS, relPath string) bool { - // Handle root/empty path - never ignore - if relPath == "" || relPath == "." { - return false - } - - // Load ignore patterns from root to the target path - patterns := loadIgnoredPatternsForPath(ctx, fsys, relPath) - - // If no patterns, nothing to ignore - if len(patterns) == 0 { - return false - } - - // Compile and check - matcher := ignore.CompileIgnoreLines(patterns...) - return isScanIgnored(ctx, matcher, relPath) -} diff --git a/scanner/watcher_test.go b/scanner/watcher_test.go index c0fb24dee..6f10d9010 100644 --- a/scanner/watcher_test.go +++ b/scanner/watcher_test.go @@ -279,107 +279,6 @@ var _ = Describe("Watcher", func() { }) }) -var _ = Describe("shouldIgnorePath", func() { - var ctx context.Context - var mockFS fs.FS - - BeforeEach(func() { - ctx = context.Background() - - // Create a mock filesystem with .ndignore files - mockFS = fstest.MapFS{ - // Root .ndignore ignoring "temp/*" - ".ndignore": &fstest.MapFile{Data: []byte("temp/*\n*.log\n")}, - - // Normal directories - "music": &fstest.MapFile{Mode: fs.ModeDir}, - "music/artist1": &fstest.MapFile{Mode: fs.ModeDir}, - "music/artist1/song.mp3": &fstest.MapFile{Data: []byte("audio")}, - - // Temp directory (should be ignored) - "temp": &fstest.MapFile{Mode: fs.ModeDir}, - "temp/cache": &fstest.MapFile{Mode: fs.ModeDir}, - "temp/cache/file.mp3": &fstest.MapFile{Data: []byte("audio")}, - - // Directory with hierarchical .ndignore - "project": &fstest.MapFile{Mode: fs.ModeDir}, - "project/.ndignore": &fstest.MapFile{Data: []byte("drafts\n")}, - "project/final": &fstest.MapFile{Mode: fs.ModeDir}, - "project/final/album.mp3": &fstest.MapFile{Data: []byte("audio")}, - "project/drafts": &fstest.MapFile{Mode: fs.ModeDir}, - "project/drafts/test.mp3": &fstest.MapFile{Data: []byte("audio")}, - - // Directory with empty .ndignore (should ignore everything) - "empty": &fstest.MapFile{Mode: fs.ModeDir}, - "empty/.ndignore": &fstest.MapFile{Data: []byte("")}, - "empty/subdir": &fstest.MapFile{Mode: fs.ModeDir}, - - // Log file at root level (should be ignored by *.log pattern) - "debug.log": &fstest.MapFile{Data: []byte("logs")}, - } - }) - - It("does not ignore paths without .ndignore patterns", func() { - result := shouldIgnorePath(ctx, mockFS, "music/artist1") - Expect(result).To(BeFalse()) - }) - - It("ignores paths matching root .ndignore patterns", func() { - result := shouldIgnorePath(ctx, mockFS, "temp/cache") - Expect(result).To(BeTrue()) - }) - - It("ignores log files matching *.log pattern", func() { - result := shouldIgnorePath(ctx, mockFS, "debug.log") - Expect(result).To(BeTrue()) - }) - - It("applies hierarchical .ndignore patterns", func() { - // project/drafts should be ignored by project/.ndignore - result := shouldIgnorePath(ctx, mockFS, "project/drafts") - Expect(result).To(BeTrue()) - - // project/final should NOT be ignored - result = shouldIgnorePath(ctx, mockFS, "project/final") - Expect(result).To(BeFalse()) - }) - - It("ignores directories with empty .ndignore file", func() { - result := shouldIgnorePath(ctx, mockFS, "empty/subdir") - Expect(result).To(BeTrue()) - }) - - It("does not ignore root or empty paths", func() { - Expect(shouldIgnorePath(ctx, mockFS, "")).To(BeFalse()) - Expect(shouldIgnorePath(ctx, mockFS, ".")).To(BeFalse()) - }) - - It("combines patterns from multiple .ndignore files", func() { - // Create a more complex hierarchy - complexFS := fstest.MapFS{ - ".ndignore": &fstest.MapFile{Data: []byte("*.tmp\n")}, - "parent": &fstest.MapFile{Mode: fs.ModeDir}, - "parent/.ndignore": &fstest.MapFile{Data: []byte("test\n")}, - "parent/test": &fstest.MapFile{Mode: fs.ModeDir}, - "parent/test/file.mp3": &fstest.MapFile{Data: []byte("audio")}, - "parent/prod": &fstest.MapFile{Mode: fs.ModeDir}, - "parent/prod/cache.tmp": &fstest.MapFile{Data: []byte("tmp")}, - } - - // parent/test should be ignored by parent/.ndignore - result := shouldIgnorePath(ctx, complexFS, "parent/test") - Expect(result).To(BeTrue()) - - // parent/prod/cache.tmp path should be ignored by root .ndignore (*.tmp) - result = shouldIgnorePath(ctx, complexFS, "parent/prod/cache.tmp") - Expect(result).To(BeTrue()) - - // parent/prod directory itself should NOT be ignored - result = shouldIgnorePath(ctx, complexFS, "parent/prod") - Expect(result).To(BeFalse()) - }) -}) - var _ = Describe("resolveFolderPath", func() { var mockFS fs.FS From b06d90a4db1848e7b059ae8408744a0f298378b7 Mon Sep 17 00:00:00 2001 From: Deluan Date: Tue, 11 Nov 2025 11:00:37 -0500 Subject: [PATCH 11/40] refactor(ignore_checker): rename scanner to lineScanner for clarity Signed-off-by: Deluan --- scanner/ignore_checker.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scanner/ignore_checker.go b/scanner/ignore_checker.go index 2ec907842..53bfae8f9 100644 --- a/scanner/ignore_checker.go +++ b/scanner/ignore_checker.go @@ -121,16 +121,16 @@ func (ic *IgnoreChecker) loadPatternsFromFolder(ctx context.Context, folder stri } defer ignoreFile.Close() - scanner := bufio.NewScanner(ignoreFile) - for scanner.Scan() { - line := strings.TrimSpace(scanner.Text()) + lineScanner := bufio.NewScanner(ignoreFile) + for lineScanner.Scan() { + line := strings.TrimSpace(lineScanner.Text()) if line == "" || strings.HasPrefix(line, "#") { continue // Skip empty lines, whitespace-only lines, and comments } patterns = append(patterns, line) } - if err := scanner.Err(); err != nil { + if err := lineScanner.Err(); err != nil { log.Warn(ctx, "Scanner: Error reading .ndignore file", "path", ignoreFilePath, err) return patterns } From f64b51f161071acdfa88c6ef190a242bd50fa9ab Mon Sep 17 00:00:00 2001 From: Deluan Date: Tue, 11 Nov 2025 11:14:51 -0500 Subject: [PATCH 12/40] refactor(scanner): enhance ScanTarget struct with String method for better target representation Signed-off-by: Deluan --- scanner/controller.go | 5 +++++ scanner/external.go | 11 +++-------- scanner/watcher.go | 5 +++-- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/scanner/controller.go b/scanner/controller.go index d2ebdbfd4..28dbb7287 100644 --- a/scanner/controller.go +++ b/scanner/controller.go @@ -27,11 +27,16 @@ var ( ) // ScanTarget represents a specific folder within a library to be scanned. +// NOTE: This struct is used as a map key, so it should only contain comparable types. type ScanTarget struct { LibraryID int FolderPath string // Relative path within the library, or "" for entire library } +func (st ScanTarget) String() string { + return fmt.Sprintf("%d:%s", st.LibraryID, st.FolderPath) +} + type Scanner interface { // ScanAll starts a full scan of the music library. This is a blocking operation. ScanAll(ctx context.Context, fullScan bool) (warnings []string, err error) diff --git a/scanner/external.go b/scanner/external.go index 869034cc2..690748d8e 100644 --- a/scanner/external.go +++ b/scanner/external.go @@ -8,10 +8,11 @@ import ( "io" "os" "os/exec" - "strconv" + "strings" "github.com/navidrome/navidrome/conf" "github.com/navidrome/navidrome/log" + "github.com/navidrome/navidrome/utils/slice" ) // scannerExternal is a scanner that runs an external process to do the scanning. It is used to avoid @@ -49,13 +50,7 @@ func (s *scannerExternal) scan(ctx context.Context, fullScan bool, targets []Sca // Add targets if provided if len(targets) > 0 { - var targetsStr string - for i, target := range targets { - if i > 0 { - targetsStr += "," - } - targetsStr += strconv.Itoa(target.LibraryID) + ":" + target.FolderPath - } + targetsStr := strings.Join(slice.Map(targets, func(t ScanTarget) string { return t.String() }), ",") args = append(args, "--targets", targetsStr) log.Debug(ctx, "Spawning external scanner process with targets", "fullScan", fullScan, "path", exe, "targets", targetsStr) } else { diff --git a/scanner/watcher.go b/scanner/watcher.go index 122af1c08..dd0d1cca2 100644 --- a/scanner/watcher.go +++ b/scanner/watcher.go @@ -121,10 +121,11 @@ func (w *watcher) Run(ctx context.Context) error { folderPath := notification.FolderPath // If already scheduled for scan, skip - if _, exists := targets[ScanTarget{LibraryID: lib.ID, FolderPath: folderPath}]; exists { + target := ScanTarget{LibraryID: lib.ID, FolderPath: folderPath} + if _, exists := targets[target]; exists { continue } - targets[ScanTarget{LibraryID: lib.ID, FolderPath: folderPath}] = struct{}{} + targets[target] = struct{}{} trigger.Reset(w.triggerWait) log.Debug(ctx, "Watcher: Detected changes. Waiting for more changes before triggering scan", From 01f68f4a75c5a6410dad00499c70504f300fdd88 Mon Sep 17 00:00:00 2001 From: Deluan Date: Tue, 11 Nov 2025 11:23:55 -0500 Subject: [PATCH 13/40] fix(scanner): validate library ID to prevent negative values Signed-off-by: Deluan --- cmd/scan.go | 3 +++ cmd/scan_test.go | 8 ++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/cmd/scan.go b/cmd/scan.go index 06d8c6c25..9c92de966 100644 --- a/cmd/scan.go +++ b/cmd/scan.go @@ -123,6 +123,9 @@ func parseTargets(targetsStr string) ([]scanner.ScanTarget, error) { if err != nil { return nil, fmt.Errorf("invalid library ID %q: %w", libIDStr, err) } + if libID <= 0 { + return nil, fmt.Errorf("invalid library ID %q", libIDStr) + } targets = append(targets, scanner.ScanTarget{ LibraryID: libID, diff --git a/cmd/scan_test.go b/cmd/scan_test.go index 2844d2ed8..c8dce51d6 100644 --- a/cmd/scan_test.go +++ b/cmd/scan_test.go @@ -68,10 +68,10 @@ var _ = Describe("parseTargets", func() { Expect(err.Error()).To(ContainSubstring("invalid library ID")) }) - It("handles negative library ID", func() { - targets, err := parseTargets("-1:Music") - Expect(err).ToNot(HaveOccurred()) // Actually valid - strconv.Atoi accepts negative numbers - Expect(targets[0].LibraryID).To(Equal(-1)) + It("return error on negative library ID", func() { + _, err := parseTargets("-1:Music") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("invalid library ID")) }) It("handles only whitespace", func() { From 33704edc1c542081f1176790a225e99c23eb1d82 Mon Sep 17 00:00:00 2001 From: Deluan Date: Tue, 11 Nov 2025 11:48:48 -0500 Subject: [PATCH 14/40] refactor(scanner): simplify GC method by removing library ID parameter Signed-off-by: Deluan --- core/maintenance_test.go | 4 ++-- model/datastore.go | 2 +- persistence/persistence.go | 8 +------- scanner/scanner.go | 2 +- tests/mock_data_store.go | 2 +- 5 files changed, 6 insertions(+), 12 deletions(-) diff --git a/core/maintenance_test.go b/core/maintenance_test.go index e83d1f8bd..8e8796ffa 100644 --- a/core/maintenance_test.go +++ b/core/maintenance_test.go @@ -373,10 +373,10 @@ type extendedDataStore struct { gcError error } -func (ds *extendedDataStore) GC(ctx context.Context, libraryIDs ...int) error { +func (ds *extendedDataStore) GC(ctx context.Context) error { ds.gcCalled = true if ds.gcError != nil { return ds.gcError } - return ds.MockDataStore.GC(ctx, libraryIDs...) + return ds.MockDataStore.GC(ctx) } diff --git a/model/datastore.go b/model/datastore.go index 536a37274..4290e2134 100644 --- a/model/datastore.go +++ b/model/datastore.go @@ -43,5 +43,5 @@ type DataStore interface { WithTx(block func(tx DataStore) error, scope ...string) error WithTxImmediate(block func(tx DataStore) error, scope ...string) error - GC(ctx context.Context, libraryIDs ...int) error + GC(ctx context.Context) error } diff --git a/persistence/persistence.go b/persistence/persistence.go index 6db3f8575..ac607f85f 100644 --- a/persistence/persistence.go +++ b/persistence/persistence.go @@ -157,7 +157,7 @@ func (s *SQLStore) WithTxImmediate(block func(tx model.DataStore) error, scope . }, scope...) } -func (s *SQLStore) GC(ctx context.Context, libraryIDs ...int) error { +func (s *SQLStore) GC(ctx context.Context) error { trace := func(ctx context.Context, msg string, f func() error) func() error { return func() error { start := time.Now() @@ -167,12 +167,6 @@ func (s *SQLStore) GC(ctx context.Context, libraryIDs ...int) error { } } - // TODO: Implement library-specific filtering for GC operations - // For now, GC runs globally even in selective scans - if len(libraryIDs) > 0 { - log.Debug(ctx, "GC: Running with library filter (not implemented)", "libraries", libraryIDs) - } - err := run.Sequentially( trace(ctx, "purge empty albums", func() error { return s.Album(ctx).(*albumRepository).purgeEmpty() }), trace(ctx, "purge empty artists", func() error { return s.Artist(ctx).(*artistRepository).purgeEmpty() }), diff --git a/scanner/scanner.go b/scanner/scanner.go index 7b3c14d4b..be11ec5a3 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -187,7 +187,7 @@ func (s *scannerImpl) runGC(ctx context.Context, state *scanState) func() error return s.ds.WithTx(func(tx model.DataStore) error { if state.changesDetected.Load() { start := time.Now() - err := tx.GC(ctx, state.affectedLibIDs...) + err := tx.GC(ctx) if err != nil { log.Error(ctx, "Scanner: Error running GC", err) return fmt.Errorf("running GC: %w", err) diff --git a/tests/mock_data_store.go b/tests/mock_data_store.go index 2c0c90f62..56f68a74b 100644 --- a/tests/mock_data_store.go +++ b/tests/mock_data_store.go @@ -258,6 +258,6 @@ func (db *MockDataStore) Resource(ctx context.Context, m any) model.ResourceRepo } } -func (db *MockDataStore) GC(context.Context, ...int) error { +func (db *MockDataStore) GC(context.Context) error { return nil } From 847cc92e8843b56daaa007b28109b6082a004825 Mon Sep 17 00:00:00 2001 From: Deluan Date: Tue, 11 Nov 2025 13:06:11 -0500 Subject: [PATCH 15/40] feat(scanner): update folder scanning to include all descendants of specified folders Signed-off-by: Deluan --- persistence/folder_repository.go | 35 ++++- scanner/phase_1_folders.go | 2 +- scanner/scanner_test.go | 22 ++- scanner/selective_scan_test.go | 240 +++++++++++++++++++++++++++++++ scanner/walk_dir_tree.go | 18 +-- scanner/walk_dir_tree_test.go | 86 +++++++++++ 6 files changed, 371 insertions(+), 32 deletions(-) create mode 100644 scanner/selective_scan_test.go diff --git a/persistence/folder_repository.go b/persistence/folder_repository.go index 1a4caae57..74edf2c12 100644 --- a/persistence/folder_repository.go +++ b/persistence/folder_repository.go @@ -4,7 +4,9 @@ import ( "context" "encoding/json" "fmt" + "os" "slices" + "strings" "time" . "github.com/Masterminds/squirrel" @@ -97,16 +99,37 @@ func (r folderRepository) GetFolderUpdateInfo(lib model.Library, targetPaths ... Eq{"missing": false}, } - // If specific paths are requested, generate folder IDs and filter by them + // If specific paths are requested, include those folders and all their descendants if len(targetPaths) > 0 { + // Collect folder IDs for exact target folders and path conditions for descendants folderIDs := make([]string, 0, len(targetPaths)) - for _, path := range targetPaths { - if path == "" { - path = "." + pathConditions := make(Or, 0, len(targetPaths)*2) + + for _, targetPath := range targetPaths { + if targetPath == "" || targetPath == "." { + // Root path - include everything in this library + pathConditions = Or{} + folderIDs = nil + break } - folderIDs = append(folderIDs, model.FolderID(lib, path)) + // Clean the path to normalize it. Paths stored in the folder table do not have leading/trailing slashes. + cleanPath := strings.Trim(targetPath, string(os.PathSeparator)) + + // Include the target folder itself by ID + folderIDs = append(folderIDs, model.FolderID(lib, cleanPath)) + + // Include all descendants: folders whose path field equals or starts with the target path + // Note: Folder.Path is the directory path, so children have path = targetPath + pathConditions = append(pathConditions, Eq{"path": cleanPath}) + pathConditions = append(pathConditions, Like{"path": cleanPath + "/%"}) + } + + // Combine conditions: exact folder IDs OR descendant path patterns + if len(folderIDs) > 0 { + where = append(where, Or{Eq{"id": folderIDs}, pathConditions}) + } else if len(pathConditions) > 0 { + where = append(where, pathConditions) } - where = append(where, Eq{"id": folderIDs}) } sq := r.newSelect().Columns("id", "updated_at", "hash").Where(where) diff --git a/scanner/phase_1_folders.go b/scanner/phase_1_folders.go index 20554fe97..b33814fdc 100644 --- a/scanner/phase_1_folders.go +++ b/scanner/phase_1_folders.go @@ -76,7 +76,7 @@ type scanJob struct { fs storage.MusicFS cw artwork.CacheWarmer lastUpdates map[string]model.FolderUpdateInfo - targetFolders []string // Specific folders to scan (non-recursive) + targetFolders []string // Specific folders to scan (including all descendants) lock sync.Mutex numFolders atomic.Int64 } diff --git a/scanner/scanner_test.go b/scanner/scanner_test.go index 1e0614573..ecad6f501 100644 --- a/scanner/scanner_test.go +++ b/scanner/scanner_test.go @@ -719,7 +719,7 @@ var _ = Describe("Scanner", Ordered, func() { }) Describe("ScanFolders", func() { - It("scans only specified folders without recursion", func() { + It("scans specified folders recursively including all subdirectories", func() { rock := template(_t{"albumartist": "Rock Artist", "album": "Rock Album"}) jazz := template(_t{"albumartist": "Jazz Artist", "album": "Jazz Album"}) pop := template(_t{"albumartist": "Pop Artist", "album": "Pop Album"}) @@ -735,7 +735,7 @@ var _ = Describe("Scanner", Ordered, func() { // Use the existing library from BeforeEach // (lib is already created with the path "fake:///music") - // Scan only the "rock" and "jazz" folders (not their subdirectories or pop) + // Scan only the "rock" and "jazz" folders (including their subdirectories) targets := []scanner.ScanTarget{ {LibraryID: lib.ID, FolderPath: "rock"}, {LibraryID: lib.ID, FolderPath: "jazz"}, @@ -745,31 +745,29 @@ var _ = Describe("Scanner", Ordered, func() { Expect(err).ToNot(HaveOccurred()) Expect(warnings).To(BeEmpty()) - // Verify only track1, track2, and track4 were imported (not track3, track5, or track6) + // Verify all tracks in rock and jazz folders (including subdirectories) were imported allFiles, err := ds.MediaFile(ctx).GetAll() Expect(err).ToNot(HaveOccurred()) - // Should have exactly 3 tracks (rock/track1, rock/track2, jazz/track4) - Expect(allFiles).To(HaveLen(3)) + // Should have 5 tracks (all rock and jazz tracks including subdirectories) + Expect(allFiles).To(HaveLen(5)) // Get the file paths paths := slice.Map(allFiles, func(mf model.MediaFile) string { return filepath.ToSlash(mf.Path) }) - // Verify the correct files were scanned + // Verify the correct files were scanned (including subdirectories) Expect(paths).To(ContainElements( "rock/track1.mp3", "rock/track2.mp3", + "rock/subdir/track3.mp3", "jazz/track4.mp3", + "jazz/subdir/track5.mp3", )) - // Verify files in subdirectories and pop folder were NOT scanned - Expect(paths).ToNot(ContainElements( - "rock/subdir/track3.mp3", - "jazz/subdir/track5.mp3", - "pop/track6.mp3", - )) + // Verify files in the pop folder were NOT scanned + Expect(paths).ToNot(ContainElement("pop/track6.mp3")) }) }) }) diff --git a/scanner/selective_scan_test.go b/scanner/selective_scan_test.go new file mode 100644 index 000000000..a706af159 --- /dev/null +++ b/scanner/selective_scan_test.go @@ -0,0 +1,240 @@ +package scanner_test + +import ( + "context" + "path/filepath" + "testing/fstest" + + "github.com/Masterminds/squirrel" + "github.com/navidrome/navidrome/conf" + "github.com/navidrome/navidrome/conf/configtest" + "github.com/navidrome/navidrome/core" + "github.com/navidrome/navidrome/core/artwork" + "github.com/navidrome/navidrome/core/metrics" + "github.com/navidrome/navidrome/core/storage/storagetest" + "github.com/navidrome/navidrome/db" + "github.com/navidrome/navidrome/log" + "github.com/navidrome/navidrome/model" + "github.com/navidrome/navidrome/model/request" + "github.com/navidrome/navidrome/persistence" + "github.com/navidrome/navidrome/scanner" + "github.com/navidrome/navidrome/server/events" + "github.com/navidrome/navidrome/tests" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Selective Scan - Deleted Child Folders", Ordered, func() { + var ctx context.Context + var lib model.Library + var ds model.DataStore + var s scanner.Scanner + var fsys storagetest.FakeFS + + BeforeAll(func() { + ctx = request.WithUser(GinkgoT().Context(), model.User{ID: "123", IsAdmin: true}) + tmpDir := GinkgoT().TempDir() + conf.Server.DbPath = filepath.Join(tmpDir, "test-selective-scan.db?_journal_mode=WAL") + log.Warn("Using DB at " + conf.Server.DbPath) + db.Db().SetMaxOpenConns(1) + }) + + BeforeEach(func() { + DeferCleanup(configtest.SetupConfig()) + conf.Server.MusicFolder = "fake:///music" + conf.Server.DevExternalScanner = false + + db.Init(ctx) + DeferCleanup(func() { + Expect(tests.ClearDB()).To(Succeed()) + }) + + ds = persistence.New(db.Db()) + + // Create the admin user in the database to match the context + adminUser := model.User{ + ID: "123", + UserName: "admin", + Name: "Admin User", + IsAdmin: true, + NewPassword: "password", + } + Expect(ds.User(ctx).Put(&adminUser)).To(Succeed()) + + s = scanner.New(ctx, ds, artwork.NoopCacheWarmer(), events.NoopBroker(), + core.NewPlaylists(ds), metrics.NewNoopInstance()) + + lib = model.Library{ID: 1, Name: "Fake Library", Path: "fake:///music"} + Expect(ds.Library(ctx).Put(&lib)).To(Succeed()) + + // Initialize fake filesystem + fsys = storagetest.FakeFS{} + storagetest.Register("fake", &fsys) + }) + + Context("when a child folder is deleted", func() { + var ( + revolver, help func(...map[string]any) *fstest.MapFile + artistFolderID string + album1FolderID string + album2FolderID string + album1TrackIDs []string + album2TrackIDs []string + ) + + BeforeEach(func() { + // Setup template functions for creating test files + revolver = storagetest.Template(_t{"albumartist": "The Beatles", "album": "Revolver", "year": 1966}) + help = storagetest.Template(_t{"albumartist": "The Beatles", "album": "Help!", "year": 1965}) + + // Initial filesystem with nested folders + fsys.SetFiles(fstest.MapFS{ + "The Beatles/Revolver/01 - Taxman.mp3": revolver(storagetest.Track(1, "Taxman")), + "The Beatles/Revolver/02 - Eleanor Rigby.mp3": revolver(storagetest.Track(2, "Eleanor Rigby")), + "The Beatles/Help!/01 - Help!.mp3": help(storagetest.Track(1, "Help!")), + "The Beatles/Help!/02 - The Night Before.mp3": help(storagetest.Track(2, "The Night Before")), + }) + + // First scan - import everything + _, err := s.ScanAll(ctx, true) + Expect(err).ToNot(HaveOccurred()) + + // Verify initial state - all folders exist + folders, err := ds.Folder(ctx).GetAll(model.QueryOptions{Filters: squirrel.Eq{"library_id": lib.ID}}) + Expect(err).ToNot(HaveOccurred()) + Expect(folders).To(HaveLen(4)) // root, Artist, Album1, Album2 + + // Store folder IDs for later verification + for _, f := range folders { + switch f.Name { + case "The Beatles": + artistFolderID = f.ID + case "Revolver": + album1FolderID = f.ID + case "Help!": + album2FolderID = f.ID + } + } + + // Verify all tracks exist + allTracks, err := ds.MediaFile(ctx).GetAll() + Expect(err).ToNot(HaveOccurred()) + Expect(allTracks).To(HaveLen(4)) + + // Store track IDs for later verification + for _, t := range allTracks { + if t.Album == "Revolver" { + album1TrackIDs = append(album1TrackIDs, t.ID) + } else if t.Album == "Help!" { + album2TrackIDs = append(album2TrackIDs, t.ID) + } + } + + // Verify no tracks are missing initially + for _, t := range allTracks { + Expect(t.Missing).To(BeFalse()) + } + }) + + It("should mark child folder and its tracks as missing when parent is scanned", func() { + // Delete the child folder (Help!) from the filesystem + fsys.SetFiles(fstest.MapFS{ + "The Beatles/Revolver/01 - Taxman.mp3": revolver(storagetest.Track(1, "Taxman")), + "The Beatles/Revolver/02 - Eleanor Rigby.mp3": revolver(storagetest.Track(2, "Eleanor Rigby")), + // "The Beatles/Help!" folder and its contents are DELETED + }) + + // Run selective scan on the parent folder (Artist) + // This simulates what the watcher does when a child folder is deleted + _, err := s.ScanFolders(ctx, false, []scanner.ScanTarget{ + {LibraryID: lib.ID, FolderPath: "The Beatles"}, + }) + Expect(err).ToNot(HaveOccurred()) + + // Verify the deleted child folder is now marked as missing + deletedFolder, err := ds.Folder(ctx).Get(album2FolderID) + Expect(err).ToNot(HaveOccurred()) + Expect(deletedFolder.Missing).To(BeTrue(), "Deleted child folder should be marked as missing") + + // Verify the deleted folder's tracks are marked as missing + for _, trackID := range album2TrackIDs { + track, err := ds.MediaFile(ctx).Get(trackID) + Expect(err).ToNot(HaveOccurred()) + Expect(track.Missing).To(BeTrue(), "Track in deleted folder should be marked as missing") + } + + // Verify the parent folder is still present and not marked as missing + parentFolder, err := ds.Folder(ctx).Get(artistFolderID) + Expect(err).ToNot(HaveOccurred()) + Expect(parentFolder.Missing).To(BeFalse(), "Parent folder should not be marked as missing") + + // Verify the sibling folder and its tracks are still present and not missing + siblingFolder, err := ds.Folder(ctx).Get(album1FolderID) + Expect(err).ToNot(HaveOccurred()) + Expect(siblingFolder.Missing).To(BeFalse(), "Sibling folder should not be marked as missing") + + for _, trackID := range album1TrackIDs { + track, err := ds.MediaFile(ctx).Get(trackID) + Expect(err).ToNot(HaveOccurred()) + Expect(track.Missing).To(BeFalse(), "Track in sibling folder should not be marked as missing") + } + }) + + It("should mark deeply nested child folders as missing", func() { + // Add a deeply nested folder structure + fsys.SetFiles(fstest.MapFS{ + "The Beatles/Revolver/01 - Taxman.mp3": revolver(storagetest.Track(1, "Taxman")), + "The Beatles/Revolver/02 - Eleanor Rigby.mp3": revolver(storagetest.Track(2, "Eleanor Rigby")), + "The Beatles/Help!/01 - Help!.mp3": help(storagetest.Track(1, "Help!")), + "The Beatles/Help!/02 - The Night Before.mp3": help(storagetest.Track(2, "The Night Before")), + "The Beatles/Help!/Bonus/01 - Bonus Track.mp3": help(storagetest.Track(99, "Bonus Track")), + "The Beatles/Help!/Bonus/Nested/01 - Deep Track.mp3": help(storagetest.Track(100, "Deep Track")), + }) + + // Rescan to import the new nested structure + _, err := s.ScanAll(ctx, true) + Expect(err).ToNot(HaveOccurred()) + + // Verify nested folders were created + allFolders, err := ds.Folder(ctx).GetAll(model.QueryOptions{Filters: squirrel.Eq{"library_id": lib.ID}}) + Expect(err).ToNot(HaveOccurred()) + Expect(len(allFolders)).To(BeNumerically(">", 4), "Should have more folders with nested structure") + + // Now delete the entire Help! folder including nested children + fsys.SetFiles(fstest.MapFS{ + "The Beatles/Revolver/01 - Taxman.mp3": revolver(storagetest.Track(1, "Taxman")), + "The Beatles/Revolver/02 - Eleanor Rigby.mp3": revolver(storagetest.Track(2, "Eleanor Rigby")), + // All Help! subfolders are deleted + }) + + // Run selective scan on parent + _, err = s.ScanFolders(ctx, false, []scanner.ScanTarget{ + {LibraryID: lib.ID, FolderPath: "The Beatles"}, + }) + Expect(err).ToNot(HaveOccurred()) + + // Verify all Help! folders (including nested ones) are marked as missing + missingFolders, err := ds.Folder(ctx).GetAll(model.QueryOptions{ + Filters: squirrel.And{ + squirrel.Eq{"library_id": lib.ID}, + squirrel.Eq{"missing": true}, + }, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(len(missingFolders)).To(BeNumerically(">", 0), "At least one folder should be marked as missing") + + // Verify all tracks in deleted folders are marked as missing + allTracks, err := ds.MediaFile(ctx).GetAll() + Expect(err).ToNot(HaveOccurred()) + Expect(allTracks).To(HaveLen(6)) + + for _, track := range allTracks { + if track.Album == "Help!" { + Expect(track.Missing).To(BeTrue(), "All tracks in deleted Help! folder should be marked as missing") + } else if track.Album == "Revolver" { + Expect(track.Missing).To(BeFalse(), "Tracks in Revolver folder should not be marked as missing") + } + } + }) + }) +}) diff --git a/scanner/walk_dir_tree.go b/scanner/walk_dir_tree.go index d431fef23..59c7000e5 100644 --- a/scanner/walk_dir_tree.go +++ b/scanner/walk_dir_tree.go @@ -30,7 +30,7 @@ func walkDirTree(ctx context.Context, job *scanJob) (<-chan *folderEntry, error) return results, nil } -// loadSpecificFolders loads only the specified folders without recursing into subdirectories +// loadSpecificFolders loads the specified folders and recursively walks their subdirectories func loadSpecificFolders(ctx context.Context, job *scanJob, targetFolders []string) (<-chan *folderEntry, error) { results := make(chan *folderEntry) go func() { @@ -44,22 +44,14 @@ func loadSpecificFolders(ctx context.Context, job *scanJob, targetFolders []stri checker := newIgnoreChecker(job.fs) _ = checker.PushAllParents(ctx, folderPath) - // Load only this specific folder (no recursion) - folder, _, err := loadDir(ctx, job, folderPath, checker) + // Recursively walk this folder and all its children + err := walkFolder(ctx, job, folderPath, checker, results) if err != nil { - log.Warn(ctx, "Scanner: Error loading target folder. Skipping", "path", folderPath, err) + log.Error(ctx, "Scanner: Error walking target folder", "path", folderPath, err) continue } - - folder.path = path.Clean(folderPath) - folder.elapsed.Start() - log.Trace(ctx, "Scanner: Found target directory", " path", folder.path, "audioFiles", maps.Keys(folder.audioFiles), - "images", maps.Keys(folder.imageFiles), "playlists", folder.numPlaylists, "imagesUpdatedAt", folder.imagesUpdatedAt, - "updTime", folder.updTime, "modTime", folder.modTime) - - results <- folder } - log.Debug(ctx, "Scanner: Finished reading target folders", "lib", job.lib.Name, "path", job.lib.Path, "numFolders", len(targetFolders)) + log.Debug(ctx, "Scanner: Finished reading target folders", "lib", job.lib.Name, "path", job.lib.Path, "numFolders", job.numFolders.Load()) }() return results, nil } diff --git a/scanner/walk_dir_tree_test.go b/scanner/walk_dir_tree_test.go index 1cab8a0b7..840cd5dd2 100644 --- a/scanner/walk_dir_tree_test.go +++ b/scanner/walk_dir_tree_test.go @@ -103,6 +103,92 @@ var _ = Describe("walk_dir_tree", func() { ) }) + Describe("loadSpecificFolders", func() { + var ( + fsys storage.MusicFS + job *scanJob + ctx context.Context + ) + + BeforeEach(func() { + DeferCleanup(configtest.SetupConfig()) + ctx = GinkgoT().Context() + fsys = &mockMusicFS{ + FS: fstest.MapFS{ + "Artist/Album1/track1.mp3": {}, + "Artist/Album1/track2.mp3": {}, + "Artist/Album2/track1.mp3": {}, + "Artist/Album2/track2.mp3": {}, + "Artist/Album2/Sub/track3.mp3": {}, + "OtherArtist/Album3/track1.mp3": {}, + }, + } + job = &scanJob{ + fs: fsys, + lib: model.Library{Path: "/music"}, + } + }) + + It("should recursively walk all subdirectories of target folders", func() { + results, err := loadSpecificFolders(ctx, job, []string{"Artist"}) + Expect(err).ToNot(HaveOccurred()) + + folders := map[string]*folderEntry{} + g := errgroup.Group{} + g.Go(func() error { + for folder := range results { + folders[folder.path] = folder + } + return nil + }) + _ = g.Wait() + + // Should include the target folder and all its descendants + Expect(folders).To(SatisfyAll( + HaveKey("Artist"), + HaveKey("Artist/Album1"), + HaveKey("Artist/Album2"), + HaveKey("Artist/Album2/Sub"), + )) + + // Should not include folders outside the target + Expect(folders).ToNot(HaveKey("OtherArtist")) + Expect(folders).ToNot(HaveKey("OtherArtist/Album3")) + + // Verify audio files are present + Expect(folders["Artist/Album1"].audioFiles).To(HaveLen(2)) + Expect(folders["Artist/Album2"].audioFiles).To(HaveLen(2)) + Expect(folders["Artist/Album2/Sub"].audioFiles).To(HaveLen(1)) + }) + + It("should handle multiple target folders", func() { + results, err := loadSpecificFolders(ctx, job, []string{"Artist/Album1", "OtherArtist"}) + Expect(err).ToNot(HaveOccurred()) + + folders := map[string]*folderEntry{} + g := errgroup.Group{} + g.Go(func() error { + for folder := range results { + folders[folder.path] = folder + } + return nil + }) + _ = g.Wait() + + // Should include both target folders and their descendants + Expect(folders).To(SatisfyAll( + HaveKey("Artist/Album1"), + HaveKey("OtherArtist"), + HaveKey("OtherArtist/Album3"), + )) + + // Should not include other folders + Expect(folders).ToNot(HaveKey("Artist")) + Expect(folders).ToNot(HaveKey("Artist/Album2")) + Expect(folders).ToNot(HaveKey("Artist/Album2/Sub")) + }) + }) + Describe("helper functions", func() { dir, _ := os.Getwd() fsys := os.DirFS(dir) From 265a6973bc8a05935e897c938c0d83b68cad9d0d Mon Sep 17 00:00:00 2001 From: Deluan Date: Tue, 11 Nov 2025 14:43:50 -0500 Subject: [PATCH 16/40] feat(subsonic): allow selective scan in the /startScan endpoint Signed-off-by: Deluan --- cmd/scan.go | 43 +--- cmd/scan_test.go | 57 +--- scanner/controller.go | 49 +++- scanner/controller_test.go | 82 ++++++ server/subsonic/library_scanning.go | 25 +- server/subsonic/library_scanning_test.go | 315 +++++++++++++++++++++++ 6 files changed, 468 insertions(+), 103 deletions(-) create mode 100644 server/subsonic/library_scanning_test.go diff --git a/cmd/scan.go b/cmd/scan.go index 9c92de966..8f2dc8acf 100644 --- a/cmd/scan.go +++ b/cmd/scan.go @@ -3,9 +3,7 @@ package cmd import ( "context" "encoding/gob" - "fmt" "os" - "strconv" "strings" "github.com/navidrome/navidrome/core" @@ -98,44 +96,7 @@ func runScanner(ctx context.Context) { } // parseTargets parses the comma-separated targets string into ScanTarget structs -// Format: "libraryID:folderPath,libraryID:folderPath,..." -// Example: "1:Music/Rock,1:Music/Jazz,2:Classical" func parseTargets(targetsStr string) ([]scanner.ScanTarget, error) { - parts := strings.Split(targetsStr, ",") - targets := make([]scanner.ScanTarget, 0, len(parts)) - - for _, part := range parts { - part = strings.TrimSpace(part) - if part == "" { - continue - } - - // Split by the first colon - colonIdx := strings.Index(part, ":") - if colonIdx == -1 { - return nil, fmt.Errorf("invalid target format: %q (expected libraryID:folderPath)", part) - } - - libIDStr := part[:colonIdx] - folderPath := part[colonIdx+1:] - - libID, err := strconv.Atoi(libIDStr) - if err != nil { - return nil, fmt.Errorf("invalid library ID %q: %w", libIDStr, err) - } - if libID <= 0 { - return nil, fmt.Errorf("invalid library ID %q", libIDStr) - } - - targets = append(targets, scanner.ScanTarget{ - LibraryID: libID, - FolderPath: folderPath, - }) - } - - if len(targets) == 0 { - return nil, fmt.Errorf("no valid targets found in %q", targetsStr) - } - - return targets, nil + targets := strings.Split(targetsStr, ",") + return scanner.ParseTargets(targets) } diff --git a/cmd/scan_test.go b/cmd/scan_test.go index c8dce51d6..fecd79c4b 100644 --- a/cmd/scan_test.go +++ b/cmd/scan_test.go @@ -8,14 +8,6 @@ import ( var _ = Describe("parseTargets", func() { Context("Valid targets", func() { - It("parses a single target", func() { - targets, err := parseTargets("1:Music/Rock") - Expect(err).ToNot(HaveOccurred()) - Expect(targets).To(HaveLen(1)) - Expect(targets[0].LibraryID).To(Equal(1)) - Expect(targets[0].FolderPath).To(Equal("Music/Rock")) - }) - It("parses multiple targets", func() { targets, err := parseTargets("1:Music/Rock,2:Jazz,3:Classical/Beethoven") Expect(err).ToNot(HaveOccurred()) @@ -25,59 +17,12 @@ var _ = Describe("parseTargets", func() { Expect(targets[2]).To(Equal(scanner.ScanTarget{LibraryID: 3, FolderPath: "Classical/Beethoven"})) }) - It("handles targets with spaces around commas", func() { - targets, err := parseTargets("1:Music/Rock And Roll, 2:Jazz , 3:Classical") - Expect(err).ToNot(HaveOccurred()) - Expect(targets).To(HaveLen(3)) - Expect(targets[0].FolderPath).To(Equal("Music/Rock And Roll")) - }) - - It("handles paths with colons after the first colon", func() { - targets, err := parseTargets("1:C:/Music/Rock") - Expect(err).ToNot(HaveOccurred()) - Expect(targets).To(HaveLen(1)) - Expect(targets[0].LibraryID).To(Equal(1)) - Expect(targets[0].FolderPath).To(Equal("C:/Music/Rock")) - }) - - It("handles empty folder paths", func() { - targets, err := parseTargets("1:,2:") - Expect(err).ToNot(HaveOccurred()) - Expect(targets).To(HaveLen(2)) - Expect(targets[0].FolderPath).To(BeEmpty()) - Expect(targets[1].FolderPath).To(BeEmpty()) - }) - }) - - Context("Invalid targets", func() { It("returns error for empty string", func() { _, err := parseTargets("") Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("no valid targets")) }) - It("returns error for missing colon", func() { - _, err := parseTargets("1Music") - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("invalid target format")) - }) - - It("returns error for invalid library ID", func() { - _, err := parseTargets("abc:Music") - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("invalid library ID")) - }) - - It("return error on negative library ID", func() { - _, err := parseTargets("-1:Music") - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("invalid library ID")) - }) - - It("handles only whitespace", func() { - _, err := parseTargets(" , , ") - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("no valid targets")) - }) + // Other test cases are covered in scanner/controller_test.go }) }) diff --git a/scanner/controller.go b/scanner/controller.go index 28dbb7287..f6a80cec7 100644 --- a/scanner/controller.go +++ b/scanner/controller.go @@ -4,6 +4,8 @@ import ( "context" "errors" "fmt" + "strconv" + "strings" "sync/atomic" "time" @@ -37,11 +39,52 @@ func (st ScanTarget) String() string { return fmt.Sprintf("%d:%s", st.LibraryID, st.FolderPath) } +// ParseTargets parses scan targets strings into ScanTarget structs. +// Example: []string{"1:Music/Rock", "2:Classical"} +func ParseTargets(libFolders []string) ([]ScanTarget, error) { + targets := make([]ScanTarget, 0, len(libFolders)) + + for _, part := range libFolders { + part = strings.TrimSpace(part) + if part == "" { + continue + } + + // Split by the first colon + colonIdx := strings.Index(part, ":") + if colonIdx == -1 { + return nil, fmt.Errorf("invalid target format: %q (expected libraryID:folderPath)", part) + } + + libIDStr := part[:colonIdx] + folderPath := part[colonIdx+1:] + + libID, err := strconv.Atoi(libIDStr) + if err != nil { + return nil, fmt.Errorf("invalid library ID %q: %w", libIDStr, err) + } + if libID <= 0 { + return nil, fmt.Errorf("invalid library ID %q", libIDStr) + } + + targets = append(targets, ScanTarget{ + LibraryID: libID, + FolderPath: folderPath, + }) + } + + if len(targets) == 0 { + return nil, fmt.Errorf("no valid targets found") + } + + return targets, nil +} + type Scanner interface { - // ScanAll starts a full scan of the music library. This is a blocking operation. + // ScanAll starts a scan of all libraries. This is a blocking operation. ScanAll(ctx context.Context, fullScan bool) (warnings []string, err error) - // ScanFolders scans specific library/folder pairs without recursing into subdirectories. - // This is a blocking operation. + // ScanFolders scans specific library/folder pairs, recursing into subdirectories. + // If targets is nil, it scans all libraries. This is a blocking operation. ScanFolders(ctx context.Context, fullScan bool, targets []ScanTarget) (warnings []string, err error) Status(context.Context) (*StatusInfo, error) } diff --git a/scanner/controller_test.go b/scanner/controller_test.go index e551e15b1..d1109b0be 100644 --- a/scanner/controller_test.go +++ b/scanner/controller_test.go @@ -53,3 +53,85 @@ var _ = Describe("Controller", func() { }) }) }) + +var _ = Describe("ParseTargets", func() { + It("parses multiple entries in slice", func() { + targets, err := scanner.ParseTargets([]string{"1:Music/Rock", "1:Music/Jazz", "2:Classical"}) + Expect(err).ToNot(HaveOccurred()) + Expect(targets).To(HaveLen(3)) + Expect(targets[0].LibraryID).To(Equal(1)) + Expect(targets[0].FolderPath).To(Equal("Music/Rock")) + Expect(targets[1].LibraryID).To(Equal(1)) + Expect(targets[1].FolderPath).To(Equal("Music/Jazz")) + Expect(targets[2].LibraryID).To(Equal(2)) + Expect(targets[2].FolderPath).To(Equal("Classical")) + }) + + It("handles empty folder paths", func() { + targets, err := scanner.ParseTargets([]string{"1:", "2:"}) + Expect(err).ToNot(HaveOccurred()) + Expect(targets).To(HaveLen(2)) + Expect(targets[0].FolderPath).To(Equal("")) + Expect(targets[1].FolderPath).To(Equal("")) + }) + + It("trims whitespace from entries", func() { + targets, err := scanner.ParseTargets([]string{" 1:Music/Rock", " 2:Classical "}) + Expect(err).ToNot(HaveOccurred()) + Expect(targets).To(HaveLen(2)) + Expect(targets[0].LibraryID).To(Equal(1)) + Expect(targets[0].FolderPath).To(Equal("Music/Rock")) + Expect(targets[1].LibraryID).To(Equal(2)) + Expect(targets[1].FolderPath).To(Equal("Classical")) + }) + + It("skips empty strings", func() { + targets, err := scanner.ParseTargets([]string{"1:Music/Rock", "", "2:Classical"}) + Expect(err).ToNot(HaveOccurred()) + Expect(targets).To(HaveLen(2)) + }) + + It("handles paths with colons", func() { + targets, err := scanner.ParseTargets([]string{"1:C:/Music/Rock", "2:/path:with:colons"}) + Expect(err).ToNot(HaveOccurred()) + Expect(targets).To(HaveLen(2)) + Expect(targets[0].FolderPath).To(Equal("C:/Music/Rock")) + Expect(targets[1].FolderPath).To(Equal("/path:with:colons")) + }) + + It("returns error for invalid format without colon", func() { + _, err := scanner.ParseTargets([]string{"1Music/Rock"}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("invalid target format")) + }) + + It("returns error for non-numeric library ID", func() { + _, err := scanner.ParseTargets([]string{"abc:Music/Rock"}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("invalid library ID")) + }) + + It("returns error for negative library ID", func() { + _, err := scanner.ParseTargets([]string{"-1:Music/Rock"}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("invalid library ID")) + }) + + It("returns error for zero library ID", func() { + _, err := scanner.ParseTargets([]string{"0:Music/Rock"}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("invalid library ID")) + }) + + It("returns error for empty input", func() { + _, err := scanner.ParseTargets([]string{}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("no valid targets found")) + }) + + It("returns error for all empty strings", func() { + _, err := scanner.ParseTargets([]string{"", " ", ""}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("no valid targets found")) + }) +}) diff --git a/server/subsonic/library_scanning.go b/server/subsonic/library_scanning.go index b6ccb9ae6..abd388428 100644 --- a/server/subsonic/library_scanning.go +++ b/server/subsonic/library_scanning.go @@ -1,11 +1,13 @@ package subsonic import ( + "fmt" "net/http" "time" "github.com/navidrome/navidrome/log" "github.com/navidrome/navidrome/model/request" + "github.com/navidrome/navidrome/scanner" "github.com/navidrome/navidrome/server/subsonic/responses" "github.com/navidrome/navidrome/utils/req" ) @@ -44,15 +46,32 @@ func (api *Router) StartScan(r *http.Request) (*responses.Subsonic, error) { p := req.Params(r) fullScan := p.BoolOr("fullScan", false) + // Parse optional path parameters for selective scanning + var targets []scanner.ScanTarget + if pathParams, err := p.Strings("path"); err == nil && len(pathParams) > 0 { + targets, err = scanner.ParseTargets(pathParams) + if err != nil { + return nil, newError(responses.ErrorGeneric, fmt.Sprintf("Invalid path parameter: %v", err)) + } + } + go func() { start := time.Now() - log.Info(ctx, "Triggering manual scan", "fullScan", fullScan, "user", loggedUser.UserName) - _, err := api.scanner.ScanAll(ctx, fullScan) + var err error + + if len(targets) > 0 { + log.Info(ctx, "Triggering on-demand scan", "fullScan", fullScan, "targets", len(targets), "user", loggedUser.UserName) + _, err = api.scanner.ScanFolders(ctx, fullScan, targets) + } else { + log.Info(ctx, "Triggering on-demand scan", "fullScan", fullScan, "user", loggedUser.UserName) + _, err = api.scanner.ScanAll(ctx, fullScan) + } + if err != nil { log.Error(ctx, "Error scanning", err) return } - log.Info(ctx, "Manual scan complete", "user", loggedUser.UserName, "elapsed", time.Since(start)) + log.Info(ctx, "On-demand scan complete", "user", loggedUser.UserName, "elapsed", time.Since(start)) }() return api.GetScanStatus(r) diff --git a/server/subsonic/library_scanning_test.go b/server/subsonic/library_scanning_test.go new file mode 100644 index 000000000..413cd8606 --- /dev/null +++ b/server/subsonic/library_scanning_test.go @@ -0,0 +1,315 @@ +package subsonic + +import ( + "context" + "net/http/httptest" + + "github.com/navidrome/navidrome/model" + "github.com/navidrome/navidrome/model/request" + "github.com/navidrome/navidrome/scanner" + "github.com/navidrome/navidrome/server/subsonic/responses" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("LibraryScanning", func() { + var api *Router + var ms *mockScanner + + BeforeEach(func() { + ms = &mockScanner{} + api = &Router{scanner: ms} + }) + + Describe("StartScan", func() { + It("requires admin authentication", func() { + // Create non-admin user + ctx := request.WithUser(context.Background(), model.User{ + ID: "user-id", + IsAdmin: false, + }) + + // Create request + r := httptest.NewRequest("GET", "/rest/startScan", nil) + r = r.WithContext(ctx) + + // Call endpoint + response, err := api.StartScan(r) + + // Should return authorization error + Expect(err).To(HaveOccurred()) + Expect(response).To(BeNil()) + subErr, ok := err.(subError) + Expect(ok).To(BeTrue()) + Expect(subErr.code).To(Equal(responses.ErrorAuthorizationFail)) + }) + + It("triggers a full scan with no parameters", func() { + // Create admin user + ctx := request.WithUser(context.Background(), model.User{ + ID: "admin-id", + IsAdmin: true, + }) + + // Create request with no parameters + r := httptest.NewRequest("GET", "/rest/startScan", nil) + r = r.WithContext(ctx) + + // Call endpoint + response, err := api.StartScan(r) + + // Should succeed + Expect(err).ToNot(HaveOccurred()) + Expect(response).ToNot(BeNil()) + + // Verify ScanAll was called (eventually, since it's in a goroutine) + Eventually(func() bool { + return ms.scanAllCalled + }).Should(BeTrue()) + Expect(ms.scanAllFullScan).To(BeFalse()) + }) + + It("triggers a full scan with fullScan=true", func() { + // Create admin user + ctx := request.WithUser(context.Background(), model.User{ + ID: "admin-id", + IsAdmin: true, + }) + + // Create request with fullScan parameter + r := httptest.NewRequest("GET", "/rest/startScan?fullScan=true", nil) + r = r.WithContext(ctx) + + // Call endpoint + response, err := api.StartScan(r) + + // Should succeed + Expect(err).ToNot(HaveOccurred()) + Expect(response).ToNot(BeNil()) + + // Verify ScanAll was called with fullScan=true + Eventually(func() bool { + return ms.scanAllCalled + }).Should(BeTrue()) + Expect(ms.scanAllFullScan).To(BeTrue()) + }) + + It("triggers a selective scan with single path parameter", func() { + // Create admin user + ctx := request.WithUser(context.Background(), model.User{ + ID: "admin-id", + IsAdmin: true, + }) + + // Create request with single path parameter + r := httptest.NewRequest("GET", "/rest/startScan?path=1:Music/Rock", nil) + r = r.WithContext(ctx) + + // Call endpoint + response, err := api.StartScan(r) + + // Should succeed + Expect(err).ToNot(HaveOccurred()) + Expect(response).ToNot(BeNil()) + + // Verify ScanFolders was called with correct targets + Eventually(func() bool { + return ms.scanFoldersCalled + }).Should(BeTrue()) + Expect(ms.scanFoldersTargets).To(HaveLen(1)) + Expect(ms.scanFoldersTargets[0].LibraryID).To(Equal(1)) + Expect(ms.scanFoldersTargets[0].FolderPath).To(Equal("Music/Rock")) + }) + + It("triggers a selective scan with multiple path parameters", func() { + // Create admin user + ctx := request.WithUser(context.Background(), model.User{ + ID: "admin-id", + IsAdmin: true, + }) + + // Create request with multiple path parameters + r := httptest.NewRequest("GET", "/rest/startScan?path=1:Music/Reggae&path=2:Classical/Bach", nil) + r = r.WithContext(ctx) + + // Call endpoint + response, err := api.StartScan(r) + + // Should succeed + Expect(err).ToNot(HaveOccurred()) + Expect(response).ToNot(BeNil()) + + // Verify ScanFolders was called with correct targets + Eventually(func() bool { + return ms.scanFoldersCalled + }).Should(BeTrue()) + Expect(ms.scanFoldersTargets).To(HaveLen(2)) + Expect(ms.scanFoldersTargets[0].LibraryID).To(Equal(1)) + Expect(ms.scanFoldersTargets[0].FolderPath).To(Equal("Music/Reggae")) + Expect(ms.scanFoldersTargets[1].LibraryID).To(Equal(2)) + Expect(ms.scanFoldersTargets[1].FolderPath).To(Equal("Classical/Bach")) + }) + + It("triggers a selective full scan with path and fullScan parameters", func() { + // Create admin user + ctx := request.WithUser(context.Background(), model.User{ + ID: "admin-id", + IsAdmin: true, + }) + + // Create request with path and fullScan parameters + r := httptest.NewRequest("GET", "/rest/startScan?path=1:Music/Jazz&fullScan=true", nil) + r = r.WithContext(ctx) + + // Call endpoint + response, err := api.StartScan(r) + + // Should succeed + Expect(err).ToNot(HaveOccurred()) + Expect(response).ToNot(BeNil()) + + // Verify ScanFolders was called with fullScan=true + Eventually(func() bool { + return ms.scanFoldersCalled + }).Should(BeTrue()) + Expect(ms.scanFoldersFullScan).To(BeTrue()) + Expect(ms.scanFoldersTargets).To(HaveLen(1)) + }) + + It("returns error for invalid path format", func() { + // Create admin user + ctx := request.WithUser(context.Background(), model.User{ + ID: "admin-id", + IsAdmin: true, + }) + + // Create request with invalid path format (missing colon) + r := httptest.NewRequest("GET", "/rest/startScan?path=1MusicRock", nil) + r = r.WithContext(ctx) + + // Call endpoint + response, err := api.StartScan(r) + + // Should return error + Expect(err).To(HaveOccurred()) + Expect(response).To(BeNil()) + subErr, ok := err.(subError) + Expect(ok).To(BeTrue()) + Expect(subErr.code).To(Equal(responses.ErrorGeneric)) + }) + + It("returns error for invalid library ID", func() { + // Create admin user + ctx := request.WithUser(context.Background(), model.User{ + ID: "admin-id", + IsAdmin: true, + }) + + // Create request with invalid library ID + r := httptest.NewRequest("GET", "/rest/startScan?path=0:Music/Rock", nil) + r = r.WithContext(ctx) + + // Call endpoint + response, err := api.StartScan(r) + + // Should return error + Expect(err).To(HaveOccurred()) + Expect(response).To(BeNil()) + subErr, ok := err.(subError) + Expect(ok).To(BeTrue()) + Expect(subErr.code).To(Equal(responses.ErrorGeneric)) + }) + + It("handles URL-encoded paths", func() { + // Create admin user + ctx := request.WithUser(context.Background(), model.User{ + ID: "admin-id", + IsAdmin: true, + }) + + // Create request with URL-encoded path + r := httptest.NewRequest("GET", "/rest/startScan?path=1:The%20Beatles", nil) + r = r.WithContext(ctx) + + // Call endpoint + response, err := api.StartScan(r) + + // Should succeed + Expect(err).ToNot(HaveOccurred()) + Expect(response).ToNot(BeNil()) + + // Verify path was decoded correctly + Eventually(func() bool { + return ms.scanFoldersCalled + }).Should(BeTrue()) + Expect(ms.scanFoldersTargets[0].FolderPath).To(Equal("The Beatles")) + }) + }) + + Describe("GetScanStatus", func() { + It("returns scan status", func() { + // Setup mock scanner status + ms.statusResponse = &scanner.StatusInfo{ + Scanning: false, + Count: 100, + FolderCount: 10, + } + + // Create request + ctx := context.Background() + r := httptest.NewRequest("GET", "/rest/getScanStatus", nil) + r = r.WithContext(ctx) + + // Call endpoint + response, err := api.GetScanStatus(r) + + // Should succeed + Expect(err).ToNot(HaveOccurred()) + Expect(response).ToNot(BeNil()) + Expect(response.ScanStatus).ToNot(BeNil()) + Expect(response.ScanStatus.Scanning).To(BeFalse()) + Expect(response.ScanStatus.Count).To(Equal(int64(100))) + Expect(response.ScanStatus.FolderCount).To(Equal(int64(10))) + }) + }) +}) + +// mockScanner is a test double for the scanner.Scanner interface +type mockScanner struct { + // ScanAll tracking + scanAllCalled bool + scanAllFullScan bool + scanAllError error + scanAllWarnings []string + + // ScanFolders tracking + scanFoldersCalled bool + scanFoldersFullScan bool + scanFoldersTargets []scanner.ScanTarget + scanFoldersError error + scanFoldersWarnings []string + + // Status tracking + statusResponse *scanner.StatusInfo + statusError error +} + +func (m *mockScanner) ScanAll(ctx context.Context, fullScan bool) ([]string, error) { + m.scanAllCalled = true + m.scanAllFullScan = fullScan + return m.scanAllWarnings, m.scanAllError +} + +func (m *mockScanner) ScanFolders(ctx context.Context, fullScan bool, targets []scanner.ScanTarget) ([]string, error) { + m.scanFoldersCalled = true + m.scanFoldersFullScan = fullScan + m.scanFoldersTargets = targets + return m.scanFoldersWarnings, m.scanFoldersError +} + +func (m *mockScanner) Status(ctx context.Context) (*scanner.StatusInfo, error) { + if m.statusResponse == nil { + return &scanner.StatusInfo{}, m.statusError + } + return m.statusResponse, m.statusError +} From 7e396fd6ce08314957ecb1b73d48a6732195a620 Mon Sep 17 00:00:00 2001 From: Deluan Date: Tue, 11 Nov 2025 15:00:01 -0500 Subject: [PATCH 17/40] refactor(scanner): update CallScan to handle specific library/folder pairs Signed-off-by: Deluan --- cmd/scan.go | 2 +- scanner/controller.go | 15 +++++---------- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/cmd/scan.go b/cmd/scan.go index 8f2dc8acf..2c43baf62 100644 --- a/cmd/scan.go +++ b/cmd/scan.go @@ -82,7 +82,7 @@ func runScanner(ctx context.Context) { log.Info(ctx, "Scanning specific folders", "numTargets", len(scanTargets)) } - progress, err := scanner.CallScanFolders(ctx, ds, pls, fullScan, scanTargets) + progress, err := scanner.CallScan(ctx, ds, pls, fullScan, scanTargets) if err != nil { log.Fatal(ctx, "Failed to scan", err) } diff --git a/scanner/controller.go b/scanner/controller.go index f6a80cec7..e34246c4b 100644 --- a/scanner/controller.go +++ b/scanner/controller.go @@ -122,15 +122,10 @@ func (s *controller) getScanner() scanner { return &scannerImpl{ds: s.ds, cw: s.cw, pls: s.pls} } -// CallScan starts an in-process scan of the music library. +// CallScan starts an in-process scan of specific library/folder pairs. +// If targets is empty, it scans all libraries. // This is meant to be called from the command line (see cmd/scan.go). -func CallScan(ctx context.Context, ds model.DataStore, pls core.Playlists, fullScan bool) (<-chan *ProgressInfo, error) { - return CallScanFolders(ctx, ds, pls, fullScan, nil) -} - -// CallScanFolders starts an in-process scan of specific library/folder pairs. -// If targets is nil, it scans all libraries. This is meant to be called from the command line. -func CallScanFolders(ctx context.Context, ds model.DataStore, pls core.Playlists, fullScan bool, targets []ScanTarget) (<-chan *ProgressInfo, error) { +func CallScan(ctx context.Context, ds model.DataStore, pls core.Playlists, fullScan bool, targets []ScanTarget) (<-chan *ProgressInfo, error) { release, err := lockScan(ctx) if err != nil { return nil, err @@ -142,7 +137,7 @@ func CallScanFolders(ctx context.Context, ds model.DataStore, pls core.Playlists go func() { defer close(progress) scanner := &scannerImpl{ds: ds, cw: artwork.NoopCacheWarmer(), pls: pls} - if targets == nil { + if len(targets) == 0 { scanner.scanAll(ctx, fullScan, progress) } else { scanner.scanFolders(ctx, fullScan, targets, progress) @@ -296,7 +291,7 @@ func (s *controller) ScanFolders(requestCtx context.Context, fullScan bool, targ go func() { defer close(progress) scanner := s.getScanner() - if targets == nil { + if len(targets) == 0 { scanner.scanAll(ctx, fullScan, progress) } else { scanner.scanFolders(ctx, fullScan, targets, progress) From dec658c235502f2315cbc68cb419df5e3f14d0e1 Mon Sep 17 00:00:00 2001 From: Deluan Date: Tue, 11 Nov 2025 15:22:06 -0500 Subject: [PATCH 18/40] refactor(scanner): streamline scanning logic by removing scanAll method Signed-off-by: Deluan --- core/library.go | 2 +- scanner/controller.go | 16 +++++----------- scanner/external.go | 4 ---- scanner/scanner.go | 4 ---- scanner/watcher_test.go | 26 +++++++++++++------------- 5 files changed, 19 insertions(+), 33 deletions(-) diff --git a/core/library.go b/core/library.go index 7abd35c8f..8e17445ec 100644 --- a/core/library.go +++ b/core/library.go @@ -21,7 +21,7 @@ import ( "github.com/navidrome/navidrome/utils/slice" ) -// Scanner interface for triggering scans +// Scanner interface for triggering scans. This is a subset of the full scanner.Scanner interface. type Scanner interface { ScanAll(ctx context.Context, fullScan bool) (warnings []string, err error) } diff --git a/scanner/controller.go b/scanner/controller.go index e34246c4b..70e2d6a69 100644 --- a/scanner/controller.go +++ b/scanner/controller.go @@ -137,11 +137,7 @@ func CallScan(ctx context.Context, ds model.DataStore, pls core.Playlists, fullS go func() { defer close(progress) scanner := &scannerImpl{ds: ds, cw: artwork.NoopCacheWarmer(), pls: pls} - if len(targets) == 0 { - scanner.scanAll(ctx, fullScan, progress) - } else { - scanner.scanFolders(ctx, fullScan, targets, progress) - } + scanner.scanFolders(ctx, fullScan, targets, progress) }() return progress, nil } @@ -161,8 +157,10 @@ type ProgressInfo struct { ForceUpdate bool } +// scanner defines the interface for different scanner implementations. +// This allows for swapping between in-process and external scanners. type scanner interface { - scanAll(ctx context.Context, fullScan bool, progress chan<- *ProgressInfo) + // scanFolders performs the actual scanning of folders. If targets is nil, it scans all libraries. scanFolders(ctx context.Context, fullScan bool, targets []ScanTarget, progress chan<- *ProgressInfo) } @@ -291,11 +289,7 @@ func (s *controller) ScanFolders(requestCtx context.Context, fullScan bool, targ go func() { defer close(progress) scanner := s.getScanner() - if len(targets) == 0 { - scanner.scanAll(ctx, fullScan, progress) - } else { - scanner.scanFolders(ctx, fullScan, targets, progress) - } + scanner.scanFolders(ctx, fullScan, targets, progress) }() // Wait for the scan to finish, sending progress events to all connected clients diff --git a/scanner/external.go b/scanner/external.go index 690748d8e..155fdf58b 100644 --- a/scanner/external.go +++ b/scanner/external.go @@ -24,10 +24,6 @@ import ( // process will forward them to the caller. type scannerExternal struct{} -func (s *scannerExternal) scanAll(ctx context.Context, fullScan bool, progress chan<- *ProgressInfo) { - s.scan(ctx, fullScan, nil, progress) -} - func (s *scannerExternal) scanFolders(ctx context.Context, fullScan bool, targets []ScanTarget, progress chan<- *ProgressInfo) { s.scan(ctx, fullScan, targets, progress) } diff --git a/scanner/scanner.go b/scanner/scanner.go index be11ec5a3..f1b42a911 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -47,10 +47,6 @@ func (s *scanState) sendError(err error) { s.sendProgress(&ProgressInfo{Error: err.Error()}) } -func (s *scannerImpl) scanAll(ctx context.Context, fullScan bool, progress chan<- *ProgressInfo) { - s.scanFolders(ctx, fullScan, nil, progress) -} - func (s *scannerImpl) scanFolders(ctx context.Context, fullScan bool, targets []ScanTarget, progress chan<- *ProgressInfo) { startTime := time.Now() diff --git a/scanner/watcher_test.go b/scanner/watcher_test.go index 6f10d9010..be813a10b 100644 --- a/scanner/watcher_test.go +++ b/scanner/watcher_test.go @@ -18,7 +18,7 @@ import ( var _ = Describe("Watcher", func() { var ctx context.Context var cancel context.CancelFunc - var mockScanner *MockScanner + var mockScanner *mockScanner var mockDS *tests.MockDataStore var w *watcher var lib *model.Library @@ -337,8 +337,8 @@ var _ = Describe("resolveFolderPath", func() { }) }) -// MockScanner implements scanner.Scanner for testing -type MockScanner struct { +// mockScanner implements scanner.Scanner for testing +type mockScanner struct { mu sync.Mutex scanAllCalls []ScanAllCall scanFoldersCalls []ScanFoldersCall @@ -354,14 +354,14 @@ type ScanFoldersCall struct { Targets []ScanTarget } -func NewMockScanner() *MockScanner { - return &MockScanner{ +func NewMockScanner() *mockScanner { + return &mockScanner{ scanAllCalls: make([]ScanAllCall, 0), scanFoldersCalls: make([]ScanFoldersCall, 0), } } -func (m *MockScanner) ScanAll(_ context.Context, fullScan bool) ([]string, error) { +func (m *mockScanner) ScanAll(_ context.Context, fullScan bool) ([]string, error) { m.mu.Lock() defer m.mu.Unlock() @@ -370,7 +370,7 @@ func (m *MockScanner) ScanAll(_ context.Context, fullScan bool) ([]string, error return nil, nil } -func (m *MockScanner) ScanFolders(_ context.Context, fullScan bool, targets []ScanTarget) ([]string, error) { +func (m *mockScanner) ScanFolders(_ context.Context, fullScan bool, targets []ScanTarget) ([]string, error) { m.mu.Lock() defer m.mu.Unlock() @@ -386,7 +386,7 @@ func (m *MockScanner) ScanFolders(_ context.Context, fullScan bool, targets []Sc return nil, nil } -func (m *MockScanner) Status(_ context.Context) (*StatusInfo, error) { +func (m *mockScanner) Status(_ context.Context) (*StatusInfo, error) { m.mu.Lock() defer m.mu.Unlock() @@ -395,19 +395,19 @@ func (m *MockScanner) Status(_ context.Context) (*StatusInfo, error) { }, nil } -func (m *MockScanner) GetScanAllCallCount() int { +func (m *mockScanner) GetScanAllCallCount() int { m.mu.Lock() defer m.mu.Unlock() return len(m.scanAllCalls) } -func (m *MockScanner) GetScanFoldersCallCount() int { +func (m *mockScanner) GetScanFoldersCallCount() int { m.mu.Lock() defer m.mu.Unlock() return len(m.scanFoldersCalls) } -func (m *MockScanner) GetScanFoldersCalls() []ScanFoldersCall { +func (m *mockScanner) GetScanFoldersCalls() []ScanFoldersCall { m.mu.Lock() defer m.mu.Unlock() // Return a copy to avoid race conditions @@ -416,14 +416,14 @@ func (m *MockScanner) GetScanFoldersCalls() []ScanFoldersCall { return calls } -func (m *MockScanner) Reset() { +func (m *mockScanner) Reset() { m.mu.Lock() defer m.mu.Unlock() m.scanAllCalls = make([]ScanAllCall, 0) m.scanFoldersCalls = make([]ScanFoldersCall, 0) } -func (m *MockScanner) SetScanning(scanning bool) { +func (m *mockScanner) SetScanning(scanning bool) { m.mu.Lock() defer m.mu.Unlock() m.scanningStatus = scanning From 63c3a958144e8aecf3cc4888dca6eb3767ca06a9 Mon Sep 17 00:00:00 2001 From: Deluan Date: Tue, 11 Nov 2025 15:27:22 -0500 Subject: [PATCH 19/40] test: enhance mockScanner for thread safety and improve test reliability Signed-off-by: Deluan --- server/subsonic/library_scanning_test.go | 94 ++++++++++++++++++------ 1 file changed, 73 insertions(+), 21 deletions(-) diff --git a/server/subsonic/library_scanning_test.go b/server/subsonic/library_scanning_test.go index 413cd8606..1e242597d 100644 --- a/server/subsonic/library_scanning_test.go +++ b/server/subsonic/library_scanning_test.go @@ -3,6 +3,7 @@ package subsonic import ( "context" "net/http/httptest" + "sync" "github.com/navidrome/navidrome/model" "github.com/navidrome/navidrome/model/request" @@ -64,9 +65,9 @@ var _ = Describe("LibraryScanning", func() { // Verify ScanAll was called (eventually, since it's in a goroutine) Eventually(func() bool { - return ms.scanAllCalled + return ms.getScanAllCalled() }).Should(BeTrue()) - Expect(ms.scanAllFullScan).To(BeFalse()) + Expect(ms.getScanAllFullScan()).To(BeFalse()) }) It("triggers a full scan with fullScan=true", func() { @@ -89,9 +90,9 @@ var _ = Describe("LibraryScanning", func() { // Verify ScanAll was called with fullScan=true Eventually(func() bool { - return ms.scanAllCalled + return ms.getScanAllCalled() }).Should(BeTrue()) - Expect(ms.scanAllFullScan).To(BeTrue()) + Expect(ms.getScanAllFullScan()).To(BeTrue()) }) It("triggers a selective scan with single path parameter", func() { @@ -114,11 +115,12 @@ var _ = Describe("LibraryScanning", func() { // Verify ScanFolders was called with correct targets Eventually(func() bool { - return ms.scanFoldersCalled + return ms.getScanFoldersCalled() }).Should(BeTrue()) - Expect(ms.scanFoldersTargets).To(HaveLen(1)) - Expect(ms.scanFoldersTargets[0].LibraryID).To(Equal(1)) - Expect(ms.scanFoldersTargets[0].FolderPath).To(Equal("Music/Rock")) + targets := ms.getScanFoldersTargets() + Expect(targets).To(HaveLen(1)) + Expect(targets[0].LibraryID).To(Equal(1)) + Expect(targets[0].FolderPath).To(Equal("Music/Rock")) }) It("triggers a selective scan with multiple path parameters", func() { @@ -141,13 +143,14 @@ var _ = Describe("LibraryScanning", func() { // Verify ScanFolders was called with correct targets Eventually(func() bool { - return ms.scanFoldersCalled + return ms.getScanFoldersCalled() }).Should(BeTrue()) - Expect(ms.scanFoldersTargets).To(HaveLen(2)) - Expect(ms.scanFoldersTargets[0].LibraryID).To(Equal(1)) - Expect(ms.scanFoldersTargets[0].FolderPath).To(Equal("Music/Reggae")) - Expect(ms.scanFoldersTargets[1].LibraryID).To(Equal(2)) - Expect(ms.scanFoldersTargets[1].FolderPath).To(Equal("Classical/Bach")) + targets := ms.getScanFoldersTargets() + Expect(targets).To(HaveLen(2)) + Expect(targets[0].LibraryID).To(Equal(1)) + Expect(targets[0].FolderPath).To(Equal("Music/Reggae")) + Expect(targets[1].LibraryID).To(Equal(2)) + Expect(targets[1].FolderPath).To(Equal("Classical/Bach")) }) It("triggers a selective full scan with path and fullScan parameters", func() { @@ -170,10 +173,11 @@ var _ = Describe("LibraryScanning", func() { // Verify ScanFolders was called with fullScan=true Eventually(func() bool { - return ms.scanFoldersCalled + return ms.getScanFoldersCalled() }).Should(BeTrue()) - Expect(ms.scanFoldersFullScan).To(BeTrue()) - Expect(ms.scanFoldersTargets).To(HaveLen(1)) + Expect(ms.getScanFoldersFullScan()).To(BeTrue()) + targets := ms.getScanFoldersTargets() + Expect(targets).To(HaveLen(1)) }) It("returns error for invalid path format", func() { @@ -240,9 +244,10 @@ var _ = Describe("LibraryScanning", func() { // Verify path was decoded correctly Eventually(func() bool { - return ms.scanFoldersCalled + return ms.getScanFoldersCalled() }).Should(BeTrue()) - Expect(ms.scanFoldersTargets[0].FolderPath).To(Equal("The Beatles")) + targets := ms.getScanFoldersTargets() + Expect(targets[0].FolderPath).To(Equal("The Beatles")) }) }) @@ -274,8 +279,10 @@ var _ = Describe("LibraryScanning", func() { }) }) -// mockScanner is a test double for the scanner.Scanner interface +// mockScanner is a test double for the scanner.Scanner interface with proper synchronization type mockScanner struct { + mu sync.Mutex + // ScanAll tracking scanAllCalled bool scanAllFullScan bool @@ -295,21 +302,66 @@ type mockScanner struct { } func (m *mockScanner) ScanAll(ctx context.Context, fullScan bool) ([]string, error) { + m.mu.Lock() + defer m.mu.Unlock() + m.scanAllCalled = true m.scanAllFullScan = fullScan return m.scanAllWarnings, m.scanAllError } func (m *mockScanner) ScanFolders(ctx context.Context, fullScan bool, targets []scanner.ScanTarget) ([]string, error) { + m.mu.Lock() + defer m.mu.Unlock() + m.scanFoldersCalled = true m.scanFoldersFullScan = fullScan - m.scanFoldersTargets = targets + // Make a copy of targets to avoid race conditions + m.scanFoldersTargets = make([]scanner.ScanTarget, len(targets)) + copy(m.scanFoldersTargets, targets) return m.scanFoldersWarnings, m.scanFoldersError } func (m *mockScanner) Status(ctx context.Context) (*scanner.StatusInfo, error) { + m.mu.Lock() + defer m.mu.Unlock() + if m.statusResponse == nil { return &scanner.StatusInfo{}, m.statusError } return m.statusResponse, m.statusError } + +// Helper methods for safe read access in tests +func (m *mockScanner) getScanAllCalled() bool { + m.mu.Lock() + defer m.mu.Unlock() + return m.scanAllCalled +} + +func (m *mockScanner) getScanAllFullScan() bool { + m.mu.Lock() + defer m.mu.Unlock() + return m.scanAllFullScan +} + +func (m *mockScanner) getScanFoldersCalled() bool { + m.mu.Lock() + defer m.mu.Unlock() + return m.scanFoldersCalled +} + +func (m *mockScanner) getScanFoldersFullScan() bool { + m.mu.Lock() + defer m.mu.Unlock() + return m.scanFoldersFullScan +} + +func (m *mockScanner) getScanFoldersTargets() []scanner.ScanTarget { + m.mu.Lock() + defer m.mu.Unlock() + // Return a copy to avoid race conditions + targets := make([]scanner.ScanTarget, len(m.scanFoldersTargets)) + copy(targets, m.scanFoldersTargets) + return targets +} From 54f19c598aa4c3bf913f0c7ee2d6cf6bab122fa6 Mon Sep 17 00:00:00 2001 From: Deluan Date: Tue, 11 Nov 2025 15:33:00 -0500 Subject: [PATCH 20/40] refactor(scanner): move scanner.ScanTarget to model.ScanTarget Signed-off-by: Deluan --- cmd/scan.go | 5 +++-- cmd/scan_test.go | 8 ++++---- model/folder.go | 11 +++++++++++ scanner/controller.go | 25 +++++++----------------- scanner/external.go | 7 ++++--- scanner/scanner.go | 2 +- scanner/scanner_test.go | 2 +- scanner/selective_scan_test.go | 4 ++-- scanner/watcher.go | 8 ++++---- scanner/watcher_test.go | 6 +++--- server/subsonic/library_scanning.go | 3 ++- server/subsonic/library_scanning_test.go | 10 +++++----- 12 files changed, 47 insertions(+), 44 deletions(-) diff --git a/cmd/scan.go b/cmd/scan.go index 2c43baf62..057d4902f 100644 --- a/cmd/scan.go +++ b/cmd/scan.go @@ -9,6 +9,7 @@ import ( "github.com/navidrome/navidrome/core" "github.com/navidrome/navidrome/db" "github.com/navidrome/navidrome/log" + "github.com/navidrome/navidrome/model" "github.com/navidrome/navidrome/persistence" "github.com/navidrome/navidrome/scanner" "github.com/navidrome/navidrome/utils/pl" @@ -72,7 +73,7 @@ func runScanner(ctx context.Context) { pls := core.NewPlaylists(ds) // Parse targets if provided - var scanTargets []scanner.ScanTarget + var scanTargets []model.ScanTarget if targets != "" { var err error scanTargets, err = parseTargets(targets) @@ -96,7 +97,7 @@ func runScanner(ctx context.Context) { } // parseTargets parses the comma-separated targets string into ScanTarget structs -func parseTargets(targetsStr string) ([]scanner.ScanTarget, error) { +func parseTargets(targetsStr string) ([]model.ScanTarget, error) { targets := strings.Split(targetsStr, ",") return scanner.ParseTargets(targets) } diff --git a/cmd/scan_test.go b/cmd/scan_test.go index fecd79c4b..7abc8ef88 100644 --- a/cmd/scan_test.go +++ b/cmd/scan_test.go @@ -1,7 +1,7 @@ package cmd import ( - "github.com/navidrome/navidrome/scanner" + "github.com/navidrome/navidrome/model" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) @@ -12,9 +12,9 @@ var _ = Describe("parseTargets", func() { targets, err := parseTargets("1:Music/Rock,2:Jazz,3:Classical/Beethoven") Expect(err).ToNot(HaveOccurred()) Expect(targets).To(HaveLen(3)) - Expect(targets[0]).To(Equal(scanner.ScanTarget{LibraryID: 1, FolderPath: "Music/Rock"})) - Expect(targets[1]).To(Equal(scanner.ScanTarget{LibraryID: 2, FolderPath: "Jazz"})) - Expect(targets[2]).To(Equal(scanner.ScanTarget{LibraryID: 3, FolderPath: "Classical/Beethoven"})) + Expect(targets[0]).To(Equal(model.ScanTarget{LibraryID: 1, FolderPath: "Music/Rock"})) + Expect(targets[1]).To(Equal(model.ScanTarget{LibraryID: 2, FolderPath: "Jazz"})) + Expect(targets[2]).To(Equal(model.ScanTarget{LibraryID: 3, FolderPath: "Classical/Beethoven"})) }) It("returns error for empty string", func() { diff --git a/model/folder.go b/model/folder.go index 7a769735e..c59e9d465 100644 --- a/model/folder.go +++ b/model/folder.go @@ -90,3 +90,14 @@ type FolderRepository interface { MarkMissing(missing bool, ids ...string) error GetTouchedWithPlaylists() (FolderCursor, error) } + +// ScanTarget represents a specific folder within a library to be scanned. +// NOTE: This struct is used as a map key, so it should only contain comparable types. +type ScanTarget struct { + LibraryID int + FolderPath string // Relative path within the library, or "" for entire library +} + +func (st ScanTarget) String() string { + return fmt.Sprintf("%d:%s", st.LibraryID, st.FolderPath) +} diff --git a/scanner/controller.go b/scanner/controller.go index 70e2d6a69..caff41b0a 100644 --- a/scanner/controller.go +++ b/scanner/controller.go @@ -28,21 +28,10 @@ var ( ErrAlreadyScanning = errors.New("already scanning") ) -// ScanTarget represents a specific folder within a library to be scanned. -// NOTE: This struct is used as a map key, so it should only contain comparable types. -type ScanTarget struct { - LibraryID int - FolderPath string // Relative path within the library, or "" for entire library -} - -func (st ScanTarget) String() string { - return fmt.Sprintf("%d:%s", st.LibraryID, st.FolderPath) -} - // ParseTargets parses scan targets strings into ScanTarget structs. // Example: []string{"1:Music/Rock", "2:Classical"} -func ParseTargets(libFolders []string) ([]ScanTarget, error) { - targets := make([]ScanTarget, 0, len(libFolders)) +func ParseTargets(libFolders []string) ([]model.ScanTarget, error) { + targets := make([]model.ScanTarget, 0, len(libFolders)) for _, part := range libFolders { part = strings.TrimSpace(part) @@ -67,7 +56,7 @@ func ParseTargets(libFolders []string) ([]ScanTarget, error) { return nil, fmt.Errorf("invalid library ID %q", libIDStr) } - targets = append(targets, ScanTarget{ + targets = append(targets, model.ScanTarget{ LibraryID: libID, FolderPath: folderPath, }) @@ -85,7 +74,7 @@ type Scanner interface { ScanAll(ctx context.Context, fullScan bool) (warnings []string, err error) // ScanFolders scans specific library/folder pairs, recursing into subdirectories. // If targets is nil, it scans all libraries. This is a blocking operation. - ScanFolders(ctx context.Context, fullScan bool, targets []ScanTarget) (warnings []string, err error) + ScanFolders(ctx context.Context, fullScan bool, targets []model.ScanTarget) (warnings []string, err error) Status(context.Context) (*StatusInfo, error) } @@ -125,7 +114,7 @@ func (s *controller) getScanner() scanner { // CallScan starts an in-process scan of specific library/folder pairs. // If targets is empty, it scans all libraries. // This is meant to be called from the command line (see cmd/scan.go). -func CallScan(ctx context.Context, ds model.DataStore, pls core.Playlists, fullScan bool, targets []ScanTarget) (<-chan *ProgressInfo, error) { +func CallScan(ctx context.Context, ds model.DataStore, pls core.Playlists, fullScan bool, targets []model.ScanTarget) (<-chan *ProgressInfo, error) { release, err := lockScan(ctx) if err != nil { return nil, err @@ -161,7 +150,7 @@ type ProgressInfo struct { // This allows for swapping between in-process and external scanners. type scanner interface { // scanFolders performs the actual scanning of folders. If targets is nil, it scans all libraries. - scanFolders(ctx context.Context, fullScan bool, targets []ScanTarget, progress chan<- *ProgressInfo) + scanFolders(ctx context.Context, fullScan bool, targets []model.ScanTarget, progress chan<- *ProgressInfo) } type controller struct { @@ -272,7 +261,7 @@ func (s *controller) ScanAll(requestCtx context.Context, fullScan bool) ([]strin return s.ScanFolders(requestCtx, fullScan, nil) } -func (s *controller) ScanFolders(requestCtx context.Context, fullScan bool, targets []ScanTarget) ([]string, error) { +func (s *controller) ScanFolders(requestCtx context.Context, fullScan bool, targets []model.ScanTarget) ([]string, error) { release, err := lockScan(requestCtx) if err != nil { return nil, err diff --git a/scanner/external.go b/scanner/external.go index 155fdf58b..b6d7639be 100644 --- a/scanner/external.go +++ b/scanner/external.go @@ -12,6 +12,7 @@ import ( "github.com/navidrome/navidrome/conf" "github.com/navidrome/navidrome/log" + "github.com/navidrome/navidrome/model" "github.com/navidrome/navidrome/utils/slice" ) @@ -24,11 +25,11 @@ import ( // process will forward them to the caller. type scannerExternal struct{} -func (s *scannerExternal) scanFolders(ctx context.Context, fullScan bool, targets []ScanTarget, progress chan<- *ProgressInfo) { +func (s *scannerExternal) scanFolders(ctx context.Context, fullScan bool, targets []model.ScanTarget, progress chan<- *ProgressInfo) { s.scan(ctx, fullScan, targets, progress) } -func (s *scannerExternal) scan(ctx context.Context, fullScan bool, targets []ScanTarget, progress chan<- *ProgressInfo) { +func (s *scannerExternal) scan(ctx context.Context, fullScan bool, targets []model.ScanTarget, progress chan<- *ProgressInfo) { exe, err := os.Executable() if err != nil { progress <- &ProgressInfo{Error: fmt.Sprintf("failed to get executable path: %s", err)} @@ -46,7 +47,7 @@ func (s *scannerExternal) scan(ctx context.Context, fullScan bool, targets []Sca // Add targets if provided if len(targets) > 0 { - targetsStr := strings.Join(slice.Map(targets, func(t ScanTarget) string { return t.String() }), ",") + targetsStr := strings.Join(slice.Map(targets, func(t model.ScanTarget) string { return t.String() }), ",") args = append(args, "--targets", targetsStr) log.Debug(ctx, "Spawning external scanner process with targets", "fullScan", fullScan, "path", exe, "targets", targetsStr) } else { diff --git a/scanner/scanner.go b/scanner/scanner.go index f1b42a911..96fdb1604 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -47,7 +47,7 @@ func (s *scanState) sendError(err error) { s.sendProgress(&ProgressInfo{Error: err.Error()}) } -func (s *scannerImpl) scanFolders(ctx context.Context, fullScan bool, targets []ScanTarget, progress chan<- *ProgressInfo) { +func (s *scannerImpl) scanFolders(ctx context.Context, fullScan bool, targets []model.ScanTarget, progress chan<- *ProgressInfo) { startTime := time.Now() state := scanState{ diff --git a/scanner/scanner_test.go b/scanner/scanner_test.go index ecad6f501..e1b2e6f32 100644 --- a/scanner/scanner_test.go +++ b/scanner/scanner_test.go @@ -736,7 +736,7 @@ var _ = Describe("Scanner", Ordered, func() { // (lib is already created with the path "fake:///music") // Scan only the "rock" and "jazz" folders (including their subdirectories) - targets := []scanner.ScanTarget{ + targets := []model.ScanTarget{ {LibraryID: lib.ID, FolderPath: "rock"}, {LibraryID: lib.ID, FolderPath: "jazz"}, } diff --git a/scanner/selective_scan_test.go b/scanner/selective_scan_test.go index a706af159..4bac1c2e1 100644 --- a/scanner/selective_scan_test.go +++ b/scanner/selective_scan_test.go @@ -146,7 +146,7 @@ var _ = Describe("Selective Scan - Deleted Child Folders", Ordered, func() { // Run selective scan on the parent folder (Artist) // This simulates what the watcher does when a child folder is deleted - _, err := s.ScanFolders(ctx, false, []scanner.ScanTarget{ + _, err := s.ScanFolders(ctx, false, []model.ScanTarget{ {LibraryID: lib.ID, FolderPath: "The Beatles"}, }) Expect(err).ToNot(HaveOccurred()) @@ -208,7 +208,7 @@ var _ = Describe("Selective Scan - Deleted Child Folders", Ordered, func() { }) // Run selective scan on parent - _, err = s.ScanFolders(ctx, false, []scanner.ScanTarget{ + _, err = s.ScanFolders(ctx, false, []model.ScanTarget{ {LibraryID: lib.ID, FolderPath: "The Beatles"}, }) Expect(err).ToNot(HaveOccurred()) diff --git a/scanner/watcher.go b/scanner/watcher.go index dd0d1cca2..911ea310b 100644 --- a/scanner/watcher.go +++ b/scanner/watcher.go @@ -73,7 +73,7 @@ func (w *watcher) Run(ctx context.Context) error { // Main scan triggering loop trigger := time.NewTimer(w.triggerWait) trigger.Stop() - targets := make(map[ScanTarget]struct{}) + targets := make(map[model.ScanTarget]struct{}) for { select { case <-trigger.C: @@ -90,13 +90,13 @@ func (w *watcher) Run(ctx context.Context) error { } // Convert targets map to slice - targetSlice := make([]ScanTarget, 0, len(targets)) + targetSlice := make([]model.ScanTarget, 0, len(targets)) for target := range targets { targetSlice = append(targetSlice, target) } // Clear targets for next batch - targets = make(map[ScanTarget]struct{}) + targets = make(map[model.ScanTarget]struct{}) go func() { _, err := w.scanner.ScanFolders(ctx, false, targetSlice) @@ -121,7 +121,7 @@ func (w *watcher) Run(ctx context.Context) error { folderPath := notification.FolderPath // If already scheduled for scan, skip - target := ScanTarget{LibraryID: lib.ID, FolderPath: folderPath} + target := model.ScanTarget{LibraryID: lib.ID, FolderPath: folderPath} if _, exists := targets[target]; exists { continue } diff --git a/scanner/watcher_test.go b/scanner/watcher_test.go index be813a10b..c10ffe00e 100644 --- a/scanner/watcher_test.go +++ b/scanner/watcher_test.go @@ -351,7 +351,7 @@ type ScanAllCall struct { type ScanFoldersCall struct { FullScan bool - Targets []ScanTarget + Targets []model.ScanTarget } func NewMockScanner() *mockScanner { @@ -370,12 +370,12 @@ func (m *mockScanner) ScanAll(_ context.Context, fullScan bool) ([]string, error return nil, nil } -func (m *mockScanner) ScanFolders(_ context.Context, fullScan bool, targets []ScanTarget) ([]string, error) { +func (m *mockScanner) ScanFolders(_ context.Context, fullScan bool, targets []model.ScanTarget) ([]string, error) { m.mu.Lock() defer m.mu.Unlock() // Make a copy of targets to avoid race conditions - targetsCopy := make([]ScanTarget, len(targets)) + targetsCopy := make([]model.ScanTarget, len(targets)) copy(targetsCopy, targets) m.scanFoldersCalls = append(m.scanFoldersCalls, ScanFoldersCall{ diff --git a/server/subsonic/library_scanning.go b/server/subsonic/library_scanning.go index abd388428..5f180dc4c 100644 --- a/server/subsonic/library_scanning.go +++ b/server/subsonic/library_scanning.go @@ -6,6 +6,7 @@ import ( "time" "github.com/navidrome/navidrome/log" + "github.com/navidrome/navidrome/model" "github.com/navidrome/navidrome/model/request" "github.com/navidrome/navidrome/scanner" "github.com/navidrome/navidrome/server/subsonic/responses" @@ -47,7 +48,7 @@ func (api *Router) StartScan(r *http.Request) (*responses.Subsonic, error) { fullScan := p.BoolOr("fullScan", false) // Parse optional path parameters for selective scanning - var targets []scanner.ScanTarget + var targets []model.ScanTarget if pathParams, err := p.Strings("path"); err == nil && len(pathParams) > 0 { targets, err = scanner.ParseTargets(pathParams) if err != nil { diff --git a/server/subsonic/library_scanning_test.go b/server/subsonic/library_scanning_test.go index 1e242597d..4eb1703e8 100644 --- a/server/subsonic/library_scanning_test.go +++ b/server/subsonic/library_scanning_test.go @@ -292,7 +292,7 @@ type mockScanner struct { // ScanFolders tracking scanFoldersCalled bool scanFoldersFullScan bool - scanFoldersTargets []scanner.ScanTarget + scanFoldersTargets []model.ScanTarget scanFoldersError error scanFoldersWarnings []string @@ -310,14 +310,14 @@ func (m *mockScanner) ScanAll(ctx context.Context, fullScan bool) ([]string, err return m.scanAllWarnings, m.scanAllError } -func (m *mockScanner) ScanFolders(ctx context.Context, fullScan bool, targets []scanner.ScanTarget) ([]string, error) { +func (m *mockScanner) ScanFolders(ctx context.Context, fullScan bool, targets []model.ScanTarget) ([]string, error) { m.mu.Lock() defer m.mu.Unlock() m.scanFoldersCalled = true m.scanFoldersFullScan = fullScan // Make a copy of targets to avoid race conditions - m.scanFoldersTargets = make([]scanner.ScanTarget, len(targets)) + m.scanFoldersTargets = make([]model.ScanTarget, len(targets)) copy(m.scanFoldersTargets, targets) return m.scanFoldersWarnings, m.scanFoldersError } @@ -357,11 +357,11 @@ func (m *mockScanner) getScanFoldersFullScan() bool { return m.scanFoldersFullScan } -func (m *mockScanner) getScanFoldersTargets() []scanner.ScanTarget { +func (m *mockScanner) getScanFoldersTargets() []model.ScanTarget { m.mu.Lock() defer m.mu.Unlock() // Return a copy to avoid race conditions - targets := make([]scanner.ScanTarget, len(m.scanFoldersTargets)) + targets := make([]model.ScanTarget, len(m.scanFoldersTargets)) copy(targets, m.scanFoldersTargets) return targets } From ed781e8da01631c61e65fa06eb742089fee12158 Mon Sep 17 00:00:00 2001 From: Deluan Date: Tue, 11 Nov 2025 15:43:26 -0500 Subject: [PATCH 21/40] refactor: move scanner types to model,implement MockScanner Signed-off-by: Deluan --- model/folder.go | 11 -- model/scanner.go | 28 ++++ scanner/controller.go | 18 +-- scanner/watcher_test.go | 97 +------------ server/subsonic/library_scanning_test.go | 169 +++++++---------------- tests/mock_scanner.go | 120 ++++++++++++++++ 6 files changed, 202 insertions(+), 241 deletions(-) create mode 100644 model/scanner.go create mode 100644 tests/mock_scanner.go diff --git a/model/folder.go b/model/folder.go index c59e9d465..7a769735e 100644 --- a/model/folder.go +++ b/model/folder.go @@ -90,14 +90,3 @@ type FolderRepository interface { MarkMissing(missing bool, ids ...string) error GetTouchedWithPlaylists() (FolderCursor, error) } - -// ScanTarget represents a specific folder within a library to be scanned. -// NOTE: This struct is used as a map key, so it should only contain comparable types. -type ScanTarget struct { - LibraryID int - FolderPath string // Relative path within the library, or "" for entire library -} - -func (st ScanTarget) String() string { - return fmt.Sprintf("%d:%s", st.LibraryID, st.FolderPath) -} diff --git a/model/scanner.go b/model/scanner.go new file mode 100644 index 000000000..756688a23 --- /dev/null +++ b/model/scanner.go @@ -0,0 +1,28 @@ +package model + +import ( + "fmt" + "time" +) + +// ScanTarget represents a specific folder within a library to be scanned. +// NOTE: This struct is used as a map key, so it should only contain comparable types. +type ScanTarget struct { + LibraryID int + FolderPath string // Relative path within the library, or "" for entire library +} + +func (st ScanTarget) String() string { + return fmt.Sprintf("%d:%s", st.LibraryID, st.FolderPath) +} + +// ScannerStatus holds information about the current scan status +type ScannerStatus struct { + Scanning bool + LastScan time.Time + Count uint32 + FolderCount uint32 + LastError string + ScanType string + ElapsedTime time.Duration +} diff --git a/scanner/controller.go b/scanner/controller.go index caff41b0a..b5ba2ddb7 100644 --- a/scanner/controller.go +++ b/scanner/controller.go @@ -75,17 +75,7 @@ type Scanner interface { // ScanFolders scans specific library/folder pairs, recursing into subdirectories. // If targets is nil, it scans all libraries. This is a blocking operation. ScanFolders(ctx context.Context, fullScan bool, targets []model.ScanTarget) (warnings []string, err error) - Status(context.Context) (*StatusInfo, error) -} - -type StatusInfo struct { - Scanning bool - LastScan time.Time - Count uint32 - FolderCount uint32 - LastError string - ScanType string - ElapsedTime time.Duration + Status(context.Context) (*model.ScannerStatus, error) } func New(rootCtx context.Context, ds model.DataStore, cw artwork.CacheWarmer, broker events.Broker, @@ -208,7 +198,7 @@ func (s *controller) getScanInfo(ctx context.Context) (scanType string, elapsed return scanType, elapsed, lastErr } -func (s *controller) Status(ctx context.Context) (*StatusInfo, error) { +func (s *controller) Status(ctx context.Context) (*model.ScannerStatus, error) { lastScanTime, err := s.getLastScanTime(ctx) if err != nil { return nil, fmt.Errorf("getting last scan time: %w", err) @@ -217,7 +207,7 @@ func (s *controller) Status(ctx context.Context) (*StatusInfo, error) { scanType, elapsed, lastErr := s.getScanInfo(ctx) if running.Load() { - status := &StatusInfo{ + status := &model.ScannerStatus{ Scanning: true, LastScan: lastScanTime, Count: s.count.Load(), @@ -233,7 +223,7 @@ func (s *controller) Status(ctx context.Context) (*StatusInfo, error) { if err != nil { return nil, fmt.Errorf("getting library stats: %w", err) } - return &StatusInfo{ + return &model.ScannerStatus{ Scanning: false, LastScan: lastScanTime, Count: uint32(count), diff --git a/scanner/watcher_test.go b/scanner/watcher_test.go index c10ffe00e..7ae52e725 100644 --- a/scanner/watcher_test.go +++ b/scanner/watcher_test.go @@ -3,7 +3,6 @@ package scanner import ( "context" "io/fs" - "sync" "testing/fstest" "time" @@ -18,7 +17,7 @@ import ( var _ = Describe("Watcher", func() { var ctx context.Context var cancel context.CancelFunc - var mockScanner *mockScanner + var mockScanner *tests.MockScanner var mockDS *tests.MockDataStore var w *watcher var lib *model.Library @@ -37,7 +36,7 @@ var _ = Describe("Watcher", func() { } // Set up mocks - mockScanner = NewMockScanner() + mockScanner = tests.NewMockScanner() mockDS = &tests.MockDataStore{} mockLibRepo := &tests.MockLibraryRepo{} mockLibRepo.SetData(model.Libraries{*lib}) @@ -336,95 +335,3 @@ var _ = Describe("resolveFolderPath", func() { Expect(result).To(Equal("artist2")) }) }) - -// mockScanner implements scanner.Scanner for testing -type mockScanner struct { - mu sync.Mutex - scanAllCalls []ScanAllCall - scanFoldersCalls []ScanFoldersCall - scanningStatus bool -} - -type ScanAllCall struct { - FullScan bool -} - -type ScanFoldersCall struct { - FullScan bool - Targets []model.ScanTarget -} - -func NewMockScanner() *mockScanner { - return &mockScanner{ - scanAllCalls: make([]ScanAllCall, 0), - scanFoldersCalls: make([]ScanFoldersCall, 0), - } -} - -func (m *mockScanner) ScanAll(_ context.Context, fullScan bool) ([]string, error) { - m.mu.Lock() - defer m.mu.Unlock() - - m.scanAllCalls = append(m.scanAllCalls, ScanAllCall{FullScan: fullScan}) - - return nil, nil -} - -func (m *mockScanner) ScanFolders(_ context.Context, fullScan bool, targets []model.ScanTarget) ([]string, error) { - m.mu.Lock() - defer m.mu.Unlock() - - // Make a copy of targets to avoid race conditions - targetsCopy := make([]model.ScanTarget, len(targets)) - copy(targetsCopy, targets) - - m.scanFoldersCalls = append(m.scanFoldersCalls, ScanFoldersCall{ - FullScan: fullScan, - Targets: targetsCopy, - }) - - return nil, nil -} - -func (m *mockScanner) Status(_ context.Context) (*StatusInfo, error) { - m.mu.Lock() - defer m.mu.Unlock() - - return &StatusInfo{ - Scanning: m.scanningStatus, - }, nil -} - -func (m *mockScanner) GetScanAllCallCount() int { - m.mu.Lock() - defer m.mu.Unlock() - return len(m.scanAllCalls) -} - -func (m *mockScanner) GetScanFoldersCallCount() int { - m.mu.Lock() - defer m.mu.Unlock() - return len(m.scanFoldersCalls) -} - -func (m *mockScanner) GetScanFoldersCalls() []ScanFoldersCall { - m.mu.Lock() - defer m.mu.Unlock() - // Return a copy to avoid race conditions - calls := make([]ScanFoldersCall, len(m.scanFoldersCalls)) - copy(calls, m.scanFoldersCalls) - return calls -} - -func (m *mockScanner) Reset() { - m.mu.Lock() - defer m.mu.Unlock() - m.scanAllCalls = make([]ScanAllCall, 0) - m.scanFoldersCalls = make([]ScanFoldersCall, 0) -} - -func (m *mockScanner) SetScanning(scanning bool) { - m.mu.Lock() - defer m.mu.Unlock() - m.scanningStatus = scanning -} diff --git a/server/subsonic/library_scanning_test.go b/server/subsonic/library_scanning_test.go index 4eb1703e8..85fdea5ad 100644 --- a/server/subsonic/library_scanning_test.go +++ b/server/subsonic/library_scanning_test.go @@ -2,23 +2,23 @@ package subsonic import ( "context" + "errors" "net/http/httptest" - "sync" "github.com/navidrome/navidrome/model" "github.com/navidrome/navidrome/model/request" - "github.com/navidrome/navidrome/scanner" "github.com/navidrome/navidrome/server/subsonic/responses" + "github.com/navidrome/navidrome/tests" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) var _ = Describe("LibraryScanning", func() { var api *Router - var ms *mockScanner + var ms *tests.MockScanner BeforeEach(func() { - ms = &mockScanner{} + ms = tests.NewMockScanner() api = &Router{scanner: ms} }) @@ -40,7 +40,8 @@ var _ = Describe("LibraryScanning", func() { // Should return authorization error Expect(err).To(HaveOccurred()) Expect(response).To(BeNil()) - subErr, ok := err.(subError) + var subErr subError + ok := errors.As(err, &subErr) Expect(ok).To(BeTrue()) Expect(subErr.code).To(Equal(responses.ErrorAuthorizationFail)) }) @@ -64,10 +65,12 @@ var _ = Describe("LibraryScanning", func() { Expect(response).ToNot(BeNil()) // Verify ScanAll was called (eventually, since it's in a goroutine) - Eventually(func() bool { - return ms.getScanAllCalled() - }).Should(BeTrue()) - Expect(ms.getScanAllFullScan()).To(BeFalse()) + Eventually(func() int { + return ms.GetScanAllCallCount() + }).Should(BeNumerically(">", 0)) + calls := ms.GetScanAllCalls() + Expect(calls).To(HaveLen(1)) + Expect(calls[0].FullScan).To(BeFalse()) }) It("triggers a full scan with fullScan=true", func() { @@ -89,10 +92,12 @@ var _ = Describe("LibraryScanning", func() { Expect(response).ToNot(BeNil()) // Verify ScanAll was called with fullScan=true - Eventually(func() bool { - return ms.getScanAllCalled() - }).Should(BeTrue()) - Expect(ms.getScanAllFullScan()).To(BeTrue()) + Eventually(func() int { + return ms.GetScanAllCallCount() + }).Should(BeNumerically(">", 0)) + calls := ms.GetScanAllCalls() + Expect(calls).To(HaveLen(1)) + Expect(calls[0].FullScan).To(BeTrue()) }) It("triggers a selective scan with single path parameter", func() { @@ -114,10 +119,12 @@ var _ = Describe("LibraryScanning", func() { Expect(response).ToNot(BeNil()) // Verify ScanFolders was called with correct targets - Eventually(func() bool { - return ms.getScanFoldersCalled() - }).Should(BeTrue()) - targets := ms.getScanFoldersTargets() + Eventually(func() int { + return ms.GetScanFoldersCallCount() + }).Should(BeNumerically(">", 0)) + calls := ms.GetScanFoldersCalls() + Expect(calls).To(HaveLen(1)) + targets := calls[0].Targets Expect(targets).To(HaveLen(1)) Expect(targets[0].LibraryID).To(Equal(1)) Expect(targets[0].FolderPath).To(Equal("Music/Rock")) @@ -142,10 +149,12 @@ var _ = Describe("LibraryScanning", func() { Expect(response).ToNot(BeNil()) // Verify ScanFolders was called with correct targets - Eventually(func() bool { - return ms.getScanFoldersCalled() - }).Should(BeTrue()) - targets := ms.getScanFoldersTargets() + Eventually(func() int { + return ms.GetScanFoldersCallCount() + }).Should(BeNumerically(">", 0)) + calls := ms.GetScanFoldersCalls() + Expect(calls).To(HaveLen(1)) + targets := calls[0].Targets Expect(targets).To(HaveLen(2)) Expect(targets[0].LibraryID).To(Equal(1)) Expect(targets[0].FolderPath).To(Equal("Music/Reggae")) @@ -172,11 +181,13 @@ var _ = Describe("LibraryScanning", func() { Expect(response).ToNot(BeNil()) // Verify ScanFolders was called with fullScan=true - Eventually(func() bool { - return ms.getScanFoldersCalled() - }).Should(BeTrue()) - Expect(ms.getScanFoldersFullScan()).To(BeTrue()) - targets := ms.getScanFoldersTargets() + Eventually(func() int { + return ms.GetScanFoldersCallCount() + }).Should(BeNumerically(">", 0)) + calls := ms.GetScanFoldersCalls() + Expect(calls).To(HaveLen(1)) + Expect(calls[0].FullScan).To(BeTrue()) + targets := calls[0].Targets Expect(targets).To(HaveLen(1)) }) @@ -197,7 +208,8 @@ var _ = Describe("LibraryScanning", func() { // Should return error Expect(err).To(HaveOccurred()) Expect(response).To(BeNil()) - subErr, ok := err.(subError) + var subErr subError + ok := errors.As(err, &subErr) Expect(ok).To(BeTrue()) Expect(subErr.code).To(Equal(responses.ErrorGeneric)) }) @@ -219,7 +231,8 @@ var _ = Describe("LibraryScanning", func() { // Should return error Expect(err).To(HaveOccurred()) Expect(response).To(BeNil()) - subErr, ok := err.(subError) + var subErr subError + ok := errors.As(err, &subErr) Expect(ok).To(BeTrue()) Expect(subErr.code).To(Equal(responses.ErrorGeneric)) }) @@ -243,10 +256,11 @@ var _ = Describe("LibraryScanning", func() { Expect(response).ToNot(BeNil()) // Verify path was decoded correctly - Eventually(func() bool { - return ms.getScanFoldersCalled() - }).Should(BeTrue()) - targets := ms.getScanFoldersTargets() + Eventually(func() int { + return ms.GetScanFoldersCallCount() + }).Should(BeNumerically(">", 0)) + calls := ms.GetScanFoldersCalls() + targets := calls[0].Targets Expect(targets[0].FolderPath).To(Equal("The Beatles")) }) }) @@ -254,11 +268,11 @@ var _ = Describe("LibraryScanning", func() { Describe("GetScanStatus", func() { It("returns scan status", func() { // Setup mock scanner status - ms.statusResponse = &scanner.StatusInfo{ + ms.SetStatusResponse(&model.ScannerStatus{ Scanning: false, Count: 100, FolderCount: 10, - } + }) // Create request ctx := context.Background() @@ -278,90 +292,3 @@ var _ = Describe("LibraryScanning", func() { }) }) }) - -// mockScanner is a test double for the scanner.Scanner interface with proper synchronization -type mockScanner struct { - mu sync.Mutex - - // ScanAll tracking - scanAllCalled bool - scanAllFullScan bool - scanAllError error - scanAllWarnings []string - - // ScanFolders tracking - scanFoldersCalled bool - scanFoldersFullScan bool - scanFoldersTargets []model.ScanTarget - scanFoldersError error - scanFoldersWarnings []string - - // Status tracking - statusResponse *scanner.StatusInfo - statusError error -} - -func (m *mockScanner) ScanAll(ctx context.Context, fullScan bool) ([]string, error) { - m.mu.Lock() - defer m.mu.Unlock() - - m.scanAllCalled = true - m.scanAllFullScan = fullScan - return m.scanAllWarnings, m.scanAllError -} - -func (m *mockScanner) ScanFolders(ctx context.Context, fullScan bool, targets []model.ScanTarget) ([]string, error) { - m.mu.Lock() - defer m.mu.Unlock() - - m.scanFoldersCalled = true - m.scanFoldersFullScan = fullScan - // Make a copy of targets to avoid race conditions - m.scanFoldersTargets = make([]model.ScanTarget, len(targets)) - copy(m.scanFoldersTargets, targets) - return m.scanFoldersWarnings, m.scanFoldersError -} - -func (m *mockScanner) Status(ctx context.Context) (*scanner.StatusInfo, error) { - m.mu.Lock() - defer m.mu.Unlock() - - if m.statusResponse == nil { - return &scanner.StatusInfo{}, m.statusError - } - return m.statusResponse, m.statusError -} - -// Helper methods for safe read access in tests -func (m *mockScanner) getScanAllCalled() bool { - m.mu.Lock() - defer m.mu.Unlock() - return m.scanAllCalled -} - -func (m *mockScanner) getScanAllFullScan() bool { - m.mu.Lock() - defer m.mu.Unlock() - return m.scanAllFullScan -} - -func (m *mockScanner) getScanFoldersCalled() bool { - m.mu.Lock() - defer m.mu.Unlock() - return m.scanFoldersCalled -} - -func (m *mockScanner) getScanFoldersFullScan() bool { - m.mu.Lock() - defer m.mu.Unlock() - return m.scanFoldersFullScan -} - -func (m *mockScanner) getScanFoldersTargets() []model.ScanTarget { - m.mu.Lock() - defer m.mu.Unlock() - // Return a copy to avoid race conditions - targets := make([]model.ScanTarget, len(m.scanFoldersTargets)) - copy(targets, m.scanFoldersTargets) - return targets -} diff --git a/tests/mock_scanner.go b/tests/mock_scanner.go new file mode 100644 index 000000000..52396723f --- /dev/null +++ b/tests/mock_scanner.go @@ -0,0 +1,120 @@ +package tests + +import ( + "context" + "sync" + + "github.com/navidrome/navidrome/model" +) + +// MockScanner implements scanner.Scanner for testing with proper synchronization +type MockScanner struct { + mu sync.Mutex + scanAllCalls []ScanAllCall + scanFoldersCalls []ScanFoldersCall + scanningStatus bool + statusResponse *model.ScannerStatus +} + +type ScanAllCall struct { + FullScan bool +} + +type ScanFoldersCall struct { + FullScan bool + Targets []model.ScanTarget +} + +func NewMockScanner() *MockScanner { + return &MockScanner{ + scanAllCalls: make([]ScanAllCall, 0), + scanFoldersCalls: make([]ScanFoldersCall, 0), + } +} + +func (m *MockScanner) ScanAll(_ context.Context, fullScan bool) ([]string, error) { + m.mu.Lock() + defer m.mu.Unlock() + + m.scanAllCalls = append(m.scanAllCalls, ScanAllCall{FullScan: fullScan}) + + return nil, nil +} + +func (m *MockScanner) ScanFolders(_ context.Context, fullScan bool, targets []model.ScanTarget) ([]string, error) { + m.mu.Lock() + defer m.mu.Unlock() + + // Make a copy of targets to avoid race conditions + targetsCopy := make([]model.ScanTarget, len(targets)) + copy(targetsCopy, targets) + + m.scanFoldersCalls = append(m.scanFoldersCalls, ScanFoldersCall{ + FullScan: fullScan, + Targets: targetsCopy, + }) + + return nil, nil +} + +func (m *MockScanner) Status(_ context.Context) (*model.ScannerStatus, error) { + m.mu.Lock() + defer m.mu.Unlock() + + if m.statusResponse != nil { + return m.statusResponse, nil + } + + return &model.ScannerStatus{ + Scanning: m.scanningStatus, + }, nil +} + +func (m *MockScanner) GetScanAllCallCount() int { + m.mu.Lock() + defer m.mu.Unlock() + return len(m.scanAllCalls) +} + +func (m *MockScanner) GetScanAllCalls() []ScanAllCall { + m.mu.Lock() + defer m.mu.Unlock() + // Return a copy to avoid race conditions + calls := make([]ScanAllCall, len(m.scanAllCalls)) + copy(calls, m.scanAllCalls) + return calls +} + +func (m *MockScanner) GetScanFoldersCallCount() int { + m.mu.Lock() + defer m.mu.Unlock() + return len(m.scanFoldersCalls) +} + +func (m *MockScanner) GetScanFoldersCalls() []ScanFoldersCall { + m.mu.Lock() + defer m.mu.Unlock() + // Return a copy to avoid race conditions + calls := make([]ScanFoldersCall, len(m.scanFoldersCalls)) + copy(calls, m.scanFoldersCalls) + return calls +} + +func (m *MockScanner) Reset() { + m.mu.Lock() + defer m.mu.Unlock() + m.scanAllCalls = make([]ScanAllCall, 0) + m.scanFoldersCalls = make([]ScanFoldersCall, 0) +} + +func (m *MockScanner) SetScanning(scanning bool) { + m.mu.Lock() + defer m.mu.Unlock() + m.scanningStatus = scanning +} + +func (m *MockScanner) SetStatusResponse(status *model.ScannerStatus) { + m.mu.Lock() + defer m.mu.Unlock() + m.statusResponse = status +} From bfa31a246f0f93a4884e4c8e798cec3d7c44072c Mon Sep 17 00:00:00 2001 From: Deluan Date: Tue, 11 Nov 2025 16:02:03 -0500 Subject: [PATCH 22/40] refactor(scanner): update scanner interface and implementations to use model.Scanner Signed-off-by: Deluan --- cmd/wire_gen.go | 22 ++++++------ cmd/wire_injectors.go | 3 +- core/library.go | 13 +++---- core/library_test.go | 52 ++++++++-------------------- model/scanner.go | 10 ++++++ scanner/controller.go | 11 +----- scanner/controller_test.go | 3 +- scanner/scanner_multilibrary_test.go | 2 +- scanner/scanner_test.go | 2 +- scanner/selective_scan_test.go | 2 +- scanner/watcher.go | 4 +-- server/subsonic/api.go | 5 ++- 12 files changed, 51 insertions(+), 78 deletions(-) diff --git a/cmd/wire_gen.go b/cmd/wire_gen.go index bf13dc731..d7b6a3ad2 100644 --- a/cmd/wire_gen.go +++ b/cmd/wire_gen.go @@ -69,9 +69,9 @@ func CreateNativeAPIRouter(ctx context.Context) *nativeapi.Router { artworkArtwork := artwork.NewArtwork(dataStore, fileCache, fFmpeg, provider) cacheWarmer := artwork.NewCacheWarmer(artworkArtwork, fileCache) broker := events.GetBroker() - scannerScanner := scanner.New(ctx, dataStore, cacheWarmer, broker, playlists, metricsMetrics) - watcher := scanner.GetWatcher(dataStore, scannerScanner) - library := core.NewLibrary(dataStore, scannerScanner, watcher, broker) + modelScanner := scanner.New(ctx, dataStore, cacheWarmer, broker, playlists, metricsMetrics) + watcher := scanner.GetWatcher(dataStore, modelScanner) + library := core.NewLibrary(dataStore, modelScanner, watcher, broker) maintenance := core.NewMaintenance(dataStore) router := nativeapi.New(dataStore, share, playlists, insights, library, maintenance) return router @@ -95,10 +95,10 @@ func CreateSubsonicAPIRouter(ctx context.Context) *subsonic.Router { cacheWarmer := artwork.NewCacheWarmer(artworkArtwork, fileCache) broker := events.GetBroker() playlists := core.NewPlaylists(dataStore) - scannerScanner := scanner.New(ctx, dataStore, cacheWarmer, broker, playlists, metricsMetrics) + modelScanner := scanner.New(ctx, dataStore, cacheWarmer, broker, playlists, metricsMetrics) playTracker := scrobbler.GetPlayTracker(dataStore, broker, manager) playbackServer := playback.GetInstance(dataStore) - router := subsonic.New(dataStore, artworkArtwork, mediaStreamer, archiver, players, provider, scannerScanner, broker, playlists, playTracker, share, playbackServer, metricsMetrics) + router := subsonic.New(dataStore, artworkArtwork, mediaStreamer, archiver, players, provider, modelScanner, broker, playlists, playTracker, share, playbackServer, metricsMetrics) return router } @@ -150,7 +150,7 @@ func CreatePrometheus() metrics.Metrics { return metricsMetrics } -func CreateScanner(ctx context.Context) scanner.Scanner { +func CreateScanner(ctx context.Context) model.Scanner { sqlDB := db.Db() dataStore := persistence.New(sqlDB) fileCache := artwork.GetImageCache() @@ -163,8 +163,8 @@ func CreateScanner(ctx context.Context) scanner.Scanner { cacheWarmer := artwork.NewCacheWarmer(artworkArtwork, fileCache) broker := events.GetBroker() playlists := core.NewPlaylists(dataStore) - scannerScanner := scanner.New(ctx, dataStore, cacheWarmer, broker, playlists, metricsMetrics) - return scannerScanner + modelScanner := scanner.New(ctx, dataStore, cacheWarmer, broker, playlists, metricsMetrics) + return modelScanner } func CreateScanWatcher(ctx context.Context) scanner.Watcher { @@ -180,8 +180,8 @@ func CreateScanWatcher(ctx context.Context) scanner.Watcher { cacheWarmer := artwork.NewCacheWarmer(artworkArtwork, fileCache) broker := events.GetBroker() playlists := core.NewPlaylists(dataStore) - scannerScanner := scanner.New(ctx, dataStore, cacheWarmer, broker, playlists, metricsMetrics) - watcher := scanner.GetWatcher(dataStore, scannerScanner) + modelScanner := scanner.New(ctx, dataStore, cacheWarmer, broker, playlists, metricsMetrics) + watcher := scanner.GetWatcher(dataStore, modelScanner) return watcher } @@ -202,7 +202,7 @@ func getPluginManager() plugins.Manager { // wire_injectors.go: -var allProviders = wire.NewSet(core.Set, artwork.Set, server.New, subsonic.New, nativeapi.New, public.New, persistence.New, lastfm.NewRouter, listenbrainz.NewRouter, events.GetBroker, scanner.New, scanner.GetWatcher, plugins.GetManager, metrics.GetPrometheusInstance, db.Db, wire.Bind(new(agents.PluginLoader), new(plugins.Manager)), wire.Bind(new(scrobbler.PluginLoader), new(plugins.Manager)), wire.Bind(new(metrics.PluginLoader), new(plugins.Manager)), wire.Bind(new(core.Scanner), new(scanner.Scanner)), wire.Bind(new(core.Watcher), new(scanner.Watcher))) +var allProviders = wire.NewSet(core.Set, artwork.Set, server.New, subsonic.New, nativeapi.New, public.New, persistence.New, lastfm.NewRouter, listenbrainz.NewRouter, events.GetBroker, scanner.New, scanner.GetWatcher, plugins.GetManager, metrics.GetPrometheusInstance, db.Db, wire.Bind(new(agents.PluginLoader), new(plugins.Manager)), wire.Bind(new(scrobbler.PluginLoader), new(plugins.Manager)), wire.Bind(new(metrics.PluginLoader), new(plugins.Manager)), wire.Bind(new(core.Watcher), new(scanner.Watcher))) func GetPluginManager(ctx context.Context) plugins.Manager { manager := getPluginManager() diff --git a/cmd/wire_injectors.go b/cmd/wire_injectors.go index e8759ac53..595d406b9 100644 --- a/cmd/wire_injectors.go +++ b/cmd/wire_injectors.go @@ -45,7 +45,6 @@ var allProviders = wire.NewSet( wire.Bind(new(agents.PluginLoader), new(plugins.Manager)), wire.Bind(new(scrobbler.PluginLoader), new(plugins.Manager)), wire.Bind(new(metrics.PluginLoader), new(plugins.Manager)), - wire.Bind(new(core.Scanner), new(scanner.Scanner)), wire.Bind(new(core.Watcher), new(scanner.Watcher)), ) @@ -103,7 +102,7 @@ func CreatePrometheus() metrics.Metrics { )) } -func CreateScanner(ctx context.Context) scanner.Scanner { +func CreateScanner(ctx context.Context) model.Scanner { panic(wire.Build( allProviders, )) diff --git a/core/library.go b/core/library.go index 8e17445ec..f4f55ec5a 100644 --- a/core/library.go +++ b/core/library.go @@ -21,11 +21,6 @@ import ( "github.com/navidrome/navidrome/utils/slice" ) -// Scanner interface for triggering scans. This is a subset of the full scanner.Scanner interface. -type Scanner interface { - ScanAll(ctx context.Context, fullScan bool) (warnings []string, err error) -} - // Watcher interface for managing file system watchers type Watcher interface { Watch(ctx context.Context, lib *model.Library) error @@ -43,13 +38,13 @@ type Library interface { type libraryService struct { ds model.DataStore - scanner Scanner + scanner model.Scanner watcher Watcher broker events.Broker } // NewLibrary creates a new Library service -func NewLibrary(ds model.DataStore, scanner Scanner, watcher Watcher, broker events.Broker) Library { +func NewLibrary(ds model.DataStore, scanner model.Scanner, watcher Watcher, broker events.Broker) Library { return &libraryService{ ds: ds, scanner: scanner, @@ -155,7 +150,7 @@ type libraryRepositoryWrapper struct { model.LibraryRepository ctx context.Context ds model.DataStore - scanner Scanner + scanner model.Scanner watcher Watcher broker events.Broker } @@ -192,7 +187,7 @@ func (r *libraryRepositoryWrapper) Save(entity interface{}) (string, error) { return strconv.Itoa(lib.ID), nil } -func (r *libraryRepositoryWrapper) Update(id string, entity interface{}, cols ...string) error { +func (r *libraryRepositoryWrapper) Update(id string, entity interface{}, _ ...string) error { lib := entity.(*model.Library) libID, err := strconv.Atoi(id) if err != nil { diff --git a/core/library_test.go b/core/library_test.go index bfbb4300a..bf73a62b7 100644 --- a/core/library_test.go +++ b/core/library_test.go @@ -29,7 +29,7 @@ var _ = Describe("Library Service", func() { var userRepo *tests.MockedUserRepo var ctx context.Context var tempDir string - var scanner *mockScanner + var scanner *tests.MockScanner var watcherManager *mockWatcherManager var broker *mockEventBroker @@ -43,7 +43,7 @@ var _ = Describe("Library Service", func() { ds.MockedUser = userRepo // Create a mock scanner that tracks calls - scanner = &mockScanner{} + scanner = tests.NewMockScanner() // Create a mock watcher manager watcherManager = &mockWatcherManager{ libraryStates: make(map[int]model.Library), @@ -616,11 +616,12 @@ var _ = Describe("Library Service", func() { // Wait briefly for the goroutine to complete Eventually(func() int { - return scanner.len() + return scanner.GetScanAllCallCount() }, "1s", "10ms").Should(Equal(1)) // Verify scan was called with correct parameters - Expect(scanner.ScanCalls[0].FullScan).To(BeFalse()) // Should be quick scan + calls := scanner.GetScanAllCalls() + Expect(calls[0].FullScan).To(BeFalse()) // Should be quick scan }) It("triggers scan when updating library path", func() { @@ -641,11 +642,12 @@ var _ = Describe("Library Service", func() { // Wait briefly for the goroutine to complete Eventually(func() int { - return scanner.len() + return scanner.GetScanAllCallCount() }, "1s", "10ms").Should(Equal(1)) // Verify scan was called with correct parameters - Expect(scanner.ScanCalls[0].FullScan).To(BeFalse()) // Should be quick scan + calls := scanner.GetScanAllCalls() + Expect(calls[0].FullScan).To(BeFalse()) // Should be quick scan }) It("does not trigger scan when updating library without path change", func() { @@ -661,7 +663,7 @@ var _ = Describe("Library Service", func() { // Wait a bit to ensure no scan was triggered Consistently(func() int { - return scanner.len() + return scanner.GetScanAllCallCount() }, "100ms", "10ms").Should(Equal(0)) }) @@ -674,7 +676,7 @@ var _ = Describe("Library Service", func() { // Ensure no scan was triggered since creation failed Consistently(func() int { - return scanner.len() + return scanner.GetScanAllCallCount() }, "100ms", "10ms").Should(Equal(0)) }) @@ -691,7 +693,7 @@ var _ = Describe("Library Service", func() { // Ensure no scan was triggered since update failed Consistently(func() int { - return scanner.len() + return scanner.GetScanAllCallCount() }, "100ms", "10ms").Should(Equal(0)) }) @@ -707,11 +709,12 @@ var _ = Describe("Library Service", func() { // Wait briefly for the goroutine to complete Eventually(func() int { - return scanner.len() + return scanner.GetScanAllCallCount() }, "1s", "10ms").Should(Equal(1)) // Verify scan was called with correct parameters - Expect(scanner.ScanCalls[0].FullScan).To(BeFalse()) // Should be quick scan + calls := scanner.GetScanAllCalls() + Expect(calls[0].FullScan).To(BeFalse()) // Should be quick scan }) It("does not trigger scan when library deletion fails", func() { @@ -721,7 +724,7 @@ var _ = Describe("Library Service", func() { // Ensure no scan was triggered since deletion failed Consistently(func() int { - return scanner.len() + return scanner.GetScanAllCallCount() }, "100ms", "10ms").Should(Equal(0)) }) @@ -868,31 +871,6 @@ var _ = Describe("Library Service", func() { }) }) -// mockScanner provides a simple mock implementation of core.Scanner for testing -type mockScanner struct { - ScanCalls []ScanCall - mu sync.RWMutex -} - -type ScanCall struct { - FullScan bool -} - -func (m *mockScanner) ScanAll(ctx context.Context, fullScan bool) (warnings []string, err error) { - m.mu.Lock() - defer m.mu.Unlock() - m.ScanCalls = append(m.ScanCalls, ScanCall{ - FullScan: fullScan, - }) - return []string{}, nil -} - -func (m *mockScanner) len() int { - m.mu.RLock() - defer m.mu.RUnlock() - return len(m.ScanCalls) -} - // mockWatcherManager provides a simple mock implementation of core.Watcher for testing type mockWatcherManager struct { StartedWatchers []model.Library diff --git a/model/scanner.go b/model/scanner.go index 756688a23..12a386ea0 100644 --- a/model/scanner.go +++ b/model/scanner.go @@ -1,6 +1,7 @@ package model import ( + "context" "fmt" "time" ) @@ -26,3 +27,12 @@ type ScannerStatus struct { ScanType string ElapsedTime time.Duration } + +type Scanner interface { + // ScanAll starts a scan of all libraries. This is a blocking operation. + ScanAll(ctx context.Context, fullScan bool) (warnings []string, err error) + // ScanFolders scans specific library/folder pairs, recursing into subdirectories. + // If targets is nil, it scans all libraries. This is a blocking operation. + ScanFolders(ctx context.Context, fullScan bool, targets []ScanTarget) (warnings []string, err error) + Status(context.Context) (*ScannerStatus, error) +} diff --git a/scanner/controller.go b/scanner/controller.go index b5ba2ddb7..8e308f51d 100644 --- a/scanner/controller.go +++ b/scanner/controller.go @@ -69,17 +69,8 @@ func ParseTargets(libFolders []string) ([]model.ScanTarget, error) { return targets, nil } -type Scanner interface { - // ScanAll starts a scan of all libraries. This is a blocking operation. - ScanAll(ctx context.Context, fullScan bool) (warnings []string, err error) - // ScanFolders scans specific library/folder pairs, recursing into subdirectories. - // If targets is nil, it scans all libraries. This is a blocking operation. - ScanFolders(ctx context.Context, fullScan bool, targets []model.ScanTarget) (warnings []string, err error) - Status(context.Context) (*model.ScannerStatus, error) -} - func New(rootCtx context.Context, ds model.DataStore, cw artwork.CacheWarmer, broker events.Broker, - pls core.Playlists, m metrics.Metrics) Scanner { + pls core.Playlists, m metrics.Metrics) model.Scanner { c := &controller{ rootCtx: rootCtx, ds: ds, diff --git a/scanner/controller_test.go b/scanner/controller_test.go index d1109b0be..929fa09ba 100644 --- a/scanner/controller_test.go +++ b/scanner/controller_test.go @@ -9,6 +9,7 @@ import ( "github.com/navidrome/navidrome/core/artwork" "github.com/navidrome/navidrome/core/metrics" "github.com/navidrome/navidrome/db" + "github.com/navidrome/navidrome/model" "github.com/navidrome/navidrome/persistence" "github.com/navidrome/navidrome/scanner" "github.com/navidrome/navidrome/server/events" @@ -20,7 +21,7 @@ import ( var _ = Describe("Controller", func() { var ctx context.Context var ds *tests.MockDataStore - var ctrl scanner.Scanner + var ctrl model.Scanner Describe("Status", func() { BeforeEach(func() { diff --git a/scanner/scanner_multilibrary_test.go b/scanner/scanner_multilibrary_test.go index f27ad52fc..66db62edf 100644 --- a/scanner/scanner_multilibrary_test.go +++ b/scanner/scanner_multilibrary_test.go @@ -32,7 +32,7 @@ var _ = Describe("Scanner - Multi-Library", Ordered, func() { var ctx context.Context var lib1, lib2 model.Library var ds *tests.MockDataStore - var s scanner.Scanner + var s model.Scanner createFS := func(path string, files fstest.MapFS) storagetest.FakeFS { fs := storagetest.FakeFS{} diff --git a/scanner/scanner_test.go b/scanner/scanner_test.go index e1b2e6f32..604561058 100644 --- a/scanner/scanner_test.go +++ b/scanner/scanner_test.go @@ -39,7 +39,7 @@ var _ = Describe("Scanner", Ordered, func() { var lib model.Library var ds *tests.MockDataStore var mfRepo *mockMediaFileRepo - var s scanner.Scanner + var s model.Scanner createFS := func(files fstest.MapFS) storagetest.FakeFS { fs := storagetest.FakeFS{} diff --git a/scanner/selective_scan_test.go b/scanner/selective_scan_test.go index 4bac1c2e1..13d0f02cb 100644 --- a/scanner/selective_scan_test.go +++ b/scanner/selective_scan_test.go @@ -28,7 +28,7 @@ var _ = Describe("Selective Scan - Deleted Child Folders", Ordered, func() { var ctx context.Context var lib model.Library var ds model.DataStore - var s scanner.Scanner + var s model.Scanner var fsys storagetest.FakeFS BeforeAll(func() { diff --git a/scanner/watcher.go b/scanner/watcher.go index 911ea310b..849ddf91a 100644 --- a/scanner/watcher.go +++ b/scanner/watcher.go @@ -24,7 +24,7 @@ type Watcher interface { type watcher struct { mainCtx context.Context ds model.DataStore - scanner Scanner + scanner model.Scanner triggerWait time.Duration watcherNotify chan scanNotification libraryWatchers map[int]*libraryWatcherInstance @@ -42,7 +42,7 @@ type scanNotification struct { } // GetWatcher returns the watcher singleton -func GetWatcher(ds model.DataStore, s Scanner) Watcher { +func GetWatcher(ds model.DataStore, s model.Scanner) Watcher { return singleton.GetInstance(func() *watcher { return &watcher{ ds: ds, diff --git a/server/subsonic/api.go b/server/subsonic/api.go index d08d3eb5b..f0e73c3d2 100644 --- a/server/subsonic/api.go +++ b/server/subsonic/api.go @@ -18,7 +18,6 @@ import ( "github.com/navidrome/navidrome/core/scrobbler" "github.com/navidrome/navidrome/log" "github.com/navidrome/navidrome/model" - "github.com/navidrome/navidrome/scanner" "github.com/navidrome/navidrome/server" "github.com/navidrome/navidrome/server/events" "github.com/navidrome/navidrome/server/subsonic/responses" @@ -39,7 +38,7 @@ type Router struct { players core.Players provider external.Provider playlists core.Playlists - scanner scanner.Scanner + scanner model.Scanner broker events.Broker scrobbler scrobbler.PlayTracker share core.Share @@ -48,7 +47,7 @@ type Router struct { } func New(ds model.DataStore, artwork artwork.Artwork, streamer core.MediaStreamer, archiver core.Archiver, - players core.Players, provider external.Provider, scanner scanner.Scanner, broker events.Broker, + players core.Players, provider external.Provider, scanner model.Scanner, broker events.Broker, playlists core.Playlists, scrobbler scrobbler.PlayTracker, share core.Share, playback playback.PlaybackServer, metrics metrics.Metrics, ) *Router { From eeb73d24f08af8e2316c60ccd5b039778eb944c1 Mon Sep 17 00:00:00 2001 From: Deluan Date: Tue, 11 Nov 2025 16:08:24 -0500 Subject: [PATCH 23/40] refactor(folder_repository): normalize target path handling by using filepath.Clean Signed-off-by: Deluan --- persistence/folder_repository.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/persistence/folder_repository.go b/persistence/folder_repository.go index 74edf2c12..4d35da62e 100644 --- a/persistence/folder_repository.go +++ b/persistence/folder_repository.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "os" + "path/filepath" "slices" "strings" "time" @@ -113,7 +114,8 @@ func (r folderRepository) GetFolderUpdateInfo(lib model.Library, targetPaths ... break } // Clean the path to normalize it. Paths stored in the folder table do not have leading/trailing slashes. - cleanPath := strings.Trim(targetPath, string(os.PathSeparator)) + cleanPath := strings.TrimPrefix(targetPath, string(os.PathSeparator)) + cleanPath = filepath.Clean(cleanPath) // Include the target folder itself by ID folderIDs = append(folderIDs, model.FolderID(lib, cleanPath)) From 4386c3a876cbf0e4043cc87c642d04c75cb1450c Mon Sep 17 00:00:00 2001 From: Deluan Date: Tue, 11 Nov 2025 16:15:51 -0500 Subject: [PATCH 24/40] test(folder_repository): add comprehensive tests for folder retrieval and child exclusion Signed-off-by: Deluan --- persistence/folder_repository_test.go | 82 +++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/persistence/folder_repository_test.go b/persistence/folder_repository_test.go index 166797933..6c24741c9 100644 --- a/persistence/folder_repository_test.go +++ b/persistence/folder_repository_test.go @@ -97,6 +97,88 @@ var _ = Describe("FolderRepository", func() { Expect(results[folder1.ID].Hash).To(Equal(folder1.Hash)) }) + It("includes all child folders when querying parent", func() { + // Create a parent folder with multiple children + parent := model.NewFolder(testLib, "TestParent/Music") + child1 := model.NewFolder(testLib, "TestParent/Music/Rock/Queen") + child2 := model.NewFolder(testLib, "TestParent/Music/Jazz") + otherParent := model.NewFolder(testLib, "TestParent2/Music/Jazz") + + Expect(repo.Put(parent)).To(Succeed()) + Expect(repo.Put(child1)).To(Succeed()) + Expect(repo.Put(child2)).To(Succeed()) + + // Query the parent folder - should return parent and all children + results, err := repo.GetFolderUpdateInfo(testLib, "TestParent/Music") + Expect(err).ToNot(HaveOccurred()) + Expect(results).To(HaveLen(3)) + Expect(results).To(HaveKey(parent.ID)) + Expect(results).To(HaveKey(child1.ID)) + Expect(results).To(HaveKey(child2.ID)) + Expect(results).ToNot(HaveKey(otherParent.ID)) + }) + + It("excludes children from other libraries", func() { + // Create parent in testLib + parent := model.NewFolder(testLib, "TestIsolation/Parent") + child := model.NewFolder(testLib, "TestIsolation/Parent/Child") + + Expect(repo.Put(parent)).To(Succeed()) + Expect(repo.Put(child)).To(Succeed()) + + // Create similar path in other library + otherParent := model.NewFolder(otherLib, "TestIsolation/Parent") + otherChild := model.NewFolder(otherLib, "TestIsolation/Parent/Child") + + Expect(repo.Put(otherParent)).To(Succeed()) + Expect(repo.Put(otherChild)).To(Succeed()) + + // Query should only return folders from testLib + results, err := repo.GetFolderUpdateInfo(testLib, "TestIsolation/Parent") + Expect(err).ToNot(HaveOccurred()) + Expect(results).To(HaveLen(2)) + Expect(results).To(HaveKey(parent.ID)) + Expect(results).To(HaveKey(child.ID)) + Expect(results).ToNot(HaveKey(otherParent.ID)) + Expect(results).ToNot(HaveKey(otherChild.ID)) + }) + + It("excludes missing children when querying parent", func() { + // Create parent and children, mark one as missing + parent := model.NewFolder(testLib, "TestMissingChild/Parent") + child1 := model.NewFolder(testLib, "TestMissingChild/Parent/Child1") + child2 := model.NewFolder(testLib, "TestMissingChild/Parent/Child2") + child2.Missing = true + + Expect(repo.Put(parent)).To(Succeed()) + Expect(repo.Put(child1)).To(Succeed()) + Expect(repo.Put(child2)).To(Succeed()) + + // Query parent - should only return parent and non-missing child + results, err := repo.GetFolderUpdateInfo(testLib, "TestMissingChild/Parent") + Expect(err).ToNot(HaveOccurred()) + Expect(results).To(HaveLen(2)) + Expect(results).To(HaveKey(parent.ID)) + Expect(results).To(HaveKey(child1.ID)) + Expect(results).ToNot(HaveKey(child2.ID)) + }) + + It("handles mix of existing and non-existing target paths", func() { + // Create folders for one path but not the other + existingParent := model.NewFolder(testLib, "TestMixed/Exists") + existingChild := model.NewFolder(testLib, "TestMixed/Exists/Child") + + Expect(repo.Put(existingParent)).To(Succeed()) + Expect(repo.Put(existingChild)).To(Succeed()) + + // Query both existing and non-existing paths + results, err := repo.GetFolderUpdateInfo(testLib, "TestMixed/Exists", "TestMixed/DoesNotExist") + Expect(err).ToNot(HaveOccurred()) + Expect(results).To(HaveLen(2)) + Expect(results).To(HaveKey(existingParent.ID)) + Expect(results).To(HaveKey(existingChild.ID)) + }) + It("handles empty folder path as root", func() { // Test querying for root folder without creating it (fixtures should have one) rootFolderID := model.FolderID(testLib, ".") From fadaaf3ad14e628162b4b559dbf1c344ed75da98 Mon Sep 17 00:00:00 2001 From: Deluan Date: Tue, 11 Nov 2025 17:35:45 -0500 Subject: [PATCH 25/40] refactor(scanner): simplify selective scan logic using slice.Filter Signed-off-by: Deluan --- scanner/scanner.go | 14 +++++--------- utils/slice/slice.go | 11 +++++++++++ utils/slice/slice_test.go | 38 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 9 deletions(-) diff --git a/scanner/scanner.go b/scanner/scanner.go index 96fdb1604..543b9b46a 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -15,6 +15,7 @@ import ( "github.com/navidrome/navidrome/log" "github.com/navidrome/navidrome/model" "github.com/navidrome/navidrome/utils/run" + "github.com/navidrome/navidrome/utils/slice" ) type scannerImpl struct { @@ -30,7 +31,6 @@ type scanState struct { changesDetected atomic.Bool libraries model.Libraries // Store libraries list for consistency across phases targets map[int][]string // Optional: map[libraryID][]folderPaths for selective scans - affectedLibIDs []int // IDs of libraries involved in the scan (for GC scoping) } func (s *scanState) sendProgress(info *ProgressInfo) { @@ -74,7 +74,6 @@ func (s *scannerImpl) scanFolders(ctx context.Context, fullScan bool, targets [] if isSelectiveScan { // Selective scan: filter libraries and build targets map state.targets = make(map[int][]string) - affectedLibIDSet := make(map[int]bool) for _, target := range targets { folderPath := target.FolderPath @@ -82,15 +81,12 @@ func (s *scannerImpl) scanFolders(ctx context.Context, fullScan bool, targets [] folderPath = "." } state.targets[target.LibraryID] = append(state.targets[target.LibraryID], folderPath) - affectedLibIDSet[target.LibraryID] = true } - for _, lib := range allLibs { - if affectedLibIDSet[lib.ID] { - libs = append(libs, lib) - state.affectedLibIDs = append(state.affectedLibIDs, lib.ID) - } - } + // Filter libraries to only those in targets + libs = slice.Filter(allLibs, func(lib model.Library) bool { + return len(state.targets[lib.ID]) > 0 + }) log.Info(ctx, "Scanner: Starting selective scan", "fullScan", state.fullScan, "numLibraries", len(libs), "numTargets", len(targets)) } else { diff --git a/utils/slice/slice.go b/utils/slice/slice.go index 1d7c64f50..b1f50afcc 100644 --- a/utils/slice/slice.go +++ b/utils/slice/slice.go @@ -171,3 +171,14 @@ func SeqFunc[I, O any](s []I, f func(I) O) iter.Seq[O] { } } } + +// Filter returns a new slice containing only the elements of s for which filterFunc returns true +func Filter[T any](s []T, filterFunc func(T) bool) []T { + var result []T + for _, item := range s { + if filterFunc(item) { + result = append(result, item) + } + } + return result +} diff --git a/utils/slice/slice_test.go b/utils/slice/slice_test.go index c6d4be1e0..65e5f0934 100644 --- a/utils/slice/slice_test.go +++ b/utils/slice/slice_test.go @@ -172,4 +172,42 @@ var _ = Describe("Slice Utils", func() { Expect(result).To(ConsistOf("2", "4", "6", "8")) }) }) + + Describe("Filter", func() { + It("returns empty slice for an empty input", func() { + filterFunc := func(v int) bool { return v > 0 } + result := slice.Filter([]int{}, filterFunc) + Expect(result).To(BeEmpty()) + }) + + It("returns all elements when filter matches all", func() { + filterFunc := func(v int) bool { return v > 0 } + result := slice.Filter([]int{1, 2, 3, 4}, filterFunc) + Expect(result).To(HaveExactElements(1, 2, 3, 4)) + }) + + It("returns empty slice when filter matches none", func() { + filterFunc := func(v int) bool { return v > 10 } + result := slice.Filter([]int{1, 2, 3, 4}, filterFunc) + Expect(result).To(BeEmpty()) + }) + + It("returns only matching elements", func() { + filterFunc := func(v int) bool { return v%2 == 0 } + result := slice.Filter([]int{1, 2, 3, 4, 5, 6}, filterFunc) + Expect(result).To(HaveExactElements(2, 4, 6)) + }) + + It("works with string slices", func() { + filterFunc := func(s string) bool { return len(s) > 3 } + result := slice.Filter([]string{"a", "abc", "abcd", "ab", "abcde"}, filterFunc) + Expect(result).To(HaveExactElements("abcd", "abcde")) + }) + + It("preserves order of elements", func() { + filterFunc := func(v int) bool { return v%2 == 1 } + result := slice.Filter([]int{9, 8, 7, 6, 5, 4, 3, 2, 1}, filterFunc) + Expect(result).To(HaveExactElements(9, 7, 5, 3, 1)) + }) + }) }) From b9f368ad0d1276b7274c376bec1055ee36351649 Mon Sep 17 00:00:00 2001 From: Deluan Date: Tue, 11 Nov 2025 18:38:40 -0500 Subject: [PATCH 26/40] refactor(scanner): streamline phase folder and album creation by removing unnecessary library parameter Signed-off-by: Deluan --- scanner/phase_1_folders.go | 4 ++-- scanner/phase_3_refresh_albums.go | 7 +++--- scanner/phase_3_refresh_albums_test.go | 4 ++-- scanner/scanner.go | 30 +++++++++++++------------- 4 files changed, 22 insertions(+), 23 deletions(-) diff --git a/scanner/phase_1_folders.go b/scanner/phase_1_folders.go index b33814fdc..3d04d525d 100644 --- a/scanner/phase_1_folders.go +++ b/scanner/phase_1_folders.go @@ -26,10 +26,10 @@ import ( "github.com/navidrome/navidrome/utils/slice" ) -func createPhaseFolders(ctx context.Context, state *scanState, ds model.DataStore, cw artwork.CacheWarmer, libs []model.Library) *phaseFolders { +func createPhaseFolders(ctx context.Context, state *scanState, ds model.DataStore, cw artwork.CacheWarmer) *phaseFolders { var jobs []*scanJob var updatedLibs []model.Library - for _, lib := range libs { + for _, lib := range state.libraries { if lib.LastScanStartedAt.IsZero() { err := ds.Library(ctx).ScanBegin(lib.ID, state.fullScan) if err != nil { diff --git a/scanner/phase_3_refresh_albums.go b/scanner/phase_3_refresh_albums.go index f51aa8f4b..33e0fed01 100644 --- a/scanner/phase_3_refresh_albums.go +++ b/scanner/phase_3_refresh_albums.go @@ -27,14 +27,13 @@ import ( type phaseRefreshAlbums struct { ds model.DataStore ctx context.Context - libs model.Libraries refreshed atomic.Uint32 skipped atomic.Uint32 state *scanState } -func createPhaseRefreshAlbums(ctx context.Context, state *scanState, ds model.DataStore, libs model.Libraries) *phaseRefreshAlbums { - return &phaseRefreshAlbums{ctx: ctx, ds: ds, libs: libs, state: state} +func createPhaseRefreshAlbums(ctx context.Context, state *scanState, ds model.DataStore) *phaseRefreshAlbums { + return &phaseRefreshAlbums{ctx: ctx, ds: ds, state: state} } func (p *phaseRefreshAlbums) description() string { @@ -47,7 +46,7 @@ func (p *phaseRefreshAlbums) producer() ppl.Producer[*model.Album] { func (p *phaseRefreshAlbums) produce(put func(album *model.Album)) error { count := 0 - for _, lib := range p.libs { + for _, lib := range p.state.libraries { cursor, err := p.ds.Album(p.ctx).GetTouchedAlbums(lib.ID) if err != nil { return fmt.Errorf("loading touched albums: %w", err) diff --git a/scanner/phase_3_refresh_albums_test.go b/scanner/phase_3_refresh_albums_test.go index dea2556f0..1f0baf428 100644 --- a/scanner/phase_3_refresh_albums_test.go +++ b/scanner/phase_3_refresh_albums_test.go @@ -32,8 +32,8 @@ var _ = Describe("phaseRefreshAlbums", func() { {ID: 1, Name: "Library 1"}, {ID: 2, Name: "Library 2"}, } - state = &scanState{} - phase = createPhaseRefreshAlbums(ctx, state, ds, libs) + state = &scanState{libraries: libs} + phase = createPhaseRefreshAlbums(ctx, state, ds) }) Describe("description", func() { diff --git a/scanner/scanner.go b/scanner/scanner.go index 543b9b46a..c0bf98c26 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -39,6 +39,10 @@ func (s *scanState) sendProgress(info *ProgressInfo) { } } +func (s *scanState) isSelectiveScan() bool { + return len(s.targets) > 0 +} + func (s *scanState) sendWarning(msg string) { s.sendProgress(&ProgressInfo{Warning: msg}) } @@ -68,10 +72,7 @@ func (s *scannerImpl) scanFolders(ctx context.Context, fullScan bool, targets [] return } - var libs model.Libraries - isSelectiveScan := len(targets) > 0 - - if isSelectiveScan { + if len(targets) > 0 { // Selective scan: filter libraries and build targets map state.targets = make(map[int][]string) @@ -84,24 +85,23 @@ func (s *scannerImpl) scanFolders(ctx context.Context, fullScan bool, targets [] } // Filter libraries to only those in targets - libs = slice.Filter(allLibs, func(lib model.Library) bool { + state.libraries = slice.Filter(allLibs, func(lib model.Library) bool { return len(state.targets[lib.ID]) > 0 }) - log.Info(ctx, "Scanner: Starting selective scan", "fullScan", state.fullScan, "numLibraries", len(libs), "numTargets", len(targets)) + log.Info(ctx, "Scanner: Starting selective scan", "fullScan", state.fullScan, "numLibraries", len(state.libraries), "numTargets", len(targets)) } else { // Full library scan - libs = allLibs - log.Info(ctx, "Scanner: Starting scan", "fullScan", state.fullScan, "numLibraries", len(libs)) + state.libraries = allLibs + log.Info(ctx, "Scanner: Starting scan", "fullScan", state.fullScan, "numLibraries", len(state.libraries)) } - state.libraries = libs // Store scan type and start time scanType := "quick" if state.fullScan { scanType = "full" } - if isSelectiveScan { + if state.isSelectiveScan() { scanType += "-selective" } _ = s.ds.Property(ctx).Put(consts.LastScanTypeKey, scanType) @@ -109,11 +109,11 @@ func (s *scannerImpl) scanFolders(ctx context.Context, fullScan bool, targets [] // if there was a full scan in progress, force a full scan if !state.fullScan { - for _, lib := range libs { + for _, lib := range state.libraries { if lib.FullScanInProgress { log.Info(ctx, "Scanner: Interrupted full scan detected", "lib", lib.Name) state.fullScan = true - if isSelectiveScan { + if state.isSelectiveScan() { _ = s.ds.Property(ctx).Put(consts.LastScanTypeKey, "full-selective") } else { _ = s.ds.Property(ctx).Put(consts.LastScanTypeKey, "full") @@ -125,7 +125,7 @@ func (s *scannerImpl) scanFolders(ctx context.Context, fullScan bool, targets [] err = run.Sequentially( // Phase 1: Scan all libraries and import new/updated files - runPhase[*folderEntry](ctx, 1, createPhaseFolders(ctx, &state, s.ds, s.cw, libs)), + runPhase[*folderEntry](ctx, 1, createPhaseFolders(ctx, &state, s.ds, s.cw)), // Phase 2: Process missing files, checking for moves runPhase[*missingTracks](ctx, 2, createPhaseMissingTracks(ctx, &state, s.ds)), @@ -133,7 +133,7 @@ func (s *scannerImpl) scanFolders(ctx context.Context, fullScan bool, targets [] // Phases 3 and 4 can be run in parallel run.Parallel( // Phase 3: Refresh all new/changed albums and update artists - runPhase[*model.Album](ctx, 3, createPhaseRefreshAlbums(ctx, &state, s.ds, libs)), + runPhase[*model.Album](ctx, 3, createPhaseRefreshAlbums(ctx, &state, s.ds)), // Phase 4: Import/update playlists runPhase[*model.Folder](ctx, 4, createPhasePlaylists(ctx, &state, s.ds, s.pls, s.cw)), @@ -166,7 +166,7 @@ func (s *scannerImpl) scanFolders(ctx context.Context, fullScan bool, targets [] state.sendProgress(&ProgressInfo{ChangesDetected: true}) } - if isSelectiveScan { + if state.isSelectiveScan() { log.Info(ctx, "Scanner: Finished scanning selected folders", "duration", time.Since(startTime), "numTargets", len(targets)) } else { log.Info(ctx, "Scanner: Finished scanning all libraries", "duration", time.Since(startTime)) From 78904257dfef3d2e493c023494babe24dc3cad43 Mon Sep 17 00:00:00 2001 From: Deluan Date: Tue, 11 Nov 2025 19:43:36 -0500 Subject: [PATCH 27/40] refactor(scanner): move initialization logic from phase_1 to the scanner itself Signed-off-by: Deluan --- scanner/phase_1_folders.go | 29 +++--------------- scanner/phase_2_missing_tracks.go | 3 -- scanner/scanner.go | 50 +++++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 28 deletions(-) diff --git a/scanner/phase_1_folders.go b/scanner/phase_1_folders.go index 3d04d525d..fb78e81a4 100644 --- a/scanner/phase_1_folders.go +++ b/scanner/phase_1_folders.go @@ -27,31 +27,14 @@ import ( ) func createPhaseFolders(ctx context.Context, state *scanState, ds model.DataStore, cw artwork.CacheWarmer) *phaseFolders { + // At this point, all libraries in state.libraries have been initialized + // (LastScanStartedAt has been set by prepareLibrariesForScan in scanner.go) var jobs []*scanJob - var updatedLibs []model.Library - for _, lib := range state.libraries { - if lib.LastScanStartedAt.IsZero() { - err := ds.Library(ctx).ScanBegin(lib.ID, state.fullScan) - if err != nil { - log.Error(ctx, "Scanner: Error updating last scan started at", "lib", lib.Name, err) - state.sendWarning(err.Error()) - continue - } - // Reload library to get updated state - l, err := ds.Library(ctx).Get(lib.ID) - if err != nil { - log.Error(ctx, "Scanner: Error reloading library", "lib", lib.Name, err) - state.sendWarning(err.Error()) - continue - } - lib = *l - } else { - log.Debug(ctx, "Scanner: Resuming previous scan", "lib", lib.Name, "lastScanStartedAt", lib.LastScanStartedAt, "fullScan", lib.FullScanInProgress) - } + for _, lib := range state.libraries { // Get target folders for this library if selective scan var targetFolders []string - if state.targets != nil { + if state.isSelectiveScan() { targetFolders = state.targets[lib.ID] } @@ -62,12 +45,8 @@ func createPhaseFolders(ctx context.Context, state *scanState, ds model.DataStor continue } jobs = append(jobs, job) - updatedLibs = append(updatedLibs, lib) } - // Update the state with the libraries that have been processed and have their scan timestamps set - state.libraries = updatedLibs - return &phaseFolders{jobs: jobs, ctx: ctx, ds: ds, state: state} } diff --git a/scanner/phase_2_missing_tracks.go b/scanner/phase_2_missing_tracks.go index a6c0e261e..de93ed6ee 100644 --- a/scanner/phase_2_missing_tracks.go +++ b/scanner/phase_2_missing_tracks.go @@ -69,9 +69,6 @@ func (p *phaseMissingTracks) produce(put func(tracks *missingTracks)) error { } } for _, lib := range p.state.libraries { - if lib.LastScanStartedAt.IsZero() { - continue - } log.Debug(p.ctx, "Scanner: Checking missing tracks", "libraryId", lib.ID, "libraryName", lib.Name) cursor, err := p.ds.MediaFile(p.ctx).GetMissingAndMatching(lib.ID) if err != nil { diff --git a/scanner/scanner.go b/scanner/scanner.go index c0bf98c26..539e52933 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -123,6 +123,14 @@ func (s *scannerImpl) scanFolders(ctx context.Context, fullScan bool, targets [] } } + // Prepare libraries for scanning (initialize LastScanStartedAt if needed) + err = s.prepareLibrariesForScan(ctx, &state) + if err != nil { + log.Error(ctx, "Scanner: Error preparing libraries for scan", err) + state.sendError(err) + return + } + err = run.Sequentially( // Phase 1: Scan all libraries and import new/updated files runPhase[*folderEntry](ctx, 1, createPhaseFolders(ctx, &state, s.ds, s.cw)), @@ -173,6 +181,48 @@ func (s *scannerImpl) scanFolders(ctx context.Context, fullScan bool, targets [] } } +// prepareLibrariesForScan initializes the scan for all libraries in the state. +// It calls ScanBegin for libraries that haven't started scanning yet (LastScanStartedAt is zero), +// reloads them to get the updated state, and filters out any libraries that fail to initialize. +func (s *scannerImpl) prepareLibrariesForScan(ctx context.Context, state *scanState) error { + var successfulLibs []model.Library + + for _, lib := range state.libraries { + if lib.LastScanStartedAt.IsZero() { + // This is a new scan - mark it as started + err := s.ds.Library(ctx).ScanBegin(lib.ID, state.fullScan) + if err != nil { + log.Error(ctx, "Scanner: Error marking scan start", "lib", lib.Name, err) + state.sendWarning(err.Error()) + continue + } + + // Reload library to get updated state (timestamps, etc.) + reloadedLib, err := s.ds.Library(ctx).Get(lib.ID) + if err != nil { + log.Error(ctx, "Scanner: Error reloading library", "lib", lib.Name, err) + state.sendWarning(err.Error()) + continue + } + lib = *reloadedLib + } else { + // This is a resumed scan + log.Debug(ctx, "Scanner: Resuming previous scan", "lib", lib.Name, + "lastScanStartedAt", lib.LastScanStartedAt, "fullScan", lib.FullScanInProgress) + } + + successfulLibs = append(successfulLibs, lib) + } + + if len(successfulLibs) == 0 { + return fmt.Errorf("no libraries available for scanning") + } + + // Update state with only successfully initialized libraries + state.libraries = successfulLibs + return nil +} + func (s *scannerImpl) runGC(ctx context.Context, state *scanState) func() error { return func() error { state.sendProgress(&ProgressInfo{ForceUpdate: true}) From 71acd02d1b24cdc469a071228e868de65155d209 Mon Sep 17 00:00:00 2001 From: Deluan Date: Tue, 11 Nov 2025 19:47:54 -0500 Subject: [PATCH 28/40] refactor(tests): rename selective scan test file to scanner_selective_test.go Signed-off-by: Deluan --- scanner/{selective_scan_test.go => scanner_selective_test.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename scanner/{selective_scan_test.go => scanner_selective_test.go} (100%) diff --git a/scanner/selective_scan_test.go b/scanner/scanner_selective_test.go similarity index 100% rename from scanner/selective_scan_test.go rename to scanner/scanner_selective_test.go From 7ed817c1c03ca4b40d98610a6580cc313273e051 Mon Sep 17 00:00:00 2001 From: Deluan Date: Tue, 11 Nov 2025 20:04:54 -0500 Subject: [PATCH 29/40] feat(configuration): add DevSelectiveWatcher configuration option Signed-off-by: Deluan --- conf/configuration.go | 2 ++ scanner/watcher.go | 7 ++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/conf/configuration.go b/conf/configuration.go index 7292c7dfe..a9fee00e4 100644 --- a/conf/configuration.go +++ b/conf/configuration.go @@ -125,6 +125,7 @@ type configOptions struct { DevAlbumInfoTimeToLive time.Duration DevExternalScanner bool DevScannerThreads uint + DevSelectiveWatcher bool DevInsightsInitialDelay time.Duration DevEnablePlayerInsights bool DevEnablePluginsInsights bool @@ -600,6 +601,7 @@ func setViperDefaults() { viper.SetDefault("devalbuminfotimetolive", consts.AlbumInfoTimeToLive) viper.SetDefault("devexternalscanner", true) viper.SetDefault("devscannerthreads", 5) + viper.SetDefault("devselectivewatcher", true) viper.SetDefault("devinsightsinitialdelay", consts.InsightsInitialDelay) viper.SetDefault("devenableplayerinsights", true) viper.SetDefault("devenablepluginsinsights", true) diff --git a/scanner/watcher.go b/scanner/watcher.go index 849ddf91a..80cada050 100644 --- a/scanner/watcher.go +++ b/scanner/watcher.go @@ -99,7 +99,12 @@ func (w *watcher) Run(ctx context.Context) error { targets = make(map[model.ScanTarget]struct{}) go func() { - _, err := w.scanner.ScanFolders(ctx, false, targetSlice) + var err error + if conf.Server.DevSelectiveWatcher { + _, err = w.scanner.ScanFolders(ctx, false, targetSlice) + } else { + _, err = w.scanner.ScanAll(ctx, false) + } if err != nil { log.Error(ctx, "Watcher: Error scanning", err) } else { From 587249bf816b9bbca3823f0c9b3e23fbb08b30a9 Mon Sep 17 00:00:00 2001 From: Deluan Date: Tue, 11 Nov 2025 20:42:46 -0500 Subject: [PATCH 30/40] feat(watcher): enhance .ndignore handling for folder deletions and file changes Signed-off-by: Deluan --- scanner/watcher.go | 20 +++++- scanner/watcher_test.go | 154 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 171 insertions(+), 3 deletions(-) diff --git a/scanner/watcher.go b/scanner/watcher.go index 80cada050..3b47f8416 100644 --- a/scanner/watcher.go +++ b/scanner/watcher.go @@ -225,13 +225,18 @@ func (w *watcher) watchLibrary(ctx context.Context, lib *model.Library) error { log.Info(ctx, "Watcher started for library", "libraryID", lib.ID, "name", lib.Name, "path", lib.Path, "absoluteLibPath", absLibPath) + return w.processLibraryEvents(ctx, lib, fsys, c, absLibPath) +} + +// processLibraryEvents processes filesystem events for a library. +func (w *watcher) processLibraryEvents(ctx context.Context, lib *model.Library, fsys storage.MusicFS, events <-chan string, absLibPath string) error { for { select { case <-ctx.Done(): log.Debug(ctx, "Watcher stopped due to context cancellation", "libraryID", lib.ID, "name", lib.Name) return nil - case path := <-c: - path, err = filepath.Rel(absLibPath, path) + case path := <-events: + path, err := filepath.Rel(absLibPath, path) if err != nil { log.Error(ctx, "Error getting relative path", "libraryID", lib.ID, "absolutePath", absLibPath, "path", path, err) continue @@ -243,9 +248,18 @@ func (w *watcher) watchLibrary(ctx context.Context, lib *model.Library) error { } log.Trace(ctx, "Detected change", "libraryID", lib.ID, "path", path, "absoluteLibPath", absLibPath) + // Check if the original path (before resolution) matches .ndignore patterns + // This is crucial for deleted folders - if a deleted folder matches .ndignore, + // we should ignore it BEFORE resolveFolderPath walks up to the parent + if w.shouldIgnoreFolderPath(ctx, fsys, path) { + log.Debug(ctx, "Ignoring change matching .ndignore pattern", "libraryID", lib.ID, "path", path) + continue + } + // Find the folder to scan - validate path exists as directory, walk up if needed folderPath := resolveFolderPath(fsys, path) - if w.shouldIgnoreFolderPath(ctx, fsys, folderPath) { + // Double-check after resolution in case the resolved path is different and also matches patterns + if folderPath != path && w.shouldIgnoreFolderPath(ctx, fsys, folderPath) { log.Trace(ctx, "Ignoring change in folder matching .ndignore pattern", "libraryID", lib.ID, "folderPath", folderPath) continue } diff --git a/scanner/watcher_test.go b/scanner/watcher_test.go index 7ae52e725..01bfb2491 100644 --- a/scanner/watcher_test.go +++ b/scanner/watcher_test.go @@ -3,6 +3,7 @@ package scanner import ( "context" "io/fs" + "path/filepath" "testing/fstest" "time" @@ -276,6 +277,159 @@ var _ = Describe("Watcher", func() { Expect(libraryIDs).To(HaveKey(2)) }) }) + + Describe(".ndignore handling", func() { + var ctx context.Context + var cancel context.CancelFunc + var w *watcher + var mockFS *mockMusicFS + var lib *model.Library + var eventChan chan string + var absLibPath string + + BeforeEach(func() { + ctx, cancel = context.WithCancel(GinkgoT().Context()) + DeferCleanup(cancel) + + // Set up library + var err error + absLibPath, err = filepath.Abs(".") + Expect(err).NotTo(HaveOccurred()) + + lib = &model.Library{ + ID: 1, + Name: "Test Library", + Path: absLibPath, + } + + // Create watcher with notification channel + w = &watcher{ + watcherNotify: make(chan scanNotification, 10), + } + + eventChan = make(chan string, 10) + }) + + // Helper to send an event - converts relative path to absolute + sendEvent := func(relativePath string) { + path := filepath.Join(absLibPath, relativePath) + eventChan <- path + } + + // Helper to start the real event processing loop + startEventProcessing := func() { + go func() { + defer GinkgoRecover() + // Call the actual processLibraryEvents method - testing the real implementation! + _ = w.processLibraryEvents(ctx, lib, mockFS, eventChan, absLibPath) + }() + } + + Context("when a folder matching .ndignore is deleted", func() { + BeforeEach(func() { + // Create filesystem with .ndignore containing _TEMP pattern + // The deleted folder (_TEMP) will NOT exist in the filesystem + mockFS = &mockMusicFS{ + FS: fstest.MapFS{ + "rock": &fstest.MapFile{Mode: fs.ModeDir}, + "rock/.ndignore": &fstest.MapFile{Data: []byte("_TEMP\n")}, + "rock/valid_album": &fstest.MapFile{Mode: fs.ModeDir}, + "rock/valid_album/track.mp3": &fstest.MapFile{Data: []byte("audio")}, + }, + } + }) + + It("should NOT send scan notification when deleted folder matches .ndignore", func() { + startEventProcessing() + + // Simulate deletion event for rock/_TEMP + sendEvent("rock/_TEMP") + + // Wait a bit to ensure event is processed + time.Sleep(50 * time.Millisecond) + + // No notification should have been sent + Consistently(eventChan, 100*time.Millisecond).Should(BeEmpty()) + }) + + It("should send scan notification for valid folder deletion", func() { + startEventProcessing() + + // Simulate deletion event for rock/other_folder (not in .ndignore and doesn't exist) + // Since it doesn't exist in mockFS, resolveFolderPath will walk up to "rock" + sendEvent("rock/other_folder") + + // Should receive notification for parent folder + Eventually(w.watcherNotify, 200*time.Millisecond).Should(Receive(Equal(scanNotification{ + Library: lib, + FolderPath: "rock", + }))) + }) + }) + + Context("with nested folder patterns", func() { + BeforeEach(func() { + mockFS = &mockMusicFS{ + FS: fstest.MapFS{ + "music": &fstest.MapFile{Mode: fs.ModeDir}, + "music/.ndignore": &fstest.MapFile{Data: []byte("**/temp\n**/cache\n")}, + "music/rock": &fstest.MapFile{Mode: fs.ModeDir}, + "music/rock/artist": &fstest.MapFile{Mode: fs.ModeDir}, + }, + } + }) + + It("should NOT send notification when nested ignored folder is deleted", func() { + startEventProcessing() + + // Simulate deletion of music/rock/artist/temp (matches **/temp) + sendEvent("music/rock/artist/temp") + + // Wait to ensure event is processed + time.Sleep(50 * time.Millisecond) + + // No notification should be sent + Expect(w.watcherNotify).To(BeEmpty(), "Expected no scan notification for nested ignored folder") + }) + + It("should send notification for non-ignored nested folder", func() { + startEventProcessing() + + // Simulate change in music/rock/artist (doesn't match any pattern) + sendEvent("music/rock/artist") + + // Should receive notification + Eventually(w.watcherNotify, 200*time.Millisecond).Should(Receive(Equal(scanNotification{ + Library: lib, + FolderPath: "music/rock/artist", + }))) + }) + }) + + Context("with file events in ignored folders", func() { + BeforeEach(func() { + mockFS = &mockMusicFS{ + FS: fstest.MapFS{ + "rock": &fstest.MapFile{Mode: fs.ModeDir}, + "rock/.ndignore": &fstest.MapFile{Data: []byte("_TEMP\n")}, + }, + } + }) + + It("should NOT send notification for file changes in ignored folders", func() { + startEventProcessing() + + // Simulate file change in rock/_TEMP/file.mp3 + sendEvent("rock/_TEMP/file.mp3") + + // Wait to ensure event is processed + time.Sleep(50 * time.Millisecond) + + // No notification should be sent + Expect(w.watcherNotify).To(BeEmpty(), "Expected no scan notification for file in ignored folder") + }) + }) + }) }) var _ = Describe("resolveFolderPath", func() { From 08bccad02073e9d29667cd792dd11c2bb4d99ba1 Mon Sep 17 00:00:00 2001 From: Deluan Date: Tue, 11 Nov 2025 20:52:57 -0500 Subject: [PATCH 31/40] docs(scanner): comments Signed-off-by: Deluan --- scanner/phase_1_folders.go | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/scanner/phase_1_folders.go b/scanner/phase_1_folders.go index fb78e81a4..52bed9fde 100644 --- a/scanner/phase_1_folders.go +++ b/scanner/phase_1_folders.go @@ -27,10 +27,9 @@ import ( ) func createPhaseFolders(ctx context.Context, state *scanState, ds model.DataStore, cw artwork.CacheWarmer) *phaseFolders { - // At this point, all libraries in state.libraries have been initialized - // (LastScanStartedAt has been set by prepareLibrariesForScan in scanner.go) var jobs []*scanJob + // Create scan jobs for all libraries for _, lib := range state.libraries { // Get target folders for this library if selective scan var targetFolders []string @@ -61,11 +60,8 @@ type scanJob struct { } func newScanJob(ctx context.Context, ds model.DataStore, cw artwork.CacheWarmer, lib model.Library, fullScan bool, targetFolders []string) (*scanJob, error) { - var lastUpdates map[string]model.FolderUpdateInfo - var err error - // Get folder updates, optionally filtered to specific target folders - lastUpdates, err = ds.Folder(ctx).GetFolderUpdateInfo(lib, targetFolders...) + lastUpdates, err := ds.Folder(ctx).GetFolderUpdateInfo(lib, targetFolders...) if err != nil { return nil, fmt.Errorf("getting last updates: %w", err) } From e222d8d8db7a857a244af70eaebb3e94a0d36f25 Mon Sep 17 00:00:00 2001 From: Deluan Date: Tue, 11 Nov 2025 21:03:53 -0500 Subject: [PATCH 32/40] refactor(scanner): enhance walkDirTree to support target folder scanning Signed-off-by: Deluan --- scanner/phase_1_folders.go | 4 +- scanner/walk_dir_tree.go | 28 ++-- scanner/walk_dir_tree_test.go | 294 +++++++++++++++++----------------- 3 files changed, 156 insertions(+), 170 deletions(-) diff --git a/scanner/phase_1_folders.go b/scanner/phase_1_folders.go index 52bed9fde..2dd5c6639 100644 --- a/scanner/phase_1_folders.go +++ b/scanner/phase_1_folders.go @@ -140,10 +140,8 @@ func (p *phaseFolders) producer() ppl.Producer[*folderEntry] { // Use selective folder loading if target folders are specified if len(job.targetFolders) > 0 { log.Debug(p.ctx, "Scanner: Loading specific folders only (non-recursive)", "lib", job.lib.Name, "numTargets", len(job.targetFolders)) - outputChan, err = loadSpecificFolders(p.ctx, job, job.targetFolders) - } else { - outputChan, err = walkDirTree(p.ctx, job) } + outputChan, err = walkDirTree(p.ctx, job, job.targetFolders...) if err != nil { log.Warn(p.ctx, "Scanner: Error scanning library", "lib", job.lib.Name, err) diff --git a/scanner/walk_dir_tree.go b/scanner/walk_dir_tree.go index 59c7000e5..8500824cf 100644 --- a/scanner/walk_dir_tree.go +++ b/scanner/walk_dir_tree.go @@ -15,27 +15,19 @@ import ( "github.com/navidrome/navidrome/utils" ) -func walkDirTree(ctx context.Context, job *scanJob) (<-chan *folderEntry, error) { +// walkDirTree recursively walks the directory tree starting from the given targetFolders. +// If no targetFolders are provided, it starts from the root folder ("."). +// It returns a channel of folderEntry pointers representing each folder found. +func walkDirTree(ctx context.Context, job *scanJob, targetFolders ...string) (<-chan *folderEntry, error) { results := make(chan *folderEntry) + folders := targetFolders + if len(targetFolders) == 0 { + // No specific folders provided, scan the root folder + folders = []string{"."} + } go func() { defer close(results) - checker := newIgnoreChecker(job.fs) - err := walkFolder(ctx, job, ".", checker, results) - if err != nil { - log.Error(ctx, "Scanner: There were errors reading directories from filesystem", "path", job.lib.Path, err) - return - } - log.Debug(ctx, "Scanner: Finished reading folders", "lib", job.lib.Name, "path", job.lib.Path, "numFolders", job.numFolders.Load()) - }() - return results, nil -} - -// loadSpecificFolders loads the specified folders and recursively walks their subdirectories -func loadSpecificFolders(ctx context.Context, job *scanJob, targetFolders []string) (<-chan *folderEntry, error) { - results := make(chan *folderEntry) - go func() { - defer close(results) - for _, folderPath := range targetFolders { + for _, folderPath := range folders { if utils.IsCtxDone(ctx) { return } diff --git a/scanner/walk_dir_tree_test.go b/scanner/walk_dir_tree_test.go index 840cd5dd2..5cd1f3ab6 100644 --- a/scanner/walk_dir_tree_test.go +++ b/scanner/walk_dir_tree_test.go @@ -25,167 +25,163 @@ var _ = Describe("walk_dir_tree", func() { ctx context.Context ) - BeforeEach(func() { - DeferCleanup(configtest.SetupConfig()) - ctx = GinkgoT().Context() - fsys = &mockMusicFS{ - FS: fstest.MapFS{ - "root/a/.ndignore": {Data: []byte("ignored/*")}, - "root/a/f1.mp3": {}, - "root/a/f2.mp3": {}, - "root/a/ignored/bad.mp3": {}, - "root/b/cover.jpg": {}, - "root/c/f3": {}, - "root/d": {}, - "root/d/.ndignore": {}, - "root/d/f1.mp3": {}, - "root/d/f2.mp3": {}, - "root/d/f3.mp3": {}, - "root/e/original/f1.mp3": {}, - "root/e/symlink": {Mode: fs.ModeSymlink, Data: []byte("original")}, + Context("full library", func() { + BeforeEach(func() { + DeferCleanup(configtest.SetupConfig()) + ctx = GinkgoT().Context() + fsys = &mockMusicFS{ + FS: fstest.MapFS{ + "root/a/.ndignore": {Data: []byte("ignored/*")}, + "root/a/f1.mp3": {}, + "root/a/f2.mp3": {}, + "root/a/ignored/bad.mp3": {}, + "root/b/cover.jpg": {}, + "root/c/f3": {}, + "root/d": {}, + "root/d/.ndignore": {}, + "root/d/f1.mp3": {}, + "root/d/f2.mp3": {}, + "root/d/f3.mp3": {}, + "root/e/original/f1.mp3": {}, + "root/e/symlink": {Mode: fs.ModeSymlink, Data: []byte("original")}, + }, + } + job = &scanJob{ + fs: fsys, + lib: model.Library{Path: "/music"}, + } + }) + + // Helper function to call walkDirTree and collect folders from the results channel + getFolders := func() map[string]*folderEntry { + results, err := walkDirTree(ctx, job) + Expect(err).ToNot(HaveOccurred()) + + folders := map[string]*folderEntry{} + g := errgroup.Group{} + g.Go(func() error { + for folder := range results { + folders[folder.path] = folder + } + return nil + }) + _ = g.Wait() + return folders + } + + DescribeTable("symlink handling", + func(followSymlinks bool, expectedFolderCount int) { + conf.Server.Scanner.FollowSymlinks = followSymlinks + folders := getFolders() + + Expect(folders).To(HaveLen(expectedFolderCount + 2)) // +2 for `.` and `root` + + // Basic folder structure checks + Expect(folders["root/a"].audioFiles).To(SatisfyAll( + HaveLen(2), + HaveKey("f1.mp3"), + HaveKey("f2.mp3"), + )) + Expect(folders["root/a"].imageFiles).To(BeEmpty()) + Expect(folders["root/b"].audioFiles).To(BeEmpty()) + Expect(folders["root/b"].imageFiles).To(SatisfyAll( + HaveLen(1), + HaveKey("cover.jpg"), + )) + Expect(folders["root/c"].audioFiles).To(BeEmpty()) + Expect(folders["root/c"].imageFiles).To(BeEmpty()) + Expect(folders).ToNot(HaveKey("root/d")) + + // Symlink specific checks + if followSymlinks { + Expect(folders["root/e/symlink"].audioFiles).To(HaveLen(1)) + } else { + Expect(folders).ToNot(HaveKey("root/e/symlink")) + } }, - } - job = &scanJob{ - fs: fsys, - lib: model.Library{Path: "/music"}, - } + Entry("with symlinks enabled", true, 7), + Entry("with symlinks disabled", false, 6), + ) }) - // Helper function to call walkDirTree and collect folders from the results channel - getFolders := func() map[string]*folderEntry { - results, err := walkDirTree(ctx, job) - Expect(err).ToNot(HaveOccurred()) - - folders := map[string]*folderEntry{} - g := errgroup.Group{} - g.Go(func() error { - for folder := range results { - folders[folder.path] = folder + Context("with target folders", func() { + BeforeEach(func() { + DeferCleanup(configtest.SetupConfig()) + ctx = GinkgoT().Context() + fsys = &mockMusicFS{ + FS: fstest.MapFS{ + "Artist/Album1/track1.mp3": {}, + "Artist/Album1/track2.mp3": {}, + "Artist/Album2/track1.mp3": {}, + "Artist/Album2/track2.mp3": {}, + "Artist/Album2/Sub/track3.mp3": {}, + "OtherArtist/Album3/track1.mp3": {}, + }, + } + job = &scanJob{ + fs: fsys, + lib: model.Library{Path: "/music"}, } - return nil }) - _ = g.Wait() - return folders - } - DescribeTable("symlink handling", - func(followSymlinks bool, expectedFolderCount int) { - conf.Server.Scanner.FollowSymlinks = followSymlinks - folders := getFolders() + It("should recursively walk all subdirectories of target folders", func() { + results, err := walkDirTree(ctx, job, "Artist") + Expect(err).ToNot(HaveOccurred()) - Expect(folders).To(HaveLen(expectedFolderCount + 2)) // +2 for `.` and `root` + folders := map[string]*folderEntry{} + g := errgroup.Group{} + g.Go(func() error { + for folder := range results { + folders[folder.path] = folder + } + return nil + }) + _ = g.Wait() - // Basic folder structure checks - Expect(folders["root/a"].audioFiles).To(SatisfyAll( - HaveLen(2), - HaveKey("f1.mp3"), - HaveKey("f2.mp3"), + // Should include the target folder and all its descendants + Expect(folders).To(SatisfyAll( + HaveKey("Artist"), + HaveKey("Artist/Album1"), + HaveKey("Artist/Album2"), + HaveKey("Artist/Album2/Sub"), )) - Expect(folders["root/a"].imageFiles).To(BeEmpty()) - Expect(folders["root/b"].audioFiles).To(BeEmpty()) - Expect(folders["root/b"].imageFiles).To(SatisfyAll( - HaveLen(1), - HaveKey("cover.jpg"), + + // Should not include folders outside the target + Expect(folders).ToNot(HaveKey("OtherArtist")) + Expect(folders).ToNot(HaveKey("OtherArtist/Album3")) + + // Verify audio files are present + Expect(folders["Artist/Album1"].audioFiles).To(HaveLen(2)) + Expect(folders["Artist/Album2"].audioFiles).To(HaveLen(2)) + Expect(folders["Artist/Album2/Sub"].audioFiles).To(HaveLen(1)) + }) + + It("should handle multiple target folders", func() { + results, err := walkDirTree(ctx, job, "Artist/Album1", "OtherArtist") + Expect(err).ToNot(HaveOccurred()) + + folders := map[string]*folderEntry{} + g := errgroup.Group{} + g.Go(func() error { + for folder := range results { + folders[folder.path] = folder + } + return nil + }) + _ = g.Wait() + + // Should include both target folders and their descendants + Expect(folders).To(SatisfyAll( + HaveKey("Artist/Album1"), + HaveKey("OtherArtist"), + HaveKey("OtherArtist/Album3"), )) - Expect(folders["root/c"].audioFiles).To(BeEmpty()) - Expect(folders["root/c"].imageFiles).To(BeEmpty()) - Expect(folders).ToNot(HaveKey("root/d")) - // Symlink specific checks - if followSymlinks { - Expect(folders["root/e/symlink"].audioFiles).To(HaveLen(1)) - } else { - Expect(folders).ToNot(HaveKey("root/e/symlink")) - } - }, - Entry("with symlinks enabled", true, 7), - Entry("with symlinks disabled", false, 6), - ) - }) - - Describe("loadSpecificFolders", func() { - var ( - fsys storage.MusicFS - job *scanJob - ctx context.Context - ) - - BeforeEach(func() { - DeferCleanup(configtest.SetupConfig()) - ctx = GinkgoT().Context() - fsys = &mockMusicFS{ - FS: fstest.MapFS{ - "Artist/Album1/track1.mp3": {}, - "Artist/Album1/track2.mp3": {}, - "Artist/Album2/track1.mp3": {}, - "Artist/Album2/track2.mp3": {}, - "Artist/Album2/Sub/track3.mp3": {}, - "OtherArtist/Album3/track1.mp3": {}, - }, - } - job = &scanJob{ - fs: fsys, - lib: model.Library{Path: "/music"}, - } - }) - - It("should recursively walk all subdirectories of target folders", func() { - results, err := loadSpecificFolders(ctx, job, []string{"Artist"}) - Expect(err).ToNot(HaveOccurred()) - - folders := map[string]*folderEntry{} - g := errgroup.Group{} - g.Go(func() error { - for folder := range results { - folders[folder.path] = folder - } - return nil + // Should not include other folders + Expect(folders).ToNot(HaveKey("Artist")) + Expect(folders).ToNot(HaveKey("Artist/Album2")) + Expect(folders).ToNot(HaveKey("Artist/Album2/Sub")) }) - _ = g.Wait() - - // Should include the target folder and all its descendants - Expect(folders).To(SatisfyAll( - HaveKey("Artist"), - HaveKey("Artist/Album1"), - HaveKey("Artist/Album2"), - HaveKey("Artist/Album2/Sub"), - )) - - // Should not include folders outside the target - Expect(folders).ToNot(HaveKey("OtherArtist")) - Expect(folders).ToNot(HaveKey("OtherArtist/Album3")) - - // Verify audio files are present - Expect(folders["Artist/Album1"].audioFiles).To(HaveLen(2)) - Expect(folders["Artist/Album2"].audioFiles).To(HaveLen(2)) - Expect(folders["Artist/Album2/Sub"].audioFiles).To(HaveLen(1)) - }) - - It("should handle multiple target folders", func() { - results, err := loadSpecificFolders(ctx, job, []string{"Artist/Album1", "OtherArtist"}) - Expect(err).ToNot(HaveOccurred()) - - folders := map[string]*folderEntry{} - g := errgroup.Group{} - g.Go(func() error { - for folder := range results { - folders[folder.path] = folder - } - return nil - }) - _ = g.Wait() - - // Should include both target folders and their descendants - Expect(folders).To(SatisfyAll( - HaveKey("Artist/Album1"), - HaveKey("OtherArtist"), - HaveKey("OtherArtist/Album3"), - )) - - // Should not include other folders - Expect(folders).ToNot(HaveKey("Artist")) - Expect(folders).ToNot(HaveKey("Artist/Album2")) - Expect(folders).ToNot(HaveKey("Artist/Album2/Sub")) }) }) From 2998cd3f7cbe02e646d84e9a57fe385ef1318cf5 Mon Sep 17 00:00:00 2001 From: Deluan Date: Tue, 11 Nov 2025 21:10:34 -0500 Subject: [PATCH 33/40] fix(scanner, watcher): handle errors when pushing ignore patterns for folders Signed-off-by: Deluan --- scanner/walk_dir_tree.go | 8 ++++++-- scanner/watcher.go | 5 ++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/scanner/walk_dir_tree.go b/scanner/walk_dir_tree.go index 8500824cf..59b73f56a 100644 --- a/scanner/walk_dir_tree.go +++ b/scanner/walk_dir_tree.go @@ -34,10 +34,14 @@ func walkDirTree(ctx context.Context, job *scanJob, targetFolders ...string) (<- // Create checker and push patterns from root to this folder checker := newIgnoreChecker(job.fs) - _ = checker.PushAllParents(ctx, folderPath) + err := checker.PushAllParents(ctx, folderPath) + if err != nil { + log.Error(ctx, "Scanner: Error pushing ignore patterns for target folder", "path", folderPath, err) + continue + } // Recursively walk this folder and all its children - err := walkFolder(ctx, job, folderPath, checker, results) + err = walkFolder(ctx, job, folderPath, checker, results) if err != nil { log.Error(ctx, "Scanner: Error walking target folder", "path", folderPath, err) continue diff --git a/scanner/watcher.go b/scanner/watcher.go index 3b47f8416..ad9a06421 100644 --- a/scanner/watcher.go +++ b/scanner/watcher.go @@ -308,7 +308,10 @@ func resolveFolderPath(fsys fs.FS, path string) string { // in the library. It pushes all parent folders onto the IgnoreChecker stack before checking. func (w *watcher) shouldIgnoreFolderPath(ctx context.Context, fsys storage.MusicFS, folderPath string) bool { checker := newIgnoreChecker(fsys) - _ = checker.PushAllParents(ctx, folderPath) + err := checker.PushAllParents(ctx, folderPath) + if err != nil { + log.Warn(ctx, "Watcher: Error pushing ignore patterns for folder", "path", folderPath, err) + } return checker.ShouldIgnore(ctx, folderPath) } From d30f6cfc9259a4f528941fd21036f385da54d699 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Deluan=20Quint=C3=A3o?= Date: Tue, 11 Nov 2025 22:49:05 -0500 Subject: [PATCH 34/40] Update scanner/phase_1_folders.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- scanner/phase_1_folders.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scanner/phase_1_folders.go b/scanner/phase_1_folders.go index 2dd5c6639..178176c1b 100644 --- a/scanner/phase_1_folders.go +++ b/scanner/phase_1_folders.go @@ -139,7 +139,7 @@ func (p *phaseFolders) producer() ppl.Producer[*folderEntry] { // Use selective folder loading if target folders are specified if len(job.targetFolders) > 0 { - log.Debug(p.ctx, "Scanner: Loading specific folders only (non-recursive)", "lib", job.lib.Name, "numTargets", len(job.targetFolders)) + log.Debug(p.ctx, "Scanner: Loading specific folders and all their subdirectories (recursive)", "lib", job.lib.Name, "numTargets", len(job.targetFolders)) } outputChan, err = walkDirTree(p.ctx, job, job.targetFolders...) From 5eb867b6399aeca1fd9791b52a3ba1c2e13d5054 Mon Sep 17 00:00:00 2001 From: Deluan Date: Tue, 11 Nov 2025 22:53:39 -0500 Subject: [PATCH 35/40] refactor(scanner): replace parseTargets function with direct call to scanner.ParseTargets Signed-off-by: Deluan --- cmd/scan.go | 8 +------- cmd/scan_test.go | 28 ---------------------------- 2 files changed, 1 insertion(+), 35 deletions(-) delete mode 100644 cmd/scan_test.go diff --git a/cmd/scan.go b/cmd/scan.go index 057d4902f..a2b510d56 100644 --- a/cmd/scan.go +++ b/cmd/scan.go @@ -76,7 +76,7 @@ func runScanner(ctx context.Context) { var scanTargets []model.ScanTarget if targets != "" { var err error - scanTargets, err = parseTargets(targets) + scanTargets, err = scanner.ParseTargets(strings.Split(targets, ",")) if err != nil { log.Fatal(ctx, "Failed to parse targets", err) } @@ -95,9 +95,3 @@ func runScanner(ctx context.Context) { trackScanInteractively(ctx, progress) } } - -// parseTargets parses the comma-separated targets string into ScanTarget structs -func parseTargets(targetsStr string) ([]model.ScanTarget, error) { - targets := strings.Split(targetsStr, ",") - return scanner.ParseTargets(targets) -} diff --git a/cmd/scan_test.go b/cmd/scan_test.go deleted file mode 100644 index 7abc8ef88..000000000 --- a/cmd/scan_test.go +++ /dev/null @@ -1,28 +0,0 @@ -package cmd - -import ( - "github.com/navidrome/navidrome/model" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -var _ = Describe("parseTargets", func() { - Context("Valid targets", func() { - It("parses multiple targets", func() { - targets, err := parseTargets("1:Music/Rock,2:Jazz,3:Classical/Beethoven") - Expect(err).ToNot(HaveOccurred()) - Expect(targets).To(HaveLen(3)) - Expect(targets[0]).To(Equal(model.ScanTarget{LibraryID: 1, FolderPath: "Music/Rock"})) - Expect(targets[1]).To(Equal(model.ScanTarget{LibraryID: 2, FolderPath: "Jazz"})) - Expect(targets[2]).To(Equal(model.ScanTarget{LibraryID: 3, FolderPath: "Classical/Beethoven"})) - }) - - It("returns error for empty string", func() { - _, err := parseTargets("") - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("no valid targets")) - }) - - // Other test cases are covered in scanner/controller_test.go - }) -}) From d8dfbc07325c479ef47c3af35f9fe63802604956 Mon Sep 17 00:00:00 2001 From: Deluan Date: Wed, 12 Nov 2025 11:24:38 -0500 Subject: [PATCH 36/40] test(scanner): add tests for ScanBegin and ScanEnd functionality Signed-off-by: Deluan --- persistence/library_repository_test.go | 58 ++++++++++++++++++++++++++ scanner/phase_1_folders.go | 1 - 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/persistence/library_repository_test.go b/persistence/library_repository_test.go index 6f4df1beb..3e3972bdb 100644 --- a/persistence/library_repository_test.go +++ b/persistence/library_repository_test.go @@ -142,4 +142,62 @@ var _ = Describe("LibraryRepository", func() { Expect(libAfter.TotalSize).To(Equal(sizeRes.Sum)) Expect(libAfter.TotalDuration).To(Equal(durationRes.Sum)) }) + + Describe("ScanBegin and ScanEnd", func() { + var lib *model.Library + + BeforeEach(func() { + lib = &model.Library{ + ID: 0, + Name: "Test Scan Library", + Path: "/music/test-scan", + } + err := repo.Put(lib) + Expect(err).ToNot(HaveOccurred()) + }) + + DescribeTable("ScanBegin", + func(fullScan bool, expectedFullScanInProgress bool) { + err := repo.ScanBegin(lib.ID, fullScan) + Expect(err).ToNot(HaveOccurred()) + + updatedLib, err := repo.Get(lib.ID) + Expect(err).ToNot(HaveOccurred()) + Expect(updatedLib.LastScanStartedAt).ToNot(BeZero()) + Expect(updatedLib.FullScanInProgress).To(Equal(expectedFullScanInProgress)) + }, + Entry("sets FullScanInProgress to true for full scan", true, true), + Entry("sets FullScanInProgress to false for quick scan", false, false), + ) + + Context("ScanEnd", func() { + BeforeEach(func() { + err := repo.ScanBegin(lib.ID, true) + Expect(err).ToNot(HaveOccurred()) + }) + + It("sets LastScanAt and clears FullScanInProgress and LastScanStartedAt", func() { + err := repo.ScanEnd(lib.ID) + Expect(err).ToNot(HaveOccurred()) + + updatedLib, err := repo.Get(lib.ID) + Expect(err).ToNot(HaveOccurred()) + Expect(updatedLib.LastScanAt).ToNot(BeZero()) + Expect(updatedLib.FullScanInProgress).To(BeFalse()) + Expect(updatedLib.LastScanStartedAt).To(BeZero()) + }) + + It("sets LastScanAt to be after LastScanStartedAt", func() { + libBefore, err := repo.Get(lib.ID) + Expect(err).ToNot(HaveOccurred()) + + err = repo.ScanEnd(lib.ID) + Expect(err).ToNot(HaveOccurred()) + + libAfter, err := repo.Get(lib.ID) + Expect(err).ToNot(HaveOccurred()) + Expect(libAfter.LastScanAt).To(BeTemporally(">=", libBefore.LastScanStartedAt)) + }) + }) + }) }) diff --git a/scanner/phase_1_folders.go b/scanner/phase_1_folders.go index 178176c1b..4a687b310 100644 --- a/scanner/phase_1_folders.go +++ b/scanner/phase_1_folders.go @@ -76,7 +76,6 @@ func newScanJob(ctx context.Context, ds model.DataStore, cw artwork.CacheWarmer, log.Error(ctx, "Error getting fs for library", "library", lib.Name, "path", lib.Path, err) return nil, fmt.Errorf("getting fs for library: %w", err) } - lib.FullScanInProgress = lib.FullScanInProgress || fullScan return &scanJob{ lib: lib, fs: fsys, From e898deaa9f498dffddd5aeb33e69b777cfe3a57c Mon Sep 17 00:00:00 2001 From: Deluan Date: Wed, 12 Nov 2025 12:04:07 -0500 Subject: [PATCH 37/40] fix(library): update PRAGMA optimize to check table sizes without ANALYZE Signed-off-by: Deluan --- persistence/library_repository.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/persistence/library_repository.go b/persistence/library_repository.go index 314b682bb..5621e1719 100644 --- a/persistence/library_repository.go +++ b/persistence/library_repository.go @@ -177,7 +177,9 @@ func (r *libraryRepository) ScanEnd(id int) error { return err } // https://www.sqlite.org/pragma.html#pragma_optimize - _, err = r.executeSQL(Expr("PRAGMA optimize=0x10012;")) + // Use mask 0x10000 to check table sizes without running ANALYZE + // Running ANALYZE can cause query planner issues with expression-based collation indexes + _, err = r.executeSQL(Expr("PRAGMA optimize=0x10000;")) return err } From 62a8d85772391a776cd75ca35e0148b774b4c515 Mon Sep 17 00:00:00 2001 From: Deluan Date: Wed, 12 Nov 2025 14:35:14 -0500 Subject: [PATCH 38/40] test(scanner): refactor tests Signed-off-by: Deluan --- scanner/scanner_selective_test.go | 348 +++++++++++++++++------------- scanner/scanner_test.go | 67 +----- 2 files changed, 209 insertions(+), 206 deletions(-) diff --git a/scanner/scanner_selective_test.go b/scanner/scanner_selective_test.go index 13d0f02cb..bf289b203 100644 --- a/scanner/scanner_selective_test.go +++ b/scanner/scanner_selective_test.go @@ -20,11 +20,12 @@ import ( "github.com/navidrome/navidrome/scanner" "github.com/navidrome/navidrome/server/events" "github.com/navidrome/navidrome/tests" + "github.com/navidrome/navidrome/utils/slice" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) -var _ = Describe("Selective Scan - Deleted Child Folders", Ordered, func() { +var _ = Describe("ScanFolders", Ordered, func() { var ctx context.Context var lib model.Library var ds model.DataStore @@ -72,169 +73,224 @@ var _ = Describe("Selective Scan - Deleted Child Folders", Ordered, func() { storagetest.Register("fake", &fsys) }) - Context("when a child folder is deleted", func() { - var ( - revolver, help func(...map[string]any) *fstest.MapFile - artistFolderID string - album1FolderID string - album2FolderID string - album1TrackIDs []string - album2TrackIDs []string - ) - - BeforeEach(func() { - // Setup template functions for creating test files - revolver = storagetest.Template(_t{"albumartist": "The Beatles", "album": "Revolver", "year": 1966}) - help = storagetest.Template(_t{"albumartist": "The Beatles", "album": "Help!", "year": 1965}) - - // Initial filesystem with nested folders - fsys.SetFiles(fstest.MapFS{ - "The Beatles/Revolver/01 - Taxman.mp3": revolver(storagetest.Track(1, "Taxman")), - "The Beatles/Revolver/02 - Eleanor Rigby.mp3": revolver(storagetest.Track(2, "Eleanor Rigby")), - "The Beatles/Help!/01 - Help!.mp3": help(storagetest.Track(1, "Help!")), - "The Beatles/Help!/02 - The Night Before.mp3": help(storagetest.Track(2, "The Night Before")), + Describe("Adding tracks to the library", func() { + It("scans specified folders recursively including all subdirectories", func() { + rock := template(_t{"albumartist": "Rock Artist", "album": "Rock Album"}) + jazz := template(_t{"albumartist": "Jazz Artist", "album": "Jazz Album"}) + pop := template(_t{"albumartist": "Pop Artist", "album": "Pop Album"}) + createFS(fstest.MapFS{ + "rock/track1.mp3": rock(track(1, "Rock Track 1")), + "rock/track2.mp3": rock(track(2, "Rock Track 2")), + "rock/subdir/track3.mp3": rock(track(3, "Rock Track 3")), + "jazz/track4.mp3": jazz(track(1, "Jazz Track 1")), + "jazz/subdir/track5.mp3": jazz(track(2, "Jazz Track 2")), + "pop/track6.mp3": pop(track(1, "Pop Track 1")), }) - // First scan - import everything - _, err := s.ScanAll(ctx, true) - Expect(err).ToNot(HaveOccurred()) + // Use the existing library from BeforeEach + // (lib is already created with the path "fake:///music") - // Verify initial state - all folders exist - folders, err := ds.Folder(ctx).GetAll(model.QueryOptions{Filters: squirrel.Eq{"library_id": lib.ID}}) - Expect(err).ToNot(HaveOccurred()) - Expect(folders).To(HaveLen(4)) // root, Artist, Album1, Album2 - - // Store folder IDs for later verification - for _, f := range folders { - switch f.Name { - case "The Beatles": - artistFolderID = f.ID - case "Revolver": - album1FolderID = f.ID - case "Help!": - album2FolderID = f.ID - } + // Scan only the "rock" and "jazz" folders (including their subdirectories) + targets := []model.ScanTarget{ + {LibraryID: lib.ID, FolderPath: "rock"}, + {LibraryID: lib.ID, FolderPath: "jazz"}, } - // Verify all tracks exist - allTracks, err := ds.MediaFile(ctx).GetAll() + warnings, err := s.ScanFolders(ctx, false, targets) Expect(err).ToNot(HaveOccurred()) - Expect(allTracks).To(HaveLen(4)) + Expect(warnings).To(BeEmpty()) - // Store track IDs for later verification - for _, t := range allTracks { - if t.Album == "Revolver" { - album1TrackIDs = append(album1TrackIDs, t.ID) - } else if t.Album == "Help!" { - album2TrackIDs = append(album2TrackIDs, t.ID) - } - } + // Verify all tracks in rock and jazz folders (including subdirectories) were imported + allFiles, err := ds.MediaFile(ctx).GetAll() + Expect(err).ToNot(HaveOccurred()) - // Verify no tracks are missing initially - for _, t := range allTracks { - Expect(t.Missing).To(BeFalse()) - } + // Should have 5 tracks (all rock and jazz tracks including subdirectories) + Expect(allFiles).To(HaveLen(5)) + + // Get the file paths + paths := slice.Map(allFiles, func(mf model.MediaFile) string { + return filepath.ToSlash(mf.Path) + }) + + // Verify the correct files were scanned (including subdirectories) + Expect(paths).To(ContainElements( + "rock/track1.mp3", + "rock/track2.mp3", + "rock/subdir/track3.mp3", + "jazz/track4.mp3", + "jazz/subdir/track5.mp3", + )) + + // Verify files in the pop folder were NOT scanned + Expect(paths).ToNot(ContainElement("pop/track6.mp3")) }) + }) - It("should mark child folder and its tracks as missing when parent is scanned", func() { - // Delete the child folder (Help!) from the filesystem - fsys.SetFiles(fstest.MapFS{ - "The Beatles/Revolver/01 - Taxman.mp3": revolver(storagetest.Track(1, "Taxman")), - "The Beatles/Revolver/02 - Eleanor Rigby.mp3": revolver(storagetest.Track(2, "Eleanor Rigby")), - // "The Beatles/Help!" folder and its contents are DELETED - }) + Describe("Deleting folders", func() { + Context("when a child folder is deleted", func() { + var ( + revolver, help func(...map[string]any) *fstest.MapFile + artistFolderID string + album1FolderID string + album2FolderID string + album1TrackIDs []string + album2TrackIDs []string + ) - // Run selective scan on the parent folder (Artist) - // This simulates what the watcher does when a child folder is deleted - _, err := s.ScanFolders(ctx, false, []model.ScanTarget{ - {LibraryID: lib.ID, FolderPath: "The Beatles"}, - }) - Expect(err).ToNot(HaveOccurred()) + BeforeEach(func() { + // Setup template functions for creating test files + revolver = storagetest.Template(_t{"albumartist": "The Beatles", "album": "Revolver", "year": 1966}) + help = storagetest.Template(_t{"albumartist": "The Beatles", "album": "Help!", "year": 1965}) - // Verify the deleted child folder is now marked as missing - deletedFolder, err := ds.Folder(ctx).Get(album2FolderID) - Expect(err).ToNot(HaveOccurred()) - Expect(deletedFolder.Missing).To(BeTrue(), "Deleted child folder should be marked as missing") + // Initial filesystem with nested folders + fsys.SetFiles(fstest.MapFS{ + "The Beatles/Revolver/01 - Taxman.mp3": revolver(storagetest.Track(1, "Taxman")), + "The Beatles/Revolver/02 - Eleanor Rigby.mp3": revolver(storagetest.Track(2, "Eleanor Rigby")), + "The Beatles/Help!/01 - Help!.mp3": help(storagetest.Track(1, "Help!")), + "The Beatles/Help!/02 - The Night Before.mp3": help(storagetest.Track(2, "The Night Before")), + }) - // Verify the deleted folder's tracks are marked as missing - for _, trackID := range album2TrackIDs { - track, err := ds.MediaFile(ctx).Get(trackID) + // First scan - import everything + _, err := s.ScanAll(ctx, true) Expect(err).ToNot(HaveOccurred()) - Expect(track.Missing).To(BeTrue(), "Track in deleted folder should be marked as missing") - } - // Verify the parent folder is still present and not marked as missing - parentFolder, err := ds.Folder(ctx).Get(artistFolderID) - Expect(err).ToNot(HaveOccurred()) - Expect(parentFolder.Missing).To(BeFalse(), "Parent folder should not be marked as missing") - - // Verify the sibling folder and its tracks are still present and not missing - siblingFolder, err := ds.Folder(ctx).Get(album1FolderID) - Expect(err).ToNot(HaveOccurred()) - Expect(siblingFolder.Missing).To(BeFalse(), "Sibling folder should not be marked as missing") - - for _, trackID := range album1TrackIDs { - track, err := ds.MediaFile(ctx).Get(trackID) + // Verify initial state - all folders exist + folders, err := ds.Folder(ctx).GetAll(model.QueryOptions{Filters: squirrel.Eq{"library_id": lib.ID}}) Expect(err).ToNot(HaveOccurred()) - Expect(track.Missing).To(BeFalse(), "Track in sibling folder should not be marked as missing") - } - }) + Expect(folders).To(HaveLen(4)) // root, Artist, Album1, Album2 - It("should mark deeply nested child folders as missing", func() { - // Add a deeply nested folder structure - fsys.SetFiles(fstest.MapFS{ - "The Beatles/Revolver/01 - Taxman.mp3": revolver(storagetest.Track(1, "Taxman")), - "The Beatles/Revolver/02 - Eleanor Rigby.mp3": revolver(storagetest.Track(2, "Eleanor Rigby")), - "The Beatles/Help!/01 - Help!.mp3": help(storagetest.Track(1, "Help!")), - "The Beatles/Help!/02 - The Night Before.mp3": help(storagetest.Track(2, "The Night Before")), - "The Beatles/Help!/Bonus/01 - Bonus Track.mp3": help(storagetest.Track(99, "Bonus Track")), - "The Beatles/Help!/Bonus/Nested/01 - Deep Track.mp3": help(storagetest.Track(100, "Deep Track")), - }) - - // Rescan to import the new nested structure - _, err := s.ScanAll(ctx, true) - Expect(err).ToNot(HaveOccurred()) - - // Verify nested folders were created - allFolders, err := ds.Folder(ctx).GetAll(model.QueryOptions{Filters: squirrel.Eq{"library_id": lib.ID}}) - Expect(err).ToNot(HaveOccurred()) - Expect(len(allFolders)).To(BeNumerically(">", 4), "Should have more folders with nested structure") - - // Now delete the entire Help! folder including nested children - fsys.SetFiles(fstest.MapFS{ - "The Beatles/Revolver/01 - Taxman.mp3": revolver(storagetest.Track(1, "Taxman")), - "The Beatles/Revolver/02 - Eleanor Rigby.mp3": revolver(storagetest.Track(2, "Eleanor Rigby")), - // All Help! subfolders are deleted - }) - - // Run selective scan on parent - _, err = s.ScanFolders(ctx, false, []model.ScanTarget{ - {LibraryID: lib.ID, FolderPath: "The Beatles"}, - }) - Expect(err).ToNot(HaveOccurred()) - - // Verify all Help! folders (including nested ones) are marked as missing - missingFolders, err := ds.Folder(ctx).GetAll(model.QueryOptions{ - Filters: squirrel.And{ - squirrel.Eq{"library_id": lib.ID}, - squirrel.Eq{"missing": true}, - }, - }) - Expect(err).ToNot(HaveOccurred()) - Expect(len(missingFolders)).To(BeNumerically(">", 0), "At least one folder should be marked as missing") - - // Verify all tracks in deleted folders are marked as missing - allTracks, err := ds.MediaFile(ctx).GetAll() - Expect(err).ToNot(HaveOccurred()) - Expect(allTracks).To(HaveLen(6)) - - for _, track := range allTracks { - if track.Album == "Help!" { - Expect(track.Missing).To(BeTrue(), "All tracks in deleted Help! folder should be marked as missing") - } else if track.Album == "Revolver" { - Expect(track.Missing).To(BeFalse(), "Tracks in Revolver folder should not be marked as missing") + // Store folder IDs for later verification + for _, f := range folders { + switch f.Name { + case "The Beatles": + artistFolderID = f.ID + case "Revolver": + album1FolderID = f.ID + case "Help!": + album2FolderID = f.ID + } } - } + + // Verify all tracks exist + allTracks, err := ds.MediaFile(ctx).GetAll() + Expect(err).ToNot(HaveOccurred()) + Expect(allTracks).To(HaveLen(4)) + + // Store track IDs for later verification + for _, t := range allTracks { + if t.Album == "Revolver" { + album1TrackIDs = append(album1TrackIDs, t.ID) + } else if t.Album == "Help!" { + album2TrackIDs = append(album2TrackIDs, t.ID) + } + } + + // Verify no tracks are missing initially + for _, t := range allTracks { + Expect(t.Missing).To(BeFalse()) + } + }) + + It("should mark child folder and its tracks as missing when parent is scanned", func() { + // Delete the child folder (Help!) from the filesystem + fsys.SetFiles(fstest.MapFS{ + "The Beatles/Revolver/01 - Taxman.mp3": revolver(storagetest.Track(1, "Taxman")), + "The Beatles/Revolver/02 - Eleanor Rigby.mp3": revolver(storagetest.Track(2, "Eleanor Rigby")), + // "The Beatles/Help!" folder and its contents are DELETED + }) + + // Run selective scan on the parent folder (Artist) + // This simulates what the watcher does when a child folder is deleted + _, err := s.ScanFolders(ctx, false, []model.ScanTarget{ + {LibraryID: lib.ID, FolderPath: "The Beatles"}, + }) + Expect(err).ToNot(HaveOccurred()) + + // Verify the deleted child folder is now marked as missing + deletedFolder, err := ds.Folder(ctx).Get(album2FolderID) + Expect(err).ToNot(HaveOccurred()) + Expect(deletedFolder.Missing).To(BeTrue(), "Deleted child folder should be marked as missing") + + // Verify the deleted folder's tracks are marked as missing + for _, trackID := range album2TrackIDs { + track, err := ds.MediaFile(ctx).Get(trackID) + Expect(err).ToNot(HaveOccurred()) + Expect(track.Missing).To(BeTrue(), "Track in deleted folder should be marked as missing") + } + + // Verify the parent folder is still present and not marked as missing + parentFolder, err := ds.Folder(ctx).Get(artistFolderID) + Expect(err).ToNot(HaveOccurred()) + Expect(parentFolder.Missing).To(BeFalse(), "Parent folder should not be marked as missing") + + // Verify the sibling folder and its tracks are still present and not missing + siblingFolder, err := ds.Folder(ctx).Get(album1FolderID) + Expect(err).ToNot(HaveOccurred()) + Expect(siblingFolder.Missing).To(BeFalse(), "Sibling folder should not be marked as missing") + + for _, trackID := range album1TrackIDs { + track, err := ds.MediaFile(ctx).Get(trackID) + Expect(err).ToNot(HaveOccurred()) + Expect(track.Missing).To(BeFalse(), "Track in sibling folder should not be marked as missing") + } + }) + + It("should mark deeply nested child folders as missing", func() { + // Add a deeply nested folder structure + fsys.SetFiles(fstest.MapFS{ + "The Beatles/Revolver/01 - Taxman.mp3": revolver(storagetest.Track(1, "Taxman")), + "The Beatles/Revolver/02 - Eleanor Rigby.mp3": revolver(storagetest.Track(2, "Eleanor Rigby")), + "The Beatles/Help!/01 - Help!.mp3": help(storagetest.Track(1, "Help!")), + "The Beatles/Help!/02 - The Night Before.mp3": help(storagetest.Track(2, "The Night Before")), + "The Beatles/Help!/Bonus/01 - Bonus Track.mp3": help(storagetest.Track(99, "Bonus Track")), + "The Beatles/Help!/Bonus/Nested/01 - Deep Track.mp3": help(storagetest.Track(100, "Deep Track")), + }) + + // Rescan to import the new nested structure + _, err := s.ScanAll(ctx, true) + Expect(err).ToNot(HaveOccurred()) + + // Verify nested folders were created + allFolders, err := ds.Folder(ctx).GetAll(model.QueryOptions{Filters: squirrel.Eq{"library_id": lib.ID}}) + Expect(err).ToNot(HaveOccurred()) + Expect(len(allFolders)).To(BeNumerically(">", 4), "Should have more folders with nested structure") + + // Now delete the entire Help! folder including nested children + fsys.SetFiles(fstest.MapFS{ + "The Beatles/Revolver/01 - Taxman.mp3": revolver(storagetest.Track(1, "Taxman")), + "The Beatles/Revolver/02 - Eleanor Rigby.mp3": revolver(storagetest.Track(2, "Eleanor Rigby")), + // All Help! subfolders are deleted + }) + + // Run selective scan on parent + _, err = s.ScanFolders(ctx, false, []model.ScanTarget{ + {LibraryID: lib.ID, FolderPath: "The Beatles"}, + }) + Expect(err).ToNot(HaveOccurred()) + + // Verify all Help! folders (including nested ones) are marked as missing + missingFolders, err := ds.Folder(ctx).GetAll(model.QueryOptions{ + Filters: squirrel.And{ + squirrel.Eq{"library_id": lib.ID}, + squirrel.Eq{"missing": true}, + }, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(len(missingFolders)).To(BeNumerically(">", 0), "At least one folder should be marked as missing") + + // Verify all tracks in deleted folders are marked as missing + allTracks, err := ds.MediaFile(ctx).GetAll() + Expect(err).ToNot(HaveOccurred()) + Expect(allTracks).To(HaveLen(6)) + + for _, track := range allTracks { + if track.Album == "Help!" { + Expect(track.Missing).To(BeTrue(), "All tracks in deleted Help! folder should be marked as missing") + } else if track.Album == "Revolver" { + Expect(track.Missing).To(BeFalse(), "Tracks in Revolver folder should not be marked as missing") + } + } + }) }) }) }) diff --git a/scanner/scanner_test.go b/scanner/scanner_test.go index 604561058..4dabcf893 100644 --- a/scanner/scanner_test.go +++ b/scanner/scanner_test.go @@ -34,6 +34,13 @@ type _t = map[string]any var template = storagetest.Template var track = storagetest.Track +func createFS(files fstest.MapFS) storagetest.FakeFS { + fs := storagetest.FakeFS{} + fs.SetFiles(files) + storagetest.Register("fake", &fs) + return fs +} + var _ = Describe("Scanner", Ordered, func() { var ctx context.Context var lib model.Library @@ -41,13 +48,6 @@ var _ = Describe("Scanner", Ordered, func() { var mfRepo *mockMediaFileRepo var s model.Scanner - createFS := func(files fstest.MapFS) storagetest.FakeFS { - fs := storagetest.FakeFS{} - fs.SetFiles(files) - storagetest.Register("fake", &fs) - return fs - } - BeforeAll(func() { ctx = request.WithUser(GinkgoT().Context(), model.User{ID: "123", IsAdmin: true}) tmpDir := GinkgoT().TempDir() @@ -717,59 +717,6 @@ var _ = Describe("Scanner", Ordered, func() { Expect(albumArtistStats.SongCount).To(Equal(3)) // 3 songs }) }) - - Describe("ScanFolders", func() { - It("scans specified folders recursively including all subdirectories", func() { - rock := template(_t{"albumartist": "Rock Artist", "album": "Rock Album"}) - jazz := template(_t{"albumartist": "Jazz Artist", "album": "Jazz Album"}) - pop := template(_t{"albumartist": "Pop Artist", "album": "Pop Album"}) - createFS(fstest.MapFS{ - "rock/track1.mp3": rock(track(1, "Rock Track 1")), - "rock/track2.mp3": rock(track(2, "Rock Track 2")), - "rock/subdir/track3.mp3": rock(track(3, "Rock Track 3")), - "jazz/track4.mp3": jazz(track(1, "Jazz Track 1")), - "jazz/subdir/track5.mp3": jazz(track(2, "Jazz Track 2")), - "pop/track6.mp3": pop(track(1, "Pop Track 1")), - }) - - // Use the existing library from BeforeEach - // (lib is already created with the path "fake:///music") - - // Scan only the "rock" and "jazz" folders (including their subdirectories) - targets := []model.ScanTarget{ - {LibraryID: lib.ID, FolderPath: "rock"}, - {LibraryID: lib.ID, FolderPath: "jazz"}, - } - - warnings, err := s.ScanFolders(ctx, false, targets) - Expect(err).ToNot(HaveOccurred()) - Expect(warnings).To(BeEmpty()) - - // Verify all tracks in rock and jazz folders (including subdirectories) were imported - allFiles, err := ds.MediaFile(ctx).GetAll() - Expect(err).ToNot(HaveOccurred()) - - // Should have 5 tracks (all rock and jazz tracks including subdirectories) - Expect(allFiles).To(HaveLen(5)) - - // Get the file paths - paths := slice.Map(allFiles, func(mf model.MediaFile) string { - return filepath.ToSlash(mf.Path) - }) - - // Verify the correct files were scanned (including subdirectories) - Expect(paths).To(ContainElements( - "rock/track1.mp3", - "rock/track2.mp3", - "rock/subdir/track3.mp3", - "jazz/track4.mp3", - "jazz/subdir/track5.mp3", - )) - - // Verify files in the pop folder were NOT scanned - Expect(paths).ToNot(ContainElement("pop/track6.mp3")) - }) - }) }) func createFindByPath(ctx context.Context, ds model.DataStore) func(string) (*model.MediaFile, error) { From acae163b0f0139884d63639684b648f3c6c8337b Mon Sep 17 00:00:00 2001 From: Deluan Date: Wed, 12 Nov 2025 15:28:53 -0500 Subject: [PATCH 39/40] feat(ui): add selective scan options and update translations Signed-off-by: Deluan --- resources/i18n/pt-br.json | 7 ++++--- ui/src/i18n/en.json | 7 ++++--- ui/src/layout/ActivityPanel.jsx | 3 +++ 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/resources/i18n/pt-br.json b/resources/i18n/pt-br.json index 9c22d509f..bde22e8bd 100644 --- a/resources/i18n/pt-br.json +++ b/resources/i18n/pt-br.json @@ -598,11 +598,12 @@ "activity": { "title": "Atividade", "totalScanned": "Total de pastas scaneadas", - "quickScan": "Scan rápido", - "fullScan": "Scan completo", + "quickScan": "Rápido", + "fullScan": "Completo", + "selectiveScan": "Seletivo", "serverUptime": "Uptime do servidor", "serverDown": "DESCONECTADO", - "scanType": "Tipo", + "scanType": "Último Scan", "status": "Erro", "elapsedTime": "Duração" }, diff --git a/ui/src/i18n/en.json b/ui/src/i18n/en.json index 4a9039a67..c28ea8cff 100644 --- a/ui/src/i18n/en.json +++ b/ui/src/i18n/en.json @@ -600,11 +600,12 @@ "activity": { "title": "Activity", "totalScanned": "Total Folders Scanned", - "quickScan": "Quick Scan", - "fullScan": "Full Scan", + "quickScan": "Quick", + "fullScan": "Full", + "selectiveScan": "Selective", "serverUptime": "Server Uptime", "serverDown": "OFFLINE", - "scanType": "Type", + "scanType": "Last Scan", "status": "Scan Error", "elapsedTime": "Elapsed Time" }, diff --git a/ui/src/layout/ActivityPanel.jsx b/ui/src/layout/ActivityPanel.jsx index 18af8dc93..6d5d32d31 100644 --- a/ui/src/layout/ActivityPanel.jsx +++ b/ui/src/layout/ActivityPanel.jsx @@ -113,6 +113,9 @@ const ActivityPanel = () => { return translate('activity.fullScan') case 'quick': return translate('activity.quickScan') + case 'full-selective': + case 'quick-selective': + return translate('activity.selectiveScan') default: return '' } From 8cebf82590a80666307240acfcf5bf1674bf1a9c Mon Sep 17 00:00:00 2001 From: Deluan Date: Wed, 12 Nov 2025 16:09:35 -0500 Subject: [PATCH 40/40] feat(ui): add quick and full scan options for individual libraries Signed-off-by: Deluan --- resources/i18n/pt-br.json | 5 ++ ui/src/i18n/en.json | 5 ++ ui/src/library/LibraryList.jsx | 3 +- ui/src/library/LibraryListBulkActions.jsx | 11 ++++ ui/src/library/LibraryScanButton.jsx | 77 +++++++++++++++++++++++ ui/src/subsonic/index.js | 8 ++- 6 files changed, 107 insertions(+), 2 deletions(-) create mode 100644 ui/src/library/LibraryListBulkActions.jsx create mode 100644 ui/src/library/LibraryScanButton.jsx diff --git a/resources/i18n/pt-br.json b/resources/i18n/pt-br.json index bde22e8bd..3f095b025 100644 --- a/resources/i18n/pt-br.json +++ b/resources/i18n/pt-br.json @@ -300,6 +300,8 @@ }, "actions": { "scan": "Scanear Biblioteca", + "quickScan": "Scan Rápido", + "fullScan": "Scan Completo", "manageUsers": "Gerenciar Acesso do Usuário", "viewDetails": "Ver Detalhes" }, @@ -308,6 +310,9 @@ "updated": "Biblioteca atualizada com sucesso", "deleted": "Biblioteca excluída com sucesso", "scanStarted": "Scan da biblioteca iniciada", + "quickScanStarted": "Scan rápido iniciado", + "fullScanStarted": "Scan completo iniciado", + "scanError": "Erro ao iniciar o scan. Verifique os logs", "scanCompleted": "Scan da biblioteca concluída" }, "validation": { diff --git a/ui/src/i18n/en.json b/ui/src/i18n/en.json index c28ea8cff..9ef65d668 100644 --- a/ui/src/i18n/en.json +++ b/ui/src/i18n/en.json @@ -302,6 +302,8 @@ }, "actions": { "scan": "Scan Library", + "quickScan": "Quick Scan", + "fullScan": "Full Scan", "manageUsers": "Manage User Access", "viewDetails": "View Details" }, @@ -310,6 +312,9 @@ "updated": "Library updated successfully", "deleted": "Library deleted successfully", "scanStarted": "Library scan started", + "quickScanStarted": "Quick scan started", + "fullScanStarted": "Full scan started", + "scanError": "Error starting scan. Check logs", "scanCompleted": "Library scan completed" }, "validation": { diff --git a/ui/src/library/LibraryList.jsx b/ui/src/library/LibraryList.jsx index 932732b10..91ed97dfc 100644 --- a/ui/src/library/LibraryList.jsx +++ b/ui/src/library/LibraryList.jsx @@ -10,6 +10,7 @@ import { } from 'react-admin' import { useMediaQuery } from '@material-ui/core' import { List, DateField, useResourceRefresh, SizeField } from '../common' +import LibraryListBulkActions from './LibraryListBulkActions' const LibraryFilter = (props) => ( @@ -26,7 +27,7 @@ const LibraryList = (props) => { {...props} sort={{ field: 'name', order: 'ASC' }} exporter={false} - bulkActionButtons={false} + bulkActionButtons={!isXsmall && } filters={} > {isXsmall ? ( diff --git a/ui/src/library/LibraryListBulkActions.jsx b/ui/src/library/LibraryListBulkActions.jsx new file mode 100644 index 000000000..8862a4f51 --- /dev/null +++ b/ui/src/library/LibraryListBulkActions.jsx @@ -0,0 +1,11 @@ +import React from 'react' +import LibraryScanButton from './LibraryScanButton' + +const LibraryListBulkActions = (props) => ( + <> + + + +) + +export default LibraryListBulkActions diff --git a/ui/src/library/LibraryScanButton.jsx b/ui/src/library/LibraryScanButton.jsx new file mode 100644 index 000000000..cf4e8dfc5 --- /dev/null +++ b/ui/src/library/LibraryScanButton.jsx @@ -0,0 +1,77 @@ +import React, { useState } from 'react' +import PropTypes from 'prop-types' +import { + Button, + useNotify, + useRefresh, + useTranslate, + useUnselectAll, +} from 'react-admin' +import { useSelector } from 'react-redux' +import SyncIcon from '@material-ui/icons/Sync' +import CachedIcon from '@material-ui/icons/Cached' +import subsonic from '../subsonic' + +const LibraryScanButton = ({ fullScan, selectedIds, className }) => { + const [loading, setLoading] = useState(false) + const notify = useNotify() + const refresh = useRefresh() + const translate = useTranslate() + const unselectAll = useUnselectAll() + const scanStatus = useSelector((state) => state.activity.scanStatus) + + const handleClick = async () => { + setLoading(true) + try { + // Build scan options + const options = { fullScan } + + // If specific libraries are selected, scan only those + // Format: "libraryID:" to scan entire library (no folder path specified) + if (selectedIds && selectedIds.length > 0) { + options.path = selectedIds.map((id) => `${id}:`) + } + + await subsonic.startScan(options) + const notificationKey = fullScan + ? 'resources.library.notifications.fullScanStarted' + : 'resources.library.notifications.quickScanStarted' + notify(notificationKey, 'info') + refresh() + + // Unselect all items after successful scan + unselectAll('library') + } catch (error) { + notify('resources.library.notifications.scanError', 'warning') + } finally { + setLoading(false) + } + } + + const isDisabled = loading || scanStatus.scanning + + const label = fullScan + ? translate('resources.library.actions.fullScan') + : translate('resources.library.actions.quickScan') + + const icon = fullScan ? : + + return ( + + ) +} + +LibraryScanButton.propTypes = { + fullScan: PropTypes.bool.isRequired, + selectedIds: PropTypes.array, + className: PropTypes.string, +} + +export default LibraryScanButton diff --git a/ui/src/subsonic/index.js b/ui/src/subsonic/index.js index ad7a391e0..cfcc01043 100644 --- a/ui/src/subsonic/index.js +++ b/ui/src/subsonic/index.js @@ -23,7 +23,13 @@ const url = (command, id, options) => { delete options.ts } Object.keys(options).forEach((k) => { - params.append(k, options[k]) + const value = options[k] + // Handle array parameters by appending each value separately + if (Array.isArray(value)) { + value.forEach((v) => params.append(k, v)) + } else { + params.append(k, value) + } }) } return `/rest/${command}?${params.toString()}`