diff --git a/cmd/scan.go b/cmd/scan.go index d37ccd69f..06d8c6c25 100644 --- a/cmd/scan.go +++ b/cmd/scan.go @@ -3,7 +3,10 @@ package cmd import ( "context" "encoding/gob" + "fmt" "os" + "strconv" + "strings" "github.com/navidrome/navidrome/core" "github.com/navidrome/navidrome/db" @@ -17,11 +20,13 @@ import ( var ( fullScan bool subprocess bool + targets string ) func init() { scanCmd.Flags().BoolVarP(&fullScan, "full", "f", false, "check all subfolders, ignoring timestamps") scanCmd.Flags().BoolVarP(&subprocess, "subprocess", "", false, "run as subprocess (internal use)") + scanCmd.Flags().StringVarP(&targets, "targets", "t", "", "comma-separated list of libraryID:folderPath pairs (e.g., \"1:Music/Rock,1:Music/Jazz,2:Classical\")") rootCmd.AddCommand(scanCmd) } @@ -68,7 +73,18 @@ func runScanner(ctx context.Context) { ds := persistence.New(sqlDB) pls := core.NewPlaylists(ds) - progress, err := scanner.CallScan(ctx, ds, pls, fullScan) + // Parse targets if provided + var scanTargets []scanner.ScanTarget + if targets != "" { + var err error + scanTargets, err = parseTargets(targets) + if err != nil { + log.Fatal(ctx, "Failed to parse targets", err) + } + log.Info(ctx, "Scanning specific folders", "numTargets", len(scanTargets)) + } + + progress, err := scanner.CallScanFolders(ctx, ds, pls, fullScan, scanTargets) if err != nil { log.Fatal(ctx, "Failed to scan", err) } @@ -80,3 +96,43 @@ func runScanner(ctx context.Context) { trackScanInteractively(ctx, progress) } } + +// parseTargets parses the comma-separated targets string into ScanTarget structs +// Format: "libraryID:folderPath,libraryID:folderPath,..." +// Example: "1:Music/Rock,1:Music/Jazz,2:Classical" +func parseTargets(targetsStr string) ([]scanner.ScanTarget, error) { + parts := strings.Split(targetsStr, ",") + targets := make([]scanner.ScanTarget, 0, len(parts)) + + for _, part := range parts { + part = strings.TrimSpace(part) + if part == "" { + continue + } + + // Split by the first colon + colonIdx := strings.Index(part, ":") + if colonIdx == -1 { + return nil, fmt.Errorf("invalid target format: %q (expected libraryID:folderPath)", part) + } + + libIDStr := part[:colonIdx] + folderPath := part[colonIdx+1:] + + libID, err := strconv.Atoi(libIDStr) + if err != nil { + return nil, fmt.Errorf("invalid library ID %q: %w", libIDStr, err) + } + + targets = append(targets, scanner.ScanTarget{ + LibraryID: libID, + FolderPath: folderPath, + }) + } + + if len(targets) == 0 { + return nil, fmt.Errorf("no valid targets found in %q", targetsStr) + } + + return targets, nil +} diff --git a/cmd/scan_test.go b/cmd/scan_test.go new file mode 100644 index 000000000..191e1096c --- /dev/null +++ b/cmd/scan_test.go @@ -0,0 +1,82 @@ +package cmd + +import ( + "github.com/navidrome/navidrome/scanner" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("parseTargets", func() { + Context("Valid targets", func() { + It("parses a single target", func() { + targets, err := parseTargets("1:Music/Rock") + Expect(err).ToNot(HaveOccurred()) + Expect(targets).To(HaveLen(1)) + Expect(targets[0].LibraryID).To(Equal(1)) + Expect(targets[0].FolderPath).To(Equal("Music/Rock")) + }) + + It("parses multiple targets", func() { + targets, err := parseTargets("1:Music/Rock,2:Jazz,3:Classical/Beethoven") + Expect(err).ToNot(HaveOccurred()) + Expect(targets).To(HaveLen(3)) + Expect(targets[0]).To(Equal(scanner.ScanTarget{LibraryID: 1, FolderPath: "Music/Rock"})) + Expect(targets[1]).To(Equal(scanner.ScanTarget{LibraryID: 2, FolderPath: "Jazz"})) + Expect(targets[2]).To(Equal(scanner.ScanTarget{LibraryID: 3, FolderPath: "Classical/Beethoven"})) + }) + + It("handles targets with spaces around commas", func() { + targets, err := parseTargets("1:Music/Rock , 2:Jazz , 3:Classical") + Expect(err).ToNot(HaveOccurred()) + Expect(targets).To(HaveLen(3)) + }) + + It("handles paths with colons after the first colon", func() { + targets, err := parseTargets("1:C:/Music/Rock") + Expect(err).ToNot(HaveOccurred()) + Expect(targets).To(HaveLen(1)) + Expect(targets[0].LibraryID).To(Equal(1)) + Expect(targets[0].FolderPath).To(Equal("C:/Music/Rock")) + }) + + It("handles empty folder paths", func() { + targets, err := parseTargets("1:,2:") + Expect(err).ToNot(HaveOccurred()) + Expect(targets).To(HaveLen(2)) + Expect(targets[0].FolderPath).To(BeEmpty()) + Expect(targets[1].FolderPath).To(BeEmpty()) + }) + }) + + Context("Invalid targets", func() { + It("returns error for empty string", func() { + _, err := parseTargets("") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("no valid targets")) + }) + + It("returns error for missing colon", func() { + _, err := parseTargets("1Music") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("invalid target format")) + }) + + It("returns error for invalid library ID", func() { + _, err := parseTargets("abc:Music") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("invalid library ID")) + }) + + It("handles negative library ID", func() { + targets, err := parseTargets("-1:Music") + Expect(err).ToNot(HaveOccurred()) // Actually valid - strconv.Atoi accepts negative numbers + Expect(targets[0].LibraryID).To(Equal(-1)) + }) + + It("handles only whitespace", func() { + _, err := parseTargets(" , , ") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("no valid targets")) + }) + }) +}) diff --git a/model/datastore.go b/model/datastore.go index 4290e2134..536a37274 100644 --- a/model/datastore.go +++ b/model/datastore.go @@ -43,5 +43,5 @@ type DataStore interface { WithTx(block func(tx DataStore) error, scope ...string) error WithTxImmediate(block func(tx DataStore) error, scope ...string) error - GC(ctx context.Context) error + GC(ctx context.Context, libraryIDs ...int) error } diff --git a/model/folder.go b/model/folder.go index f715f8c11..7ac2bf031 100644 --- a/model/folder.go +++ b/model/folder.go @@ -83,6 +83,7 @@ type FolderUpdateInfo struct { type FolderRepository interface { Get(id string) (*Folder, error) GetByPath(lib Library, path string) (*Folder, error) + GetByPaths(targets []LibraryPath) (map[string]FolderUpdateInfo, error) GetAll(...QueryOptions) ([]Folder, error) CountAll(...QueryOptions) (int64, error) GetLastUpdates(lib Library) (map[string]FolderUpdateInfo, error) @@ -90,3 +91,9 @@ type FolderRepository interface { MarkMissing(missing bool, ids ...string) error GetTouchedWithPlaylists() (FolderCursor, error) } + +// LibraryPath represents a folder path within a specific library +type LibraryPath struct { + LibraryID int + FolderPath string +} diff --git a/persistence/folder_repository.go b/persistence/folder_repository.go index 96a9bae82..646c7e61d 100644 --- a/persistence/folder_repository.go +++ b/persistence/folder_repository.go @@ -78,6 +78,67 @@ func (r folderRepository) GetByPath(lib model.Library, path string) (*model.Fold return r.Get(id) } +func (r folderRepository) GetByPaths(targets []model.LibraryPath) (map[string]model.FolderUpdateInfo, error) { + if len(targets) == 0 { + return make(map[string]model.FolderUpdateInfo), nil + } + + // Group targets by library to build efficient queries + targetsByLib := make(map[int][]string) + folderIDs := make([]string, 0, len(targets)) + + // We need to resolve library paths to generate folder IDs + // Get all libraries first + libRepo := NewLibraryRepository(r.ctx, r.db) + allLibs, err := libRepo.GetAll() + if err != nil { + return nil, fmt.Errorf("getting libraries: %w", err) + } + libMap := make(map[int]model.Library) + for _, lib := range allLibs { + libMap[lib.ID] = lib + } + + // Generate folder IDs for all targets + for _, target := range targets { + lib, ok := libMap[target.LibraryID] + if !ok { + continue // Skip invalid library IDs + } + folderPath := target.FolderPath + if folderPath == "" { + folderPath = "." + } + folderID := model.FolderID(lib, folderPath) + folderIDs = append(folderIDs, folderID) + targetsByLib[target.LibraryID] = append(targetsByLib[target.LibraryID], folderPath) + } + + if len(folderIDs) == 0 { + return make(map[string]model.FolderUpdateInfo), nil + } + + // Query folders by IDs + sq := r.newSelect().Columns("id", "updated_at", "hash").Where(And{ + Eq{"id": folderIDs}, + Eq{"missing": false}, + }) + var res []struct { + ID string + UpdatedAt time.Time + Hash string + } + err = r.queryAll(sq, &res) + if err != nil { + return nil, err + } + m := make(map[string]model.FolderUpdateInfo, len(res)) + for _, f := range res { + m[f.ID] = model.FolderUpdateInfo{UpdatedAt: f.UpdatedAt, Hash: f.Hash} + } + return m, nil +} + func (r folderRepository) GetAll(opt ...model.QueryOptions) ([]model.Folder, error) { sq := r.selectFolder(opt...) var res dbFolders diff --git a/persistence/folder_repository_test.go b/persistence/folder_repository_test.go new file mode 100644 index 000000000..93190c494 --- /dev/null +++ b/persistence/folder_repository_test.go @@ -0,0 +1,172 @@ +package persistence + +import ( + "context" + + "github.com/navidrome/navidrome/log" + "github.com/navidrome/navidrome/model" + "github.com/navidrome/navidrome/model/request" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/pocketbase/dbx" +) + +var _ = Describe("FolderRepository", func() { + var repo model.FolderRepository + var ctx context.Context + var conn *dbx.DB + var testLib model.Library + + BeforeEach(func() { + ctx = request.WithUser(log.NewContext(context.TODO()), model.User{ID: "userid"}) + conn = GetDBXBuilder() + repo = newFolderRepository(ctx, conn) + + // Use existing library ID 1 from test fixtures + libRepo := NewLibraryRepository(ctx, conn) + lib, err := libRepo.Get(1) + Expect(err).ToNot(HaveOccurred()) + testLib = *lib + }) + + AfterEach(func() { + // Clean up test folders created by these tests + // Only delete folders with paths starting with our test prefix + _, _ = conn.NewQuery("DELETE FROM folder WHERE library_id = 1 AND (path LIKE 'TestFolder%' OR path LIKE 'Music/%' OR path = 'Classical' OR path = 'Podcasts')").Execute() + }) + + Describe("GetByPaths", func() { + Context("with valid targets", func() { + It("returns folder info for existing folders", func() { + // Create test folders + folder1 := model.NewFolder(testLib, "Music/Rock") + folder2 := model.NewFolder(testLib, "Music/Jazz") + folder3 := model.NewFolder(testLib, "Classical") + + err := repo.Put(folder1) + Expect(err).ToNot(HaveOccurred()) + err = repo.Put(folder2) + Expect(err).ToNot(HaveOccurred()) + err = repo.Put(folder3) + Expect(err).ToNot(HaveOccurred()) + + // Query by paths + targets := []model.LibraryPath{ + {LibraryID: testLib.ID, FolderPath: "Music/Rock"}, + {LibraryID: testLib.ID, FolderPath: "Classical"}, + } + + results, err := repo.GetByPaths(targets) + Expect(err).ToNot(HaveOccurred()) + Expect(results).To(HaveLen(2)) + + // Verify folder IDs are in results + Expect(results).To(HaveKey(folder1.ID)) + Expect(results).To(HaveKey(folder3.ID)) + Expect(results).ToNot(HaveKey(folder2.ID)) + + // Verify update info is populated + Expect(results[folder1.ID].UpdatedAt).ToNot(BeZero()) + Expect(results[folder1.ID].Hash).To(Equal(folder1.Hash)) + }) + + It("handles empty folder path as root", func() { + // Create root folder + rootFolder := model.NewFolder(testLib, ".") + err := repo.Put(rootFolder) + Expect(err).ToNot(HaveOccurred()) + + targets := []model.LibraryPath{ + {LibraryID: testLib.ID, FolderPath: ""}, + } + + results, err := repo.GetByPaths(targets) + Expect(err).ToNot(HaveOccurred()) + Expect(results).To(HaveLen(1)) + Expect(results).To(HaveKey(rootFolder.ID)) + }) + + It("returns empty map for non-existent folders", func() { + targets := []model.LibraryPath{ + {LibraryID: testLib.ID, FolderPath: "NonExistent/Path"}, + } + + results, err := repo.GetByPaths(targets) + Expect(err).ToNot(HaveOccurred()) + Expect(results).To(BeEmpty()) + }) + + It("skips missing folders", func() { + // Create a folder and mark it as missing + folder := model.NewFolder(testLib, "Music/Missing") + folder.Missing = true + err := repo.Put(folder) + Expect(err).ToNot(HaveOccurred()) + + targets := []model.LibraryPath{ + {LibraryID: testLib.ID, FolderPath: "Music/Missing"}, + } + + results, err := repo.GetByPaths(targets) + Expect(err).ToNot(HaveOccurred()) + Expect(results).To(BeEmpty()) + }) + }) + + Context("with invalid library IDs", func() { + It("returns empty map for non-existent library", func() { + targets := []model.LibraryPath{ + {LibraryID: 99999, FolderPath: "Music"}, + } + + results, err := repo.GetByPaths(targets) + Expect(err).ToNot(HaveOccurred()) + Expect(results).To(BeEmpty()) + }) + }) + + Context("with empty targets", func() { + It("returns empty map", func() { + results, err := repo.GetByPaths([]model.LibraryPath{}) + Expect(err).ToNot(HaveOccurred()) + Expect(results).To(BeEmpty()) + }) + + It("returns empty map for nil targets", func() { + results, err := repo.GetByPaths(nil) + Expect(err).ToNot(HaveOccurred()) + Expect(results).To(BeEmpty()) + }) + }) + + Context("with multiple paths in same library", func() { + It("returns multiple folders", func() { + // Create multiple folders in the same library + folder1 := model.NewFolder(testLib, "Music/Pop") + folder2 := model.NewFolder(testLib, "Music/Electronic") + folder3 := model.NewFolder(testLib, "Podcasts") + + err := repo.Put(folder1) + Expect(err).ToNot(HaveOccurred()) + err = repo.Put(folder2) + Expect(err).ToNot(HaveOccurred()) + err = repo.Put(folder3) + Expect(err).ToNot(HaveOccurred()) + + // Query multiple paths + targets := []model.LibraryPath{ + {LibraryID: testLib.ID, FolderPath: "Music/Pop"}, + {LibraryID: testLib.ID, FolderPath: "Music/Electronic"}, + {LibraryID: testLib.ID, FolderPath: "Podcasts"}, + } + + results, err := repo.GetByPaths(targets) + Expect(err).ToNot(HaveOccurred()) + Expect(results).To(HaveLen(3)) + Expect(results).To(HaveKey(folder1.ID)) + Expect(results).To(HaveKey(folder2.ID)) + Expect(results).To(HaveKey(folder3.ID)) + }) + }) + }) +}) diff --git a/persistence/persistence.go b/persistence/persistence.go index ac607f85f..b0c5a7f50 100644 --- a/persistence/persistence.go +++ b/persistence/persistence.go @@ -157,7 +157,7 @@ func (s *SQLStore) WithTxImmediate(block func(tx model.DataStore) error, scope . }, scope...) } -func (s *SQLStore) GC(ctx context.Context) error { +func (s *SQLStore) GC(ctx context.Context, libraryIDs ...int) error { trace := func(ctx context.Context, msg string, f func() error) func() error { return func() error { start := time.Now() @@ -167,6 +167,12 @@ func (s *SQLStore) GC(ctx context.Context) error { } } + // TODO: Implement library-specific filtering for GC operations + // For now, GC runs globally even in selective scans + if len(libraryIDs) > 0 { + log.Debug(ctx, "GC: Running with library filter", "libraries", libraryIDs) + } + err := run.Sequentially( trace(ctx, "purge empty albums", func() error { return s.Album(ctx).(*albumRepository).purgeEmpty() }), trace(ctx, "purge empty artists", func() error { return s.Artist(ctx).(*artistRepository).purgeEmpty() }), diff --git a/scanner/controller.go b/scanner/controller.go index c1347077a..d2ebdbfd4 100644 --- a/scanner/controller.go +++ b/scanner/controller.go @@ -26,9 +26,18 @@ var ( ErrAlreadyScanning = errors.New("already scanning") ) +// ScanTarget represents a specific folder within a library to be scanned. +type ScanTarget struct { + LibraryID int + FolderPath string // Relative path within the library, or "" for entire library +} + type Scanner interface { // ScanAll starts a full scan of the music library. This is a blocking operation. ScanAll(ctx context.Context, fullScan bool) (warnings []string, err error) + // ScanFolders scans specific library/folder pairs without recursing into subdirectories. + // This is a blocking operation. + ScanFolders(ctx context.Context, fullScan bool, targets []ScanTarget) (warnings []string, err error) Status(context.Context) (*StatusInfo, error) } @@ -68,6 +77,12 @@ func (s *controller) getScanner() scanner { // CallScan starts an in-process scan of the music library. // This is meant to be called from the command line (see cmd/scan.go). func CallScan(ctx context.Context, ds model.DataStore, pls core.Playlists, fullScan bool) (<-chan *ProgressInfo, error) { + return CallScanFolders(ctx, ds, pls, fullScan, nil) +} + +// CallScanFolders starts an in-process scan of specific library/folder pairs. +// If targets is nil, it scans all libraries. This is meant to be called from the command line. +func CallScanFolders(ctx context.Context, ds model.DataStore, pls core.Playlists, fullScan bool, targets []ScanTarget) (<-chan *ProgressInfo, error) { release, err := lockScan(ctx) if err != nil { return nil, err @@ -79,7 +94,11 @@ func CallScan(ctx context.Context, ds model.DataStore, pls core.Playlists, fullS go func() { defer close(progress) scanner := &scannerImpl{ds: ds, cw: artwork.NoopCacheWarmer(), pls: pls} - scanner.scanAll(ctx, fullScan, progress) + if targets == nil { + scanner.scanAll(ctx, fullScan, progress) + } else { + scanner.scanFolders(ctx, fullScan, targets, progress) + } }() return progress, nil } @@ -101,6 +120,7 @@ type ProgressInfo struct { type scanner interface { scanAll(ctx context.Context, fullScan bool, progress chan<- *ProgressInfo) + scanFolders(ctx context.Context, fullScan bool, targets []ScanTarget, progress chan<- *ProgressInfo) } type controller struct { @@ -208,6 +228,10 @@ func (s *controller) getCounters(ctx context.Context) (int64, int64, error) { } func (s *controller) ScanAll(requestCtx context.Context, fullScan bool) ([]string, error) { + return s.ScanFolders(requestCtx, fullScan, nil) +} + +func (s *controller) ScanFolders(requestCtx context.Context, fullScan bool, targets []ScanTarget) ([]string, error) { release, err := lockScan(requestCtx) if err != nil { return nil, err @@ -224,7 +248,11 @@ func (s *controller) ScanAll(requestCtx context.Context, fullScan bool) ([]strin go func() { defer close(progress) scanner := s.getScanner() - scanner.scanAll(ctx, fullScan, progress) + if targets == nil { + scanner.scanAll(ctx, fullScan, progress) + } else { + scanner.scanFolders(ctx, fullScan, targets, progress) + } }() // Wait for the scan to finish, sending progress events to all connected clients diff --git a/scanner/external.go b/scanner/external.go index c4a29efa3..86d171346 100644 --- a/scanner/external.go +++ b/scanner/external.go @@ -8,6 +8,7 @@ import ( "io" "os" "os/exec" + "strconv" "github.com/navidrome/navidrome/conf" "github.com/navidrome/navidrome/log" @@ -62,6 +63,56 @@ func (s *scannerExternal) scanAll(ctx context.Context, fullScan bool, progress c } } +func (s *scannerExternal) scanFolders(ctx context.Context, fullScan bool, targets []ScanTarget, progress chan<- *ProgressInfo) { + exe, err := os.Executable() + if err != nil { + progress <- &ProgressInfo{Error: fmt.Sprintf("failed to get executable path: %s", err)} + return + } + + // Build targets string for CLI + var targetsStr string + for i, target := range targets { + if i > 0 { + targetsStr += "," + } + targetsStr += strconv.Itoa(target.LibraryID) + ":" + target.FolderPath + } + + log.Debug(ctx, "Spawning external scanner process with targets", "fullScan", fullScan, "path", exe, "targets", targetsStr) + cmd := exec.CommandContext(ctx, exe, "scan", + "--nobanner", "--subprocess", + "--configfile", conf.Server.ConfigFile, + "--datafolder", conf.Server.DataFolder, + "--cachefolder", conf.Server.CacheFolder, + "--targets", targetsStr, + If(fullScan, "--full", "")) + + in, out := io.Pipe() + defer in.Close() + defer out.Close() + cmd.Stdout = out + cmd.Stderr = os.Stderr + + if err := cmd.Start(); err != nil { + progress <- &ProgressInfo{Error: fmt.Sprintf("failed to start scanner process: %s", err)} + return + } + go s.wait(cmd, out) + + decoder := gob.NewDecoder(in) + for { + var p ProgressInfo + if err := decoder.Decode(&p); err != nil { + if !errors.Is(err, io.EOF) { + progress <- &ProgressInfo{Error: fmt.Sprintf("failed to read status from scanner: %s", err)} + } + break + } + progress <- &p + } +} + func (s *scannerExternal) wait(cmd *exec.Cmd, out *io.PipeWriter) { if err := cmd.Wait(); err != nil { var exitErr *exec.ExitError diff --git a/scanner/phase_1_folders.go b/scanner/phase_1_folders.go index e04f10c70..ebdf74b54 100644 --- a/scanner/phase_1_folders.go +++ b/scanner/phase_1_folders.go @@ -48,7 +48,14 @@ func createPhaseFolders(ctx context.Context, state *scanState, ds model.DataStor } else { log.Debug(ctx, "Scanner: Resuming previous scan", "lib", lib.Name, "lastScanStartedAt", lib.LastScanStartedAt, "fullScan", lib.FullScanInProgress) } - job, err := newScanJob(ctx, ds, cw, lib, state.fullScan) + + // Get target folders for this library if selective scan + var targetFolders []string + if state.targets != nil { + targetFolders = state.targets[lib.ID] + } + + job, err := newScanJob(ctx, ds, cw, lib, state.fullScan, targetFolders) if err != nil { log.Error(ctx, "Scanner: Error creating scan context", "lib", lib.Name, err) state.sendWarning(err.Error()) @@ -65,19 +72,36 @@ func createPhaseFolders(ctx context.Context, state *scanState, ds model.DataStor } type scanJob struct { - lib model.Library - fs storage.MusicFS - cw artwork.CacheWarmer - lastUpdates map[string]model.FolderUpdateInfo - lock sync.Mutex - numFolders atomic.Int64 + lib model.Library + fs storage.MusicFS + cw artwork.CacheWarmer + lastUpdates map[string]model.FolderUpdateInfo + targetFolders []string // Optional: specific folders to scan (non-recursive) + lock sync.Mutex + numFolders atomic.Int64 } -func newScanJob(ctx context.Context, ds model.DataStore, cw artwork.CacheWarmer, lib model.Library, fullScan bool) (*scanJob, error) { - lastUpdates, err := ds.Folder(ctx).GetLastUpdates(lib) +func newScanJob(ctx context.Context, ds model.DataStore, cw artwork.CacheWarmer, lib model.Library, fullScan bool, targetFolders []string) (*scanJob, error) { + var lastUpdates map[string]model.FolderUpdateInfo + var err error + + // If we have target folders, get only those folder updates. Otherwise get all updates for the library + if len(targetFolders) > 0 { + var targets []model.LibraryPath + for _, folderPath := range targetFolders { + targets = append(targets, model.LibraryPath{ + LibraryID: lib.ID, + FolderPath: folderPath, + }) + } + lastUpdates, err = ds.Folder(ctx).GetByPaths(targets) + } else { + lastUpdates, err = ds.Folder(ctx).GetLastUpdates(lib) + } if err != nil { return nil, fmt.Errorf("getting last updates: %w", err) } + fileStore, err := storage.For(lib.Path) if err != nil { log.Error(ctx, "Error getting storage for library", "library", lib.Name, "path", lib.Path, err) @@ -90,10 +114,11 @@ func newScanJob(ctx context.Context, ds model.DataStore, cw artwork.CacheWarmer, } lib.FullScanInProgress = lib.FullScanInProgress || fullScan return &scanJob{ - lib: lib, - fs: fsys, - cw: cw, - lastUpdates: lastUpdates, + lib: lib, + fs: fsys, + cw: cw, + lastUpdates: lastUpdates, + targetFolders: targetFolders, }, nil } @@ -144,7 +169,18 @@ func (p *phaseFolders) producer() ppl.Producer[*folderEntry] { if utils.IsCtxDone(p.ctx) { break } - outputChan, err := walkDirTree(p.ctx, job) + + var outputChan <-chan *folderEntry + var err error + + // Use selective folder loading if target folders are specified + if len(job.targetFolders) > 0 { + log.Debug(p.ctx, "Scanner: Loading specific folders only (non-recursive)", "lib", job.lib.Name, "numTargets", len(job.targetFolders)) + outputChan, err = loadSpecificFolders(p.ctx, job, job.targetFolders) + } else { + outputChan, err = walkDirTree(p.ctx, job) + } + if err != nil { log.Warn(p.ctx, "Scanner: Error scanning library", "lib", job.lib.Name, err) } diff --git a/scanner/scanner.go b/scanner/scanner.go index 04a5c2456..3c4d41d31 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -28,7 +28,9 @@ type scanState struct { progress chan<- *ProgressInfo fullScan bool changesDetected atomic.Bool - libraries model.Libraries // Store libraries list for consistency across phases + libraries model.Libraries // Store libraries list for consistency across phases + targets map[int][]string // Optional: map[libraryID][]folderPaths for selective scans + affectedLibIDs []int // IDs of libraries involved in the scan (for GC scoping) } func (s *scanState) sendProgress(info *ProgressInfo) { @@ -134,13 +136,123 @@ func (s *scannerImpl) scanAll(ctx context.Context, fullScan bool, progress chan< log.Info(ctx, "Scanner: Finished scanning all libraries", "duration", time.Since(startTime)) } +func (s *scannerImpl) scanFolders(ctx context.Context, fullScan bool, targets []ScanTarget, progress chan<- *ProgressInfo) { + startTime := time.Now() + + state := scanState{ + progress: progress, + fullScan: fullScan, + changesDetected: atomic.Bool{}, + targets: make(map[int][]string), + } + + // Set changesDetected to true for full scans to ensure all maintenance operations run + if fullScan { + state.changesDetected.Store(true) + } + + // Group targets by library and collect affected library IDs + affectedLibIDSet := make(map[int]bool) + for _, target := range targets { + folderPath := target.FolderPath + if folderPath == "" { + folderPath = "." + } + state.targets[target.LibraryID] = append(state.targets[target.LibraryID], folderPath) + affectedLibIDSet[target.LibraryID] = true + } + + // Get affected libraries + allLibs, err := s.ds.Library(ctx).GetAll() + if err != nil { + state.sendWarning(fmt.Sprintf("getting libraries: %s", err)) + return + } + + var libs model.Libraries + for _, lib := range allLibs { + if affectedLibIDSet[lib.ID] { + libs = append(libs, lib) + state.affectedLibIDs = append(state.affectedLibIDs, lib.ID) + } + } + state.libraries = libs + + log.Info(ctx, "Scanner: Starting selective scan", "fullScan", state.fullScan, "numLibraries", len(libs), "numTargets", len(targets)) + + // Store scan type and start time + scanType := "quick-selective" + if state.fullScan { + scanType = "full-selective" + } + _ = s.ds.Property(ctx).Put(consts.LastScanTypeKey, scanType) + _ = s.ds.Property(ctx).Put(consts.LastScanStartTimeKey, startTime.Format(time.RFC3339)) + + // if there was a full scan in progress, force a full scan + if !state.fullScan { + for _, lib := range libs { + if lib.FullScanInProgress { + log.Info(ctx, "Scanner: Interrupted full scan detected", "lib", lib.Name) + state.fullScan = true + _ = s.ds.Property(ctx).Put(consts.LastScanTypeKey, "full-selective") + break + } + } + } + + err = run.Sequentially( + // Phase 1: Scan specified folders and import new/updated files + runPhase[*folderEntry](ctx, 1, createPhaseFolders(ctx, &state, s.ds, s.cw, libs)), + + // Phase 2: Process missing files in scanned folders only + runPhase[*missingTracks](ctx, 2, createPhaseMissingTracks(ctx, &state, s.ds)), + + // Phases 3 and 4 can be run in parallel + run.Parallel( + // Phase 3: Refresh all new/changed albums (from affected libraries only) + runPhase[*model.Album](ctx, 3, createPhaseRefreshAlbums(ctx, &state, s.ds, libs)), + + // Phase 4: Import/update playlists (from affected libraries only) + runPhase[*model.Folder](ctx, 4, createPhasePlaylists(ctx, &state, s.ds, s.pls, s.cw)), + ), + + // Final Steps (cannot be parallelized): + + // Run GC scoped to affected libraries only + s.runGC(ctx, &state), + + // Refresh artist and tags stats + s.runRefreshStats(ctx, &state), + + // Update last_scan_completed_at for affected libraries + s.runUpdateLibraries(ctx, &state), + + // Optimize DB + s.runOptimize(ctx), + ) + if err != nil { + log.Error(ctx, "Scanner: Finished with error", "duration", time.Since(startTime), err) + _ = s.ds.Property(ctx).Put(consts.LastScanErrorKey, err.Error()) + state.sendError(err) + return + } + + _ = s.ds.Property(ctx).Put(consts.LastScanErrorKey, "") + + if state.changesDetected.Load() { + state.sendProgress(&ProgressInfo{ChangesDetected: true}) + } + + log.Info(ctx, "Scanner: Finished scanning selected folders", "duration", time.Since(startTime), "numTargets", len(targets)) +} + func (s *scannerImpl) runGC(ctx context.Context, state *scanState) func() error { return func() error { state.sendProgress(&ProgressInfo{ForceUpdate: true}) return s.ds.WithTx(func(tx model.DataStore) error { if state.changesDetected.Load() { start := time.Now() - err := tx.GC(ctx) + err := tx.GC(ctx, state.affectedLibIDs...) if err != nil { log.Error(ctx, "Scanner: Error running GC", err) return fmt.Errorf("running GC: %w", err) diff --git a/scanner/scanner_test.go b/scanner/scanner_test.go index e7e354f21..1e0614573 100644 --- a/scanner/scanner_test.go +++ b/scanner/scanner_test.go @@ -717,6 +717,61 @@ var _ = Describe("Scanner", Ordered, func() { Expect(albumArtistStats.SongCount).To(Equal(3)) // 3 songs }) }) + + Describe("ScanFolders", func() { + It("scans only specified folders without recursion", func() { + rock := template(_t{"albumartist": "Rock Artist", "album": "Rock Album"}) + jazz := template(_t{"albumartist": "Jazz Artist", "album": "Jazz Album"}) + pop := template(_t{"albumartist": "Pop Artist", "album": "Pop Album"}) + createFS(fstest.MapFS{ + "rock/track1.mp3": rock(track(1, "Rock Track 1")), + "rock/track2.mp3": rock(track(2, "Rock Track 2")), + "rock/subdir/track3.mp3": rock(track(3, "Rock Track 3")), + "jazz/track4.mp3": jazz(track(1, "Jazz Track 1")), + "jazz/subdir/track5.mp3": jazz(track(2, "Jazz Track 2")), + "pop/track6.mp3": pop(track(1, "Pop Track 1")), + }) + + // Use the existing library from BeforeEach + // (lib is already created with the path "fake:///music") + + // Scan only the "rock" and "jazz" folders (not their subdirectories or pop) + targets := []scanner.ScanTarget{ + {LibraryID: lib.ID, FolderPath: "rock"}, + {LibraryID: lib.ID, FolderPath: "jazz"}, + } + + warnings, err := s.ScanFolders(ctx, false, targets) + Expect(err).ToNot(HaveOccurred()) + Expect(warnings).To(BeEmpty()) + + // Verify only track1, track2, and track4 were imported (not track3, track5, or track6) + allFiles, err := ds.MediaFile(ctx).GetAll() + Expect(err).ToNot(HaveOccurred()) + + // Should have exactly 3 tracks (rock/track1, rock/track2, jazz/track4) + Expect(allFiles).To(HaveLen(3)) + + // Get the file paths + paths := slice.Map(allFiles, func(mf model.MediaFile) string { + return filepath.ToSlash(mf.Path) + }) + + // Verify the correct files were scanned + Expect(paths).To(ContainElements( + "rock/track1.mp3", + "rock/track2.mp3", + "jazz/track4.mp3", + )) + + // Verify files in subdirectories and pop folder were NOT scanned + Expect(paths).ToNot(ContainElements( + "rock/subdir/track3.mp3", + "jazz/subdir/track5.mp3", + "pop/track6.mp3", + )) + }) + }) }) func createFindByPath(ctx context.Context, ds model.DataStore) func(string) (*model.MediaFile, error) { diff --git a/scanner/walk_dir_tree.go b/scanner/walk_dir_tree.go index 63854d262..6afe76755 100644 --- a/scanner/walk_dir_tree.go +++ b/scanner/walk_dir_tree.go @@ -32,6 +32,64 @@ func walkDirTree(ctx context.Context, job *scanJob) (<-chan *folderEntry, error) return results, nil } +// loadSpecificFolders loads only the specified folders without recursing into subdirectories +func loadSpecificFolders(ctx context.Context, job *scanJob, targetFolders []string) (<-chan *folderEntry, error) { + results := make(chan *folderEntry) + go func() { + defer close(results) + for _, folderPath := range targetFolders { + if utils.IsCtxDone(ctx) { + return + } + + // Load ignore patterns from parent directories up to this folder + ignorePatterns := loadIgnorePatternsForPath(ctx, job.fs, folderPath) + + // Load only this specific folder (no recursion) + folder, _, err := loadDir(ctx, job, folderPath, ignorePatterns) + if err != nil { + log.Warn(ctx, "Scanner: Error loading target folder. Skipping", "path", folderPath, err) + continue + } + + folder.path = path.Clean(folderPath) + folder.elapsed.Start() + log.Trace(ctx, "Scanner: Found target directory", " path", folder.path, "audioFiles", maps.Keys(folder.audioFiles), + "images", maps.Keys(folder.imageFiles), "playlists", folder.numPlaylists, "imagesUpdatedAt", folder.imagesUpdatedAt, + "updTime", folder.updTime, "modTime", folder.modTime) + + results <- folder + } + log.Debug(ctx, "Scanner: Finished reading target folders", "lib", job.lib.Name, "path", job.lib.Path, "numFolders", len(targetFolders)) + }() + return results, nil +} + +// loadIgnorePatternsForPath loads all .ndignore patterns from the root down to the specified path +func loadIgnorePatternsForPath(ctx context.Context, fsys fs.FS, targetPath string) []string { + var patterns []string + currentPath := "." + + // If target is root, just check root + if targetPath == "." { + return loadIgnoredPatterns(ctx, fsys, ".", nil) + } + + // Walk from root to target, collecting ignore patterns + parts := strings.Split(path.Clean(targetPath), "/") + for _, part := range parts { + if part == "." { + continue + } + patterns = loadIgnoredPatterns(ctx, fsys, currentPath, patterns) + currentPath = path.Join(currentPath, part) + } + // Load patterns from the target folder itself + patterns = loadIgnoredPatterns(ctx, fsys, currentPath, patterns) + + return patterns +} + func walkFolder(ctx context.Context, job *scanJob, currentFolder string, ignorePatterns []string, results chan<- *folderEntry) error { ignorePatterns = loadIgnoredPatterns(ctx, job.fs, currentFolder, ignorePatterns) diff --git a/tests/mock_data_store.go b/tests/mock_data_store.go index 56f68a74b..2c0c90f62 100644 --- a/tests/mock_data_store.go +++ b/tests/mock_data_store.go @@ -258,6 +258,6 @@ func (db *MockDataStore) Resource(ctx context.Context, m any) model.ResourceRepo } } -func (db *MockDataStore) GC(context.Context) error { +func (db *MockDataStore) GC(context.Context, ...int) error { return nil }