mirror of
https://github.com/navidrome/navidrome.git
synced 2026-08-31 07:30:32 +00:00
feat: Add selective folder scanning capability
Implement targeted scanning of specific library/folder pairs without full recursion. This enables efficient rescanning of individual folders when changes are detected, significantly reducing scan time for large libraries. Key changes: - Add ScanTarget struct and ScanFolders API to Scanner interface - Implement CLI flag --targets for specifying libraryID:folderPath pairs - Add FolderRepository.GetByPaths() for batch folder info retrieval - Create loadSpecificFolders() for non-recursive directory loading - Scope GC operations to affected libraries only (with TODO for full impl) - Add comprehensive tests for selective scanning behavior The selective scan: - Only processes specified folders (no subdirectory recursion) - Maintains library isolation - Runs full maintenance pipeline scoped to affected libraries - Supports both full and quick scan modes Examples: navidrome scan --targets "1:Music/Rock,1:Music/Jazz" navidrome scan --full --targets "2:Classical"
This commit is contained in:
parent
f939ad84f3
commit
8f1a6116fe
58
cmd/scan.go
58
cmd/scan.go
@ -3,7 +3,10 @@ package cmd
|
||||
import (
|
||||
"context"
|
||||
"encoding/gob"
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/navidrome/navidrome/core"
|
||||
"github.com/navidrome/navidrome/db"
|
||||
@ -17,11 +20,13 @@ import (
|
||||
var (
|
||||
fullScan bool
|
||||
subprocess bool
|
||||
targets string
|
||||
)
|
||||
|
||||
func init() {
|
||||
scanCmd.Flags().BoolVarP(&fullScan, "full", "f", false, "check all subfolders, ignoring timestamps")
|
||||
scanCmd.Flags().BoolVarP(&subprocess, "subprocess", "", false, "run as subprocess (internal use)")
|
||||
scanCmd.Flags().StringVarP(&targets, "targets", "t", "", "comma-separated list of libraryID:folderPath pairs (e.g., \"1:Music/Rock,1:Music/Jazz,2:Classical\")")
|
||||
rootCmd.AddCommand(scanCmd)
|
||||
}
|
||||
|
||||
@ -68,7 +73,18 @@ func runScanner(ctx context.Context) {
|
||||
ds := persistence.New(sqlDB)
|
||||
pls := core.NewPlaylists(ds)
|
||||
|
||||
progress, err := scanner.CallScan(ctx, ds, pls, fullScan)
|
||||
// Parse targets if provided
|
||||
var scanTargets []scanner.ScanTarget
|
||||
if targets != "" {
|
||||
var err error
|
||||
scanTargets, err = parseTargets(targets)
|
||||
if err != nil {
|
||||
log.Fatal(ctx, "Failed to parse targets", err)
|
||||
}
|
||||
log.Info(ctx, "Scanning specific folders", "numTargets", len(scanTargets))
|
||||
}
|
||||
|
||||
progress, err := scanner.CallScanFolders(ctx, ds, pls, fullScan, scanTargets)
|
||||
if err != nil {
|
||||
log.Fatal(ctx, "Failed to scan", err)
|
||||
}
|
||||
@ -80,3 +96,43 @@ func runScanner(ctx context.Context) {
|
||||
trackScanInteractively(ctx, progress)
|
||||
}
|
||||
}
|
||||
|
||||
// parseTargets parses the comma-separated targets string into ScanTarget structs
|
||||
// Format: "libraryID:folderPath,libraryID:folderPath,..."
|
||||
// Example: "1:Music/Rock,1:Music/Jazz,2:Classical"
|
||||
func parseTargets(targetsStr string) ([]scanner.ScanTarget, error) {
|
||||
parts := strings.Split(targetsStr, ",")
|
||||
targets := make([]scanner.ScanTarget, 0, len(parts))
|
||||
|
||||
for _, part := range parts {
|
||||
part = strings.TrimSpace(part)
|
||||
if part == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
// Split by the first colon
|
||||
colonIdx := strings.Index(part, ":")
|
||||
if colonIdx == -1 {
|
||||
return nil, fmt.Errorf("invalid target format: %q (expected libraryID:folderPath)", part)
|
||||
}
|
||||
|
||||
libIDStr := part[:colonIdx]
|
||||
folderPath := part[colonIdx+1:]
|
||||
|
||||
libID, err := strconv.Atoi(libIDStr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid library ID %q: %w", libIDStr, err)
|
||||
}
|
||||
|
||||
targets = append(targets, scanner.ScanTarget{
|
||||
LibraryID: libID,
|
||||
FolderPath: folderPath,
|
||||
})
|
||||
}
|
||||
|
||||
if len(targets) == 0 {
|
||||
return nil, fmt.Errorf("no valid targets found in %q", targetsStr)
|
||||
}
|
||||
|
||||
return targets, nil
|
||||
}
|
||||
|
||||
82
cmd/scan_test.go
Normal file
82
cmd/scan_test.go
Normal file
@ -0,0 +1,82 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"github.com/navidrome/navidrome/scanner"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
var _ = Describe("parseTargets", func() {
|
||||
Context("Valid targets", func() {
|
||||
It("parses a single target", func() {
|
||||
targets, err := parseTargets("1:Music/Rock")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(targets).To(HaveLen(1))
|
||||
Expect(targets[0].LibraryID).To(Equal(1))
|
||||
Expect(targets[0].FolderPath).To(Equal("Music/Rock"))
|
||||
})
|
||||
|
||||
It("parses multiple targets", func() {
|
||||
targets, err := parseTargets("1:Music/Rock,2:Jazz,3:Classical/Beethoven")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(targets).To(HaveLen(3))
|
||||
Expect(targets[0]).To(Equal(scanner.ScanTarget{LibraryID: 1, FolderPath: "Music/Rock"}))
|
||||
Expect(targets[1]).To(Equal(scanner.ScanTarget{LibraryID: 2, FolderPath: "Jazz"}))
|
||||
Expect(targets[2]).To(Equal(scanner.ScanTarget{LibraryID: 3, FolderPath: "Classical/Beethoven"}))
|
||||
})
|
||||
|
||||
It("handles targets with spaces around commas", func() {
|
||||
targets, err := parseTargets("1:Music/Rock , 2:Jazz , 3:Classical")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(targets).To(HaveLen(3))
|
||||
})
|
||||
|
||||
It("handles paths with colons after the first colon", func() {
|
||||
targets, err := parseTargets("1:C:/Music/Rock")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(targets).To(HaveLen(1))
|
||||
Expect(targets[0].LibraryID).To(Equal(1))
|
||||
Expect(targets[0].FolderPath).To(Equal("C:/Music/Rock"))
|
||||
})
|
||||
|
||||
It("handles empty folder paths", func() {
|
||||
targets, err := parseTargets("1:,2:")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(targets).To(HaveLen(2))
|
||||
Expect(targets[0].FolderPath).To(BeEmpty())
|
||||
Expect(targets[1].FolderPath).To(BeEmpty())
|
||||
})
|
||||
})
|
||||
|
||||
Context("Invalid targets", func() {
|
||||
It("returns error for empty string", func() {
|
||||
_, err := parseTargets("")
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring("no valid targets"))
|
||||
})
|
||||
|
||||
It("returns error for missing colon", func() {
|
||||
_, err := parseTargets("1Music")
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring("invalid target format"))
|
||||
})
|
||||
|
||||
It("returns error for invalid library ID", func() {
|
||||
_, err := parseTargets("abc:Music")
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring("invalid library ID"))
|
||||
})
|
||||
|
||||
It("handles negative library ID", func() {
|
||||
targets, err := parseTargets("-1:Music")
|
||||
Expect(err).ToNot(HaveOccurred()) // Actually valid - strconv.Atoi accepts negative numbers
|
||||
Expect(targets[0].LibraryID).To(Equal(-1))
|
||||
})
|
||||
|
||||
It("handles only whitespace", func() {
|
||||
_, err := parseTargets(" , , ")
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring("no valid targets"))
|
||||
})
|
||||
})
|
||||
})
|
||||
@ -43,5 +43,5 @@ type DataStore interface {
|
||||
|
||||
WithTx(block func(tx DataStore) error, scope ...string) error
|
||||
WithTxImmediate(block func(tx DataStore) error, scope ...string) error
|
||||
GC(ctx context.Context) error
|
||||
GC(ctx context.Context, libraryIDs ...int) error
|
||||
}
|
||||
|
||||
@ -83,6 +83,7 @@ type FolderUpdateInfo struct {
|
||||
type FolderRepository interface {
|
||||
Get(id string) (*Folder, error)
|
||||
GetByPath(lib Library, path string) (*Folder, error)
|
||||
GetByPaths(targets []LibraryPath) (map[string]FolderUpdateInfo, error)
|
||||
GetAll(...QueryOptions) ([]Folder, error)
|
||||
CountAll(...QueryOptions) (int64, error)
|
||||
GetLastUpdates(lib Library) (map[string]FolderUpdateInfo, error)
|
||||
@ -90,3 +91,9 @@ type FolderRepository interface {
|
||||
MarkMissing(missing bool, ids ...string) error
|
||||
GetTouchedWithPlaylists() (FolderCursor, error)
|
||||
}
|
||||
|
||||
// LibraryPath represents a folder path within a specific library
|
||||
type LibraryPath struct {
|
||||
LibraryID int
|
||||
FolderPath string
|
||||
}
|
||||
|
||||
@ -78,6 +78,67 @@ func (r folderRepository) GetByPath(lib model.Library, path string) (*model.Fold
|
||||
return r.Get(id)
|
||||
}
|
||||
|
||||
func (r folderRepository) GetByPaths(targets []model.LibraryPath) (map[string]model.FolderUpdateInfo, error) {
|
||||
if len(targets) == 0 {
|
||||
return make(map[string]model.FolderUpdateInfo), nil
|
||||
}
|
||||
|
||||
// Group targets by library to build efficient queries
|
||||
targetsByLib := make(map[int][]string)
|
||||
folderIDs := make([]string, 0, len(targets))
|
||||
|
||||
// We need to resolve library paths to generate folder IDs
|
||||
// Get all libraries first
|
||||
libRepo := NewLibraryRepository(r.ctx, r.db)
|
||||
allLibs, err := libRepo.GetAll()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("getting libraries: %w", err)
|
||||
}
|
||||
libMap := make(map[int]model.Library)
|
||||
for _, lib := range allLibs {
|
||||
libMap[lib.ID] = lib
|
||||
}
|
||||
|
||||
// Generate folder IDs for all targets
|
||||
for _, target := range targets {
|
||||
lib, ok := libMap[target.LibraryID]
|
||||
if !ok {
|
||||
continue // Skip invalid library IDs
|
||||
}
|
||||
folderPath := target.FolderPath
|
||||
if folderPath == "" {
|
||||
folderPath = "."
|
||||
}
|
||||
folderID := model.FolderID(lib, folderPath)
|
||||
folderIDs = append(folderIDs, folderID)
|
||||
targetsByLib[target.LibraryID] = append(targetsByLib[target.LibraryID], folderPath)
|
||||
}
|
||||
|
||||
if len(folderIDs) == 0 {
|
||||
return make(map[string]model.FolderUpdateInfo), nil
|
||||
}
|
||||
|
||||
// Query folders by IDs
|
||||
sq := r.newSelect().Columns("id", "updated_at", "hash").Where(And{
|
||||
Eq{"id": folderIDs},
|
||||
Eq{"missing": false},
|
||||
})
|
||||
var res []struct {
|
||||
ID string
|
||||
UpdatedAt time.Time
|
||||
Hash string
|
||||
}
|
||||
err = r.queryAll(sq, &res)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m := make(map[string]model.FolderUpdateInfo, len(res))
|
||||
for _, f := range res {
|
||||
m[f.ID] = model.FolderUpdateInfo{UpdatedAt: f.UpdatedAt, Hash: f.Hash}
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func (r folderRepository) GetAll(opt ...model.QueryOptions) ([]model.Folder, error) {
|
||||
sq := r.selectFolder(opt...)
|
||||
var res dbFolders
|
||||
|
||||
172
persistence/folder_repository_test.go
Normal file
172
persistence/folder_repository_test.go
Normal file
@ -0,0 +1,172 @@
|
||||
package persistence
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/navidrome/navidrome/log"
|
||||
"github.com/navidrome/navidrome/model"
|
||||
"github.com/navidrome/navidrome/model/request"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
"github.com/pocketbase/dbx"
|
||||
)
|
||||
|
||||
var _ = Describe("FolderRepository", func() {
|
||||
var repo model.FolderRepository
|
||||
var ctx context.Context
|
||||
var conn *dbx.DB
|
||||
var testLib model.Library
|
||||
|
||||
BeforeEach(func() {
|
||||
ctx = request.WithUser(log.NewContext(context.TODO()), model.User{ID: "userid"})
|
||||
conn = GetDBXBuilder()
|
||||
repo = newFolderRepository(ctx, conn)
|
||||
|
||||
// Use existing library ID 1 from test fixtures
|
||||
libRepo := NewLibraryRepository(ctx, conn)
|
||||
lib, err := libRepo.Get(1)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
testLib = *lib
|
||||
})
|
||||
|
||||
AfterEach(func() {
|
||||
// Clean up test folders created by these tests
|
||||
// Only delete folders with paths starting with our test prefix
|
||||
_, _ = conn.NewQuery("DELETE FROM folder WHERE library_id = 1 AND (path LIKE 'TestFolder%' OR path LIKE 'Music/%' OR path = 'Classical' OR path = 'Podcasts')").Execute()
|
||||
})
|
||||
|
||||
Describe("GetByPaths", func() {
|
||||
Context("with valid targets", func() {
|
||||
It("returns folder info for existing folders", func() {
|
||||
// Create test folders
|
||||
folder1 := model.NewFolder(testLib, "Music/Rock")
|
||||
folder2 := model.NewFolder(testLib, "Music/Jazz")
|
||||
folder3 := model.NewFolder(testLib, "Classical")
|
||||
|
||||
err := repo.Put(folder1)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
err = repo.Put(folder2)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
err = repo.Put(folder3)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
// Query by paths
|
||||
targets := []model.LibraryPath{
|
||||
{LibraryID: testLib.ID, FolderPath: "Music/Rock"},
|
||||
{LibraryID: testLib.ID, FolderPath: "Classical"},
|
||||
}
|
||||
|
||||
results, err := repo.GetByPaths(targets)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(results).To(HaveLen(2))
|
||||
|
||||
// Verify folder IDs are in results
|
||||
Expect(results).To(HaveKey(folder1.ID))
|
||||
Expect(results).To(HaveKey(folder3.ID))
|
||||
Expect(results).ToNot(HaveKey(folder2.ID))
|
||||
|
||||
// Verify update info is populated
|
||||
Expect(results[folder1.ID].UpdatedAt).ToNot(BeZero())
|
||||
Expect(results[folder1.ID].Hash).To(Equal(folder1.Hash))
|
||||
})
|
||||
|
||||
It("handles empty folder path as root", func() {
|
||||
// Create root folder
|
||||
rootFolder := model.NewFolder(testLib, ".")
|
||||
err := repo.Put(rootFolder)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
targets := []model.LibraryPath{
|
||||
{LibraryID: testLib.ID, FolderPath: ""},
|
||||
}
|
||||
|
||||
results, err := repo.GetByPaths(targets)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(results).To(HaveLen(1))
|
||||
Expect(results).To(HaveKey(rootFolder.ID))
|
||||
})
|
||||
|
||||
It("returns empty map for non-existent folders", func() {
|
||||
targets := []model.LibraryPath{
|
||||
{LibraryID: testLib.ID, FolderPath: "NonExistent/Path"},
|
||||
}
|
||||
|
||||
results, err := repo.GetByPaths(targets)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(results).To(BeEmpty())
|
||||
})
|
||||
|
||||
It("skips missing folders", func() {
|
||||
// Create a folder and mark it as missing
|
||||
folder := model.NewFolder(testLib, "Music/Missing")
|
||||
folder.Missing = true
|
||||
err := repo.Put(folder)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
targets := []model.LibraryPath{
|
||||
{LibraryID: testLib.ID, FolderPath: "Music/Missing"},
|
||||
}
|
||||
|
||||
results, err := repo.GetByPaths(targets)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(results).To(BeEmpty())
|
||||
})
|
||||
})
|
||||
|
||||
Context("with invalid library IDs", func() {
|
||||
It("returns empty map for non-existent library", func() {
|
||||
targets := []model.LibraryPath{
|
||||
{LibraryID: 99999, FolderPath: "Music"},
|
||||
}
|
||||
|
||||
results, err := repo.GetByPaths(targets)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(results).To(BeEmpty())
|
||||
})
|
||||
})
|
||||
|
||||
Context("with empty targets", func() {
|
||||
It("returns empty map", func() {
|
||||
results, err := repo.GetByPaths([]model.LibraryPath{})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(results).To(BeEmpty())
|
||||
})
|
||||
|
||||
It("returns empty map for nil targets", func() {
|
||||
results, err := repo.GetByPaths(nil)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(results).To(BeEmpty())
|
||||
})
|
||||
})
|
||||
|
||||
Context("with multiple paths in same library", func() {
|
||||
It("returns multiple folders", func() {
|
||||
// Create multiple folders in the same library
|
||||
folder1 := model.NewFolder(testLib, "Music/Pop")
|
||||
folder2 := model.NewFolder(testLib, "Music/Electronic")
|
||||
folder3 := model.NewFolder(testLib, "Podcasts")
|
||||
|
||||
err := repo.Put(folder1)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
err = repo.Put(folder2)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
err = repo.Put(folder3)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
// Query multiple paths
|
||||
targets := []model.LibraryPath{
|
||||
{LibraryID: testLib.ID, FolderPath: "Music/Pop"},
|
||||
{LibraryID: testLib.ID, FolderPath: "Music/Electronic"},
|
||||
{LibraryID: testLib.ID, FolderPath: "Podcasts"},
|
||||
}
|
||||
|
||||
results, err := repo.GetByPaths(targets)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(results).To(HaveLen(3))
|
||||
Expect(results).To(HaveKey(folder1.ID))
|
||||
Expect(results).To(HaveKey(folder2.ID))
|
||||
Expect(results).To(HaveKey(folder3.ID))
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
@ -157,7 +157,7 @@ func (s *SQLStore) WithTxImmediate(block func(tx model.DataStore) error, scope .
|
||||
}, scope...)
|
||||
}
|
||||
|
||||
func (s *SQLStore) GC(ctx context.Context) error {
|
||||
func (s *SQLStore) GC(ctx context.Context, libraryIDs ...int) error {
|
||||
trace := func(ctx context.Context, msg string, f func() error) func() error {
|
||||
return func() error {
|
||||
start := time.Now()
|
||||
@ -167,6 +167,12 @@ func (s *SQLStore) GC(ctx context.Context) error {
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: Implement library-specific filtering for GC operations
|
||||
// For now, GC runs globally even in selective scans
|
||||
if len(libraryIDs) > 0 {
|
||||
log.Debug(ctx, "GC: Running with library filter", "libraries", libraryIDs)
|
||||
}
|
||||
|
||||
err := run.Sequentially(
|
||||
trace(ctx, "purge empty albums", func() error { return s.Album(ctx).(*albumRepository).purgeEmpty() }),
|
||||
trace(ctx, "purge empty artists", func() error { return s.Artist(ctx).(*artistRepository).purgeEmpty() }),
|
||||
|
||||
@ -26,9 +26,18 @@ var (
|
||||
ErrAlreadyScanning = errors.New("already scanning")
|
||||
)
|
||||
|
||||
// ScanTarget represents a specific folder within a library to be scanned.
|
||||
type ScanTarget struct {
|
||||
LibraryID int
|
||||
FolderPath string // Relative path within the library, or "" for entire library
|
||||
}
|
||||
|
||||
type Scanner interface {
|
||||
// ScanAll starts a full scan of the music library. This is a blocking operation.
|
||||
ScanAll(ctx context.Context, fullScan bool) (warnings []string, err error)
|
||||
// ScanFolders scans specific library/folder pairs without recursing into subdirectories.
|
||||
// This is a blocking operation.
|
||||
ScanFolders(ctx context.Context, fullScan bool, targets []ScanTarget) (warnings []string, err error)
|
||||
Status(context.Context) (*StatusInfo, error)
|
||||
}
|
||||
|
||||
@ -68,6 +77,12 @@ func (s *controller) getScanner() scanner {
|
||||
// CallScan starts an in-process scan of the music library.
|
||||
// This is meant to be called from the command line (see cmd/scan.go).
|
||||
func CallScan(ctx context.Context, ds model.DataStore, pls core.Playlists, fullScan bool) (<-chan *ProgressInfo, error) {
|
||||
return CallScanFolders(ctx, ds, pls, fullScan, nil)
|
||||
}
|
||||
|
||||
// CallScanFolders starts an in-process scan of specific library/folder pairs.
|
||||
// If targets is nil, it scans all libraries. This is meant to be called from the command line.
|
||||
func CallScanFolders(ctx context.Context, ds model.DataStore, pls core.Playlists, fullScan bool, targets []ScanTarget) (<-chan *ProgressInfo, error) {
|
||||
release, err := lockScan(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@ -79,7 +94,11 @@ func CallScan(ctx context.Context, ds model.DataStore, pls core.Playlists, fullS
|
||||
go func() {
|
||||
defer close(progress)
|
||||
scanner := &scannerImpl{ds: ds, cw: artwork.NoopCacheWarmer(), pls: pls}
|
||||
scanner.scanAll(ctx, fullScan, progress)
|
||||
if targets == nil {
|
||||
scanner.scanAll(ctx, fullScan, progress)
|
||||
} else {
|
||||
scanner.scanFolders(ctx, fullScan, targets, progress)
|
||||
}
|
||||
}()
|
||||
return progress, nil
|
||||
}
|
||||
@ -101,6 +120,7 @@ type ProgressInfo struct {
|
||||
|
||||
type scanner interface {
|
||||
scanAll(ctx context.Context, fullScan bool, progress chan<- *ProgressInfo)
|
||||
scanFolders(ctx context.Context, fullScan bool, targets []ScanTarget, progress chan<- *ProgressInfo)
|
||||
}
|
||||
|
||||
type controller struct {
|
||||
@ -208,6 +228,10 @@ func (s *controller) getCounters(ctx context.Context) (int64, int64, error) {
|
||||
}
|
||||
|
||||
func (s *controller) ScanAll(requestCtx context.Context, fullScan bool) ([]string, error) {
|
||||
return s.ScanFolders(requestCtx, fullScan, nil)
|
||||
}
|
||||
|
||||
func (s *controller) ScanFolders(requestCtx context.Context, fullScan bool, targets []ScanTarget) ([]string, error) {
|
||||
release, err := lockScan(requestCtx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@ -224,7 +248,11 @@ func (s *controller) ScanAll(requestCtx context.Context, fullScan bool) ([]strin
|
||||
go func() {
|
||||
defer close(progress)
|
||||
scanner := s.getScanner()
|
||||
scanner.scanAll(ctx, fullScan, progress)
|
||||
if targets == nil {
|
||||
scanner.scanAll(ctx, fullScan, progress)
|
||||
} else {
|
||||
scanner.scanFolders(ctx, fullScan, targets, progress)
|
||||
}
|
||||
}()
|
||||
|
||||
// Wait for the scan to finish, sending progress events to all connected clients
|
||||
|
||||
@ -8,6 +8,7 @@ import (
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strconv"
|
||||
|
||||
"github.com/navidrome/navidrome/conf"
|
||||
"github.com/navidrome/navidrome/log"
|
||||
@ -62,6 +63,56 @@ func (s *scannerExternal) scanAll(ctx context.Context, fullScan bool, progress c
|
||||
}
|
||||
}
|
||||
|
||||
func (s *scannerExternal) scanFolders(ctx context.Context, fullScan bool, targets []ScanTarget, progress chan<- *ProgressInfo) {
|
||||
exe, err := os.Executable()
|
||||
if err != nil {
|
||||
progress <- &ProgressInfo{Error: fmt.Sprintf("failed to get executable path: %s", err)}
|
||||
return
|
||||
}
|
||||
|
||||
// Build targets string for CLI
|
||||
var targetsStr string
|
||||
for i, target := range targets {
|
||||
if i > 0 {
|
||||
targetsStr += ","
|
||||
}
|
||||
targetsStr += strconv.Itoa(target.LibraryID) + ":" + target.FolderPath
|
||||
}
|
||||
|
||||
log.Debug(ctx, "Spawning external scanner process with targets", "fullScan", fullScan, "path", exe, "targets", targetsStr)
|
||||
cmd := exec.CommandContext(ctx, exe, "scan",
|
||||
"--nobanner", "--subprocess",
|
||||
"--configfile", conf.Server.ConfigFile,
|
||||
"--datafolder", conf.Server.DataFolder,
|
||||
"--cachefolder", conf.Server.CacheFolder,
|
||||
"--targets", targetsStr,
|
||||
If(fullScan, "--full", ""))
|
||||
|
||||
in, out := io.Pipe()
|
||||
defer in.Close()
|
||||
defer out.Close()
|
||||
cmd.Stdout = out
|
||||
cmd.Stderr = os.Stderr
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
progress <- &ProgressInfo{Error: fmt.Sprintf("failed to start scanner process: %s", err)}
|
||||
return
|
||||
}
|
||||
go s.wait(cmd, out)
|
||||
|
||||
decoder := gob.NewDecoder(in)
|
||||
for {
|
||||
var p ProgressInfo
|
||||
if err := decoder.Decode(&p); err != nil {
|
||||
if !errors.Is(err, io.EOF) {
|
||||
progress <- &ProgressInfo{Error: fmt.Sprintf("failed to read status from scanner: %s", err)}
|
||||
}
|
||||
break
|
||||
}
|
||||
progress <- &p
|
||||
}
|
||||
}
|
||||
|
||||
func (s *scannerExternal) wait(cmd *exec.Cmd, out *io.PipeWriter) {
|
||||
if err := cmd.Wait(); err != nil {
|
||||
var exitErr *exec.ExitError
|
||||
|
||||
@ -48,7 +48,14 @@ func createPhaseFolders(ctx context.Context, state *scanState, ds model.DataStor
|
||||
} else {
|
||||
log.Debug(ctx, "Scanner: Resuming previous scan", "lib", lib.Name, "lastScanStartedAt", lib.LastScanStartedAt, "fullScan", lib.FullScanInProgress)
|
||||
}
|
||||
job, err := newScanJob(ctx, ds, cw, lib, state.fullScan)
|
||||
|
||||
// Get target folders for this library if selective scan
|
||||
var targetFolders []string
|
||||
if state.targets != nil {
|
||||
targetFolders = state.targets[lib.ID]
|
||||
}
|
||||
|
||||
job, err := newScanJob(ctx, ds, cw, lib, state.fullScan, targetFolders)
|
||||
if err != nil {
|
||||
log.Error(ctx, "Scanner: Error creating scan context", "lib", lib.Name, err)
|
||||
state.sendWarning(err.Error())
|
||||
@ -65,19 +72,36 @@ func createPhaseFolders(ctx context.Context, state *scanState, ds model.DataStor
|
||||
}
|
||||
|
||||
type scanJob struct {
|
||||
lib model.Library
|
||||
fs storage.MusicFS
|
||||
cw artwork.CacheWarmer
|
||||
lastUpdates map[string]model.FolderUpdateInfo
|
||||
lock sync.Mutex
|
||||
numFolders atomic.Int64
|
||||
lib model.Library
|
||||
fs storage.MusicFS
|
||||
cw artwork.CacheWarmer
|
||||
lastUpdates map[string]model.FolderUpdateInfo
|
||||
targetFolders []string // Optional: specific folders to scan (non-recursive)
|
||||
lock sync.Mutex
|
||||
numFolders atomic.Int64
|
||||
}
|
||||
|
||||
func newScanJob(ctx context.Context, ds model.DataStore, cw artwork.CacheWarmer, lib model.Library, fullScan bool) (*scanJob, error) {
|
||||
lastUpdates, err := ds.Folder(ctx).GetLastUpdates(lib)
|
||||
func newScanJob(ctx context.Context, ds model.DataStore, cw artwork.CacheWarmer, lib model.Library, fullScan bool, targetFolders []string) (*scanJob, error) {
|
||||
var lastUpdates map[string]model.FolderUpdateInfo
|
||||
var err error
|
||||
|
||||
// If we have target folders, get only those folder updates. Otherwise get all updates for the library
|
||||
if len(targetFolders) > 0 {
|
||||
var targets []model.LibraryPath
|
||||
for _, folderPath := range targetFolders {
|
||||
targets = append(targets, model.LibraryPath{
|
||||
LibraryID: lib.ID,
|
||||
FolderPath: folderPath,
|
||||
})
|
||||
}
|
||||
lastUpdates, err = ds.Folder(ctx).GetByPaths(targets)
|
||||
} else {
|
||||
lastUpdates, err = ds.Folder(ctx).GetLastUpdates(lib)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("getting last updates: %w", err)
|
||||
}
|
||||
|
||||
fileStore, err := storage.For(lib.Path)
|
||||
if err != nil {
|
||||
log.Error(ctx, "Error getting storage for library", "library", lib.Name, "path", lib.Path, err)
|
||||
@ -90,10 +114,11 @@ func newScanJob(ctx context.Context, ds model.DataStore, cw artwork.CacheWarmer,
|
||||
}
|
||||
lib.FullScanInProgress = lib.FullScanInProgress || fullScan
|
||||
return &scanJob{
|
||||
lib: lib,
|
||||
fs: fsys,
|
||||
cw: cw,
|
||||
lastUpdates: lastUpdates,
|
||||
lib: lib,
|
||||
fs: fsys,
|
||||
cw: cw,
|
||||
lastUpdates: lastUpdates,
|
||||
targetFolders: targetFolders,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@ -144,7 +169,18 @@ func (p *phaseFolders) producer() ppl.Producer[*folderEntry] {
|
||||
if utils.IsCtxDone(p.ctx) {
|
||||
break
|
||||
}
|
||||
outputChan, err := walkDirTree(p.ctx, job)
|
||||
|
||||
var outputChan <-chan *folderEntry
|
||||
var err error
|
||||
|
||||
// Use selective folder loading if target folders are specified
|
||||
if len(job.targetFolders) > 0 {
|
||||
log.Debug(p.ctx, "Scanner: Loading specific folders only (non-recursive)", "lib", job.lib.Name, "numTargets", len(job.targetFolders))
|
||||
outputChan, err = loadSpecificFolders(p.ctx, job, job.targetFolders)
|
||||
} else {
|
||||
outputChan, err = walkDirTree(p.ctx, job)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
log.Warn(p.ctx, "Scanner: Error scanning library", "lib", job.lib.Name, err)
|
||||
}
|
||||
|
||||
@ -28,7 +28,9 @@ type scanState struct {
|
||||
progress chan<- *ProgressInfo
|
||||
fullScan bool
|
||||
changesDetected atomic.Bool
|
||||
libraries model.Libraries // Store libraries list for consistency across phases
|
||||
libraries model.Libraries // Store libraries list for consistency across phases
|
||||
targets map[int][]string // Optional: map[libraryID][]folderPaths for selective scans
|
||||
affectedLibIDs []int // IDs of libraries involved in the scan (for GC scoping)
|
||||
}
|
||||
|
||||
func (s *scanState) sendProgress(info *ProgressInfo) {
|
||||
@ -134,13 +136,123 @@ func (s *scannerImpl) scanAll(ctx context.Context, fullScan bool, progress chan<
|
||||
log.Info(ctx, "Scanner: Finished scanning all libraries", "duration", time.Since(startTime))
|
||||
}
|
||||
|
||||
func (s *scannerImpl) scanFolders(ctx context.Context, fullScan bool, targets []ScanTarget, progress chan<- *ProgressInfo) {
|
||||
startTime := time.Now()
|
||||
|
||||
state := scanState{
|
||||
progress: progress,
|
||||
fullScan: fullScan,
|
||||
changesDetected: atomic.Bool{},
|
||||
targets: make(map[int][]string),
|
||||
}
|
||||
|
||||
// Set changesDetected to true for full scans to ensure all maintenance operations run
|
||||
if fullScan {
|
||||
state.changesDetected.Store(true)
|
||||
}
|
||||
|
||||
// Group targets by library and collect affected library IDs
|
||||
affectedLibIDSet := make(map[int]bool)
|
||||
for _, target := range targets {
|
||||
folderPath := target.FolderPath
|
||||
if folderPath == "" {
|
||||
folderPath = "."
|
||||
}
|
||||
state.targets[target.LibraryID] = append(state.targets[target.LibraryID], folderPath)
|
||||
affectedLibIDSet[target.LibraryID] = true
|
||||
}
|
||||
|
||||
// Get affected libraries
|
||||
allLibs, err := s.ds.Library(ctx).GetAll()
|
||||
if err != nil {
|
||||
state.sendWarning(fmt.Sprintf("getting libraries: %s", err))
|
||||
return
|
||||
}
|
||||
|
||||
var libs model.Libraries
|
||||
for _, lib := range allLibs {
|
||||
if affectedLibIDSet[lib.ID] {
|
||||
libs = append(libs, lib)
|
||||
state.affectedLibIDs = append(state.affectedLibIDs, lib.ID)
|
||||
}
|
||||
}
|
||||
state.libraries = libs
|
||||
|
||||
log.Info(ctx, "Scanner: Starting selective scan", "fullScan", state.fullScan, "numLibraries", len(libs), "numTargets", len(targets))
|
||||
|
||||
// Store scan type and start time
|
||||
scanType := "quick-selective"
|
||||
if state.fullScan {
|
||||
scanType = "full-selective"
|
||||
}
|
||||
_ = s.ds.Property(ctx).Put(consts.LastScanTypeKey, scanType)
|
||||
_ = s.ds.Property(ctx).Put(consts.LastScanStartTimeKey, startTime.Format(time.RFC3339))
|
||||
|
||||
// if there was a full scan in progress, force a full scan
|
||||
if !state.fullScan {
|
||||
for _, lib := range libs {
|
||||
if lib.FullScanInProgress {
|
||||
log.Info(ctx, "Scanner: Interrupted full scan detected", "lib", lib.Name)
|
||||
state.fullScan = true
|
||||
_ = s.ds.Property(ctx).Put(consts.LastScanTypeKey, "full-selective")
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
err = run.Sequentially(
|
||||
// Phase 1: Scan specified folders and import new/updated files
|
||||
runPhase[*folderEntry](ctx, 1, createPhaseFolders(ctx, &state, s.ds, s.cw, libs)),
|
||||
|
||||
// Phase 2: Process missing files in scanned folders only
|
||||
runPhase[*missingTracks](ctx, 2, createPhaseMissingTracks(ctx, &state, s.ds)),
|
||||
|
||||
// Phases 3 and 4 can be run in parallel
|
||||
run.Parallel(
|
||||
// Phase 3: Refresh all new/changed albums (from affected libraries only)
|
||||
runPhase[*model.Album](ctx, 3, createPhaseRefreshAlbums(ctx, &state, s.ds, libs)),
|
||||
|
||||
// Phase 4: Import/update playlists (from affected libraries only)
|
||||
runPhase[*model.Folder](ctx, 4, createPhasePlaylists(ctx, &state, s.ds, s.pls, s.cw)),
|
||||
),
|
||||
|
||||
// Final Steps (cannot be parallelized):
|
||||
|
||||
// Run GC scoped to affected libraries only
|
||||
s.runGC(ctx, &state),
|
||||
|
||||
// Refresh artist and tags stats
|
||||
s.runRefreshStats(ctx, &state),
|
||||
|
||||
// Update last_scan_completed_at for affected libraries
|
||||
s.runUpdateLibraries(ctx, &state),
|
||||
|
||||
// Optimize DB
|
||||
s.runOptimize(ctx),
|
||||
)
|
||||
if err != nil {
|
||||
log.Error(ctx, "Scanner: Finished with error", "duration", time.Since(startTime), err)
|
||||
_ = s.ds.Property(ctx).Put(consts.LastScanErrorKey, err.Error())
|
||||
state.sendError(err)
|
||||
return
|
||||
}
|
||||
|
||||
_ = s.ds.Property(ctx).Put(consts.LastScanErrorKey, "")
|
||||
|
||||
if state.changesDetected.Load() {
|
||||
state.sendProgress(&ProgressInfo{ChangesDetected: true})
|
||||
}
|
||||
|
||||
log.Info(ctx, "Scanner: Finished scanning selected folders", "duration", time.Since(startTime), "numTargets", len(targets))
|
||||
}
|
||||
|
||||
func (s *scannerImpl) runGC(ctx context.Context, state *scanState) func() error {
|
||||
return func() error {
|
||||
state.sendProgress(&ProgressInfo{ForceUpdate: true})
|
||||
return s.ds.WithTx(func(tx model.DataStore) error {
|
||||
if state.changesDetected.Load() {
|
||||
start := time.Now()
|
||||
err := tx.GC(ctx)
|
||||
err := tx.GC(ctx, state.affectedLibIDs...)
|
||||
if err != nil {
|
||||
log.Error(ctx, "Scanner: Error running GC", err)
|
||||
return fmt.Errorf("running GC: %w", err)
|
||||
|
||||
@ -717,6 +717,61 @@ var _ = Describe("Scanner", Ordered, func() {
|
||||
Expect(albumArtistStats.SongCount).To(Equal(3)) // 3 songs
|
||||
})
|
||||
})
|
||||
|
||||
Describe("ScanFolders", func() {
|
||||
It("scans only specified folders without recursion", func() {
|
||||
rock := template(_t{"albumartist": "Rock Artist", "album": "Rock Album"})
|
||||
jazz := template(_t{"albumartist": "Jazz Artist", "album": "Jazz Album"})
|
||||
pop := template(_t{"albumartist": "Pop Artist", "album": "Pop Album"})
|
||||
createFS(fstest.MapFS{
|
||||
"rock/track1.mp3": rock(track(1, "Rock Track 1")),
|
||||
"rock/track2.mp3": rock(track(2, "Rock Track 2")),
|
||||
"rock/subdir/track3.mp3": rock(track(3, "Rock Track 3")),
|
||||
"jazz/track4.mp3": jazz(track(1, "Jazz Track 1")),
|
||||
"jazz/subdir/track5.mp3": jazz(track(2, "Jazz Track 2")),
|
||||
"pop/track6.mp3": pop(track(1, "Pop Track 1")),
|
||||
})
|
||||
|
||||
// Use the existing library from BeforeEach
|
||||
// (lib is already created with the path "fake:///music")
|
||||
|
||||
// Scan only the "rock" and "jazz" folders (not their subdirectories or pop)
|
||||
targets := []scanner.ScanTarget{
|
||||
{LibraryID: lib.ID, FolderPath: "rock"},
|
||||
{LibraryID: lib.ID, FolderPath: "jazz"},
|
||||
}
|
||||
|
||||
warnings, err := s.ScanFolders(ctx, false, targets)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(warnings).To(BeEmpty())
|
||||
|
||||
// Verify only track1, track2, and track4 were imported (not track3, track5, or track6)
|
||||
allFiles, err := ds.MediaFile(ctx).GetAll()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
// Should have exactly 3 tracks (rock/track1, rock/track2, jazz/track4)
|
||||
Expect(allFiles).To(HaveLen(3))
|
||||
|
||||
// Get the file paths
|
||||
paths := slice.Map(allFiles, func(mf model.MediaFile) string {
|
||||
return filepath.ToSlash(mf.Path)
|
||||
})
|
||||
|
||||
// Verify the correct files were scanned
|
||||
Expect(paths).To(ContainElements(
|
||||
"rock/track1.mp3",
|
||||
"rock/track2.mp3",
|
||||
"jazz/track4.mp3",
|
||||
))
|
||||
|
||||
// Verify files in subdirectories and pop folder were NOT scanned
|
||||
Expect(paths).ToNot(ContainElements(
|
||||
"rock/subdir/track3.mp3",
|
||||
"jazz/subdir/track5.mp3",
|
||||
"pop/track6.mp3",
|
||||
))
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
func createFindByPath(ctx context.Context, ds model.DataStore) func(string) (*model.MediaFile, error) {
|
||||
|
||||
@ -32,6 +32,64 @@ func walkDirTree(ctx context.Context, job *scanJob) (<-chan *folderEntry, error)
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// loadSpecificFolders loads only the specified folders without recursing into subdirectories
|
||||
func loadSpecificFolders(ctx context.Context, job *scanJob, targetFolders []string) (<-chan *folderEntry, error) {
|
||||
results := make(chan *folderEntry)
|
||||
go func() {
|
||||
defer close(results)
|
||||
for _, folderPath := range targetFolders {
|
||||
if utils.IsCtxDone(ctx) {
|
||||
return
|
||||
}
|
||||
|
||||
// Load ignore patterns from parent directories up to this folder
|
||||
ignorePatterns := loadIgnorePatternsForPath(ctx, job.fs, folderPath)
|
||||
|
||||
// Load only this specific folder (no recursion)
|
||||
folder, _, err := loadDir(ctx, job, folderPath, ignorePatterns)
|
||||
if err != nil {
|
||||
log.Warn(ctx, "Scanner: Error loading target folder. Skipping", "path", folderPath, err)
|
||||
continue
|
||||
}
|
||||
|
||||
folder.path = path.Clean(folderPath)
|
||||
folder.elapsed.Start()
|
||||
log.Trace(ctx, "Scanner: Found target directory", " path", folder.path, "audioFiles", maps.Keys(folder.audioFiles),
|
||||
"images", maps.Keys(folder.imageFiles), "playlists", folder.numPlaylists, "imagesUpdatedAt", folder.imagesUpdatedAt,
|
||||
"updTime", folder.updTime, "modTime", folder.modTime)
|
||||
|
||||
results <- folder
|
||||
}
|
||||
log.Debug(ctx, "Scanner: Finished reading target folders", "lib", job.lib.Name, "path", job.lib.Path, "numFolders", len(targetFolders))
|
||||
}()
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// loadIgnorePatternsForPath loads all .ndignore patterns from the root down to the specified path
|
||||
func loadIgnorePatternsForPath(ctx context.Context, fsys fs.FS, targetPath string) []string {
|
||||
var patterns []string
|
||||
currentPath := "."
|
||||
|
||||
// If target is root, just check root
|
||||
if targetPath == "." {
|
||||
return loadIgnoredPatterns(ctx, fsys, ".", nil)
|
||||
}
|
||||
|
||||
// Walk from root to target, collecting ignore patterns
|
||||
parts := strings.Split(path.Clean(targetPath), "/")
|
||||
for _, part := range parts {
|
||||
if part == "." {
|
||||
continue
|
||||
}
|
||||
patterns = loadIgnoredPatterns(ctx, fsys, currentPath, patterns)
|
||||
currentPath = path.Join(currentPath, part)
|
||||
}
|
||||
// Load patterns from the target folder itself
|
||||
patterns = loadIgnoredPatterns(ctx, fsys, currentPath, patterns)
|
||||
|
||||
return patterns
|
||||
}
|
||||
|
||||
func walkFolder(ctx context.Context, job *scanJob, currentFolder string, ignorePatterns []string, results chan<- *folderEntry) error {
|
||||
ignorePatterns = loadIgnoredPatterns(ctx, job.fs, currentFolder, ignorePatterns)
|
||||
|
||||
|
||||
@ -258,6 +258,6 @@ func (db *MockDataStore) Resource(ctx context.Context, m any) model.ResourceRepo
|
||||
}
|
||||
}
|
||||
|
||||
func (db *MockDataStore) GC(context.Context) error {
|
||||
func (db *MockDataStore) GC(context.Context, ...int) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user