navidrome/server/subsonic/library_scanning_test.go
Deluan Quintão 28d5299ffc
feat(scanner): implement selective folder scanning and file system watcher improvements (#4674)
* feat: Add selective folder scanning capability

Implement targeted scanning of specific library/folder pairs without
full recursion. This enables efficient rescanning of individual folders
when changes are detected, significantly reducing scan time for large
libraries.

Key changes:
- Add ScanTarget struct and ScanFolders API to Scanner interface
- Implement CLI flag --targets for specifying libraryID:folderPath pairs
- Add FolderRepository.GetByPaths() for batch folder info retrieval
- Create loadSpecificFolders() for non-recursive directory loading
- Scope GC operations to affected libraries only (with TODO for full impl)
- Add comprehensive tests for selective scanning behavior

The selective scan:
- Only processes specified folders (no subdirectory recursion)
- Maintains library isolation
- Runs full maintenance pipeline scoped to affected libraries
- Supports both full and quick scan modes

Examples:
  navidrome scan --targets "1:Music/Rock,1:Music/Jazz"
  navidrome scan --full --targets "2:Classical"

* feat(folder): replace GetByPaths with GetFolderUpdateInfo for improved folder updates retrieval

Signed-off-by: Deluan <deluan@navidrome.org>

* test: update parseTargets test to handle folder names with spaces

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor(folder): remove unused LibraryPath struct and update GC logging message

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor(folder): enhance external scanner to support target-specific scanning

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor(scanner): simplify scanner methods

Signed-off-by: Deluan <deluan@navidrome.org>

* feat(watcher): implement folder scanning notifications with deduplication

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor(watcher): add resolveFolderPath function for testability

Signed-off-by: Deluan <deluan@navidrome.org>

* feat(watcher): implement path ignoring based on .ndignore patterns

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor(scanner): implement IgnoreChecker for managing .ndignore patterns

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor(ignore_checker): rename scanner to lineScanner for clarity

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor(scanner): enhance ScanTarget struct with String method for better target representation

Signed-off-by: Deluan <deluan@navidrome.org>

* fix(scanner): validate library ID to prevent negative values

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor(scanner): simplify GC method by removing library ID parameter

Signed-off-by: Deluan <deluan@navidrome.org>

* feat(scanner): update folder scanning to include all descendants of specified folders

Signed-off-by: Deluan <deluan@navidrome.org>

* feat(subsonic): allow selective scan in the /startScan endpoint

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor(scanner): update CallScan to handle specific library/folder pairs

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor(scanner): streamline scanning logic by removing scanAll method

Signed-off-by: Deluan <deluan@navidrome.org>

* test: enhance mockScanner for thread safety and improve test reliability

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor(scanner): move scanner.ScanTarget to model.ScanTarget

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor: move scanner types to model,implement MockScanner

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor(scanner): update scanner interface and implementations to use model.Scanner

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor(folder_repository): normalize target path handling by using filepath.Clean

Signed-off-by: Deluan <deluan@navidrome.org>

* test(folder_repository): add comprehensive tests for folder retrieval and child exclusion

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor(scanner): simplify selective scan logic using slice.Filter

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor(scanner): streamline phase folder and album creation by removing unnecessary library parameter

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor(scanner): move initialization logic from phase_1 to the scanner itself

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor(tests): rename selective scan test file to scanner_selective_test.go

Signed-off-by: Deluan <deluan@navidrome.org>

* feat(configuration): add DevSelectiveWatcher configuration option

Signed-off-by: Deluan <deluan@navidrome.org>

* feat(watcher): enhance .ndignore handling for folder deletions and file changes

Signed-off-by: Deluan <deluan@navidrome.org>

* docs(scanner): comments

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor(scanner): enhance walkDirTree to support target folder scanning

Signed-off-by: Deluan <deluan@navidrome.org>

* fix(scanner, watcher): handle errors when pushing ignore patterns for folders

Signed-off-by: Deluan <deluan@navidrome.org>

* Update scanner/phase_1_folders.go

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* refactor(scanner): replace parseTargets function with direct call to scanner.ParseTargets

Signed-off-by: Deluan <deluan@navidrome.org>

* test(scanner): add tests for ScanBegin and ScanEnd functionality

Signed-off-by: Deluan <deluan@navidrome.org>

* fix(library): update PRAGMA optimize to check table sizes without ANALYZE

Signed-off-by: Deluan <deluan@navidrome.org>

* test(scanner): refactor tests

Signed-off-by: Deluan <deluan@navidrome.org>

* feat(ui): add selective scan options and update translations

Signed-off-by: Deluan <deluan@navidrome.org>

* feat(ui): add quick and full scan options for individual libraries

Signed-off-by: Deluan <deluan@navidrome.org>

* feat(ui): add Scan buttonsto the LibraryList

Signed-off-by: Deluan <deluan@navidrome.org>

* feat(scan): update scanning parameters from 'path' to 'target' for selective scans.

* refactor(scan): move ParseTargets function to model package

* test(scan): suppress unused return value from SetUserLibraries in tests

* feat(gc): enhance garbage collection to support selective library purging

Signed-off-by: Deluan <deluan@navidrome.org>

* fix(scanner): prevent race condition when scanning deleted folders

When the watcher detects changes in a folder that gets deleted before
the scanner runs (due to the 10-second delay), the scanner was
prematurely removing these folders from the tracking map, preventing
them from being marked as missing.

The issue occurred because `newFolderEntry` was calling `popLastUpdate`
before verifying the folder actually exists on the filesystem.

Changes:
- Move fs.Stat check before newFolderEntry creation in loadDir to
  ensure deleted folders remain in lastUpdates for finalize() to handle
- Add early existence check in walkDirTree to skip non-existent target
  folders with a warning log
- Add unit test verifying non-existent folders aren't removed from
  lastUpdates prematurely
- Add integration test for deleted folder scenario with ScanFolders

Fixes the issue where deleting entire folders (e.g., /music/AC_DC)
wouldn't mark tracks as missing when using selective folder scanning.

* refactor(scan): streamline folder entry creation and update handling

Signed-off-by: Deluan <deluan@navidrome.org>

* feat(scan): add '@Recycle' (QNAP) to ignored directories list

Signed-off-by: Deluan <deluan@navidrome.org>

* fix(log): improve thread safety in logging level management

* test(scan): move unit tests for ParseTargets function

Signed-off-by: Deluan <deluan@navidrome.org>

* review

Signed-off-by: Deluan <deluan@navidrome.org>

---------

Signed-off-by: Deluan <deluan@navidrome.org>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: deluan <deluan.quintao@mechanical-orchard.com>
2025-11-14 22:15:43 -05:00

397 lines
11 KiB
Go

package subsonic
import (
"context"
"errors"
"net/http/httptest"
"github.com/navidrome/navidrome/model"
"github.com/navidrome/navidrome/model/request"
"github.com/navidrome/navidrome/server/subsonic/responses"
"github.com/navidrome/navidrome/tests"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
var _ = Describe("LibraryScanning", func() {
var api *Router
var ms *tests.MockScanner
BeforeEach(func() {
ms = tests.NewMockScanner()
api = &Router{scanner: ms}
})
Describe("StartScan", func() {
It("requires admin authentication", func() {
// Create non-admin user
ctx := request.WithUser(context.Background(), model.User{
ID: "user-id",
IsAdmin: false,
})
// Create request
r := httptest.NewRequest("GET", "/rest/startScan", nil)
r = r.WithContext(ctx)
// Call endpoint
response, err := api.StartScan(r)
// Should return authorization error
Expect(err).To(HaveOccurred())
Expect(response).To(BeNil())
var subErr subError
ok := errors.As(err, &subErr)
Expect(ok).To(BeTrue())
Expect(subErr.code).To(Equal(responses.ErrorAuthorizationFail))
})
It("triggers a full scan with no parameters", func() {
// Create admin user
ctx := request.WithUser(context.Background(), model.User{
ID: "admin-id",
IsAdmin: true,
})
// Create request with no parameters
r := httptest.NewRequest("GET", "/rest/startScan", nil)
r = r.WithContext(ctx)
// Call endpoint
response, err := api.StartScan(r)
// Should succeed
Expect(err).ToNot(HaveOccurred())
Expect(response).ToNot(BeNil())
// Verify ScanAll was called (eventually, since it's in a goroutine)
Eventually(func() int {
return ms.GetScanAllCallCount()
}).Should(BeNumerically(">", 0))
calls := ms.GetScanAllCalls()
Expect(calls).To(HaveLen(1))
Expect(calls[0].FullScan).To(BeFalse())
})
It("triggers a full scan with fullScan=true", func() {
// Create admin user
ctx := request.WithUser(context.Background(), model.User{
ID: "admin-id",
IsAdmin: true,
})
// Create request with fullScan parameter
r := httptest.NewRequest("GET", "/rest/startScan?fullScan=true", nil)
r = r.WithContext(ctx)
// Call endpoint
response, err := api.StartScan(r)
// Should succeed
Expect(err).ToNot(HaveOccurred())
Expect(response).ToNot(BeNil())
// Verify ScanAll was called with fullScan=true
Eventually(func() int {
return ms.GetScanAllCallCount()
}).Should(BeNumerically(">", 0))
calls := ms.GetScanAllCalls()
Expect(calls).To(HaveLen(1))
Expect(calls[0].FullScan).To(BeTrue())
})
It("triggers a selective scan with single target parameter", func() {
// Setup mocks
mockUserRepo := tests.CreateMockUserRepo()
_ = mockUserRepo.SetUserLibraries("admin-id", []int{1, 2})
mockDS := &tests.MockDataStore{MockedUser: mockUserRepo}
api.ds = mockDS
// Create admin user
ctx := request.WithUser(context.Background(), model.User{
ID: "admin-id",
IsAdmin: true,
})
// Create request with single target parameter
r := httptest.NewRequest("GET", "/rest/startScan?target=1:Music/Rock", nil)
r = r.WithContext(ctx)
// Call endpoint
response, err := api.StartScan(r)
// Should succeed
Expect(err).ToNot(HaveOccurred())
Expect(response).ToNot(BeNil())
// Verify ScanFolders was called with correct targets
Eventually(func() int {
return ms.GetScanFoldersCallCount()
}).Should(BeNumerically(">", 0))
calls := ms.GetScanFoldersCalls()
Expect(calls).To(HaveLen(1))
targets := calls[0].Targets
Expect(targets).To(HaveLen(1))
Expect(targets[0].LibraryID).To(Equal(1))
Expect(targets[0].FolderPath).To(Equal("Music/Rock"))
})
It("triggers a selective scan with multiple target parameters", func() {
// Setup mocks
mockUserRepo := tests.CreateMockUserRepo()
_ = mockUserRepo.SetUserLibraries("admin-id", []int{1, 2})
mockDS := &tests.MockDataStore{MockedUser: mockUserRepo}
api.ds = mockDS
// Create admin user
ctx := request.WithUser(context.Background(), model.User{
ID: "admin-id",
IsAdmin: true,
})
// Create request with multiple target parameters
r := httptest.NewRequest("GET", "/rest/startScan?target=1:Music/Reggae&target=2:Classical/Bach", nil)
r = r.WithContext(ctx)
// Call endpoint
response, err := api.StartScan(r)
// Should succeed
Expect(err).ToNot(HaveOccurred())
Expect(response).ToNot(BeNil())
// Verify ScanFolders was called with correct targets
Eventually(func() int {
return ms.GetScanFoldersCallCount()
}).Should(BeNumerically(">", 0))
calls := ms.GetScanFoldersCalls()
Expect(calls).To(HaveLen(1))
targets := calls[0].Targets
Expect(targets).To(HaveLen(2))
Expect(targets[0].LibraryID).To(Equal(1))
Expect(targets[0].FolderPath).To(Equal("Music/Reggae"))
Expect(targets[1].LibraryID).To(Equal(2))
Expect(targets[1].FolderPath).To(Equal("Classical/Bach"))
})
It("triggers a selective full scan with target and fullScan parameters", func() {
// Setup mocks
mockUserRepo := tests.CreateMockUserRepo()
_ = mockUserRepo.SetUserLibraries("admin-id", []int{1})
mockDS := &tests.MockDataStore{MockedUser: mockUserRepo}
api.ds = mockDS
// Create admin user
ctx := request.WithUser(context.Background(), model.User{
ID: "admin-id",
IsAdmin: true,
})
// Create request with target and fullScan parameters
r := httptest.NewRequest("GET", "/rest/startScan?target=1:Music/Jazz&fullScan=true", nil)
r = r.WithContext(ctx)
// Call endpoint
response, err := api.StartScan(r)
// Should succeed
Expect(err).ToNot(HaveOccurred())
Expect(response).ToNot(BeNil())
// Verify ScanFolders was called with fullScan=true
Eventually(func() int {
return ms.GetScanFoldersCallCount()
}).Should(BeNumerically(">", 0))
calls := ms.GetScanFoldersCalls()
Expect(calls).To(HaveLen(1))
Expect(calls[0].FullScan).To(BeTrue())
targets := calls[0].Targets
Expect(targets).To(HaveLen(1))
})
It("returns error for invalid target format", func() {
// Create admin user
ctx := request.WithUser(context.Background(), model.User{
ID: "admin-id",
IsAdmin: true,
})
// Create request with invalid target format (missing colon)
r := httptest.NewRequest("GET", "/rest/startScan?target=1MusicRock", nil)
r = r.WithContext(ctx)
// Call endpoint
response, err := api.StartScan(r)
// Should return error
Expect(err).To(HaveOccurred())
Expect(response).To(BeNil())
var subErr subError
ok := errors.As(err, &subErr)
Expect(ok).To(BeTrue())
Expect(subErr.code).To(Equal(responses.ErrorGeneric))
})
It("returns error for invalid library ID in target", func() {
// Create admin user
ctx := request.WithUser(context.Background(), model.User{
ID: "admin-id",
IsAdmin: true,
})
// Create request with invalid library ID
r := httptest.NewRequest("GET", "/rest/startScan?target=0:Music/Rock", nil)
r = r.WithContext(ctx)
// Call endpoint
response, err := api.StartScan(r)
// Should return error
Expect(err).To(HaveOccurred())
Expect(response).To(BeNil())
var subErr subError
ok := errors.As(err, &subErr)
Expect(ok).To(BeTrue())
Expect(subErr.code).To(Equal(responses.ErrorGeneric))
})
It("returns error when library does not exist", func() {
// Setup mocks - user has access to library 1 and 2 only
mockUserRepo := tests.CreateMockUserRepo()
_ = mockUserRepo.SetUserLibraries("admin-id", []int{1, 2})
mockDS := &tests.MockDataStore{MockedUser: mockUserRepo}
api.ds = mockDS
// Create admin user
ctx := request.WithUser(context.Background(), model.User{
ID: "admin-id",
IsAdmin: true,
})
// Create request with library ID that doesn't exist
r := httptest.NewRequest("GET", "/rest/startScan?target=999:Music/Rock", nil)
r = r.WithContext(ctx)
// Call endpoint
response, err := api.StartScan(r)
// Should return ErrorDataNotFound
Expect(err).To(HaveOccurred())
Expect(response).To(BeNil())
var subErr subError
ok := errors.As(err, &subErr)
Expect(ok).To(BeTrue())
Expect(subErr.code).To(Equal(responses.ErrorDataNotFound))
})
It("calls ScanAll when single library with empty path and only one library exists", func() {
// Setup mocks - single library in DB
mockUserRepo := tests.CreateMockUserRepo()
_ = mockUserRepo.SetUserLibraries("admin-id", []int{1})
mockLibraryRepo := &tests.MockLibraryRepo{}
mockLibraryRepo.SetData(model.Libraries{
{ID: 1, Name: "Music Library", Path: "/music"},
})
mockDS := &tests.MockDataStore{
MockedUser: mockUserRepo,
MockedLibrary: mockLibraryRepo,
}
api.ds = mockDS
// Create admin user
ctx := request.WithUser(context.Background(), model.User{
ID: "admin-id",
IsAdmin: true,
})
// Create request with single library and empty path
r := httptest.NewRequest("GET", "/rest/startScan?target=1:", nil)
r = r.WithContext(ctx)
// Call endpoint
response, err := api.StartScan(r)
// Should succeed
Expect(err).ToNot(HaveOccurred())
Expect(response).ToNot(BeNil())
// Verify ScanAll was called instead of ScanFolders
Eventually(func() int {
return ms.GetScanAllCallCount()
}).Should(BeNumerically(">", 0))
Expect(ms.GetScanFoldersCallCount()).To(Equal(0))
})
It("calls ScanFolders when single library with empty path but multiple libraries exist", func() {
// Setup mocks - multiple libraries in DB
mockUserRepo := tests.CreateMockUserRepo()
_ = mockUserRepo.SetUserLibraries("admin-id", []int{1, 2})
mockLibraryRepo := &tests.MockLibraryRepo{}
mockLibraryRepo.SetData(model.Libraries{
{ID: 1, Name: "Music Library", Path: "/music"},
{ID: 2, Name: "Audiobooks", Path: "/audiobooks"},
})
mockDS := &tests.MockDataStore{
MockedUser: mockUserRepo,
MockedLibrary: mockLibraryRepo,
}
api.ds = mockDS
// Create admin user
ctx := request.WithUser(context.Background(), model.User{
ID: "admin-id",
IsAdmin: true,
})
// Create request with single library and empty path
r := httptest.NewRequest("GET", "/rest/startScan?target=1:", nil)
r = r.WithContext(ctx)
// Call endpoint
response, err := api.StartScan(r)
// Should succeed
Expect(err).ToNot(HaveOccurred())
Expect(response).ToNot(BeNil())
// Verify ScanFolders was called (not ScanAll)
Eventually(func() int {
return ms.GetScanFoldersCallCount()
}).Should(BeNumerically(">", 0))
calls := ms.GetScanFoldersCalls()
Expect(calls).To(HaveLen(1))
targets := calls[0].Targets
Expect(targets).To(HaveLen(1))
Expect(targets[0].LibraryID).To(Equal(1))
Expect(targets[0].FolderPath).To(Equal(""))
})
})
Describe("GetScanStatus", func() {
It("returns scan status", func() {
// Setup mock scanner status
ms.SetStatusResponse(&model.ScannerStatus{
Scanning: false,
Count: 100,
FolderCount: 10,
})
// Create request
ctx := context.Background()
r := httptest.NewRequest("GET", "/rest/getScanStatus", nil)
r = r.WithContext(ctx)
// Call endpoint
response, err := api.GetScanStatus(r)
// Should succeed
Expect(err).ToNot(HaveOccurred())
Expect(response).ToNot(BeNil())
Expect(response.ScanStatus).ToNot(BeNil())
Expect(response.ScanStatus.Scanning).To(BeFalse())
Expect(response.ScanStatus.Count).To(Equal(int64(100)))
Expect(response.ScanStatus.FolderCount).To(Equal(int64(10)))
})
})
})