mirror of
https://github.com/navidrome/navidrome.git
synced 2026-03-04 06:35:52 +00:00
* feat: Add selective folder scanning capability Implement targeted scanning of specific library/folder pairs without full recursion. This enables efficient rescanning of individual folders when changes are detected, significantly reducing scan time for large libraries. Key changes: - Add ScanTarget struct and ScanFolders API to Scanner interface - Implement CLI flag --targets for specifying libraryID:folderPath pairs - Add FolderRepository.GetByPaths() for batch folder info retrieval - Create loadSpecificFolders() for non-recursive directory loading - Scope GC operations to affected libraries only (with TODO for full impl) - Add comprehensive tests for selective scanning behavior The selective scan: - Only processes specified folders (no subdirectory recursion) - Maintains library isolation - Runs full maintenance pipeline scoped to affected libraries - Supports both full and quick scan modes Examples: navidrome scan --targets "1:Music/Rock,1:Music/Jazz" navidrome scan --full --targets "2:Classical" * feat(folder): replace GetByPaths with GetFolderUpdateInfo for improved folder updates retrieval Signed-off-by: Deluan <deluan@navidrome.org> * test: update parseTargets test to handle folder names with spaces Signed-off-by: Deluan <deluan@navidrome.org> * refactor(folder): remove unused LibraryPath struct and update GC logging message Signed-off-by: Deluan <deluan@navidrome.org> * refactor(folder): enhance external scanner to support target-specific scanning Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): simplify scanner methods Signed-off-by: Deluan <deluan@navidrome.org> * feat(watcher): implement folder scanning notifications with deduplication Signed-off-by: Deluan <deluan@navidrome.org> * refactor(watcher): add resolveFolderPath function for testability Signed-off-by: Deluan <deluan@navidrome.org> * feat(watcher): implement path ignoring based on .ndignore patterns Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): implement IgnoreChecker for managing .ndignore patterns Signed-off-by: Deluan <deluan@navidrome.org> * refactor(ignore_checker): rename scanner to lineScanner for clarity Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): enhance ScanTarget struct with String method for better target representation Signed-off-by: Deluan <deluan@navidrome.org> * fix(scanner): validate library ID to prevent negative values Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): simplify GC method by removing library ID parameter Signed-off-by: Deluan <deluan@navidrome.org> * feat(scanner): update folder scanning to include all descendants of specified folders Signed-off-by: Deluan <deluan@navidrome.org> * feat(subsonic): allow selective scan in the /startScan endpoint Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): update CallScan to handle specific library/folder pairs Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): streamline scanning logic by removing scanAll method Signed-off-by: Deluan <deluan@navidrome.org> * test: enhance mockScanner for thread safety and improve test reliability Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): move scanner.ScanTarget to model.ScanTarget Signed-off-by: Deluan <deluan@navidrome.org> * refactor: move scanner types to model,implement MockScanner Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): update scanner interface and implementations to use model.Scanner Signed-off-by: Deluan <deluan@navidrome.org> * refactor(folder_repository): normalize target path handling by using filepath.Clean Signed-off-by: Deluan <deluan@navidrome.org> * test(folder_repository): add comprehensive tests for folder retrieval and child exclusion Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): simplify selective scan logic using slice.Filter Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): streamline phase folder and album creation by removing unnecessary library parameter Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): move initialization logic from phase_1 to the scanner itself Signed-off-by: Deluan <deluan@navidrome.org> * refactor(tests): rename selective scan test file to scanner_selective_test.go Signed-off-by: Deluan <deluan@navidrome.org> * feat(configuration): add DevSelectiveWatcher configuration option Signed-off-by: Deluan <deluan@navidrome.org> * feat(watcher): enhance .ndignore handling for folder deletions and file changes Signed-off-by: Deluan <deluan@navidrome.org> * docs(scanner): comments Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): enhance walkDirTree to support target folder scanning Signed-off-by: Deluan <deluan@navidrome.org> * fix(scanner, watcher): handle errors when pushing ignore patterns for folders Signed-off-by: Deluan <deluan@navidrome.org> * Update scanner/phase_1_folders.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * refactor(scanner): replace parseTargets function with direct call to scanner.ParseTargets Signed-off-by: Deluan <deluan@navidrome.org> * test(scanner): add tests for ScanBegin and ScanEnd functionality Signed-off-by: Deluan <deluan@navidrome.org> * fix(library): update PRAGMA optimize to check table sizes without ANALYZE Signed-off-by: Deluan <deluan@navidrome.org> * test(scanner): refactor tests Signed-off-by: Deluan <deluan@navidrome.org> * feat(ui): add selective scan options and update translations Signed-off-by: Deluan <deluan@navidrome.org> * feat(ui): add quick and full scan options for individual libraries Signed-off-by: Deluan <deluan@navidrome.org> * feat(ui): add Scan buttonsto the LibraryList Signed-off-by: Deluan <deluan@navidrome.org> * feat(scan): update scanning parameters from 'path' to 'target' for selective scans. * refactor(scan): move ParseTargets function to model package * test(scan): suppress unused return value from SetUserLibraries in tests * feat(gc): enhance garbage collection to support selective library purging Signed-off-by: Deluan <deluan@navidrome.org> * fix(scanner): prevent race condition when scanning deleted folders When the watcher detects changes in a folder that gets deleted before the scanner runs (due to the 10-second delay), the scanner was prematurely removing these folders from the tracking map, preventing them from being marked as missing. The issue occurred because `newFolderEntry` was calling `popLastUpdate` before verifying the folder actually exists on the filesystem. Changes: - Move fs.Stat check before newFolderEntry creation in loadDir to ensure deleted folders remain in lastUpdates for finalize() to handle - Add early existence check in walkDirTree to skip non-existent target folders with a warning log - Add unit test verifying non-existent folders aren't removed from lastUpdates prematurely - Add integration test for deleted folder scenario with ScanFolders Fixes the issue where deleting entire folders (e.g., /music/AC_DC) wouldn't mark tracks as missing when using selective folder scanning. * refactor(scan): streamline folder entry creation and update handling Signed-off-by: Deluan <deluan@navidrome.org> * feat(scan): add '@Recycle' (QNAP) to ignored directories list Signed-off-by: Deluan <deluan@navidrome.org> * fix(log): improve thread safety in logging level management * test(scan): move unit tests for ParseTargets function Signed-off-by: Deluan <deluan@navidrome.org> * review Signed-off-by: Deluan <deluan@navidrome.org> --------- Signed-off-by: Deluan <deluan@navidrome.org> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: deluan <deluan.quintao@mechanical-orchard.com>
397 lines
11 KiB
Go
397 lines
11 KiB
Go
package subsonic
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"net/http/httptest"
|
|
|
|
"github.com/navidrome/navidrome/model"
|
|
"github.com/navidrome/navidrome/model/request"
|
|
"github.com/navidrome/navidrome/server/subsonic/responses"
|
|
"github.com/navidrome/navidrome/tests"
|
|
. "github.com/onsi/ginkgo/v2"
|
|
. "github.com/onsi/gomega"
|
|
)
|
|
|
|
var _ = Describe("LibraryScanning", func() {
|
|
var api *Router
|
|
var ms *tests.MockScanner
|
|
|
|
BeforeEach(func() {
|
|
ms = tests.NewMockScanner()
|
|
api = &Router{scanner: ms}
|
|
})
|
|
|
|
Describe("StartScan", func() {
|
|
It("requires admin authentication", func() {
|
|
// Create non-admin user
|
|
ctx := request.WithUser(context.Background(), model.User{
|
|
ID: "user-id",
|
|
IsAdmin: false,
|
|
})
|
|
|
|
// Create request
|
|
r := httptest.NewRequest("GET", "/rest/startScan", nil)
|
|
r = r.WithContext(ctx)
|
|
|
|
// Call endpoint
|
|
response, err := api.StartScan(r)
|
|
|
|
// Should return authorization error
|
|
Expect(err).To(HaveOccurred())
|
|
Expect(response).To(BeNil())
|
|
var subErr subError
|
|
ok := errors.As(err, &subErr)
|
|
Expect(ok).To(BeTrue())
|
|
Expect(subErr.code).To(Equal(responses.ErrorAuthorizationFail))
|
|
})
|
|
|
|
It("triggers a full scan with no parameters", func() {
|
|
// Create admin user
|
|
ctx := request.WithUser(context.Background(), model.User{
|
|
ID: "admin-id",
|
|
IsAdmin: true,
|
|
})
|
|
|
|
// Create request with no parameters
|
|
r := httptest.NewRequest("GET", "/rest/startScan", nil)
|
|
r = r.WithContext(ctx)
|
|
|
|
// Call endpoint
|
|
response, err := api.StartScan(r)
|
|
|
|
// Should succeed
|
|
Expect(err).ToNot(HaveOccurred())
|
|
Expect(response).ToNot(BeNil())
|
|
|
|
// Verify ScanAll was called (eventually, since it's in a goroutine)
|
|
Eventually(func() int {
|
|
return ms.GetScanAllCallCount()
|
|
}).Should(BeNumerically(">", 0))
|
|
calls := ms.GetScanAllCalls()
|
|
Expect(calls).To(HaveLen(1))
|
|
Expect(calls[0].FullScan).To(BeFalse())
|
|
})
|
|
|
|
It("triggers a full scan with fullScan=true", func() {
|
|
// Create admin user
|
|
ctx := request.WithUser(context.Background(), model.User{
|
|
ID: "admin-id",
|
|
IsAdmin: true,
|
|
})
|
|
|
|
// Create request with fullScan parameter
|
|
r := httptest.NewRequest("GET", "/rest/startScan?fullScan=true", nil)
|
|
r = r.WithContext(ctx)
|
|
|
|
// Call endpoint
|
|
response, err := api.StartScan(r)
|
|
|
|
// Should succeed
|
|
Expect(err).ToNot(HaveOccurred())
|
|
Expect(response).ToNot(BeNil())
|
|
|
|
// Verify ScanAll was called with fullScan=true
|
|
Eventually(func() int {
|
|
return ms.GetScanAllCallCount()
|
|
}).Should(BeNumerically(">", 0))
|
|
calls := ms.GetScanAllCalls()
|
|
Expect(calls).To(HaveLen(1))
|
|
Expect(calls[0].FullScan).To(BeTrue())
|
|
})
|
|
|
|
It("triggers a selective scan with single target parameter", func() {
|
|
// Setup mocks
|
|
mockUserRepo := tests.CreateMockUserRepo()
|
|
_ = mockUserRepo.SetUserLibraries("admin-id", []int{1, 2})
|
|
mockDS := &tests.MockDataStore{MockedUser: mockUserRepo}
|
|
api.ds = mockDS
|
|
|
|
// Create admin user
|
|
ctx := request.WithUser(context.Background(), model.User{
|
|
ID: "admin-id",
|
|
IsAdmin: true,
|
|
})
|
|
|
|
// Create request with single target parameter
|
|
r := httptest.NewRequest("GET", "/rest/startScan?target=1:Music/Rock", nil)
|
|
r = r.WithContext(ctx)
|
|
|
|
// Call endpoint
|
|
response, err := api.StartScan(r)
|
|
|
|
// Should succeed
|
|
Expect(err).ToNot(HaveOccurred())
|
|
Expect(response).ToNot(BeNil())
|
|
|
|
// Verify ScanFolders was called with correct targets
|
|
Eventually(func() int {
|
|
return ms.GetScanFoldersCallCount()
|
|
}).Should(BeNumerically(">", 0))
|
|
calls := ms.GetScanFoldersCalls()
|
|
Expect(calls).To(HaveLen(1))
|
|
targets := calls[0].Targets
|
|
Expect(targets).To(HaveLen(1))
|
|
Expect(targets[0].LibraryID).To(Equal(1))
|
|
Expect(targets[0].FolderPath).To(Equal("Music/Rock"))
|
|
})
|
|
|
|
It("triggers a selective scan with multiple target parameters", func() {
|
|
// Setup mocks
|
|
mockUserRepo := tests.CreateMockUserRepo()
|
|
_ = mockUserRepo.SetUserLibraries("admin-id", []int{1, 2})
|
|
mockDS := &tests.MockDataStore{MockedUser: mockUserRepo}
|
|
api.ds = mockDS
|
|
|
|
// Create admin user
|
|
ctx := request.WithUser(context.Background(), model.User{
|
|
ID: "admin-id",
|
|
IsAdmin: true,
|
|
})
|
|
|
|
// Create request with multiple target parameters
|
|
r := httptest.NewRequest("GET", "/rest/startScan?target=1:Music/Reggae&target=2:Classical/Bach", nil)
|
|
r = r.WithContext(ctx)
|
|
|
|
// Call endpoint
|
|
response, err := api.StartScan(r)
|
|
|
|
// Should succeed
|
|
Expect(err).ToNot(HaveOccurred())
|
|
Expect(response).ToNot(BeNil())
|
|
|
|
// Verify ScanFolders was called with correct targets
|
|
Eventually(func() int {
|
|
return ms.GetScanFoldersCallCount()
|
|
}).Should(BeNumerically(">", 0))
|
|
calls := ms.GetScanFoldersCalls()
|
|
Expect(calls).To(HaveLen(1))
|
|
targets := calls[0].Targets
|
|
Expect(targets).To(HaveLen(2))
|
|
Expect(targets[0].LibraryID).To(Equal(1))
|
|
Expect(targets[0].FolderPath).To(Equal("Music/Reggae"))
|
|
Expect(targets[1].LibraryID).To(Equal(2))
|
|
Expect(targets[1].FolderPath).To(Equal("Classical/Bach"))
|
|
})
|
|
|
|
It("triggers a selective full scan with target and fullScan parameters", func() {
|
|
// Setup mocks
|
|
mockUserRepo := tests.CreateMockUserRepo()
|
|
_ = mockUserRepo.SetUserLibraries("admin-id", []int{1})
|
|
mockDS := &tests.MockDataStore{MockedUser: mockUserRepo}
|
|
api.ds = mockDS
|
|
|
|
// Create admin user
|
|
ctx := request.WithUser(context.Background(), model.User{
|
|
ID: "admin-id",
|
|
IsAdmin: true,
|
|
})
|
|
|
|
// Create request with target and fullScan parameters
|
|
r := httptest.NewRequest("GET", "/rest/startScan?target=1:Music/Jazz&fullScan=true", nil)
|
|
r = r.WithContext(ctx)
|
|
|
|
// Call endpoint
|
|
response, err := api.StartScan(r)
|
|
|
|
// Should succeed
|
|
Expect(err).ToNot(HaveOccurred())
|
|
Expect(response).ToNot(BeNil())
|
|
|
|
// Verify ScanFolders was called with fullScan=true
|
|
Eventually(func() int {
|
|
return ms.GetScanFoldersCallCount()
|
|
}).Should(BeNumerically(">", 0))
|
|
calls := ms.GetScanFoldersCalls()
|
|
Expect(calls).To(HaveLen(1))
|
|
Expect(calls[0].FullScan).To(BeTrue())
|
|
targets := calls[0].Targets
|
|
Expect(targets).To(HaveLen(1))
|
|
})
|
|
|
|
It("returns error for invalid target format", func() {
|
|
// Create admin user
|
|
ctx := request.WithUser(context.Background(), model.User{
|
|
ID: "admin-id",
|
|
IsAdmin: true,
|
|
})
|
|
|
|
// Create request with invalid target format (missing colon)
|
|
r := httptest.NewRequest("GET", "/rest/startScan?target=1MusicRock", nil)
|
|
r = r.WithContext(ctx)
|
|
|
|
// Call endpoint
|
|
response, err := api.StartScan(r)
|
|
|
|
// Should return error
|
|
Expect(err).To(HaveOccurred())
|
|
Expect(response).To(BeNil())
|
|
var subErr subError
|
|
ok := errors.As(err, &subErr)
|
|
Expect(ok).To(BeTrue())
|
|
Expect(subErr.code).To(Equal(responses.ErrorGeneric))
|
|
})
|
|
|
|
It("returns error for invalid library ID in target", func() {
|
|
// Create admin user
|
|
ctx := request.WithUser(context.Background(), model.User{
|
|
ID: "admin-id",
|
|
IsAdmin: true,
|
|
})
|
|
|
|
// Create request with invalid library ID
|
|
r := httptest.NewRequest("GET", "/rest/startScan?target=0:Music/Rock", nil)
|
|
r = r.WithContext(ctx)
|
|
|
|
// Call endpoint
|
|
response, err := api.StartScan(r)
|
|
|
|
// Should return error
|
|
Expect(err).To(HaveOccurred())
|
|
Expect(response).To(BeNil())
|
|
var subErr subError
|
|
ok := errors.As(err, &subErr)
|
|
Expect(ok).To(BeTrue())
|
|
Expect(subErr.code).To(Equal(responses.ErrorGeneric))
|
|
})
|
|
|
|
It("returns error when library does not exist", func() {
|
|
// Setup mocks - user has access to library 1 and 2 only
|
|
mockUserRepo := tests.CreateMockUserRepo()
|
|
_ = mockUserRepo.SetUserLibraries("admin-id", []int{1, 2})
|
|
mockDS := &tests.MockDataStore{MockedUser: mockUserRepo}
|
|
api.ds = mockDS
|
|
|
|
// Create admin user
|
|
ctx := request.WithUser(context.Background(), model.User{
|
|
ID: "admin-id",
|
|
IsAdmin: true,
|
|
})
|
|
|
|
// Create request with library ID that doesn't exist
|
|
r := httptest.NewRequest("GET", "/rest/startScan?target=999:Music/Rock", nil)
|
|
r = r.WithContext(ctx)
|
|
|
|
// Call endpoint
|
|
response, err := api.StartScan(r)
|
|
|
|
// Should return ErrorDataNotFound
|
|
Expect(err).To(HaveOccurred())
|
|
Expect(response).To(BeNil())
|
|
var subErr subError
|
|
ok := errors.As(err, &subErr)
|
|
Expect(ok).To(BeTrue())
|
|
Expect(subErr.code).To(Equal(responses.ErrorDataNotFound))
|
|
})
|
|
|
|
It("calls ScanAll when single library with empty path and only one library exists", func() {
|
|
// Setup mocks - single library in DB
|
|
mockUserRepo := tests.CreateMockUserRepo()
|
|
_ = mockUserRepo.SetUserLibraries("admin-id", []int{1})
|
|
mockLibraryRepo := &tests.MockLibraryRepo{}
|
|
mockLibraryRepo.SetData(model.Libraries{
|
|
{ID: 1, Name: "Music Library", Path: "/music"},
|
|
})
|
|
mockDS := &tests.MockDataStore{
|
|
MockedUser: mockUserRepo,
|
|
MockedLibrary: mockLibraryRepo,
|
|
}
|
|
api.ds = mockDS
|
|
|
|
// Create admin user
|
|
ctx := request.WithUser(context.Background(), model.User{
|
|
ID: "admin-id",
|
|
IsAdmin: true,
|
|
})
|
|
|
|
// Create request with single library and empty path
|
|
r := httptest.NewRequest("GET", "/rest/startScan?target=1:", nil)
|
|
r = r.WithContext(ctx)
|
|
|
|
// Call endpoint
|
|
response, err := api.StartScan(r)
|
|
|
|
// Should succeed
|
|
Expect(err).ToNot(HaveOccurred())
|
|
Expect(response).ToNot(BeNil())
|
|
|
|
// Verify ScanAll was called instead of ScanFolders
|
|
Eventually(func() int {
|
|
return ms.GetScanAllCallCount()
|
|
}).Should(BeNumerically(">", 0))
|
|
Expect(ms.GetScanFoldersCallCount()).To(Equal(0))
|
|
})
|
|
|
|
It("calls ScanFolders when single library with empty path but multiple libraries exist", func() {
|
|
// Setup mocks - multiple libraries in DB
|
|
mockUserRepo := tests.CreateMockUserRepo()
|
|
_ = mockUserRepo.SetUserLibraries("admin-id", []int{1, 2})
|
|
mockLibraryRepo := &tests.MockLibraryRepo{}
|
|
mockLibraryRepo.SetData(model.Libraries{
|
|
{ID: 1, Name: "Music Library", Path: "/music"},
|
|
{ID: 2, Name: "Audiobooks", Path: "/audiobooks"},
|
|
})
|
|
mockDS := &tests.MockDataStore{
|
|
MockedUser: mockUserRepo,
|
|
MockedLibrary: mockLibraryRepo,
|
|
}
|
|
api.ds = mockDS
|
|
|
|
// Create admin user
|
|
ctx := request.WithUser(context.Background(), model.User{
|
|
ID: "admin-id",
|
|
IsAdmin: true,
|
|
})
|
|
|
|
// Create request with single library and empty path
|
|
r := httptest.NewRequest("GET", "/rest/startScan?target=1:", nil)
|
|
r = r.WithContext(ctx)
|
|
|
|
// Call endpoint
|
|
response, err := api.StartScan(r)
|
|
|
|
// Should succeed
|
|
Expect(err).ToNot(HaveOccurred())
|
|
Expect(response).ToNot(BeNil())
|
|
|
|
// Verify ScanFolders was called (not ScanAll)
|
|
Eventually(func() int {
|
|
return ms.GetScanFoldersCallCount()
|
|
}).Should(BeNumerically(">", 0))
|
|
calls := ms.GetScanFoldersCalls()
|
|
Expect(calls).To(HaveLen(1))
|
|
targets := calls[0].Targets
|
|
Expect(targets).To(HaveLen(1))
|
|
Expect(targets[0].LibraryID).To(Equal(1))
|
|
Expect(targets[0].FolderPath).To(Equal(""))
|
|
})
|
|
})
|
|
|
|
Describe("GetScanStatus", func() {
|
|
It("returns scan status", func() {
|
|
// Setup mock scanner status
|
|
ms.SetStatusResponse(&model.ScannerStatus{
|
|
Scanning: false,
|
|
Count: 100,
|
|
FolderCount: 10,
|
|
})
|
|
|
|
// Create request
|
|
ctx := context.Background()
|
|
r := httptest.NewRequest("GET", "/rest/getScanStatus", nil)
|
|
r = r.WithContext(ctx)
|
|
|
|
// Call endpoint
|
|
response, err := api.GetScanStatus(r)
|
|
|
|
// Should succeed
|
|
Expect(err).ToNot(HaveOccurred())
|
|
Expect(response).ToNot(BeNil())
|
|
Expect(response.ScanStatus).ToNot(BeNil())
|
|
Expect(response.ScanStatus.Scanning).To(BeFalse())
|
|
Expect(response.ScanStatus.Count).To(Equal(int64(100)))
|
|
Expect(response.ScanStatus.FolderCount).To(Equal(int64(10)))
|
|
})
|
|
})
|
|
})
|