mirror of
https://github.com/navidrome/navidrome.git
synced 2026-01-03 06:15:22 +00:00
* feat: Add selective folder scanning capability Implement targeted scanning of specific library/folder pairs without full recursion. This enables efficient rescanning of individual folders when changes are detected, significantly reducing scan time for large libraries. Key changes: - Add ScanTarget struct and ScanFolders API to Scanner interface - Implement CLI flag --targets for specifying libraryID:folderPath pairs - Add FolderRepository.GetByPaths() for batch folder info retrieval - Create loadSpecificFolders() for non-recursive directory loading - Scope GC operations to affected libraries only (with TODO for full impl) - Add comprehensive tests for selective scanning behavior The selective scan: - Only processes specified folders (no subdirectory recursion) - Maintains library isolation - Runs full maintenance pipeline scoped to affected libraries - Supports both full and quick scan modes Examples: navidrome scan --targets "1:Music/Rock,1:Music/Jazz" navidrome scan --full --targets "2:Classical" * feat(folder): replace GetByPaths with GetFolderUpdateInfo for improved folder updates retrieval Signed-off-by: Deluan <deluan@navidrome.org> * test: update parseTargets test to handle folder names with spaces Signed-off-by: Deluan <deluan@navidrome.org> * refactor(folder): remove unused LibraryPath struct and update GC logging message Signed-off-by: Deluan <deluan@navidrome.org> * refactor(folder): enhance external scanner to support target-specific scanning Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): simplify scanner methods Signed-off-by: Deluan <deluan@navidrome.org> * feat(watcher): implement folder scanning notifications with deduplication Signed-off-by: Deluan <deluan@navidrome.org> * refactor(watcher): add resolveFolderPath function for testability Signed-off-by: Deluan <deluan@navidrome.org> * feat(watcher): implement path ignoring based on .ndignore patterns Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): implement IgnoreChecker for managing .ndignore patterns Signed-off-by: Deluan <deluan@navidrome.org> * refactor(ignore_checker): rename scanner to lineScanner for clarity Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): enhance ScanTarget struct with String method for better target representation Signed-off-by: Deluan <deluan@navidrome.org> * fix(scanner): validate library ID to prevent negative values Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): simplify GC method by removing library ID parameter Signed-off-by: Deluan <deluan@navidrome.org> * feat(scanner): update folder scanning to include all descendants of specified folders Signed-off-by: Deluan <deluan@navidrome.org> * feat(subsonic): allow selective scan in the /startScan endpoint Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): update CallScan to handle specific library/folder pairs Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): streamline scanning logic by removing scanAll method Signed-off-by: Deluan <deluan@navidrome.org> * test: enhance mockScanner for thread safety and improve test reliability Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): move scanner.ScanTarget to model.ScanTarget Signed-off-by: Deluan <deluan@navidrome.org> * refactor: move scanner types to model,implement MockScanner Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): update scanner interface and implementations to use model.Scanner Signed-off-by: Deluan <deluan@navidrome.org> * refactor(folder_repository): normalize target path handling by using filepath.Clean Signed-off-by: Deluan <deluan@navidrome.org> * test(folder_repository): add comprehensive tests for folder retrieval and child exclusion Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): simplify selective scan logic using slice.Filter Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): streamline phase folder and album creation by removing unnecessary library parameter Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): move initialization logic from phase_1 to the scanner itself Signed-off-by: Deluan <deluan@navidrome.org> * refactor(tests): rename selective scan test file to scanner_selective_test.go Signed-off-by: Deluan <deluan@navidrome.org> * feat(configuration): add DevSelectiveWatcher configuration option Signed-off-by: Deluan <deluan@navidrome.org> * feat(watcher): enhance .ndignore handling for folder deletions and file changes Signed-off-by: Deluan <deluan@navidrome.org> * docs(scanner): comments Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): enhance walkDirTree to support target folder scanning Signed-off-by: Deluan <deluan@navidrome.org> * fix(scanner, watcher): handle errors when pushing ignore patterns for folders Signed-off-by: Deluan <deluan@navidrome.org> * Update scanner/phase_1_folders.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * refactor(scanner): replace parseTargets function with direct call to scanner.ParseTargets Signed-off-by: Deluan <deluan@navidrome.org> * test(scanner): add tests for ScanBegin and ScanEnd functionality Signed-off-by: Deluan <deluan@navidrome.org> * fix(library): update PRAGMA optimize to check table sizes without ANALYZE Signed-off-by: Deluan <deluan@navidrome.org> * test(scanner): refactor tests Signed-off-by: Deluan <deluan@navidrome.org> * feat(ui): add selective scan options and update translations Signed-off-by: Deluan <deluan@navidrome.org> * feat(ui): add quick and full scan options for individual libraries Signed-off-by: Deluan <deluan@navidrome.org> * feat(ui): add Scan buttonsto the LibraryList Signed-off-by: Deluan <deluan@navidrome.org> * feat(scan): update scanning parameters from 'path' to 'target' for selective scans. * refactor(scan): move ParseTargets function to model package * test(scan): suppress unused return value from SetUserLibraries in tests * feat(gc): enhance garbage collection to support selective library purging Signed-off-by: Deluan <deluan@navidrome.org> * fix(scanner): prevent race condition when scanning deleted folders When the watcher detects changes in a folder that gets deleted before the scanner runs (due to the 10-second delay), the scanner was prematurely removing these folders from the tracking map, preventing them from being marked as missing. The issue occurred because `newFolderEntry` was calling `popLastUpdate` before verifying the folder actually exists on the filesystem. Changes: - Move fs.Stat check before newFolderEntry creation in loadDir to ensure deleted folders remain in lastUpdates for finalize() to handle - Add early existence check in walkDirTree to skip non-existent target folders with a warning log - Add unit test verifying non-existent folders aren't removed from lastUpdates prematurely - Add integration test for deleted folder scenario with ScanFolders Fixes the issue where deleting entire folders (e.g., /music/AC_DC) wouldn't mark tracks as missing when using selective folder scanning. * refactor(scan): streamline folder entry creation and update handling Signed-off-by: Deluan <deluan@navidrome.org> * feat(scan): add '@Recycle' (QNAP) to ignored directories list Signed-off-by: Deluan <deluan@navidrome.org> * fix(log): improve thread safety in logging level management * test(scan): move unit tests for ParseTargets function Signed-off-by: Deluan <deluan@navidrome.org> * review Signed-off-by: Deluan <deluan@navidrome.org> --------- Signed-off-by: Deluan <deluan@navidrome.org> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: deluan <deluan.quintao@mechanical-orchard.com>
214 lines
7.8 KiB
Go
214 lines
7.8 KiB
Go
package persistence
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
|
|
"github.com/navidrome/navidrome/log"
|
|
"github.com/navidrome/navidrome/model"
|
|
"github.com/navidrome/navidrome/model/request"
|
|
. "github.com/onsi/ginkgo/v2"
|
|
. "github.com/onsi/gomega"
|
|
"github.com/pocketbase/dbx"
|
|
)
|
|
|
|
var _ = Describe("FolderRepository", func() {
|
|
var repo model.FolderRepository
|
|
var ctx context.Context
|
|
var conn *dbx.DB
|
|
var testLib, otherLib model.Library
|
|
|
|
BeforeEach(func() {
|
|
ctx = request.WithUser(log.NewContext(context.TODO()), model.User{ID: "userid"})
|
|
conn = GetDBXBuilder()
|
|
repo = newFolderRepository(ctx, conn)
|
|
|
|
// Use existing library ID 1 from test fixtures
|
|
libRepo := NewLibraryRepository(ctx, conn)
|
|
lib, err := libRepo.Get(1)
|
|
Expect(err).ToNot(HaveOccurred())
|
|
testLib = *lib
|
|
|
|
// Create a second library with its own folder to verify isolation
|
|
otherLib = model.Library{Name: "Other Library", Path: "/other/path"}
|
|
Expect(libRepo.Put(&otherLib)).To(Succeed())
|
|
})
|
|
|
|
AfterEach(func() {
|
|
// Clean up only test folders created by our tests (paths starting with "Test")
|
|
// This prevents interference with fixture data needed by other tests
|
|
_, _ = conn.NewQuery("DELETE FROM folder WHERE library_id = 1 AND path LIKE 'Test%'").Execute()
|
|
_, _ = conn.NewQuery(fmt.Sprintf("DELETE FROM library WHERE id = %d", otherLib.ID)).Execute()
|
|
})
|
|
|
|
Describe("GetFolderUpdateInfo", func() {
|
|
Context("with no target paths", func() {
|
|
It("returns all folders in the library", func() {
|
|
// Create test folders with unique names to avoid conflicts
|
|
folder1 := model.NewFolder(testLib, "TestGetLastUpdates/Folder1")
|
|
folder2 := model.NewFolder(testLib, "TestGetLastUpdates/Folder2")
|
|
|
|
err := repo.Put(folder1)
|
|
Expect(err).ToNot(HaveOccurred())
|
|
err = repo.Put(folder2)
|
|
Expect(err).ToNot(HaveOccurred())
|
|
|
|
otherFolder := model.NewFolder(otherLib, "TestOtherLib/Folder")
|
|
err = repo.Put(otherFolder)
|
|
Expect(err).ToNot(HaveOccurred())
|
|
|
|
// Query all folders (no target paths) - should only return folders from testLib
|
|
results, err := repo.GetFolderUpdateInfo(testLib)
|
|
Expect(err).ToNot(HaveOccurred())
|
|
// Should include folders from testLib
|
|
Expect(results).To(HaveKey(folder1.ID))
|
|
Expect(results).To(HaveKey(folder2.ID))
|
|
// Should NOT include folders from other library
|
|
Expect(results).ToNot(HaveKey(otherFolder.ID))
|
|
})
|
|
})
|
|
|
|
Context("with specific target paths", func() {
|
|
It("returns folder info for existing folders", func() {
|
|
// Create test folders with unique names
|
|
folder1 := model.NewFolder(testLib, "TestSpecific/Rock")
|
|
folder2 := model.NewFolder(testLib, "TestSpecific/Jazz")
|
|
folder3 := model.NewFolder(testLib, "TestSpecific/Classical")
|
|
|
|
err := repo.Put(folder1)
|
|
Expect(err).ToNot(HaveOccurred())
|
|
err = repo.Put(folder2)
|
|
Expect(err).ToNot(HaveOccurred())
|
|
err = repo.Put(folder3)
|
|
Expect(err).ToNot(HaveOccurred())
|
|
|
|
// Query specific paths
|
|
results, err := repo.GetFolderUpdateInfo(testLib, "TestSpecific/Rock", "TestSpecific/Classical")
|
|
Expect(err).ToNot(HaveOccurred())
|
|
Expect(results).To(HaveLen(2))
|
|
|
|
// Verify folder IDs are in results
|
|
Expect(results).To(HaveKey(folder1.ID))
|
|
Expect(results).To(HaveKey(folder3.ID))
|
|
Expect(results).ToNot(HaveKey(folder2.ID))
|
|
|
|
// Verify update info is populated
|
|
Expect(results[folder1.ID].UpdatedAt).ToNot(BeZero())
|
|
Expect(results[folder1.ID].Hash).To(Equal(folder1.Hash))
|
|
})
|
|
|
|
It("includes all child folders when querying parent", func() {
|
|
// Create a parent folder with multiple children
|
|
parent := model.NewFolder(testLib, "TestParent/Music")
|
|
child1 := model.NewFolder(testLib, "TestParent/Music/Rock/Queen")
|
|
child2 := model.NewFolder(testLib, "TestParent/Music/Jazz")
|
|
otherParent := model.NewFolder(testLib, "TestParent2/Music/Jazz")
|
|
|
|
Expect(repo.Put(parent)).To(Succeed())
|
|
Expect(repo.Put(child1)).To(Succeed())
|
|
Expect(repo.Put(child2)).To(Succeed())
|
|
|
|
// Query the parent folder - should return parent and all children
|
|
results, err := repo.GetFolderUpdateInfo(testLib, "TestParent/Music")
|
|
Expect(err).ToNot(HaveOccurred())
|
|
Expect(results).To(HaveLen(3))
|
|
Expect(results).To(HaveKey(parent.ID))
|
|
Expect(results).To(HaveKey(child1.ID))
|
|
Expect(results).To(HaveKey(child2.ID))
|
|
Expect(results).ToNot(HaveKey(otherParent.ID))
|
|
})
|
|
|
|
It("excludes children from other libraries", func() {
|
|
// Create parent in testLib
|
|
parent := model.NewFolder(testLib, "TestIsolation/Parent")
|
|
child := model.NewFolder(testLib, "TestIsolation/Parent/Child")
|
|
|
|
Expect(repo.Put(parent)).To(Succeed())
|
|
Expect(repo.Put(child)).To(Succeed())
|
|
|
|
// Create similar path in other library
|
|
otherParent := model.NewFolder(otherLib, "TestIsolation/Parent")
|
|
otherChild := model.NewFolder(otherLib, "TestIsolation/Parent/Child")
|
|
|
|
Expect(repo.Put(otherParent)).To(Succeed())
|
|
Expect(repo.Put(otherChild)).To(Succeed())
|
|
|
|
// Query should only return folders from testLib
|
|
results, err := repo.GetFolderUpdateInfo(testLib, "TestIsolation/Parent")
|
|
Expect(err).ToNot(HaveOccurred())
|
|
Expect(results).To(HaveLen(2))
|
|
Expect(results).To(HaveKey(parent.ID))
|
|
Expect(results).To(HaveKey(child.ID))
|
|
Expect(results).ToNot(HaveKey(otherParent.ID))
|
|
Expect(results).ToNot(HaveKey(otherChild.ID))
|
|
})
|
|
|
|
It("excludes missing children when querying parent", func() {
|
|
// Create parent and children, mark one as missing
|
|
parent := model.NewFolder(testLib, "TestMissingChild/Parent")
|
|
child1 := model.NewFolder(testLib, "TestMissingChild/Parent/Child1")
|
|
child2 := model.NewFolder(testLib, "TestMissingChild/Parent/Child2")
|
|
child2.Missing = true
|
|
|
|
Expect(repo.Put(parent)).To(Succeed())
|
|
Expect(repo.Put(child1)).To(Succeed())
|
|
Expect(repo.Put(child2)).To(Succeed())
|
|
|
|
// Query parent - should only return parent and non-missing child
|
|
results, err := repo.GetFolderUpdateInfo(testLib, "TestMissingChild/Parent")
|
|
Expect(err).ToNot(HaveOccurred())
|
|
Expect(results).To(HaveLen(2))
|
|
Expect(results).To(HaveKey(parent.ID))
|
|
Expect(results).To(HaveKey(child1.ID))
|
|
Expect(results).ToNot(HaveKey(child2.ID))
|
|
})
|
|
|
|
It("handles mix of existing and non-existing target paths", func() {
|
|
// Create folders for one path but not the other
|
|
existingParent := model.NewFolder(testLib, "TestMixed/Exists")
|
|
existingChild := model.NewFolder(testLib, "TestMixed/Exists/Child")
|
|
|
|
Expect(repo.Put(existingParent)).To(Succeed())
|
|
Expect(repo.Put(existingChild)).To(Succeed())
|
|
|
|
// Query both existing and non-existing paths
|
|
results, err := repo.GetFolderUpdateInfo(testLib, "TestMixed/Exists", "TestMixed/DoesNotExist")
|
|
Expect(err).ToNot(HaveOccurred())
|
|
Expect(results).To(HaveLen(2))
|
|
Expect(results).To(HaveKey(existingParent.ID))
|
|
Expect(results).To(HaveKey(existingChild.ID))
|
|
})
|
|
|
|
It("handles empty folder path as root", func() {
|
|
// Test querying for root folder without creating it (fixtures should have one)
|
|
rootFolderID := model.FolderID(testLib, ".")
|
|
|
|
results, err := repo.GetFolderUpdateInfo(testLib, "")
|
|
Expect(err).ToNot(HaveOccurred())
|
|
// Should return the root folder if it exists
|
|
if len(results) > 0 {
|
|
Expect(results).To(HaveKey(rootFolderID))
|
|
}
|
|
})
|
|
|
|
It("returns empty map for non-existent folders", func() {
|
|
results, err := repo.GetFolderUpdateInfo(testLib, "NonExistent/Path")
|
|
Expect(err).ToNot(HaveOccurred())
|
|
Expect(results).To(BeEmpty())
|
|
})
|
|
|
|
It("skips missing folders", func() {
|
|
// Create a folder and mark it as missing
|
|
folder := model.NewFolder(testLib, "TestMissing/Folder")
|
|
folder.Missing = true
|
|
err := repo.Put(folder)
|
|
Expect(err).ToNot(HaveOccurred())
|
|
|
|
results, err := repo.GetFolderUpdateInfo(testLib, "TestMissing/Folder")
|
|
Expect(err).ToNot(HaveOccurred())
|
|
Expect(results).To(BeEmpty())
|
|
})
|
|
})
|
|
})
|
|
})
|