mirror of
https://github.com/navidrome/navidrome.git
synced 2026-01-03 06:15:22 +00:00
* feat: Add selective folder scanning capability Implement targeted scanning of specific library/folder pairs without full recursion. This enables efficient rescanning of individual folders when changes are detected, significantly reducing scan time for large libraries. Key changes: - Add ScanTarget struct and ScanFolders API to Scanner interface - Implement CLI flag --targets for specifying libraryID:folderPath pairs - Add FolderRepository.GetByPaths() for batch folder info retrieval - Create loadSpecificFolders() for non-recursive directory loading - Scope GC operations to affected libraries only (with TODO for full impl) - Add comprehensive tests for selective scanning behavior The selective scan: - Only processes specified folders (no subdirectory recursion) - Maintains library isolation - Runs full maintenance pipeline scoped to affected libraries - Supports both full and quick scan modes Examples: navidrome scan --targets "1:Music/Rock,1:Music/Jazz" navidrome scan --full --targets "2:Classical" * feat(folder): replace GetByPaths with GetFolderUpdateInfo for improved folder updates retrieval Signed-off-by: Deluan <deluan@navidrome.org> * test: update parseTargets test to handle folder names with spaces Signed-off-by: Deluan <deluan@navidrome.org> * refactor(folder): remove unused LibraryPath struct and update GC logging message Signed-off-by: Deluan <deluan@navidrome.org> * refactor(folder): enhance external scanner to support target-specific scanning Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): simplify scanner methods Signed-off-by: Deluan <deluan@navidrome.org> * feat(watcher): implement folder scanning notifications with deduplication Signed-off-by: Deluan <deluan@navidrome.org> * refactor(watcher): add resolveFolderPath function for testability Signed-off-by: Deluan <deluan@navidrome.org> * feat(watcher): implement path ignoring based on .ndignore patterns Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): implement IgnoreChecker for managing .ndignore patterns Signed-off-by: Deluan <deluan@navidrome.org> * refactor(ignore_checker): rename scanner to lineScanner for clarity Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): enhance ScanTarget struct with String method for better target representation Signed-off-by: Deluan <deluan@navidrome.org> * fix(scanner): validate library ID to prevent negative values Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): simplify GC method by removing library ID parameter Signed-off-by: Deluan <deluan@navidrome.org> * feat(scanner): update folder scanning to include all descendants of specified folders Signed-off-by: Deluan <deluan@navidrome.org> * feat(subsonic): allow selective scan in the /startScan endpoint Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): update CallScan to handle specific library/folder pairs Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): streamline scanning logic by removing scanAll method Signed-off-by: Deluan <deluan@navidrome.org> * test: enhance mockScanner for thread safety and improve test reliability Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): move scanner.ScanTarget to model.ScanTarget Signed-off-by: Deluan <deluan@navidrome.org> * refactor: move scanner types to model,implement MockScanner Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): update scanner interface and implementations to use model.Scanner Signed-off-by: Deluan <deluan@navidrome.org> * refactor(folder_repository): normalize target path handling by using filepath.Clean Signed-off-by: Deluan <deluan@navidrome.org> * test(folder_repository): add comprehensive tests for folder retrieval and child exclusion Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): simplify selective scan logic using slice.Filter Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): streamline phase folder and album creation by removing unnecessary library parameter Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): move initialization logic from phase_1 to the scanner itself Signed-off-by: Deluan <deluan@navidrome.org> * refactor(tests): rename selective scan test file to scanner_selective_test.go Signed-off-by: Deluan <deluan@navidrome.org> * feat(configuration): add DevSelectiveWatcher configuration option Signed-off-by: Deluan <deluan@navidrome.org> * feat(watcher): enhance .ndignore handling for folder deletions and file changes Signed-off-by: Deluan <deluan@navidrome.org> * docs(scanner): comments Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): enhance walkDirTree to support target folder scanning Signed-off-by: Deluan <deluan@navidrome.org> * fix(scanner, watcher): handle errors when pushing ignore patterns for folders Signed-off-by: Deluan <deluan@navidrome.org> * Update scanner/phase_1_folders.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * refactor(scanner): replace parseTargets function with direct call to scanner.ParseTargets Signed-off-by: Deluan <deluan@navidrome.org> * test(scanner): add tests for ScanBegin and ScanEnd functionality Signed-off-by: Deluan <deluan@navidrome.org> * fix(library): update PRAGMA optimize to check table sizes without ANALYZE Signed-off-by: Deluan <deluan@navidrome.org> * test(scanner): refactor tests Signed-off-by: Deluan <deluan@navidrome.org> * feat(ui): add selective scan options and update translations Signed-off-by: Deluan <deluan@navidrome.org> * feat(ui): add quick and full scan options for individual libraries Signed-off-by: Deluan <deluan@navidrome.org> * feat(ui): add Scan buttonsto the LibraryList Signed-off-by: Deluan <deluan@navidrome.org> * feat(scan): update scanning parameters from 'path' to 'target' for selective scans. * refactor(scan): move ParseTargets function to model package * test(scan): suppress unused return value from SetUserLibraries in tests * feat(gc): enhance garbage collection to support selective library purging Signed-off-by: Deluan <deluan@navidrome.org> * fix(scanner): prevent race condition when scanning deleted folders When the watcher detects changes in a folder that gets deleted before the scanner runs (due to the 10-second delay), the scanner was prematurely removing these folders from the tracking map, preventing them from being marked as missing. The issue occurred because `newFolderEntry` was calling `popLastUpdate` before verifying the folder actually exists on the filesystem. Changes: - Move fs.Stat check before newFolderEntry creation in loadDir to ensure deleted folders remain in lastUpdates for finalize() to handle - Add early existence check in walkDirTree to skip non-existent target folders with a warning log - Add unit test verifying non-existent folders aren't removed from lastUpdates prematurely - Add integration test for deleted folder scenario with ScanFolders Fixes the issue where deleting entire folders (e.g., /music/AC_DC) wouldn't mark tracks as missing when using selective folder scanning. * refactor(scan): streamline folder entry creation and update handling Signed-off-by: Deluan <deluan@navidrome.org> * feat(scan): add '@Recycle' (QNAP) to ignored directories list Signed-off-by: Deluan <deluan@navidrome.org> * fix(log): improve thread safety in logging level management * test(scan): move unit tests for ParseTargets function Signed-off-by: Deluan <deluan@navidrome.org> * review Signed-off-by: Deluan <deluan@navidrome.org> --------- Signed-off-by: Deluan <deluan@navidrome.org> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: deluan <deluan.quintao@mechanical-orchard.com>
214 lines
6.8 KiB
Go
214 lines
6.8 KiB
Go
package slice_test
|
|
|
|
import (
|
|
"os"
|
|
"slices"
|
|
"strconv"
|
|
"testing"
|
|
|
|
"github.com/navidrome/navidrome/tests"
|
|
"github.com/navidrome/navidrome/utils/slice"
|
|
. "github.com/onsi/ginkgo/v2"
|
|
. "github.com/onsi/gomega"
|
|
)
|
|
|
|
func TestSlice(t *testing.T) {
|
|
tests.Init(t, false)
|
|
RegisterFailHandler(Fail)
|
|
RunSpecs(t, "Slice Suite")
|
|
}
|
|
|
|
var _ = Describe("Slice Utils", func() {
|
|
Describe("Map", func() {
|
|
It("returns empty slice for an empty input", func() {
|
|
mapFunc := func(v int) string { return strconv.Itoa(v * 2) }
|
|
result := slice.Map([]int{}, mapFunc)
|
|
Expect(result).To(BeEmpty())
|
|
})
|
|
|
|
It("returns a new slice with elements mapped", func() {
|
|
mapFunc := func(v int) string { return strconv.Itoa(v * 2) }
|
|
result := slice.Map([]int{1, 2, 3, 4}, mapFunc)
|
|
Expect(result).To(ConsistOf("2", "4", "6", "8"))
|
|
})
|
|
})
|
|
|
|
Describe("MapWithArg", func() {
|
|
It("returns empty slice for an empty input", func() {
|
|
mapFunc := func(a int, v int) string { return strconv.Itoa(a + v) }
|
|
result := slice.MapWithArg([]int{}, 10, mapFunc)
|
|
Expect(result).To(BeEmpty())
|
|
})
|
|
|
|
It("returns a new slice with elements mapped", func() {
|
|
mapFunc := func(a int, v int) string { return strconv.Itoa(a + v) }
|
|
result := slice.MapWithArg([]int{1, 2, 3, 4}, 10, mapFunc)
|
|
Expect(result).To(ConsistOf("11", "12", "13", "14"))
|
|
})
|
|
})
|
|
|
|
Describe("Group", func() {
|
|
It("returns empty map for an empty input", func() {
|
|
keyFunc := func(v int) int { return v % 2 }
|
|
result := slice.Group([]int{}, keyFunc)
|
|
Expect(result).To(BeEmpty())
|
|
})
|
|
|
|
It("groups by the result of the key function", func() {
|
|
keyFunc := func(v int) int { return v % 2 }
|
|
result := slice.Group([]int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}, keyFunc)
|
|
Expect(result).To(HaveLen(2))
|
|
Expect(result[0]).To(ConsistOf(2, 4, 6, 8, 10))
|
|
Expect(result[1]).To(ConsistOf(1, 3, 5, 7, 9, 11))
|
|
})
|
|
})
|
|
|
|
Describe("ToMap", func() {
|
|
It("returns empty map for an empty input", func() {
|
|
transformFunc := func(v int) (int, string) { return v, strconv.Itoa(v) }
|
|
result := slice.ToMap([]int{}, transformFunc)
|
|
Expect(result).To(BeEmpty())
|
|
})
|
|
|
|
It("returns a map with the result of the transform function", func() {
|
|
transformFunc := func(v int) (int, string) { return v * 2, strconv.Itoa(v * 2) }
|
|
result := slice.ToMap([]int{1, 2, 3, 4}, transformFunc)
|
|
Expect(result).To(HaveLen(4))
|
|
Expect(result).To(HaveKeyWithValue(2, "2"))
|
|
Expect(result).To(HaveKeyWithValue(4, "4"))
|
|
Expect(result).To(HaveKeyWithValue(6, "6"))
|
|
Expect(result).To(HaveKeyWithValue(8, "8"))
|
|
})
|
|
})
|
|
|
|
Describe("CompactByFrequency", func() {
|
|
It("returns empty slice for an empty input", func() {
|
|
Expect(slice.CompactByFrequency([]int{})).To(BeEmpty())
|
|
})
|
|
|
|
It("groups by frequency", func() {
|
|
Expect(slice.CompactByFrequency([]int{1, 2, 1, 2, 3, 2})).To(HaveExactElements(2, 1, 3))
|
|
})
|
|
})
|
|
|
|
Describe("MostFrequent", func() {
|
|
It("returns zero value if no arguments are passed", func() {
|
|
Expect(slice.MostFrequent([]int{})).To(BeZero())
|
|
})
|
|
|
|
It("returns the single item", func() {
|
|
Expect(slice.MostFrequent([]string{"123"})).To(Equal("123"))
|
|
})
|
|
It("returns the item that appeared more times", func() {
|
|
Expect(slice.MostFrequent([]string{"1", "2", "1", "2", "3", "2"})).To(Equal("2"))
|
|
})
|
|
It("ignores zero values", func() {
|
|
Expect(slice.MostFrequent([]int{0, 0, 0, 2, 2})).To(Equal(2))
|
|
})
|
|
})
|
|
|
|
Describe("Move", func() {
|
|
It("moves item to end of slice", func() {
|
|
Expect(slice.Move([]string{"1", "2", "3"}, 0, 2)).To(HaveExactElements("2", "3", "1"))
|
|
})
|
|
It("moves item to beginning of slice", func() {
|
|
Expect(slice.Move([]string{"1", "2", "3"}, 2, 0)).To(HaveExactElements("3", "1", "2"))
|
|
})
|
|
It("keeps item in same position if srcIndex == dstIndex", func() {
|
|
Expect(slice.Move([]string{"1", "2", "3"}, 1, 1)).To(HaveExactElements("1", "2", "3"))
|
|
})
|
|
})
|
|
|
|
Describe("Unique", func() {
|
|
It("returns empty slice for an empty input", func() {
|
|
Expect(slice.Unique([]int{})).To(BeEmpty())
|
|
})
|
|
|
|
It("returns the unique elements", func() {
|
|
Expect(slice.Unique([]int{1, 2, 1, 2, 3, 2})).To(HaveExactElements(1, 2, 3))
|
|
})
|
|
})
|
|
|
|
DescribeTable("LinesFrom",
|
|
func(path string, expected int) {
|
|
count := 0
|
|
file, _ := os.Open(path)
|
|
defer file.Close()
|
|
for _ = range slice.LinesFrom(file) {
|
|
count++
|
|
}
|
|
Expect(count).To(Equal(expected))
|
|
},
|
|
Entry("returns empty slice for an empty input", "tests/fixtures/empty.txt", 0),
|
|
Entry("returns the lines of a file", "tests/fixtures/playlists/pls1.m3u", 2),
|
|
Entry("returns empty if file does not exist", "tests/fixtures/NON-EXISTENT", 0),
|
|
)
|
|
|
|
DescribeTable("CollectChunks",
|
|
func(input []int, n int, expected [][]int) {
|
|
var result [][]int
|
|
for chunks := range slice.CollectChunks(slices.Values(input), n) {
|
|
result = append(result, chunks)
|
|
}
|
|
Expect(result).To(Equal(expected))
|
|
},
|
|
Entry("returns empty slice (nil) for an empty input", []int{}, 1, nil),
|
|
Entry("returns the slice in one chunk if len < chunkSize", []int{1, 2, 3}, 10, [][]int{{1, 2, 3}}),
|
|
Entry("breaks up the slice if len > chunkSize", []int{1, 2, 3, 4, 5}, 3, [][]int{{1, 2, 3}, {4, 5}}),
|
|
)
|
|
|
|
Describe("SeqFunc", func() {
|
|
It("returns empty slice for an empty input", func() {
|
|
it := slice.SeqFunc([]int{}, func(v int) int { return v })
|
|
|
|
result := slices.Collect(it)
|
|
Expect(result).To(BeEmpty())
|
|
})
|
|
|
|
It("returns a new slice with mapped elements", func() {
|
|
it := slice.SeqFunc([]int{1, 2, 3, 4}, func(v int) string { return strconv.Itoa(v * 2) })
|
|
|
|
result := slices.Collect(it)
|
|
Expect(result).To(ConsistOf("2", "4", "6", "8"))
|
|
})
|
|
})
|
|
|
|
Describe("Filter", func() {
|
|
It("returns empty slice for an empty input", func() {
|
|
filterFunc := func(v int) bool { return v > 0 }
|
|
result := slice.Filter([]int{}, filterFunc)
|
|
Expect(result).To(BeEmpty())
|
|
})
|
|
|
|
It("returns all elements when filter matches all", func() {
|
|
filterFunc := func(v int) bool { return v > 0 }
|
|
result := slice.Filter([]int{1, 2, 3, 4}, filterFunc)
|
|
Expect(result).To(HaveExactElements(1, 2, 3, 4))
|
|
})
|
|
|
|
It("returns empty slice when filter matches none", func() {
|
|
filterFunc := func(v int) bool { return v > 10 }
|
|
result := slice.Filter([]int{1, 2, 3, 4}, filterFunc)
|
|
Expect(result).To(BeEmpty())
|
|
})
|
|
|
|
It("returns only matching elements", func() {
|
|
filterFunc := func(v int) bool { return v%2 == 0 }
|
|
result := slice.Filter([]int{1, 2, 3, 4, 5, 6}, filterFunc)
|
|
Expect(result).To(HaveExactElements(2, 4, 6))
|
|
})
|
|
|
|
It("works with string slices", func() {
|
|
filterFunc := func(s string) bool { return len(s) > 3 }
|
|
result := slice.Filter([]string{"a", "abc", "abcd", "ab", "abcde"}, filterFunc)
|
|
Expect(result).To(HaveExactElements("abcd", "abcde"))
|
|
})
|
|
|
|
It("preserves order of elements", func() {
|
|
filterFunc := func(v int) bool { return v%2 == 1 }
|
|
result := slice.Filter([]int{9, 8, 7, 6, 5, 4, 3, 2, 1}, filterFunc)
|
|
Expect(result).To(HaveExactElements(9, 7, 5, 3, 1))
|
|
})
|
|
})
|
|
})
|