mirror of
https://github.com/navidrome/navidrome.git
synced 2026-02-02 06:24:14 +00:00
* feat: Add selective folder scanning capability Implement targeted scanning of specific library/folder pairs without full recursion. This enables efficient rescanning of individual folders when changes are detected, significantly reducing scan time for large libraries. Key changes: - Add ScanTarget struct and ScanFolders API to Scanner interface - Implement CLI flag --targets for specifying libraryID:folderPath pairs - Add FolderRepository.GetByPaths() for batch folder info retrieval - Create loadSpecificFolders() for non-recursive directory loading - Scope GC operations to affected libraries only (with TODO for full impl) - Add comprehensive tests for selective scanning behavior The selective scan: - Only processes specified folders (no subdirectory recursion) - Maintains library isolation - Runs full maintenance pipeline scoped to affected libraries - Supports both full and quick scan modes Examples: navidrome scan --targets "1:Music/Rock,1:Music/Jazz" navidrome scan --full --targets "2:Classical" * feat(folder): replace GetByPaths with GetFolderUpdateInfo for improved folder updates retrieval Signed-off-by: Deluan <deluan@navidrome.org> * test: update parseTargets test to handle folder names with spaces Signed-off-by: Deluan <deluan@navidrome.org> * refactor(folder): remove unused LibraryPath struct and update GC logging message Signed-off-by: Deluan <deluan@navidrome.org> * refactor(folder): enhance external scanner to support target-specific scanning Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): simplify scanner methods Signed-off-by: Deluan <deluan@navidrome.org> * feat(watcher): implement folder scanning notifications with deduplication Signed-off-by: Deluan <deluan@navidrome.org> * refactor(watcher): add resolveFolderPath function for testability Signed-off-by: Deluan <deluan@navidrome.org> * feat(watcher): implement path ignoring based on .ndignore patterns Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): implement IgnoreChecker for managing .ndignore patterns Signed-off-by: Deluan <deluan@navidrome.org> * refactor(ignore_checker): rename scanner to lineScanner for clarity Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): enhance ScanTarget struct with String method for better target representation Signed-off-by: Deluan <deluan@navidrome.org> * fix(scanner): validate library ID to prevent negative values Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): simplify GC method by removing library ID parameter Signed-off-by: Deluan <deluan@navidrome.org> * feat(scanner): update folder scanning to include all descendants of specified folders Signed-off-by: Deluan <deluan@navidrome.org> * feat(subsonic): allow selective scan in the /startScan endpoint Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): update CallScan to handle specific library/folder pairs Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): streamline scanning logic by removing scanAll method Signed-off-by: Deluan <deluan@navidrome.org> * test: enhance mockScanner for thread safety and improve test reliability Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): move scanner.ScanTarget to model.ScanTarget Signed-off-by: Deluan <deluan@navidrome.org> * refactor: move scanner types to model,implement MockScanner Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): update scanner interface and implementations to use model.Scanner Signed-off-by: Deluan <deluan@navidrome.org> * refactor(folder_repository): normalize target path handling by using filepath.Clean Signed-off-by: Deluan <deluan@navidrome.org> * test(folder_repository): add comprehensive tests for folder retrieval and child exclusion Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): simplify selective scan logic using slice.Filter Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): streamline phase folder and album creation by removing unnecessary library parameter Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): move initialization logic from phase_1 to the scanner itself Signed-off-by: Deluan <deluan@navidrome.org> * refactor(tests): rename selective scan test file to scanner_selective_test.go Signed-off-by: Deluan <deluan@navidrome.org> * feat(configuration): add DevSelectiveWatcher configuration option Signed-off-by: Deluan <deluan@navidrome.org> * feat(watcher): enhance .ndignore handling for folder deletions and file changes Signed-off-by: Deluan <deluan@navidrome.org> * docs(scanner): comments Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): enhance walkDirTree to support target folder scanning Signed-off-by: Deluan <deluan@navidrome.org> * fix(scanner, watcher): handle errors when pushing ignore patterns for folders Signed-off-by: Deluan <deluan@navidrome.org> * Update scanner/phase_1_folders.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * refactor(scanner): replace parseTargets function with direct call to scanner.ParseTargets Signed-off-by: Deluan <deluan@navidrome.org> * test(scanner): add tests for ScanBegin and ScanEnd functionality Signed-off-by: Deluan <deluan@navidrome.org> * fix(library): update PRAGMA optimize to check table sizes without ANALYZE Signed-off-by: Deluan <deluan@navidrome.org> * test(scanner): refactor tests Signed-off-by: Deluan <deluan@navidrome.org> * feat(ui): add selective scan options and update translations Signed-off-by: Deluan <deluan@navidrome.org> * feat(ui): add quick and full scan options for individual libraries Signed-off-by: Deluan <deluan@navidrome.org> * feat(ui): add Scan buttonsto the LibraryList Signed-off-by: Deluan <deluan@navidrome.org> * feat(scan): update scanning parameters from 'path' to 'target' for selective scans. * refactor(scan): move ParseTargets function to model package * test(scan): suppress unused return value from SetUserLibraries in tests * feat(gc): enhance garbage collection to support selective library purging Signed-off-by: Deluan <deluan@navidrome.org> * fix(scanner): prevent race condition when scanning deleted folders When the watcher detects changes in a folder that gets deleted before the scanner runs (due to the 10-second delay), the scanner was prematurely removing these folders from the tracking map, preventing them from being marked as missing. The issue occurred because `newFolderEntry` was calling `popLastUpdate` before verifying the folder actually exists on the filesystem. Changes: - Move fs.Stat check before newFolderEntry creation in loadDir to ensure deleted folders remain in lastUpdates for finalize() to handle - Add early existence check in walkDirTree to skip non-existent target folders with a warning log - Add unit test verifying non-existent folders aren't removed from lastUpdates prematurely - Add integration test for deleted folder scenario with ScanFolders Fixes the issue where deleting entire folders (e.g., /music/AC_DC) wouldn't mark tracks as missing when using selective folder scanning. * refactor(scan): streamline folder entry creation and update handling Signed-off-by: Deluan <deluan@navidrome.org> * feat(scan): add '@Recycle' (QNAP) to ignored directories list Signed-off-by: Deluan <deluan@navidrome.org> * fix(log): improve thread safety in logging level management * test(scan): move unit tests for ParseTargets function Signed-off-by: Deluan <deluan@navidrome.org> * review Signed-off-by: Deluan <deluan@navidrome.org> --------- Signed-off-by: Deluan <deluan@navidrome.org> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: deluan <deluan.quintao@mechanical-orchard.com>
185 lines
3.8 KiB
Go
185 lines
3.8 KiB
Go
package slice
|
|
|
|
import (
|
|
"bufio"
|
|
"bytes"
|
|
"cmp"
|
|
"io"
|
|
"iter"
|
|
"slices"
|
|
|
|
"golang.org/x/exp/maps"
|
|
)
|
|
|
|
func Map[T any, R any](t []T, mapFunc func(T) R) []R {
|
|
r := make([]R, len(t))
|
|
for i, e := range t {
|
|
r[i] = mapFunc(e)
|
|
}
|
|
return r
|
|
}
|
|
|
|
func MapWithArg[I any, O any, A any](t []I, arg A, mapFunc func(A, I) O) []O {
|
|
return Map(t, func(e I) O {
|
|
return mapFunc(arg, e)
|
|
})
|
|
}
|
|
|
|
func Group[T any, K comparable](s []T, keyFunc func(T) K) map[K][]T {
|
|
m := map[K][]T{}
|
|
for _, item := range s {
|
|
k := keyFunc(item)
|
|
m[k] = append(m[k], item)
|
|
}
|
|
return m
|
|
}
|
|
|
|
func ToMap[T any, K comparable, V any](s []T, transformFunc func(T) (K, V)) map[K]V {
|
|
m := make(map[K]V, len(s))
|
|
for _, item := range s {
|
|
k, v := transformFunc(item)
|
|
m[k] = v
|
|
}
|
|
return m
|
|
}
|
|
|
|
func CompactByFrequency[T comparable](list []T) []T {
|
|
counters := make(map[T]int)
|
|
for _, item := range list {
|
|
counters[item]++
|
|
}
|
|
|
|
sorted := maps.Keys(counters)
|
|
slices.SortFunc(sorted, func(i, j T) int {
|
|
return cmp.Compare(counters[j], counters[i])
|
|
})
|
|
return sorted
|
|
}
|
|
|
|
func MostFrequent[T comparable](list []T) T {
|
|
var zero T
|
|
if len(list) == 0 {
|
|
return zero
|
|
}
|
|
|
|
counters := make(map[T]int)
|
|
var topItem T
|
|
var topCount int
|
|
|
|
for _, value := range list {
|
|
if value == zero {
|
|
continue
|
|
}
|
|
counters[value]++
|
|
if counters[value] > topCount {
|
|
topItem = value
|
|
topCount = counters[value]
|
|
}
|
|
}
|
|
|
|
return topItem
|
|
}
|
|
|
|
func Insert[T any](slice []T, value T, index int) []T {
|
|
return append(slice[:index], append([]T{value}, slice[index:]...)...)
|
|
}
|
|
|
|
func Remove[T any](slice []T, index int) []T {
|
|
return append(slice[:index], slice[index+1:]...)
|
|
}
|
|
|
|
func Move[T any](slice []T, srcIndex int, dstIndex int) []T {
|
|
value := slice[srcIndex]
|
|
return Insert(Remove(slice, srcIndex), value, dstIndex)
|
|
}
|
|
|
|
func Unique[T comparable](list []T) []T {
|
|
seen := make(map[T]struct{})
|
|
var result []T
|
|
for _, item := range list {
|
|
if _, ok := seen[item]; !ok {
|
|
seen[item] = struct{}{}
|
|
result = append(result, item)
|
|
}
|
|
}
|
|
return result
|
|
}
|
|
|
|
// LinesFrom returns a Seq that reads lines from the given reader
|
|
func LinesFrom(reader io.Reader) iter.Seq[string] {
|
|
return func(yield func(string) bool) {
|
|
scanner := bufio.NewScanner(reader)
|
|
scanner.Split(scanLines)
|
|
for scanner.Scan() {
|
|
if !yield(scanner.Text()) {
|
|
return
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// From https://stackoverflow.com/a/41433698
|
|
func scanLines(data []byte, atEOF bool) (advance int, token []byte, err error) {
|
|
if atEOF && len(data) == 0 {
|
|
return 0, nil, nil
|
|
}
|
|
if i := bytes.IndexAny(data, "\r\n"); i >= 0 {
|
|
if data[i] == '\n' {
|
|
// We have a line terminated by single newline.
|
|
return i + 1, data[0:i], nil
|
|
}
|
|
advance = i + 1
|
|
if len(data) > i+1 && data[i+1] == '\n' {
|
|
advance += 1
|
|
}
|
|
return advance, data[0:i], nil
|
|
}
|
|
// If we're at EOF, we have a final, non-terminated line. Return it.
|
|
if atEOF {
|
|
return len(data), data, nil
|
|
}
|
|
// Request more data.
|
|
return 0, nil, nil
|
|
}
|
|
|
|
// CollectChunks collects chunks of n elements from the input sequence and return a Seq of chunks
|
|
func CollectChunks[T any](it iter.Seq[T], n int) iter.Seq[[]T] {
|
|
return func(yield func([]T) bool) {
|
|
s := make([]T, 0, n)
|
|
for x := range it {
|
|
s = append(s, x)
|
|
if len(s) >= n {
|
|
if !yield(s) {
|
|
return
|
|
}
|
|
s = make([]T, 0, n)
|
|
}
|
|
}
|
|
if len(s) > 0 {
|
|
yield(s)
|
|
}
|
|
}
|
|
}
|
|
|
|
// SeqFunc returns a Seq that iterates over the slice with the given mapping function
|
|
func SeqFunc[I, O any](s []I, f func(I) O) iter.Seq[O] {
|
|
return func(yield func(O) bool) {
|
|
for _, x := range s {
|
|
if !yield(f(x)) {
|
|
return
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Filter returns a new slice containing only the elements of s for which filterFunc returns true
|
|
func Filter[T any](s []T, filterFunc func(T) bool) []T {
|
|
var result []T
|
|
for _, item := range s {
|
|
if filterFunc(item) {
|
|
result = append(result, item)
|
|
}
|
|
}
|
|
return result
|
|
}
|