mirror of
https://github.com/navidrome/navidrome.git
synced 2026-04-03 06:41:01 +00:00
refactor(scanner): implement IgnoreChecker for managing .ndignore patterns
Signed-off-by: Deluan <deluan@navidrome.org>
This commit is contained in:
parent
3cbbb440c2
commit
0b06ecb034
165
scanner/ignore_checker.go
Normal file
165
scanner/ignore_checker.go
Normal file
@ -0,0 +1,165 @@
|
||||
package scanner
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"io/fs"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"github.com/navidrome/navidrome/consts"
|
||||
"github.com/navidrome/navidrome/log"
|
||||
ignore "github.com/sabhiram/go-gitignore"
|
||||
)
|
||||
|
||||
// IgnoreChecker manages .ndignore patterns using a stack-based approach.
|
||||
// Use Push() to add patterns when entering a folder, Pop() when leaving,
|
||||
// and ShouldIgnore() to check if a path should be ignored.
|
||||
type IgnoreChecker struct {
|
||||
fsys fs.FS
|
||||
patternStack [][]string // Stack of patterns for each folder level
|
||||
currentPatterns []string // Flattened current patterns
|
||||
matcher *ignore.GitIgnore // Compiled matcher for current patterns
|
||||
}
|
||||
|
||||
// newIgnoreChecker creates a new IgnoreChecker for the given filesystem.
|
||||
func newIgnoreChecker(fsys fs.FS) *IgnoreChecker {
|
||||
return &IgnoreChecker{
|
||||
fsys: fsys,
|
||||
patternStack: make([][]string, 0),
|
||||
}
|
||||
}
|
||||
|
||||
// Push loads .ndignore patterns from the specified folder and adds them to the pattern stack.
|
||||
// Use this when entering a folder during directory tree traversal.
|
||||
func (ic *IgnoreChecker) Push(ctx context.Context, folder string) error {
|
||||
patterns := ic.loadPatternsFromFolder(ctx, folder)
|
||||
ic.patternStack = append(ic.patternStack, patterns)
|
||||
ic.rebuildCurrentPatterns()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Pop removes the most recent patterns from the stack.
|
||||
// Use this when leaving a folder during directory tree traversal.
|
||||
func (ic *IgnoreChecker) Pop() {
|
||||
if len(ic.patternStack) > 0 {
|
||||
ic.patternStack = ic.patternStack[:len(ic.patternStack)-1]
|
||||
ic.rebuildCurrentPatterns()
|
||||
}
|
||||
}
|
||||
|
||||
// PushAllParents pushes patterns from root down to the target path.
|
||||
// This is a convenience method for when you need to check a specific path
|
||||
// without recursively walking the tree. It handles the common pattern of
|
||||
// pushing all parent directories from root to the target.
|
||||
// This method is optimized to compile patterns only once at the end.
|
||||
func (ic *IgnoreChecker) PushAllParents(ctx context.Context, targetPath string) error {
|
||||
if targetPath == "." || targetPath == "" {
|
||||
// Simple case: just push root
|
||||
return ic.Push(ctx, ".")
|
||||
}
|
||||
|
||||
// Load patterns for root
|
||||
patterns := ic.loadPatternsFromFolder(ctx, ".")
|
||||
ic.patternStack = append(ic.patternStack, patterns)
|
||||
|
||||
// Load patterns for each parent directory
|
||||
currentPath := "."
|
||||
parts := strings.Split(path.Clean(targetPath), "/")
|
||||
for _, part := range parts {
|
||||
if part == "." || part == "" {
|
||||
continue
|
||||
}
|
||||
currentPath = path.Join(currentPath, part)
|
||||
patterns = ic.loadPatternsFromFolder(ctx, currentPath)
|
||||
ic.patternStack = append(ic.patternStack, patterns)
|
||||
}
|
||||
|
||||
// Rebuild and compile patterns only once at the end
|
||||
ic.rebuildCurrentPatterns()
|
||||
return nil
|
||||
}
|
||||
|
||||
// ShouldIgnore checks if the given path should be ignored based on the current patterns.
|
||||
// Returns true if the path matches any ignore pattern, false otherwise.
|
||||
func (ic *IgnoreChecker) ShouldIgnore(ctx context.Context, relPath string) bool {
|
||||
// Handle root/empty path - never ignore
|
||||
if relPath == "" || relPath == "." {
|
||||
return false
|
||||
}
|
||||
|
||||
// If no patterns loaded, nothing to ignore
|
||||
if ic.matcher == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
matches := ic.matcher.MatchesPath(relPath)
|
||||
if matches {
|
||||
log.Trace(ctx, "Scanner: Ignoring entry matching .ndignore", "path", relPath)
|
||||
}
|
||||
return matches
|
||||
}
|
||||
|
||||
// loadPatternsFromFolder reads the .ndignore file in the specified folder and returns the patterns.
|
||||
// If the file doesn't exist, returns an empty slice.
|
||||
// If the file exists but is empty, returns a pattern to ignore everything ("**/*").
|
||||
func (ic *IgnoreChecker) loadPatternsFromFolder(ctx context.Context, folder string) []string {
|
||||
ignoreFilePath := path.Join(folder, consts.ScanIgnoreFile)
|
||||
var patterns []string
|
||||
|
||||
// Check if .ndignore file exists
|
||||
if _, err := fs.Stat(ic.fsys, ignoreFilePath); err != nil {
|
||||
// No .ndignore file in this folder
|
||||
return patterns
|
||||
}
|
||||
|
||||
// Read and parse the .ndignore file
|
||||
ignoreFile, err := ic.fsys.Open(ignoreFilePath)
|
||||
if err != nil {
|
||||
log.Warn(ctx, "Scanner: Error opening .ndignore file", "path", ignoreFilePath, err)
|
||||
return patterns
|
||||
}
|
||||
defer ignoreFile.Close()
|
||||
|
||||
scanner := bufio.NewScanner(ignoreFile)
|
||||
for scanner.Scan() {
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue // Skip empty lines, whitespace-only lines, and comments
|
||||
}
|
||||
patterns = append(patterns, line)
|
||||
}
|
||||
|
||||
if err := scanner.Err(); err != nil {
|
||||
log.Warn(ctx, "Scanner: Error reading .ndignore file", "path", ignoreFilePath, err)
|
||||
return patterns
|
||||
}
|
||||
|
||||
// If the .ndignore file is empty, ignore everything
|
||||
if len(patterns) == 0 {
|
||||
log.Trace(ctx, "Scanner: .ndignore file is empty, ignoring everything", "path", folder)
|
||||
patterns = []string{"**/*"}
|
||||
} else {
|
||||
log.Trace(ctx, "Scanner: .ndignore file found", "path", ignoreFilePath, "patterns", patterns)
|
||||
}
|
||||
|
||||
return patterns
|
||||
}
|
||||
|
||||
// rebuildCurrentPatterns flattens the pattern stack into currentPatterns and recompiles the matcher.
|
||||
func (ic *IgnoreChecker) rebuildCurrentPatterns() {
|
||||
ic.currentPatterns = make([]string, 0)
|
||||
for _, patterns := range ic.patternStack {
|
||||
ic.currentPatterns = append(ic.currentPatterns, patterns...)
|
||||
}
|
||||
ic.compilePatterns()
|
||||
}
|
||||
|
||||
// compilePatterns compiles the current patterns into a GitIgnore matcher.
|
||||
func (ic *IgnoreChecker) compilePatterns() {
|
||||
if len(ic.currentPatterns) == 0 {
|
||||
ic.matcher = nil
|
||||
return
|
||||
}
|
||||
ic.matcher = ignore.CompileIgnoreLines(ic.currentPatterns...)
|
||||
}
|
||||
313
scanner/ignore_checker_test.go
Normal file
313
scanner/ignore_checker_test.go
Normal file
@ -0,0 +1,313 @@
|
||||
package scanner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing/fstest"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
var _ = Describe("IgnoreChecker", func() {
|
||||
Describe("loadPatternsFromFolder", func() {
|
||||
var ic *IgnoreChecker
|
||||
var ctx context.Context
|
||||
|
||||
BeforeEach(func() {
|
||||
ctx = context.Background()
|
||||
})
|
||||
|
||||
Context("when .ndignore file does not exist", func() {
|
||||
It("should return empty patterns", func() {
|
||||
fsys := fstest.MapFS{}
|
||||
ic = newIgnoreChecker(fsys)
|
||||
patterns := ic.loadPatternsFromFolder(ctx, ".")
|
||||
Expect(patterns).To(BeEmpty())
|
||||
})
|
||||
})
|
||||
|
||||
Context("when .ndignore file is empty", func() {
|
||||
It("should return wildcard to ignore everything", func() {
|
||||
fsys := fstest.MapFS{
|
||||
".ndignore": &fstest.MapFile{Data: []byte("")},
|
||||
}
|
||||
ic = newIgnoreChecker(fsys)
|
||||
patterns := ic.loadPatternsFromFolder(ctx, ".")
|
||||
Expect(patterns).To(Equal([]string{"**/*"}))
|
||||
})
|
||||
})
|
||||
|
||||
DescribeTable("parsing .ndignore content",
|
||||
func(content string, expectedPatterns []string) {
|
||||
fsys := fstest.MapFS{
|
||||
".ndignore": &fstest.MapFile{Data: []byte(content)},
|
||||
}
|
||||
ic = newIgnoreChecker(fsys)
|
||||
patterns := ic.loadPatternsFromFolder(ctx, ".")
|
||||
Expect(patterns).To(Equal(expectedPatterns))
|
||||
},
|
||||
Entry("single pattern", "*.txt", []string{"*.txt"}),
|
||||
Entry("multiple patterns", "*.txt\n*.log", []string{"*.txt", "*.log"}),
|
||||
Entry("with comments", "# comment\n*.txt\n# another\n*.log", []string{"*.txt", "*.log"}),
|
||||
Entry("with empty lines", "*.txt\n\n*.log\n\n", []string{"*.txt", "*.log"}),
|
||||
Entry("mixed content", "# header\n\n*.txt\n# middle\n*.log\n\n", []string{"*.txt", "*.log"}),
|
||||
Entry("only comments and empty lines", "# comment\n\n# another\n", []string{"**/*"}),
|
||||
Entry("trailing newline", "*.txt\n*.log\n", []string{"*.txt", "*.log"}),
|
||||
Entry("directory pattern", "temp/", []string{"temp/"}),
|
||||
Entry("wildcard pattern", "**/*.mp3", []string{"**/*.mp3"}),
|
||||
Entry("multiple wildcards", "**/*.mp3\n**/*.flac\n*.log", []string{"**/*.mp3", "**/*.flac", "*.log"}),
|
||||
Entry("negation pattern", "!important.txt", []string{"!important.txt"}),
|
||||
Entry("comment with hash not at start is pattern", "not#comment", []string{"not#comment"}),
|
||||
Entry("whitespace-only lines skipped", "*.txt\n \n*.log\n\t\n", []string{"*.txt", "*.log"}),
|
||||
Entry("patterns with whitespace trimmed", " *.txt \n\t*.log\t", []string{"*.txt", "*.log"}),
|
||||
)
|
||||
})
|
||||
|
||||
Describe("Push and Pop", func() {
|
||||
var ic *IgnoreChecker
|
||||
var fsys fstest.MapFS
|
||||
var ctx context.Context
|
||||
|
||||
BeforeEach(func() {
|
||||
ctx = context.Background()
|
||||
fsys = fstest.MapFS{
|
||||
".ndignore": &fstest.MapFile{Data: []byte("*.txt")},
|
||||
"folder1/.ndignore": &fstest.MapFile{Data: []byte("*.mp3")},
|
||||
"folder2/.ndignore": &fstest.MapFile{Data: []byte("*.flac")},
|
||||
}
|
||||
ic = newIgnoreChecker(fsys)
|
||||
})
|
||||
|
||||
Context("Push", func() {
|
||||
It("should add patterns to stack", func() {
|
||||
err := ic.Push(ctx, ".")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(len(ic.patternStack)).To(Equal(1))
|
||||
Expect(ic.currentPatterns).To(ContainElement("*.txt"))
|
||||
})
|
||||
|
||||
It("should compile matcher after push", func() {
|
||||
err := ic.Push(ctx, ".")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(ic.matcher).ToNot(BeNil())
|
||||
})
|
||||
|
||||
It("should accumulate patterns from multiple levels", func() {
|
||||
err := ic.Push(ctx, ".")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
err = ic.Push(ctx, "folder1")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(len(ic.patternStack)).To(Equal(2))
|
||||
Expect(ic.currentPatterns).To(ConsistOf("*.txt", "*.mp3"))
|
||||
})
|
||||
|
||||
It("should handle push when no .ndignore exists", func() {
|
||||
err := ic.Push(ctx, "nonexistent")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(len(ic.patternStack)).To(Equal(1))
|
||||
Expect(ic.currentPatterns).To(BeEmpty())
|
||||
})
|
||||
})
|
||||
|
||||
Context("Pop", func() {
|
||||
It("should remove most recent patterns", func() {
|
||||
err := ic.Push(ctx, ".")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
err = ic.Push(ctx, "folder1")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
ic.Pop()
|
||||
Expect(len(ic.patternStack)).To(Equal(1))
|
||||
Expect(ic.currentPatterns).To(Equal([]string{"*.txt"}))
|
||||
})
|
||||
|
||||
It("should handle Pop on empty stack gracefully", func() {
|
||||
Expect(func() { ic.Pop() }).ToNot(Panic())
|
||||
Expect(ic.patternStack).To(BeEmpty())
|
||||
})
|
||||
|
||||
It("should set matcher to nil when all patterns popped", func() {
|
||||
err := ic.Push(ctx, ".")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(ic.matcher).ToNot(BeNil())
|
||||
ic.Pop()
|
||||
Expect(ic.matcher).To(BeNil())
|
||||
})
|
||||
|
||||
It("should update matcher after pop", func() {
|
||||
err := ic.Push(ctx, ".")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
err = ic.Push(ctx, "folder1")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
matcher1 := ic.matcher
|
||||
ic.Pop()
|
||||
matcher2 := ic.matcher
|
||||
Expect(matcher1).ToNot(Equal(matcher2))
|
||||
})
|
||||
})
|
||||
|
||||
Context("multiple Push/Pop cycles", func() {
|
||||
It("should maintain correct state through cycles", func() {
|
||||
err := ic.Push(ctx, ".")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(ic.currentPatterns).To(Equal([]string{"*.txt"}))
|
||||
|
||||
err = ic.Push(ctx, "folder1")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(ic.currentPatterns).To(ConsistOf("*.txt", "*.mp3"))
|
||||
|
||||
ic.Pop()
|
||||
Expect(ic.currentPatterns).To(Equal([]string{"*.txt"}))
|
||||
|
||||
err = ic.Push(ctx, "folder2")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(ic.currentPatterns).To(ConsistOf("*.txt", "*.flac"))
|
||||
|
||||
ic.Pop()
|
||||
Expect(ic.currentPatterns).To(Equal([]string{"*.txt"}))
|
||||
|
||||
ic.Pop()
|
||||
Expect(ic.currentPatterns).To(BeEmpty())
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
Describe("PushAllParents", func() {
|
||||
var ic *IgnoreChecker
|
||||
var ctx context.Context
|
||||
|
||||
BeforeEach(func() {
|
||||
ctx = context.Background()
|
||||
fsys := fstest.MapFS{
|
||||
".ndignore": &fstest.MapFile{Data: []byte("root.txt")},
|
||||
"folder1/.ndignore": &fstest.MapFile{Data: []byte("level1.txt")},
|
||||
"folder1/folder2/.ndignore": &fstest.MapFile{Data: []byte("level2.txt")},
|
||||
"folder1/folder2/folder3/.ndignore": &fstest.MapFile{Data: []byte("level3.txt")},
|
||||
}
|
||||
ic = newIgnoreChecker(fsys)
|
||||
})
|
||||
|
||||
DescribeTable("loading parent patterns",
|
||||
func(targetPath string, expectedStackDepth int, expectedPatterns []string) {
|
||||
err := ic.PushAllParents(ctx, targetPath)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(len(ic.patternStack)).To(Equal(expectedStackDepth))
|
||||
Expect(ic.currentPatterns).To(ConsistOf(expectedPatterns))
|
||||
},
|
||||
Entry("root path", ".", 1, []string{"root.txt"}),
|
||||
Entry("empty path", "", 1, []string{"root.txt"}),
|
||||
Entry("single level", "folder1", 2, []string{"root.txt", "level1.txt"}),
|
||||
Entry("two levels", "folder1/folder2", 3, []string{"root.txt", "level1.txt", "level2.txt"}),
|
||||
Entry("three levels", "folder1/folder2/folder3", 4, []string{"root.txt", "level1.txt", "level2.txt", "level3.txt"}),
|
||||
)
|
||||
|
||||
It("should only compile patterns once at the end", func() {
|
||||
// This is more of a behavioral test - we verify the matcher is not nil after PushAllParents
|
||||
err := ic.PushAllParents(ctx, "folder1/folder2")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(ic.matcher).ToNot(BeNil())
|
||||
})
|
||||
|
||||
It("should handle paths with dot", func() {
|
||||
err := ic.PushAllParents(ctx, "./folder1")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(len(ic.patternStack)).To(Equal(2))
|
||||
})
|
||||
|
||||
Context("when some parent folders have no .ndignore", func() {
|
||||
BeforeEach(func() {
|
||||
fsys := fstest.MapFS{
|
||||
".ndignore": &fstest.MapFile{Data: []byte("root.txt")},
|
||||
"folder1/folder2/.ndignore": &fstest.MapFile{Data: []byte("level2.txt")},
|
||||
}
|
||||
ic = newIgnoreChecker(fsys)
|
||||
})
|
||||
|
||||
It("should still push all parent levels", func() {
|
||||
err := ic.PushAllParents(ctx, "folder1/folder2")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(len(ic.patternStack)).To(Equal(3)) // root, folder1 (empty), folder2
|
||||
Expect(ic.currentPatterns).To(ConsistOf("root.txt", "level2.txt"))
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
Describe("ShouldIgnore", func() {
|
||||
var ic *IgnoreChecker
|
||||
var ctx context.Context
|
||||
|
||||
BeforeEach(func() {
|
||||
ctx = context.Background()
|
||||
})
|
||||
|
||||
Context("with no patterns loaded", func() {
|
||||
It("should not ignore any path", func() {
|
||||
fsys := fstest.MapFS{}
|
||||
ic = newIgnoreChecker(fsys)
|
||||
Expect(ic.ShouldIgnore(ctx, "anything.txt")).To(BeFalse())
|
||||
Expect(ic.ShouldIgnore(ctx, "folder/file.mp3")).To(BeFalse())
|
||||
})
|
||||
})
|
||||
|
||||
Context("special paths", func() {
|
||||
BeforeEach(func() {
|
||||
fsys := fstest.MapFS{
|
||||
".ndignore": &fstest.MapFile{Data: []byte("**/*")},
|
||||
}
|
||||
ic = newIgnoreChecker(fsys)
|
||||
err := ic.Push(ctx, ".")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
|
||||
It("should never ignore root or empty paths", func() {
|
||||
Expect(ic.ShouldIgnore(ctx, "")).To(BeFalse())
|
||||
Expect(ic.ShouldIgnore(ctx, ".")).To(BeFalse())
|
||||
})
|
||||
|
||||
It("should ignore all other paths with wildcard", func() {
|
||||
Expect(ic.ShouldIgnore(ctx, "file.txt")).To(BeTrue())
|
||||
Expect(ic.ShouldIgnore(ctx, "folder/file.mp3")).To(BeTrue())
|
||||
})
|
||||
})
|
||||
|
||||
DescribeTable("pattern matching",
|
||||
func(pattern string, path string, shouldMatch bool) {
|
||||
fsys := fstest.MapFS{
|
||||
".ndignore": &fstest.MapFile{Data: []byte(pattern)},
|
||||
}
|
||||
ic = newIgnoreChecker(fsys)
|
||||
err := ic.Push(ctx, ".")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(ic.ShouldIgnore(ctx, path)).To(Equal(shouldMatch))
|
||||
},
|
||||
Entry("glob match", "*.txt", "file.txt", true),
|
||||
Entry("glob no match", "*.txt", "file.mp3", false),
|
||||
Entry("directory pattern match", "tmp/", "tmp/file.txt", true),
|
||||
Entry("directory pattern no match", "tmp/", "temporary/file.txt", false),
|
||||
Entry("nested glob match", "**/*.log", "deep/nested/file.log", true),
|
||||
Entry("nested glob no match", "**/*.log", "deep/nested/file.txt", false),
|
||||
Entry("specific file match", "ignore.me", "ignore.me", true),
|
||||
Entry("specific file no match", "ignore.me", "keep.me", false),
|
||||
Entry("wildcard all", "**/*", "any/path/file.txt", true),
|
||||
Entry("nested specific match", "temp/*", "temp/cache.db", true),
|
||||
Entry("nested specific no match", "temp/*", "temporary/cache.db", false),
|
||||
)
|
||||
|
||||
Context("with multiple patterns", func() {
|
||||
BeforeEach(func() {
|
||||
fsys := fstest.MapFS{
|
||||
".ndignore": &fstest.MapFile{Data: []byte("*.txt\n*.log\ntemp/")},
|
||||
}
|
||||
ic = newIgnoreChecker(fsys)
|
||||
err := ic.Push(ctx, ".")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
|
||||
It("should match any of the patterns", func() {
|
||||
Expect(ic.ShouldIgnore(ctx, "file.txt")).To(BeTrue())
|
||||
Expect(ic.ShouldIgnore(ctx, "debug.log")).To(BeTrue())
|
||||
Expect(ic.ShouldIgnore(ctx, "temp/cache")).To(BeTrue())
|
||||
Expect(ic.ShouldIgnore(ctx, "music.mp3")).To(BeFalse())
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
@ -1,7 +1,6 @@
|
||||
package scanner
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"io/fs"
|
||||
"maps"
|
||||
@ -11,18 +10,17 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/navidrome/navidrome/conf"
|
||||
"github.com/navidrome/navidrome/consts"
|
||||
"github.com/navidrome/navidrome/log"
|
||||
"github.com/navidrome/navidrome/model"
|
||||
"github.com/navidrome/navidrome/utils"
|
||||
ignore "github.com/sabhiram/go-gitignore"
|
||||
)
|
||||
|
||||
func walkDirTree(ctx context.Context, job *scanJob) (<-chan *folderEntry, error) {
|
||||
results := make(chan *folderEntry)
|
||||
go func() {
|
||||
defer close(results)
|
||||
err := walkFolder(ctx, job, ".", nil, results)
|
||||
checker := newIgnoreChecker(job.fs)
|
||||
err := walkFolder(ctx, job, ".", checker, results)
|
||||
if err != nil {
|
||||
log.Error(ctx, "Scanner: There were errors reading directories from filesystem", "path", job.lib.Path, err)
|
||||
return
|
||||
@ -42,11 +40,12 @@ func loadSpecificFolders(ctx context.Context, job *scanJob, targetFolders []stri
|
||||
return
|
||||
}
|
||||
|
||||
// Load ignore patterns from parent directories up to this folder
|
||||
ignorePatterns := loadIgnoredPatternsForPath(ctx, job.fs, folderPath)
|
||||
// Create checker and push patterns from root to this folder
|
||||
checker := newIgnoreChecker(job.fs)
|
||||
_ = checker.PushAllParents(ctx, folderPath)
|
||||
|
||||
// Load only this specific folder (no recursion)
|
||||
folder, _, err := loadDir(ctx, job, folderPath, ignorePatterns)
|
||||
folder, _, err := loadDir(ctx, job, folderPath, checker)
|
||||
if err != nil {
|
||||
log.Warn(ctx, "Scanner: Error loading target folder. Skipping", "path", folderPath, err)
|
||||
continue
|
||||
@ -65,78 +64,18 @@ func loadSpecificFolders(ctx context.Context, job *scanJob, targetFolders []stri
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// loadIgnoredPatternsForPath loads all .ndignore patterns from the root down to the specified path
|
||||
func loadIgnoredPatternsForPath(ctx context.Context, fsys fs.FS, targetPath string) []string {
|
||||
var patterns []string
|
||||
currentPath := "."
|
||||
func walkFolder(ctx context.Context, job *scanJob, currentFolder string, checker *IgnoreChecker, results chan<- *folderEntry) error {
|
||||
// Push patterns for this folder onto the stack
|
||||
_ = checker.Push(ctx, currentFolder)
|
||||
defer checker.Pop() // Pop patterns when leaving this folder
|
||||
|
||||
// If target is root, just check root
|
||||
if targetPath == "." {
|
||||
return loadIgnoredPatterns(ctx, fsys, ".", nil)
|
||||
}
|
||||
|
||||
// Walk from root to target, collecting ignore patterns
|
||||
parts := strings.Split(path.Clean(targetPath), "/")
|
||||
for _, part := range parts {
|
||||
if part == "." {
|
||||
continue
|
||||
}
|
||||
patterns = loadIgnoredPatterns(ctx, fsys, currentPath, patterns)
|
||||
currentPath = path.Join(currentPath, part)
|
||||
}
|
||||
// Load patterns from the target folder itself
|
||||
patterns = loadIgnoredPatterns(ctx, fsys, currentPath, patterns)
|
||||
|
||||
return patterns
|
||||
}
|
||||
|
||||
// loadIgnoredPatterns loads .ndignore patterns from the specified folder and combines them with currentPatterns
|
||||
func loadIgnoredPatterns(ctx context.Context, fsys fs.FS, currentFolder string, currentPatterns []string) []string {
|
||||
ignoreFilePath := path.Join(currentFolder, consts.ScanIgnoreFile)
|
||||
var newPatterns []string
|
||||
if _, err := fs.Stat(fsys, ignoreFilePath); err == nil {
|
||||
// Read and parse the .ndignore file
|
||||
ignoreFile, err := fsys.Open(ignoreFilePath)
|
||||
if err != nil {
|
||||
log.Warn(ctx, "Scanner: Error opening .ndignore file", "path", ignoreFilePath, err)
|
||||
// Continue with previous patterns
|
||||
} else {
|
||||
defer ignoreFile.Close()
|
||||
scanner := bufio.NewScanner(ignoreFile)
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue // Skip empty lines and comments
|
||||
}
|
||||
newPatterns = append(newPatterns, line)
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
log.Warn(ctx, "Scanner: Error reading .ignore file", "path", ignoreFilePath, err)
|
||||
}
|
||||
}
|
||||
// If the .ndignore file is empty, mimic the current behavior and ignore everything
|
||||
if len(newPatterns) == 0 {
|
||||
log.Trace(ctx, "Scanner: .ndignore file is empty, ignoring everything", "path", currentFolder)
|
||||
newPatterns = []string{"**/*"}
|
||||
} else {
|
||||
log.Trace(ctx, "Scanner: .ndignore file found ", "path", ignoreFilePath, "patterns", newPatterns)
|
||||
}
|
||||
}
|
||||
// Combine the patterns from the .ndignore file with the ones passed as argument
|
||||
combinedPatterns := append([]string{}, currentPatterns...)
|
||||
return append(combinedPatterns, newPatterns...)
|
||||
}
|
||||
|
||||
func walkFolder(ctx context.Context, job *scanJob, currentFolder string, ignorePatterns []string, results chan<- *folderEntry) error {
|
||||
ignorePatterns = loadIgnoredPatterns(ctx, job.fs, currentFolder, ignorePatterns)
|
||||
|
||||
folder, children, err := loadDir(ctx, job, currentFolder, ignorePatterns)
|
||||
folder, children, err := loadDir(ctx, job, currentFolder, checker)
|
||||
if err != nil {
|
||||
log.Warn(ctx, "Scanner: Error loading dir. Skipping", "path", currentFolder, err)
|
||||
return nil
|
||||
}
|
||||
for _, c := range children {
|
||||
err := walkFolder(ctx, job, c, ignorePatterns, results)
|
||||
err := walkFolder(ctx, job, c, checker, results)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -154,7 +93,7 @@ func walkFolder(ctx context.Context, job *scanJob, currentFolder string, ignoreP
|
||||
return nil
|
||||
}
|
||||
|
||||
func loadDir(ctx context.Context, job *scanJob, dirPath string, ignorePatterns []string) (folder *folderEntry, children []string, err error) {
|
||||
func loadDir(ctx context.Context, job *scanJob, dirPath string, checker *IgnoreChecker) (folder *folderEntry, children []string, err error) {
|
||||
folder = newFolderEntry(job, dirPath)
|
||||
|
||||
dirInfo, err := fs.Stat(job.fs, dirPath)
|
||||
@ -176,12 +115,11 @@ func loadDir(ctx context.Context, job *scanJob, dirPath string, ignorePatterns [
|
||||
return folder, children, err
|
||||
}
|
||||
|
||||
ignoreMatcher := ignore.CompileIgnoreLines(ignorePatterns...)
|
||||
entries := fullReadDir(ctx, dirFile)
|
||||
children = make([]string, 0, len(entries))
|
||||
for _, entry := range entries {
|
||||
entryPath := path.Join(dirPath, entry.Name())
|
||||
if len(ignorePatterns) > 0 && isScanIgnored(ctx, ignoreMatcher, entryPath) {
|
||||
if checker.ShouldIgnore(ctx, entryPath) {
|
||||
log.Trace(ctx, "Scanner: Ignoring entry", "path", entryPath)
|
||||
continue
|
||||
}
|
||||
@ -313,11 +251,3 @@ func isDirIgnored(name string) bool {
|
||||
func isEntryIgnored(name string) bool {
|
||||
return strings.HasPrefix(name, ".") && !strings.HasPrefix(name, "..")
|
||||
}
|
||||
|
||||
func isScanIgnored(ctx context.Context, matcher *ignore.GitIgnore, entryPath string) bool {
|
||||
matches := matcher.MatchesPath(entryPath)
|
||||
if matches {
|
||||
log.Trace(ctx, "Scanner: Ignoring entry matching .ndignore: ", "path", entryPath)
|
||||
}
|
||||
return matches
|
||||
}
|
||||
|
||||
@ -13,7 +13,6 @@ import (
|
||||
"github.com/navidrome/navidrome/log"
|
||||
"github.com/navidrome/navidrome/model"
|
||||
"github.com/navidrome/navidrome/utils/singleton"
|
||||
ignore "github.com/sabhiram/go-gitignore"
|
||||
)
|
||||
|
||||
type Watcher interface {
|
||||
@ -236,14 +235,11 @@ func (w *watcher) watchLibrary(ctx context.Context, lib *model.Library) error {
|
||||
log.Trace(ctx, "Ignoring change", "libraryID", lib.ID, "path", path)
|
||||
continue
|
||||
}
|
||||
|
||||
log.Trace(ctx, "Detected change", "libraryID", lib.ID, "path", path, "absoluteLibPath", absLibPath)
|
||||
|
||||
// Find the folder to scan - validate path exists as directory, walk up if needed
|
||||
folderPath := resolveFolderPath(fsys, path)
|
||||
|
||||
// Check if the folder should be ignored based on .ndignore patterns
|
||||
if shouldIgnorePath(ctx, fsys, folderPath) {
|
||||
if w.shouldIgnoreFolderPath(ctx, fsys, folderPath) {
|
||||
log.Trace(ctx, "Ignoring change in folder matching .ndignore pattern", "libraryID", lib.ID, "folderPath", folderPath)
|
||||
continue
|
||||
}
|
||||
@ -288,6 +284,14 @@ func resolveFolderPath(fsys fs.FS, path string) string {
|
||||
}
|
||||
}
|
||||
|
||||
// shouldIgnoreFolderPath checks if the given folderPath should be ignored based on .ndignore patterns
|
||||
// in the library. It pushes all parent folders onto the IgnoreChecker stack before checking.
|
||||
func (w *watcher) shouldIgnoreFolderPath(ctx context.Context, fsys storage.MusicFS, folderPath string) bool {
|
||||
checker := newIgnoreChecker(fsys)
|
||||
_ = checker.PushAllParents(ctx, folderPath)
|
||||
return checker.ShouldIgnore(ctx, folderPath)
|
||||
}
|
||||
|
||||
func isIgnoredPath(_ context.Context, _ fs.FS, path string) bool {
|
||||
baseDir, name := filepath.Split(path)
|
||||
switch {
|
||||
@ -304,26 +308,3 @@ func isIgnoredPath(_ context.Context, _ fs.FS, path string) bool {
|
||||
// But at this point, we can assume it's a directory. If it's a file, it would be ignored anyway
|
||||
return isDirIgnored(baseDir)
|
||||
}
|
||||
|
||||
// shouldIgnorePath checks if the given path should be ignored based on .ndignore patterns.
|
||||
// It loads all .ndignore files from the root down to the path and returns true if the path
|
||||
// matches any ignore pattern. This function is suitable for checking paths without recursion,
|
||||
// such as in the watcher.
|
||||
func shouldIgnorePath(ctx context.Context, fsys fs.FS, relPath string) bool {
|
||||
// Handle root/empty path - never ignore
|
||||
if relPath == "" || relPath == "." {
|
||||
return false
|
||||
}
|
||||
|
||||
// Load ignore patterns from root to the target path
|
||||
patterns := loadIgnoredPatternsForPath(ctx, fsys, relPath)
|
||||
|
||||
// If no patterns, nothing to ignore
|
||||
if len(patterns) == 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
// Compile and check
|
||||
matcher := ignore.CompileIgnoreLines(patterns...)
|
||||
return isScanIgnored(ctx, matcher, relPath)
|
||||
}
|
||||
|
||||
@ -279,107 +279,6 @@ var _ = Describe("Watcher", func() {
|
||||
})
|
||||
})
|
||||
|
||||
var _ = Describe("shouldIgnorePath", func() {
|
||||
var ctx context.Context
|
||||
var mockFS fs.FS
|
||||
|
||||
BeforeEach(func() {
|
||||
ctx = context.Background()
|
||||
|
||||
// Create a mock filesystem with .ndignore files
|
||||
mockFS = fstest.MapFS{
|
||||
// Root .ndignore ignoring "temp/*"
|
||||
".ndignore": &fstest.MapFile{Data: []byte("temp/*\n*.log\n")},
|
||||
|
||||
// Normal directories
|
||||
"music": &fstest.MapFile{Mode: fs.ModeDir},
|
||||
"music/artist1": &fstest.MapFile{Mode: fs.ModeDir},
|
||||
"music/artist1/song.mp3": &fstest.MapFile{Data: []byte("audio")},
|
||||
|
||||
// Temp directory (should be ignored)
|
||||
"temp": &fstest.MapFile{Mode: fs.ModeDir},
|
||||
"temp/cache": &fstest.MapFile{Mode: fs.ModeDir},
|
||||
"temp/cache/file.mp3": &fstest.MapFile{Data: []byte("audio")},
|
||||
|
||||
// Directory with hierarchical .ndignore
|
||||
"project": &fstest.MapFile{Mode: fs.ModeDir},
|
||||
"project/.ndignore": &fstest.MapFile{Data: []byte("drafts\n")},
|
||||
"project/final": &fstest.MapFile{Mode: fs.ModeDir},
|
||||
"project/final/album.mp3": &fstest.MapFile{Data: []byte("audio")},
|
||||
"project/drafts": &fstest.MapFile{Mode: fs.ModeDir},
|
||||
"project/drafts/test.mp3": &fstest.MapFile{Data: []byte("audio")},
|
||||
|
||||
// Directory with empty .ndignore (should ignore everything)
|
||||
"empty": &fstest.MapFile{Mode: fs.ModeDir},
|
||||
"empty/.ndignore": &fstest.MapFile{Data: []byte("")},
|
||||
"empty/subdir": &fstest.MapFile{Mode: fs.ModeDir},
|
||||
|
||||
// Log file at root level (should be ignored by *.log pattern)
|
||||
"debug.log": &fstest.MapFile{Data: []byte("logs")},
|
||||
}
|
||||
})
|
||||
|
||||
It("does not ignore paths without .ndignore patterns", func() {
|
||||
result := shouldIgnorePath(ctx, mockFS, "music/artist1")
|
||||
Expect(result).To(BeFalse())
|
||||
})
|
||||
|
||||
It("ignores paths matching root .ndignore patterns", func() {
|
||||
result := shouldIgnorePath(ctx, mockFS, "temp/cache")
|
||||
Expect(result).To(BeTrue())
|
||||
})
|
||||
|
||||
It("ignores log files matching *.log pattern", func() {
|
||||
result := shouldIgnorePath(ctx, mockFS, "debug.log")
|
||||
Expect(result).To(BeTrue())
|
||||
})
|
||||
|
||||
It("applies hierarchical .ndignore patterns", func() {
|
||||
// project/drafts should be ignored by project/.ndignore
|
||||
result := shouldIgnorePath(ctx, mockFS, "project/drafts")
|
||||
Expect(result).To(BeTrue())
|
||||
|
||||
// project/final should NOT be ignored
|
||||
result = shouldIgnorePath(ctx, mockFS, "project/final")
|
||||
Expect(result).To(BeFalse())
|
||||
})
|
||||
|
||||
It("ignores directories with empty .ndignore file", func() {
|
||||
result := shouldIgnorePath(ctx, mockFS, "empty/subdir")
|
||||
Expect(result).To(BeTrue())
|
||||
})
|
||||
|
||||
It("does not ignore root or empty paths", func() {
|
||||
Expect(shouldIgnorePath(ctx, mockFS, "")).To(BeFalse())
|
||||
Expect(shouldIgnorePath(ctx, mockFS, ".")).To(BeFalse())
|
||||
})
|
||||
|
||||
It("combines patterns from multiple .ndignore files", func() {
|
||||
// Create a more complex hierarchy
|
||||
complexFS := fstest.MapFS{
|
||||
".ndignore": &fstest.MapFile{Data: []byte("*.tmp\n")},
|
||||
"parent": &fstest.MapFile{Mode: fs.ModeDir},
|
||||
"parent/.ndignore": &fstest.MapFile{Data: []byte("test\n")},
|
||||
"parent/test": &fstest.MapFile{Mode: fs.ModeDir},
|
||||
"parent/test/file.mp3": &fstest.MapFile{Data: []byte("audio")},
|
||||
"parent/prod": &fstest.MapFile{Mode: fs.ModeDir},
|
||||
"parent/prod/cache.tmp": &fstest.MapFile{Data: []byte("tmp")},
|
||||
}
|
||||
|
||||
// parent/test should be ignored by parent/.ndignore
|
||||
result := shouldIgnorePath(ctx, complexFS, "parent/test")
|
||||
Expect(result).To(BeTrue())
|
||||
|
||||
// parent/prod/cache.tmp path should be ignored by root .ndignore (*.tmp)
|
||||
result = shouldIgnorePath(ctx, complexFS, "parent/prod/cache.tmp")
|
||||
Expect(result).To(BeTrue())
|
||||
|
||||
// parent/prod directory itself should NOT be ignored
|
||||
result = shouldIgnorePath(ctx, complexFS, "parent/prod")
|
||||
Expect(result).To(BeFalse())
|
||||
})
|
||||
})
|
||||
|
||||
var _ = Describe("resolveFolderPath", func() {
|
||||
var mockFS fs.FS
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user