diff --git a/conf/configuration.go b/conf/configuration.go index 0aa0a4f18..ba907883f 100644 --- a/conf/configuration.go +++ b/conf/configuration.go @@ -635,7 +635,7 @@ func setViperDefaults() { viper.SetDefault("inspect.backlogtimeout", consts.RequestThrottleBacklogTimeout) viper.SetDefault("plugins.folder", "") viper.SetDefault("plugins.enabled", false) - viper.SetDefault("plugins.cachesize", "100MB") + viper.SetDefault("plugins.cachesize", "200MB") viper.SetDefault("plugins.autoreload", false) // DevFlags. These are used to enable/disable debugging and incomplete features diff --git a/plugins/cache_purge_test.go b/plugins/cache_purge_test.go new file mode 100644 index 000000000..1afe26d41 --- /dev/null +++ b/plugins/cache_purge_test.go @@ -0,0 +1,189 @@ +//go:build !windows + +package plugins + +import ( + "context" + "os" + "path/filepath" + "time" + + "github.com/dustin/go-humanize" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("purgeCacheBySize", func() { + var ( + tmpDir string + ctx context.Context + ) + + BeforeEach(func() { + var err error + ctx = GinkgoT().Context() + tmpDir, err = os.MkdirTemp("", "cache-purge-test-*") + Expect(err).ToNot(HaveOccurred()) + }) + + AfterEach(func() { + os.RemoveAll(tmpDir) + }) + + createFileWithSize := func(path string, sizeBytes int64, modTime time.Time) { + dir := filepath.Dir(path) + err := os.MkdirAll(dir, 0755) + Expect(err).ToNot(HaveOccurred()) + + f, err := os.Create(path) + Expect(err).ToNot(HaveOccurred()) + defer f.Close() + + // Write random data to reach desired size + if sizeBytes > 0 { + err = f.Truncate(sizeBytes) + Expect(err).ToNot(HaveOccurred()) + } + + // Set modification time + err = os.Chtimes(path, modTime, modTime) + Expect(err).ToNot(HaveOccurred()) + } + + getDirSize := func(dir string) uint64 { + var total uint64 + err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error { + if err != nil || d.IsDir() { + return nil + } + info, err := d.Info() + if err != nil { + return nil + } + total += uint64(info.Size()) + return nil + }) + Expect(err).ToNot(HaveOccurred()) + return total + } + + Context("when maxSize is invalid or zero", func() { + It("should not remove any files with invalid size", func() { + cacheDir := filepath.Join(tmpDir, "cache") + createFileWithSize(filepath.Join(cacheDir, "file1.bin"), 1000, time.Now()) + createFileWithSize(filepath.Join(cacheDir, "file2.bin"), 1000, time.Now()) + + purgeCacheBySize(ctx, cacheDir, "invalid") + + Expect(getDirSize(cacheDir)).To(Equal(uint64(2000))) + }) + + It("should not remove any files when maxSize is 0", func() { + cacheDir := filepath.Join(tmpDir, "cache") + createFileWithSize(filepath.Join(cacheDir, "file1.bin"), 1000, time.Now()) + createFileWithSize(filepath.Join(cacheDir, "file2.bin"), 1000, time.Now()) + + purgeCacheBySize(ctx, cacheDir, "0") + + Expect(getDirSize(cacheDir)).To(Equal(uint64(2000))) + }) + }) + + Context("when cache directory doesn't exist", func() { + It("should not error", func() { + nonExistentDir := filepath.Join(tmpDir, "nonexistent") + Expect(func() { + purgeCacheBySize(ctx, nonExistentDir, "100MB") + }).ToNot(Panic()) + }) + }) + + Context("when total size is under limit", func() { + It("should not remove any files", func() { + cacheDir := filepath.Join(tmpDir, "cache") + createFileWithSize(filepath.Join(cacheDir, "file1.bin"), 1000, time.Now()) + createFileWithSize(filepath.Join(cacheDir, "file2.bin"), 1000, time.Now()) + + purgeCacheBySize(ctx, cacheDir, "10KB") + + Expect(getDirSize(cacheDir)).To(Equal(uint64(2000))) + }) + }) + + Context("when total size exceeds limit", func() { + It("should remove oldest files first", func() { + cacheDir := filepath.Join(tmpDir, "cache") + now := time.Now() + + // Create files with different ages (1MB each) + oldestFile := filepath.Join(cacheDir, "old.bin") + middleFile := filepath.Join(cacheDir, "middle.bin") + newestFile := filepath.Join(cacheDir, "new.bin") + + createFileWithSize(oldestFile, 1*1024*1024, now.Add(-3*time.Hour)) + createFileWithSize(middleFile, 1*1024*1024, now.Add(-2*time.Hour)) + createFileWithSize(newestFile, 1*1024*1024, now.Add(-1*time.Hour)) + + // Set limit to 2MiB - should remove oldest file + purgeCacheBySize(ctx, cacheDir, "2MiB") + + // Oldest should be removed + _, err := os.Stat(oldestFile) + Expect(os.IsNotExist(err)).To(BeTrue(), "oldest file should be removed") + + // Others should remain + _, err = os.Stat(middleFile) + Expect(err).ToNot(HaveOccurred(), "middle file should remain") + + _, err = os.Stat(newestFile) + Expect(err).ToNot(HaveOccurred(), "newest file should remain") + }) + + It("should remove multiple files to get under limit", func() { + cacheDir := filepath.Join(tmpDir, "cache") + now := time.Now() + + // Create 5 files, 1MiB each (total 5MiB) + for i := 0; i < 5; i++ { + path := filepath.Join(cacheDir, filepath.Join("dir", "file"+string(rune('0'+i))+".bin")) + createFileWithSize(path, 1*1024*1024, now.Add(-time.Duration(5-i)*time.Hour)) + } + + // Set limit to 2.5MiB - should remove oldest 3 files (leaving 2MiB) + purgeCacheBySize(ctx, cacheDir, "2.5MiB") + + finalSize := getDirSize(cacheDir) + limit, _ := humanize.ParseBytes("2.5MiB") + Expect(finalSize).To(BeNumerically("<=", limit)) + }) + + It("should remove empty parent directories after removing files", func() { + cacheDir := filepath.Join(tmpDir, "cache") + now := time.Now() + + // Create files in subdirectories + oldFile := filepath.Join(cacheDir, "subdir1", "old.bin") + newFile := filepath.Join(cacheDir, "subdir2", "new.bin") + + createFileWithSize(oldFile, 2*1024*1024, now.Add(-2*time.Hour)) + createFileWithSize(newFile, 2*1024*1024, now.Add(-1*time.Hour)) + + // Set limit to 2MiB - should remove old file and its parent dir + purgeCacheBySize(ctx, cacheDir, "2MiB") + + // Old file and its parent dir should be removed + _, err := os.Stat(oldFile) + Expect(os.IsNotExist(err)).To(BeTrue()) + + _, err = os.Stat(filepath.Join(cacheDir, "subdir1")) + Expect(os.IsNotExist(err)).To(BeTrue(), "empty parent directory should be removed") + + // New file and its parent dir should remain + _, err = os.Stat(newFile) + Expect(err).ToNot(HaveOccurred()) + + _, err = os.Stat(filepath.Join(cacheDir, "subdir2")) + Expect(err).ToNot(HaveOccurred()) + }) + }) +}) diff --git a/plugins/manager.go b/plugins/manager.go index 051853c9e..9f02c5d54 100644 --- a/plugins/manager.go +++ b/plugins/manager.go @@ -1,20 +1,24 @@ package plugins import ( + "cmp" "context" "crypto/rand" "encoding/json" "errors" "fmt" "io" + "io/fs" "net/http" "os" "path/filepath" + "slices" "strings" "sync" "sync/atomic" "time" + "github.com/dustin/go-humanize" extism "github.com/extism/go-sdk" "github.com/navidrome/navidrome/conf" "github.com/navidrome/navidrome/core/agents" @@ -140,8 +144,11 @@ func (m *Manager) Start(ctx context.Context) error { m.ctx, m.cancel = context.WithCancel(ctx) // Initialize wazero compilation cache for better performance + cacheDir := filepath.Join(conf.Server.CacheFolder, "plugins") + purgeCacheBySize(ctx, cacheDir, conf.Server.Plugins.CacheSize) + var err error - m.cache, err = wazero.NewCompilationCacheWithDir(filepath.Join(conf.Server.CacheFolder, "plugins")) + m.cache, err = wazero.NewCompilationCacheWithDir(cacheDir) if err != nil { log.Error(ctx, "Failed to create wazero compilation cache", err) return fmt.Errorf("creating wazero compilation cache: %w", err) @@ -685,3 +692,81 @@ func toExtismLogLevel(level log.Level) extism.LogLevel { return extism.LogLevelInfo } } + +// purgeCacheBySize removes the oldest files in dir until its total size is +// lower than or equal to maxSize. maxSize should be a human-readable string +// like "10MB" or "200K". If parsing fails or maxSize is "0", the function is +// a no-op. +func purgeCacheBySize(ctx context.Context, dir, maxSize string) { + sizeLimit, err := humanize.ParseBytes(maxSize) + if err != nil || sizeLimit == 0 { + return + } + + type fileInfo struct { + path string + size uint64 + mod int64 + } + + var files []fileInfo + var total uint64 + + walk := func(path string, d fs.DirEntry, err error) error { + if err != nil { + log.Trace(ctx, "Failed to access plugin cache entry", "path", path, err) + return nil //nolint:nilerr + } + if d.IsDir() { + return nil + } + info, err := d.Info() + if err != nil { + log.Trace(ctx, "Failed to get file info for plugin cache entry", "path", path, err) + return nil //nolint:nilerr + } + files = append(files, fileInfo{ + path: path, + size: uint64(info.Size()), + mod: info.ModTime().UnixMilli(), + }) + total += uint64(info.Size()) + return nil + } + + if err := filepath.WalkDir(dir, walk); err != nil { + if !os.IsNotExist(err) { + log.Warn(ctx, "Failed to traverse plugin cache directory", "path", dir, err) + } + return + } + + log.Trace(ctx, "Current plugin cache size", "path", dir, "size", humanize.Bytes(total), "sizeLimit", humanize.Bytes(sizeLimit)) + if total <= sizeLimit { + return + } + + log.Debug(ctx, "Purging plugin cache", "path", dir, "sizeLimit", humanize.Bytes(sizeLimit), "currentSize", humanize.Bytes(total)) + slices.SortFunc(files, func(i, j fileInfo) int { return cmp.Compare(i.mod, j.mod) }) + + for _, f := range files { + if total <= sizeLimit { + break + } + if err := os.Remove(f.path); err != nil { + log.Warn(ctx, "Failed to remove plugin cache entry", "path", f.path, "size", humanize.Bytes(f.size), err) + continue + } + total -= f.size + log.Debug(ctx, "Removed plugin cache entry", "path", f.path, "size", humanize.Bytes(f.size), "time", time.UnixMilli(f.mod), "remainingSize", humanize.Bytes(total)) + + // Remove empty parent directories + dirPath := filepath.Dir(f.path) + for dirPath != dir { + if err := os.Remove(dirPath); err != nil { + break + } + dirPath = filepath.Dir(dirPath) + } + } +}