mirror of
https://github.com/navidrome/navidrome.git
synced 2026-08-31 07:30:32 +00:00
feat: implement plugin cache purging functionality
Signed-off-by: Deluan <deluan@navidrome.org>
This commit is contained in:
Deluan
parent
a05fddbf7d
commit
8e9737ab95
@ -635,7 +635,7 @@ func setViperDefaults() {
|
||||
viper.SetDefault("inspect.backlogtimeout", consts.RequestThrottleBacklogTimeout)
|
||||
viper.SetDefault("plugins.folder", "")
|
||||
viper.SetDefault("plugins.enabled", false)
|
||||
viper.SetDefault("plugins.cachesize", "100MB")
|
||||
viper.SetDefault("plugins.cachesize", "200MB")
|
||||
viper.SetDefault("plugins.autoreload", false)
|
||||
|
||||
// DevFlags. These are used to enable/disable debugging and incomplete features
|
||||
|
||||
189
plugins/cache_purge_test.go
Normal file
189
plugins/cache_purge_test.go
Normal file
@ -0,0 +1,189 @@
|
||||
//go:build !windows
|
||||
|
||||
package plugins
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/dustin/go-humanize"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
var _ = Describe("purgeCacheBySize", func() {
|
||||
var (
|
||||
tmpDir string
|
||||
ctx context.Context
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
var err error
|
||||
ctx = GinkgoT().Context()
|
||||
tmpDir, err = os.MkdirTemp("", "cache-purge-test-*")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
|
||||
AfterEach(func() {
|
||||
os.RemoveAll(tmpDir)
|
||||
})
|
||||
|
||||
createFileWithSize := func(path string, sizeBytes int64, modTime time.Time) {
|
||||
dir := filepath.Dir(path)
|
||||
err := os.MkdirAll(dir, 0755)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
f, err := os.Create(path)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
defer f.Close()
|
||||
|
||||
// Write random data to reach desired size
|
||||
if sizeBytes > 0 {
|
||||
err = f.Truncate(sizeBytes)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
}
|
||||
|
||||
// Set modification time
|
||||
err = os.Chtimes(path, modTime, modTime)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
}
|
||||
|
||||
getDirSize := func(dir string) uint64 {
|
||||
var total uint64
|
||||
err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error {
|
||||
if err != nil || d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
info, err := d.Info()
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
total += uint64(info.Size())
|
||||
return nil
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
return total
|
||||
}
|
||||
|
||||
Context("when maxSize is invalid or zero", func() {
|
||||
It("should not remove any files with invalid size", func() {
|
||||
cacheDir := filepath.Join(tmpDir, "cache")
|
||||
createFileWithSize(filepath.Join(cacheDir, "file1.bin"), 1000, time.Now())
|
||||
createFileWithSize(filepath.Join(cacheDir, "file2.bin"), 1000, time.Now())
|
||||
|
||||
purgeCacheBySize(ctx, cacheDir, "invalid")
|
||||
|
||||
Expect(getDirSize(cacheDir)).To(Equal(uint64(2000)))
|
||||
})
|
||||
|
||||
It("should not remove any files when maxSize is 0", func() {
|
||||
cacheDir := filepath.Join(tmpDir, "cache")
|
||||
createFileWithSize(filepath.Join(cacheDir, "file1.bin"), 1000, time.Now())
|
||||
createFileWithSize(filepath.Join(cacheDir, "file2.bin"), 1000, time.Now())
|
||||
|
||||
purgeCacheBySize(ctx, cacheDir, "0")
|
||||
|
||||
Expect(getDirSize(cacheDir)).To(Equal(uint64(2000)))
|
||||
})
|
||||
})
|
||||
|
||||
Context("when cache directory doesn't exist", func() {
|
||||
It("should not error", func() {
|
||||
nonExistentDir := filepath.Join(tmpDir, "nonexistent")
|
||||
Expect(func() {
|
||||
purgeCacheBySize(ctx, nonExistentDir, "100MB")
|
||||
}).ToNot(Panic())
|
||||
})
|
||||
})
|
||||
|
||||
Context("when total size is under limit", func() {
|
||||
It("should not remove any files", func() {
|
||||
cacheDir := filepath.Join(tmpDir, "cache")
|
||||
createFileWithSize(filepath.Join(cacheDir, "file1.bin"), 1000, time.Now())
|
||||
createFileWithSize(filepath.Join(cacheDir, "file2.bin"), 1000, time.Now())
|
||||
|
||||
purgeCacheBySize(ctx, cacheDir, "10KB")
|
||||
|
||||
Expect(getDirSize(cacheDir)).To(Equal(uint64(2000)))
|
||||
})
|
||||
})
|
||||
|
||||
Context("when total size exceeds limit", func() {
|
||||
It("should remove oldest files first", func() {
|
||||
cacheDir := filepath.Join(tmpDir, "cache")
|
||||
now := time.Now()
|
||||
|
||||
// Create files with different ages (1MB each)
|
||||
oldestFile := filepath.Join(cacheDir, "old.bin")
|
||||
middleFile := filepath.Join(cacheDir, "middle.bin")
|
||||
newestFile := filepath.Join(cacheDir, "new.bin")
|
||||
|
||||
createFileWithSize(oldestFile, 1*1024*1024, now.Add(-3*time.Hour))
|
||||
createFileWithSize(middleFile, 1*1024*1024, now.Add(-2*time.Hour))
|
||||
createFileWithSize(newestFile, 1*1024*1024, now.Add(-1*time.Hour))
|
||||
|
||||
// Set limit to 2MiB - should remove oldest file
|
||||
purgeCacheBySize(ctx, cacheDir, "2MiB")
|
||||
|
||||
// Oldest should be removed
|
||||
_, err := os.Stat(oldestFile)
|
||||
Expect(os.IsNotExist(err)).To(BeTrue(), "oldest file should be removed")
|
||||
|
||||
// Others should remain
|
||||
_, err = os.Stat(middleFile)
|
||||
Expect(err).ToNot(HaveOccurred(), "middle file should remain")
|
||||
|
||||
_, err = os.Stat(newestFile)
|
||||
Expect(err).ToNot(HaveOccurred(), "newest file should remain")
|
||||
})
|
||||
|
||||
It("should remove multiple files to get under limit", func() {
|
||||
cacheDir := filepath.Join(tmpDir, "cache")
|
||||
now := time.Now()
|
||||
|
||||
// Create 5 files, 1MiB each (total 5MiB)
|
||||
for i := 0; i < 5; i++ {
|
||||
path := filepath.Join(cacheDir, filepath.Join("dir", "file"+string(rune('0'+i))+".bin"))
|
||||
createFileWithSize(path, 1*1024*1024, now.Add(-time.Duration(5-i)*time.Hour))
|
||||
}
|
||||
|
||||
// Set limit to 2.5MiB - should remove oldest 3 files (leaving 2MiB)
|
||||
purgeCacheBySize(ctx, cacheDir, "2.5MiB")
|
||||
|
||||
finalSize := getDirSize(cacheDir)
|
||||
limit, _ := humanize.ParseBytes("2.5MiB")
|
||||
Expect(finalSize).To(BeNumerically("<=", limit))
|
||||
})
|
||||
|
||||
It("should remove empty parent directories after removing files", func() {
|
||||
cacheDir := filepath.Join(tmpDir, "cache")
|
||||
now := time.Now()
|
||||
|
||||
// Create files in subdirectories
|
||||
oldFile := filepath.Join(cacheDir, "subdir1", "old.bin")
|
||||
newFile := filepath.Join(cacheDir, "subdir2", "new.bin")
|
||||
|
||||
createFileWithSize(oldFile, 2*1024*1024, now.Add(-2*time.Hour))
|
||||
createFileWithSize(newFile, 2*1024*1024, now.Add(-1*time.Hour))
|
||||
|
||||
// Set limit to 2MiB - should remove old file and its parent dir
|
||||
purgeCacheBySize(ctx, cacheDir, "2MiB")
|
||||
|
||||
// Old file and its parent dir should be removed
|
||||
_, err := os.Stat(oldFile)
|
||||
Expect(os.IsNotExist(err)).To(BeTrue())
|
||||
|
||||
_, err = os.Stat(filepath.Join(cacheDir, "subdir1"))
|
||||
Expect(os.IsNotExist(err)).To(BeTrue(), "empty parent directory should be removed")
|
||||
|
||||
// New file and its parent dir should remain
|
||||
_, err = os.Stat(newFile)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
_, err = os.Stat(filepath.Join(cacheDir, "subdir2"))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
})
|
||||
})
|
||||
@ -1,20 +1,24 @@
|
||||
package plugins
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/dustin/go-humanize"
|
||||
extism "github.com/extism/go-sdk"
|
||||
"github.com/navidrome/navidrome/conf"
|
||||
"github.com/navidrome/navidrome/core/agents"
|
||||
@ -140,8 +144,11 @@ func (m *Manager) Start(ctx context.Context) error {
|
||||
m.ctx, m.cancel = context.WithCancel(ctx)
|
||||
|
||||
// Initialize wazero compilation cache for better performance
|
||||
cacheDir := filepath.Join(conf.Server.CacheFolder, "plugins")
|
||||
purgeCacheBySize(ctx, cacheDir, conf.Server.Plugins.CacheSize)
|
||||
|
||||
var err error
|
||||
m.cache, err = wazero.NewCompilationCacheWithDir(filepath.Join(conf.Server.CacheFolder, "plugins"))
|
||||
m.cache, err = wazero.NewCompilationCacheWithDir(cacheDir)
|
||||
if err != nil {
|
||||
log.Error(ctx, "Failed to create wazero compilation cache", err)
|
||||
return fmt.Errorf("creating wazero compilation cache: %w", err)
|
||||
@ -685,3 +692,81 @@ func toExtismLogLevel(level log.Level) extism.LogLevel {
|
||||
return extism.LogLevelInfo
|
||||
}
|
||||
}
|
||||
|
||||
// purgeCacheBySize removes the oldest files in dir until its total size is
|
||||
// lower than or equal to maxSize. maxSize should be a human-readable string
|
||||
// like "10MB" or "200K". If parsing fails or maxSize is "0", the function is
|
||||
// a no-op.
|
||||
func purgeCacheBySize(ctx context.Context, dir, maxSize string) {
|
||||
sizeLimit, err := humanize.ParseBytes(maxSize)
|
||||
if err != nil || sizeLimit == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
type fileInfo struct {
|
||||
path string
|
||||
size uint64
|
||||
mod int64
|
||||
}
|
||||
|
||||
var files []fileInfo
|
||||
var total uint64
|
||||
|
||||
walk := func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
log.Trace(ctx, "Failed to access plugin cache entry", "path", path, err)
|
||||
return nil //nolint:nilerr
|
||||
}
|
||||
if d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
info, err := d.Info()
|
||||
if err != nil {
|
||||
log.Trace(ctx, "Failed to get file info for plugin cache entry", "path", path, err)
|
||||
return nil //nolint:nilerr
|
||||
}
|
||||
files = append(files, fileInfo{
|
||||
path: path,
|
||||
size: uint64(info.Size()),
|
||||
mod: info.ModTime().UnixMilli(),
|
||||
})
|
||||
total += uint64(info.Size())
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := filepath.WalkDir(dir, walk); err != nil {
|
||||
if !os.IsNotExist(err) {
|
||||
log.Warn(ctx, "Failed to traverse plugin cache directory", "path", dir, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
log.Trace(ctx, "Current plugin cache size", "path", dir, "size", humanize.Bytes(total), "sizeLimit", humanize.Bytes(sizeLimit))
|
||||
if total <= sizeLimit {
|
||||
return
|
||||
}
|
||||
|
||||
log.Debug(ctx, "Purging plugin cache", "path", dir, "sizeLimit", humanize.Bytes(sizeLimit), "currentSize", humanize.Bytes(total))
|
||||
slices.SortFunc(files, func(i, j fileInfo) int { return cmp.Compare(i.mod, j.mod) })
|
||||
|
||||
for _, f := range files {
|
||||
if total <= sizeLimit {
|
||||
break
|
||||
}
|
||||
if err := os.Remove(f.path); err != nil {
|
||||
log.Warn(ctx, "Failed to remove plugin cache entry", "path", f.path, "size", humanize.Bytes(f.size), err)
|
||||
continue
|
||||
}
|
||||
total -= f.size
|
||||
log.Debug(ctx, "Removed plugin cache entry", "path", f.path, "size", humanize.Bytes(f.size), "time", time.UnixMilli(f.mod), "remainingSize", humanize.Bytes(total))
|
||||
|
||||
// Remove empty parent directories
|
||||
dirPath := filepath.Dir(f.path)
|
||||
for dirPath != dir {
|
||||
if err := os.Remove(dirPath); err != nil {
|
||||
break
|
||||
}
|
||||
dirPath = filepath.Dir(dirPath)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user