From b72597821a0bdbf2624985196bf16c516ed09a2b Mon Sep 17 00:00:00 2001 From: Deluan Date: Tue, 21 Jul 2026 22:53:00 -0400 Subject: [PATCH] feat(artwork): add content-addressed originals store --- core/artwork/originals/originals.go | 112 ++++++++++++++++++ .../artwork/originals/originals_suite_test.go | 17 +++ core/artwork/originals/originals_test.go | 80 +++++++++++++ go.mod | 2 +- 4 files changed, 210 insertions(+), 1 deletion(-) create mode 100644 core/artwork/originals/originals.go create mode 100644 core/artwork/originals/originals_suite_test.go create mode 100644 core/artwork/originals/originals_test.go diff --git a/core/artwork/originals/originals.go b/core/artwork/originals/originals.go new file mode 100644 index 000000000..12cfe56cc --- /dev/null +++ b/core/artwork/originals/originals.go @@ -0,0 +1,112 @@ +// Package originals is the content-addressed store for artwork images that have +// no library file backing them (external downloads, embedded extractions, generated). +package originals + +import ( + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + + "github.com/zeebo/xxh3" +) + +func Hash(r io.Reader) (string, error) { + d := xxh3.New() + if _, err := io.Copy(d, r); err != nil { + return "", err + } + return fmt.Sprintf("%016x", d.Sum64()), nil +} + +type Store struct { + root string +} + +func New(rootDir string) *Store { + return &Store{root: rootDir} +} + +// extForMime is deliberately NOT mime.ExtensionsByType: extensions are baked into +// content-addressed paths and re-derived on Open, so they must be stable across OSes. +func extForMime(m string) string { + switch m { + case "image/jpeg": + return ".jpg" + case "image/png": + return ".png" + case "image/gif": + return ".gif" + case "image/webp": + return ".webp" + } + return ".img" +} + +func (s *Store) path(hash, mimeType string) string { + return filepath.Join(s.root, hash[0:2], hash[2:4], hash+extForMime(mimeType)) +} + +func (s *Store) Write(hash, mimeType string, r io.Reader) error { + dst := s.path(hash, mimeType) + if _, err := os.Stat(dst); err == nil { + return nil + } + if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil { + return err + } + tmp, err := os.CreateTemp(filepath.Dir(dst), "."+hash+".tmp*") + if err != nil { + return err + } + defer os.Remove(tmp.Name()) + if _, err := io.Copy(tmp, r); err != nil { + tmp.Close() + return err + } + if err := tmp.Close(); err != nil { + return err + } + return os.Rename(tmp.Name(), dst) +} + +func (s *Store) Open(hash, mimeType string) (io.ReadCloser, error) { + return os.Open(s.path(hash, mimeType)) +} + +func (s *Store) Remove(hash, mimeType string) error { + err := os.Remove(s.path(hash, mimeType)) + if errors.Is(err, fs.ErrNotExist) { + return nil + } + return err +} + +func (s *Store) Sweep(keep func(hash string) bool) (int, error) { + removed := 0 + err := filepath.WalkDir(s.root, func(path string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() { + return err + } + name := d.Name() + if strings.HasPrefix(name, ".") { // in-flight temp files + return nil + } + hash := strings.TrimSuffix(name, filepath.Ext(name)) + if !keep(hash) { + // #nosec G122 -- path comes from WalkDir over our own store root, no attacker-controlled symlinks + if err := os.Remove(path); err != nil { + return err + } + removed++ + } + return nil + }) + if errors.Is(err, fs.ErrNotExist) { + return removed, nil + } + return removed, err +} diff --git a/core/artwork/originals/originals_suite_test.go b/core/artwork/originals/originals_suite_test.go new file mode 100644 index 000000000..2686f9ec9 --- /dev/null +++ b/core/artwork/originals/originals_suite_test.go @@ -0,0 +1,17 @@ +package originals_test + +import ( + "testing" + + "github.com/navidrome/navidrome/log" + "github.com/navidrome/navidrome/tests" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestOriginals(t *testing.T) { + tests.Init(t, false) + log.SetLevel(log.LevelFatal) + RegisterFailHandler(Fail) + RunSpecs(t, "Originals Store Suite") +} diff --git a/core/artwork/originals/originals_test.go b/core/artwork/originals/originals_test.go new file mode 100644 index 000000000..caa861a6f --- /dev/null +++ b/core/artwork/originals/originals_test.go @@ -0,0 +1,80 @@ +package originals_test + +import ( + "bytes" + "io" + "os" + "path/filepath" + + "github.com/navidrome/navidrome/core/artwork/originals" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Originals Store", func() { + var store *originals.Store + var root string + + BeforeEach(func() { + root = GinkgoT().TempDir() + store = originals.New(root) + }) + + It("hashes deterministically", func() { + h1, err := originals.Hash(bytes.NewReader([]byte("some image bytes"))) + Expect(err).ToNot(HaveOccurred()) + h2, _ := originals.Hash(bytes.NewReader([]byte("some image bytes"))) + Expect(h1).To(Equal(h2)) + Expect(h1).To(HaveLen(16)) + h3, _ := originals.Hash(bytes.NewReader([]byte("other bytes"))) + Expect(h3).ToNot(Equal(h1)) + }) + + It("writes sharded and reads back", func() { + data := []byte("jpeg-bytes") + h, _ := originals.Hash(bytes.NewReader(data)) + Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed()) + + Expect(filepath.Join(root, h[0:2], h[2:4], h+".jpg")).To(BeAnExistingFile()) + + rc, err := store.Open(h, "image/jpeg") + Expect(err).ToNot(HaveOccurred()) + defer rc.Close() + got, _ := io.ReadAll(rc) + Expect(got).To(Equal(data)) + }) + + It("is idempotent on duplicate writes", func() { + data := []byte("dup") + h, _ := originals.Hash(bytes.NewReader(data)) + Expect(store.Write(h, "image/png", bytes.NewReader(data))).To(Succeed()) + Expect(store.Write(h, "image/png", bytes.NewReader(data))).To(Succeed()) + }) + + It("returns fs.ErrNotExist for missing images", func() { + _, err := store.Open("beefbeefbeefbeef", "image/jpeg") + Expect(os.IsNotExist(err)).To(BeTrue()) + }) + + It("removes without error when already gone", func() { + Expect(store.Remove("beefbeefbeefbeef", "image/jpeg")).To(Succeed()) + }) + + It("sweeps unknown files, keeps known ones", func() { + d1 := []byte("keep-me") + h1, _ := originals.Hash(bytes.NewReader(d1)) + Expect(store.Write(h1, "image/jpeg", bytes.NewReader(d1))).To(Succeed()) + d2 := []byte("orphan") + h2, _ := originals.Hash(bytes.NewReader(d2)) + Expect(store.Write(h2, "image/jpeg", bytes.NewReader(d2))).To(Succeed()) + + removed, err := store.Sweep(func(h string) bool { return h == h1 }) + Expect(err).ToNot(HaveOccurred()) + Expect(removed).To(Equal(1)) + _, err = store.Open(h2, "image/jpeg") + Expect(os.IsNotExist(err)).To(BeTrue()) + rc, err := store.Open(h1, "image/jpeg") + Expect(err).ToNot(HaveOccurred()) + rc.Close() + }) +}) diff --git a/go.mod b/go.mod index 5488b41e4..b72f4035c 100644 --- a/go.mod +++ b/go.mod @@ -57,6 +57,7 @@ require ( github.com/tetratelabs/wazero v1.12.0 github.com/unrolled/secure v1.17.0 github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342 + github.com/zeebo/xxh3 v1.1.0 go.senan.xyz/taglib v0.11.1 go.uber.org/goleak v1.3.0 golang.org/x/image v0.44.0 @@ -128,7 +129,6 @@ require ( github.com/subosito/gotenv v1.6.0 // indirect github.com/tetratelabs/wabin v0.0.0-20230304001439-f6f874872834 // indirect github.com/valyala/fastjson v1.6.10 // indirect - github.com/zeebo/xxh3 v1.1.0 // indirect go.opentelemetry.io/proto/otlp v1.10.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect