From aa0824e03bfd617caeddfe52a41a421a77e6289f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Deluan=20Quint=C3=A3o?= Date: Fri, 14 Aug 2026 09:17:00 -0400 Subject: [PATCH 01/27] feat(jellyfin): add System/Endpoint so Finamp's connection test passes (#5955) Finamp's connection test GETs /System/Endpoint and treats anything other than a 200 carrying an IsInNetwork key as "not a Jellyfin server", so the test failed against Navidrome and dual-connection setups could never switch to the local address. IsInNetwork mirrors Jellyfin's default LAN set (NetworkManager with no LocalNetworkSubnets configured): loopback, the RFC 1918 ranges, fc00::/7 and fe80::/10. Notably that set omits 169.254.0.0/16, so Go's IsLinkLocalUnicast is deliberately restricted to its IPv6 half. IsLocal mirrors HttpContext.IsLocal(): the caller shares the connection's local address, not merely "is loopback". It falls back to the loopback check when the local address is unavailable. --- server/jellyfin/README.md | 2 +- server/jellyfin/api.go | 1 + server/jellyfin/dto/dto.go | 6 ++++ server/jellyfin/e2e/system_test.go | 16 +++++++++ server/jellyfin/system.go | 44 +++++++++++++++++++++++ server/jellyfin/system_test.go | 56 ++++++++++++++++++++++++++++++ 6 files changed, 124 insertions(+), 1 deletion(-) diff --git a/server/jellyfin/README.md b/server/jellyfin/README.md index 195173c2a..5759e6cc0 100644 --- a/server/jellyfin/README.md +++ b/server/jellyfin/README.md @@ -112,7 +112,7 @@ returns direct children only (no tracks — no track is a library's direct child | Area | Endpoints | |---|---| -| Handshake / system | `GET System/Info/Public`, `GET System/Info` (authenticated), `GET`/`POST System/Ping`, `GET QuickConnect/Enabled` | +| Handshake / system | `GET System/Info/Public`, `GET System/Info` (authenticated), `GET`/`POST System/Ping`, `GET System/Endpoint` (authenticated), `GET QuickConnect/Enabled` | | Auth | `POST Users/AuthenticateByName`, `GET Users/Public` | | Users | `GET UserViews`, `GET Users/{userId}/Views`, `GET Users/Me`, `GET Users/{userId}` | | Browsing | `GET Items`, `GET Users/{userId}/Items`, `GET Items/{itemId}`, `GET Users/{userId}/Items/{itemId}`, `GET Users/{userId}/Items/Latest`, `DELETE Items/{itemId}` (playlists only) | diff --git a/server/jellyfin/api.go b/server/jellyfin/api.go index 8a175df35..d64366a63 100644 --- a/server/jellyfin/api.go +++ b/server/jellyfin/api.go @@ -104,6 +104,7 @@ func (api *Router) routes() http.Handler { // player) even before the first playback report. r.Use(api.withPlayer) r.Get("/system/info", api.getSystemInfo) + r.Get("/system/endpoint", api.getEndpointInfo) r.Get("/userviews", api.getUserViews) r.Get("/users/{userId}/views", api.getUserViews) r.Get("/users/me", api.getCurrentUser) diff --git a/server/jellyfin/dto/dto.go b/server/jellyfin/dto/dto.go index 5a2a4c2b1..58a6b919e 100644 --- a/server/jellyfin/dto/dto.go +++ b/server/jellyfin/dto/dto.go @@ -20,6 +20,12 @@ type SystemInfo struct { CachePath string `json:"CachePath,omitempty"` } +// EndPointInfo describes the caller's network location (GET /System/Endpoint). +type EndPointInfo struct { + IsLocal bool `json:"IsLocal"` + IsInNetwork bool `json:"IsInNetwork"` +} + type NameGuidPair struct { Name string `json:"Name"` Id string `json:"Id"` diff --git a/server/jellyfin/e2e/system_test.go b/server/jellyfin/e2e/system_test.go index c6f9145d5..ac8c350c4 100644 --- a/server/jellyfin/e2e/system_test.go +++ b/server/jellyfin/e2e/system_test.go @@ -51,6 +51,22 @@ var _ = Describe("System", func() { }) }) + Describe("GET /System/Endpoint", func() { + It("always reports IsInNetwork, which Finamp's connection test probes for", func() { + w := getAs(regularUser, "/System/Endpoint") + Expect(w.Code).To(Equal(http.StatusOK)) + var info map[string]any + parseInto(w, &info) + Expect(info).To(HaveKey("IsInNetwork")) + Expect(info).To(HaveKey("IsLocal")) + }) + + It("rejects unauthenticated requests", func() { + w := rawReq("GET", "/System/Endpoint", "") + Expect(w.Code).To(Equal(http.StatusUnauthorized)) + }) + }) + Describe("GET/POST /System/Ping", func() { It("answers GET with a plain-text server name", func() { w := rawReq("GET", "/System/Ping", "") diff --git a/server/jellyfin/system.go b/server/jellyfin/system.go index ea553fd87..8b847987f 100644 --- a/server/jellyfin/system.go +++ b/server/jellyfin/system.go @@ -5,7 +5,9 @@ import ( "encoding/hex" "errors" "fmt" + "net" "net/http" + "net/netip" "path" "strings" @@ -108,6 +110,48 @@ func (api *Router) ping(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte(api.serverName())) } +// getEndpointInfo answers /System/Endpoint, which Finamp's connection test uses to pick between a +// dual-connection setup's addresses; a missing IsInNetwork reads to it as "not a Jellyfin server". +func (api *Router) getEndpointInfo(w http.ResponseWriter, r *http.Request) { + remote := remoteIP(r) + api.ok(w, r, dto.EndPointInfo{ + IsLocal: isSameMachine(r, remote), + IsInNetwork: isInLocalNetwork(remote), + }) +} + +// isInLocalNetwork mirrors Jellyfin's default LAN set (NetworkManager.UpdateSettings with no +// LocalNetworkSubnets configured): loopback, the RFC 1918 ranges, fc00::/7 and fe80::/10. +func isInLocalNetwork(ip netip.Addr) bool { + return ip.IsLoopback() || ip.IsPrivate() || (ip.Is6() && ip.IsLinkLocalUnicast()) +} + +// isSameMachine mirrors Jellyfin's HttpContext.IsLocal(): the caller shares the connection's local +// address. The local address is missing in tests and unreliable behind a proxy, so fall back to loopback. +func isSameMachine(r *http.Request, remote netip.Addr) bool { + local, ok := r.Context().Value(http.LocalAddrContextKey).(net.Addr) + if !ok { + return remote.IsLoopback() + } + return parseIP(local.String()) == remote +} + +// remoteIP parses RemoteAddr, which the RealIP middleware may have rewritten to a bare IP. +func remoteIP(r *http.Request) netip.Addr { + return parseIP(r.RemoteAddr) +} + +func parseIP(addr string) netip.Addr { + if h, _, err := net.SplitHostPort(addr); err == nil { + addr = h + } + ip, err := netip.ParseAddr(addr) + if err != nil { + return netip.Addr{} + } + return ip.Unmap() +} + func (api *Router) quickConnectEnabled(w http.ResponseWriter, r *http.Request) { api.ok(w, r, false) } diff --git a/server/jellyfin/system_test.go b/server/jellyfin/system_test.go index a7f7df6a4..4b3ac240e 100644 --- a/server/jellyfin/system_test.go +++ b/server/jellyfin/system_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "net" "net/http" "net/http/httptest" @@ -88,6 +89,61 @@ var _ = Describe("System", func() { Expect(w.Body.String()).To(HavePrefix("Navidrome")) }) + DescribeTable("reports the caller's network location on /System/Endpoint", + func(remoteAddr string, isLocal, isInNetwork bool) { + w := httptest.NewRecorder() + r := httptest.NewRequest("GET", "/System/Endpoint", nil) + r.RemoteAddr = remoteAddr + api.getEndpointInfo(w, r) + + Expect(w.Code).To(Equal(http.StatusOK)) + // Finamp's connection test only probes for the key's presence, so it must always be emitted. + Expect(w.Body.String()).To(ContainSubstring(`"IsInNetwork"`)) + var info dto.EndPointInfo + Expect(json.Unmarshal(w.Body.Bytes(), &info)).To(Succeed()) + Expect(info.IsLocal).To(Equal(isLocal)) + Expect(info.IsInNetwork).To(Equal(isInNetwork)) + }, + Entry("loopback", "127.0.0.1:12345", true, true), + Entry("IPv6 loopback", "[::1]:12345", true, true), + Entry("LAN address", "192.168.1.20:54321", false, true), + Entry("bare IP, as left by the RealIP middleware", "10.0.0.5", false, true), + Entry("IPv4-mapped IPv6 LAN address", "[::ffff:172.16.0.9]:80", false, true), + Entry("IPv6 link-local", "[fe80::1]:80", false, true), + Entry("IPv6 unique-local", "[fd00::1]:80", false, true), + // Jellyfin's default LAN set omits 169.254.0.0/16, so we do too. + Entry("IPv4 link-local", "169.254.1.1:80", false, false), + Entry("public address", "8.8.8.8:443", false, false), + Entry("unparseable address", "not-an-ip", false, false), + ) + + It("reports IsLocal when the caller shares the connection's local address", func() { + w := httptest.NewRecorder() + r := httptest.NewRequest("GET", "/System/Endpoint", nil) + r.RemoteAddr = "192.168.1.20:54321" + ctx := context.WithValue(r.Context(), http.LocalAddrContextKey, + &net.TCPAddr{IP: net.ParseIP("192.168.1.20"), Port: 4533}) + api.getEndpointInfo(w, r.WithContext(ctx)) + + var info dto.EndPointInfo + Expect(json.Unmarshal(w.Body.Bytes(), &info)).To(Succeed()) + Expect(info.IsLocal).To(BeTrue()) + }) + + It("does not report IsLocal for a different host on the same LAN", func() { + w := httptest.NewRecorder() + r := httptest.NewRequest("GET", "/System/Endpoint", nil) + r.RemoteAddr = "192.168.1.99:54321" + ctx := context.WithValue(r.Context(), http.LocalAddrContextKey, + &net.TCPAddr{IP: net.ParseIP("192.168.1.20"), Port: 4533}) + api.getEndpointInfo(w, r.WithContext(ctx)) + + var info dto.EndPointInfo + Expect(json.Unmarshal(w.Body.Bytes(), &info)).To(Succeed()) + Expect(info.IsLocal).To(BeFalse()) + Expect(info.IsInNetwork).To(BeTrue()) + }) + It("reports quick connect as disabled", func() { w := httptest.NewRecorder() r := httptest.NewRequest("GET", "/QuickConnect/Enabled", nil) From 95615bcb18ece3f88af0d4dccb0e9faa25fb3e21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Deluan=20Quint=C3=A3o?= Date: Fri, 14 Aug 2026 09:43:13 -0400 Subject: [PATCH 02/27] fix(artwork): serve images whose format has no registered decoder (#5952) * fix(artwork): serve images whose format has no registered decoder The new pipeline derives dimensions, mime and the placeholder hashes at resolution time, so a decode became a precondition for recording artwork at all. An image in a format Go has no decoder for therefore failed acquisition, retried until the 12h budget ran out, and then settled as absent, serving a placeholder from that point on. The old pipeline decoded only to resize and fell back to the original bytes when that failed, so these covers used to work. A local file is picked by matching an image extension, so bytes it cannot decode are most likely a codec we lack: image.ErrFormat on a folder, upload or embedded source now yields an Artwork row carrying just the hash and mime, and the bytes stay servable. An external response carries no such guarantee, so it still fails and retries rather than pinning a non-image body as a cover. A corrupt image of a known format and an over-cap declared size still fail, so the decompression bomb guard is unchanged. Absent rows recorded by earlier builds are re-resolved by the existing stale-absent recheck within a day, so no epoch bump is needed to repair them. Reusing a stored image now re-decodes when it carries no dimensions, so a row recorded while a decoder was missing can still be upgraded later. Registers jxl, heic and heif in mime_types.yaml: image detection resolves the extension through the host MIME table, and the Alpine release image ships no /etc/mime.types, so those covers were never recorded in folder.ImageFiles there and never reached the pipeline at all. * fix(artwork): never record empty bytes as artwork image.DecodeConfig returns image.ErrFormat for an empty payload just as it does for a codec with no registered decoder, so a zero-byte cover file was recorded as found artwork and served as an empty response instead of falling back to the placeholder. A truncated image of a known format already fails with unexpected EOF rather than ErrFormat, so only the empty case needed the guard. --- core/artwork/e2e/acquire_serve_test.go | 23 ++++++ core/artwork/processor.go | 23 +++++- core/artwork/processor_test.go | 108 ++++++++++++++++++++++++- resources/mime_types.yaml | 3 + 4 files changed, 154 insertions(+), 3 deletions(-) diff --git a/core/artwork/e2e/acquire_serve_test.go b/core/artwork/e2e/acquire_serve_test.go index d6c374e39..0a9363b5d 100644 --- a/core/artwork/e2e/acquire_serve_test.go +++ b/core/artwork/e2e/acquire_serve_test.go @@ -6,6 +6,7 @@ import ( "errors" "io" "os" + "path/filepath" "time" "github.com/navidrome/navidrome/conf" @@ -116,6 +117,25 @@ var _ = Describe("Acquisition → serve loop", func() { albumRepo.SetData(model.Albums{{ID: albumID, Name: "Album", FolderIDs: []string{"f1"}, LibraryID: 0}}) } + It("acquires and serves a cover whose format has no registered decoder (#5950)", func() { + libDir := GinkgoT().TempDir() + Expect(os.MkdirAll(filepath.Join(libDir, "an-album"), 0755)).To(Succeed()) + Expect(os.WriteFile(filepath.Join(libDir, "an-album", "cover.jxl"), jxlFixture, 0600)).To(Succeed()) + + conf.Server.CoverArtPriority = "cover.*" + libRepo.SetData(model.Libraries{{ID: 0, Path: libDir}}) + folderRepo.result = []model.Folder{{Path: "an-album", ImageFiles: []string{"cover.jxl"}}} + albumRepo.SetData(model.Albums{{ID: "al1", Name: "Album", FolderIDs: []string{"f1"}, LibraryID: 0}}) + + bump("al", "al1") + runWorkerUntil(ctx, worker, itemFound(model.KindAlbumArtwork, "al1")) + + img, err := svc.Get(ctx, model.MustParseArtworkID("al-al1"), 0, false) + Expect(err).ToNot(HaveOccurred()) + Expect(img.Placeholder).To(BeFalse()) + Expect(readAll(img)).To(Equal(jxlFixture)) + }) + It("acquires album folder art and serves the exact bytes under its hash", func() { seedFolderAlbum("al1") bump("al", "al1") @@ -317,6 +337,9 @@ func mustGet(img *artwork.Image, err error) *artwork.Image { } // Raw bytes on purpose: encoding a GIF here would register image/gif in the test binary, masking +// jxlFixture is a JPEG XL bare codestream header: a real image format, with no stdlib decoder. +var jxlFixture = []byte{0xff, 0x0a, 0x00, 0x10, 0x00} + // the production import the spec above guards. var gifFixture = []byte{ 0x47, 0x49, 0x46, 0x38, 0x39, 0x61, 0x04, 0x00, 0x04, 0x00, 0x80, 0x00, diff --git a/core/artwork/processor.go b/core/artwork/processor.go index 6ce76bf45..4d38ced95 100644 --- a/core/artwork/processor.go +++ b/core/artwork/processor.go @@ -123,12 +123,20 @@ func (p *processor) acquire(ctx context.Context, item model.ArtworkQueueItem) (o art, err := repo.GetImage(hash) switch { - case err == nil: + case err == nil && art.Width > 0: log.Debug(ctx, "Artwork: Reusing a known image, skipping decode", "kind", item.ItemKind, "id", item.ItemID, "hash", hash) - case errors.Is(err, model.ErrNotFound): + // A row with no dimensions was stored when no decoder matched; retry in case one exists now. + case err == nil, errors.Is(err, model.ErrNotFound): decodeStart := time.Now() art, err = decodeArtwork(ctx, hash, data) + // Extension-matched local bytes we cannot decode are most likely a codec we lack; an + // external body carries no such guarantee, and empty bytes are no image at all. + if errors.Is(err, image.ErrFormat) && len(data) > 0 && isLocalSource(res.source) { + log.Debug(ctx, "Artwork: No decoder for this image format, storing it without placeholders", + "kind", item.ItemKind, "id", item.ItemID, "source", res.source, "bytes", len(data)) + art, err = undecodedArtwork(hash), nil + } if err != nil { log.Warn(ctx, "Artwork: Failed to decode resolved image", "kind", item.ItemKind, "id", item.ItemID, err) return outcomeFailed, nil @@ -234,6 +242,12 @@ func decodeCapped(data []byte) (image.Image, string, error) { return img, format, nil } +// undecodedArtwork is the row for bytes no decoder matched: servable, but with no dimensions +// and none of the placeholders a decode would have produced. +func undecodedArtwork(hash string) *model.Artwork { + return &model.Artwork{Hash: hash, Mime: mimeForFormat("")} +} + // decodeArtwork builds a new Artwork row from raw bytes: dimensions, mime and the two // placeholder hashes, both encoded from one shared downscaled thumbnail. func decodeArtwork(ctx context.Context, hash string, data []byte) (*model.Artwork, error) { @@ -288,6 +302,11 @@ func isFileBacked(source string) bool { return source == "folder" || source == "upload" } +// isLocalSource reports whether the bytes came off disk rather than off the network. +func isLocalSource(source string) bool { + return isFileBacked(source) || source == "embedded" +} + // placeBytes reports the item's backing-file provenance and writes the bytes into the store // for the sources that have none. func placeBytes(store *ImageStore, art *model.Artwork, res resolution, data []byte) (sourcePath string, refMtime int64, err error) { diff --git a/core/artwork/processor_test.go b/core/artwork/processor_test.go index 8d30975cd..1ada8415d 100644 --- a/core/artwork/processor_test.go +++ b/core/artwork/processor_test.go @@ -1,6 +1,7 @@ package artwork import ( + "bytes" "context" "encoding/binary" "errors" @@ -24,6 +25,9 @@ import ( . "github.com/onsi/gomega" ) +// jxlCodestream is a JPEG XL bare codestream header: a real image format, with no stdlib decoder. +var jxlCodestream = []byte{0xff, 0x0a, 0x00, 0x10, 0x00} + // DecodeConfig reads only the header, so the pixel data can be omitted entirely. func pngHeaderWithDims(w, h uint32) []byte { ihdr := make([]byte, 13) @@ -259,6 +263,59 @@ var _ = Describe("processor.acquire", func() { Expect(ia.Source).To(Equal("folder")) }) + It("undecodable local file: acquires it anyway, with no placeholder metadata", func() { + libRoot := GinkgoT().TempDir() + Expect(os.MkdirAll(filepath.Join(libRoot, "album"), 0755)).To(Succeed()) + Expect(os.WriteFile(filepath.Join(libRoot, "album", "cover.jpg"), jxlCodestream, 0600)).To(Succeed()) + libRepo.SetData(model.Libraries{{ID: 0, Path: testFileLibPath(libRoot)}}) + ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{{ID: "alU", Name: "Album", FolderIDs: []string{"f1"}}}) + folderRepo.result = []model.Folder{{Path: "album", ImageFiles: []string{"cover.jpg"}}} + + out, _ := proc.acquire(ctx, model.ArtworkQueueItem{ItemKind: "al", ItemID: "alU"}) + Expect(out).To(Equal(outcomeFound)) + + ia, err := artRepo.GetItemArtwork(model.KindAlbumArtwork, "alU", model.ImageTypePrimary) + Expect(err).ToNot(HaveOccurred()) + Expect(ia.Source).To(Equal("folder")) + art, err := artRepo.GetImage(ia.Hash) + Expect(err).ToNot(HaveOccurred()) + Expect(art.Width).To(BeZero()) + Expect(art.BlurHash).To(BeEmpty()) + }) + + It("empty local file: fails without writing state", func() { + libRoot := GinkgoT().TempDir() + Expect(os.MkdirAll(filepath.Join(libRoot, "album"), 0755)).To(Succeed()) + Expect(os.WriteFile(filepath.Join(libRoot, "album", "cover.jpg"), nil, 0600)).To(Succeed()) + libRepo.SetData(model.Libraries{{ID: 0, Path: testFileLibPath(libRoot)}}) + ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{{ID: "alE", Name: "Album", FolderIDs: []string{"f1"}}}) + folderRepo.result = []model.Folder{{Path: "album", ImageFiles: []string{"cover.jpg"}}} + + out, _ := proc.acquire(ctx, model.ArtworkQueueItem{ItemKind: "al", ItemID: "alE"}) + Expect(out).To(Equal(outcomeFailed)) + + _, err := artRepo.GetItemArtwork(model.KindAlbumArtwork, "alE", model.ImageTypePrimary) + Expect(err).To(MatchError(model.ErrNotFound)) + }) + + // An agent answering 200 with a non-image body must keep retrying, not pin garbage as a cover. + It("undecodable external body: fails without writing state", func() { + conf.Server.CoverArtPriority = "external" + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte("rate limited")) + })) + DeferCleanup(srv.Close) + + ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{{ID: "alX", Name: "Album"}}) + imageAgents(&fakeImageAgent{name: "deezerFake", imgs: []agents.ExternalImage{{URL: srv.URL, Size: 500}}}) + + out, _ := proc.acquire(ctx, model.ArtworkQueueItem{ItemKind: "al", ItemID: "alX"}) + Expect(out).To(Equal(outcomeFailed)) + + _, err := artRepo.GetItemArtwork(model.KindAlbumArtwork, "alX", model.ImageTypePrimary) + Expect(err).To(MatchError(model.ErrNotFound)) + }) + It("found-external: persists source as external: and stores the fetched bytes", func() { conf.Server.CoverArtPriority = "external" imgBytes, err := os.ReadFile(filepath.Join(repoRoot, "tests/fixtures/artist/an-album/cover.jpg")) @@ -374,7 +431,8 @@ var _ = Describe("processor.acquire", func() { conf.Server.DataFolder = conf.NewDir(tmpDir) Expect(os.MkdirAll(filepath.Join(tmpDir, "artwork", "radio"), 0755)).To(Succeed()) imgPath := filepath.Join(tmpDir, "artwork", "radio", "ra1_test.jpg") - Expect(os.WriteFile(imgPath, []byte("not actually an image"), 0600)).To(Succeed()) + // Truncated PNG: a known format, so this is a real decode failure, not a missing decoder. + Expect(os.WriteFile(imgPath, pngHeaderWithDims(100, 100), 0600)).To(Succeed()) radioRepo := tests.CreateMockedRadioRepo() radioRepo.Data = map[string]*model.Radio{"ra1": {ID: "ra1", Name: "Radio", UploadedImage: "ra1_test.jpg"}} @@ -428,6 +486,54 @@ var _ = Describe("processor.acquire", func() { Expect(err.Error()).To(ContainSubstring("dimensions")) }) + It("unknown format: reports ErrFormat so the caller can decide", func() { + _, err := decodeArtwork(ctx, "jxl", jxlCodestream) + Expect(err).To(MatchError(image.ErrFormat)) + }) + + It("undecodedArtwork: carries the hash and mime, and nothing a decode would add", func() { + art := undecodedArtwork("jxl") + Expect(art.Hash).To(Equal("jxl")) + Expect(art.Mime).To(Equal("application/octet-stream")) + Expect(art.Width).To(BeZero()) + Expect(art.Height).To(BeZero()) + Expect(art.BlurHash).To(BeEmpty()) + Expect(art.ThumbHash).To(BeEmpty()) + Expect(art.DominantColor).To(BeEmpty()) + }) + + It("corrupt image of a known format: still fails", func() { + data := pngHeaderWithDims(100, 100) // header declares a decodable size, body is missing + _, err := decodeArtwork(ctx, "truncated", data) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("decode image")) + }) + + // Without this a metadata-less row would be reused forever, so a decoder added later + // could never upgrade it. + It("metadata-less row: re-decodes on reuse instead of skipping", func() { + libRoot := GinkgoT().TempDir() + imgBytes, err := os.ReadFile(filepath.Join(repoRoot, "tests/fixtures/artist/an-album/cover.jpg")) + Expect(err).ToNot(HaveOccurred()) + Expect(os.MkdirAll(filepath.Join(libRoot, "album"), 0755)).To(Succeed()) + Expect(os.WriteFile(filepath.Join(libRoot, "album", "cover.jpg"), imgBytes, 0600)).To(Succeed()) + libRepo.SetData(model.Libraries{{ID: 0, Path: testFileLibPath(libRoot)}}) + ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{{ID: "alM", Name: "Album", FolderIDs: []string{"f1"}}}) + folderRepo.result = []model.Folder{{Path: "album", ImageFiles: []string{"cover.jpg"}}} + + hash, err := hashImage(bytes.NewReader(imgBytes)) + Expect(err).ToNot(HaveOccurred()) + Expect(artRepo.PutImage(&model.Artwork{Hash: hash, Mime: "application/octet-stream"})).To(Succeed()) + + out, _ := proc.acquire(ctx, model.ArtworkQueueItem{ItemKind: "al", ItemID: "alM"}) + Expect(out).To(Equal(outcomeFound)) + + upgraded, err := artRepo.GetImage(hash) + Expect(err).ToNot(HaveOccurred()) + Expect(upgraded.Width).To(BeNumerically(">", 0)) + Expect(upgraded.BlurHash).ToNot(BeEmpty()) + }) + It("store write failure: fails without writing state", func() { ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{ {ID: "al7", Name: "Album", EmbedArtPath: "tests/fixtures/artist/an-album/test.mp3", FolderIDs: []string{"f1"}}, diff --git a/resources/mime_types.yaml b/resources/mime_types.yaml index f67b26af4..83abf2e5c 100644 --- a/resources/mime_types.yaml +++ b/resources/mime_types.yaml @@ -37,6 +37,9 @@ types: .webp: image/webp .png: image/png .bmp: image/bmp + .jxl: image/jxl + .heic: image/heic + .heif: image/heif # List of audio formats that are considered lossless lossless: From b617a878b90a9834579c0a36e20dc1e10d13a6fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Deluan=20Quint=C3=A3o?= Date: Fri, 14 Aug 2026 13:46:52 -0400 Subject: [PATCH 03/27] feat(insights): report the app store or hosting platform via ND_PLATFORM (#5956) * feat(insights): report the app store or hosting platform via ND_PLATFORM Insights had no way to tell where an instance is deployed. The existing `os.package` field is written only by our own packagers and holds just `deb`, `rpm` or `msi`, so it answers "which installer", not "which platform". Overloading it would mix two unrelated dimensions in the same field. This adds a separate top-level `platform` field, self-declared by the deployer through the `ND_PLATFORM` environment variable. App stores and hosting providers (ZimaOS, PikaPods, TrueNAS, Unraid, and others) generally deploy our container image unmodified and can only inject environment variables, so an env var is the one marker they can all set. It is deliberately not a config option: it is a packager marker, not something users should tune, and it stays out of the config surface. Both values are now whitespace-trimmed. The msi packager writes the file with `echo`, so `os.package` has been arriving as `"msi\n"` and sorting separately from `"msi"` in any aggregation. * test(insights): isolate hostingPlatform specs from an inherited ND_PLATFORM The spec asserting an empty result read the real environment, so it failed on any machine that already had ND_PLATFORM set. Unset it per-spec, using Setenv first so Ginkgo restores the original value on cleanup. --- core/metrics/insights.go | 19 +++++-- core/metrics/insights/data.go | 4 +- core/metrics/insights_internal_test.go | 70 ++++++++++++++++++++++++++ core/metrics/metrics_suite_test.go | 17 +++++++ 4 files changed, 104 insertions(+), 6 deletions(-) create mode 100644 core/metrics/insights_internal_test.go create mode 100644 core/metrics/metrics_suite_test.go diff --git a/core/metrics/insights.go b/core/metrics/insights.go index 78391779a..66d0b89bd 100644 --- a/core/metrics/insights.go +++ b/core/metrics/insights.go @@ -10,6 +10,7 @@ import ( "path/filepath" "runtime" "runtime/debug" + "strings" "sync" "sync/atomic" "time" @@ -153,6 +154,17 @@ func getFSInfo(path string) *insights.FSInfo { return &info } +// installedPackage returns the official installer format used, as written by our own packagers. +func installedPackage() string { + data, _ := os.ReadFile(filepath.Join(conf.Server.DataFolder.String(), ".package")) + return strings.TrimSpace(string(data)) +} + +// hostingPlatform is env-based, not a file, as app stores can only inject env vars into our image. +func hostingPlatform() string { + return strings.TrimSpace(os.Getenv("ND_PLATFORM")) +} + var staticData = sync.OnceValue(func() insights.Data { // Basic info data := insights.Data{ @@ -165,11 +177,8 @@ var staticData = sync.OnceValue(func() insights.Data { data.OS.Containerized = consts.InContainer // Install info - packageFilename := filepath.Join(conf.Server.DataFolder.String(), ".package") - packageFileData, err := os.ReadFile(packageFilename) - if err == nil { - data.OS.Package = string(packageFileData) - } + data.OS.Package = installedPackage() + data.Platform = hostingPlatform() // OS info data.OS.Type = runtime.GOOS diff --git a/core/metrics/insights/data.go b/core/metrics/insights/data.go index 126d759bc..8559d4204 100644 --- a/core/metrics/insights/data.go +++ b/core/metrics/insights/data.go @@ -4,7 +4,9 @@ type Data struct { InsightsID string `json:"id"` Version string `json:"version"` Uptime int64 `json:"uptime"` - Build struct { + // Platform is the app store or hosting provider this instance runs on, self-declared via ND_PLATFORM + Platform string `json:"platform,omitempty"` + Build struct { // build settings used by the Go compiler Settings map[string]string `json:"settings"` GoVersion string `json:"goVersion"` diff --git a/core/metrics/insights_internal_test.go b/core/metrics/insights_internal_test.go new file mode 100644 index 000000000..74c8ce236 --- /dev/null +++ b/core/metrics/insights_internal_test.go @@ -0,0 +1,70 @@ +package metrics + +import ( + "os" + "path/filepath" + + "github.com/navidrome/navidrome/conf" + "github.com/navidrome/navidrome/conf/configtest" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("installedPackage", func() { + BeforeEach(func() { + DeferCleanup(configtest.SetupConfig()) + conf.Server.DataFolder = conf.NewDir(GinkgoT().TempDir()) + }) + + It("returns empty when there's no .package file", func() { + Expect(installedPackage()).To(BeEmpty()) + }) + + It("reads the .package file from the data folder", func() { + writePackageFile("deb") + + Expect(installedPackage()).To(Equal("deb")) + }) + + It("trims surrounding whitespace, as the msi packager writes a trailing newline", func() { + writePackageFile("msi\n") + + Expect(installedPackage()).To(Equal("msi")) + }) + + It("ignores ND_PLATFORM", func() { + GinkgoT().Setenv("ND_PLATFORM", "zimaos") + + Expect(installedPackage()).To(BeEmpty()) + }) +}) + +var _ = Describe("hostingPlatform", func() { + BeforeEach(func() { + // Setenv registers the restore, then unset so an inherited value can't leak in + GinkgoT().Setenv("ND_PLATFORM", "") + Expect(os.Unsetenv("ND_PLATFORM")).To(Succeed()) + }) + + It("returns empty when ND_PLATFORM is not set", func() { + Expect(hostingPlatform()).To(BeEmpty()) + }) + + It("reads ND_PLATFORM", func() { + GinkgoT().Setenv("ND_PLATFORM", "zimaos") + + Expect(hostingPlatform()).To(Equal("zimaos")) + }) + + It("trims surrounding whitespace", func() { + GinkgoT().Setenv("ND_PLATFORM", " pikapods\n") + + Expect(hostingPlatform()).To(Equal("pikapods")) + }) +}) + +func writePackageFile(content string) { + GinkgoHelper() + path := filepath.Join(conf.Server.DataFolder.String(), ".package") + Expect(os.WriteFile(path, []byte(content), 0600)).To(Succeed()) +} diff --git a/core/metrics/metrics_suite_test.go b/core/metrics/metrics_suite_test.go new file mode 100644 index 000000000..bae622e90 --- /dev/null +++ b/core/metrics/metrics_suite_test.go @@ -0,0 +1,17 @@ +package metrics + +import ( + "testing" + + "github.com/navidrome/navidrome/log" + "github.com/navidrome/navidrome/tests" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestMetrics(t *testing.T) { + tests.Init(t, false) + log.SetLevel(log.LevelFatal) + RegisterFailHandler(Fail) + RunSpecs(t, "Metrics Suite") +} From dc40bcaf808a40a2b5d6e62c9338e84f40ca9407 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Deluan=20Quint=C3=A3o?= Date: Fri, 14 Aug 2026 21:07:56 -0400 Subject: [PATCH 04/27] feat(cli): add an artwork command group for diagnosing and re-driving artwork (#5957) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(artwork): add a resolution chain trace collector * feat(artwork): trace the local priority chain * fix(artwork): record priority candidates the chain never evaluated * refactor(artwork): report never-evaluated candidates as skipped * feat(artwork): trace external agents at the gate seam * feat(artwork): add repository queries to enqueue by current source * feat(artwork): expose a tracing resolver for the CLI * feat(artwork): read a single queue row by item The explain CLI must report whether an item is queued, at what priority and when it retries; the queue repository could only be drained in eligibility batches, which cannot see a row that is still backing off. * feat(cli): add artwork explain Prints why an item has the artwork it has: the stored state, its queue row, the governing config, the resolver's priority-chain walk and the verdict. Offline by default so a diagnostic run cannot add load to an external provider; --live asks the agents for real. Playlists and radios do not walk a priority chain, so they report that instead of an empty chain table. * fix(artwork): trace an external tier that never reaches an agent A configured 'external' token vanished from the chain when no enabled agent provided images for that entity type, and for synthetic artists, leaving the trace unable to say whether the tier was even considered. * fix(cli): never state an artwork outcome the walk did not observe A transient external failure traced as 'error' fell through to 'not resolved', which is the most common state behind a missing-artwork report. It is now indeterminate, and an offline win that a skipped higher-priority external candidate could have taken says so instead of naming a winner the live chain might not pick. * feat(cli): add artwork refresh * feat(cli): add artwork reprocess Bulk re-enqueues artwork by kind and/or by the source an item currently resolves from, previewing the matched count and confirming before queueing. The preview counts with CountBySource (rows matched) and reports separately what EnqueueBySource inserted: its DO NOTHING conflict policy leaves an already-queued row untouched, so the two numbers differ and the output must not claim the skipped rows were re-queued. An unknown --source is rejected against the sources present in item_artwork, rather than silently matching nothing and printing a reassuring 0. * fix(cli): cover the reprocess selection rule and validate sources table-wide The reconciliation that makes --source alone target every kind was only exercised through runReprocess, which no test calls: mutating it to `all := reprocessAll` left the suite green. It is now reprocessSelectsAll, covered for all three selectors. Scoping source validation to the selected kinds made the same well-formed filter valid or invalid depending on which other kinds were selected, and its error read the same for a typo as for a source that simply does not apply to the chosen kind. Validation is now table-wide: a typo still aborts, while a valid-but-inapplicable source falls through to "Nothing matches". Also: the prompt now counts only the kinds that reach an external agent as external cost, and --dry-run on an empty selection reports a dry run. * fix(cli): cover the reprocess --yes guard and preview the external cost Mutating the --yes check to `if true` left the suite green, so the one bypass of the confirmation was unverified. The choice is now reprocessConfirm(yes, in), covered in both directions. The external estimate only reached the operator through the prompt, which --dry-run skips — hiding the number in the one mode that exists to show it before committing. The preview now carries it, and the prompt drops the clause when no lookup will be made. An empty selection says so again under --dry-run. * feat(artwork): add read-only queue and absent counters Both are needed by the artwork status CLI: a queue breakdown by kind and priority, and the absent totals split against the recheck cutoff. * feat(cli): add artwork status Reports the queue, where artwork currently resolves from, absent counts against the 24h recheck window, and the stored config fingerprint versus the current one — the line that turns 'why is my server re-resolving everything?' into one command. fingerprint() and staleAbsentAge are exported so the CLI reports the values backfill itself compares, instead of a second copy of the formula that can silently drift. * fix(cli): lead the artwork status backfill line with the queued backlog By the time anyone runs a diagnostic, backfill has usually already stored the new fingerprint, so 'up to date' was printed while thousands of items churned through external providers. The backlog is the finding; the fingerprint is context. Also echoes the config inputs the fingerprint covers, so a change can be traced to the setting that caused it, and pins the rendered rows: the Absent values, the queue TOTAL and a queue-scoped kind/priority pair were all unasserted, so kindName and priorityName were effectively untested. FingerprintInputs is now the single listing ConfigFingerprint hashes; a pinned hash proves the value did not change. * refactor(artwork): export the trace outcome vocabulary The CLI hardcoded the outcome literals and the "external:" prefix, so renaming a constant's value in core/artwork left cmd compiling and the suite green while `artwork explain` silently degraded its verdict. Renaming a value now fails the golden vocabulary test in core/artwork and the explainResult tests in cmd. * fix(cli): keep the re-enqueue warning when a backfill is already running A stale stored fingerprint with items already queued is the worst state the system can be in: a second full re-enqueue is pending on top of the one running. The line carried the weakest wording of the three, and was untested. * refactor(artwork): drop the unreachable breaker branch from the tracing gate --live wires the tracing gate straight to passthroughGate, so errBreakerOpen can never reach it; the test only passed by injecting a fake gate. * refactor(artwork): delete the never-emitted not-reached outcome Candidates after the winner are lower priority and say nothing about why a source won; the ones that matter sit above it and are already recorded. * refactor(artwork): make the trace nil-safe in one place only add already handles a nil trace, so record's own guard was dead; Steps was the odd one out and would panic where every other method tolerates nil. * refactor(artwork): export the trace types directly ChainTrace and TraceStep were unexported types re-exported through aliases, which existed only so the CLI had a name to refer to them by. The types are public API — Resolver.Steps returns []TraceStep and the CLI constructs a ChainTrace — so name them that way and drop the indirection. Encapsulation is unchanged: add, mu and steps stay unexported, so only this package can write a step. * refactor(cli): simplify parseArtworkKind with slices.Contains Replaces a nested loop and a manual append with slices.Contains and the repo's slice.Map helper. Same behaviour, same error message. * fix(cli): print the absent artwork source under the name --source accepts `artwork explain` rendered the stored empty source as "(absent)", while `artwork reprocess --source` only accepts "absent", so pasting what explain printed straight back into reprocess was rejected as an unknown source. * refactor(artwork): own the kind list and the chain predicate in the package Export RecheckKinds and add WalksPriorityChain so the CLI stops keeping its own copies of both, and unexport externalCandidate, which nothing outside the package consumes. * refactor(cli): drop the artwork command's duplicated state and formatting Reuse artwork.RecheckKinds and artwork.WalksPriorityChain, extract newTabWriter and externalEstimate, fold reprocessSelectsAll into selectedKinds, and derive the queue total and the walks-chain flag instead of carrying them in the report structs. * test(persistence): drop two artwork-queue specs that cannot fail One seeded hash and source together and then asserted the two counts agree, so its setup guaranteed the result; the other repeated the count-does-not- enqueue property already covered by the CountBySource spec. * refactor(artwork): rename Resolver to TracingResolver for clarity * fix(cli): count playlists in the artwork reprocess external estimate The estimate used WalksPriorityChain, which is true only for artist and album, so a playlist-only reprocess reported "External lookups: none" and the confirmation prompt dropped the external-cost warning. Playlists do reach the network: through the m3u ExternalImageURL fetch when EnableM3UExternalAlbumArt is on, and — verified by test — through the generated grid, whose tiles resolve album art via the full album priority chain. Adds artwork.MayFetchExternal, a config-aware predicate for "can this kind's resolver reach the network", and uses it for the estimate. WalksPriorityChain keeps its separate job of deciding whether explain prints a chain block. * fix(cli): estimate artwork reprocess external lookups per agent, not per item The reprocess prompt billed one external lookup per externally-capable item. fetchArtistImage/fetchAlbumImage try every enabled image agent and stop early only on a hit, and resolvePlaylist can fetch the m3u image and then resolve up to four sampled albums for the grid, each walking the album agents again. The number the operator confirmed could understate real provider traffic several fold, in the prompt whose whole job is to stop a provider flood. ExternalLookupsPerItem now multiplies by the visible image-agent count and adds the playlist grid factor. It stays a floor: the CLI never calls Manager.Start(), so the plugin registry is empty and plugin-provided agents are dropped by getEnabledAgentNames. On an install with 5 agents of which 3 are plugins the count is well under the truth, so the wording is now "at least N" rather than "up to N" — a zero visible count still bills one lookup for the same reason. Fixing the plugin visibility is out of scope: Manager.Start() needs a Subsonic router and writes to the DB via syncPlugins, breaking this command group's read-only guarantee. * fix(cli): state the artwork reprocess estimate as an estimate, not a bound Neither bound is true. A ceiling is false because plugin agents are invisible to a CLI that never starts the plugin manager, and a floor is false because a local hit ends the walk before any agent is asked and a hit on the first agent skips the rest. "at least N" traded one wrong claim for another. The line now names its blind spots instead: External lookups: ~340 estimated (plugin agents not counted; local hits may need fewer). The same line is reused in the confirmation prompt, and the zero case still reads "External lookups: none." with the prompt dropping the clause entirely. The count itself is unchanged. * fix(cli): account for every configured agent in artwork explain The Agents: line printed the raw config while the Chain only showed the agents the CLI could construct, with nothing explaining the gap: plugin agents are never registered in a CLI that does not start the plugin manager, and a built-in without credentials returns nil. Three of five agents could vanish, including ones ranked above the one shown. Also treat a live external error before the winning hit like the already-handled would-try case: the resolver serves such a hit provisionally and retries later, so the verdict is indeterminate. The Result line is still not qualified when an unavailable agent might have won; that needs agent ranking, and is left to the follow-up that makes the CLI load plugin agents for real. * fix(cli): do not call an external artwork win indeterminate explainResult qualified the verdict whenever an external OutcomeError appeared before the winning hit. When a later external agent returns an image, fetchArtistImage/fetchAlbumImage discard the earlier error, so extError is false: the worker settles the item and schedules no retry. Telling the operator it may resolve differently on a retry was wrong. The warning is only correct when a lower-priority local source won while an external error was recorded, which is the case that carries extError. * fix(cli): accept --source absent when nothing is currently absent validateSources checks the requested sources against the ones item_artwork actually uses, to catch a typo. The reserved empty source (spelled 'absent' on the CLI) is a valid filter even when it matches nothing, so a scheduled 'artwork reprocess --source absent --yes' stopped working the moment the library finished resolving. Treat it as intrinsically valid and let the existing zero-match path report it. * feat(artwork): explain disc and media file artwork from the CLI `artwork explain` rejected `dc` and `mf` because it validated against RecheckKinds, the list of kinds the backfill revisits. Those are different questions: a kind with no recheck path still has artwork someone can report as wrong. Disc artwork now walks DiscArtPriority under a trace, so explain reports which entry won and why the others lost, including entries that map to no source at all (external is unsupported, a disc with no subtitle, an album folder with no images). Media file artwork traces its single embedded candidate, separating "EnableMediaFileCoverArt is off" from "the track has no embedded art" — stored state cannot tell those apart. Each command now validates against the kinds it can actually serve: explain takes all six, refresh takes artwork.RefreshableKinds (which nativeapi now shares instead of keeping its own copy), reprocess still takes RecheckKinds. Disc artwork stays out of refresh: the worker cannot resolve it, so the queue row would be rejected on every drain. WalksPriorityChain becomes Explainable, and ResolveArtist/ResolveAlbum collapse into Resolve(kind, id). * refactor(artwork): one disc-artwork walk for serving and explain resolveDisc duplicated the loop selectImageReader already ran: try each source in priority order, take the first that yields an image. The serving path and the CLI diverged on two details as a result — only selectImageReader checked ctx between candidates and logged each attempt. Both now call discArtworkReader.selectImage, which takes the chainState the CLI already uses for the other kinds. The serving path passes an untraced one, whose nil trace makes recording a no-op. selectImageReader had no other caller and is gone. The disc tests move from fromDiscArtPriority to discCandidates, so they assert the skip reason for an entry that maps to no source rather than that it silently vanished, and cancellation mid-walk is now covered. * fix(artwork): reject a nil reader in the resize cache instead of panicking resizedItem.Reader closes what open() hands back, so an open() that reports "no image" as (nil, nil) rather than an error takes the request down with a nil-pointer panic. Every caller returns an error today, and no test covered it: the resolution e2e harness stubs the resize reader out entirely, so no e2e path reaches this code at all. Guard it and cover Reader directly. * refactor(artwork): move the keeps-state fact into core, drop a redundant guard keepsArtworkState lived in package cmd and re-derived by hand what RefreshableKinds already encodes: the same five-of-six kinds. It is now artwork.KeepsState, beside the list, with a test pinning the two together — nothing else stopped them drifting, and a drift would have explain report stored state for a kind that keeps none. serveDisc's closure also hand-rolled a nil-reader error that both consumers of open() now produce themselves: serveSource for a full-size request, resizedItem.Reader for a resized one. * fix(artwork): route disc candidates through the shared resolvers openCandidate ran its own source loop and threw the error away, so a disc track that exists but cannot be parsed traced as "miss" — indistinguishable from a track with no embedded art. fromTag and fromFFmpegTag already report that case as errSourceUnreadable; only this loop was discarding it. Telling those two apart is what the trace is for. Candidates now carry a resolve func instead of raw sources: embedded goes to resolveEmbedded, and the folder-backed entries to resolveFolderSource, extracted from resolveFolderFile so both callers classify an unopenable file the same way. openCandidate and its absolute-path special case go away with it. Disc's own fromExternalFile and fromDiscSubtitle still swallow open errors, so folder candidates cannot report unreadable yet; that is a change to their error contracts. * fix(artwork): report an unreadable local candidate as indeterminate processor.acquire treats resolution.localError exactly as it treats extError: a fault is not a definitive "no image", so it retries instead of settling absent. explainResult qualified only the external case, so a chain that ended on an unreadable local candidate printed "not resolved" — the one verdict that says the walk was conclusive. The qualification belongs only to the unresolved branch. chainState.try stamps extErr onto a hit and deliberately drops localErr, so an unreadable step followed by a hit is settled as found and must not carry a warning; a test pins that. Found by Codex on 5f65d7cfa. --- cmd/artwork.go | 778 ++++++++++++++++ cmd/artwork_test.go | 899 +++++++++++++++++++ cmd/plugin.go | 3 +- cmd/utils.go | 7 + cmd/wire_gen.go | 12 + cmd/wire_injectors.go | 7 + core/artwork/agent_images.go | 17 +- core/artwork/agent_images_test.go | 36 + core/artwork/artwork.go | 60 +- core/artwork/disc.go | 72 +- core/artwork/disc_test.go | 179 ++-- core/artwork/housekeeping.go | 55 +- core/artwork/housekeeping_test.go | 55 +- core/artwork/image_cache.go | 4 + core/artwork/image_cache_test.go | 41 + core/artwork/resolve.go | 177 +++- core/artwork/resolve_test.go | 129 +++ core/artwork/sources.go | 17 - core/artwork/trace.go | 99 ++ core/artwork/trace_test.go | 628 +++++++++++++ core/artwork/worker.go | 2 +- core/artwork/worker_test.go | 2 +- model/artwork.go | 26 + persistence/artwork_queue_repository.go | 72 +- persistence/artwork_queue_repository_test.go | 138 +++ server/nativeapi/artwork.go | 10 +- tests/mock_artwork_queue_repo.go | 116 +++ 27 files changed, 3469 insertions(+), 172 deletions(-) create mode 100644 cmd/artwork.go create mode 100644 cmd/artwork_test.go create mode 100644 core/artwork/image_cache_test.go create mode 100644 core/artwork/trace.go create mode 100644 core/artwork/trace_test.go diff --git a/cmd/artwork.go b/cmd/artwork.go new file mode 100644 index 000000000..2b6a50164 --- /dev/null +++ b/cmd/artwork.go @@ -0,0 +1,778 @@ +package cmd + +import ( + "cmp" + "context" + "errors" + "fmt" + "io" + "os" + "slices" + "strconv" + "strings" + "time" + + "github.com/navidrome/navidrome/conf" + "github.com/navidrome/navidrome/consts" + "github.com/navidrome/navidrome/core/agents" + "github.com/navidrome/navidrome/core/artwork" + "github.com/navidrome/navidrome/db" + "github.com/navidrome/navidrome/log" + "github.com/navidrome/navidrome/model" + "github.com/navidrome/navidrome/utils/slice" + "github.com/spf13/cobra" +) + +var explainLive bool + +var ( + reprocessKinds []string + reprocessSources []string + reprocessAll bool + reprocessDryRun bool + reprocessYes bool +) + +func init() { + artworkExplainCmd.Flags().BoolVar(&explainLive, "live", false, + "perform real external lookups instead of reporting what would be tried") + artworkReprocessCmd.Flags().StringSliceVar(&reprocessKinds, "kind", nil, + "kinds to reprocess ("+kindPrefixes(artwork.RecheckKinds)+"); repeatable") + artworkReprocessCmd.Flags().StringSliceVar(&reprocessSources, "source", nil, + "only items currently resolved from these sources (e.g. folder, external:deezer, absent)") + artworkReprocessCmd.Flags().BoolVar(&reprocessAll, "all", false, "reprocess every kind") + artworkReprocessCmd.Flags().BoolVar(&reprocessDryRun, "dry-run", false, + "report what would be queued and exit without queueing") + artworkReprocessCmd.Flags().BoolVarP(&reprocessYes, "yes", "y", false, "skip the confirmation prompt") + artworkCmd.AddCommand(artworkExplainCmd) + artworkCmd.AddCommand(artworkRefreshCmd) + artworkCmd.AddCommand(artworkReprocessCmd) + artworkCmd.AddCommand(artworkStatusCmd) + rootCmd.AddCommand(artworkCmd) +} + +var artworkCmd = &cobra.Command{ + Use: "artwork", + Short: "Inspect and re-resolve artwork", +} + +var artworkExplainCmd = &cobra.Command{ + Use: "explain ", + Short: "Explain why an item's artwork resolved the way it did", + Long: "Explain why an item's artwork resolved the way it did.\n\n" + + " is one of: " + kindPrefixes(explainKinds) + ".\n" + + "A disc artwork id is the album id and the disc number, joined by a colon: :2", + Args: cobra.ExactArgs(2), + Run: func(cmd *cobra.Command, args []string) { + kind, err := parseArtworkKind(args[0], explainKinds) + if err != nil { + log.Fatal(cmd.Context(), err) + } + runExplain(cmd.Context(), kind, args[1]) + }, +} + +var artworkRefreshCmd = &cobra.Command{ + Use: "refresh ...", + Short: "Clear an item's artwork state and re-resolve it", + Args: cobra.MinimumNArgs(2), + Run: func(cmd *cobra.Command, args []string) { + kind, err := parseArtworkKind(args[0], artwork.RefreshableKinds) + if err != nil { + log.Fatal(cmd.Context(), err) + } + runRefresh(cmd.Context(), kind, args[1:]) + }, +} + +var artworkReprocessCmd = &cobra.Command{ + Use: "reprocess", + Short: "Re-enqueue artwork in bulk, by kind and/or by the source it currently resolves from", + Args: cobra.NoArgs, + Run: func(cmd *cobra.Command, args []string) { + runReprocess(cmd.Context()) + }, +} + +var artworkStatusCmd = &cobra.Command{ + Use: "status", + Short: "Report the artwork queue, where artwork resolves from, and the backfill state", + Args: cobra.NoArgs, + Run: func(cmd *cobra.Command, args []string) { + runStatus(cmd.Context()) + }, +} + +func runStatus(ctx context.Context) { + defer db.Init(ctx)() + ds, ctx := getAdminContext(ctx) + + rep, err := collectStatus(ctx, ds) + if err != nil { + log.Fatal(ctx, err) + } + fmt.Print(formatStatus(rep)) +} + +type sourceCount struct { + kind model.Kind + source string + count int64 +} + +type absentCount struct { + kind model.Kind + model.ArtworkAbsentStat +} + +type statusReport struct { + queue []model.ArtworkQueueStat + sources []sourceCount + absent []absentCount + inputs []artwork.FingerprintInput + stored string + current string +} + +func (r statusReport) queueTotal() int64 { + var n int64 + for _, s := range r.queue { + n += s.Count + } + return n +} + +func (r statusReport) backfillQueued() int64 { + var n int64 + for _, s := range r.queue { + if s.Priority == model.ArtworkPriorityBackfill { + n += s.Count + } + } + return n +} + +func collectStatus(ctx context.Context, ds model.DataStore) (statusReport, error) { + q := ds.ArtworkQueue(ctx) + var rep statusReport + var err error + if rep.queue, err = q.CountByKindAndPriority(); err != nil { + return rep, fmt.Errorf("breaking the artwork queue down by kind: %w", err) + } + + cutoff := time.Now().Add(-artwork.StaleAbsentAge) + for _, k := range artwork.RecheckKinds { + sources, err := q.SourcesInUse(k) + if err != nil { + return rep, fmt.Errorf("listing the sources in use by %s artwork: %w", k, err) + } + slices.Sort(sources) + for _, s := range sources { + n, err := q.CountBySource(k, []string{s}) + if err != nil { + return rep, fmt.Errorf("counting %s artwork resolved from %s: %w", k, displaySource(s), err) + } + rep.sources = append(rep.sources, sourceCount{kind: k, source: s, count: n}) + } + stat, err := q.CountAbsent(k, cutoff) + if err != nil { + return rep, fmt.Errorf("counting absent %s artwork: %w", k, err) + } + rep.absent = append(rep.absent, absentCount{kind: k, ArtworkAbsentStat: stat}) + } + + rep.current, rep.inputs = artwork.ConfigFingerprint(), artwork.FingerprintInputs() + if rep.stored, err = ds.Property(ctx).DefaultGet(consts.ArtConfFingerprintPropertyKey, ""); err != nil { + return rep, fmt.Errorf("reading the stored artwork fingerprint: %w", err) + } + return rep, nil +} + +func formatStatus(rep statusReport) string { + var sb strings.Builder + w := newTabWriter(&sb) + + fmt.Fprintln(w, "Queue") + if len(rep.queue) == 0 { + fmt.Fprintln(w, " (empty)") + } else { + fmt.Fprintln(w, " KIND\tPRIORITY\tITEMS") + for _, s := range rep.queue { + fmt.Fprintf(w, " %s\t%s\t%d\n", kindName(s.ItemKind), priorityName(s.Priority), s.Count) + } + fmt.Fprintf(w, " TOTAL\t\t%d\n", rep.queueTotal()) + } + + fmt.Fprintln(w, "\nSources") + fmt.Fprintln(w, " KIND\tSOURCE\tITEMS") + for _, s := range rep.sources { + fmt.Fprintf(w, " %s\t%s\t%d\n", s.kind, displaySource(s.source), s.count) + } + + fmt.Fprintln(w, "\nAbsent (resolved, no image found)") + fmt.Fprintln(w, " KIND\tABSENT\tDUE FOR RECHECK") + for _, a := range rep.absent { + fmt.Fprintf(w, " %s\t%d\t%d\n", a.kind, a.Total, a.Stale) + } + fmt.Fprintf(w, " (rechecked once the last attempt is older than %gh)\n", artwork.StaleAbsentAge.Hours()) + + fmt.Fprintln(w, "\nBackfill") + fmt.Fprintf(w, " State:\t%s\n", backfillState(rep)) + fmt.Fprintf(w, " Stored fingerprint:\t%s\n", cmp.Or(rep.stored, "(none)")) + fmt.Fprintf(w, " Current fingerprint:\t%s\n", rep.current) + if len(rep.inputs) > 0 { + fmt.Fprintln(w, " Fingerprint inputs (changing any of these re-resolves the whole library):") + for _, in := range rep.inputs { + fmt.Fprintf(w, " %s:\t%s\n", in.Name, in.Value) + } + } + + w.Flush() + return sb.String() +} + +// backfillState leads with the queued backlog: by the time anyone runs this, backfill has usually +// already stored the new fingerprint, and "up to date" would bury the flood it is still working through. +func backfillState(rep statusReport) string { + pending := "fingerprint changed — every artist, album, playlist and radio will be re-enqueued on the next startup" + if n := rep.backfillQueued(); n > 0 { + if rep.stored != rep.current { + return fmt.Sprintf("backfill running: %d items queued, and %s", n, pending) + } + return fmt.Sprintf("backfill running: %d items queued (fingerprint up to date)", n) + } + if rep.stored != rep.current { + return pending + } + return "up to date" +} + +func kindName(prefix string) string { + if k, ok := model.ParseKind(prefix); ok { + return k.String() + } + return prefix +} + +func priorityName(p int) string { + switch p { + case model.ArtworkPriorityRecheck: + return "recheck" + case model.ArtworkPriorityBackfill: + return "backfill" + case model.ArtworkPriorityScan: + return "scan" + case model.ArtworkPriorityBump: + return "bump" + } + return strconv.Itoa(p) +} + +func runReprocess(ctx context.Context) { + kinds, err := selectedKinds(reprocessKinds, reprocessSources, reprocessAll) + if err != nil { + log.Fatal(ctx, err) + } + + defer db.Init(ctx)() + ds, ctx := getAdminContext(ctx) + + if err := reprocessArtwork(ctx, ds, kinds, repositorySources(reprocessSources), imageAgentCount(ds), + reprocessDryRun, reprocessConfirm(reprocessYes, os.Stdin), os.Stdout); err != nil { + log.Fatal(ctx, err) + } +} + +func selectedKinds(kinds, sources []string, all bool) ([]model.Kind, error) { + // A source filter on its own is already a complete selection, so it does not also need a kind. + if all || (len(kinds) == 0 && len(sources) > 0) { + return artwork.RecheckKinds, nil + } + if len(kinds) == 0 { + return nil, fmt.Errorf("no selector given: pass --kind, --source or --all") + } + out := make([]model.Kind, 0, len(kinds)) + for _, k := range kinds { + kind, err := parseArtworkKind(k, artwork.RecheckKinds) + if err != nil { + return nil, err + } + out = append(out, kind) + } + // A repeated kind would be counted twice, overstating the cost the operator confirms. + return slice.Unique(out), nil +} + +// absentSource is how the stored empty source — resolved, no image — is spelled on the CLI. +const absentSource = "absent" + +func repositorySources(sources []string) []string { + return slice.Map(sources, func(s string) string { + if s == absentSource { + return "" + } + return s + }) +} + +func displaySource(s string) string { return cmp.Or(s, absentSource) } + +type confirmFunc func(out io.Writer, total, external int64) bool + +func reprocessConfirm(yes bool, in io.Reader) confirmFunc { + if yes { + return func(io.Writer, int64, int64) bool { return true } + } + return promptConfirm(in) +} + +// externalEstimate claims no bound: a local hit ends the walk before any agent is asked, and plugin +// agents are unregistered in a CLI that never starts the plugin manager. +func externalEstimate(n int64) string { + if n == 0 { + return "none" + } + return fmt.Sprintf("~%d estimated (plugin agents not counted; local hits may need fewer)", n) +} + +func externalLookupLine(n int64) string { + return fmt.Sprintf("External lookups: %s.", externalEstimate(n)) +} + +// imageAgentCount counts only the built-in image agents, for the same reason. +func imageAgentCount(ds model.DataStore) artwork.ImageAgentCount { + ag := agents.GetAgents(ds, getPluginManager()) + return artwork.ImageAgentCount{Artist: len(ag.ArtistImageAgents()), Album: len(ag.AlbumImageAgents())} +} + +func promptConfirm(in io.Reader) confirmFunc { + return func(out io.Writer, total, external int64) bool { + var cost string + if external > 0 { + cost = fmt.Sprintf(" %s", externalLookupLine(external)) + } + fmt.Fprintf(out, "\nThis will re-resolve %d items.%s Continue? [y/N] ", total, cost) + var answer string + if _, err := fmt.Fscanln(in, &answer); err != nil { + return false + } + answer = strings.ToLower(strings.TrimSpace(answer)) + return answer == "y" || answer == "yes" + } +} + +// validateSources rejects a typo'd source: matching nothing silently reads as "nothing to do" when +// it means the filter was wrong. Checked table-wide, so a filter is never a typo for one --kind only. +func validateSources(q model.ArtworkQueueRepository, sources []string) error { + if len(sources) == 0 { + return nil + } + var inUse []string + for _, k := range artwork.RecheckKinds { + found, err := q.SourcesInUse(k) + if err != nil { + return fmt.Errorf("listing the sources in use by %s artwork: %w", k, err) + } + inUse = slice.Unique(append(inUse, found...)) + } + var unknown []string + for _, s := range sources { + if s != "" && !slices.Contains(inUse, s) { // the reserved absent source is valid even when nothing is absent + unknown = append(unknown, displaySource(s)) + } + } + if len(unknown) == 0 { + return nil + } + valid := slice.Map(inUse, displaySource) + slices.Sort(valid) + return fmt.Errorf("no artwork resolves from %s; sources in use: %s", + strings.Join(unknown, ", "), cmp.Or(strings.Join(valid, ", "), "(none)")) +} + +// reprocessArtwork previews from CountBySource — rows matched — then reports what EnqueueBySource +// actually inserted; the two differ because an already-queued row is left untouched. +func reprocessArtwork(ctx context.Context, ds model.DataStore, kinds []model.Kind, sources []string, + imageAgents artwork.ImageAgentCount, dryRun bool, confirm confirmFunc, out io.Writer) error { + q := ds.ArtworkQueue(ctx) + if err := validateSources(q, sources); err != nil { + return err + } + + matched := make([]int64, len(kinds)) + var total, external int64 + for i, k := range kinds { + n, err := q.CountBySource(k, sources) + if err != nil { + return fmt.Errorf("counting %s artwork: %w", k, err) + } + matched[i] = n + total += n + external += n * artwork.ExternalLookupsPerItem(k, imageAgents) + } + printReprocessPreview(out, kinds, matched, total, external, sources) + + switch { + case dryRun: + fmt.Fprintln(out, "\nDry run: nothing was queued.") + return nil + case total == 0: + fmt.Fprintln(out, "Nothing was queued.") + return nil + case !confirm(out, total, external): + fmt.Fprintln(out, "Aborted: nothing was queued.") + return nil + } + + var queued int64 + for i, k := range kinds { + if matched[i] == 0 { + continue + } + n, err := q.EnqueueBySource(k, sources, model.ArtworkPriorityRecheck) + if err != nil { + return fmt.Errorf("queueing %s artwork: %w", k, err) + } + queued += n + fmt.Fprintf(out, "%s: %d queued\n", k, n) + } + fmt.Fprintf(out, "Queued %d of %d matched items.\n", queued, total) + if skipped := total - queued; skipped > 0 { + fmt.Fprintf(out, "Already queued, left unchanged: %d (priority and retry backoff untouched).\n", skipped) + } + return nil +} + +// printReprocessPreview also states the external estimate, which --dry-run must show because it +// skips the prompt that would otherwise carry it. +func printReprocessPreview(out io.Writer, kinds []model.Kind, matched []int64, total, external int64, sources []string) { + w := newTabWriter(out) + shown := slice.Map(sources, displaySource) + fmt.Fprintf(w, "Sources:\t%s\n\n", cmp.Or(strings.Join(shown, ", "), "(any)")) + fmt.Fprintln(w, "KIND\tMATCHED") + for i, k := range kinds { + fmt.Fprintf(w, "%s\t%d\n", k, matched[i]) + } + fmt.Fprintf(w, "TOTAL\t%d\n", total) + w.Flush() + + fmt.Fprintf(out, "\n%s\n", externalLookupLine(external)) + if total == 0 { + fmt.Fprintln(out, "\nNothing matches this selection.") + } +} + +func runRefresh(ctx context.Context, kind model.Kind, ids []string) { + defer db.Init(ctx)() + ds, ctx := getAdminContext(ctx) + + if failed := refreshItems(ctx, ds, kind, ids, os.Stdout); failed > 0 { + log.Fatal(ctx, "Failed to refresh artwork", "kind", kind, "failed", failed, "total", len(ids)) + } +} + +// refreshItems keeps going after a failure — the ids are independent — and returns how many failed. +func refreshItems(ctx context.Context, ds model.DataStore, kind model.Kind, ids []string, out io.Writer) int { + var failed int + for _, id := range ids { + // artwork.Refresh would happily queue an id that does not exist, orphaning a queue row. + if _, err := artworkItemName(ctx, ds, kind, id); err != nil { + log.Error(ctx, "Item not found", "kind", kind, "id", id, err) + failed++ + continue + } + if err := artwork.Refresh(ctx, ds, kind, id); err != nil { + log.Error(ctx, "Error refreshing artwork", "kind", kind, "id", id, err) + failed++ + continue + } + fmt.Fprintf(out, "%s/%s: queued\n", kind.Prefix(), id) + } + return failed +} + +// explainKinds is every kind explain accepts: it reports stored state and config too, so a kind +// with no chain to walk still has something to answer with. +var explainKinds = []model.Kind{ + model.KindArtistArtwork, model.KindAlbumArtwork, model.KindDiscArtwork, + model.KindMediaFileArtwork, model.KindPlaylistArtwork, model.KindRadioArtwork, +} + +func kindPrefixes(kinds []model.Kind) string { + return strings.Join(slice.Map(kinds, func(k model.Kind) string { return k.Prefix() }), ", ") +} + +func parseArtworkKind(s string, valid []model.Kind) (model.Kind, error) { + kind, ok := model.ParseKind(s) + if ok && slices.Contains(valid, kind) { + return kind, nil + } + return kind, fmt.Errorf("invalid kind %q, expected one of: %s", s, kindPrefixes(valid)) +} + +// explainAgents accounts for every configured agent: one the CLI cannot construct (a plugin, or a +// built-in missing its credentials) never reaches the Chain, so the raw list alone overstates it. +func explainAgents(configured string, available []string) string { + if strings.TrimSpace(configured) == "" { + return "(none)" + } + var unavailable bool + names := slice.Map(strings.Split(configured, ","), func(name string) string { + name = strings.TrimSpace(name) + if slices.Contains(available, name) { + return name + } + unavailable = true + return name + "*" + }) + line := strings.Join(names, ", ") + if unavailable { + line += " (* not available to the CLI)" + } + return line +} + +// availableImageAgents names the agents that can actually supply an image for kind. +func availableImageAgents(ds model.DataStore, kind model.Kind) []string { + ag := agents.GetAgents(ds, getPluginManager()) + if kind == model.KindArtistArtwork { + return slice.Map(ag.ArtistImageAgents(), func(a agents.ArtistImageAgent) string { return a.Name }) + } + return slice.Map(ag.AlbumImageAgents(), func(a agents.AlbumImageAgent) string { return a.Name }) +} + +// explainResult states the verdict of the walk. A skipped or failed external tier, or a local +// candidate that would not open, leaves the outcome unknown: nothing observed that there is no artwork. +func explainResult(source string, steps []artwork.TraceStep) string { + if source != "" { + for _, s := range steps { + if s.Outcome == artwork.OutcomeHit { + break + } + if s.Outcome == artwork.OutcomeWouldTry { + return "resolved from " + source + + " (offline: a higher-priority external candidate was not tried; re-run with --live)" + } + // An external winner discards the earlier error, so the resolver settles it with no retry. + if s.Outcome == artwork.OutcomeError && strings.HasPrefix(s.Candidate, artwork.ExternalPrefix) && + !strings.HasPrefix(source, artwork.ExternalPrefix) { + return "resolved from " + source + + " (indeterminate: a higher-priority external lookup failed; this may resolve differently on a retry)" + } + } + return "resolved from " + source + } + for _, s := range steps { + switch { + case s.Outcome == artwork.OutcomeWouldTry: + return "indeterminate (external agents not called; re-run with --live)" + case s.Outcome == artwork.OutcomeError && strings.HasPrefix(s.Candidate, artwork.ExternalPrefix): + return "indeterminate (an external lookup failed; the item may resolve on a later attempt)" + // The worker treats an unreadable local candidate exactly as it treats a failed external one: + // it retries instead of settling absent, so the verdict must not read as a clean miss. + case s.Outcome == artwork.OutcomeUnreadable: + return "indeterminate (a candidate exists but could not be read; the worker retries rather than settling absent)" + } + } + return "not resolved" +} + +// explainConfig names the setting that decides where a kind's artwork comes from, and its value. +func explainConfig(kind model.Kind) (name, value string) { + switch kind { + case model.KindArtistArtwork: + return "ArtistArtPriority", conf.Server.ArtistArtPriority + case model.KindAlbumArtwork: + return "CoverArtPriority", conf.Server.CoverArtPriority + case model.KindDiscArtwork: + return "DiscArtPriority", conf.Server.DiscArtPriority + case model.KindMediaFileArtwork: + return "EnableMediaFileCoverArt", strconv.FormatBool(conf.Server.EnableMediaFileCoverArt) + } + return "", "" +} + +type explainReport struct { + kind model.Kind + id string + name string + stored *model.ItemArtwork + queued *model.ArtworkQueueItem + agents string + steps []artwork.TraceStep + source string + resolveErr error +} + +func formatExplain(rep explainReport) string { + var sb strings.Builder + w := newTabWriter(&sb) + explainable := artwork.Explainable(rep.kind) + stateful := artwork.KeepsState(rep.kind) + + fmt.Fprintln(w, "Item") + fmt.Fprintf(w, " Kind:\t%s (%s)\n", rep.kind, rep.kind.Prefix()) + fmt.Fprintf(w, " ID:\t%s\n", rep.id) + fmt.Fprintf(w, " Name:\t%s\n", rep.name) + + fmt.Fprintln(w, "\nStored") + switch { + case !stateful: + fmt.Fprintf(w, " (%s artwork is resolved on every request and never recorded)\n", rep.kind) + case rep.stored == nil: + fmt.Fprintln(w, " (no artwork state recorded)") + default: + fmt.Fprintf(w, " Source:\t%s\n", displaySource(rep.stored.Source)) + fmt.Fprintf(w, " Hash:\t%s\n", cmp.Or(rep.stored.Hash, "(absent)")) + if rep.stored.SourcePath != "" { + fmt.Fprintf(w, " Source path:\t%s\n", rep.stored.SourcePath) + } + fmt.Fprintf(w, " Attempted at:\t%s\n", formatTime(rep.stored.AttemptedAt)) + } + + fmt.Fprintln(w, "\nQueue") + switch { + case !stateful: + fmt.Fprintln(w, " (never queued)") + case rep.queued == nil: + fmt.Fprintln(w, " (not queued)") + default: + fmt.Fprintf(w, " Priority:\t%s (%d)\n", priorityName(rep.queued.Priority), rep.queued.Priority) + fmt.Fprintf(w, " Attempts:\t%d\n", rep.queued.Attempts) + fmt.Fprintf(w, " Retry at:\t%s\n", formatTime(rep.queued.RetryAt)) + } + + fmt.Fprintln(w, "\nConfig") + if setting, value := explainConfig(rep.kind); setting == "" { + fmt.Fprintln(w, " (no artwork source configuration applies)") + } else { + fmt.Fprintf(w, " %s:\t%s\n", setting, value) + if rep.agents != "" { + fmt.Fprintf(w, " Agents:\t%s\n", rep.agents) + } + } + + fmt.Fprintln(w, "\nChain") + if !explainable { + fmt.Fprintf(w, " (%s artwork does not walk a priority chain)\n", rep.kind) + } else { + fmt.Fprintln(w, " CANDIDATE\tOUTCOME\tDETAIL") + for _, s := range rep.steps { + // A row with an empty last cell would end tabwriter's column block, breaking alignment. + fmt.Fprintf(w, " %s\t%s\t%s\n", s.Candidate, s.Outcome, cmp.Or(s.Detail, "-")) + } + } + + fmt.Fprintln(w, "\nResult") + switch { + case rep.resolveErr != nil: + fmt.Fprintf(w, " resolution failed: %s\n", rep.resolveErr) + case !explainable: + fmt.Fprintln(w, " not evaluated (no chain was walked; see Stored above)") + default: + fmt.Fprintf(w, " %s\n", explainResult(rep.source, rep.steps)) + } + + w.Flush() + return sb.String() +} + +func formatTime(t time.Time) string { + if t.IsZero() { + return "-" + } + return t.Format(time.RFC3339) +} + +func runExplain(ctx context.Context, kind model.Kind, id string) { + defer db.Init(ctx)() + ds, ctx := getAdminContext(ctx) + + name, err := artworkItemName(ctx, ds, kind, id) + if err != nil { + log.Fatal(ctx, "Item not found", "kind", kind, "id", id, err) + } + rep := explainReport{kind: kind, id: id, name: name} + if artwork.KeepsState(kind) { + rep.stored, err = ds.Artwork(ctx).GetItemArtwork(kind, id, model.ImageTypePrimary) + if err != nil && !errors.Is(err, model.ErrNotFound) { + log.Fatal(ctx, "Failed to read artwork state", "kind", kind, "id", id, err) + } + rep.queued, err = ds.ArtworkQueue(ctx).Get(kind, id, model.ImageTypePrimary) + if err != nil && !errors.Is(err, model.ErrNotFound) { + log.Fatal(ctx, "Failed to read the artwork queue", "kind", kind, "id", id, err) + } + } + + if artwork.Explainable(kind) { + if kind == model.KindArtistArtwork || kind == model.KindAlbumArtwork { + rep.agents = explainAgents(conf.Server.Agents, availableImageAgents(ds, kind)) + } + trace := &artwork.ChainTrace{} + rep.source, rep.resolveErr = CreateArtworkResolver(trace, explainLive).Resolve(ctx, kind, id) + rep.steps = trace.Steps() + } + + fmt.Print(formatExplain(rep)) + // The steps taken before a failed walk are the diagnosis, so report them before exiting. + if rep.resolveErr != nil { + log.Fatal(ctx, "Failed to resolve artwork", "kind", kind, "id", id, rep.resolveErr) + } +} + +// artworkItemName looks the entity up under its own kind, so a mismatched kind/id pair is +// reported as not found instead of silently explaining another entity's artwork. +func artworkItemName(ctx context.Context, ds model.DataStore, kind model.Kind, id string) (string, error) { + switch kind { + case model.KindArtistArtwork: + ar, err := ds.Artist(ctx).Get(id) + if err != nil { + return "", err + } + return ar.Name, nil + case model.KindAlbumArtwork: + al, err := ds.Album(ctx).Get(id) + if err != nil { + return "", err + } + return al.Name, nil + case model.KindPlaylistArtwork: + pls, err := ds.Playlist(ctx).Get(id) + if err != nil { + return "", err + } + return pls.Name, nil + case model.KindRadioArtwork: + rd, err := ds.Radio(ctx).Get(id) + if err != nil { + return "", err + } + return rd.Name, nil + case model.KindMediaFileArtwork: + mf, err := ds.MediaFile(ctx).Get(id) + if err != nil { + return "", err + } + return mf.Title, nil + case model.KindDiscArtwork: + return discArtworkName(ctx, ds, id) + } + return "", fmt.Errorf("unsupported kind %q", kind.Prefix()) +} + +func discArtworkName(ctx context.Context, ds model.DataStore, id string) (string, error) { + albumID, discNumber, err := model.ParseDiscArtworkID(id) + if err != nil { + return "", err + } + al, err := ds.Album(ctx).Get(albumID) + if err != nil { + return "", err + } + name := fmt.Sprintf("%s (disc %d)", al.Name, discNumber) + // The subtitle is itself a DiscArtPriority candidate, so name it where the chain can be read against it. + if subtitle := strings.TrimSpace(al.Discs[discNumber]); subtitle != "" { + name += ": " + subtitle + } + return name, nil +} diff --git a/cmd/artwork_test.go b/cmd/artwork_test.go new file mode 100644 index 000000000..949fdb42a --- /dev/null +++ b/cmd/artwork_test.go @@ -0,0 +1,899 @@ +package cmd + +import ( + "context" + "errors" + "io" + "strings" + "time" + + "github.com/navidrome/navidrome/conf" + "github.com/navidrome/navidrome/conf/configtest" + "github.com/navidrome/navidrome/consts" + "github.com/navidrome/navidrome/core/artwork" + "github.com/navidrome/navidrome/model" + "github.com/navidrome/navidrome/tests" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("parseArtworkKind", func() { + It("accepts a supported kind", func() { + k, err := parseArtworkKind("ar", artwork.RecheckKinds) + Expect(err).ToNot(HaveOccurred()) + Expect(k).To(Equal(model.KindArtistArtwork)) + }) + + It("rejects an unknown kind and lists the valid ones", func() { + _, err := parseArtworkKind("zz", artwork.RecheckKinds) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("ar")) + Expect(err.Error()).To(ContainSubstring("al")) + }) + + It("rejects a known kind the command does not accept", func() { + _, err := parseArtworkKind("mf", artwork.RecheckKinds) + Expect(err).To(HaveOccurred()) + }) + + DescribeTable("accepts the kinds each command supports", + func(prefix string, valid []model.Kind) { + _, err := parseArtworkKind(prefix, valid) + Expect(err).ToNot(HaveOccurred()) + }, + Entry("explain reads disc artwork", "dc", explainKinds), + Entry("explain reads media file artwork", "mf", explainKinds), + // Disc artwork has no state to clear and the worker cannot resolve it, so refresh must not + // accept it: the queue row would be rejected on every drain. + Entry("refresh re-queues media files", "mf", artwork.RefreshableKinds), + ) + + It("rejects disc artwork for refresh", func() { + _, err := parseArtworkKind("dc", artwork.RefreshableKinds) + Expect(err).To(HaveOccurred()) + }) +}) + +var _ = Describe("explainResult", func() { + It("reports the winning source", func() { + steps := []artwork.TraceStep{{Candidate: "folder", Outcome: "hit", Detail: "/music/a.jpg"}} + Expect(explainResult("folder", steps)).To(ContainSubstring("resolved from folder")) + }) + + It("reports not resolved when every candidate was tried and missed", func() { + steps := []artwork.TraceStep{ + {Candidate: "artist.*", Outcome: "miss"}, + {Candidate: "external:deezer", Outcome: "miss"}, + } + Expect(explainResult("", steps)).To(Equal("not resolved")) + }) + + It("reports indeterminate when a local candidate exists but could not be read", func() { + steps := []artwork.TraceStep{ + {Candidate: "cover.*", Outcome: "miss"}, + {Candidate: "embedded", Outcome: "unreadable"}, + } + Expect(explainResult("", steps)).To(ContainSubstring("indeterminate"), + "the worker retries an unreadable candidate instead of settling absent, so this is not a clean miss") + }) + + It("does not qualify a hit that an earlier unreadable candidate preceded", func() { + // chainState.try stamps only the external error onto a hit and drops the local one, so the + // worker settles this as found; warning about it would be a false alarm. + steps := []artwork.TraceStep{ + {Candidate: "embedded", Outcome: "unreadable"}, + {Candidate: "cover.*", Outcome: "hit", Detail: "/music/cover.jpg"}, + } + Expect(explainResult("folder", steps)).To(Equal("resolved from folder")) + }) + + It("reports indeterminate when an external lookup failed transiently", func() { + steps := []artwork.TraceStep{ + {Candidate: "artist.*", Outcome: "miss"}, + {Candidate: "external:deezer", Outcome: "error", Detail: "context deadline exceeded"}, + } + Expect(explainResult("", steps)).To(ContainSubstring("indeterminate"), + "a failed network call is not evidence that the item has no artwork") + }) + + It("qualifies a win a skipped higher-priority external candidate could have taken", func() { + steps := []artwork.TraceStep{ + {Candidate: "external:deezer", Outcome: "would-try"}, + {Candidate: "artist.*", Outcome: "hit", Detail: "/music/artist.jpg"}, + } + res := explainResult("artist.*", steps) + Expect(res).To(ContainSubstring("resolved from artist.*")) + Expect(res).To(ContainSubstring("--live"), + "offline, the winner is only the winner because the external tier was skipped") + }) + + It("does not qualify a win that no skipped candidate outranked", func() { + steps := []artwork.TraceStep{ + {Candidate: "artist.*", Outcome: "hit"}, + {Candidate: "external:deezer", Outcome: "would-try"}, + } + Expect(explainResult("artist.*", steps)).To(Equal("resolved from artist.*")) + }) + + It("reports indeterminate when external agents were never called", func() { + steps := []artwork.TraceStep{ + {Candidate: "artist.*", Outcome: "miss"}, + {Candidate: "external:deezer", Outcome: "would-try"}, + } + Expect(explainResult("", steps)).To(ContainSubstring("indeterminate"), + "an offline run must not claim an item is unresolvable when external agents were skipped") + }) + + It("qualifies a win a failed higher-priority external lookup could have taken", func() { + steps := []artwork.TraceStep{ + {Candidate: "external:deezer", Outcome: "error", Detail: "context deadline exceeded"}, + {Candidate: "artist.*", Outcome: "hit", Detail: "/music/artist.jpg"}, + } + res := explainResult("artist.*", steps) + Expect(res).To(ContainSubstring("resolved from artist.*")) + Expect(res).To(ContainSubstring("indeterminate"), + "the resolver serves this hit but retries later, so the winner is provisional") + }) + + It("does not qualify an external win that followed a failed external lookup", func() { + steps := []artwork.TraceStep{ + {Candidate: "external:deezer", Outcome: "error", Detail: "context deadline exceeded"}, + {Candidate: "external:lastfm", Outcome: "hit", Detail: "http://img"}, + } + Expect(explainResult("external:lastfm", steps)).To(Equal("resolved from external:lastfm"), + "a later agent supplying the image discards the earlier error, so there is no retry to warn about") + }) + + It("does not qualify a win that outranked the failed external lookup", func() { + steps := []artwork.TraceStep{ + {Candidate: "artist.*", Outcome: "hit"}, + {Candidate: "external:deezer", Outcome: "error", Detail: "context deadline exceeded"}, + } + Expect(explainResult("artist.*", steps)).To(Equal("resolved from artist.*")) + }) +}) + +var _ = Describe("explainAgents", func() { + It("accounts for every configured agent, marking the ones the CLI could not use", func() { + out := explainAgents("artist-nfo-metadata,apple-music,deezer,lastfm", []string{"deezer"}) + for _, name := range []string{"artist-nfo-metadata", "apple-music", "deezer", "lastfm"} { + Expect(out).To(ContainSubstring(name), + "a configured agent missing from this line reads as if it had never been configured") + } + Expect(out).To(ContainSubstring("not available to the CLI")) + }) + + It("does not mark anything when every configured agent is available", func() { + out := explainAgents("deezer, lastfm", []string{"lastfm", "deezer"}) + Expect(out).To(Equal("deezer, lastfm")) + }) + + It("reports an empty configuration as none, not as an unavailable agent", func() { + Expect(explainAgents("", nil)).To(Equal("(none)")) + }) +}) + +var _ = Describe("formatExplain", func() { + var rep explainReport + + BeforeEach(func() { + DeferCleanup(configtest.SetupConfig()) + conf.Server.ArtistArtPriority = "external, artist.*" + rep = explainReport{ + kind: model.KindArtistArtwork, + id: "ar-1", + name: "Radiohead", + agents: "lastfm,spotify", + steps: []artwork.TraceStep{ + {Candidate: "upload", Outcome: "skipped", Detail: "no uploaded image"}, + {Candidate: "external:deezer", Outcome: "would-try"}, + }, + source: "", + } + }) + + It("reports the item, its config and the chain it walked", func() { + out := formatExplain(rep) + Expect(out).To(ContainSubstring("Radiohead")) + Expect(out).To(ContainSubstring("ar-1")) + Expect(out).To(ContainSubstring("ArtistArtPriority")) + Expect(out).To(ContainSubstring("lastfm,spotify")) + Expect(out).To(ContainSubstring("external:deezer")) + Expect(out).To(ContainSubstring("would-try")) + Expect(out).To(ContainSubstring("indeterminate")) + }) + + It("reports the absence of stored state and of a queue row", func() { + out := formatExplain(rep) + Expect(out).To(ContainSubstring("no artwork state recorded")) + Expect(out).To(ContainSubstring("not queued")) + }) + + It("prints the stored state and the queue row when they exist", func() { + attempted := time.Date(2026, 8, 13, 10, 0, 0, 0, time.UTC) + rep.stored = &model.ItemArtwork{Source: "folder", Hash: "abc123", + SourcePath: "/music/cover.jpg", AttemptedAt: attempted} + rep.queued = &model.ArtworkQueueItem{Priority: model.ArtworkPriorityScan, Attempts: 2, + RetryAt: attempted.Add(time.Hour)} + rep.source = "folder" + + out := formatExplain(rep) + Expect(out).To(ContainSubstring("abc123")) + Expect(out).To(ContainSubstring("/music/cover.jpg")) + Expect(out).To(ContainSubstring("2026-08-13T10:00:00Z")) + Expect(out).To(ContainSubstring("scan (50)"), "a bare 50 makes the operator look the priority up") + Expect(out).To(ContainSubstring("resolved from folder")) + }) + + It("marks a known-absent stored state instead of printing an empty hash", func() { + rep.stored = &model.ItemArtwork{AttemptedAt: time.Now()} + Expect(formatExplain(rep)).To(ContainSubstring("absent")) + }) + + It("reports a failed walk as failed, not as unresolved", func() { + rep.resolveErr = errors.New("no such directory") + + out := formatExplain(rep) + Expect(out).To(ContainSubstring("resolution failed: no such directory")) + Expect(out).ToNot(ContainSubstring("indeterminate")) + Expect(out).To(ContainSubstring("would-try"), "the steps taken before the failure still print") + }) + + It("says a kind that does not walk a chain has no chain, without an empty table", func() { + conf.Server.CoverArtPriority = "cover.*, embedded" + rep.kind = model.KindPlaylistArtwork + rep.steps = nil + rep.agents = "" + + out := formatExplain(rep) + Expect(out).To(ContainSubstring("does not walk a priority chain")) + Expect(out).ToNot(ContainSubstring("CANDIDATE"), + "an empty chain table reads as 'nothing was tried', which is false") + Expect(out).ToNot(ContainSubstring("not resolved"), + "nothing was resolved because nothing was attempted") + Expect(out).ToNot(ContainSubstring("CoverArtPriority"), + "the priority chain config does not govern this kind") + }) + + It("says disc artwork keeps no state instead of reporting it as unresolved state", func() { + conf.Server.DiscArtPriority = "cover.jpg, embedded" + rep = explainReport{ + kind: model.KindDiscArtwork, id: "al-1:2", name: "OK Computer (disc 2)", + steps: []artwork.TraceStep{{Candidate: "cover.jpg", Outcome: "hit", Detail: "/music/cover.jpg"}}, + source: "folder", + } + + out := formatExplain(rep) + Expect(out).To(ContainSubstring("never recorded")) + Expect(out).To(ContainSubstring("never queued")) + Expect(out).ToNot(ContainSubstring("no artwork state recorded"), + "a missing row would read as a lookup that failed, when disc artwork has no row by design") + Expect(out).To(ContainSubstring("DiscArtPriority")) + Expect(out).ToNot(ContainSubstring("Agents:"), "disc artwork never asks an agent") + Expect(out).To(ContainSubstring("resolved from folder")) + }) + + It("reports the setting that governs media file artwork", func() { + conf.Server.EnableMediaFileCoverArt = false + rep = explainReport{ + kind: model.KindMediaFileArtwork, id: "mf-1", name: "Airbag", + steps: []artwork.TraceStep{ + {Candidate: "embedded", Outcome: "skipped", Detail: "EnableMediaFileCoverArt is off"}, + }, + } + + out := formatExplain(rep) + Expect(out).To(ContainSubstring("EnableMediaFileCoverArt")) + Expect(out).To(ContainSubstring("false")) + Expect(out).To(ContainSubstring("not resolved")) + Expect(out).To(ContainSubstring("no artwork state recorded"), "media files do keep state") + }) +}) + +var _ = Describe("explainConfig", func() { + BeforeEach(func() { + DeferCleanup(configtest.SetupConfig()) + conf.Server.DiscArtPriority = "cover.jpg" + conf.Server.EnableMediaFileCoverArt = true + }) + + DescribeTable("names the setting that decides where a kind's artwork comes from", + func(kind model.Kind, setting, value string) { + gotSetting, gotValue := explainConfig(kind) + Expect(gotSetting).To(Equal(setting)) + Expect(gotValue).To(Equal(value)) + }, + Entry("disc", model.KindDiscArtwork, "DiscArtPriority", "cover.jpg"), + Entry("media file", model.KindMediaFileArtwork, "EnableMediaFileCoverArt", "true"), + Entry("playlist has none", model.KindPlaylistArtwork, "", ""), + ) +}) + +var _ = Describe("discArtworkName", func() { + var ds *tests.MockDataStore + + BeforeEach(func() { + albumRepo := tests.CreateMockAlbumRepo() + albumRepo.SetData(model.Albums{{ID: "al-1", Name: "Sandinista!", Discs: model.Discs{2: "Side Three"}}}) + ds = &tests.MockDataStore{MockedAlbum: albumRepo} + }) + + It("names the album, the disc and its subtitle", func() { + name, err := artworkItemName(context.Background(), ds, model.KindDiscArtwork, "al-1:2") + Expect(err).ToNot(HaveOccurred()) + Expect(name).To(Equal("Sandinista! (disc 2): Side Three")) + }) + + It("omits the subtitle when the disc has none", func() { + name, err := artworkItemName(context.Background(), ds, model.KindDiscArtwork, "al-1:1") + Expect(err).ToNot(HaveOccurred()) + Expect(name).To(Equal("Sandinista! (disc 1)")) + }) + + It("rejects an id that is not :", func() { + _, err := artworkItemName(context.Background(), ds, model.KindDiscArtwork, "al-1") + Expect(err).To(HaveOccurred()) + }) +}) + +var _ = Describe("artwork refresh command", func() { + It("requires at least a kind and one id", func() { + Expect(artworkRefreshCmd.Args(artworkRefreshCmd, []string{"ar"})).To(HaveOccurred()) + Expect(artworkRefreshCmd.Args(artworkRefreshCmd, []string{"ar", "id1"})).ToNot(HaveOccurred()) + Expect(artworkRefreshCmd.Args(artworkRefreshCmd, []string{"ar", "id1", "id2"})).ToNot(HaveOccurred()) + }) +}) + +var _ = Describe("artwork reprocess selection", func() { + It("errors when no selector is given", func() { + _, err := selectedKinds(nil, nil, false) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("--all")) + }) + + It("returns every kind for --all", func() { + ks, err := selectedKinds(nil, nil, true) + Expect(err).ToNot(HaveOccurred()) + Expect(ks).To(ConsistOf(artwork.RecheckKinds)) + }) + + It("returns every kind for a source filter given without a kind", func() { + ks, err := selectedKinds(nil, []string{"folder"}, false) + Expect(err).ToNot(HaveOccurred()) + Expect(ks).To(ConsistOf(artwork.RecheckKinds), "--source alone is already a complete selection") + }) + + It("returns only the named kinds", func() { + ks, err := selectedKinds([]string{"ar"}, nil, false) + Expect(err).ToNot(HaveOccurred()) + Expect(ks).To(Equal([]model.Kind{model.KindArtistArtwork})) + }) + + It("keeps a named kind selection alongside a source filter", func() { + ks, err := selectedKinds([]string{"ar"}, []string{"folder"}, false) + Expect(err).ToNot(HaveOccurred()) + Expect(ks).To(Equal([]model.Kind{model.KindArtistArtwork})) + }) + + It("rejects an unknown kind", func() { + _, err := selectedKinds([]string{"zz"}, nil, false) + Expect(err).To(HaveOccurred()) + }) + + It("counts a repeated kind once", func() { + ks, err := selectedKinds([]string{"ar", "ar"}, nil, false) + Expect(err).ToNot(HaveOccurred()) + Expect(ks).To(Equal([]model.Kind{model.KindArtistArtwork})) + }) +}) + +var _ = Describe("explain/reprocess source round trip", func() { + ctx := context.Background() + + // storedSource reads back the Source line explain printed, as an operator would copy it. + storedSource := func(out string) string { + GinkgoHelper() + for line := range strings.SplitSeq(out, "\n") { + if after, ok := strings.CutPrefix(strings.TrimSpace(line), "Source:"); ok { + return strings.TrimSpace(after) + } + } + Fail("explain printed no Source line") + return "" + } + + It("names the absent state as reprocess --source accepts it", func() { + ds := &tests.MockDataStore{} + art := ds.Artwork(ctx).(*tests.MockArtworkRepo) + Expect(art.PutItemArtwork(&model.ItemArtwork{ItemKind: model.KindArtistArtwork.Prefix(), + ItemID: "ar-1", ImageType: model.ImageTypePrimary})).To(Succeed()) + + shown := storedSource(formatExplain(explainReport{kind: model.KindArtistArtwork, id: "ar-1", + stored: &model.ItemArtwork{AttemptedAt: time.Now()}})) + + q := ds.ArtworkQueue(ctx) + Expect(validateSources(q, repositorySources([]string{shown}))).To(Succeed(), + "explain's spelling of a source must be pasteable into --source") + Expect(validateSources(q, repositorySources([]string{"(" + shown + ")"}))).ToNot(Succeed(), + "a parenthesised name would be rejected, so explain must not print one") + }) +}) + +var _ = Describe("repositorySources", func() { + It("maps the user-facing absent name onto the stored empty source", func() { + Expect(repositorySources([]string{"absent", "folder"})).To(Equal([]string{"", "folder"})) + }) + + It("keeps an empty selection empty, meaning every source", func() { + Expect(repositorySources(nil)).To(BeEmpty()) + }) +}) + +var _ = Describe("promptConfirm", func() { + var out strings.Builder + + BeforeEach(func() { out.Reset() }) + + It("states the external cost and accepts an explicit yes", func() { + Expect(promptConfirm(strings.NewReader("y\n"))(&out, 42, 7)).To(BeTrue()) + Expect(out.String()).To(ContainSubstring("re-resolve 42 items")) + Expect(out.String()).To(ContainSubstring("External lookups: ~7 estimated")) + }) + + It("defaults to no on anything else", func() { + Expect(promptConfirm(strings.NewReader("\n"))(&out, 1, 1)).To(BeFalse()) + Expect(promptConfirm(strings.NewReader("nope\n"))(&out, 1, 1)).To(BeFalse()) + Expect(promptConfirm(strings.NewReader(""))(&out, 1, 1)).To(BeFalse()) + }) + + It("drops the external clause when no lookup will be made", func() { + Expect(promptConfirm(strings.NewReader("y\n"))(&out, 3, 0)).To(BeTrue()) + Expect(out.String()).To(ContainSubstring("re-resolve 3 items.")) + Expect(out.String()).ToNot(ContainSubstring("External lookups")) + }) +}) + +var _ = Describe("reprocessConfirm", func() { + var out strings.Builder + + BeforeEach(func() { out.Reset() }) + + It("prompts when --yes was not given", func() { + Expect(reprocessConfirm(false, strings.NewReader("n\n"))(&out, 5, 5)).To(BeFalse()) + Expect(out.String()).To(ContainSubstring("Continue?")) + }) + + It("bypasses the prompt only for --yes", func() { + Expect(reprocessConfirm(true, strings.NewReader(""))(&out, 5, 5)).To(BeTrue()) + Expect(out.String()).To(BeEmpty(), "--yes must not print a prompt it never reads") + }) +}) + +var _ = Describe("reprocessArtwork", func() { + var ds *tests.MockDataStore + var art *tests.MockArtworkRepo + var queue *tests.MockArtworkQueueRepo + var out strings.Builder + var imageAgents artwork.ImageAgentCount + ctx := context.Background() + kinds := []model.Kind{model.KindArtistArtwork, model.KindAlbumArtwork} + accept := func(io.Writer, int64, int64) bool { return true } + decline := func(io.Writer, int64, int64) bool { return false } + + put := func(kind model.Kind, id, source string) { + Expect(art.PutItemArtwork(&model.ItemArtwork{ItemKind: kind.Prefix(), ItemID: id, + ImageType: model.ImageTypePrimary, Hash: "h" + id, Source: source})).To(Succeed()) + } + + BeforeEach(func() { + DeferCleanup(configtest.SetupConfig()) + conf.Server.CoverArtPriority = "cover.*, external" + conf.Server.ArtistArtPriority = "artist.*, external" + conf.Server.EnableM3UExternalAlbumArt = false + imageAgents = artwork.ImageAgentCount{Artist: 1, Album: 1} + ds = &tests.MockDataStore{} + art = ds.Artwork(ctx).(*tests.MockArtworkRepo) + queue = ds.ArtworkQueue(ctx).(*tests.MockArtworkQueueRepo) + out.Reset() + put(model.KindArtistArtwork, "ar-1", "external:deezer") + put(model.KindArtistArtwork, "ar-2", "") + put(model.KindAlbumArtwork, "al-1", "external:deezer") + put(model.KindAlbumArtwork, "al-2", "folder") + }) + + It("previews the per-kind breakdown and queues nothing on a dry run", func() { + Expect(reprocessArtwork(ctx, ds, kinds, []string{"external:deezer"}, imageAgents, true, accept, &out)).To(Succeed()) + + Expect(out.String()).To(ContainSubstring("external:deezer")) + Expect(out.String()).To(ContainSubstring("artist")) + Expect(out.String()).To(ContainSubstring("album")) + Expect(out.String()).To(ContainSubstring("TOTAL")) + Expect(out.String()).To(ContainSubstring("Dry run")) + Expect(queue.Count()).To(BeZero()) + }) + + It("queues nothing when the operator declines", func() { + Expect(reprocessArtwork(ctx, ds, kinds, nil, imageAgents, false, decline, &out)).To(Succeed()) + + Expect(out.String()).To(ContainSubstring("Aborted")) + Expect(queue.Count()).To(BeZero()) + }) + + It("queues the matching items at recheck priority, leaving their artwork state alone", func() { + Expect(reprocessArtwork(ctx, ds, kinds, []string{"external:deezer"}, imageAgents, false, accept, &out)).To(Succeed()) + + Expect(queue.Count()).To(Equal(int64(2))) + queued, err := queue.Get(model.KindAlbumArtwork, "al-1", model.ImageTypePrimary) + Expect(err).ToNot(HaveOccurred()) + Expect(queued.Priority).To(Equal(model.ArtworkPriorityRecheck)) + _, err = queue.Get(model.KindAlbumArtwork, "al-2", model.ImageTypePrimary) + Expect(err).To(MatchError(model.ErrNotFound), "a non-matching source must not be queued") + + stored, err := art.GetItemArtwork(model.KindAlbumArtwork, "al-1", model.ImageTypePrimary) + Expect(err).ToNot(HaveOccurred()) + Expect(stored.Hash).To(Equal("hal-1"), "bulk reprocessing must not blank the current artwork") + }) + + It("targets the absent state", func() { + Expect(reprocessArtwork(ctx, ds, kinds, []string{""}, imageAgents, false, accept, &out)).To(Succeed()) + + Expect(queue.Count()).To(Equal(int64(1))) + _, err := queue.Get(model.KindArtistArtwork, "ar-2", model.ImageTypePrimary) + Expect(err).ToNot(HaveOccurred()) + }) + + It("reports matched and queued separately when part of the set is already queued", func() { + Expect(queue.Enqueue(model.ArtworkQueueItem{ItemKind: "ar", ItemID: "ar-1", + ImageType: model.ImageTypePrimary, Priority: model.ArtworkPriorityBump})).To(Succeed()) + + Expect(reprocessArtwork(ctx, ds, kinds, []string{"external:deezer"}, imageAgents, false, accept, &out)).To(Succeed()) + + Expect(out.String()).To(ContainSubstring("Queued 1 of 2 matched items")) + Expect(out.String()).To(ContainSubstring("Already queued, left unchanged: 1")) + queued, err := queue.Get(model.KindArtistArtwork, "ar-1", model.ImageTypePrimary) + Expect(err).ToNot(HaveOccurred()) + Expect(queued.Priority).To(Equal(model.ArtworkPriorityBump), + "an already-queued row keeps its priority and backoff") + }) + + It("stops at a selection that matches nothing instead of prompting", func() { + Expect(reprocessArtwork(ctx, ds, []model.Kind{model.KindRadioArtwork}, nil, imageAgents, false, + func(io.Writer, int64, int64) bool { + Fail("must not prompt when there is nothing to queue") + return true + }, &out)).To(Succeed()) + + Expect(out.String()).To(ContainSubstring("Nothing")) + Expect(queue.Count()).To(BeZero()) + }) + + It("reports an empty selection as a dry run when one was asked for", func() { + Expect(reprocessArtwork(ctx, ds, []model.Kind{model.KindRadioArtwork}, nil, imageAgents, true, accept, &out)).To(Succeed()) + + Expect(out.String()).To(ContainSubstring("Nothing matches")) + Expect(out.String()).To(ContainSubstring("Dry run")) + }) + + It("shows the external estimate on a dry run, which never reaches the prompt", func() { + Expect(reprocessArtwork(ctx, ds, []model.Kind{model.KindAlbumArtwork}, nil, imageAgents, true, accept, &out)).To(Succeed()) + + Expect(out.String()).To(ContainSubstring("External lookups: ~2 estimated")) + }) + + It("bills every agent per item, not one lookup per item", func() { + imageAgents = artwork.ImageAgentCount{Artist: 2, Album: 3} + var external int64 + capture := func(_ io.Writer, _, e int64) bool { external = e; return false } + + Expect(reprocessArtwork(ctx, ds, kinds, nil, imageAgents, false, capture, &out)).To(Succeed()) + + Expect(external).To(Equal(int64(2*2+2*3)), "2 artists at 2 agents plus 2 albums at 3 agents") + Expect(out.String()).To(ContainSubstring("External lookups: ~10 estimated")) + }) + + It("names the estimate's blind spots instead of claiming a bound it cannot hold", func() { + Expect(reprocessArtwork(ctx, ds, kinds, nil, imageAgents, true, accept, &out)).To(Succeed()) + + Expect(out.String()).To(ContainSubstring("plugin agents not counted")) + Expect(out.String()).To(ContainSubstring("local hits may need fewer")) + Expect(out.String()).ToNot(ContainSubstring("up to"), "plugin agents make any ceiling false") + Expect(out.String()).ToNot(ContainSubstring("at least"), "a local hit makes any floor false") + }) + + It("says so when the selection needs no external lookup", func() { + conf.Server.CoverArtPriority = "cover.*" + put(model.KindPlaylistArtwork, "pl-1", "playlist") + + Expect(reprocessArtwork(ctx, ds, []model.Kind{model.KindPlaylistArtwork}, nil, imageAgents, true, accept, &out)).To(Succeed()) + + Expect(out.String()).To(ContainSubstring("External lookups: none")) + }) + + It("counts playlists as external cost when the m3u image fetch is enabled", func() { + conf.Server.CoverArtPriority = "cover.*" + conf.Server.EnableM3UExternalAlbumArt = true + put(model.KindPlaylistArtwork, "pl-1", "playlist") + var external int64 + capture := func(_ io.Writer, _, e int64) bool { external = e; return false } + + Expect(reprocessArtwork(ctx, ds, []model.Kind{model.KindPlaylistArtwork}, nil, imageAgents, false, capture, &out)).To(Succeed()) + + Expect(external).To(Equal(int64(1))) + Expect(out.String()).To(ContainSubstring("External lookups: ~1 estimated")) + }) + + It("bills a playlist for every album its grid samples, at every agent", func() { + imageAgents = artwork.ImageAgentCount{Album: 3} + put(model.KindPlaylistArtwork, "pl-1", "playlist") + var external int64 + capture := func(_ io.Writer, _, e int64) bool { external = e; return false } + + Expect(reprocessArtwork(ctx, ds, []model.Kind{model.KindPlaylistArtwork}, nil, imageAgents, false, capture, &out)).To(Succeed()) + + Expect(external).To(Equal(int64(artwork.PlaylistGridSamples*3)), + "one playlist samples 4 albums, each walking all 3 album agents") + }) + + It("counts only the kinds that call an external agent as external cost", func() { + put(model.KindRadioArtwork, "ra-1", "upload") + var total, external int64 + capture := func(_ io.Writer, t, e int64) bool { total, external = t, e; return false } + + Expect(reprocessArtwork(ctx, ds, []model.Kind{model.KindAlbumArtwork, model.KindRadioArtwork}, + nil, imageAgents, false, capture, &out)).To(Succeed()) + + Expect(total).To(Equal(int64(3))) + Expect(external).To(Equal(int64(2)), "radio artwork never reaches an external agent") + }) + + It("rejects an unknown source and names the ones in use", func() { + err := reprocessArtwork(ctx, ds, kinds, []string{"externa:deezer"}, imageAgents, true, accept, &out) + + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("externa:deezer")) + Expect(err.Error()).To(ContainSubstring("external:deezer")) + Expect(err.Error()).To(ContainSubstring("folder")) + Expect(err.Error()).To(ContainSubstring("absent"), "the empty source prints under its user-facing name") + Expect(queue.Count()).To(BeZero()) + }) + + It("accepts the absent filter with nothing absent, still rejecting a typo", func() { + put(model.KindArtistArtwork, "ar-2", "folder") + + Expect(reprocessArtwork(ctx, ds, kinds, repositorySources([]string{absentSource}), + imageAgents, false, accept, &out)).To(Succeed(), + "a reserved source must stay valid once the library has none of it") + Expect(out.String()).To(ContainSubstring("Nothing matches")) + Expect(queue.Count()).To(BeZero()) + + Expect(reprocessArtwork(ctx, ds, kinds, repositorySources([]string{"absnt"}), + imageAgents, true, accept, &out)).ToNot(Succeed(), "a typo must still be rejected") + }) + + It("accepts a source another kind uses, letting the empty selection report itself", func() { + Expect(reprocessArtwork(ctx, ds, []model.Kind{model.KindArtistArtwork}, []string{"folder"}, + imageAgents, false, decline, &out)).To(Succeed()) + + Expect(out.String()).To(ContainSubstring("Nothing matches"), + "a well-formed filter must not be reported as a typo because of the kinds selected") + Expect(queue.Count()).To(BeZero()) + }) +}) + +var _ = Describe("artwork status command", func() { + It("takes no arguments", func() { + Expect(artworkStatusCmd.Args(artworkStatusCmd, []string{})).ToNot(HaveOccurred()) + Expect(artworkStatusCmd.Args(artworkStatusCmd, []string{"x"})).To(HaveOccurred()) + }) +}) + +var _ = Describe("collectStatus", func() { + var ds *tests.MockDataStore + var art *tests.MockArtworkRepo + var queue *tests.MockArtworkQueueRepo + ctx := context.Background() + + BeforeEach(func() { + ds = &tests.MockDataStore{} + art = ds.Artwork(ctx).(*tests.MockArtworkRepo) + queue = ds.ArtworkQueue(ctx).(*tests.MockArtworkQueueRepo) + put := func(kind model.Kind, id, source, hash string, attempted time.Time) { + Expect(art.PutItemArtwork(&model.ItemArtwork{ItemKind: kind.Prefix(), ItemID: id, + ImageType: model.ImageTypePrimary, Source: source, Hash: hash, AttemptedAt: attempted})).To(Succeed()) + } + put(model.KindArtistArtwork, "ar-1", "external:deezer", "h1", time.Now()) + put(model.KindArtistArtwork, "ar-2", "", "", time.Now().Add(-48*time.Hour)) + put(model.KindArtistArtwork, "ar-3", "", "", time.Now()) + put(model.KindAlbumArtwork, "al-1", "folder", "h2", time.Now()) + Expect(queue.Enqueue(model.ArtworkQueueItem{ItemKind: "ar", ItemID: "ar-9", + ImageType: model.ImageTypePrimary, Priority: model.ArtworkPriorityBackfill})).To(Succeed()) + }) + + It("reports the queue, the source distribution and the absent ages", func() { + rep, err := collectStatus(ctx, ds) + Expect(err).ToNot(HaveOccurred()) + + Expect(rep.queue).To(ConsistOf(model.ArtworkQueueStat{ItemKind: "ar", + Priority: model.ArtworkPriorityBackfill, Count: 1})) + Expect(rep.sources).To(ContainElements( + sourceCount{kind: model.KindArtistArtwork, source: "external:deezer", count: 1}, + sourceCount{kind: model.KindArtistArtwork, source: "", count: 2}, + sourceCount{kind: model.KindAlbumArtwork, source: "folder", count: 1}, + )) + Expect(rep.absent).To(ContainElement(absentCount{kind: model.KindArtistArtwork, + ArtworkAbsentStat: model.ArtworkAbsentStat{Total: 2, Stale: 1}})) + }) + + It("compares the stored fingerprint against the current one", func() { + Expect(ds.Property(ctx).Put(consts.ArtConfFingerprintPropertyKey, "old-fingerprint")).To(Succeed()) + + rep, err := collectStatus(ctx, ds) + Expect(err).ToNot(HaveOccurred()) + Expect(rep.stored).To(Equal("old-fingerprint")) + Expect(rep.current).To(Equal(artwork.ConfigFingerprint()), + "the CLI must report the value backfill itself compares") + }) + + It("queues nothing", func() { + _, err := collectStatus(ctx, ds) + Expect(err).ToNot(HaveOccurred()) + Expect(queue.Count()).To(Equal(int64(1)), "status must not enqueue anything") + }) +}) + +var _ = Describe("formatStatus", func() { + var rep statusReport + + BeforeEach(func() { + rep = statusReport{ + queue: []model.ArtworkQueueStat{ + {ItemKind: "ar", Priority: model.ArtworkPriorityBackfill, Count: 2}, + {ItemKind: "al", Priority: model.ArtworkPriorityScan, Count: 1}, + }, + sources: []sourceCount{ + {kind: model.KindArtistArtwork, source: "external:deezer", count: 5}, + {kind: model.KindArtistArtwork, source: "", count: 2}, + }, + absent: []absentCount{ + {kind: model.KindArtistArtwork, ArtworkAbsentStat: model.ArtworkAbsentStat{Total: 2, Stale: 1}}, + }, + inputs: []artwork.FingerprintInput{{Name: "Agents", Value: "deezer,lastfm"}}, + stored: "abc123", + current: "abc123", + } + }) + + // block isolates one section, so an assertion cannot be satisfied by a coincidence elsewhere. + block := func(out, header string) string { + GinkgoHelper() + _, after, found := strings.Cut(out, header+"\n") + Expect(found).To(BeTrue(), "the %q block must be printed", header) + body, _, _ := strings.Cut(after, "\n\n") + return body + } + + It("names the kind and the priority of every queued row", func() { + queue := block(formatStatus(rep), "Queue") + Expect(queue).To(MatchRegexp(`artist\s+backfill\s+2`)) + Expect(queue).To(MatchRegexp(`album\s+scan\s+1`)) + }) + + It("totals the queue", func() { + Expect(block(formatStatus(rep), "Queue")).To(MatchRegexp(`TOTAL\s+3`)) + }) + + It("counts each source, naming the empty one absent", func() { + sources := block(formatStatus(rep), "Sources") + Expect(sources).To(MatchRegexp(`artist\s+external:deezer\s+5`)) + Expect(sources).To(MatchRegexp(`artist\s+absent\s+2`)) + }) + + It("prints the absent total and how many are due for recheck", func() { + absent := block(formatStatus(rep), "Absent (resolved, no image found)") + Expect(absent).To(MatchRegexp(`artist\s+2\s+1`)) + }) + + It("states the recheck window the absent counts are bucketed against", func() { + Expect(formatStatus(rep)).To(ContainSubstring("24h")) + }) + + It("leads with the queued backlog, which is the finding, not with the fingerprint verdict", func() { + out := block(formatStatus(rep), "Backfill") + Expect(out).To(MatchRegexp(`State:\s+backfill running: 2 items queued`), + "an operator scanning for trouble must not read 'up to date' while 2 items churn") + Expect(out).To(ContainSubstring("fingerprint up to date")) + }) + + It("keeps the re-enqueue warning while a backfill is already running", func() { + rep.stored = "older" + + out := block(formatStatus(rep), "Backfill") + Expect(out).To(MatchRegexp(`State:\s+backfill running: 2 items queued`)) + Expect(out).To(ContainSubstring("re-enqueued"), + "the stored fingerprint is still stale, so a second full re-enqueue is pending on top of this one") + }) + + It("reports up to date only once the backfill has drained", func() { + rep.queue = []model.ArtworkQueueStat{{ItemKind: "al", Priority: model.ArtworkPriorityScan, Count: 1}} + + Expect(block(formatStatus(rep), "Backfill")).To(MatchRegexp(`State:\s+up to date`)) + }) + + It("echoes the config inputs a fingerprint change would have come from", func() { + out := block(formatStatus(rep), "Backfill") + Expect(out).To(MatchRegexp(`Agents:\s+deezer,lastfm`)) + Expect(out).To(ContainSubstring("abc123"), "the fingerprint values themselves must be printed") + }) + + It("reports a changed fingerprint as a pending re-resolve of everything", func() { + rep.stored = "older" + rep.queue = nil + + out := formatStatus(rep) + Expect(out).To(ContainSubstring("fingerprint changed")) + Expect(out).ToNot(ContainSubstring("up to date")) + }) + + It("reports a never-recorded fingerprint without printing an empty value", func() { + rep.stored = "" + + out := formatStatus(rep) + Expect(out).To(ContainSubstring("(none)")) + Expect(out).To(ContainSubstring("fingerprint changed")) + }) + + It("says the queue is empty instead of printing a headless table", func() { + rep.queue = nil + + out := formatStatus(rep) + Expect(out).To(ContainSubstring("empty")) + Expect(out).ToNot(ContainSubstring("PRIORITY")) + }) +}) + +var _ = Describe("refreshItems", func() { + var ds *tests.MockDataStore + var queue *tests.MockArtworkQueueRepo + var art *tests.MockArtworkRepo + var out strings.Builder + ctx := context.Background() + + BeforeEach(func() { + albums := tests.CreateMockAlbumRepo() + albums.SetData(model.Albums{{ID: "al-1"}, {ID: "al-3"}}) + ds = &tests.MockDataStore{MockedAlbum: albums} + art = ds.Artwork(ctx).(*tests.MockArtworkRepo) + queue = ds.ArtworkQueue(ctx).(*tests.MockArtworkQueueRepo) + out.Reset() + }) + + It("clears the stored state and queues each id at Bump priority", func() { + Expect(art.PutItemArtwork(&model.ItemArtwork{ItemKind: model.KindAlbumArtwork.Prefix(), + ItemID: "al-1", ImageType: model.ImageTypePrimary, Hash: "abc123"})).To(Succeed()) + + Expect(refreshItems(ctx, ds, model.KindAlbumArtwork, []string{"al-1", "al-3"}, &out)).To(BeZero()) + + _, err := art.GetItemArtwork(model.KindAlbumArtwork, "al-1", model.ImageTypePrimary) + Expect(err).To(MatchError(model.ErrNotFound)) + queued, err := queue.Get(model.KindAlbumArtwork, "al-1", model.ImageTypePrimary) + Expect(err).ToNot(HaveOccurred()) + Expect(queued.Priority).To(Equal(model.ArtworkPriorityBump)) + Expect(out.String()).To(Equal("al/al-1: queued\nal/al-3: queued\n")) + }) + + It("skips an id that does not exist instead of queuing it", func() { + Expect(refreshItems(ctx, ds, model.KindAlbumArtwork, []string{"al-2"}, &out)).To(Equal(1)) + + _, err := queue.Get(model.KindAlbumArtwork, "al-2", model.ImageTypePrimary) + Expect(err).To(MatchError(model.ErrNotFound), "a typo must not leave an orphan queue row") + Expect(out.String()).To(BeEmpty()) + }) + + It("continues past a failing id and counts the failures", func() { + Expect(refreshItems(ctx, ds, model.KindAlbumArtwork, + []string{"al-1", "al-2", "al-3"}, &out)).To(Equal(1)) + + Expect(out.String()).To(Equal("al/al-1: queued\nal/al-3: queued\n"), + "the ids after a failure are still refreshed") + }) +}) diff --git a/cmd/plugin.go b/cmd/plugin.go index 6cce8ea5f..ded28e969 100644 --- a/cmd/plugin.go +++ b/cmd/plugin.go @@ -9,7 +9,6 @@ import ( "os" "strconv" "strings" - "text/tabwriter" "time" "github.com/navidrome/navidrome/conf" @@ -314,7 +313,7 @@ func formatPluginList(list model.Plugins, format string) (string, error) { return sb.String(), w.Error() case "table": var sb strings.Builder - w := tabwriter.NewWriter(&sb, 0, 4, 2, ' ', 0) + w := newTabWriter(&sb) fmt.Fprintln(w, "ID\tNAME\tVERSION\tENABLED\tLAST ERROR") for _, p := range list { name, version := manifestSummary(p) diff --git a/cmd/utils.go b/cmd/utils.go index 81d646cf1..74da51828 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -4,6 +4,8 @@ import ( "context" "errors" "fmt" + "io" + "text/tabwriter" "github.com/navidrome/navidrome/core/auth" "github.com/navidrome/navidrome/db" @@ -13,6 +15,11 @@ import ( "github.com/navidrome/navidrome/persistence" ) +// newTabWriter keeps every CLI table on the same column settings. +func newTabWriter(out io.Writer) *tabwriter.Writer { + return tabwriter.NewWriter(out, 0, 4, 2, ' ', 0) +} + func getAdminContext(ctx context.Context) (model.DataStore, context.Context) { sqlDB := db.Db() ds := persistence.New(sqlDB) diff --git a/cmd/wire_gen.go b/cmd/wire_gen.go index 4bdd48422..49a99f8ca 100644 --- a/cmd/wire_gen.go +++ b/cmd/wire_gen.go @@ -223,6 +223,18 @@ func CreateArtworkWorker() *artwork.Worker { return worker } +func CreateArtworkResolver(trace *artwork.ChainTrace, live bool) *artwork.TracingResolver { + sqlDB := db.Db() + dataStore := persistence.New(sqlDB) + broker := events.GetBroker() + metricsMetrics := metrics.GetPrometheusInstance(dataStore) + manager := plugins.GetManager(dataStore, broker, metricsMetrics) + agentsAgents := agents.GetAgents(dataStore, manager) + fFmpeg := ffmpeg.New() + tracingResolver := artwork.NewTracingResolver(dataStore, agentsAgents, fFmpeg, trace, live) + return tracingResolver +} + func getPluginManager() *plugins.Manager { sqlDB := db.Db() dataStore := persistence.New(sqlDB) diff --git a/cmd/wire_injectors.go b/cmd/wire_injectors.go index ae24b61fa..8bc404fb1 100644 --- a/cmd/wire_injectors.go +++ b/cmd/wire_injectors.go @@ -144,6 +144,13 @@ func CreateArtworkWorker() *artwork.Worker { )) } +func CreateArtworkResolver(trace *artwork.ChainTrace, live bool) *artwork.TracingResolver { + panic(wire.Build( + allProviders, + artwork.NewTracingResolver, + )) +} + func getPluginManager() *plugins.Manager { panic(wire.Build( allProviders, diff --git a/core/artwork/agent_images.go b/core/artwork/agent_images.go index c7032b62b..a6f746959 100644 --- a/core/artwork/agent_images.go +++ b/core/artwork/agent_images.go @@ -47,10 +47,17 @@ func fetchArtistImage(ctx context.Context, ag *agents.Agents, gate gateFunc, ar // Synthetic artists would otherwise get an unrelated agent result assigned to them. switch ar.ID { case consts.UnknownArtistID, consts.VariousArtistsID: + traceFrom(ctx).add(TraceStep{Candidate: externalCandidate, Outcome: OutcomeSkipped, Detail: "synthetic artist"}) return nil, "", false } name := externalName(ar.Name) - for _, a := range ag.ArtistImageAgents() { + imageAgents := ag.ArtistImageAgents() + if len(imageAgents) == 0 { + traceFrom(ctx).add(TraceStep{Candidate: externalCandidate, Outcome: OutcomeSkipped, + Detail: "no enabled agent provides artist images"}) + return nil, "", false + } + for _, a := range imageAgents { reader, _, err := gate(a.Name, func() (io.ReadCloser, string, error) { imgs, err := a.Retriever.GetArtistImages(ctx, ar.ID, name, ar.MbzArtistID) if err != nil { @@ -76,7 +83,13 @@ func fetchArtistImage(ctx context.Context, ag *agents.Agents, gate gateFunc, ar // fetchAlbumImage is the album counterpart of fetchArtistImage. func fetchAlbumImage(ctx context.Context, ag *agents.Agents, gate gateFunc, al model.Album) (r io.ReadCloser, agentName string, extErr bool) { name, artist := externalName(al.Name), externalName(al.AlbumArtist) - for _, a := range ag.AlbumImageAgents() { + imageAgents := ag.AlbumImageAgents() + if len(imageAgents) == 0 { + traceFrom(ctx).add(TraceStep{Candidate: externalCandidate, Outcome: OutcomeSkipped, + Detail: "no enabled agent provides album images"}) + return nil, "", false + } + for _, a := range imageAgents { reader, _, err := gate(a.Name, func() (io.ReadCloser, string, error) { imgs, err := a.Retriever.GetAlbumImages(ctx, name, artist, al.MbzAlbumID) if err != nil { diff --git a/core/artwork/agent_images_test.go b/core/artwork/agent_images_test.go index 716245c93..60a34352d 100644 --- a/core/artwork/agent_images_test.go +++ b/core/artwork/agent_images_test.go @@ -173,6 +173,30 @@ var _ = Describe("agent images", func() { Expect(a.artistCalls).To(Equal(0), "synthetic artists never reach the agents") }) + It("records a skipped external candidate when no agent provides artist images", func() { + ag := imageAgents() + t := &ChainTrace{} + + r, _, extErr := fetchArtistImage(withTrace(ctx, t), ag, passthroughGate, model.Artist{ID: "ar1"}) + Expect(r).To(BeNil()) + Expect(extErr).To(BeFalse()) + Expect(t.Steps()).To(Equal([]TraceStep{{Candidate: "external", Outcome: OutcomeSkipped, + Detail: "no enabled agent provides artist images"}}), + "a configured external token must never be silently absent from the chain") + }) + + It("records a skipped external candidate for synthetic artists", func() { + a := &fakeImageAgent{name: "agentA", imgs: []agents.ExternalImage{img("/a", 100)}} + ag := imageAgents(a) + t := &ChainTrace{} + + _, _, _ = fetchArtistImage(withTrace(ctx, t), ag, passthroughGate, + model.Artist{ID: consts.VariousArtistsID, Name: "Various Artists"}) + Expect(t.Steps()).To(HaveLen(1)) + Expect(t.Steps()[0].Outcome).To(Equal(OutcomeSkipped)) + Expect(t.Steps()[0].Detail).To(ContainSubstring("synthetic")) + }) + It("clears typographic characters from the query name unless preserving unicode", func() { conf.Server.DevPreserveUnicodeInExternalCalls = false a := &fakeImageAgent{name: "agentA"} @@ -231,6 +255,18 @@ var _ = Describe("agent images", func() { Expect(a.albumCalls).To(Equal(1)) }) + It("records a skipped external candidate when no agent provides album images", func() { + ag := imageAgents() + t := &ChainTrace{} + + r, _, extErr := fetchAlbumImage(withTrace(ctx, t), ag, passthroughGate, model.Album{Name: "Album"}) + Expect(r).To(BeNil()) + Expect(extErr).To(BeFalse()) + Expect(t.Steps()).To(Equal([]TraceStep{{Candidate: "external", Outcome: OutcomeSkipped, + Detail: "no enabled agent provides album images"}}), + "a configured external token must never be silently absent from the chain") + }) + It("reports extErr when the only agent fails transiently", func() { a := &fakeImageAgent{name: "agentA", err: context.DeadlineExceeded} ag := imageAgents(a) diff --git a/core/artwork/artwork.go b/core/artwork/artwork.go index 8ac8d9c9d..7edc80e99 100644 --- a/core/artwork/artwork.go +++ b/core/artwork/artwork.go @@ -11,6 +11,7 @@ import ( "github.com/navidrome/navidrome/conf" "github.com/navidrome/navidrome/consts" + "github.com/navidrome/navidrome/core/agents" "github.com/navidrome/navidrome/core/ffmpeg" "github.com/navidrome/navidrome/log" "github.com/navidrome/navidrome/model" @@ -118,7 +119,7 @@ func (s *service) Get(ctx context.Context, artID model.ArtworkID, size int, squa } // requestRecheckAge throttles view-triggered rechecks so reopening a genuinely-absent page can't -// hammer external services; below staleAbsentAge to catch younger absences. +// hammer external services; below StaleAbsentAge to catch younger absences. const requestRecheckAge = time.Hour func (s *service) serveEntity(ctx context.Context, artID model.ArtworkID, size int, square bool) (*Image, error) { @@ -317,16 +318,15 @@ func (s *service) serveDisc(ctx context.Context, artID model.ArtworkID, size int return nil, err } // Single-disc albums run the chain too: a disc can carry art distinct from the album cover. - selectImage := func() (io.ReadCloser, string, error) { - funcs := dr.fromDiscArtPriority(ctx, s.ffmpeg, conf.Server.DiscArtPriority) - return selectImageReader(ctx, artID, funcs...) + selectImage := func() (io.ReadCloser, error) { + res, err := dr.selectImage(ctx, s.ffmpeg, conf.Server.DiscArtPriority, &chainState{}) + return res.reader, err } albumArtID := model.ArtworkID{Kind: model.KindAlbumArtwork, ID: dr.album.ID} // Disc art has no state row, hence no content hash: keying on id, album mtime and // DiscArtPriority lets a warm cache answer without running the chain or touching the disk. key := fmt.Sprintf("%s|%d|%s", artID.ID, dr.cacheTime().UnixNano(), conf.Server.DiscArtPriority) - img, err := s.serveSource(ctx, key, "", dr.cacheTime(), size, square, - func() (io.ReadCloser, error) { rc, _, err := selectImage(); return rc, err }) + img, err := s.serveSource(ctx, key, "", dr.cacheTime(), size, square, selectImage) if err != nil { if errors.Is(err, context.Canceled) { return nil, err @@ -386,6 +386,54 @@ func (s *service) parseArtworkID(ctx context.Context, id string) (model.ArtworkI return model.ArtworkID{}, model.ErrNotFound } +// TracingResolver is the CLI's read-only view of resolution: it walks the priority chain, records +// the walk and reports the winning source, without ever writing artwork state. +type TracingResolver struct { + inner *resolver + trace *ChainTrace +} + +// NewTracingResolver builds a TracingResolver that records its priority-chain walk. With live +// false the external tier is reported but never called. +func NewTracingResolver(ds model.DataStore, ag *agents.Agents, ffm ffmpeg.FFmpeg, t *ChainTrace, live bool) *TracingResolver { + gate := offlineGate(t) + if live { + // A diagnostic must show the provider's real answer, and one item is at most one call + // per agent, so --live deliberately bypasses the rate limiter and circuit breaker. + gate = tracingGate(t, passthroughGate) + } + return &TracingResolver{inner: newResolver(ds, ag, ffm, gate), trace: t} +} + +// Resolve walks kind's sources for id, recording the walk, and reports the winning source +// ("" when none produced an image). +func (r *TracingResolver) Resolve(ctx context.Context, kind model.Kind, id string) (string, error) { + switch kind { + case model.KindArtistArtwork: + return r.explain(ctx, r.inner.resolveArtist, id) + case model.KindAlbumArtwork: + return r.explain(ctx, r.inner.resolveAlbum, id) + case model.KindDiscArtwork: + return r.explain(ctx, r.inner.resolveDisc, id) + case model.KindMediaFileArtwork: + return r.explain(ctx, r.inner.resolveMediaFile, id) + } + return "", fmt.Errorf("artwork: %s artwork has no chain to explain", kind) +} + +// explain discards the bytes: nothing downstream persists this resolution, so nothing else +// would close the reader either. +func (r *TracingResolver) explain(ctx context.Context, resolve func(context.Context, string) (resolution, error), id string) (string, error) { + res, err := resolve(withTrace(ctx, r.trace), id) + if err != nil { + return "", err + } + if res.reader != nil { + _ = res.reader.Close() + } + return res.source, nil +} + func unixMtime(mtime int64) time.Time { if mtime <= 0 { return time.Time{} diff --git a/core/artwork/disc.go b/core/artwork/disc.go index 21f596b60..acd8a3740 100644 --- a/core/artwork/disc.go +++ b/core/artwork/disc.go @@ -113,27 +113,71 @@ func newDiscArtworkReader(ctx context.Context, ds model.DataStore, artID model.A }, nil } -func (d *discArtworkReader) fromDiscArtPriority(ctx context.Context, ffmpeg ffmpeg.FFmpeg, priority string) []sourceFunc { - var ff []sourceFunc +// discCandidate is one DiscArtPriority entry. skip is set when the entry maps to no source at +// all, so a chain walk can say why instead of leaving a configured entry unaccounted for. +type discCandidate struct { + pattern string + resolve func() (resolution, bool) + skip string +} + +func (d *discArtworkReader) discCandidates(ctx context.Context, ffmpeg ffmpeg.FFmpeg, priority string) []discCandidate { + folder := func(sf sourceFunc) func() (resolution, bool) { + return func() (resolution, bool) { return resolveFolderSource(d.lib, sf) } + } + var cc []discCandidate for pattern := range strings.SplitSeq(strings.ToLower(priority), ",") { pattern = strings.TrimSpace(pattern) + if pattern == "" { + continue + } + c := discCandidate{pattern: pattern} switch { case pattern == "embedded": - ff = append(ff, - fromTag(ctx, d.lib.FS, d.firstTrackRel), - fromFFmpegTag(ctx, ffmpeg, d.lib.Abs(d.firstTrackRel)), - ) - case pattern == "external": - // Not supported for disc art, silently ignore - case pattern == "discsubtitle": - if subtitle := strings.TrimSpace(d.album.Discs[d.discNumber]); subtitle != "" { - ff = append(ff, d.fromDiscSubtitle(ctx, subtitle)) + c.resolve = func() (resolution, bool) { + return resolveEmbedded(ctx, d.lib, ffmpeg, d.firstTrackRel) } - case len(d.imgFiles) > 0: - ff = append(ff, d.fromExternalFile(ctx, pattern)) + case pattern == externalCandidate: + c.skip = "external sources are not supported for disc artwork" + case pattern == "discsubtitle": + subtitle := strings.TrimSpace(d.album.Discs[d.discNumber]) + if subtitle == "" { + c.skip = "disc has no subtitle" + } else { + c.resolve = folder(d.fromDiscSubtitle(ctx, subtitle)) + } + case len(d.imgFiles) == 0: + c.skip = "no images in album folder" + default: + c.resolve = folder(d.fromExternalFile(ctx, pattern)) + } + cc = append(cc, c) + } + return cc +} + +// selectImage walks the DiscArtPriority entries and returns the first that yields an image. +// chain records the walk; the serving path passes an untraced one and pays nothing for it. +func (d *discArtworkReader) selectImage(ctx context.Context, ffmpeg ffmpeg.FFmpeg, priority string, + chain *chainState) (resolution, error) { + for _, c := range d.discCandidates(ctx, ffmpeg, priority) { + if err := ctx.Err(); err != nil { + return resolution{}, err + } + if c.skip != "" { + chain.record(c.pattern, OutcomeSkipped, c.skip) + continue + } + start := time.Now() + res, ok := c.resolve() + log.Trace(ctx, "Artwork: Tried a disc artwork candidate", "albumID", d.album.ID, + "disc", d.discNumber, "pattern", c.pattern, "hit", ok, "path", res.sourcePath, + "elapsed", time.Since(start)) + if res, ok = chain.try(c.pattern, res, ok); ok { + return res, nil } } - return ff + return chain.exhausted(), nil } // fromDiscSubtitle returns a sourceFunc that matches image files whose stem diff --git a/core/artwork/disc_test.go b/core/artwork/disc_test.go index 8264ee27b..8852741cf 100644 --- a/core/artwork/disc_test.go +++ b/core/artwork/disc_test.go @@ -6,6 +6,8 @@ import ( "path/filepath" "github.com/navidrome/navidrome/model" + "github.com/navidrome/navidrome/tests" + "github.com/navidrome/navidrome/utils/slice" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) @@ -179,19 +181,19 @@ var _ = Describe("Disc Artwork Reader", func() { lib: libraryView{FS: osDirFS{os.DirFS(tmpDir)}, absRoot: tmpDir}, } - ff := reader.fromDiscArtPriority(ctx, nil, "disc*.*, cover.*") - Expect(ff).To(HaveLen(2)) - r, path, err := ff[0]() - Expect(err).ToNot(HaveOccurred()) - Expect(path).To(Equal(f2)) - r.Close() + cc := reader.discCandidates(ctx, nil, "disc*.*, cover.*") + Expect(cc).To(HaveLen(2)) + res, ok := cc[0].resolve() + Expect(ok).To(BeTrue()) + Expect(res.sourcePath).To(Equal(reader.lib.Abs(f2))) + res.reader.Close() - ff = reader.fromDiscArtPriority(ctx, nil, "cover.*, disc*.*") - Expect(ff).To(HaveLen(2)) - r, path, err = ff[0]() - Expect(err).ToNot(HaveOccurred()) - Expect(path).To(Equal(f1)) - r.Close() + cc = reader.discCandidates(ctx, nil, "cover.*, disc*.*") + Expect(cc).To(HaveLen(2)) + res, ok = cc[0].resolve() + Expect(ok).To(BeTrue()) + Expect(res.sourcePath).To(Equal(reader.lib.Abs(f1))) + res.reader.Close() }) DescribeTable("numbered match wins over shared fallback within a pattern", @@ -428,64 +430,109 @@ var _ = Describe("Disc Artwork Reader", func() { }) Describe("discArtworkReader", func() { - Describe("fromDiscArtPriority", func() { - var ( - reader *discArtworkReader - tmpDir string + var ( + reader *discArtworkReader + tmpDir string + ) + + BeforeEach(func() { + tmpDir = GinkgoT().TempDir() + reader = &discArtworkReader{ + discNumber: 2, + isMultiFolder: true, + discFoldersRel: map[string]bool{"music/album/cd2": true}, + imgFiles: []string{ + "music/album/cd1/disc.jpg", + "music/album/cd2/disc.jpg", + "music/album/cd2/disc2.jpg", + }, + firstTrackRel: "music/album/cd2/track1.flac", + lib: libraryView{FS: osDirFS{os.DirFS(tmpDir)}, absRoot: tmpDir}, + } + }) + + Describe("selectImage", func() { + It("abandons the walk when the context is cancelled", func() { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + res, err := reader.selectImage(ctx, nil, "disc*.*, cover.*", &chainState{}) + + Expect(err).To(MatchError(context.Canceled)) + Expect(res.reader).To(BeNil()) + }) + + // "the track has no embedded art" and "the track is there but unreadable" are the two + // answers a wrong-artwork report needs told apart; only the second is worth retrying. + It("reports a track it cannot parse as unreadable, not as a miss", func() { + trace := &ChainTrace{} + track := filepath.Join(tmpDir, filepath.FromSlash(reader.firstTrackRel)) + Expect(os.MkdirAll(filepath.Dir(track), 0755)).To(Succeed()) + Expect(os.WriteFile(track, []byte("not audio"), 0600)).To(Succeed()) + + res, err := reader.selectImage(context.Background(), tests.NewMockFFmpeg(""), "embedded", + &chainState{trace: trace}) + + Expect(err).ToNot(HaveOccurred()) + Expect(res.localError).To(BeTrue()) + Expect(trace.Steps()).To(Equal([]TraceStep{{Candidate: "embedded", Outcome: OutcomeUnreadable}})) + }) + + It("reports a disc with no tracks to read as a miss", func() { + trace := &ChainTrace{} + reader.firstTrackRel = "" + + res, err := reader.selectImage(context.Background(), tests.NewMockFFmpeg(""), "embedded", + &chainState{trace: trace}) + + Expect(err).ToNot(HaveOccurred()) + Expect(res.localError).To(BeFalse(), "there was nothing to read, so nothing failed to read") + Expect(trace.Steps()).To(Equal([]TraceStep{{Candidate: "embedded", Outcome: OutcomeMiss}})) + }) + }) + + Describe("discCandidates", func() { + It("returns a resolvable candidate for glob patterns", func() { + cc := reader.discCandidates(context.Background(), nil, "disc*.*") + Expect(cc).To(HaveLen(1)) + Expect(cc[0].resolve).ToNot(BeNil()) + }) + + It("returns one candidate per entry, in order", func() { + cc := reader.discCandidates(context.Background(), nil, "disc*.*, cd*.*, embedded") + Expect(slice.Map(cc, func(c discCandidate) string { return c.pattern })). + To(Equal([]string{"disc*.*", "cd*.*", "embedded"})) + }) + + It("skips an empty entry rather than building a glob that matches nothing", func() { + cc := reader.discCandidates(context.Background(), nil, "disc*.*,") + Expect(cc).To(HaveLen(1)) + }) + + // The skip reasons below are what `artwork explain` prints, so an entry that maps to no + // source must say why instead of vanishing from the walk. + DescribeTable("keeps an entry that maps to no source, with its reason", + func(setup func(), priority, reason string) { + setup() + cc := reader.discCandidates(context.Background(), nil, priority) + Expect(cc).To(HaveLen(1)) + Expect(cc[0].resolve).To(BeNil()) + Expect(cc[0].skip).To(Equal(reason)) + }, + Entry("external is unsupported", func() {}, "external", + "external sources are not supported for disc artwork"), + Entry("no images in the album folder", func() { reader.imgFiles = nil }, "disc*.*", + "no images in album folder"), + Entry("the disc has no subtitle", + func() { reader.album = model.Album{Discs: model.Discs{2: ""}} }, "discsubtitle", + "disc has no subtitle"), ) - BeforeEach(func() { - tmpDir = GinkgoT().TempDir() - reader = &discArtworkReader{ - discNumber: 2, - isMultiFolder: true, - discFoldersRel: map[string]bool{"music/album/cd2": true}, - imgFiles: []string{ - "music/album/cd1/disc.jpg", - "music/album/cd2/disc.jpg", - "music/album/cd2/disc2.jpg", - }, - firstTrackRel: "music/album/cd2/track1.flac", - lib: libraryView{FS: osDirFS{os.DirFS(tmpDir)}, absRoot: tmpDir}, - } - }) - - It("returns source funcs for glob patterns", func() { - ff := reader.fromDiscArtPriority(context.Background(), nil, "disc*.*") - Expect(ff).To(HaveLen(1)) - }) - - It("returns source funcs for embedded pattern", func() { - ff := reader.fromDiscArtPriority(context.Background(), nil, "embedded") - Expect(ff).To(HaveLen(2)) // fromTag + fromFFmpegTag - }) - - It("handles multiple comma-separated patterns", func() { - ff := reader.fromDiscArtPriority(context.Background(), nil, "disc*.*, cd*.*, embedded") - Expect(ff).To(HaveLen(4)) // disc*.* + cd*.* + fromTag + fromFFmpegTag - }) - - It("ignores 'external' pattern silently", func() { - ff := reader.fromDiscArtPriority(context.Background(), nil, "external") - Expect(ff).To(HaveLen(0)) - }) - - It("returns no source funcs when imgFiles is empty and pattern is not embedded", func() { - reader.imgFiles = nil - ff := reader.fromDiscArtPriority(context.Background(), nil, "disc*.*") - Expect(ff).To(HaveLen(0)) - }) - It("returns source func for discsubtitle pattern", func() { reader.album = model.Album{Discs: model.Discs{2: "Bonus Tracks"}} - ff := reader.fromDiscArtPriority(context.Background(), nil, "discsubtitle") - Expect(ff).To(HaveLen(1)) - }) - - It("returns no source func for discsubtitle when disc has no subtitle", func() { - reader.album = model.Album{Discs: model.Discs{2: ""}} - ff := reader.fromDiscArtPriority(context.Background(), nil, "discsubtitle") - Expect(ff).To(HaveLen(0)) + cc := reader.discCandidates(context.Background(), nil, "discsubtitle") + Expect(cc).To(HaveLen(1)) + Expect(cc[0].resolve).ToNot(BeNil()) }) }) }) diff --git a/core/artwork/housekeeping.go b/core/artwork/housekeeping.go index ce72a6c8e..ae1fc0a0d 100644 --- a/core/artwork/housekeeping.go +++ b/core/artwork/housekeeping.go @@ -6,6 +6,8 @@ import ( "encoding/hex" "fmt" "slices" + "strconv" + "strings" "time" "github.com/navidrome/navidrome/conf" @@ -16,27 +18,54 @@ import ( "github.com/navidrome/navidrome/utils/slice" ) -const staleAbsentAge = 24 * time.Hour +// StaleAbsentAge is how long an absent state is trusted before a recheck retries it. +const StaleAbsentAge = 24 * time.Hour -// recheckKinds omits media files: they resolve embedded-only, at scan or on view. -var recheckKinds = []model.Kind{ +// RecheckKinds omits media files: they resolve embedded-only, at scan or on view. +var RecheckKinds = []model.Kind{ model.KindArtistArtwork, model.KindAlbumArtwork, model.KindPlaylistArtwork, model.KindRadioArtwork, } +// KeepsState reports whether a kind is recorded in item_artwork and the artwork queue. Disc +// artwork is read through on every request and cached by content key, so it has neither. +func KeepsState(kind model.Kind) bool { return kind != model.KindDiscArtwork } + +// RefreshableKinds is every kind Refresh can clear and re-queue, so it holds exactly the kinds +// KeepsState admits. Media files are absent from RecheckKinds but belong here: the worker +// resolves them, it just never revisits them on its own. +var RefreshableKinds = append(slices.Clone(RecheckKinds), model.KindMediaFileArtwork) + // hasRecheckPath reports whether a periodic job will revisit this kind, making an absent settle recoverable. func hasRecheckPath(prefix string) bool { kind, ok := model.ParseKind(prefix) - return ok && slices.Contains(recheckKinds, kind) + return ok && slices.Contains(RecheckKinds, kind) } // artworkEpoch invalidates all resolution state when bumped; bump it whenever resolution semantics change. const artworkEpoch = 1 -// fingerprint covers the inputs that affect resolution outcomes; a change invalidates stored state. -func fingerprint() string { - raw := fmt.Sprintf("%s|%s|%s|%s|%t|%t|%d", - conf.Server.CoverArtPriority, conf.Server.ArtistArtPriority, conf.Server.ArtistImageFolder, - conf.Server.Agents, conf.Server.EnableExternalServices, conf.Server.EnableM3UExternalAlbumArt, artworkEpoch) +// FingerprintInput is one config value the fingerprint covers, named after the setting it came from. +type FingerprintInput struct { + Name string + Value string +} + +// FingerprintInputs is the single listing of what ConfigFingerprint hashes. +func FingerprintInputs() []FingerprintInput { + return []FingerprintInput{ + {"CoverArtPriority", conf.Server.CoverArtPriority}, + {"ArtistArtPriority", conf.Server.ArtistArtPriority}, + {"ArtistImageFolder", conf.Server.ArtistImageFolder}, + {"Agents", conf.Server.Agents}, + {"EnableExternalServices", strconv.FormatBool(conf.Server.EnableExternalServices)}, + {"EnableM3UExternalAlbumArt", strconv.FormatBool(conf.Server.EnableM3UExternalAlbumArt)}, + } +} + +// ConfigFingerprint covers the inputs that affect resolution outcomes; a change invalidates stored state. +func ConfigFingerprint() string { + values := slice.Map(FingerprintInputs(), func(i FingerprintInput) string { return i.Value }) + raw := fmt.Sprintf("%s|%d", strings.Join(values, "|"), artworkEpoch) sum := md5.Sum([]byte(raw)) //nolint:gosec // fingerprint, not security-sensitive return hex.EncodeToString(sum[:]) } @@ -45,7 +74,7 @@ func fingerprint() string { func backfill(ctx context.Context, ds model.DataStore) (bool, error) { start := time.Now() ctx = auth.WithAdminUser(ctx, ds) - current := fingerprint() + current := ConfigFingerprint() props := ds.Property(ctx) stored, err := props.DefaultGet(consts.ArtConfFingerprintPropertyKey, "") if err != nil { @@ -95,9 +124,9 @@ func enqueueBackfillKind(ctx context.Context, ds model.DataStore, kind model.Kin } func enqueueStaleAbsentAll(ctx context.Context, ds model.DataStore) error { - cutoff := time.Now().Add(-staleAbsentAge) + cutoff := time.Now().Add(-StaleAbsentAge) queue := ds.ArtworkQueue(ctx) - for _, kind := range recheckKinds { + for _, kind := range RecheckKinds { if _, err := queue.EnqueueStaleAbsent(kind, cutoff); err != nil { return err } @@ -108,7 +137,7 @@ func enqueueStaleAbsentAll(ctx context.Context, ds model.DataStore) error { // enqueueMissingAll is the safety net for entities a scan never enqueued (added between scans, or scanner off). func enqueueMissingAll(ctx context.Context, ds model.DataStore) error { queue := ds.ArtworkQueue(ctx) - for _, kind := range recheckKinds { + for _, kind := range RecheckKinds { if _, err := queue.EnqueueAllMissing(kind, model.ArtworkPriorityRecheck); err != nil { return err } diff --git a/core/artwork/housekeeping_test.go b/core/artwork/housekeeping_test.go index 3b75c5186..32a7688b4 100644 --- a/core/artwork/housekeeping_test.go +++ b/core/artwork/housekeeping_test.go @@ -52,6 +52,19 @@ func (o *orderTrackingQueueRepo) Enqueue(items ...model.ArtworkQueueItem) error return o.MockArtworkQueueRepo.Enqueue(items...) } +var _ = Describe("RefreshableKinds", func() { + // The two are meant to describe the same fact. Nothing but this test stops them from drifting, + // and a drift would have `artwork explain` report state for a kind that keeps none. + It("holds exactly the kinds that keep state", func() { + for _, k := range []model.Kind{ + model.KindArtistArtwork, model.KindAlbumArtwork, model.KindPlaylistArtwork, + model.KindRadioArtwork, model.KindMediaFileArtwork, model.KindDiscArtwork, + } { + Expect(slices.Contains(RefreshableKinds, k)).To(Equal(KeepsState(k)), k.String()) + } + }) +}) + var _ = Describe("Housekeeping", func() { var ( ctx context.Context @@ -93,32 +106,54 @@ var _ = Describe("Housekeeping", func() { Describe("Fingerprint", func() { It("changes when a fingerprint-affecting config value changes", func() { - f1 := fingerprint() + f1 := ConfigFingerprint() conf.Server.CoverArtPriority = "folder, embedded" - f2 := fingerprint() + f2 := ConfigFingerprint() Expect(f1).NotTo(Equal(f2)) }) It("changes when ArtistImageFolder changes", func() { conf.Server.ArtistImageFolder = "/before" - f1 := fingerprint() + f1 := ConfigFingerprint() conf.Server.ArtistImageFolder = "/after" - Expect(fingerprint()).NotTo(Equal(f1)) + Expect(ConfigFingerprint()).NotTo(Equal(f1)) }) It("changes when EnableM3UExternalAlbumArt is toggled", func() { conf.Server.EnableM3UExternalAlbumArt = false - f1 := fingerprint() + f1 := ConfigFingerprint() conf.Server.EnableM3UExternalAlbumArt = true - Expect(fingerprint()).NotTo(Equal(f1)) + Expect(ConfigFingerprint()).NotTo(Equal(f1)) + }) + + // Pinned: a changed formula re-resolves every library on upgrade, flooding external providers. + It("hashes a given config to a stable value", func() { + conf.Server.CoverArtPriority = "cover.*, embedded" + conf.Server.ArtistArtPriority = "artist.*, external" + conf.Server.ArtistImageFolder = "" + conf.Server.Agents = "lastfm,spotify" + conf.Server.EnableExternalServices = true + conf.Server.EnableM3UExternalAlbumArt = false + + Expect(ConfigFingerprint()).To(Equal("7e537a22febc07d3d5ca40546e88da54")) + }) + + It("reports the config inputs it hashes, so a change can be traced to a setting", func() { + conf.Server.Agents = "lastfm,spotify" + conf.Server.CoverArtPriority = "cover.*, embedded" + + Expect(FingerprintInputs()).To(ContainElements( + FingerprintInput{Name: "Agents", Value: "lastfm,spotify"}, + FingerprintInput{Name: "CoverArtPriority", Value: "cover.*, embedded"}, + )) }) It("does not change when the server version changes", func() { original := consts.Version DeferCleanup(func() { consts.Version = original }) - f1 := fingerprint() + f1 := ConfigFingerprint() consts.Version = original + "-next" - Expect(fingerprint()).To(Equal(f1), + Expect(ConfigFingerprint()).To(Equal(f1), "the version must not invalidate artwork state: it would re-resolve every entity on every build") }) }) @@ -126,7 +161,7 @@ var _ = Describe("Housekeeping", func() { Describe("Backfill", func() { It("enqueues nothing and returns false when the stored fingerprint matches", func() { seedEntities() - Expect(propRepo.Put(consts.ArtConfFingerprintPropertyKey, fingerprint())).To(Succeed()) + Expect(propRepo.Put(consts.ArtConfFingerprintPropertyKey, ConfigFingerprint())).To(Succeed()) did, err := backfill(ctx, ds) Expect(err).ToNot(HaveOccurred()) @@ -150,7 +185,7 @@ var _ = Describe("Housekeeping", func() { stored, err := propRepo.Get(consts.ArtConfFingerprintPropertyKey) Expect(err).ToNot(HaveOccurred()) - Expect(stored).To(Equal(fingerprint())) + Expect(stored).To(Equal(ConfigFingerprint())) }) It("enqueues a private playlist by resolving it under an admin context", func() { diff --git a/core/artwork/image_cache.go b/core/artwork/image_cache.go index 39938a755..b1970d21d 100644 --- a/core/artwork/image_cache.go +++ b/core/artwork/image_cache.go @@ -55,6 +55,10 @@ func (r *resizedItem) Reader(ctx context.Context) (io.ReadCloser, error) { if err != nil { return nil, err } + // An open() that reports "no image" as a nil reader would otherwise panic on the Close below. + if orig == nil { + return nil, ErrUnavailable + } defer orig.Close() data, err := readCapped(orig) if err != nil { diff --git a/core/artwork/image_cache_test.go b/core/artwork/image_cache_test.go new file mode 100644 index 000000000..a74c51088 --- /dev/null +++ b/core/artwork/image_cache_test.go @@ -0,0 +1,41 @@ +package artwork + +import ( + "context" + "errors" + "io" + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("resizedItem", func() { + Describe("Reader", func() { + newItem := func(open func() (io.ReadCloser, error)) *resizedItem { + return &resizedItem{hash: "abc123", size: 300, open: open} + } + + It("reports a nil reader as unavailable instead of panicking on it", func() { + // Every caller is expected to report "no image" as an error, but a nil reader reaches + // the deferred Close as a nil interface, which takes the whole request down. + _, err := newItem(func() (io.ReadCloser, error) { return nil, nil }).Reader(context.Background()) + Expect(err).To(MatchError(ErrUnavailable)) + }) + + It("propagates the open error", func() { + boom := errors.New("boom") + _, err := newItem(func() (io.ReadCloser, error) { return nil, boom }).Reader(context.Background()) + Expect(err).To(MatchError(boom)) + }) + + It("serves the original bytes when they cannot be resized", func() { + rc, err := newItem(func() (io.ReadCloser, error) { + return io.NopCloser(strings.NewReader("not an image")), nil + }).Reader(context.Background()) + Expect(err).ToNot(HaveOccurred()) + defer rc.Close() + Expect(io.ReadAll(rc)).To(Equal([]byte("not an image"))) + }) + }) +}) diff --git a/core/artwork/resolve.go b/core/artwork/resolve.go index 11711469d..518b6e100 100644 --- a/core/artwork/resolve.go +++ b/core/artwork/resolve.go @@ -34,18 +34,31 @@ type resolution struct { // chainState carries what a priority walk has seen so far. A hit takes extErr with it so a // transient external failure still retries; localErr is dropped, as the scanner re-lists changes. -type chainState struct{ extErr, localErr bool } +type chainState struct { + extErr, localErr bool + trace *ChainTrace // nil unless the CLI asked for a trace +} // try stamps the accumulated external failure onto a hit, and records the miss otherwise. -func (c *chainState) try(res resolution, ok bool) (resolution, bool) { +func (c *chainState) try(candidate string, res resolution, ok bool) (resolution, bool) { if ok { res.extError = c.extErr + c.record(candidate, OutcomeHit, res.sourcePath) return res, true } c.localErr = c.localErr || res.localError + if res.localError { + c.record(candidate, OutcomeUnreadable, "") + } else { + c.record(candidate, OutcomeMiss, "") + } return resolution{}, false } +func (c *chainState) record(candidate string, out Outcome, detail string) { + c.trace.add(TraceStep{Candidate: candidate, Outcome: out, Detail: detail}) +} + // exhausted is the outcome when no source in the chain yielded an image. func (c *chainState) exhausted() resolution { return resolution{extError: c.extErr, localError: c.localErr} @@ -95,8 +108,69 @@ func (r *resolver) resolve(ctx context.Context, item model.ArtworkQueueItem) (re } } -// fetchExternalAlbum and fetchExternalArtist are the only places resolution touches the network, -// so a local-only resolver is stopped here rather than at each point in the chain walk. +// Explainable reports whether TracingResolver can walk this kind's sources and report which one +// won; playlists and radios resolve from a fixed internal order, with nothing configured to explain. +func Explainable(kind model.Kind) bool { + switch kind { + case model.KindArtistArtwork, model.KindAlbumArtwork, model.KindDiscArtwork, model.KindMediaFileArtwork: + return true + } + return false +} + +// MayFetchExternal reports whether resolving this kind can issue an external request under the +// current config. Playlists inherit the album chain: the generated grid resolves album art. +func MayFetchExternal(kind model.Kind) bool { + switch kind { + case model.KindArtistArtwork: + return chainFetchesExternal(conf.Server.ArtistArtPriority) + case model.KindAlbumArtwork: + return chainFetchesExternal(conf.Server.CoverArtPriority) + case model.KindPlaylistArtwork: + return conf.Server.EnableM3UExternalAlbumArt || chainFetchesExternal(conf.Server.CoverArtPriority) + default: + return false + } +} + +// ImageAgentCount is how many enabled agents provide artist and album images. +type ImageAgentCount struct{ Artist, Album int } + +// ExternalLookupsPerItem reports what resolving one item of this kind can cost: every image agent is +// tried, and a zero count still bills one, so agents the caller cannot see never read as free. +func ExternalLookupsPerItem(kind model.Kind, agents ImageAgentCount) int64 { + if !MayFetchExternal(kind) { + return 0 + } + switch kind { + case model.KindArtistArtwork: + return int64(max(agents.Artist, 1)) + case model.KindAlbumArtwork: + return int64(max(agents.Album, 1)) + case model.KindPlaylistArtwork: + var n int64 + if conf.Server.EnableM3UExternalAlbumArt { + n++ + } + if chainFetchesExternal(conf.Server.CoverArtPriority) { + n += PlaylistGridSamples * int64(max(agents.Album, 1)) + } + return n + } + return 0 +} + +func chainFetchesExternal(priority string) bool { + for pattern := range strings.SplitSeq(strings.ToLower(priority), ",") { + if strings.TrimSpace(pattern) == externalCandidate { + return true + } + } + return false +} + +// Album and artist fetches stop here when the resolver is local-only, rather than at each point in +// the chain walk; resolvePlaylist gates the third network path, the m3u image URL, itself. func (r *resolver) fetchExternalAlbum(ctx context.Context, al model.Album) (io.ReadCloser, string, bool) { if r.ext == nil { return nil, "", false @@ -126,24 +200,31 @@ func (r *resolver) resolveAlbum(ctx context.Context, albumID string) (resolution return resolution{}, err } - var chain chainState + chain := chainState{trace: traceFrom(ctx)} for pattern := range strings.SplitSeq(strings.ToLower(conf.Server.CoverArtPriority), ",") { pattern = strings.TrimSpace(pattern) + if pattern == "" { + continue + } switch { case pattern == "embedded": - if res, ok := chain.try(resolveEmbedded(ctx, lib, r.ffmpeg, al.EmbedArtPath)); ok { + res, ok := resolveEmbedded(ctx, lib, r.ffmpeg, al.EmbedArtPath) + if res, ok = chain.try(pattern, res, ok); ok { return res, nil } - case pattern == "external": + case pattern == externalCandidate: if rd, name, isErr := r.fetchExternalAlbum(ctx, *al); rd != nil { - return resolution{reader: rd, source: "external:" + name}, nil + return resolution{reader: rd, source: ExternalPrefix + name}, nil } else if isErr { chain.extErr = true } case len(imgFiles) > 0: - if res, ok := chain.try(resolveFolderFile(ctx, lib, imgFiles, pattern)); ok { + res, ok := resolveFolderFile(ctx, lib, imgFiles, pattern) + if res, ok = chain.try(pattern, res, ok); ok { return res, nil } + default: + chain.record(pattern, OutcomeSkipped, "no images in album folder") } } return chain.exhausted(), nil @@ -155,9 +236,10 @@ func (r *resolver) resolveArtist(ctx context.Context, artistID string) (resoluti if err != nil { return resolution{}, err } - upload, ok := resolveLocalFile(ar.UploadedImagePath(), "upload") - if ok { - return upload, nil + chain := chainState{trace: traceFrom(ctx)} + upload, uploadOK := resolveLocalFile(ar.UploadedImagePath(), "upload") + if res, ok := chain.try("upload", upload, uploadOK); ok { + return res, nil } if upload.localError { // The upload outranks every other source; falling through would persist a lower-priority @@ -191,32 +273,43 @@ func (r *resolver) resolveArtist(ctx context.Context, artistID string) (resoluti } } - var chain chainState for pattern := range strings.SplitSeq(strings.ToLower(conf.Server.ArtistArtPriority), ",") { pattern = strings.TrimSpace(pattern) + if pattern == "" { + continue + } switch { - case pattern == "external": + case pattern == externalCandidate: if rd, name, isErr := r.fetchExternalArtist(ctx, *ar); rd != nil { - return resolution{reader: rd, source: "external:" + name}, nil + return resolution{reader: rd, source: ExternalPrefix + name}, nil } else if isErr { chain.extErr = true } case pattern == "image-folder": - if res, ok := chain.try(resolveArtistImageFolder(ar)); ok { + res, ok := resolveArtistImageFolder(ar) + if res, ok = chain.try(pattern, res, ok); ok { return res, nil } case strings.HasPrefix(pattern, "album/"): if lib.FS == nil { + chain.record(pattern, OutcomeSkipped, "artist has no albums") continue } - if res, ok := chain.try(resolveFolderFile(ctx, lib, imgFiles, strings.TrimPrefix(pattern, "album/"))); ok { + res, ok := resolveFolderFile(ctx, lib, imgFiles, strings.TrimPrefix(pattern, "album/")) + if res, ok = chain.try(pattern, res, ok); ok { return res, nil } default: - if lib.FS == nil || artistFolder == "" { + if lib.FS == nil { + chain.record(pattern, OutcomeSkipped, "artist has no albums") continue } - if res, ok := chain.try(resolveArtistFolderPattern(ctx, lib, artistFolder, pattern)); ok { + if artistFolder == "" { + chain.record(pattern, OutcomeSkipped, "no artist folder") + continue + } + res, ok := resolveArtistFolderPattern(ctx, lib, artistFolder, pattern) + if res, ok = chain.try(pattern, res, ok); ok { return res, nil } } @@ -224,6 +317,9 @@ func (r *resolver) resolveArtist(ctx context.Context, artistID string) (resoluti return chain.exhausted(), nil } +// PlaylistGridSamples is how many albums resolvePlaylist samples to build the generated grid. +const PlaylistGridSamples = 4 + // resolvePlaylist tries the uploaded image, the sidecar and ExternalImageURL, then a generated grid. func (r *resolver) resolvePlaylist(ctx context.Context, playlistID string) (resolution, error) { pl, err := r.ds.Playlist(ctx).Get(playlistID) @@ -269,7 +365,8 @@ func (r *resolver) resolvePlaylist(ctx context.Context, playlistID string) (reso } } - albumIDs, err := r.ds.Playlist(ctx).Tracks(pl.ID, false).GetAlbumIDs(model.QueryOptions{Max: 4, Sort: "random()"}) + albumIDs, err := r.ds.Playlist(ctx).Tracks(pl.ID, false). + GetAlbumIDs(model.QueryOptions{Max: PlaylistGridSamples, Sort: "random()"}) if err != nil { return resolution{}, err } @@ -295,7 +392,7 @@ func (r *resolver) resolvePlaylist(ctx context.Context, playlistID string) (reso if decErr == nil { tiles = append(tiles, tile) } - if len(tiles) == 4 { + if len(tiles) == PlaylistGridSamples { break } } @@ -337,15 +434,35 @@ func (r *resolver) resolveMediaFile(ctx context.Context, id string) (resolution, if err != nil { return resolution{}, err } - if !conf.Server.EnableMediaFileCoverArt || !mf.HasCoverArt { + chain := chainState{trace: traceFrom(ctx)} + switch { + case !conf.Server.EnableMediaFileCoverArt: + chain.record("embedded", OutcomeSkipped, "EnableMediaFileCoverArt is off") + return resolution{}, nil + case !mf.HasCoverArt: + chain.record("embedded", OutcomeMiss, "the track has no embedded cover art") return resolution{}, nil } lib, err := loadLibraryView(ctx, r.ds, mf.LibraryID) if err != nil { return resolution{}, err } - res, _ := resolveEmbedded(ctx, lib, r.ffmpeg, mf.Path) - return res, nil + res, ok := resolveEmbedded(ctx, lib, r.ffmpeg, mf.Path) + if res, ok = chain.try("embedded", res, ok); ok { + return res, nil + } + return chain.exhausted(), nil +} + +// resolveDisc walks conf.Server.DiscArtPriority. Disc artwork keeps no state row and is never +// queued: the serving path reads it through on every request, so this only ever explains. +func (r *resolver) resolveDisc(ctx context.Context, id string) (resolution, error) { + dr, err := newDiscArtworkReader(ctx, r.ds, model.ArtworkID{Kind: model.KindDiscArtwork, ID: id}) + if err != nil { + return resolution{}, err + } + chain := chainState{trace: traceFrom(ctx)} + return dr.selectImage(ctx, r.ffmpeg, conf.Server.DiscArtPriority, &chain) } // resolveExternalStep runs a single external sourceFunc through the named gate. extErr excludes @@ -353,7 +470,7 @@ func (r *resolver) resolveMediaFile(ctx context.Context, id string) (resolution, func resolveExternalStep(gate gateFunc, name string, sf sourceFunc) (res resolution, ok bool, extErr bool) { r, path, err := gate(name, sf) if r != nil { - return resolution{reader: r, source: "external", sourcePath: path}, true, false + return resolution{reader: r, source: externalCandidate, sourcePath: path}, true, false } return resolution{}, false, err != nil && !errors.Is(err, model.ErrNotFound) } @@ -394,14 +511,20 @@ func resolveEmbedded(ctx context.Context, lib libraryView, ffm ffmpeg.FFmpeg, em return resolution{localError: unreadable}, false } -func resolveFolderFile(ctx context.Context, lib libraryView, imgFiles []string, pattern string) (resolution, bool) { - r, path, err := fromExternalFile(ctx, lib.FS, imgFiles, pattern)() +// resolveFolderSource turns a source that yields a library-relative image path into a folder +// resolution, keeping an existing-but-unopenable file distinct from an absent one. +func resolveFolderSource(lib libraryView, sf sourceFunc) (resolution, bool) { + r, path, err := sf() if r == nil { return resolution{localError: errors.Is(err, errSourceUnreadable)}, false } return resolution{reader: r, source: "folder", sourcePath: lib.Abs(path), refMtime: mtimeViaFS(lib.FS, path)}, true } +func resolveFolderFile(ctx context.Context, lib libraryView, imgFiles []string, pattern string) (resolution, bool) { + return resolveFolderSource(lib, fromExternalFile(ctx, lib.FS, imgFiles, pattern)) +} + func resolveArtistImageFolder(ar *model.Artist) (resolution, bool) { folder := conf.Server.ArtistImageFolder if folder == "" { diff --git a/core/artwork/resolve_test.go b/core/artwork/resolve_test.go index 9a0962cf2..25fb727de 100644 --- a/core/artwork/resolve_test.go +++ b/core/artwork/resolve_test.go @@ -394,6 +394,28 @@ var _ = Describe("resolveItem", func() { Entry("4 albums -> full grid", []string{"t1", "t2", "t3", "t4"}, tileSize-1), ) + // The grid samples album art through the full album chain, so a playlist reaches the + // network even with the m3u fetch off. + It("calls the album image agents for its grid tiles when m3u art is disabled", func() { + conf.Server.EnableM3UExternalAlbumArt = false + conf.Server.CoverArtPriority = "external" + folderRepo.result = nil + plRepo := tests.CreateMockPlaylistRepo() + plRepo.SetData(model.Playlists{{ID: "plgrid", Name: "Playlist"}}) + plRepo.TracksRepo = &tests.MockPlaylistTrackRepo{AlbumIDs: []string{"t1", "t2"}} + ds.MockedPlaylist = plRepo + imageAgents(&fakeImageAgent{name: "failAgent", err: errors.New("boom")}) + var gatedNames []string + gate := func(name string, f func() (io.ReadCloser, string, error)) (io.ReadCloser, string, error) { + gatedNames = append(gatedNames, name) + return f() + } + + _, err := newResolver(ds, ag, ffm, gate).resolve(ctx, model.ArtworkQueueItem{ItemKind: "pl", ItemID: "plgrid"}) + Expect(err).ToNot(HaveOccurred()) + Expect(gatedNames).To(Equal([]string{"failAgent", "failAgent"}), "one lookup per sampled album") + }) + It("resolves the uploaded image before the generated grid", func() { tmpDir := GinkgoT().TempDir() conf.Server.DataFolder = conf.NewDir(tmpDir) @@ -630,3 +652,110 @@ var _ = Describe("decodeTile", func() { Expect(err).To(HaveOccurred()) }) }) + +var _ = Describe("Explainable", func() { + It("is true for the kinds the resolver walks", func() { + Expect(Explainable(model.KindArtistArtwork)).To(BeTrue()) + Expect(Explainable(model.KindAlbumArtwork)).To(BeTrue()) + Expect(Explainable(model.KindDiscArtwork)).To(BeTrue()) + Expect(Explainable(model.KindMediaFileArtwork)).To(BeTrue()) + }) + + It("is false for the kinds resolved from a fixed internal order", func() { + Expect(Explainable(model.KindPlaylistArtwork)).To(BeFalse()) + Expect(Explainable(model.KindRadioArtwork)).To(BeFalse()) + }) +}) + +var _ = Describe("MayFetchExternal", func() { + BeforeEach(func() { + DeferCleanup(configtest.SetupConfig()) + conf.Server.CoverArtPriority = "cover.*, embedded" + conf.Server.ArtistArtPriority = "artist.*" + conf.Server.EnableM3UExternalAlbumArt = false + }) + + It("is true for the kinds whose chain includes the external candidate", func() { + conf.Server.CoverArtPriority = "cover.*, external" + conf.Server.ArtistArtPriority = "artist.*, external" + Expect(MayFetchExternal(model.KindAlbumArtwork)).To(BeTrue()) + Expect(MayFetchExternal(model.KindArtistArtwork)).To(BeTrue()) + }) + + It("is false for a chain with no external candidate", func() { + Expect(MayFetchExternal(model.KindAlbumArtwork)).To(BeFalse()) + Expect(MayFetchExternal(model.KindArtistArtwork)).To(BeFalse()) + }) + + It("is true for playlists when the m3u image fetch is enabled", func() { + conf.Server.EnableM3UExternalAlbumArt = true + Expect(MayFetchExternal(model.KindPlaylistArtwork)).To(BeTrue()) + }) + + It("is true for playlists whose grid tiles resolve through an external album chain", func() { + conf.Server.CoverArtPriority = "cover.*, external" + Expect(MayFetchExternal(model.KindPlaylistArtwork)).To(BeTrue()) + }) + + It("is false for playlists with both paths off", func() { + Expect(MayFetchExternal(model.KindPlaylistArtwork)).To(BeFalse()) + }) + + It("is false for the kinds that only read local files", func() { + conf.Server.CoverArtPriority = "external" + conf.Server.ArtistArtPriority = "external" + conf.Server.EnableM3UExternalAlbumArt = true + Expect(MayFetchExternal(model.KindRadioArtwork)).To(BeFalse()) + Expect(MayFetchExternal(model.KindMediaFileArtwork)).To(BeFalse()) + }) +}) + +var _ = Describe("ExternalLookupsPerItem", func() { + count := ImageAgentCount{Artist: 3, Album: 2} + + BeforeEach(func() { + DeferCleanup(configtest.SetupConfig()) + conf.Server.CoverArtPriority = "cover.*, external" + conf.Server.ArtistArtPriority = "artist.*, external" + conf.Server.EnableM3UExternalAlbumArt = false + }) + + It("bills one call per agent, since the walk only stops early on a hit", func() { + Expect(ExternalLookupsPerItem(model.KindArtistArtwork, count)).To(Equal(int64(3))) + Expect(ExternalLookupsPerItem(model.KindAlbumArtwork, count)).To(Equal(int64(2))) + }) + + It("bills a playlist for every album its grid samples", func() { + Expect(ExternalLookupsPerItem(model.KindPlaylistArtwork, count)). + To(Equal(int64(PlaylistGridSamples) * 2)) + }) + + It("adds the m3u image fetch on top of the grid", func() { + conf.Server.EnableM3UExternalAlbumArt = true + Expect(ExternalLookupsPerItem(model.KindPlaylistArtwork, count)). + To(Equal(int64(PlaylistGridSamples)*2 + 1)) + }) + + It("bills only the m3u fetch when the album chain stays local", func() { + conf.Server.CoverArtPriority = "cover.*" + conf.Server.EnableM3UExternalAlbumArt = true + Expect(ExternalLookupsPerItem(model.KindPlaylistArtwork, count)).To(Equal(int64(1))) + }) + + It("still bills a call when no agent is visible, which plugins never are offline", func() { + none := ImageAgentCount{} + Expect(ExternalLookupsPerItem(model.KindArtistArtwork, none)).To(Equal(int64(1))) + Expect(ExternalLookupsPerItem(model.KindAlbumArtwork, none)).To(Equal(int64(1))) + Expect(ExternalLookupsPerItem(model.KindPlaylistArtwork, none)). + To(Equal(int64(PlaylistGridSamples))) + }) + + It("is zero whenever the kind reaches no agent at all", func() { + conf.Server.CoverArtPriority = "cover.*" + conf.Server.ArtistArtPriority = "artist.*" + Expect(ExternalLookupsPerItem(model.KindArtistArtwork, count)).To(BeZero()) + Expect(ExternalLookupsPerItem(model.KindAlbumArtwork, count)).To(BeZero()) + Expect(ExternalLookupsPerItem(model.KindPlaylistArtwork, count)).To(BeZero()) + Expect(ExternalLookupsPerItem(model.KindRadioArtwork, count)).To(BeZero()) + }) +}) diff --git a/core/artwork/sources.go b/core/artwork/sources.go index 885ca03cf..78b7dd68d 100644 --- a/core/artwork/sources.go +++ b/core/artwork/sources.go @@ -27,23 +27,6 @@ import ( // to open it is not evidence the entity has no artwork, so callers must not settle on absent. var errSourceUnreadable = errors.New("artwork source unreadable") -func selectImageReader(ctx context.Context, artID model.ArtworkID, extractFuncs ...sourceFunc) (io.ReadCloser, string, error) { - for _, f := range extractFuncs { - if ctx.Err() != nil { - return nil, "", ctx.Err() - } - start := time.Now() - r, path, err := f() - if r != nil { - msg := fmt.Sprintf("Artwork: Found %s artwork", artID.Kind) - log.Debug(ctx, msg, "artID", artID, "path", path, "source", f, "elapsed", time.Since(start)) - return r, path, nil - } - log.Trace(ctx, "Artwork: Failed trying to extract artwork", "artID", artID, "source", f, "elapsed", time.Since(start), err) - } - return nil, "", fmt.Errorf("could not get `%s` cover art for %s: %w", artID.Kind, artID, ErrUnavailable) -} - type sourceFunc func() (r io.ReadCloser, path string, err error) func (f sourceFunc) String() string { diff --git a/core/artwork/trace.go b/core/artwork/trace.go new file mode 100644 index 000000000..5d02fa4ff --- /dev/null +++ b/core/artwork/trace.go @@ -0,0 +1,99 @@ +package artwork + +import ( + "context" + "errors" + "io" + "slices" + "sync" +) + +// Outcome is what the priority chain observed for one candidate; the CLI renders and branches on these. +type Outcome string + +const ( + OutcomeHit Outcome = "hit" + OutcomeMiss Outcome = "miss" + OutcomeUnreadable Outcome = "unreadable" + OutcomeSkipped Outcome = "skipped" + OutcomeWouldTry Outcome = "would-try" + OutcomeError Outcome = "error" +) + +const ( + // externalCandidate labels the external tier itself, for the cases that never reach an agent. + externalCandidate = "external" + // ExternalPrefix qualifies a candidate or a stored source with the agent that produced it. + ExternalPrefix = externalCandidate + ":" +) + +// TraceStep is one candidate the priority chain considered. +type TraceStep struct { + Candidate string + Outcome Outcome + Detail string +} + +// ChainTrace collects the walk of a single resolution. The artwork worker never attaches +// one; only the CLI does, so resolution stays allocation-free in the hot path. +type ChainTrace struct { + mu sync.Mutex + steps []TraceStep +} + +func (t *ChainTrace) add(step TraceStep) { + if t == nil { + return + } + t.mu.Lock() + defer t.mu.Unlock() + t.steps = append(t.steps, step) +} + +func (t *ChainTrace) Steps() []TraceStep { + if t == nil { + return nil + } + t.mu.Lock() + defer t.mu.Unlock() + return slices.Clone(t.steps) +} + +type traceCtxKey struct{} + +func withTrace(ctx context.Context, t *ChainTrace) context.Context { + return context.WithValue(ctx, traceCtxKey{}, t) +} + +func traceFrom(ctx context.Context) *ChainTrace { + t, _ := ctx.Value(traceCtxKey{}).(*ChainTrace) + return t +} + +var errOfflineSkipped = errors.New("artwork: external lookup skipped (offline)") + +// tracingGate records each external agent's outcome without changing what the gate returns. +func tracingGate(t *ChainTrace, inner gateFunc) gateFunc { + return func(name string, f func() (io.ReadCloser, string, error)) (io.ReadCloser, string, error) { + r, path, err := inner(name, f) + candidate := ExternalPrefix + name + switch { + case r != nil: + t.add(TraceStep{Candidate: candidate, Outcome: OutcomeHit, Detail: path}) + case isTransientExternal(err): + t.add(TraceStep{Candidate: candidate, Outcome: OutcomeError, Detail: err.Error()}) + default: + t.add(TraceStep{Candidate: candidate, Outcome: OutcomeMiss}) + } + return r, path, err + } +} + +// offlineGate reports which agents would be asked without asking them, so a diagnostic +// command cannot add load to a provider that is already rate-limiting us. +func offlineGate(t *ChainTrace) gateFunc { + return func(name string, _ func() (io.ReadCloser, string, error)) (io.ReadCloser, string, error) { + t.add(TraceStep{Candidate: ExternalPrefix + name, Outcome: OutcomeWouldTry}) + return nil, "", errOfflineSkipped + } +} diff --git a/core/artwork/trace_test.go b/core/artwork/trace_test.go new file mode 100644 index 000000000..5a54c9e91 --- /dev/null +++ b/core/artwork/trace_test.go @@ -0,0 +1,628 @@ +package artwork + +import ( + "context" + "errors" + "io" + "os" + "path/filepath" + "runtime" + "strings" + + "github.com/navidrome/navidrome/conf" + "github.com/navidrome/navidrome/conf/configtest" + "github.com/navidrome/navidrome/consts" + "github.com/navidrome/navidrome/core/agents" + "github.com/navidrome/navidrome/model" + "github.com/navidrome/navidrome/tests" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("trace vocabulary", func() { + // The CLI renders these verbatim and branches on them; a value change is a change to + // what `artwork explain` tells an operator, so it must be made deliberately. + It("pins the wire values the CLI reads", func() { + Expect([]Outcome{ + OutcomeHit, OutcomeMiss, OutcomeUnreadable, OutcomeSkipped, OutcomeWouldTry, OutcomeError, + }).To(Equal([]Outcome{"hit", "miss", "unreadable", "skipped", "would-try", "error"})) + Expect(externalCandidate).To(Equal("external")) + Expect(ExternalPrefix).To(Equal("external:")) + }) +}) + +var _ = Describe("chainTrace", func() { + It("returns nil when no trace is attached", func() { + Expect(traceFrom(context.Background())).To(BeNil()) + }) + + It("collects steps in order", func() { + t := &ChainTrace{} + ctx := withTrace(context.Background(), t) + + traceFrom(ctx).add(TraceStep{Candidate: "cover.*", Outcome: OutcomeMiss}) + traceFrom(ctx).add(TraceStep{Candidate: "embedded", Outcome: OutcomeHit, Detail: "/music/a.flac"}) + + Expect(t.Steps()).To(Equal([]TraceStep{ + {Candidate: "cover.*", Outcome: OutcomeMiss}, + {Candidate: "embedded", Outcome: OutcomeHit, Detail: "/music/a.flac"}, + })) + + s := t.Steps() + s[0].Candidate = "mutated" + Expect(t.Steps()[0].Candidate).To(Equal("cover.*")) + }) + + It("does not panic when the trace is nil", func() { + var t *ChainTrace + Expect(func() { t.add(TraceStep{Candidate: "cover.*", Outcome: OutcomeMiss}) }).ToNot(Panic()) + Expect(t.Steps()).To(BeEmpty(), "a nil trace collects nothing, so reading it must be as safe as writing it") + }) + + It("is safe to use concurrently", func() { + t := &ChainTrace{} + done := make(chan struct{}) + for range 10 { + go func() { + defer GinkgoRecover() + t.add(TraceStep{Candidate: "x", Outcome: OutcomeMiss}) + done <- struct{}{} + }() + } + for range 10 { + <-done + } + Expect(t.Steps()).To(HaveLen(10)) + }) +}) + +var _ = Describe("chainState tracing", func() { + It("records a miss when the candidate was absent", func() { + t := &ChainTrace{} + c := chainState{trace: t} + + _, ok := c.try("cover.*", resolution{}, false) + + Expect(ok).To(BeFalse()) + Expect(t.Steps()).To(Equal([]TraceStep{{Candidate: "cover.*", Outcome: OutcomeMiss}})) + }) + + It("records unreadable when the candidate existed but could not be read", func() { + t := &ChainTrace{} + c := chainState{trace: t} + + _, ok := c.try("cover.*", resolution{localError: true}, false) + + Expect(ok).To(BeFalse()) + Expect(t.Steps()).To(HaveLen(1)) + Expect(t.Steps()[0].Outcome).To(Equal(OutcomeUnreadable), + "a candidate that existed and failed to decode must be distinguishable from one that was absent") + }) + + It("records a hit with the backing path", func() { + t := &ChainTrace{} + c := chainState{trace: t} + + res, ok := c.try("embedded", resolution{reader: nil, source: "embedded", sourcePath: "/music/a.flac"}, true) + + Expect(ok).To(BeTrue()) + Expect(res.source).To(Equal("embedded")) + Expect(t.Steps()).To(Equal([]TraceStep{ + {Candidate: "embedded", Outcome: OutcomeHit, Detail: "/music/a.flac"}, + })) + }) +}) + +var _ = Describe("external gate tracing", func() { + hit := func() (io.ReadCloser, string, error) { + return io.NopCloser(strings.NewReader("x")), "http://img", nil + } + miss := func() (io.ReadCloser, string, error) { return nil, "", agents.ErrNotFound } + boom := func() (io.ReadCloser, string, error) { return nil, "", errors.New("returned status 429") } + + It("records a hit with the image path", func() { + t := &ChainTrace{} + g := tracingGate(t, passthroughGate) + + r, _, err := g("deezer", hit) + + Expect(err).ToNot(HaveOccurred()) + Expect(r).ToNot(BeNil()) + Expect(t.Steps()).To(Equal([]TraceStep{ + {Candidate: "external:deezer", Outcome: OutcomeHit, Detail: "http://img"}, + })) + }) + + It("records a miss for a not-found", func() { + t := &ChainTrace{} + _, _, _ = tracingGate(t, passthroughGate)("deezer", miss) + Expect(t.Steps()[0].Outcome).To(Equal(OutcomeMiss)) + }) + + It("records a miss for a model not-found", func() { + t := &ChainTrace{} + notFound := func() (io.ReadCloser, string, error) { return nil, "", model.ErrNotFound } + _, _, _ = tracingGate(t, passthroughGate)("deezer", notFound) + Expect(t.Steps()[0].Outcome).To(Equal(OutcomeMiss), + "both not-found flavours are definitive answers, not faults") + }) + + It("records an error with its reason", func() { + t := &ChainTrace{} + _, _, _ = tracingGate(t, passthroughGate)("apple-music", boom) + Expect(t.Steps()[0].Outcome).To(Equal(OutcomeError)) + Expect(t.Steps()[0].Detail).To(ContainSubstring("429")) + }) + + It("never calls the agent in offline mode", func() { + t := &ChainTrace{} + called := false + counting := func() (io.ReadCloser, string, error) { + called = true + return hit() + } + + _, _, err := offlineGate(t)("deezer", counting) + + Expect(called).To(BeFalse(), "offline mode must not perform external requests") + Expect(err).To(MatchError(errOfflineSkipped)) + Expect(t.Steps()).To(Equal([]TraceStep{ + {Candidate: "external:deezer", Outcome: OutcomeWouldTry}, + })) + }) +}) + +var _ = Describe("resolveAlbum tracing", func() { + var ( + ctx context.Context + ds *tests.MockDataStore + albumRepo *tests.MockAlbumRepo + folderRepo *fakeFolderRepo + ffm *tests.MockFFmpeg + ag *agents.Agents + t *ChainTrace + ) + + BeforeEach(func() { + DeferCleanup(configtest.SetupConfig()) + conf.Server.CoverArtPriority = "cover.jpg, embedded" + repoRoot, err := os.Getwd() + Expect(err).ToNot(HaveOccurred()) + libRepo := &tests.MockLibraryRepo{} + libRepo.SetData(model.Libraries{{ID: 0, Path: testFileLibPath(repoRoot)}}) + albumRepo = tests.CreateMockAlbumRepo() + folderRepo = &fakeFolderRepo{} + ds = &tests.MockDataStore{ + MockedAlbum: albumRepo, + MockedFolder: folderRepo, + MockedLibrary: libRepo, + } + ffm = tests.NewMockFFmpeg("") + ag = agents.GetAgents(&tests.MockDataStore{}, nil) + t = &ChainTrace{} + ctx = withTrace(context.Background(), t) + }) + + It("records a pattern skipped because the album folder holds no images", func() { + albumRepo.SetData(model.Albums{ + {ID: "al1", Name: "Album", EmbedArtPath: "tests/fixtures/artist/an-album/test.mp3", FolderIDs: []string{"f1"}}, + }) + + res, err := newResolver(ds, ag, ffm, nil).resolve(ctx, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al1"}) + Expect(err).ToNot(HaveOccurred()) + Expect(res.reader).ToNot(BeNil()) + defer res.reader.Close() + Expect(t.Steps()).To(HaveLen(2), "a configured pattern must appear even when the chain never evaluated it") + Expect(t.Steps()[0]).To(Equal(TraceStep{ + Candidate: "cover.jpg", Outcome: OutcomeSkipped, Detail: "no images in album folder", + })) + Expect(t.Steps()[1].Candidate).To(Equal("embedded")) + Expect(t.Steps()[1].Outcome).To(Equal(OutcomeHit)) + }) + + It("records an evaluated pattern that matched nothing as a miss, not a skip", func() { + folderRepo.result = []model.Folder{{ + Path: "tests/fixtures/artist/an-album", + ImageFiles: []string{"artist.png"}, + }} + albumRepo.SetData(model.Albums{ + {ID: "al3", Name: "Album", EmbedArtPath: "tests/fixtures/artist/an-album/test.mp3", FolderIDs: []string{"f1"}}, + }) + + res, err := newResolver(ds, ag, ffm, nil).resolve(ctx, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al3"}) + Expect(err).ToNot(HaveOccurred()) + Expect(res.reader).ToNot(BeNil()) + defer res.reader.Close() + Expect(t.Steps()[0]).To(Equal(TraceStep{Candidate: "cover.jpg", Outcome: OutcomeMiss}), + "the folder was searched and held no cover.jpg, which is not the same as never looking") + }) + + It("ignores an empty priority token", func() { + conf.Server.CoverArtPriority = "cover.jpg," + albumRepo.SetData(model.Albums{{ID: "al2", Name: "Album", FolderIDs: []string{"f1"}}}) + + _, err := newResolver(ds, ag, ffm, nil).resolve(ctx, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al2"}) + Expect(err).ToNot(HaveOccurred()) + Expect(t.Steps()).To(Equal([]TraceStep{ + {Candidate: "cover.jpg", Outcome: OutcomeSkipped, Detail: "no images in album folder"}, + })) + }) +}) + +var _ = Describe("resolveArtist tracing", func() { + var ( + ctx context.Context + ds *tests.MockDataStore + artistRepo *tests.MockArtistRepo + albumRepo *tests.MockAlbumRepo + folderRepo *fakeFolderRepo + ffm *tests.MockFFmpeg + ag *agents.Agents + t *ChainTrace + repoRoot string + ) + + uploadPath := func(file string) string { + path := model.UploadedImagePath(consts.EntityArtist, file) + Expect(os.MkdirAll(filepath.Dir(path), 0o755)).To(Succeed()) + Expect(os.WriteFile(path, []byte("uploaded artist image"), 0o600)).To(Succeed()) + return path + } + + BeforeEach(func() { + DeferCleanup(configtest.SetupConfig()) + conf.Server.DataFolder = conf.NewDir(GinkgoT().TempDir()) + conf.Server.ArtistArtPriority = "album/artist.*" + var err error + repoRoot, err = os.Getwd() + Expect(err).ToNot(HaveOccurred()) + libRepo := &tests.MockLibraryRepo{} + libRepo.SetData(model.Libraries{{ID: 0, Path: testFileLibPath(repoRoot)}}) + artistRepo = tests.CreateMockArtistRepo() + albumRepo = tests.CreateMockAlbumRepo() + folderRepo = &fakeFolderRepo{} + ds = &tests.MockDataStore{ + MockedArtist: artistRepo, + MockedAlbum: albumRepo, + MockedFolder: folderRepo, + MockedLibrary: libRepo, + } + ffm = tests.NewMockFFmpeg("") + ag = agents.GetAgents(&tests.MockDataStore{}, nil) + t = &ChainTrace{} + ctx = withTrace(context.Background(), t) + }) + + It("records the upload short-circuit as a hit", func() { + path := uploadPath("ar1_test.jpg") + artistRepo.SetData(model.Artists{{ID: "ar1", Name: "Artist", UploadedImage: "ar1_test.jpg"}}) + + res, err := newResolver(ds, ag, ffm, nil).resolve(ctx, model.ArtworkQueueItem{ItemKind: "ar", ItemID: "ar1"}) + Expect(err).ToNot(HaveOccurred()) + Expect(res.reader).ToNot(BeNil()) + defer res.reader.Close() + Expect(t.Steps()).To(Equal([]TraceStep{{Candidate: "upload", Outcome: OutcomeHit, Detail: path}})) + }) + + It("records an upload miss before walking the chain", func() { + artistRepo.SetData(model.Artists{{ID: "ar2", Name: "Artist"}}) + + _, err := newResolver(ds, ag, ffm, nil).resolve(ctx, model.ArtworkQueueItem{ItemKind: "ar", ItemID: "ar2"}) + Expect(err).ToNot(HaveOccurred()) + Expect(t.Steps()[0]).To(Equal(TraceStep{Candidate: "upload", Outcome: OutcomeMiss})) + }) + + It("labels each step with the configured priority token", func() { + folderRepo.result = []model.Folder{{ + LibraryPath: testFileLibPath(repoRoot), + Path: "tests/fixtures/artist/an-album", + ImageFiles: []string{"artist.png"}, + }} + artistRepo.SetData(model.Artists{{ID: "ar4", Name: "Artist"}}) + albumRepo.All = model.Albums{{ID: "al9", Name: "Album", LibraryID: 0, FolderIDs: []string{"f1"}}} + + res, err := newResolver(ds, ag, ffm, nil).resolve(ctx, model.ArtworkQueueItem{ItemKind: "ar", ItemID: "ar4"}) + Expect(err).ToNot(HaveOccurred()) + Expect(res.reader).ToNot(BeNil()) + defer res.reader.Close() + Expect(t.Steps()).To(HaveLen(2)) + Expect(t.Steps()[1].Candidate).To(Equal("album/artist.*"), + "the step must be labelled with the priority token, not the pattern it was rewritten into") + Expect(t.Steps()[1].Outcome).To(Equal(OutcomeHit)) + Expect(filepath.ToSlash(t.Steps()[1].Detail)).To(HaveSuffix("tests/fixtures/artist/an-album/artist.png")) + }) + + It("records a configured pattern that could not be evaluated", func() { + artistRepo.SetData(model.Artists{{ID: "ar5", Name: "Artist"}}) + + _, err := newResolver(ds, ag, ffm, nil).resolve(ctx, model.ArtworkQueueItem{ItemKind: "ar", ItemID: "ar5"}) + Expect(err).ToNot(HaveOccurred()) + Expect(t.Steps()).To(Equal([]TraceStep{ + {Candidate: "upload", Outcome: OutcomeMiss}, + {Candidate: "album/artist.*", Outcome: OutcomeSkipped, Detail: "artist has no albums"}, + }), "a configured pattern that was never evaluated must still appear, and say why") + }) + + It("records why an artist folder pattern was skipped", func() { + conf.Server.ArtistArtPriority = "artist.*" + artistRepo.SetData(model.Artists{{ID: "ar6", Name: "Artist"}}) + albumRepo.All = model.Albums{{ID: "al10", Name: "Album", LibraryID: 0, FolderIDs: []string{"f1"}}} + + _, err := newResolver(ds, ag, ffm, nil).resolve(ctx, model.ArtworkQueueItem{ItemKind: "ar", ItemID: "ar6"}) + Expect(err).ToNot(HaveOccurred()) + Expect(t.Steps()).To(Equal([]TraceStep{ + {Candidate: "upload", Outcome: OutcomeMiss}, + {Candidate: "artist.*", Outcome: OutcomeSkipped, Detail: "no artist folder"}, + })) + }) + + It("records an upload that exists but cannot be read as unreadable", func() { + if runtime.GOOS == "windows" { + Skip("chmod does not restrict read access on Windows") + } + path := uploadPath("ar3_test.jpg") + Expect(os.Chmod(path, 0o000)).To(Succeed()) + DeferCleanup(func() { _ = os.Chmod(path, 0o600) }) + artistRepo.SetData(model.Artists{{ID: "ar3", Name: "Artist", UploadedImage: "ar3_test.jpg"}}) + + _, err := newResolver(ds, ag, ffm, nil).resolve(ctx, model.ArtworkQueueItem{ItemKind: "ar", ItemID: "ar3"}) + Expect(err).ToNot(HaveOccurred()) + Expect(t.Steps()).To(HaveLen(1)) + Expect(t.Steps()[0].Outcome).To(Equal(OutcomeUnreadable), + "an upload that exists and will not open must not look like an absent upload") + }) +}) + +var _ = Describe("NewTracingResolver", func() { + var ( + ds *tests.MockDataStore + albumRepo *tests.MockAlbumRepo + artistRepo *tests.MockArtistRepo + artworkRepo *tests.MockArtworkRepo + queueRepo *tests.MockArtworkQueueRepo + ffm *tests.MockFFmpeg + t *ChainTrace + ) + + BeforeEach(func() { + DeferCleanup(configtest.SetupConfig()) + conf.Server.DataFolder = conf.NewDir(GinkgoT().TempDir()) + conf.Server.CoverArtPriority = "external, embedded" + conf.Server.ArtistArtPriority = "external" + repoRoot, err := os.Getwd() + Expect(err).ToNot(HaveOccurred()) + libRepo := &tests.MockLibraryRepo{} + libRepo.SetData(model.Libraries{{ID: 0, Path: testFileLibPath(repoRoot)}}) + albumRepo = tests.CreateMockAlbumRepo() + artistRepo = tests.CreateMockArtistRepo() + artworkRepo = tests.CreateMockArtworkRepo() + queueRepo = tests.CreateMockArtworkQueueRepo() + ds = &tests.MockDataStore{ + MockedAlbum: albumRepo, + MockedArtist: artistRepo, + MockedFolder: &fakeFolderRepo{}, + MockedLibrary: libRepo, + MockedArtwork: artworkRepo, + MockedArtworkQueue: queueRepo, + } + ffm = tests.NewMockFFmpeg("") + t = &ChainTrace{} + }) + + Context("offline", func() { + var fake *fakeImageAgent + + BeforeEach(func() { + fake = &fakeImageAgent{name: "offline-probe"} + albumRepo.SetData(model.Albums{{ + ID: "al1", Name: "Album", EmbedArtPath: "tests/fixtures/artist/an-album/test.mp3", FolderIDs: []string{"f1"}, + }}) + artistRepo.SetData(model.Artists{{ID: "ar1", Name: "Artist"}}) + }) + + It("reports the external tier without asking any agent", func() { + source, err := NewTracingResolver(ds, imageAgents(fake), ffm, t, false).Resolve(context.Background(), model.KindAlbumArtwork, "al1") + + Expect(err).ToNot(HaveOccurred()) + Expect(source).To(Equal("embedded")) + Expect(fake.albumCalls).To(BeZero(), "offline mode must not add load to an external provider") + Expect(t.Steps()).To(ContainElement(TraceStep{Candidate: "external:offline-probe", Outcome: OutcomeWouldTry})) + }) + + It("records the local chain steps too", func() { + _, err := NewTracingResolver(ds, imageAgents(fake), ffm, t, false).Resolve(context.Background(), model.KindAlbumArtwork, "al1") + + Expect(err).ToNot(HaveOccurred()) + last := t.Steps()[len(t.Steps())-1] + Expect(last.Candidate).To(Equal("embedded"), "the local chain must be traced, not just the external gate") + Expect(last.Outcome).To(Equal(OutcomeHit)) + }) + + It("never persists artwork state", func() { + _, err := NewTracingResolver(ds, imageAgents(fake), ffm, t, false).Resolve(context.Background(), model.KindAlbumArtwork, "al1") + + Expect(err).ToNot(HaveOccurred()) + Expect(artworkRepo.ItemData).To(BeEmpty(), + "an offline resolution carries extError, which must never be recorded as a real provider failure") + Expect(queueRepo.Data).To(BeEmpty()) + }) + + It("resolves an artist without persisting anything", func() { + source, err := NewTracingResolver(ds, imageAgents(fake), ffm, t, false).Resolve(context.Background(), model.KindArtistArtwork, "ar1") + + Expect(err).ToNot(HaveOccurred()) + Expect(source).To(BeEmpty()) + Expect(fake.artistCalls).To(BeZero()) + Expect(t.Steps()).To(ContainElement(TraceStep{Candidate: "external:offline-probe", Outcome: OutcomeWouldTry})) + Expect(artworkRepo.ItemData).To(BeEmpty()) + Expect(queueRepo.Data).To(BeEmpty()) + }) + + It("closes the reader it does not hand back", func() { + conf.Server.CoverArtPriority = "embedded" + ffm = tests.NewMockFFmpeg("fake image bytes") + albumRepo.SetData(model.Albums{{ + ID: "al2", Name: "Album", EmbedArtPath: "tests/fixtures/artist/an-album/no-such-file.mp3", FolderIDs: []string{"f1"}, + }}) + + source, err := NewTracingResolver(ds, imageAgents(fake), ffm, t, false).Resolve(context.Background(), model.KindAlbumArtwork, "al2") + + Expect(err).ToNot(HaveOccurred()) + Expect(source).To(Equal("embedded")) + Expect(ffm.IsClosed()).To(BeTrue(), "nothing downstream closes it, so a leak is one file handle per invocation") + }) + + It("propagates a lookup error", func() { + _, err := NewTracingResolver(ds, imageAgents(fake), ffm, t, false).Resolve(context.Background(), model.KindAlbumArtwork, "nope") + Expect(err).To(MatchError(model.ErrNotFound)) + }) + }) + + It("asks the agents when live is true", func() { + fake := &fakeImageAgent{name: "live-probe", err: agents.ErrNotFound} + albumRepo.SetData(model.Albums{{ + ID: "al1", Name: "Album", EmbedArtPath: "tests/fixtures/artist/an-album/test.mp3", FolderIDs: []string{"f1"}, + }}) + + _, err := NewTracingResolver(ds, imageAgents(fake), ffm, t, true).Resolve(context.Background(), model.KindAlbumArtwork, "al1") + + Expect(err).ToNot(HaveOccurred()) + Expect(fake.albumCalls).To(Equal(1)) + Expect(t.Steps()).To(ContainElement(TraceStep{Candidate: "external:live-probe", Outcome: OutcomeMiss})) + }) +}) + +var _ = Describe("resolveDisc tracing", func() { + var ( + ctx context.Context + ds *tests.MockDataStore + albumRepo *tests.MockAlbumRepo + folderRepo *fakeFolderRepo + ffm *tests.MockFFmpeg + t *ChainTrace + ) + + BeforeEach(func() { + DeferCleanup(configtest.SetupConfig()) + repoRoot, err := os.Getwd() + Expect(err).ToNot(HaveOccurred()) + libRepo := &tests.MockLibraryRepo{} + libRepo.SetData(model.Libraries{{ID: 0, Path: testFileLibPath(repoRoot)}}) + albumRepo = tests.CreateMockAlbumRepo() + albumRepo.SetData(model.Albums{{ID: "al1", Name: "Album", FolderIDs: []string{"f1"}}}) + folderRepo = &fakeFolderRepo{} + mfRepo := tests.CreateMockMediaFileRepo() + mfRepo.SetData(model.MediaFiles{{ID: "mf1", AlbumID: "al1", DiscNumber: 2, Path: "tests/fixtures/artist/an-album/test.mp3"}}) + ds = &tests.MockDataStore{ + MockedAlbum: albumRepo, + MockedMediaFile: mfRepo, + MockedFolder: folderRepo, + MockedLibrary: libRepo, + } + ffm = tests.NewMockFFmpeg("") + t = &ChainTrace{} + ctx = withTrace(context.Background(), t) + }) + + It("accounts for every configured entry, including the ones that map to no source", func() { + conf.Server.DiscArtPriority = "external, discsubtitle, cover.jpg" + + res, err := newResolver(ds, nil, ffm, nil).resolveDisc(ctx, model.DiscArtworkID("al1", 2)) + + Expect(err).ToNot(HaveOccurred()) + Expect(res.reader).To(BeNil()) + Expect(t.Steps()).To(Equal([]TraceStep{ + {Candidate: "external", Outcome: OutcomeSkipped, Detail: "external sources are not supported for disc artwork"}, + {Candidate: "discsubtitle", Outcome: OutcomeSkipped, Detail: "disc has no subtitle"}, + {Candidate: "cover.jpg", Outcome: OutcomeSkipped, Detail: "no images in album folder"}, + })) + }) + + It("records the entry that won and stops there", func() { + conf.Server.DiscArtPriority = "disc*.*, cover.jpg, embedded" + folderRepo.result = []model.Folder{{ + Path: "tests/fixtures/artist/an-album", + ImageFiles: []string{"cover.jpg"}, + }} + + res, err := newResolver(ds, nil, ffm, nil).resolveDisc(ctx, model.DiscArtworkID("al1", 2)) + + Expect(err).ToNot(HaveOccurred()) + Expect(res.reader).ToNot(BeNil()) + defer res.reader.Close() + Expect(res.source).To(Equal("folder")) + Expect(t.Steps()).To(HaveLen(2), "the walk must stop at the winner, and record nothing below it") + Expect(t.Steps()[0]).To(Equal(TraceStep{Candidate: "disc*.*", Outcome: OutcomeMiss})) + Expect(t.Steps()[1].Candidate).To(Equal("cover.jpg")) + Expect(t.Steps()[1].Outcome).To(Equal(OutcomeHit)) + Expect(t.Steps()[1].Detail).To(HaveSuffix(filepath.FromSlash("tests/fixtures/artist/an-album/cover.jpg"))) + Expect(t.Steps()[1].Detail).ToNot(Equal("tests/fixtures/artist/an-album/cover.jpg"), + "a library-relative path sends the operator looking in the wrong place") + }) + + It("reports an unparseable disc id rather than explaining another disc", func() { + conf.Server.DiscArtPriority = "cover.jpg" + _, err := newResolver(ds, nil, ffm, nil).resolveDisc(ctx, "al1") + Expect(err).To(HaveOccurred()) + }) +}) + +var _ = Describe("resolveMediaFile tracing", func() { + var ( + ctx context.Context + ds *tests.MockDataStore + mfRepo *tests.MockMediaFileRepo + ffm *tests.MockFFmpeg + t *ChainTrace + ) + + BeforeEach(func() { + DeferCleanup(configtest.SetupConfig()) + conf.Server.EnableMediaFileCoverArt = true + repoRoot, err := os.Getwd() + Expect(err).ToNot(HaveOccurred()) + libRepo := &tests.MockLibraryRepo{} + libRepo.SetData(model.Libraries{{ID: 0, Path: testFileLibPath(repoRoot)}}) + mfRepo = tests.CreateMockMediaFileRepo() + mfRepo.SetData(model.MediaFiles{{ + ID: "mf1", Title: "Song", HasCoverArt: true, Path: "tests/fixtures/artist/an-album/test.mp3", + }}) + ds = &tests.MockDataStore{MockedMediaFile: mfRepo, MockedLibrary: libRepo} + ffm = tests.NewMockFFmpeg("") + t = &ChainTrace{} + ctx = withTrace(context.Background(), t) + }) + + It("separates a disabled setting from a track with nothing embedded", func() { + conf.Server.EnableMediaFileCoverArt = false + + _, err := newResolver(ds, nil, ffm, nil).resolveMediaFile(ctx, "mf1") + + Expect(err).ToNot(HaveOccurred()) + Expect(t.Steps()).To(Equal([]TraceStep{ + {Candidate: "embedded", Outcome: OutcomeSkipped, Detail: "EnableMediaFileCoverArt is off"}, + })) + }) + + It("records a track with no embedded art as a miss", func() { + mfRepo.SetData(model.MediaFiles{{ID: "mf2", Title: "Song", HasCoverArt: false}}) + + _, err := newResolver(ds, nil, ffm, nil).resolveMediaFile(ctx, "mf2") + + Expect(err).ToNot(HaveOccurred()) + Expect(t.Steps()).To(Equal([]TraceStep{ + {Candidate: "embedded", Outcome: OutcomeMiss, Detail: "the track has no embedded cover art"}, + })) + }) + + It("records the embedded hit", func() { + res, err := newResolver(ds, nil, ffm, nil).resolveMediaFile(ctx, "mf1") + + Expect(err).ToNot(HaveOccurred()) + Expect(res.reader).ToNot(BeNil()) + defer res.reader.Close() + Expect(res.source).To(Equal("embedded")) + Expect(t.Steps()).To(HaveLen(1)) + Expect(t.Steps()[0].Outcome).To(Equal(OutcomeHit)) + }) +}) diff --git a/core/artwork/worker.go b/core/artwork/worker.go index 3ded52629..be8495305 100644 --- a/core/artwork/worker.go +++ b/core/artwork/worker.go @@ -137,7 +137,7 @@ func (w *Worker) Backfill(ctx context.Context) (bool, error) { return backfill(ctx, w.proc.ds) } -// EnqueueStaleAbsentAll requeues known-absent entries older than staleAbsentAge. +// EnqueueStaleAbsentAll requeues known-absent entries older than StaleAbsentAge. func (w *Worker) EnqueueStaleAbsentAll(ctx context.Context) error { return enqueueStaleAbsentAll(ctx, w.proc.ds) } diff --git a/core/artwork/worker_test.go b/core/artwork/worker_test.go index 9fad4b503..53b6a43b2 100644 --- a/core/artwork/worker_test.go +++ b/core/artwork/worker_test.go @@ -362,7 +362,7 @@ var _ = Describe("Worker", func() { Expect(ia.Hash).To(Equal("cafebabe"), "a persistent outage must not discard served art") }) - // Media files are excluded from recheckKinds, so an absent row here would never be + // Media files are excluded from RecheckKinds, so an absent row here would never be // revisited: a transient read error would look permanent. It("does not settle absent on exhaustion for a kind with no recheck path", func() { conf.Server.EnableMediaFileCoverArt = true diff --git a/model/artwork.go b/model/artwork.go index 87b424f33..ea724265a 100644 --- a/model/artwork.go +++ b/model/artwork.go @@ -119,6 +119,8 @@ type ArtworkRepository interface { } type ArtworkQueueRepository interface { + // Get returns the pending row for an item, or ErrNotFound when it is not queued. + Get(kind Kind, id, imageType string) (*ArtworkQueueItem, error) // Enqueue upserts; an existing row keeps the higher priority and has its retry_at reset. Enqueue(items ...ArtworkQueueItem) error // EnqueuePreservingBackoff upserts like Enqueue but preserves an existing row's retry_at, so a @@ -130,6 +132,14 @@ type ArtworkQueueRepository interface { EnqueueAllMissing(kind Kind, priority int) (int64, error) // EnqueueIfMissing inserts only for items with no item_artwork row yet. EnqueueIfMissing(items ...ArtworkQueueItem) error + // CountBySource reports how many items of a kind currently resolve from the given sources. + // An empty sources slice means every source; "" matches absent state. + CountBySource(kind Kind, sources []string) (int64, error) + // SourcesInUse lists the distinct sources items of a kind currently resolve from, "" included. + SourcesInUse(kind Kind) ([]string, error) + // EnqueueBySource inserts queue rows for items of a kind whose current source matches. + // It does not clear existing artwork state: the current image stays until it is replaced. + EnqueueBySource(kind Kind, sources []string, priority int) (int64, error) // DequeueBatch returns up to n items with retry_at <= now, priority desc, enqueued_at asc. // Restricted to the given kinds when any are passed, so one kind cannot block another's drain. DequeueBatch(n int, kinds ...string) ([]ArtworkQueueItem, error) @@ -139,6 +149,22 @@ type ArtworkQueueRepository interface { // DeleteIfUnchanged deletes only while retry_at still matches, sparing a concurrent re-enqueue. DeleteIfUnchanged(kind, id, imageType string, retryAt time.Time) error Count() (int64, error) + // CountByKindAndPriority reports the pending queue rows grouped by kind and priority. + CountByKindAndPriority() ([]ArtworkQueueStat, error) + // CountAbsent reports the absent states of a kind, and how many of those EnqueueStaleAbsent + // would pick up at the given cutoff. + CountAbsent(kind Kind, attemptedBefore time.Time) (ArtworkAbsentStat, error) // PurgeDangling removes queue rows whose entity no longer exists. PurgeDangling() (int64, error) } + +type ArtworkQueueStat struct { + ItemKind string + Priority int + Count int64 +} + +type ArtworkAbsentStat struct { + Total int64 + Stale int64 +} diff --git a/persistence/artwork_queue_repository.go b/persistence/artwork_queue_repository.go index e5469fbea..1ff754dc3 100644 --- a/persistence/artwork_queue_repository.go +++ b/persistence/artwork_queue_repository.go @@ -10,6 +10,7 @@ import ( . "github.com/Masterminds/squirrel" "github.com/navidrome/navidrome/model" + "github.com/navidrome/navidrome/utils/slice" "github.com/pocketbase/dbx" ) @@ -31,6 +32,16 @@ func NewArtworkQueueRepository(ctx context.Context, db dbx.Builder) model.Artwor return r } +func (r *artworkQueueRepository) Get(kind model.Kind, id, imageType string) (*model.ArtworkQueueItem, error) { + var res model.ArtworkQueueItem + err := r.queryOne(Select("*").From(r.tableName). + Where(Eq{"item_kind": kind.Prefix(), "item_id": id, "image_type": imageType}), &res) + if err != nil { + return nil, err + } + return &res, nil +} + // Enqueue also resets enqueued_at, so a fresh request does not inherit an old row's spent retry budget. func (r *artworkQueueRepository) Enqueue(items ...model.ArtworkQueueItem) error { return r.enqueue(`ON CONFLICT (item_kind, item_id, image_type) DO UPDATE SET @@ -87,12 +98,49 @@ func (r *artworkQueueRepository) EnqueueIfMissing(items ...model.ArtworkQueueIte return nil } -// insertIfNotQueued inserts the rows selected by the given SQL, optionally prefixed by a CTE. DO NOTHING is -// deliberate: a recheck must not bump the priority or retry_at of an already-queued item. +// DO NOTHING is deliberate: a recheck must not bump the priority or retry_at of an already-queued item. +const skipIfQueued = ` ON CONFLICT (item_kind, item_id, image_type) DO NOTHING` + +// insertIfNotQueued inserts the rows selected by the given SQL, optionally prefixed by a CTE. func (r *artworkQueueRepository) insertIfNotQueued(with, sql string, args ...any) (int64, error) { return r.executeSQL(Expr(with+`INSERT INTO `+r.tableName+ - ` (`+strings.Join(enqueueColumns, ", ")+`) `+sql+ - ` ON CONFLICT (item_kind, item_id, image_type) DO NOTHING`, args...)) + ` (`+strings.Join(enqueueColumns, ", ")+`) `+sql+skipIfQueued, args...)) +} + +// artworkSourceFilter selects item_artwork rows of a kind; no sources means every source, "" the absent state. +func artworkSourceFilter(kind model.Kind, sources []string) Sqlizer { + f := And{Eq{"item_kind": kind.Prefix()}} + if len(sources) > 0 { + f = append(f, Eq{"source": sources}) + } + return f +} + +func (r *artworkQueueRepository) CountBySource(kind model.Kind, sources []string) (int64, error) { + var res struct{ Count int64 } + err := r.queryOne(Select("count(*) as count").From(itemArtworkTable). + Where(artworkSourceFilter(kind, sources)), &res) + return res.Count, err +} + +func (r *artworkQueueRepository) SourcesInUse(kind model.Kind) ([]string, error) { + var res []struct{ Source string } + err := r.queryAll(Select("distinct source").From(itemArtworkTable). + Where(Eq{"item_kind": kind.Prefix()}), &res) + if err != nil { + return nil, err + } + return slice.Map(res, func(s struct{ Source string }) string { return s.Source }), nil +} + +// EnqueueBySource deliberately leaves item_artwork alone: clearing state in bulk would blank the +// library's artwork until every item is resolved again. +func (r *artworkQueueRepository) EnqueueBySource(kind model.Kind, sources []string, priority int) (int64, error) { + now := time.Now() + sel := Select("item_kind", "item_id", "image_type"). + Column(Expr("?", priority)).Column("0").Column(Expr("?", now)).Column(Expr("?", now)). + From(itemArtworkTable).Where(artworkSourceFilter(kind, sources)) + return r.executeSQL(Insert(r.tableName).Columns(enqueueColumns...).Select(sel).Suffix(skipIfQueued)) } func (r *artworkQueueRepository) enqueue(conflict string, items []model.ArtworkQueueItem) error { @@ -146,4 +194,20 @@ func (r *artworkQueueRepository) Count() (int64, error) { return res.Count, err } +func (r *artworkQueueRepository) CountByKindAndPriority() ([]model.ArtworkQueueStat, error) { + var res []model.ArtworkQueueStat + err := r.queryAll(Select("item_kind", "priority", "count(*) as count").From(r.tableName). + GroupBy("item_kind", "priority").OrderBy("item_kind", "priority desc"), &res) + return res, err +} + +// CountAbsent matches EnqueueStaleAbsent on hash, so the stale count is what a recheck would queue. +func (r *artworkQueueRepository) CountAbsent(kind model.Kind, attemptedBefore time.Time) (model.ArtworkAbsentStat, error) { + var res model.ArtworkAbsentStat + err := r.queryOne(Select("count(*) as total"). + Column(Expr("coalesce(sum(attempted_at < ?), 0) as stale", attemptedBefore)). + From(itemArtworkTable).Where(Eq{"item_kind": kind.Prefix(), "hash": ""}), &res) + return res, err +} + var _ model.ArtworkQueueRepository = (*artworkQueueRepository)(nil) diff --git a/persistence/artwork_queue_repository_test.go b/persistence/artwork_queue_repository_test.go index 6638a204e..d11d89a1f 100644 --- a/persistence/artwork_queue_repository_test.go +++ b/persistence/artwork_queue_repository_test.go @@ -54,6 +54,22 @@ var _ = Describe("ArtworkQueueRepository", func() { Expect(got[0].ItemID).To(Equal("high")) }) + It("Get returns a queued row, including one still backing off", func() { + Expect(repo.Enqueue(item("ar", "g1", model.ArtworkPriorityScan))).To(Succeed()) + backOff("ar", "g1", time.Now().Add(time.Hour)) + + got, err := repo.Get(model.KindArtistArtwork, "g1", model.ImageTypePrimary) + Expect(err).ToNot(HaveOccurred()) + Expect(got.Priority).To(Equal(model.ArtworkPriorityScan)) + Expect(got.Attempts).To(Equal(1)) + Expect(got.RetryAt).To(BeTemporally(">", time.Now())) + }) + + It("Get reports ErrNotFound when the item is not queued", func() { + _, err := repo.Get(model.KindArtistArtwork, "nope", model.ImageTypePrimary) + Expect(err).To(MatchError(model.ErrNotFound)) + }) + It("keeps the higher priority on duplicate enqueue", func() { Expect(repo.Enqueue(item("al", "a1", model.ArtworkPriorityBump))).To(Succeed()) Expect(repo.Enqueue(item("al", "a1", model.ArtworkPriorityBackfill))).To(Succeed()) @@ -265,6 +281,89 @@ var _ = Describe("ArtworkQueueRepository", func() { Expect(got[0].Priority).To(Equal(model.ArtworkPriorityBump), "the existing priority must survive") }) + Describe("EnqueueBySource", func() { + BeforeEach(func() { + artRepo := NewArtworkRepository(context.Background(), GetDBXBuilder()) + for _, ia := range []model.ItemArtwork{ + {ItemKind: "ar", ItemID: "ar1", ImageType: model.ImageTypePrimary, Hash: "h1", Source: "external:deezer"}, + {ItemKind: "ar", ItemID: "ar2", ImageType: model.ImageTypePrimary, Hash: "h2", Source: "external:lastfm"}, + {ItemKind: "ar", ItemID: "ar3", ImageType: model.ImageTypePrimary, Hash: "", Source: ""}, + {ItemKind: "al", ItemID: "al1", ImageType: model.ImageTypePrimary, Hash: "h4", Source: "external:deezer"}, + } { + Expect(artRepo.PutItemArtwork(&ia)).To(Succeed()) + } + }) + + It("enqueues only the matching source within the kind", func() { + n, err := repo.EnqueueBySource(model.KindArtistArtwork, []string{"external:deezer"}, model.ArtworkPriorityRecheck) + Expect(err).ToNot(HaveOccurred()) + Expect(n).To(Equal(int64(1)), "al1 is a different kind and must not be touched") + + got, err := repo.DequeueBatch(10) + Expect(err).ToNot(HaveOccurred()) + Expect(slice.Map(got, func(it model.ArtworkQueueItem) string { return it.ItemID })).To(ConsistOf("ar1")) + }) + + It("treats the empty source as absent", func() { + n, err := repo.EnqueueBySource(model.KindArtistArtwork, []string{""}, model.ArtworkPriorityRecheck) + Expect(err).ToNot(HaveOccurred()) + Expect(n).To(Equal(int64(1))) + + got, _ := repo.DequeueBatch(10) + Expect(slice.Map(got, func(it model.ArtworkQueueItem) string { return it.ItemID })).To(ConsistOf("ar3")) + }) + + It("enqueues every source when none is given", func() { + n, err := repo.EnqueueBySource(model.KindArtistArtwork, nil, model.ArtworkPriorityRecheck) + Expect(err).ToNot(HaveOccurred()) + Expect(n).To(Equal(int64(3))) + }) + + It("leaves the current artwork state in place", func() { + _, err := repo.EnqueueBySource(model.KindArtistArtwork, []string{"external:deezer"}, model.ArtworkPriorityRecheck) + Expect(err).ToNot(HaveOccurred()) + + artRepo := NewArtworkRepository(context.Background(), GetDBXBuilder()) + ia, err := artRepo.GetItemArtwork(model.KindArtistArtwork, "ar1", model.ImageTypePrimary) + Expect(err).ToNot(HaveOccurred()) + Expect(ia.Hash).To(Equal("h1"), "the current image must survive until it is replaced") + Expect(ia.Source).To(Equal("external:deezer")) + }) + + It("does not disturb an already-queued row", func() { + Expect(repo.Enqueue(item("ar", "ar1", model.ArtworkPriorityBump))).To(Succeed()) + + n, err := repo.EnqueueBySource(model.KindArtistArtwork, []string{"external:deezer"}, model.ArtworkPriorityRecheck) + Expect(err).ToNot(HaveOccurred()) + Expect(n).To(BeZero()) + + got, _ := repo.DequeueBatch(10) + Expect(got).To(HaveLen(1)) + Expect(got[0].Priority).To(Equal(model.ArtworkPriorityBump)) + }) + + It("counts without enqueueing", func() { + n, err := repo.CountBySource(model.KindArtistArtwork, []string{"external:deezer"}) + Expect(err).ToNot(HaveOccurred()) + Expect(n).To(Equal(int64(1))) + + queued, err := repo.Count() + Expect(err).ToNot(HaveOccurred()) + Expect(queued).To(BeZero(), "CountBySource must not enqueue") + }) + + It("counts the absent source and every source", func() { + Expect(repo.CountBySource(model.KindArtistArtwork, []string{""})).To(Equal(int64(1))) + Expect(repo.CountBySource(model.KindArtistArtwork, nil)).To(Equal(int64(3))) + }) + + It("lists the distinct sources in use by a kind", func() { + Expect(repo.SourcesInUse(model.KindArtistArtwork)).To(ConsistOf("", "external:deezer", "external:lastfm")) + Expect(repo.SourcesInUse(model.KindAlbumArtwork)).To(ConsistOf("external:deezer")) + Expect(repo.SourcesInUse(model.KindRadioArtwork)).To(BeEmpty()) + }) + }) + It("does not disturb an already-queued entity when enqueueing missing rows", func() { Expect(repo.Enqueue(item("al", albumRadioactivity.ID, model.ArtworkPriorityBump))).To(Succeed()) @@ -281,4 +380,43 @@ var _ = Describe("ArtworkQueueRepository", func() { } Expect(count).To(Equal(1), "the already-queued row must not be duplicated") }) + + Describe("status counters", func() { + It("groups queue rows by kind and priority", func() { + Expect(repo.Enqueue(item("ar", "a1", model.ArtworkPriorityBackfill))).To(Succeed()) + Expect(repo.Enqueue(item("ar", "a2", model.ArtworkPriorityBackfill))).To(Succeed()) + Expect(repo.Enqueue(item("ar", "a3", model.ArtworkPriorityBump))).To(Succeed()) + Expect(repo.Enqueue(item("al", "b1", model.ArtworkPriorityScan))).To(Succeed()) + + Expect(repo.CountByKindAndPriority()).To(ConsistOf( + model.ArtworkQueueStat{ItemKind: "ar", Priority: model.ArtworkPriorityBackfill, Count: 2}, + model.ArtworkQueueStat{ItemKind: "ar", Priority: model.ArtworkPriorityBump, Count: 1}, + model.ArtworkQueueStat{ItemKind: "al", Priority: model.ArtworkPriorityScan, Count: 1}, + )) + }) + + It("reports an empty queue as no rows", func() { + Expect(repo.CountByKindAndPriority()).To(BeEmpty()) + }) + + It("counts absent states and how many are due for recheck", func() { + awRepo := NewArtworkRepository(context.Background(), GetDBXBuilder()) + old := time.Now().Add(-48 * time.Hour) + for _, ia := range []model.ItemArtwork{ + {ItemKind: "ar", ItemID: "stale1", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: old}, + {ItemKind: "ar", ItemID: "fresh1", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: time.Now()}, + {ItemKind: "ar", ItemID: "found1", ImageType: model.ImageTypePrimary, Hash: "hX", AttemptedAt: old}, + {ItemKind: "al", ItemID: "stale2", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: old}, + } { + Expect(awRepo.PutItemArtwork(&ia)).To(Succeed()) + } + + Expect(repo.CountAbsent(model.KindArtistArtwork, time.Now().Add(-24*time.Hour))). + To(Equal(model.ArtworkAbsentStat{Total: 2, Stale: 1})) + }) + + It("reports a kind with no absent state as zero, not as an error", func() { + Expect(repo.CountAbsent(model.KindRadioArtwork, time.Now())).To(Equal(model.ArtworkAbsentStat{})) + }) + }) }) diff --git a/server/nativeapi/artwork.go b/server/nativeapi/artwork.go index 3583c2db7..cfd943b1f 100644 --- a/server/nativeapi/artwork.go +++ b/server/nativeapi/artwork.go @@ -10,14 +10,6 @@ import ( "github.com/navidrome/navidrome/model" ) -var refreshableArtworkKinds = []model.Kind{ - model.KindAlbumArtwork, - model.KindArtistArtwork, - model.KindPlaylistArtwork, - model.KindRadioArtwork, - model.KindMediaFileArtwork, -} - func (api *Router) addArtworkRoute(r chi.Router) { r.Post("/artwork/{kind}/{id}/refresh", api.refreshArtwork()) } @@ -28,7 +20,7 @@ func (api *Router) refreshArtwork() http.HandlerFunc { ctx := r.Context() kind, _ := model.ParseKind(chi.URLParam(r, "kind")) id := chi.URLParam(r, "id") - if !slices.Contains(refreshableArtworkKinds, kind) { + if !slices.Contains(artwork.RefreshableKinds, kind) { http.Error(w, "invalid artwork kind", http.StatusBadRequest) return } diff --git a/tests/mock_artwork_queue_repo.go b/tests/mock_artwork_queue_repo.go index 1b097ca32..c8e915daa 100644 --- a/tests/mock_artwork_queue_repo.go +++ b/tests/mock_artwork_queue_repo.go @@ -7,6 +7,7 @@ import ( "time" "github.com/navidrome/navidrome/model" + "github.com/navidrome/navidrome/utils/slice" ) type MockArtworkQueueRepo struct { @@ -25,6 +26,19 @@ func CreateMockArtworkQueueRepo() *MockArtworkQueueRepo { return &MockArtworkQueueRepo{Data: map[string]model.ArtworkQueueItem{}} } +func (m *MockArtworkQueueRepo) Get(kind model.Kind, id, imageType string) (*model.ArtworkQueueItem, error) { + m.mu.Lock() + defer m.mu.Unlock() + if m.Err != nil { + return nil, m.Err + } + it, ok := m.Data[iaKey(kind.Prefix(), id, imageType)] + if !ok { + return nil, model.ErrNotFound + } + return &it, nil +} + func (m *MockArtworkQueueRepo) Enqueue(items ...model.ArtworkQueueItem) error { m.mu.Lock() defer m.mu.Unlock() @@ -161,6 +175,49 @@ func (m *MockArtworkQueueRepo) Count() (int64, error) { return int64(len(m.Data)), nil } +func (m *MockArtworkQueueRepo) CountByKindAndPriority() ([]model.ArtworkQueueStat, error) { + m.mu.Lock() + defer m.mu.Unlock() + if m.Err != nil { + return nil, m.Err + } + var res []model.ArtworkQueueStat + for _, it := range m.Data { + i := slices.IndexFunc(res, func(s model.ArtworkQueueStat) bool { + return s.ItemKind == it.ItemKind && s.Priority == it.Priority + }) + if i < 0 { + res = append(res, model.ArtworkQueueStat{ItemKind: it.ItemKind, Priority: it.Priority, Count: 1}) + continue + } + res[i].Count++ + } + slices.SortFunc(res, func(a, b model.ArtworkQueueStat) int { + return cmp.Or(cmp.Compare(a.ItemKind, b.ItemKind), cmp.Compare(b.Priority, a.Priority)) + }) + return res, nil +} + +// CountAbsent mirrors the SQL predicate: an absent state is one with no hash. +func (m *MockArtworkQueueRepo) CountAbsent(kind model.Kind, attemptedBefore time.Time) (model.ArtworkAbsentStat, error) { + m.mu.Lock() + defer m.mu.Unlock() + var res model.ArtworkAbsentStat + if m.Err != nil || m.ItemArtworkSource == nil { + return res, m.Err + } + for _, ia := range m.ItemArtworkSource.ItemData { + if ia.ItemKind != kind.Prefix() || ia.Hash != "" { + continue + } + res.Total++ + if ia.AttemptedAt.Before(attemptedBefore) { + res.Stale++ + } + } + return res, nil +} + func (m *MockArtworkQueueRepo) EnqueuePreservingBackoff(items ...model.ArtworkQueueItem) error { m.mu.Lock() defer m.mu.Unlock() @@ -216,6 +273,65 @@ func (m *MockArtworkQueueRepo) EnqueueStaleAbsent(kind model.Kind, attemptedBefo return inserted, nil } +// matchingSource mirrors the SQL filter: no sources means every source, "" the absent state. +func (m *MockArtworkQueueRepo) matchingSource(kind model.Kind, sources []string) []model.ItemArtwork { + if m.ItemArtworkSource == nil { + return nil + } + var res []model.ItemArtwork + for _, ia := range m.ItemArtworkSource.ItemData { + if ia.ItemKind == kind.Prefix() && (len(sources) == 0 || slices.Contains(sources, ia.Source)) { + res = append(res, ia) + } + } + return res +} + +func (m *MockArtworkQueueRepo) CountBySource(kind model.Kind, sources []string) (int64, error) { + m.mu.Lock() + defer m.mu.Unlock() + if m.Err != nil { + return 0, m.Err + } + return int64(len(m.matchingSource(kind, sources))), nil +} + +func (m *MockArtworkQueueRepo) SourcesInUse(kind model.Kind) ([]string, error) { + m.mu.Lock() + defer m.mu.Unlock() + if m.Err != nil { + return nil, m.Err + } + sources := slice.Map(m.matchingSource(kind, nil), func(ia model.ItemArtwork) string { return ia.Source }) + return slice.Unique(sources), nil +} + +func (m *MockArtworkQueueRepo) EnqueueBySource(kind model.Kind, sources []string, priority int) (int64, error) { + m.mu.Lock() + defer m.mu.Unlock() + if m.Err != nil { + return 0, m.Err + } + now := time.Now() + var inserted int64 + for _, ia := range m.matchingSource(kind, sources) { + k := iaKey(ia.ItemKind, ia.ItemID, ia.ImageType) + if _, ok := m.Data[k]; ok { // DO NOTHING: never touch existing queue rows + continue + } + m.Data[k] = model.ArtworkQueueItem{ + ItemKind: ia.ItemKind, + ItemID: ia.ItemID, + ImageType: ia.ImageType, + Priority: priority, + RetryAt: now, + EnqueuedAt: now, + } + inserted++ + } + return inserted, nil +} + // EnqueueMissing mirrors the SQL set-difference insert: ExistingIDs[kind] minus ItemArtworkSource. func (m *MockArtworkQueueRepo) EnqueueAllMissing(kind model.Kind, priority int) (int64, error) { m.mu.Lock() From 24311918c7825176b2603ae9a10278283ff7ebfd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Deluan=20Quint=C3=A3o?= Date: Sat, 15 Aug 2026 10:36:14 -0400 Subject: [PATCH 05/27] fix(artwork): ramp the external circuit breaker back up instead of closing on one answer (#5961) * fix(artwork): ramp the external circuit breaker back up instead of closing on one answer The breaker went straight from open to fully closed on a single non-transient response, so recovery was a burst: the agent resumed at the limiter's full rate until five consecutive failures reopened it. A not-found counted as that response, and a provider that is blocking still answers the occasional request, so the cycle never settled. Observed on a production library over 100 minutes with apple-music blocked. Of 232 responses, 228 were 403 and 4 were not-found, and those four closed the breaker four times. Each close was followed by another open 1 to 3 seconds later, with about five requests in between: 00:59:05 closed -> 00:59:06 opened 01:23:13 closed -> 01:23:16 opened 01:41:21 closed -> 01:41:24 opened Closing now needs breakerRecoveries consecutive answers, one per probe interval, and any failure discards the count. A not-found still counts, because the provider did answer, but it can no longer close the breaker by itself. Unrelated to the plugin loading in the rest of this PR; it came out of investigating why iTunes kept returning 403 while the breaker was open. * fix(artwork): count only current-episode probes toward breaker recovery The worker drains concurrently, so when the breaker opens there are already calls past allow(), queued in the rate limiter or waiting on a response. Their answers arrive after the open and reached the recovery counter, so breakerRecoveries of them closed the breaker with no probe interval elapsed at all: the burst the ramp exists to prevent. allow() now returns the open episode a call was admitted under, zero when the breaker was closed, and only an answer whose generation matches the current episode counts. The generation also invalidates a probe whose answer lands after the breaker closed and reopened, which a plain probe flag would credit to the wrong episode. The token never crosses the gateFunc seam: allow and record are both called inside Worker.gate, so passthroughGate, tracingGate and offlineGate are untouched. The regression test needs no fake clock. The race is an ordering, not a duration, so it is reproduced by calling allow and record in the order concurrency produces, which is deterministic where a goroutine-based test would pass on a lucky schedule. Found by Codex. * test(artwork): move the breaker ordering spec into the Ginkgo suite The ordering regression does not need a fake clock, so it does not need the plain testing.T runner either. That runner is only used here because testing/synctest requires it; every other spec belongs in the Ginkgo suite. The three specs left in worker_timing_test.go all drive the fake clock. --- core/artwork/gate.go | 60 ++++++++++++++++++++++-------- core/artwork/gate_test.go | 43 +++++++++++++++++++++ core/artwork/worker_timing_test.go | 59 ++++++++++++++++++++++++----- 3 files changed, 137 insertions(+), 25 deletions(-) create mode 100644 core/artwork/gate_test.go diff --git a/core/artwork/gate.go b/core/artwork/gate.go index ff77f733d..534bd8ce3 100644 --- a/core/artwork/gate.go +++ b/core/artwork/gate.go @@ -17,6 +17,11 @@ import ( const ( breakerThreshold = 5 breakerProbeAfter = time.Minute + // breakerRecoveries is how many consecutive answers an open breaker needs before it trusts the + // provider again. One is not enough: a provider that is rate-limiting or blocking us still + // answers the occasional request, and closing on the first of those puts the agent straight + // back to full rate, which is what earns the next block. + breakerRecoveries = 3 ) var errBreakerOpen = errors.New("artwork: external circuit breaker open") @@ -44,7 +49,8 @@ type extGate struct { // gate runs a named external step through that agent's rate limiter and circuit breaker. func (w *Worker) gate(name string, f func() (io.ReadCloser, string, error)) (io.ReadCloser, string, error) { g := w.gateFor(name) - if !g.breaker.allow() { + allowed, gen := g.breaker.allow() + if !allowed { log.Debug(w.runCtx, "Artwork: Skipping agent, circuit breaker open", "agent", name) return nil, "", errBreakerOpen } @@ -55,7 +61,7 @@ func (w *Worker) gate(name string, f func() (io.ReadCloser, string, error)) (io. } callStart := time.Now() r, path, err := f() - g.breaker.record(name, err) + g.breaker.record(name, gen, err) log.Trace(w.runCtx, "Artwork: External agent call", "agent", name, "hit", r != nil, "limiterWait", callStart.Sub(waitStart), "elapsed", time.Since(callStart), err) return r, path, err @@ -84,41 +90,65 @@ type breaker struct { mu sync.Mutex failures int openedAt time.Time + // recoveries counts consecutive good answers while open; a single failure discards them. + recoveries int + // generation identifies the current open episode, so an answer from a call admitted before + // the breaker opened cannot be mistaken for evidence that it has recovered. + generation int } func newBreaker() *breaker { return &breaker{} } -func (b *breaker) allow() bool { +// allow reports whether a call may proceed, and the open episode it was admitted under: zero +// when the breaker was closed, the current generation when admitted as a half-open probe. +func (b *breaker) allow() (bool, int) { b.mu.Lock() defer b.mu.Unlock() if b.failures < breakerThreshold { - return true + return true, 0 } if time.Since(b.openedAt) >= breakerProbeAfter { b.openedAt = time.Now() // start a fresh probe window so only one caller passes - return true + return true, b.generation } - return false + return false, 0 } -func (b *breaker) record(name string, err error) { +func (b *breaker) record(name string, gen int, err error) { // A cancelled run says nothing about the provider, so it neither counts nor clears. if errors.Is(err, context.Canceled) { return } b.mu.Lock() defer b.mu.Unlock() - if !isTransientExternal(err) { - if b.failures >= breakerThreshold { - log.Info("Artwork: Circuit breaker closed for agent", "agent", name) + if isTransientExternal(err) { + b.recoveries = 0 + b.failures++ + if b.failures == breakerThreshold { + b.openedAt = time.Now() + b.generation++ + log.Warn("Artwork: Circuit breaker opened for agent", "agent", name, + "consecutiveFailures", b.failures, "probeAfter", breakerProbeAfter, err) } + return + } + if b.failures < breakerThreshold { b.failures = 0 return } - b.failures++ - if b.failures == breakerThreshold { - b.openedAt = time.Now() - log.Warn("Artwork: Circuit breaker opened for agent", "agent", name, - "consecutiveFailures", b.failures, "probeAfter", breakerProbeAfter, err) + // Only a probe from this open episode is evidence of recovery. The worker drains concurrently, + // so answers keep arriving from calls admitted before the breaker opened; counting those would + // close it with no probe interval elapsed, which is the burst this exists to prevent. + if gen == 0 || gen != b.generation { + return } + // A not-found counts because the provider did answer, but on its own it is thin evidence that + // a provider which just blocked us is well. + b.recoveries++ + if b.recoveries < breakerRecoveries { + return + } + log.Info("Artwork: Circuit breaker closed for agent", "agent", name, + "consecutiveAnswers", b.recoveries) + b.failures, b.recoveries = 0, 0 } diff --git a/core/artwork/gate_test.go b/core/artwork/gate_test.go new file mode 100644 index 000000000..abe723508 --- /dev/null +++ b/core/artwork/gate_test.go @@ -0,0 +1,43 @@ +package artwork + +import ( + "errors" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// allowed drops the generation token when a caller only cares about admission. +func allowed(b *breaker) bool { ok, _ := b.allow(); return ok } + +var _ = Describe("breaker", func() { + // The worker drains concurrently, so when the breaker opens there are already calls past + // allow(), queued in the rate limiter or waiting on a response. Their answers arrive + // afterwards. Counting those as recovery closes the breaker with no probe interval elapsed, + // which is the burst the ramp exists to prevent. No clock is involved: the race is an + // ordering, so it is reproduced by making the calls in the order concurrency produces. + It("ignores answers from calls admitted before it opened", func() { + b := newBreaker() + + // A batch clears allow() while the breaker is still closed. + for range breakerThreshold + breakerRecoveries { + ok, gen := b.allow() + Expect(ok).To(BeTrue()) + Expect(gen).To(BeZero(), "admitted with the breaker closed, so not a probe") + } + + // The fast failures in that batch open it. + for range breakerThreshold { + b.record("agentA", 0, errors.New("blocked")) + } + Expect(allowed(b)).To(BeFalse(), "breaker is open") + + // The slower answers from the same batch land now. + for range breakerRecoveries { + b.record("agentA", 0, nil) + } + + Expect(allowed(b)).To(BeFalse(), + "answers from calls admitted before the breaker opened must not close it") + }) +}) diff --git a/core/artwork/worker_timing_test.go b/core/artwork/worker_timing_test.go index f7315b20a..63f12d03c 100644 --- a/core/artwork/worker_timing_test.go +++ b/core/artwork/worker_timing_test.go @@ -20,24 +20,63 @@ func TestArtworkBreakerHalfOpen(t *testing.T) { b := newBreaker() for range breakerThreshold { - b.record("agentA", errors.New("boom")) + b.record("agentA", 0, errors.New("boom")) } - g.Expect(b.allow()).To(BeFalse(), "breaker opens after consecutive errors") + g.Expect(allowed(b)).To(BeFalse(), "breaker opens after consecutive errors") time.Sleep(breakerProbeAfter - time.Nanosecond) - g.Expect(b.allow()).To(BeFalse(), "still open before the probe interval") + g.Expect(allowed(b)).To(BeFalse(), "still open before the probe interval") time.Sleep(time.Nanosecond) - g.Expect(b.allow()).To(BeTrue(), "half-open: one probe is granted") - g.Expect(b.allow()).To(BeFalse(), "only a single probe per interval") + ok, gen := b.allow() + g.Expect(ok).To(BeTrue(), "half-open: one probe is granted") + g.Expect(gen).ToNot(BeZero(), "a probe carries the open episode it belongs to") + g.Expect(allowed(b)).To(BeFalse(), "only a single probe per interval") - b.record("agentA", errors.New("boom")) // probe fails -> stay open + b.record("agentA", gen, errors.New("boom")) // probe fails -> stay open time.Sleep(breakerProbeAfter) - g.Expect(b.allow()).To(BeTrue(), "another probe after the next interval") + ok, gen = b.allow() + g.Expect(ok).To(BeTrue(), "another probe after the next interval") - b.record("agentA", nil) // probe succeeds -> close - g.Expect(b.allow()).To(BeTrue(), "closed breaker admits freely") - g.Expect(b.allow()).To(BeTrue()) + // One good answer must not reopen the floodgates: closing here is what let a burst out at + // full rate and got the provider to escalate from throttling to blocking. + b.record("agentA", gen, nil) + g.Expect(allowed(b)).To(BeFalse(), "a single good answer does not close the breaker") + + for range breakerRecoveries - 1 { + time.Sleep(breakerProbeAfter) + ok, gen = b.allow() + g.Expect(ok).To(BeTrue()) + b.record("agentA", gen, nil) + } + g.Expect(allowed(b)).To(BeTrue(), "closed breaker admits freely") + g.Expect(allowed(b)).To(BeTrue()) + }) +} + +// The failure seen in production: while an agent was blocked, the occasional answer it did serve +// reset the breaker, releasing a burst that immediately re-tripped it. Open and closed pairs were +// seconds apart, over and over. +func TestArtworkBreakerDoesNotCloseOnAnIsolatedAnswer(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + g := NewWithT(t) + b := newBreaker() + open := func() { + for range breakerThreshold { + b.record("agentA", 0, errors.New("blocked")) + } + } + open() + + // A not-found is an answer, so it counts toward recovery, but never on its own. + for range breakerRecoveries * 2 { + time.Sleep(breakerProbeAfter) + ok, gen := b.allow() + g.Expect(ok).To(BeTrue(), "one probe per interval") + b.record("agentA", gen, agents.ErrNotFound) + b.record("agentA", 0, errors.New("blocked")) // the very next call is refused again + g.Expect(allowed(b)).To(BeFalse(), "an answer between failures must not close the breaker") + } }) } From 5b87a60b5db07dbf4e4ea93d9766825a91995ef2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Deluan=20Quint=C3=A3o?= Date: Sat, 15 Aug 2026 11:01:36 -0400 Subject: [PATCH 06/27] fix(plugins): load plugin agents in CLI commands (#5959) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(plugins): load plugin agents in CLI commands A CLI that goes through core/agents saw only built-in agents: getEnabledAgentNames asks Manager.PluginNames, which reads a map populated solely by Manager.Start, and only the server calls that. On a plugin-using install the CLI's agent list was quietly short — artwork explain --live could name deezer for an artist whose stored source was external:apple-music, because apple-music was invisible to it. Adds Manager.LoadPlugins, the read-only counterpart to Start: extism/wazero init plus loadEnabledPlugins, without the folder sync, the error clearing, the cache purge or the watcher. It follows what the read-only plugin commands already do — list, info and validate read the DB and never start the manager — except that capabilities are detected from the WASM exports, not declared in the manifest, so instantiating is the only accurate source for what a plugin provides. loadEnabledPlugins disabled a plugin and recorded LastError when a load failed. That is right for the server and wrong for a diagnostic, so it is now gated on the read-only flag: inspecting a plugin must not disable it. No Subsonic router is required. Start log.Fatals without one, but the host function it feeds already nil-checks and reports 'SubsonicAPI router not available' at call time, so a plugin that reaches for it gets an error instead of the process dying. That Fatal's message also claimed the DataStore was missing; it checks the router. * fix(cli): correct the reprocess estimate's plugin caveat imageAgentCount now receives a manager with plugins loaded, so the external estimate already includes plugin image agents — but the disclaimer still said they were not counted, which told operators the opposite of what the number meant. Replaced rather than dropped: loadPluginAgents warns and continues when LoadPlugins fails, and LoadPlugins is a no-op when plugins are disabled or no folder is set, so there are still runs where plugin agents genuinely are not counted. The wording now covers all three cases, and the stale comment above it said the CLI never starts the plugin manager, which is what this branch changed. Found by Codex on 648cf38e9. * fix(plugins): load only the configured agents, and gate plugin init Loading a plugin is not free: the service constructors create a KVStore or TaskQueue database and a Storage directory for any plugin whose manifest declares those permissions, and the plugin's own init then runs arbitrary code. loadEnabledPlugins loads every enabled plugin, so inspecting artwork was starting scrobblers, schedulers and lyrics plugins that could never supply an image. Measured on a copy of a production library: 'artwork explain' created apple-music/kvstore.db, nd-lyrics/kvstore.db and listenbrainz-daily-playlist/taskqueue.db. The last one matters most — CreateQueue resets rows with status='running' to 'pending', which against a live server sets up its in-flight tasks to run twice. LoadPlugins now takes the names to load, and the artwork CLI passes the Agents list: a plugin that is not a configured agent can never win, so there is nothing to gain by instantiating it. The same two runs now create only apple-music, which is a configured agent and therefore the cost of answering the question. Init is gated separately on the caller's intent rather than on read-only. 'explain --live' already means 'reach the provider', so it runs init; plain 'explain' and 'reprocess' promise no external requests and must not. Documented in the --live flag help. Found by Codex on 28eb39ac4. * perf(cli): load plugin agents only when the selection can consult one Explaining disc or media file artwork loaded every configured metadata plugin, though neither resolver ever reaches an agent: resolveMediaFile is embedded-only and discArtworkReader.selectImage refuses external outright. With --live that also ran plugin init for a walk that provably cannot reach the network. The load now sits inside the artist/album branch that already exists, so it is a move rather than a new condition. reprocess did the same for a radio-only selection, whose estimate is unconditionally zero. It is gated on needsImageAgents, which asks exactly what ExternalLookupsPerItem asks, so the two cannot disagree. An artist/album test would look equivalent and would silently zero the playlist estimate, whose generated grid resolves album art through those agents; a test pins that, and reverting the predicate to a whitelist fails it. Found by Codex on b78e67b07. * docs(artwork): trim the comments this branch added to the project budget Six blocks ran past the one-to-two line limit. The LoadPlugins doc was eleven lines over three paragraphs, needsImageAgents spent two of its four explaining an alternative that was rejected, and inspectOpts restated what LoadPlugins already says. What went is reviewer-facing prose that belongs in a commit message: the enumeration of what Start does that this skips, and why an artist/album predicate would have been wrong. What stayed is the reasoning a future reader needs at that line, notably that instantiating a plugin creates its declared services, and that playlists consume the album agent count. * docs(artwork): correct the breaker comment after the recovery ramp It still said a success re-closes the breaker, which stopped being true when closing started requiring breakerRecoveries consecutive answers. * refactor(plugins): rename the scoped-load options to transientLoad inspect claimed the load was only looking, which is false when runInit is true: it instantiates the plugin and runs its init, which may open sockets. transient is accurate for every use of the field, and explains all three behaviours it gates. A load that will not outlive the command has no business persisting findings, instantiating plugins it will never consult, or starting background work it is about to tear down. --- cmd/artwork.go | 63 +++++++++++++--- cmd/artwork_test.go | 49 +++++++++++- core/artwork/gate.go | 2 +- plugins/manager.go | 65 ++++++++++++---- plugins/manager_loader.go | 29 ++++--- plugins/manager_readonly_test.go | 125 +++++++++++++++++++++++++++++++ 6 files changed, 295 insertions(+), 38 deletions(-) create mode 100644 plugins/manager_readonly_test.go diff --git a/cmd/artwork.go b/cmd/artwork.go index 2b6a50164..e9c910f11 100644 --- a/cmd/artwork.go +++ b/cmd/artwork.go @@ -19,6 +19,7 @@ import ( "github.com/navidrome/navidrome/db" "github.com/navidrome/navidrome/log" "github.com/navidrome/navidrome/model" + "github.com/navidrome/navidrome/plugins" "github.com/navidrome/navidrome/utils/slice" "github.com/spf13/cobra" ) @@ -35,7 +36,8 @@ var ( func init() { artworkExplainCmd.Flags().BoolVar(&explainLive, "live", false, - "perform real external lookups instead of reporting what would be tried") + "perform real external lookups instead of reporting what would be tried; "+ + "also initializes plugin agents, which may open external connections") artworkReprocessCmd.Flags().StringSliceVar(&reprocessKinds, "kind", nil, "kinds to reprocess ("+kindPrefixes(artwork.RecheckKinds)+"); repeatable") artworkReprocessCmd.Flags().StringSliceVar(&reprocessSources, "source", nil, @@ -277,7 +279,16 @@ func runReprocess(ctx context.Context) { defer db.Init(ctx)() ds, ctx := getAdminContext(ctx) - if err := reprocessArtwork(ctx, ds, kinds, repositorySources(reprocessSources), imageAgentCount(ds), + // Only a kind that can reach an agent needs the count, and loading a plugin creates its + // services. A preview must not reach the network, so init never runs here. + var imageAgents artwork.ImageAgentCount + if needsImageAgents(kinds) { + mgr := loadPluginAgents(ctx, false) + defer func() { _ = mgr.Stop() }() + imageAgents = imageAgentCount(ds, mgr) + } + + if err := reprocessArtwork(ctx, ds, kinds, repositorySources(reprocessSources), imageAgents, reprocessDryRun, reprocessConfirm(reprocessYes, os.Stdin), os.Stdout); err != nil { log.Fatal(ctx, err) } @@ -326,25 +337,51 @@ func reprocessConfirm(yes bool, in io.Reader) confirmFunc { return promptConfirm(in) } -// externalEstimate claims no bound: a local hit ends the walk before any agent is asked, and plugin -// agents are unregistered in a CLI that never starts the plugin manager. +// externalEstimate claims no bound: a local hit ends the walk before any agent is asked, and the +// plugin agents it counts are only the ones this process managed to load. func externalEstimate(n int64) string { if n == 0 { return "none" } - return fmt.Sprintf("~%d estimated (plugin agents not counted; local hits may need fewer)", n) + return fmt.Sprintf("~%d estimated (plugin agents counted only when they load; local hits may need fewer)", n) } func externalLookupLine(n int64) string { return fmt.Sprintf("External lookups: %s.", externalEstimate(n)) } -// imageAgentCount counts only the built-in image agents, for the same reason. -func imageAgentCount(ds model.DataStore) artwork.ImageAgentCount { - ag := agents.GetAgents(ds, getPluginManager()) +func imageAgentCount(ds model.DataStore, mgr *plugins.Manager) artwork.ImageAgentCount { + ag := agents.GetAgents(ds, mgr) return artwork.ImageAgentCount{Artist: len(ag.ArtistImageAgents()), Album: len(ag.AlbumImageAgents())} } +// loadPluginAgents loads the plugins named in Agents, so the CLI resolves through the same agents a +// running server would. A load failure is reported, not fatal: the built-in agents still answer. +func loadPluginAgents(ctx context.Context, runInit bool) *plugins.Manager { + mgr := getPluginManager() + if err := mgr.LoadPlugins(ctx, configuredAgents(), runInit); err != nil { + log.Warn(ctx, "Could not load plugins; plugin-provided agents will be missing", err) + } + return mgr +} + +// needsImageAgents asks exactly what ExternalLookupsPerItem asks, so the gate cannot disagree with +// the estimate it guards. Playlists count: their generated grid resolves album art through agents. +func needsImageAgents(kinds []model.Kind) bool { + return slices.ContainsFunc(kinds, artwork.MayFetchExternal) +} + +// configuredAgents names the agents in priority order; one absent from it can never supply an image. +func configuredAgents() []string { + var names []string + for name := range strings.SplitSeq(conf.Server.Agents, ",") { + if name = strings.TrimSpace(name); name != "" { + names = append(names, name) + } + } + return names +} + func promptConfirm(in io.Reader) confirmFunc { return func(out io.Writer, total, external int64) bool { var cost string @@ -533,8 +570,8 @@ func explainAgents(configured string, available []string) string { } // availableImageAgents names the agents that can actually supply an image for kind. -func availableImageAgents(ds model.DataStore, kind model.Kind) []string { - ag := agents.GetAgents(ds, getPluginManager()) +func availableImageAgents(ds model.DataStore, mgr *plugins.Manager, kind model.Kind) []string { + ag := agents.GetAgents(ds, mgr) if kind == model.KindArtistArtwork { return slice.Map(ag.ArtistImageAgents(), func(a agents.ArtistImageAgent) string { return a.Name }) } @@ -705,8 +742,12 @@ func runExplain(ctx context.Context, kind model.Kind, id string) { } if artwork.Explainable(kind) { + // Only artist and album reach an agent, and the load must precede the resolver, which reads + // the same manager. if kind == model.KindArtistArtwork || kind == model.KindAlbumArtwork { - rep.agents = explainAgents(conf.Server.Agents, availableImageAgents(ds, kind)) + mgr := loadPluginAgents(ctx, explainLive) + defer func() { _ = mgr.Stop() }() + rep.agents = explainAgents(conf.Server.Agents, availableImageAgents(ds, mgr, kind)) } trace := &artwork.ChainTrace{} rep.source, rep.resolveErr = CreateArtworkResolver(trace, explainLive).Resolve(ctx, kind, id) diff --git a/cmd/artwork_test.go b/cmd/artwork_test.go index 949fdb42a..b28d8dac5 100644 --- a/cmd/artwork_test.go +++ b/cmd/artwork_test.go @@ -594,7 +594,9 @@ var _ = Describe("reprocessArtwork", func() { It("names the estimate's blind spots instead of claiming a bound it cannot hold", func() { Expect(reprocessArtwork(ctx, ds, kinds, nil, imageAgents, true, accept, &out)).To(Succeed()) - Expect(out.String()).To(ContainSubstring("plugin agents not counted")) + // The count includes plugin agents once they load, so the caveat is about a failed load, + // not about plugins being invisible to the CLI. + Expect(out.String()).To(ContainSubstring("plugin agents counted only when they load")) Expect(out.String()).To(ContainSubstring("local hits may need fewer")) Expect(out.String()).ToNot(ContainSubstring("up to"), "plugin agents make any ceiling false") Expect(out.String()).ToNot(ContainSubstring("at least"), "a local hit makes any floor false") @@ -897,3 +899,48 @@ var _ = Describe("refreshItems", func() { "the ids after a failure are still refreshed") }) }) + +var _ = Describe("needsImageAgents", func() { + BeforeEach(func() { + DeferCleanup(configtest.SetupConfig()) + conf.Server.CoverArtPriority = "cover.*, external" + conf.Server.ArtistArtPriority = "artist.*, external" + conf.Server.EnableM3UExternalAlbumArt = false + }) + + It("is false for a selection no agent can serve", func() { + Expect(needsImageAgents([]model.Kind{model.KindRadioArtwork})).To(BeFalse()) + }) + + // The generated playlist grid resolves album art through the image agents, so a playlist + // selection needs the count even though no agent is asked for a playlist image directly. + It("is true for playlists, whose grid tiles resolve through the album chain", func() { + Expect(needsImageAgents([]model.Kind{model.KindPlaylistArtwork})).To(BeTrue()) + }) + + It("is true when any one of several kinds can reach an agent", func() { + Expect(needsImageAgents([]model.Kind{model.KindRadioArtwork, model.KindAlbumArtwork})).To(BeTrue()) + }) + + It("is false once the chains no longer reach an agent", func() { + conf.Server.CoverArtPriority = "cover.*" + conf.Server.ArtistArtPriority = "artist.*" + Expect(needsImageAgents(artwork.RecheckKinds)).To(BeFalse()) + }) +}) + +var _ = Describe("configuredAgents", func() { + BeforeEach(func() { DeferCleanup(configtest.SetupConfig()) }) + + It("splits and trims the configured list", func() { + conf.Server.Agents = "lastfm, spotify ,deezer" + Expect(configuredAgents()).To(Equal([]string{"lastfm", "spotify", "deezer"})) + }) + + // An empty name would match no plugin, but it also must not make the list look non-empty: + // LoadPlugins treats an empty list as "load nothing". + It("drops empty entries rather than passing a name nothing can match", func() { + conf.Server.Agents = " , ," + Expect(configuredAgents()).To(BeEmpty()) + }) +}) diff --git a/core/artwork/gate.go b/core/artwork/gate.go index 534bd8ce3..ebb65bcbe 100644 --- a/core/artwork/gate.go +++ b/core/artwork/gate.go @@ -85,7 +85,7 @@ func (w *Worker) gateFor(name string) *extGate { } // breaker opens after breakerThreshold consecutive errors and admits a single probe once -// breakerProbeAfter has elapsed; a success re-closes it. +// breakerProbeAfter has elapsed; it closes after breakerRecoveries consecutive answers. type breaker struct { mu sync.Mutex failures int diff --git a/plugins/manager.go b/plugins/manager.go index b3a71ce6c..bab67e987 100644 --- a/plugins/manager.go +++ b/plugins/manager.go @@ -61,6 +61,9 @@ type Manager struct { debounceTimers map[string]*time.Timer debounceMu sync.Mutex + // transient is set by LoadPlugins, and nil for a server Start. + transient *transientLoad + // SubsonicAPI host function dependencies (set once before Start, not modified after) subsonicRouter SubsonicRouter ds model.DataStore @@ -110,27 +113,14 @@ func (m *Manager) Start(ctx context.Context) error { } if m.subsonicRouter == nil { - log.Fatal(ctx, "Plugin manager requires DataStore to be configured") + log.Fatal(ctx, "Plugin manager requires the SubsonicAPI router to be configured") } - // Set extism log level based on plugin-specific config or global log level - pluginLogLevel := conf.Server.Plugins.LogLevel - if pluginLogLevel == "" { - pluginLogLevel = conf.Server.LogLevel - } - extism.SetLogLevel(toExtismLogLevel(log.ParseLogLevel(pluginLogLevel))) - - m.ctx, m.cancel = context.WithCancel(ctx) - - // Initialize wazero compilation cache for better performance cacheDir := filepath.Join(conf.Server.CacheFolder.MustPath(), "plugins") purgeCacheBySize(ctx, cacheDir, conf.Server.Plugins.CacheSize) - var err error - m.cache, err = wazero.NewCompilationCacheWithDir(cacheDir) - if err != nil { - log.Error(ctx, "Failed to create wazero compilation cache", err) - return fmt.Errorf("creating wazero compilation cache: %w", err) + if err := m.initRuntime(ctx, cacheDir); err != nil { + return err } if conf.Server.Plugins.Folder.String() == "" { @@ -171,6 +161,49 @@ func (m *Manager) Start(ctx context.Context) error { return nil } +// initRuntime prepares the extism/wazero runtime that instantiating a plugin needs. +func (m *Manager) initRuntime(ctx context.Context, cacheDir string) error { + pluginLogLevel := conf.Server.Plugins.LogLevel + if pluginLogLevel == "" { + pluginLogLevel = conf.Server.LogLevel + } + extism.SetLogLevel(toExtismLogLevel(log.ParseLogLevel(pluginLogLevel))) + + m.ctx, m.cancel = context.WithCancel(ctx) + + var err error + m.cache, err = wazero.NewCompilationCacheWithDir(cacheDir) + if err != nil { + log.Error(ctx, "Failed to create wazero compilation cache", err) + return fmt.Errorf("creating wazero compilation cache: %w", err) + } + return nil +} + +// transientLoad scopes a load that will not outlive the command asking for it; see LoadPlugins. +type transientLoad struct { + only []string + runInit bool +} + +// LoadPlugins loads the plugins named in only, so a CLI sees the agents a server would. Each is +// instantiated, creating any KVStore, TaskQueue or Storage it declares; call Stop when done. +func (m *Manager) LoadPlugins(ctx context.Context, only []string, runInit bool) error { + if !conf.Server.Plugins.Enabled || conf.Server.Plugins.Folder.String() == "" || len(only) == 0 { + return nil + } + m.transient = &transientLoad{only: only, runInit: runInit} + + cacheDir := filepath.Join(conf.Server.CacheFolder.MustPath(), "plugins") + if err := m.initRuntime(ctx, cacheDir); err != nil { + return err + } + if err := m.loadEnabledPlugins(ctx); err != nil { + return fmt.Errorf("loading enabled plugins: %w", err) + } + return nil +} + // Stop shuts down the plugin manager and releases all resources. func (m *Manager) Stop() error { // Mark as stopped first to prevent new operations diff --git a/plugins/manager_loader.go b/plugins/manager_loader.go index 439c944f5..e5e3dbfc0 100644 --- a/plugins/manager_loader.go +++ b/plugins/manager_loader.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "io" + "slices" "time" extism "github.com/extism/go-sdk" @@ -232,6 +233,11 @@ func (m *Manager) loadEnabledPlugins(ctx context.Context) error { if !p.Enabled { continue } + // Instantiating a plugin creates its host services, so a transient load takes only the + // ones it may actually consult. + if m.transient != nil && !slices.Contains(m.transient.only, p.ID) { + continue + } plugin := p // Capture for goroutine g.Go(func() error { @@ -246,19 +252,21 @@ func (m *Manager) loadEnabledPlugins(ctx context.Context) error { }() if err := m.loadPluginWithConfig(&plugin); err != nil { - // Store error in DB - plugin.LastError = err.Error() - plugin.Enabled = false - plugin.UpdatedAt = time.Now() - if putErr := repo.Put(&plugin); putErr != nil { - log.Error(ctx, "Failed to update plugin error in DB", "plugin", plugin.ID, putErr) + // A transient load must not disable the user's plugin just for looking at it. + if m.transient == nil { + plugin.LastError = err.Error() + plugin.Enabled = false + plugin.UpdatedAt = time.Now() + if putErr := repo.Put(&plugin); putErr != nil { + log.Error(ctx, "Failed to update plugin error in DB", "plugin", plugin.ID, putErr) + } } log.Error(ctx, "Failed to load plugin", "plugin", plugin.ID, err) return nil } // Clear any previous error - if plugin.LastError != "" { + if plugin.LastError != "" && m.transient == nil { plugin.LastError = "" plugin.UpdatedAt = time.Now() if putErr := repo.Put(&plugin); putErr != nil { @@ -444,8 +452,11 @@ func (m *Manager) loadPluginWithConfig(p *model.Plugin) error { m.mu.Unlock() loaded = true - // Call plugin init function - callPluginInit(ctx, m.plugins[p.ID]) + // Init is the plugin's first chance to run arbitrary code: open sockets, create task queues, + // schedule work. Only a caller that already intends to reach the network asks for it. + if m.transient == nil || m.transient.runInit { + callPluginInit(ctx, m.plugins[p.ID]) + } return nil } diff --git a/plugins/manager_readonly_test.go b/plugins/manager_readonly_test.go new file mode 100644 index 000000000..9d16b76e9 --- /dev/null +++ b/plugins/manager_readonly_test.go @@ -0,0 +1,125 @@ +//go:build !windows + +package plugins + +import ( + "os" + "path/filepath" + + "github.com/navidrome/navidrome/conf" + "github.com/navidrome/navidrome/conf/configtest" + "github.com/navidrome/navidrome/model" + "github.com/navidrome/navidrome/tests" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Manager.LoadPlugins", func() { + var ( + mgr *Manager + repo *tests.MockPluginRepo + tmpDir string + ) + + // newManager builds a manager over rows the caller can corrupt, with no Subsonic router: a CLI + // has none, and Start would log.Fatal on that. + newManager := func(rows model.Plugins) *Manager { + DeferCleanup(configtest.SetupConfig()) + var err error + tmpDir, err = os.MkdirTemp("", "plugins-readonly-*") + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = os.RemoveAll(tmpDir) }) + + conf.Server.Plugins.Enabled = true + conf.Server.Plugins.Folder = conf.NewDir(tmpDir) + conf.Server.Plugins.AutoReload = false + conf.Server.CacheFolder = conf.NewDir(tmpDir) + + if rows == nil { + rows = installTestPlugins(tmpDir, "test-metadata-agent"+PackageExtension) + for i := range rows { + rows[i].AllUsers = true + } + } + repo = tests.CreateMockPluginRepo() + repo.Permitted = true + repo.SetData(rows) + m := &Manager{ + plugins: make(map[string]*plugin), + ds: &tests.MockDataStore{MockedPlugin: repo}, + metrics: noopMetricsRecorder{}, + } + DeferCleanup(func() { _ = m.Stop() }) + return m + } + + It("detects capabilities without a Subsonic router configured", func() { + mgr = newManager(nil) + + Expect(mgr.LoadPlugins(GinkgoT().Context(), []string{"test-metadata-agent", "broken"}, false)).To(Succeed()) + + Expect(mgr.PluginNames(string(CapabilityMetadataAgent))).To(ContainElement("test-metadata-agent")) + }) + + Context("when a plugin cannot be loaded", func() { + brokenRows := func() model.Plugins { + return model.Plugins{{ + ID: "broken", Path: filepath.Join(GinkgoT().TempDir(), "does-not-exist.ndp"), + Enabled: true, AllUsers: true, + }} + } + + It("leaves the stored row untouched", func() { + mgr = newManager(brokenRows()) + + Expect(mgr.LoadPlugins(GinkgoT().Context(), []string{"test-metadata-agent", "broken"}, false)).To(Succeed()) + + stored, err := repo.Get("broken") + Expect(err).ToNot(HaveOccurred()) + Expect(stored.Enabled).To(BeTrue(), "inspecting a plugin must never disable it") + Expect(stored.LastError).To(BeEmpty()) + }) + + // Without this the test above would pass for the wrong reason. Start cannot be used: it + // syncs the folder first, dropping a row whose file is missing before any load. + It("still disables it when not read-only", func() { + mgr = newManager(brokenRows()) + + Expect(mgr.loadEnabledPlugins(GinkgoT().Context())).To(Succeed()) + + stored, err := repo.Get("broken") + Expect(err).ToNot(HaveOccurred()) + Expect(stored.Enabled).To(BeFalse()) + Expect(stored.LastError).ToNot(BeEmpty()) + }) + }) + + // Loading a plugin creates its host services — a KVStore or task queue database on disk — so a + // plugin that could never supply an image must not be instantiated just to be ignored. + It("does not load a plugin that is not in the agent list", func() { + mgr = newManager(nil) + + Expect(mgr.LoadPlugins(GinkgoT().Context(), []string{"some-other-agent"}, false)).To(Succeed()) + + Expect(mgr.PluginNames(string(CapabilityMetadataAgent))).To(BeEmpty()) + }) + + It("does nothing when no agents are configured", func() { + mgr = newManager(nil) + + Expect(mgr.LoadPlugins(GinkgoT().Context(), nil, false)).To(Succeed()) + + Expect(mgr.PluginNames(string(CapabilityMetadataAgent))).To(BeEmpty()) + // Not even the wazero cache: with nothing to load there is nothing to compile. + Expect(filepath.Join(tmpDir, "plugins")).ToNot(BeADirectory()) + }) + + It("does nothing when the plugin system is disabled", func() { + mgr = newManager(nil) + conf.Server.Plugins.Enabled = false + + Expect(mgr.LoadPlugins(GinkgoT().Context(), []string{"test-metadata-agent", "broken"}, false)).To(Succeed()) + + Expect(mgr.PluginNames(string(CapabilityMetadataAgent))).To(BeEmpty()) + }) +}) From 82fde00eccfb36c89fd722b1dabcc8d2a902aebd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Deluan=20Quint=C3=A3o?= Date: Sat, 15 Aug 2026 16:10:53 -0400 Subject: [PATCH 07/27] feat(scrobbler): add per-user scrobble filter (#5964) * feat(scrobbler): add scrobble_filter column to user * feat(scrobbler): validate scrobble filter criteria on user save * refactor(persistence): make smart playlist join helpers package-level * feat(scrobbler): add MediaFileRepository.MatchesCriteria * feat(scrobbler): filter external scrobbles with per-user criteria * feat(ui): add scrobble filter field to user form * fix(scrobbler): default scrobble_filter to empty string for existing users * refactor(scrobbler): also gate playback reports on the scrobble filter Playback reports carry the same track metadata to plugin scrobblers, so a filtered track leaked through that third dispatch path. Skip the filter evaluation entirely when no scrobbler is active. * refactor(persistence): move criteria join building into criteria_sql.go The join set a criteria needs was decided in criteria_sql.go but built in smart_playlist_repository.go, so both callers had to pair the two by hand. * refactor(persistence): unexport smartPlaylistCriteria methods The type never leaves the package, so the exported names advertised an API that callers outside persistence could never reach. Also disambiguates where/orderBy from squirrel's SelectBuilder methods of the same name. * fix(ui): cap the scrobble filter field width fullWidth stretched it across the whole page next to 256px inputs. Bounded at 40em, with two rows and a resize handle so JSON rules stay readable. * refactor(ui): move scrobble filter input in UserEdit component * feat(ui): add pt-BR translations for the scrobble filter * fix(scrobbler): take the filter verdict before incPlay incPlay mutates play counts and dates a filter can test on, so evaluating at dispatch time let one play decide differently on either side of the increment: a track could be scrobbled despite matching, or lose only its stopped report and strand presence plugins. Reject limit/offset too, rather than silently ignoring part of a rule copied from a smart playlist. * fix(scrobbler): filter the report from an expired session The expiry callback runs with a stub user carrying no filter, so evaluating there always returned false and leaked the track to plugin scrobblers. That is the normal path for clients that never send stopped, such as legacy Subsonic now-playing. Carry the last verdict on the session instead. * refactor(scrobbler): skip the now-playing enqueue instead of threading the verdict Queuing an entry only to drop it at dispatch also cancelled a pending announcement for the previous, unfiltered track, since the queue is keyed by player and a new entry replaces the old one. * fix(scrobbler): evaluate the filter regardless of active scrobblers The verdict is stored on the session and dispatched at expiry, so skipping evaluation when no scrobbler was active let a plugin enabled mid-session receive a filtered track. The empty-filter guard above already gives servers without scrobbling the same free path, so the shortcut only ever applied to users who had a filter set. --- core/scrobbler/play_tracker.go | 69 +++++- core/scrobbler/play_tracker_test.go | 200 ++++++++++++++++++ core/scrobbler/playbackreport_worker.go | 15 +- ...0260815015320_add_user_scrobble_filter.sql | 5 + model/mediafile.go | 4 + model/user.go | 2 + persistence/criteria_sql.go | 45 +++- persistence/criteria_sql_benchmark_test.go | 4 +- persistence/criteria_sql_test.go | 88 ++++---- persistence/mediafile_repository.go | 18 ++ persistence/mediafile_repository_test.go | 28 +++ persistence/smart_playlist_repository.go | 46 +--- persistence/user_repository.go | 34 +++ persistence/user_repository_test.go | 66 ++++++ resources/i18n/pt-br.json | 9 +- tests/mock_mediafile_repo.go | 10 + ui/src/i18n/en.json | 9 +- ui/src/user/UserEdit.jsx | 11 + ui/src/user/UserEdit.test.jsx | 6 + ui/src/user/userValidation.js | 10 + ui/src/user/userValidation.test.js | 23 ++ 21 files changed, 590 insertions(+), 112 deletions(-) create mode 100644 db/migrations/20260815015320_add_user_scrobble_filter.sql diff --git a/core/scrobbler/play_tracker.go b/core/scrobbler/play_tracker.go index e21db42d2..63397f8f6 100644 --- a/core/scrobbler/play_tracker.go +++ b/core/scrobbler/play_tracker.go @@ -2,6 +2,7 @@ package scrobbler import ( "context" + "encoding/json" "maps" "slices" "sync" @@ -11,6 +12,7 @@ import ( "github.com/navidrome/navidrome/consts" "github.com/navidrome/navidrome/log" "github.com/navidrome/navidrome/model" + "github.com/navidrome/navidrome/model/criteria" "github.com/navidrome/navidrome/model/request" "github.com/navidrome/navidrome/server/events" "github.com/navidrome/navidrome/utils/cache" @@ -43,6 +45,10 @@ type PlaybackSession struct { PositionMs int64 PlaybackRate float64 LastReport time.Time + + // Verdict from the last report, for the expiry callback: its context carries only + // a stub user, so it cannot evaluate the filter itself. + filtered bool } type Submission struct { @@ -68,8 +74,9 @@ type nowPlayingEntry struct { } type playbackReportEntry struct { - ctx context.Context - info PlaybackSession + ctx context.Context + info PlaybackSession + filtered bool } type PlayTracker interface { @@ -145,7 +152,7 @@ func newPlayTracker(ds model.DataStore, broker events.Broker, pluginManager Plug log.Trace("Enqueueing PlaybackReport for expired session", "session", info) info.State = StateExpired info.LastReport = time.Now() - p.enqueuePlaybackReport(ctx, info) + p.enqueuePlaybackReport(ctx, info, info.filtered) } }) @@ -273,6 +280,10 @@ func (p *playTracker) ReportPlayback(ctx context.Context, params ReportPlaybackP now := time.Now() + // One verdict per report, reused by every dispatch below, so a filter reading + // annotations cannot decide differently on either side of incPlay. + var filtered bool + switch params.State { case StateStarting: // Clients may send starting/playing unordered; a late "starting" must not downgrade @@ -285,8 +296,10 @@ func (p *playTracker) ReportPlayback(ctx context.Context, params ReportPlaybackP if err != nil { return err } + filtered = p.isFilteredOut(ctx, mf) info := PlaybackSession{ MediaFile: *mf, + filtered: filtered, Start: now, UserId: user.ID, Username: user.UserName, @@ -309,7 +322,7 @@ func (p *playTracker) ReportPlayback(ctx context.Context, params ReportPlaybackP if err != nil { log.Warn(ctx, "Error adding PlaybackSession to cache", "clientId", clientId, "mediaId", params.MediaId, "state", params.State, err) } - p.enqueuePlaybackReport(ctx, info) + p.enqueuePlaybackReport(ctx, info, filtered) case StatePlaying, StatePaused: info, getErr := p.playMap.Get(clientId) @@ -331,6 +344,8 @@ func (p *playTracker) ReportPlayback(ctx context.Context, params ReportPlaybackP info.PositionMs = params.PositionMs info.PlaybackRate = params.PlaybackRate info.LastReport = now + filtered = p.isFilteredOut(ctx, &info.MediaFile) + info.filtered = filtered ttl := 30 * time.Minute if params.State == StatePlaying { ttl = remainingTTL(info.MediaFile.Duration, params.PositionMs, params.PlaybackRate) @@ -342,16 +357,19 @@ func (p *playTracker) ReportPlayback(ctx context.Context, params ReportPlaybackP if err != nil { log.Warn(ctx, "Error updating PlaybackSession in cache", "clientId", clientId, "mediaId", params.MediaId, "state", params.State, err) } - p.enqueuePlaybackReport(ctx, info) + p.enqueuePlaybackReport(ctx, info, filtered) case StateStopped: var loadedMF *model.MediaFile + haveVerdict := false if !params.IgnoreScrobble && player.ScrobbleEnabled { mf, err := p.ds.MediaFile(ctx).GetWithParticipants(params.MediaId) if err != nil { return err } loadedMF = mf + filtered = p.isFilteredOut(ctx, mf) + haveVerdict = true trackDurationMs := int64(mf.Duration * 1000) threshold := min(trackDurationMs*50/100, 240_000) if params.PositionMs >= threshold { @@ -359,7 +377,7 @@ func (p *playTracker) ReportPlayback(ctx context.Context, params ReportPlaybackP if err != nil { log.Warn(ctx, "Error updating play counts", "id", mf.ID, "track", mf.Title, "user", user.UserName, err) } - p.dispatchScrobble(ctx, mf, now) + p.dispatchScrobble(ctx, mf, now, filtered) } } p.sessionsMu.Lock() @@ -397,7 +415,10 @@ func (p *playTracker) ReportPlayback(ctx context.Context, params ReportPlaybackP } stoppedInfo.MediaFile = *mf } - p.enqueuePlaybackReport(ctx, stoppedInfo) + if !haveVerdict { + filtered = p.isFilteredOut(ctx, &stoppedInfo.MediaFile) + } + p.enqueuePlaybackReport(ctx, stoppedInfo, filtered) } if conf.Server.EnableNowPlaying { @@ -413,7 +434,9 @@ func (p *playTracker) ReportPlayback(ctx context.Context, params ReportPlaybackP // scrobbler plugins) returned by getActiveScrobblers; see dispatchNowPlaying. if player.ScrobbleEnabled && (params.State == StateStarting || params.State == StatePlaying) { - if info, err := p.playMap.Get(clientId); err == nil { + if filtered { + log.Debug(ctx, "Ignoring external NowPlaying update for filtered track", "mediaId", params.MediaId) + } else if info, err := p.playMap.Get(clientId); err == nil { p.enqueueNowPlaying(ctx, clientId, user.ID, &info.MediaFile, int(params.PositionMs/1000)) } } @@ -452,6 +475,7 @@ func (p *playTracker) Submit(ctx context.Context, submissions []Submission) erro log.Error(ctx, "Cannot find track for scrobbling", "id", s.TrackID, "user", username, err) continue } + filtered := p.isFilteredOut(ctx, mf) err = p.incPlay(ctx, mf, s.Timestamp) if err != nil { log.Error(ctx, "Error updating play counts", "id", mf.ID, "track", mf.Title, "user", username, err) @@ -460,7 +484,7 @@ func (p *playTracker) Submit(ctx context.Context, submissions []Submission) erro event.With("song", mf.ID).With("album", mf.AlbumID).With("artist", mf.AlbumArtistID) log.Info(ctx, "Scrobbled", "title", mf.Title, "artist", mf.Artist, "user", username, "timestamp", s.Timestamp) if player.ScrobbleEnabled { - p.dispatchScrobble(ctx, mf, s.Timestamp) + p.dispatchScrobble(ctx, mf, s.Timestamp, filtered) } } } @@ -494,11 +518,36 @@ func (p *playTracker) incPlay(ctx context.Context, track *model.MediaFile, times }) } -func (p *playTracker) dispatchScrobble(ctx context.Context, t *model.MediaFile, playTime time.Time) { +// Take this verdict before incPlay mutates what a filter reads, and independently of +// which scrobblers are active: it can be stored on a session and dispatched much later. +// Any parse or query failure fails open, because filtering must not break scrobbling. +func (p *playTracker) isFilteredOut(ctx context.Context, t *model.MediaFile) bool { + u, _ := request.UserFrom(ctx) + if u.ScrobbleFilter == "" { + return false + } + var c criteria.Criteria + if err := json.Unmarshal([]byte(u.ScrobbleFilter), &c); err != nil { + log.Warn(ctx, "Invalid scrobble filter, ignoring", "user", u.UserName, err) + return false + } + match, err := p.ds.MediaFile(ctx).MatchesCriteria(t.ID, c) + if err != nil { + log.Warn(ctx, "Error evaluating scrobble filter, ignoring", "user", u.UserName, "track", t.Title, err) + return false + } + return match +} + +func (p *playTracker) dispatchScrobble(ctx context.Context, t *model.MediaFile, playTime time.Time, filtered bool) { if t.Artist == consts.UnknownArtist { log.Debug(ctx, "Ignoring external Scrobble for track with unknown artist", "track", t.Title, "artist", t.Artist) return } + if filtered { + log.Debug(ctx, "Ignoring external Scrobble for filtered track", "track", t.Title, "artist", t.Artist) + return + } allScrobblers := p.getActiveScrobblers() u, _ := request.UserFrom(ctx) diff --git a/core/scrobbler/play_tracker_test.go b/core/scrobbler/play_tracker_test.go index f49d9a0bf..72f802c2e 100644 --- a/core/scrobbler/play_tracker_test.go +++ b/core/scrobbler/play_tracker_test.go @@ -14,6 +14,7 @@ import ( "github.com/navidrome/navidrome/consts" "github.com/navidrome/navidrome/model" + "github.com/navidrome/navidrome/model/criteria" "github.com/navidrome/navidrome/model/request" "github.com/navidrome/navidrome/server/events" "github.com/navidrome/navidrome/tests" @@ -46,6 +47,26 @@ func (m *mockPluginLoader) LoadScrobbler(name string) (Scrobbler, bool) { return s, ok } +// flipOnPlayRepo reports one filter verdict before the play is counted and another +// after, reproducing a filter that reads annotations incPlay mutates. +type flipOnPlayRepo struct { + model.MediaFileRepository + before, after bool + played atomic.Bool +} + +func (r *flipOnPlayRepo) IncPlayCount(id string, ts time.Time) error { + r.played.Store(true) + return r.MediaFileRepository.IncPlayCount(id, ts) +} + +func (r *flipOnPlayRepo) MatchesCriteria(string, criteria.Criteria) (bool, error) { + if r.played.Load() { + return r.after, nil + } + return r.before, nil +} + // slowMediaFileRepo widens the window between a report's session check and its // write, making check-then-write races reproducible. type slowMediaFileRepo struct { @@ -320,6 +341,185 @@ var _ = Describe("PlayTracker", func() { }) }) + Describe("Scrobble filter", func() { + var repo *tests.MockMediaFileRepo + + BeforeEach(func() { + ctx = request.WithUser(ctx, model.User{ID: "u-1", UserName: "user-1", + ScrobbleFilter: `{"all":[{"contains":{"title":"Track"}}]}`}) + repo = ds.MediaFile(ctx).(*tests.MockMediaFileRepo) + }) + + It("does not send a matching track to the agent", func() { + repo.MatchesCriteriaValue = true + + err := tracker.Submit(ctx, []Submission{{TrackID: "123", Timestamp: time.Now()}}) + + Expect(err).ToNot(HaveOccurred()) + Expect(fake.ScrobbleCalled.Load()).To(BeFalse()) + }) + + It("still increments play counts for a filtered track", func() { + repo.MatchesCriteriaValue = true + + err := tracker.Submit(ctx, []Submission{{TrackID: "123", Timestamp: time.Now()}}) + + Expect(err).ToNot(HaveOccurred()) + Expect(track.PlayCount).To(Equal(int64(1))) + Expect(album.PlayCount).To(Equal(int64(1))) + }) + + It("sends a non-matching track to the agent", func() { + repo.MatchesCriteriaValue = false + + err := tracker.Submit(ctx, []Submission{{TrackID: "123", Timestamp: time.Now()}}) + + Expect(err).ToNot(HaveOccurred()) + Expect(fake.ScrobbleCalled.Load()).To(BeTrue()) + }) + + It("fails open when evaluation errors", func() { + repo.MatchesCriteriaErr = errors.New("boom") + + err := tracker.Submit(ctx, []Submission{{TrackID: "123", Timestamp: time.Now()}}) + + Expect(err).ToNot(HaveOccurred()) + Expect(fake.ScrobbleCalled.Load()).To(BeTrue()) + }) + + It("fails open when the stored filter is not valid JSON", func() { + ctx = request.WithUser(ctx, model.User{ID: "u-1", UserName: "user-1", ScrobbleFilter: `{broken`}) + repo.MatchesCriteriaValue = true + + err := tracker.Submit(ctx, []Submission{{TrackID: "123", Timestamp: time.Now()}}) + + Expect(err).ToNot(HaveOccurred()) + Expect(fake.ScrobbleCalled.Load()).To(BeTrue()) + }) + + It("does not send now-playing for a filtered track", func() { + repo.MatchesCriteriaValue = true + + err := tracker.ReportPlayback(ctx, ReportPlaybackParams{ + MediaId: "123", State: StateStarting, ClientId: "player-1", ClientName: "player"}) + + Expect(err).ToNot(HaveOccurred()) + Consistently(func() bool { return fake.GetNowPlayingCalled() }).Should(BeFalse()) + }) + + It("does not send playback reports for a filtered track", func() { + repo.MatchesCriteriaValue = true + + err := tracker.ReportPlayback(ctx, ReportPlaybackParams{ + MediaId: "123", State: StateStarting, ClientId: "player-1", ClientName: "player"}) + + Expect(err).ToNot(HaveOccurred()) + Consistently(func() bool { return fake.PlaybackReportCalled.Load() }).Should(BeFalse()) + }) + + It("sends playback reports for a non-matching track", func() { + repo.MatchesCriteriaValue = false + + err := tracker.ReportPlayback(ctx, ReportPlaybackParams{ + MediaId: "123", State: StateStarting, ClientId: "player-1", ClientName: "player"}) + + Expect(err).ToNot(HaveOccurred()) + Eventually(func() bool { return fake.PlaybackReportCalled.Load() }).Should(BeTrue()) + }) + + It("evaluates the filter even when no scrobbler is active yet", func() { + // The verdict is stored on the session and dispatched at expiry, by which + // time a plugin scrobbler may have been enabled. + tracker.builtinScrobblers = map[string]Scrobbler{} + repo.MatchesCriteriaValue = true + + err := tracker.ReportPlayback(ctx, ReportPlaybackParams{ + MediaId: "123", State: StatePlaying, ClientId: "player-12", ClientName: "player"}) + + Expect(err).ToNot(HaveOccurred()) + stored, getErr := tracker.playMap.Get("player-12") + Expect(getErr).ToNot(HaveOccurred()) + Expect(stored.filtered).To(BeTrue()) + }) + + Context("when incPlay itself flips the filter", func() { + // A filter on playCount or lastPlayed changes verdict the moment incPlay + // commits, so the verdict has to be taken before it, not at dispatch time. + var flip *flipOnPlayRepo + + install := func(before, after bool) { + flip = &flipOnPlayRepo{MediaFileRepository: ds.MediaFile(ctx), before: before, after: after} + ds.(*tests.MockDataStore).MockedMediaFile = flip + } + + It("does not scrobble a track the filter matched before the play was counted", func() { + install(true, false) + + err := tracker.ReportPlayback(ctx, ReportPlaybackParams{ + MediaId: "123", State: StateStopped, PositionMs: 120_000, ClientId: "player-1", ClientName: "player"}) + + Expect(err).ToNot(HaveOccurred()) + Expect(flip.played.Load()).To(BeTrue(), "incPlay must still have run") + Expect(fake.ScrobbleCalled.Load()).To(BeFalse()) + }) + + It("still sends the stopped report when the filter only starts matching after the play", func() { + install(false, true) + + err := tracker.ReportPlayback(ctx, ReportPlaybackParams{ + MediaId: "123", State: StateStopped, PositionMs: 120_000, ClientId: "player-1", ClientName: "player"}) + + Expect(err).ToNot(HaveOccurred()) + Expect(flip.played.Load()).To(BeTrue(), "incPlay must still have run") + Eventually(func() bool { return fake.PlaybackReportCalled.Load() }).Should(BeTrue()) + }) + + It("does not report an expired session for a filtered track", func() { + // The expiry callback runs with a stub user, so it cannot evaluate the + // filter itself and must reuse the verdict stored on the session. + info := PlaybackSession{ + MediaFile: track, Start: time.Now(), UserId: "u-1", Username: "user-1", + PlayerId: "player-9", PlayerName: "test-player", State: StatePlaying, filtered: true, + } + _ = tracker.playMap.AddWithTTL("player-9", info, 10*time.Millisecond) + + Consistently(func() bool { return fake.PlaybackReportCalled.Load() }).Should(BeFalse()) + }) + + It("still reports an expired session for a track that is not filtered", func() { + info := PlaybackSession{ + MediaFile: track, Start: time.Now(), UserId: "u-1", Username: "user-1", + PlayerId: "player-10", PlayerName: "test-player", State: StatePlaying, filtered: false, + } + _ = tracker.playMap.AddWithTTL("player-10", info, 10*time.Millisecond) + + Eventually(func() bool { return fake.PlaybackReportCalled.Load() }).Should(BeTrue()) + }) + + It("stores the verdict on the session so expiry can reuse it", func() { + repo.MatchesCriteriaValue = true + + err := tracker.ReportPlayback(ctx, ReportPlaybackParams{ + MediaId: "123", State: StatePlaying, ClientId: "player-11", ClientName: "player"}) + + Expect(err).ToNot(HaveOccurred()) + stored, getErr := tracker.playMap.Get("player-11") + Expect(getErr).ToNot(HaveOccurred()) + Expect(stored.filtered).To(BeTrue()) + }) + + It("does not scrobble a Submit whose filter matched before the play was counted", func() { + install(true, false) + + err := tracker.Submit(ctx, []Submission{{TrackID: "123", Timestamp: time.Now()}}) + + Expect(err).ToNot(HaveOccurred()) + Expect(flip.played.Load()).To(BeTrue(), "incPlay must still have run") + Expect(fake.ScrobbleCalled.Load()).To(BeFalse()) + }) + }) + }) + Describe("ReportPlayback", func() { const defaultClientId = "client-1" diff --git a/core/scrobbler/playbackreport_worker.go b/core/scrobbler/playbackreport_worker.go index 78ca6e0f7..799dac616 100644 --- a/core/scrobbler/playbackreport_worker.go +++ b/core/scrobbler/playbackreport_worker.go @@ -6,13 +6,14 @@ import ( "github.com/navidrome/navidrome/log" ) -func (p *playTracker) enqueuePlaybackReport(ctx context.Context, info PlaybackSession) { +func (p *playTracker) enqueuePlaybackReport(ctx context.Context, info PlaybackSession, filtered bool) { p.prMu.Lock() defer p.prMu.Unlock() ctx = context.WithoutCancel(ctx) p.prQueue = append(p.prQueue, playbackReportEntry{ - ctx: ctx, - info: info, + ctx: ctx, + info: info, + filtered: filtered, }) p.sendPlaybackReportSignal() } @@ -44,12 +45,16 @@ func (p *playTracker) playbackReportWorker() { allScrobblers := p.getActiveScrobblers() for _, entry := range entries { - p.dispatchPlaybackReport(entry.ctx, entry.info, allScrobblers) + p.dispatchPlaybackReport(entry.ctx, entry.info, allScrobblers, entry.filtered) } } } -func (p *playTracker) dispatchPlaybackReport(ctx context.Context, info PlaybackSession, allScrobblers map[string]Scrobbler) { +func (p *playTracker) dispatchPlaybackReport(ctx context.Context, info PlaybackSession, allScrobblers map[string]Scrobbler, filtered bool) { + if filtered { + log.Debug(ctx, "Ignoring external PlaybackReport for filtered track", "track", info.MediaFile.Title, "state", info.State) + return + } for name, s := range allScrobblers { if !s.IsAuthorized(ctx, info.UserId) { continue diff --git a/db/migrations/20260815015320_add_user_scrobble_filter.sql b/db/migrations/20260815015320_add_user_scrobble_filter.sql new file mode 100644 index 000000000..82ed95f3d --- /dev/null +++ b/db/migrations/20260815015320_add_user_scrobble_filter.sql @@ -0,0 +1,5 @@ +-- +goose Up +alter table user add column scrobble_filter varchar default '' not null; + +-- +goose Down +alter table user drop column scrobble_filter; diff --git a/model/mediafile.go b/model/mediafile.go index ac1fad67b..888425d07 100644 --- a/model/mediafile.go +++ b/model/mediafile.go @@ -15,6 +15,7 @@ import ( "github.com/gohugoio/hashstructure" "github.com/navidrome/navidrome/conf" "github.com/navidrome/navidrome/consts" + "github.com/navidrome/navidrome/model/criteria" "github.com/navidrome/navidrome/utils" "github.com/navidrome/navidrome/utils/gg" "github.com/navidrome/navidrome/utils/number" @@ -548,6 +549,9 @@ type MediaFileRepository interface { // filters as GetAll. Sort/Order are ignored. GetRandom(options ...QueryOptions) (MediaFiles, error) GetAllByTags(tag TagName, values []string, options ...QueryOptions) (MediaFiles, error) + // MatchesCriteria reports whether the media file matches the criteria's rule + // expression, using the logged user's annotations. Limit and offset are ignored. + MatchesCriteria(id string, c criteria.Criteria) (bool, error) GetCursor(options ...QueryOptions) (MediaFileCursor, error) // GetAllIDs returns just the media_file IDs for the same row set as GetAll. GetAllIDs(options ...QueryOptions) ([]string, error) diff --git a/model/user.go b/model/user.go index 1c8541ccf..b6f792c9a 100644 --- a/model/user.go +++ b/model/user.go @@ -14,6 +14,8 @@ type User struct { LastAccessAt *time.Time `structs:"last_access_at" json:"lastAccessAt"` CreatedAt time.Time `structs:"created_at" json:"createdAt"` UpdatedAt time.Time `structs:"updated_at" json:"updatedAt"` + // Smart-playlist criteria JSON; matching songs are not sent to external scrobblers + ScrobbleFilter string `structs:"scrobble_filter" json:"scrobbleFilter"` // Library associations (many-to-many relationship) Libraries Libraries `structs:"-" json:"libraries,omitempty"` diff --git a/persistence/criteria_sql.go b/persistence/criteria_sql.go index 70f9a1fb5..890f757d7 100644 --- a/persistence/criteria_sql.go +++ b/persistence/criteria_sql.go @@ -129,7 +129,7 @@ var smartPlaylistFields = map[string]smartPlaylistField{ "random": {order: "random()"}, } -func (c smartPlaylistCriteria) Where() (squirrel.Sqlizer, error) { +func (c smartPlaylistCriteria) where() (squirrel.Sqlizer, error) { if c.Criteria.Expression == nil { return squirrel.Expr("1 = 1"), nil } @@ -786,7 +786,7 @@ func fieldJoinType(name string) smartPlaylistJoinType { return field.joinType } -func (c smartPlaylistCriteria) ExpressionJoins() smartPlaylistJoinType { +func (c smartPlaylistCriteria) expressionJoins() smartPlaylistJoinType { var joins smartPlaylistJoinType _ = criteria.Walk(c.Criteria.Expression, func(expr criteria.Expression) error { for field := range criteria.Fields(expr) { @@ -797,15 +797,50 @@ func (c smartPlaylistCriteria) ExpressionJoins() smartPlaylistJoinType { return joins } -func (c smartPlaylistCriteria) RequiredJoins() smartPlaylistJoinType { - joins := c.ExpressionJoins() +func (c smartPlaylistCriteria) requiredJoins() smartPlaylistJoinType { + joins := c.expressionJoins() for _, name := range c.Criteria.SortFieldNames() { joins |= fieldJoinType(name) } return joins } -func (c smartPlaylistCriteria) OrderBy() string { +// applyExpressionJoins adds every join the criteria's WHERE clause resolves against. +func (c smartPlaylistCriteria) applyExpressionJoins(sq squirrel.SelectBuilder, userID string) squirrel.SelectBuilder { + return c.applyJoins(sq, c.expressionJoins(), userID) +} + +// applyRequiredJoins adds the WHERE joins plus any the ORDER BY resolves against. +func (c smartPlaylistCriteria) applyRequiredJoins(sq squirrel.SelectBuilder, userID string) squirrel.SelectBuilder { + return c.applyJoins(sq, c.requiredJoins(), userID) +} + +// applyJoins joins the media_file annotation unconditionally — annotation fields +// COALESCE a missing row to a default, so the row has to be reachable to be absent. +func (c smartPlaylistCriteria) applyJoins(sq squirrel.SelectBuilder, joins smartPlaylistJoinType, userID string) squirrel.SelectBuilder { + sq = sq.LeftJoin("annotation on ("+ + "annotation.item_id = media_file.id"+ + " AND annotation.item_type = 'media_file'"+ + " AND annotation.user_id = ?)", userID) + if joins.has(smartPlaylistJoinAlbumAnnotation) { + sq = sq.LeftJoin("annotation AS album_annotation ON ("+ + "album_annotation.item_id = media_file.album_id"+ + " AND album_annotation.item_type = 'album'"+ + " AND album_annotation.user_id = ?)", userID) + } + if joins.has(smartPlaylistJoinArtistAnnotation) { + sq = sq.LeftJoin("annotation AS artist_annotation ON ("+ + "artist_annotation.item_id = media_file.artist_id"+ + " AND artist_annotation.item_type = 'artist'"+ + " AND artist_annotation.user_id = ?)", userID) + } + if joins.has(smartPlaylistJoinAlbum) { + sq = sq.LeftJoin("album ON album.id = media_file.album_id") + } + return sq +} + +func (c smartPlaylistCriteria) orderBy() string { sortFields := c.Criteria.OrderByFields() parts := make([]string, 0, len(sortFields)) for _, sf := range sortFields { diff --git a/persistence/criteria_sql_benchmark_test.go b/persistence/criteria_sql_benchmark_test.go index 1dcf97871..523a5d825 100644 --- a/persistence/criteria_sql_benchmark_test.go +++ b/persistence/criteria_sql_benchmark_test.go @@ -124,7 +124,7 @@ func benchmarkCriteriaPipeline(b *testing.B, ctx context.Context, expr criteria. // Build the full query matching buildSmartPlaylistQuery + addCriteria sq := squirrel.Select("media_file.id").From("media_file") - cond, err := cSQL.Where() + cond, err := cSQL.where() if err != nil { b.Fatal(err) } @@ -132,7 +132,7 @@ func benchmarkCriteriaPipeline(b *testing.B, ctx context.Context, expr criteria. if expr.Limit > 0 { sq = sq.Limit(uint64(expr.Limit)) } - if order := cSQL.OrderBy(); order != "" { + if order := cSQL.orderBy(); order != "" { sq = sq.OrderBy(order) } diff --git a/persistence/criteria_sql_test.go b/persistence/criteria_sql_test.go index 0cd7187c5..8d0069b0d 100644 --- a/persistence/criteria_sql_test.go +++ b/persistence/criteria_sql_test.go @@ -20,7 +20,7 @@ var _ = Describe("Smart playlist criteria SQL", func() { DescribeTable("expressions", func(expr criteria.Expression, expectedSQL string, expectedArgs ...any) { - sqlizer, err := newSmartPlaylistCriteria(criteria.Criteria{Expression: expr}).Where() + sqlizer, err := newSmartPlaylistCriteria(criteria.Criteria{Expression: expr}).where() Expect(err).ToNot(HaveOccurred()) sql, args, err := sqlizer.ToSql() @@ -204,7 +204,7 @@ var _ = Describe("Smart playlist criteria SQL", func() { sqlizer, err := newSmartPlaylistCriteria( criteria.Criteria{Expression: criteria.InPlaylist{"id": "deadbeef-dead-beef"}}, withSmartPlaylistOwner(model.User{ID: "owner-id", IsAdmin: false}), - ).Where() + ).where() Expect(err).ToNot(HaveOccurred()) sql, args, err := sqlizer.ToSql() @@ -217,7 +217,7 @@ var _ = Describe("Smart playlist criteria SQL", func() { sqlizer, err := newSmartPlaylistCriteria( criteria.Criteria{Expression: criteria.InPlaylist{"id": "deadbeef-dead-beef"}}, withSmartPlaylistOwner(model.User{ID: "admin-id", IsAdmin: true}), - ).Where() + ).where() Expect(err).ToNot(HaveOccurred()) sql, args, err := sqlizer.ToSql() @@ -228,7 +228,7 @@ var _ = Describe("Smart playlist criteria SQL", func() { }) It("builds relative date expressions", func() { - sqlizer, err := newSmartPlaylistCriteria(criteria.Criteria{Expression: criteria.InTheLast{"lastPlayed": 30}}).Where() + sqlizer, err := newSmartPlaylistCriteria(criteria.Criteria{Expression: criteria.InTheLast{"lastPlayed": 30}}).where() Expect(err).ToNot(HaveOccurred()) sql, args, err := sqlizer.ToSql() @@ -238,7 +238,7 @@ var _ = Describe("Smart playlist criteria SQL", func() { }) It("builds negated relative date expressions", func() { - sqlizer, err := newSmartPlaylistCriteria(criteria.Criteria{Expression: criteria.NotInTheLast{"lastPlayed": 30}}).Where() + sqlizer, err := newSmartPlaylistCriteria(criteria.Criteria{Expression: criteria.NotInTheLast{"lastPlayed": 30}}).where() Expect(err).ToNot(HaveOccurred()) sql, args, err := sqlizer.ToSql() @@ -248,76 +248,76 @@ var _ = Describe("Smart playlist criteria SQL", func() { }) It("returns an error for unknown fields", func() { - _, err := newSmartPlaylistCriteria(criteria.Criteria{Expression: criteria.EndsWith{"unknown": "value"}}).Where() + _, err := newSmartPlaylistCriteria(criteria.Criteria{Expression: criteria.EndsWith{"unknown": "value"}}).where() Expect(err).To(MatchError("invalid field in criteria: unknown")) }) It("returns an error when isMissing is used with a regular field", func() { - _, err := newSmartPlaylistCriteria(criteria.Criteria{Expression: criteria.IsMissing{"year": true}}).Where() + _, err := newSmartPlaylistCriteria(criteria.Criteria{Expression: criteria.IsMissing{"year": true}}).where() Expect(err).To(MatchError(ContainSubstring("isMissing/isPresent operator is not supported for field"))) }) It("returns an error when isPresent is used with a regular field", func() { - _, err := newSmartPlaylistCriteria(criteria.Criteria{Expression: criteria.IsPresent{"title": true}}).Where() + _, err := newSmartPlaylistCriteria(criteria.Criteria{Expression: criteria.IsPresent{"title": true}}).where() Expect(err).To(MatchError(ContainSubstring("isMissing/isPresent operator is not supported for field"))) }) It("returns an error when isMissing has a non-boolean value", func() { - _, err := newSmartPlaylistCriteria(criteria.Criteria{Expression: criteria.IsMissing{"genre": "hello"}}).Where() + _, err := newSmartPlaylistCriteria(criteria.Criteria{Expression: criteria.IsMissing{"genre": "hello"}}).where() Expect(err).To(MatchError(ContainSubstring("invalid boolean value for 'missing' expression"))) }) It("returns an error for a range over a tag/role field", func() { - _, err := newSmartPlaylistCriteria(criteria.Criteria{Expression: criteria.InTheRange{"rate": []int{1, 5}}}).Where() + _, err := newSmartPlaylistCriteria(criteria.Criteria{Expression: criteria.InTheRange{"rate": []int{1, 5}}}).where() Expect(err).To(MatchError(ContainSubstring("range operator not supported for tag/role field"))) }) It("returns a clear error for a malformed range value", func() { - _, err := newSmartPlaylistCriteria(criteria.Criteria{Expression: criteria.InTheRange{"playCount": []int{1, 2, 3}}}).Where() + _, err := newSmartPlaylistCriteria(criteria.Criteria{Expression: criteria.InTheRange{"playCount": []int{1, 2, 3}}}).where() Expect(err).To(MatchError(ContainSubstring("must be a [min, max] pair"))) }) Describe("sort", func() { It("sorts by regular fields", func() { - Expect(newSmartPlaylistCriteria(criteria.Criteria{Sort: "title"}).OrderBy()).To(Equal("media_file.title asc")) + Expect(newSmartPlaylistCriteria(criteria.Criteria{Sort: "title"}).orderBy()).To(Equal("media_file.title asc")) }) It("sorts by tag fields", func() { - Expect(newSmartPlaylistCriteria(criteria.Criteria{Sort: "genre"}).OrderBy()).To(Equal("COALESCE(json_extract(media_file.tags, '$.genre[0].value'), '') asc")) + Expect(newSmartPlaylistCriteria(criteria.Criteria{Sort: "genre"}).orderBy()).To(Equal("COALESCE(json_extract(media_file.tags, '$.genre[0].value'), '') asc")) }) It("sorts by role fields", func() { - Expect(newSmartPlaylistCriteria(criteria.Criteria{Sort: "artist"}).OrderBy()).To(Equal("COALESCE(json_extract(media_file.participants, '$.artist[0].name'), '') asc")) + Expect(newSmartPlaylistCriteria(criteria.Criteria{Sort: "artist"}).orderBy()).To(Equal("COALESCE(json_extract(media_file.participants, '$.artist[0].name'), '') asc")) }) It("casts numeric tags when sorting", func() { - Expect(newSmartPlaylistCriteria(criteria.Criteria{Sort: "rate"}).OrderBy()).To(Equal("CAST(COALESCE(json_extract(media_file.tags, '$.rate[0].value'), '') AS REAL) asc")) + Expect(newSmartPlaylistCriteria(criteria.Criteria{Sort: "rate"}).orderBy()).To(Equal("CAST(COALESCE(json_extract(media_file.tags, '$.rate[0].value'), '') AS REAL) asc")) }) It("sorts by albumtype alias", func() { - Expect(newSmartPlaylistCriteria(criteria.Criteria{Sort: "albumtype"}).OrderBy()).To(Equal("COALESCE(json_extract(media_file.tags, '$.releasetype[0].value'), '') asc")) + Expect(newSmartPlaylistCriteria(criteria.Criteria{Sort: "albumtype"}).orderBy()).To(Equal("COALESCE(json_extract(media_file.tags, '$.releasetype[0].value'), '') asc")) }) It("sorts by random", func() { - Expect(newSmartPlaylistCriteria(criteria.Criteria{Sort: "random"}).OrderBy()).To(Equal("random() asc")) + Expect(newSmartPlaylistCriteria(criteria.Criteria{Sort: "random"}).orderBy()).To(Equal("random() asc")) }) It("sorts by album columns bare, with no COALESCE default", func() { - Expect(newSmartPlaylistCriteria(criteria.Criteria{Sort: "-albumDateAdded,trackNumber"}).OrderBy()). + Expect(newSmartPlaylistCriteria(criteria.Criteria{Sort: "-albumDateAdded,trackNumber"}).orderBy()). To(Equal("album.created_at desc, media_file.track_number asc")) }) It("sorts by multiple fields", func() { - Expect(newSmartPlaylistCriteria(criteria.Criteria{Sort: "title,-rating"}).OrderBy()).To(Equal("media_file.title asc, COALESCE(annotation.rating, 0) desc")) + Expect(newSmartPlaylistCriteria(criteria.Criteria{Sort: "title,-rating"}).orderBy()).To(Equal("media_file.title asc, COALESCE(annotation.rating, 0) desc")) }) It("reverts order when order is desc", func() { - Expect(newSmartPlaylistCriteria(criteria.Criteria{Sort: "-date,artist", Order: "desc"}).OrderBy()).To(Equal("media_file.date asc, COALESCE(json_extract(media_file.participants, '$.artist[0].name'), '') desc")) + Expect(newSmartPlaylistCriteria(criteria.Criteria{Sort: "-date,artist", Order: "desc"}).orderBy()).To(Equal("media_file.date asc, COALESCE(json_extract(media_file.participants, '$.artist[0].name'), '') desc")) }) It("ignores invalid sort fields", func() { - Expect(newSmartPlaylistCriteria(criteria.Criteria{Sort: "bogus,title"}).OrderBy()).To(Equal("media_file.title asc")) + Expect(newSmartPlaylistCriteria(criteria.Criteria{Sort: "bogus,title"}).orderBy()).To(Equal("media_file.title asc")) }) }) @@ -362,7 +362,7 @@ var _ = Describe("Smart playlist criteria SQL", func() { criteria.Contains{"artist": "Kraftwerk"}, criteria.Contains{"artist": "Pink Floyd"}, } - sqlizer, err := newSmartPlaylistCriteria(criteria.Criteria{Expression: expr}).Where() + sqlizer, err := newSmartPlaylistCriteria(criteria.Criteria{Expression: expr}).where() Expect(err).ToNot(HaveOccurred()) sql, args, err := sqlizer.ToSql() @@ -376,7 +376,7 @@ var _ = Describe("Smart playlist criteria SQL", func() { criteria.Contains{"artist": "Beatles"}, criteria.Contains{"composer": "Lennon"}, } - sqlizer, err := newSmartPlaylistCriteria(criteria.Criteria{Expression: expr}).Where() + sqlizer, err := newSmartPlaylistCriteria(criteria.Criteria{Expression: expr}).where() Expect(err).ToNot(HaveOccurred()) sql, _, err := sqlizer.ToSql() @@ -391,7 +391,7 @@ var _ = Describe("Smart playlist criteria SQL", func() { criteria.NotContains{"artist": "Beatles"}, criteria.NotContains{"artist": "Kraftwerk"}, } - sqlizer, err := newSmartPlaylistCriteria(criteria.Criteria{Expression: expr}).Where() + sqlizer, err := newSmartPlaylistCriteria(criteria.Criteria{Expression: expr}).where() Expect(err).ToNot(HaveOccurred()) sql, _, err := sqlizer.ToSql() @@ -406,7 +406,7 @@ var _ = Describe("Smart playlist criteria SQL", func() { for i := range anyExprs { anyExprs[i] = criteria.Contains{"artist": fmt.Sprintf("Artist%d", i)} } - sqlizer, err := newSmartPlaylistCriteria(criteria.Criteria{Expression: anyExprs}).Where() + sqlizer, err := newSmartPlaylistCriteria(criteria.Criteria{Expression: anyExprs}).where() Expect(err).ToNot(HaveOccurred()) sql, args, err := sqlizer.ToSql() @@ -424,7 +424,7 @@ var _ = Describe("Smart playlist criteria SQL", func() { criteria.Contains{"artist": "Beatles"}, criteria.Contains{"artist": "Kraftwerk"}, } - sqlizer, err := newSmartPlaylistCriteria(criteria.Criteria{Expression: expr}).Where() + sqlizer, err := newSmartPlaylistCriteria(criteria.Criteria{Expression: expr}).where() Expect(err).ToNot(HaveOccurred()) sql, args, err := sqlizer.ToSql() @@ -440,7 +440,7 @@ var _ = Describe("Smart playlist criteria SQL", func() { criteria.Contains{"genre": "Metal"}, criteria.Contains{"genre": "Punk"}, } - sqlizer, err := newSmartPlaylistCriteria(criteria.Criteria{Expression: expr}).Where() + sqlizer, err := newSmartPlaylistCriteria(criteria.Criteria{Expression: expr}).where() Expect(err).ToNot(HaveOccurred()) sql, args, err := sqlizer.ToSql() @@ -454,7 +454,7 @@ var _ = Describe("Smart playlist criteria SQL", func() { criteria.Contains{"genre": "Rock"}, criteria.Contains{"mood": "Happy"}, } - sqlizer, err := newSmartPlaylistCriteria(criteria.Criteria{Expression: expr}).Where() + sqlizer, err := newSmartPlaylistCriteria(criteria.Criteria{Expression: expr}).where() Expect(err).ToNot(HaveOccurred()) sql, _, err := sqlizer.ToSql() @@ -467,7 +467,7 @@ var _ = Describe("Smart playlist criteria SQL", func() { criteria.NotContains{"genre": "Rock"}, criteria.NotContains{"genre": "Metal"}, } - sqlizer, err := newSmartPlaylistCriteria(criteria.Criteria{Expression: expr}).Where() + sqlizer, err := newSmartPlaylistCriteria(criteria.Criteria{Expression: expr}).where() Expect(err).ToNot(HaveOccurred()) sql, _, err := sqlizer.ToSql() @@ -482,7 +482,7 @@ var _ = Describe("Smart playlist criteria SQL", func() { criteria.Contains{"genre": "Rock"}, criteria.Contains{"genre": "Metal"}, } - sqlizer, err := newSmartPlaylistCriteria(criteria.Criteria{Expression: expr}).Where() + sqlizer, err := newSmartPlaylistCriteria(criteria.Criteria{Expression: expr}).where() Expect(err).ToNot(HaveOccurred()) sql, args, err := sqlizer.ToSql() @@ -499,7 +499,7 @@ var _ = Describe("Smart playlist criteria SQL", func() { criteria.IsNot{"artist": "Beatles"}, criteria.IsNot{"artist": "Kraftwerk"}, } - sqlizer, err := newSmartPlaylistCriteria(criteria.Criteria{Expression: expr}).Where() + sqlizer, err := newSmartPlaylistCriteria(criteria.Criteria{Expression: expr}).where() Expect(err).ToNot(HaveOccurred()) sql, args, err := sqlizer.ToSql() @@ -515,7 +515,7 @@ var _ = Describe("Smart playlist criteria SQL", func() { criteria.NotContains{"artist": "Beatles"}, criteria.NotContains{"artist": "Kraftwerk"}, } - sqlizer, err := newSmartPlaylistCriteria(criteria.Criteria{Expression: expr}).Where() + sqlizer, err := newSmartPlaylistCriteria(criteria.Criteria{Expression: expr}).where() Expect(err).ToNot(HaveOccurred()) sql, args, err := sqlizer.ToSql() @@ -530,7 +530,7 @@ var _ = Describe("Smart playlist criteria SQL", func() { criteria.NotContains{"genre": "Rock"}, criteria.NotContains{"genre": "Metal"}, } - sqlizer, err := newSmartPlaylistCriteria(criteria.Criteria{Expression: expr}).Where() + sqlizer, err := newSmartPlaylistCriteria(criteria.Criteria{Expression: expr}).where() Expect(err).ToNot(HaveOccurred()) sql, args, err := sqlizer.ToSql() @@ -547,7 +547,7 @@ var _ = Describe("Smart playlist criteria SQL", func() { criteria.Contains{"artist": "Beatles"}, criteria.IsNot{"artist": "Kraftwerk"}, } - sqlizer, err := newSmartPlaylistCriteria(criteria.Criteria{Expression: expr}).Where() + sqlizer, err := newSmartPlaylistCriteria(criteria.Criteria{Expression: expr}).where() Expect(err).ToNot(HaveOccurred()) sql, _, err := sqlizer.ToSql() @@ -562,7 +562,7 @@ var _ = Describe("Smart playlist criteria SQL", func() { criteria.IsNot{"artist": "Beatles"}, criteria.IsNot{"composer": "Lennon"}, } - sqlizer, err := newSmartPlaylistCriteria(criteria.Criteria{Expression: expr}).Where() + sqlizer, err := newSmartPlaylistCriteria(criteria.Criteria{Expression: expr}).where() Expect(err).ToNot(HaveOccurred()) sql, _, err := sqlizer.ToSql() @@ -575,7 +575,7 @@ var _ = Describe("Smart playlist criteria SQL", func() { for i := range allExprs { allExprs[i] = criteria.IsNot{"artist": fmt.Sprintf("Artist%d", i)} } - sqlizer, err := newSmartPlaylistCriteria(criteria.Criteria{Expression: allExprs}).Where() + sqlizer, err := newSmartPlaylistCriteria(criteria.Criteria{Expression: allExprs}).where() Expect(err).ToNot(HaveOccurred()) sql, args, err := sqlizer.ToSql() @@ -591,14 +591,14 @@ var _ = Describe("Smart playlist criteria SQL", func() { c := criteria.Criteria{Expression: criteria.All{criteria.Contains{"title": "love"}}, Sort: "albumRating"} cSQL := newSmartPlaylistCriteria(c) - Expect(cSQL.ExpressionJoins()).To(Equal(smartPlaylistJoinNone)) - Expect(cSQL.RequiredJoins().has(smartPlaylistJoinAlbumAnnotation)).To(BeTrue()) + Expect(cSQL.expressionJoins()).To(Equal(smartPlaylistJoinNone)) + Expect(cSQL.requiredJoins().has(smartPlaylistJoinAlbumAnnotation)).To(BeTrue()) }) It("includes expression-based joins", func() { c := criteria.Criteria{Expression: criteria.All{criteria.Gt{"albumRating": 3}}} - Expect(newSmartPlaylistCriteria(c).ExpressionJoins().has(smartPlaylistJoinAlbumAnnotation)).To(BeTrue()) + Expect(newSmartPlaylistCriteria(c).expressionJoins().has(smartPlaylistJoinAlbumAnnotation)).To(BeTrue()) }) It("detects nested album and artist joins", func() { @@ -607,7 +607,7 @@ var _ = Describe("Smart playlist criteria SQL", func() { criteria.Any{criteria.Gt{"artistPlayCount": 10}}, }} - joins := newSmartPlaylistCriteria(c).RequiredJoins() + joins := newSmartPlaylistCriteria(c).requiredJoins() Expect(joins.has(smartPlaylistJoinAlbumAnnotation)).To(BeTrue()) Expect(joins.has(smartPlaylistJoinArtistAnnotation)).To(BeTrue()) }) @@ -615,20 +615,20 @@ var _ = Describe("Smart playlist criteria SQL", func() { It("detects join types from sort fields with direction prefixes", func() { c := criteria.Criteria{Expression: criteria.All{criteria.Contains{"title": "love"}}, Sort: "-artistRating"} - Expect(newSmartPlaylistCriteria(c).RequiredJoins().has(smartPlaylistJoinArtistAnnotation)).To(BeTrue()) + Expect(newSmartPlaylistCriteria(c).requiredJoins().has(smartPlaylistJoinArtistAnnotation)).To(BeTrue()) }) It("keeps a sort-only album join out of the expression joins", func() { c := criteria.Criteria{Expression: criteria.All{criteria.Contains{"title": "love"}}, Sort: "-albumDateAdded"} cSQL := newSmartPlaylistCriteria(c) - Expect(cSQL.ExpressionJoins()).To(Equal(smartPlaylistJoinNone)) - Expect(cSQL.RequiredJoins().has(smartPlaylistJoinAlbum)).To(BeTrue()) + Expect(cSQL.expressionJoins()).To(Equal(smartPlaylistJoinNone)) + Expect(cSQL.requiredJoins().has(smartPlaylistJoinAlbum)).To(BeTrue()) }) It("distinguishes the album join from the album annotation join", func() { c := criteria.Criteria{Expression: criteria.All{criteria.Gt{"albumRating": 3}}} - joins := newSmartPlaylistCriteria(c).RequiredJoins() + joins := newSmartPlaylistCriteria(c).requiredJoins() Expect(joins.has(smartPlaylistJoinAlbumAnnotation)).To(BeTrue()) Expect(joins.has(smartPlaylistJoinAlbum)).To(BeFalse()) diff --git a/persistence/mediafile_repository.go b/persistence/mediafile_repository.go index f37e394cc..5f0addc5e 100644 --- a/persistence/mediafile_repository.go +++ b/persistence/mediafile_repository.go @@ -15,6 +15,7 @@ import ( "github.com/navidrome/navidrome/conf" "github.com/navidrome/navidrome/log" "github.com/navidrome/navidrome/model" + "github.com/navidrome/navidrome/model/criteria" "github.com/navidrome/navidrome/utils/slice" "github.com/navidrome/navidrome/utils/str" "github.com/pocketbase/dbx" @@ -517,6 +518,23 @@ var mediaFileSearchConfig = searchConfig{ MBIDFields: []string{"mbz_recording_id", "mbz_release_track_id"}, } +func (r *mediaFileRepository) MatchesCriteria(id string, c criteria.Criteria) (bool, error) { + usr := loggedUser(r.ctx) + rulesSQL := newSmartPlaylistCriteria(c, withSmartPlaylistOwner(*usr)) + cond, err := rulesSQL.where() + if err != nil { + return false, err + } + sq := Select("count(*) as count").From("media_file") + sq = rulesSQL.applyExpressionJoins(sq, usr.ID) + sq = sq.Where(And{Eq{"media_file.id": id}, cond}) + var res struct{ Count int64 } + if err := r.queryOne(sq, &res); err != nil { + return false, err + } + return res.Count > 0, nil +} + func (r *mediaFileRepository) Search(q string, options ...model.QueryOptions) (model.MediaFiles, error) { var opts model.QueryOptions if len(options) > 0 { diff --git a/persistence/mediafile_repository_test.go b/persistence/mediafile_repository_test.go index b9cba03ff..ba89b5def 100644 --- a/persistence/mediafile_repository_test.go +++ b/persistence/mediafile_repository_test.go @@ -13,6 +13,7 @@ import ( "github.com/navidrome/navidrome/conf/configtest" "github.com/navidrome/navidrome/log" "github.com/navidrome/navidrome/model" + "github.com/navidrome/navidrome/model/criteria" "github.com/navidrome/navidrome/model/id" "github.com/navidrome/navidrome/model/request" . "github.com/onsi/ginkgo/v2" @@ -1129,4 +1130,31 @@ var _ = Describe("MediaRepository", func() { Expect(NewMediaFileRepository(rctx, GetDBXBuilder()).Exists(songAntenna.ID)).To(BeFalse()) }) }) + + Describe("MatchesCriteria", func() { + It("returns true when the track matches", func() { + c := criteria.Criteria{Expression: criteria.All{criteria.Contains{"title": "Day"}}} + match, err := mr.MatchesCriteria(songDayInALife.ID, c) + Expect(err).ToNot(HaveOccurred()) + Expect(match).To(BeTrue()) + }) + It("returns false when the track does not match", func() { + c := criteria.Criteria{Expression: criteria.All{criteria.Contains{"title": "Nickelback"}}} + match, err := mr.MatchesCriteria(songDayInALife.ID, c) + Expect(err).ToNot(HaveOccurred()) + Expect(match).To(BeFalse()) + }) + It("treats missing annotations as their COALESCE default", func() { + // unrated track: rating coalesces to 0, so "rating < 4" matches + c := criteria.Criteria{Expression: criteria.All{criteria.Lt{"rating": 4}}} + match, err := mr.MatchesCriteria(songDayInALife.ID, c) + Expect(err).ToNot(HaveOccurred()) + Expect(match).To(BeTrue()) + }) + It("returns an error for an invalid field", func() { + c := criteria.Criteria{Expression: criteria.All{criteria.Is{"bogusfield": 1}}} + _, err := mr.MatchesCriteria(songDayInALife.ID, c) + Expect(err).To(HaveOccurred()) + }) + }) }) diff --git a/persistence/smart_playlist_repository.go b/persistence/smart_playlist_repository.go index 5f84981b3..65ae4656b 100644 --- a/persistence/smart_playlist_repository.go +++ b/persistence/smart_playlist_repository.go @@ -119,13 +119,11 @@ func (r *playlistRepository) resolvePercentageLimit(pls *model.Playlist, rulesSQ return nil } - exprJoins := rulesSQL.ExpressionJoins() countSq := Select("count(*) as count").From("media_file") - countSq = r.addMediaFileAnnotationJoin(countSq, userID) - countSq = r.addSmartPlaylistJoins(countSq, exprJoins, userID) + countSq = rulesSQL.applyExpressionJoins(countSq, userID) countSq = r.applyLibraryFilter(countSq, "media_file") - cond, err := rulesSQL.Where() + cond, err := rulesSQL.where() if err != nil { log.Error(r.ctx, "Error building smart playlist criteria", "playlist", pls.Name, "id", pls.ID, err) return err @@ -146,50 +144,18 @@ func (r *playlistRepository) resolvePercentageLimit(pls *model.Playlist, rulesSQ // buildSmartPlaylistQuery constructs the SQL query to select media files matching the smart playlist criteria, // including the joins its fields require and library filtering. func (r *playlistRepository) buildSmartPlaylistQuery(pls *model.Playlist, rulesSQL smartPlaylistCriteria, userID string) SelectBuilder { - orderBy := rulesSQL.OrderBy() + orderBy := rulesSQL.orderBy() sq := Select("row_number() over (order by "+orderBy+") as id", "'"+pls.ID+"' as playlist_id", "media_file.id as media_file_id"). From("media_file") - sq = r.addMediaFileAnnotationJoin(sq, userID) - - requiredJoins := rulesSQL.RequiredJoins() - sq = r.addSmartPlaylistJoins(sq, requiredJoins, userID) + sq = rulesSQL.applyRequiredJoins(sq, userID) sq = r.applyLibraryFilter(sq, "media_file") return sq } -// addMediaFileAnnotationJoin adds a left join to the annotation table for media files, filtering by user ID to include -// user-specific annotations in the smart playlist criteria evaluation. -func (r *playlistRepository) addMediaFileAnnotationJoin(sq SelectBuilder, userID string) SelectBuilder { - return sq.LeftJoin("annotation on ("+ - "annotation.item_id = media_file.id"+ - " AND annotation.item_type = 'media_file'"+ - " AND annotation.user_id = ?)", userID) -} - -// addSmartPlaylistJoins adds the left joins required by the criteria's fields. -func (r *playlistRepository) addSmartPlaylistJoins(sq SelectBuilder, joins smartPlaylistJoinType, userID string) SelectBuilder { - if joins.has(smartPlaylistJoinAlbumAnnotation) { - sq = sq.LeftJoin("annotation AS album_annotation ON ("+ - "album_annotation.item_id = media_file.album_id"+ - " AND album_annotation.item_type = 'album'"+ - " AND album_annotation.user_id = ?)", userID) - } - if joins.has(smartPlaylistJoinArtistAnnotation) { - sq = sq.LeftJoin("annotation AS artist_annotation ON ("+ - "artist_annotation.item_id = media_file.artist_id"+ - " AND artist_annotation.item_type = 'artist'"+ - " AND artist_annotation.user_id = ?)", userID) - } - if joins.has(smartPlaylistJoinAlbum) { - sq = sq.LeftJoin("album ON album.id = media_file.album_id") - } - return sq -} - // addCriteria applies the where conditions, limit, offset, and order by clauses to the SQL query based on the // smart playlist criteria. func (r *playlistRepository) addCriteria(sql SelectBuilder, cSQL smartPlaylistCriteria) (SelectBuilder, error) { - cond, err := cSQL.Where() + cond, err := cSQL.where() if err != nil { return sql, err } @@ -197,7 +163,7 @@ func (r *playlistRepository) addCriteria(sql SelectBuilder, cSQL smartPlaylistCr if cSQL.Criteria.Limit > 0 { sql = sql.Limit(uint64(cSQL.Criteria.Limit)).Offset(uint64(cSQL.Criteria.Offset)) } - if order := cSQL.OrderBy(); order != "" { + if order := cSQL.orderBy(); order != "" { sql = sql.OrderBy(order) } return sql, nil diff --git a/persistence/user_repository.go b/persistence/user_repository.go index 9decff4e5..3c030a640 100644 --- a/persistence/user_repository.go +++ b/persistence/user_repository.go @@ -16,6 +16,7 @@ import ( "github.com/navidrome/navidrome/consts" "github.com/navidrome/navidrome/log" "github.com/navidrome/navidrome/model" + "github.com/navidrome/navidrome/model/criteria" "github.com/navidrome/navidrome/model/id" "github.com/navidrome/navidrome/utils" "github.com/navidrome/navidrome/utils/slice" @@ -252,6 +253,9 @@ func (r *userRepository) Save(entity any) (string, error) { if err := validateUsernameUnique(r, u); err != nil { return "", err } + if err := validateScrobbleFilter(u); err != nil { + return "", err + } err := r.Put(u) if err != nil { return "", err @@ -284,6 +288,9 @@ func (r *userRepository) Update(id string, entity any, _ ...string) error { if err := validateUsernameUnique(r, u); err != nil { return err } + if err := validateScrobbleFilter(u); err != nil { + return err + } err := r.Put(u) if errors.Is(err, model.ErrNotFound) { return rest.ErrNotFound @@ -331,6 +338,33 @@ func validateUsernameUnique(r model.UserRepository, u *model.User) error { return nil } +func validateScrobbleFilter(u *model.User) error { + u.ScrobbleFilter = strings.TrimSpace(u.ScrobbleFilter) + if u.ScrobbleFilter == "" { + return nil + } + var c criteria.Criteria + if err := json.Unmarshal([]byte(u.ScrobbleFilter), &c); err != nil { + return invalidScrobbleFilter() + } + // A filter is a per-track test, so a result-set size means nothing here. Reject it + // rather than silently ignoring part of a rule copied from a smart playlist. + if c.Limit > 0 || c.LimitPercent > 0 || c.Offset > 0 || c.RefreshDelay > 0 { + return invalidScrobbleFilter() + } + // Building the WHERE clause is what validates field names and operators + if _, err := newSmartPlaylistCriteria(c).where(); err != nil { + return invalidScrobbleFilter() + } + return nil +} + +func invalidScrobbleFilter() error { + return &rest.ValidationError{Errors: map[string]string{ + "scrobbleFilter": "resources.user.validation.invalidScrobbleFilter", + }} +} + func (r *userRepository) Delete(id string) error { usr := loggedUser(r.ctx) if !usr.IsAdmin { diff --git a/persistence/user_repository_test.go b/persistence/user_repository_test.go index 6f8ab9161..ec417c193 100644 --- a/persistence/user_repository_test.go +++ b/persistence/user_repository_test.go @@ -70,6 +70,26 @@ var _ = Describe("UserRepository", func() { Expect(err).ToNot(HaveOccurred()) Expect(actual.Password).To(Equal("newpass")) }) + It("persists and reads back the scrobble filter", func() { + usr := model.User{ID: "u-filter", UserName: "u-filter", Name: "Filter User", + ScrobbleFilter: `{"all":[{"contains":{"title":"????"}}]}`} + Expect(repo.Put(&usr)).To(Succeed()) + + saved, err := repo.Get("u-filter") + Expect(err).ToNot(HaveOccurred()) + Expect(saved.ScrobbleFilter).To(Equal(`{"all":[{"contains":{"title":"????"}}]}`)) + }) + It("reads back a user row inserted without scrobble_filter", func() { + // Guards the column's NOT NULL DEFAULT '': rows predating the migration must stay scannable + _, err := GetDBXBuilder().NewQuery( + "insert into user (id, user_name, name, email, password, created_at, updated_at) " + + "values ('u-rawsql', 'u-rawsql', 'Raw', '', '', datetime('now'), datetime('now'))").Execute() + Expect(err).ToNot(HaveOccurred()) + + saved, err := repo.Get("u-rawsql") + Expect(err).ToNot(HaveOccurred()) + Expect(saved.ScrobbleFilter).To(Equal("")) + }) }) Describe("validatePasswordChange", func() { @@ -607,6 +627,52 @@ var _ = Describe("UserRepository", func() { }) }) + Describe("validateScrobbleFilter", func() { + It("accepts an empty filter", func() { + u := &model.User{} + Expect(validateScrobbleFilter(u)).To(Succeed()) + }) + It("trims a whitespace-only filter to empty", func() { + u := &model.User{ScrobbleFilter: " "} + Expect(validateScrobbleFilter(u)).To(Succeed()) + Expect(u.ScrobbleFilter).To(Equal("")) + }) + It("accepts valid criteria JSON", func() { + u := &model.User{ScrobbleFilter: `{"all":[{"lt":{"rating":4}}]}`} + Expect(validateScrobbleFilter(u)).To(Succeed()) + }) + It("rejects malformed JSON", func() { + u := &model.User{ScrobbleFilter: `{not json`} + var vErr *rest.ValidationError + err := validateScrobbleFilter(u) + Expect(errors.As(err, &vErr)).To(BeTrue()) + Expect(vErr.Errors).To(HaveKey("scrobbleFilter")) + }) + It("rejects criteria without rules", func() { + u := &model.User{ScrobbleFilter: `{"sort":"title"}`} + Expect(validateScrobbleFilter(u)).ToNot(Succeed()) + }) + It("rejects selection options that mean nothing for a single track", func() { + for _, f := range []string{ + `{"all":[{"lt":{"rating":4}}],"limit":100}`, + `{"all":[{"lt":{"rating":4}}],"limitPercent":10}`, + `{"all":[{"lt":{"rating":4}}],"offset":5}`, + `{"all":[{"lt":{"rating":4}}],"refreshDelay":"1h"}`, + } { + u := &model.User{ScrobbleFilter: f} + Expect(validateScrobbleFilter(u)).ToNot(Succeed(), f) + } + }) + It("accepts a sort, which cannot change a single-track match", func() { + u := &model.User{ScrobbleFilter: `{"all":[{"lt":{"rating":4}}],"sort":"title"}`} + Expect(validateScrobbleFilter(u)).To(Succeed()) + }) + It("rejects unknown fields", func() { + u := &model.User{ScrobbleFilter: `{"all":[{"is":{"bogusfield":1}}]}`} + Expect(validateScrobbleFilter(u)).ToNot(Succeed()) + }) + }) + Describe("filters", func() { It("qualifies id filter with table name", func() { r := repo.(*userRepository) diff --git a/resources/i18n/pt-br.json b/resources/i18n/pt-br.json index b3b3bab2f..9a6338ae1 100644 --- a/resources/i18n/pt-br.json +++ b/resources/i18n/pt-br.json @@ -155,11 +155,13 @@ "newPassword": "Nova Senha", "token": "Token", "lastAccessAt": "Últ. Acesso", - "libraries": "Bibliotecas" + "libraries": "Bibliotecas", + "scrobbleFilter": "Filtro de scrobble" }, "helperTexts": { "name": "Alterações no seu nome só serão refletidas no próximo login", - "libraries": "Selecione bibliotecas específicas para este usuário, ou deixe vazio para usar bibliotecas padrão" + "libraries": "Selecione bibliotecas específicas para este usuário, ou deixe vazio para usar bibliotecas padrão", + "scrobbleFilter": "Músicas que correspondem a estas regras de playlist inteligente não são enviadas ao Last.fm, ListenBrainz ou plugins de scrobble. Usa a mesma sintaxe JSON e o mesmo comportamento das playlists inteligentes. Exemplo: {\"all\":[{\"lt\":{\"rating\":4}}]}. Deixe vazio para enviar scrobbles de todas as músicas. As contagens de reprodução locais não são afetadas." }, "notifications": { "created": "Novo usuário criado", @@ -173,7 +175,8 @@ "adminAutoLibraries": "Usuários administradores têm acesso automático a todas as bibliotecas" }, "validation": { - "librariesRequired": "Pelo menos uma biblioteca deve ser selecionada para usuários não-administradores" + "librariesRequired": "Pelo menos uma biblioteca deve ser selecionada para usuários não-administradores", + "invalidScrobbleFilter": "Devem ser regras válidas de playlist inteligente. Limite, deslocamento e intervalo de atualização não são suportados." } }, "player": { diff --git a/tests/mock_mediafile_repo.go b/tests/mock_mediafile_repo.go index d993af1a8..58e84785b 100644 --- a/tests/mock_mediafile_repo.go +++ b/tests/mock_mediafile_repo.go @@ -9,6 +9,7 @@ import ( "github.com/deluan/rest" "github.com/navidrome/navidrome/model" + "github.com/navidrome/navidrome/model/criteria" "github.com/navidrome/navidrome/model/id" "github.com/navidrome/navidrome/utils/slice" ) @@ -31,6 +32,8 @@ type MockMediaFileRepo struct { // Add fields for cross-library move detection tests FindRecentFilesByMBZTrackIDFunc func(missing model.MediaFile, since time.Time) (model.MediaFiles, error) FindRecentFilesByPropertiesFunc func(missing model.MediaFile, since time.Time) (model.MediaFiles, error) + MatchesCriteriaValue bool + MatchesCriteriaErr error } func (m *MockMediaFileRepo) SetError(err bool) { @@ -369,5 +372,12 @@ func (m *MockMediaFileRepo) FindRecentFilesByProperties(missing model.MediaFile, return result, nil } +func (m *MockMediaFileRepo) MatchesCriteria(string, criteria.Criteria) (bool, error) { + if m.MatchesCriteriaErr != nil { + return false, m.MatchesCriteriaErr + } + return m.MatchesCriteriaValue, nil +} + var _ model.MediaFileRepository = (*MockMediaFileRepo)(nil) var _ model.ResourceRepository = (*MockMediaFileRepo)(nil) diff --git a/ui/src/i18n/en.json b/ui/src/i18n/en.json index c0e226453..12968afbd 100644 --- a/ui/src/i18n/en.json +++ b/ui/src/i18n/en.json @@ -155,11 +155,13 @@ "currentPassword": "Current Password", "newPassword": "New Password", "token": "Token", - "libraries": "Libraries" + "libraries": "Libraries", + "scrobbleFilter": "Scrobble filter" }, "helperTexts": { "name": "Changes to your name will only be reflected on next login", - "libraries": "Select specific libraries for this user, or leave empty to use default libraries" + "libraries": "Select specific libraries for this user, or leave empty to use default libraries", + "scrobbleFilter": "Songs matching these smart playlist rules are not sent to Last.fm, ListenBrainz or scrobbler plugins. Uses the same JSON syntax and behavior as smart playlists. Example: {\"all\":[{\"lt\":{\"rating\":4}}]}. Leave empty to scrobble everything. Local play counts are not affected." }, "notifications": { "created": "User created", @@ -167,7 +169,8 @@ "deleted": "User deleted" }, "validation": { - "librariesRequired": "At least one library must be selected for non-admin users" + "librariesRequired": "At least one library must be selected for non-admin users", + "invalidScrobbleFilter": "Must be valid smart playlist rules. Limit, offset and refresh delay are not supported." }, "message": { "listenBrainzToken": "Enter your ListenBrainz user token.", diff --git a/ui/src/user/UserEdit.jsx b/ui/src/user/UserEdit.jsx index d8302a9f9..c5d9c75a4 100644 --- a/ui/src/user/UserEdit.jsx +++ b/ui/src/user/UserEdit.jsx @@ -172,6 +172,17 @@ const UserEdit = (props) => { )} + + diff --git a/ui/src/user/UserEdit.test.jsx b/ui/src/user/UserEdit.test.jsx index 1d8290569..74405cc13 100644 --- a/ui/src/user/UserEdit.test.jsx +++ b/ui/src/user/UserEdit.test.jsx @@ -127,6 +127,12 @@ describe('', () => { expect(screen.getByTestId('date-field-createdAt')).toBeInTheDocument() }) + it('should render the scrobble filter input', () => { + render() + + expect(screen.getByTestId('text-input-scrobbleFilter')).toBeInTheDocument() + }) + it('should not render username input for non-admin users', () => { render() diff --git a/ui/src/user/userValidation.js b/ui/src/user/userValidation.js index e90fd2acb..9eb1b8f05 100644 --- a/ui/src/user/userValidation.js +++ b/ui/src/user/userValidation.js @@ -15,5 +15,15 @@ export const validateUserForm = (values, translate) => { } } + if (values.scrobbleFilter && values.scrobbleFilter.trim() !== '') { + try { + JSON.parse(values.scrobbleFilter) + } catch { + errors.scrobbleFilter = translate( + 'resources.user.validation.invalidScrobbleFilter', + ) + } + } + return errors } diff --git a/ui/src/user/userValidation.test.js b/ui/src/user/userValidation.test.js index 2ee473910..de52c17df 100644 --- a/ui/src/user/userValidation.test.js +++ b/ui/src/user/userValidation.test.js @@ -67,4 +67,27 @@ describe('User Validation Utilities', () => { ) }) }) + + describe('scrobbleFilter validation', () => { + it('accepts an empty filter', () => { + const errors = validateUserForm({ isAdmin: true }, mockTranslate) + expect(errors.scrobbleFilter).toBeUndefined() + }) + it('accepts valid JSON', () => { + const errors = validateUserForm( + { isAdmin: true, scrobbleFilter: '{"all":[{"lt":{"rating":4}}]}' }, + mockTranslate, + ) + expect(errors.scrobbleFilter).toBeUndefined() + }) + it('rejects malformed JSON', () => { + const errors = validateUserForm( + { isAdmin: true, scrobbleFilter: '{broken' }, + mockTranslate, + ) + expect(errors.scrobbleFilter).toEqual( + 'resources.user.validation.invalidScrobbleFilter', + ) + }) + }) }) From 42dfdf49da7e75f48eda3463754694e2c2362799 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Deluan=20Quint=C3=A3o?= Date: Sun, 16 Aug 2026 10:18:10 -0400 Subject: [PATCH 08/27] feat(lyrics): support [bg:] tag and skip unknown tags in LRC files (#5966) Unknown [name:value] tag lines (e.g. [al:], [by:]) were being glued onto the previous lyric line, producing junk cues like "\n[bg: " and stretching the line's timing. Now any unrecognized tag line is skipped whole, and the non-standard [bg:] background-vocal tag is parsed into cues attached to the preceding line under a bg agent, using the same agents representation the TTML parser emits for Apple-style background vocals. --- model/lyrics_lrc.go | 103 +++++++++++++++++++++++++++++++-------- model/lyrics_lrc_test.go | 43 ++++++++++++++++ 2 files changed, 127 insertions(+), 19 deletions(-) diff --git a/model/lyrics_lrc.go b/model/lyrics_lrc.go index 2cccb9d51..e5bd8d4f5 100644 --- a/model/lyrics_lrc.go +++ b/model/lyrics_lrc.go @@ -21,6 +21,12 @@ var ( timeRegex = regexp.MustCompile(timeRegexString) lrcIdRegex = regexp.MustCompile(`\[(ar|ti|offset|lang):([^]]+)]`) + // Non-standard background-vocal tag: [bg: word...] attaches to the preceding line + bgTagRegex = regexp.MustCompile(`^\[bg:(.*)]$`) + + // Any other [name:value] line is a tag we don't support (e.g. [al:], [by:]) and must be skipped whole + unknownTagRegex = regexp.MustCompile(`^\[[A-Za-z][^]:]*:[^]]*]`) + // Enhanced LRC: inline word-level timing markers like <00:12.34> enhancedLRCTimeString = `<([0-9]{1,2}:)?([0-9]{1,2}):([0-9]{1,2})(\.[0-9]{1,3})?>` enhancedLRCRegex = regexp.MustCompile(enhancedLRCTimeString) @@ -40,7 +46,26 @@ func parseLRC(language, text string) (*Lyrics, error) { priorLine := "" validLine := false repeated := false + hasBg := false var timestamps []int64 + var pendingBg []string + + flushLine := func() { + value, cues := parseEnhancedLine(priorLine) + var merged bool + value, cues, merged = mergeBgLayers(value, cues, pendingBg) + hasBg = hasBg || merged + for idx := range timestamps { + startCopy := timestamps[idx] + structuredLines = append(structuredLines, Line{ + Start: &startCopy, + Value: value, + Cue: shiftELRCCues(cues, timestamps[idx]-timestamps[0]), + }) + } + timestamps = nil + pendingBg = nil + } for _, line := range lines { line := strings.TrimSpace(line) @@ -77,6 +102,17 @@ func parseLRC(language, text string) (*Lyrics, error) { continue } + if bgMatch := bgTagRegex.FindStringSubmatch(line); bgMatch != nil { + if validLine { + pendingBg = append(pendingBg, bgMatch[1]) + } + continue + } + + if unknownTagRegex.MatchString(line) { + continue + } + times := timeRegex.FindAllStringSubmatchIndex(line, -1) if len(times) > 1 { repeated = true @@ -92,16 +128,7 @@ func parseLRC(language, text string) (*Lyrics, error) { } if validLine { - value, baseCues := parseEnhancedLine(priorLine) - for idx := range timestamps { - startCopy := timestamps[idx] - structuredLines = append(structuredLines, Line{ - Start: &startCopy, - Value: value, - Cue: shiftELRCCues(baseCues, timestamps[idx]-timestamps[0]), - }) - } - timestamps = nil + flushLine() } end := 0 @@ -143,15 +170,11 @@ func parseLRC(language, text string) (*Lyrics, error) { } if validLine { - value, baseCues := parseEnhancedLine(priorLine) - for idx := range timestamps { - startCopy := timestamps[idx] - structuredLines = append(structuredLines, Line{ - Start: &startCopy, - Value: value, - Cue: shiftELRCCues(baseCues, timestamps[idx]-timestamps[0]), - }) - } + flushLine() + } + + if hasBg { + fillMainAgentID(structuredLines) } // If there are repeated values, there is no guarantee that they are in order @@ -170,9 +193,51 @@ func parseLRC(language, text string) (*Lyrics, error) { Offset: offset, Synced: synced, } + if hasBg { + lyrics.Agents = []Agent{ + {ID: "main", Role: "main"}, + {ID: backgroundAgentID("main"), Role: "bg"}, + } + } return &lyrics, nil } +// mergeBgLayers appends parsed [bg:] content to a line's value and cues as a +// background agent layer, mirroring the TTML background-vocal representation. +func mergeBgLayers(value string, cues []Cue, bgLines []string) (string, []Cue, bool) { + merged := false + for _, bg := range bgLines { + bgValue, bgCues := parseEnhancedLine(bg) + if len(bgCues) == 0 { + continue + } + offset := 0 + if value != "" { + offset = len(value) + 1 + value += " " + } + value += bgValue + for _, c := range bgCues { + c.ByteStart += offset + c.ByteEnd += offset + c.AgentID = backgroundAgentID("main") + cues = append(cues, c) + } + merged = true + } + return value, cues, merged +} + +func fillMainAgentID(lines []Line) { + for i := range lines { + for j := range lines[i].Cue { + if lines[i].Cue[j].AgentID == "" { + lines[i].Cue[j].AgentID = "main" + } + } + } +} + // parseEnhancedLine extracts word-level timing cues from Enhanced LRC inline markers // and computes UTF-8 byte offsets against the final stripped line value. func parseEnhancedLine(text string) (string, []Cue) { diff --git a/model/lyrics_lrc_test.go b/model/lyrics_lrc_test.go index 87514caf1..e8cd05352 100644 --- a/model/lyrics_lrc_test.go +++ b/model/lyrics_lrc_test.go @@ -147,6 +147,49 @@ var _ = Describe("parseLRC", func() { })) }) + It("should skip unknown tag lines instead of merging them into the prior line", func() { + lyrics, err := parseLRC("xxx", "[00:01.00]<00:01.00>Some <00:01.50>lyrics\n[by: someone]\n[al: An album]\n[00:03.00]Next") + Expect(err).ToNot(HaveOccurred()) + Expect(lyrics.Synced).To(BeTrue()) + Expect(lyrics.Line).To(HaveLen(2)) + + t1000, t1500, t3000 := int64(1000), int64(1500), int64(3000) + Expect(lyrics.Line[0].Value).To(Equal("Some lyrics")) + Expect(lyrics.Line[0].Cue).To(Equal([]Cue{ + {Start: &t1000, End: &t1500, Value: "Some ", ByteStart: 0, ByteEnd: 4}, + {Start: &t1500, End: &t3000, Value: "lyrics", ByteStart: 5, ByteEnd: 10}, + })) + Expect(lyrics.Line[1].Value).To(Equal("Next")) + }) + + It("should attach [bg:] background vocals to the previous line as a bg agent layer", func() { + lyrics, err := parseLRC("xxx", "[00:52.00]<00:52.00>Main <00:52.50>line\n[bg: <00:53.00>Okay<00:53.50>]\n[00:55.00]Next") + Expect(err).ToNot(HaveOccurred()) + Expect(lyrics.Agents).To(Equal([]Agent{ + {ID: "main", Role: "main"}, + {ID: "__nd_bg__|main", Role: "bg"}, + })) + Expect(lyrics.Line).To(HaveLen(2)) + + line := lyrics.Line[0] + Expect(line.Value).To(Equal("Main line Okay")) + Expect(line.Cue).To(Equal([]Cue{ + {Start: new(int64(52000)), End: new(int64(52500)), Value: "Main ", ByteStart: 0, ByteEnd: 4, AgentID: "main"}, + {Start: new(int64(52500)), End: new(int64(55000)), Value: "line", ByteStart: 5, ByteEnd: 8, AgentID: "main"}, + {Start: new(int64(53000)), End: new(int64(53500)), Value: "Okay", ByteStart: 10, ByteEnd: 13, AgentID: "__nd_bg__|main"}, + })) + Expect(lyrics.Line[1].Value).To(Equal("Next")) + }) + + It("should skip a [bg:] line that has no inline timing", func() { + lyrics, err := parseLRC("xxx", "[00:01.00]Hi\n[bg: untimed]\n[00:03.00]Next") + Expect(err).ToNot(HaveOccurred()) + Expect(lyrics.Agents).To(BeNil()) + Expect(lyrics.Line).To(HaveLen(2)) + Expect(lyrics.Line[0].Value).To(Equal("Hi")) + Expect(lyrics.Line[1].Value).To(Equal("Next")) + }) + It("should handle mixed Enhanced and plain LRC lines", func() { lyrics, err := parseLRC("xxx", "[00:01.00]<00:01.00>Some <00:01.50>lyrics\n[00:03.00]Plain line\n[00:05.00]<00:05.00>More <00:05.50>words") Expect(err).ToNot(HaveOccurred()) From 5b758fc20cc11e16f0fddab19e5bb0714007e1a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Deluan=20Quint=C3=A3o?= Date: Sun, 16 Aug 2026 13:24:07 -0400 Subject: [PATCH 09/27] fix(artwork): re-resolve artwork when image files change on disk (#5965) * fix(artwork): re-resolve artwork when image files change on disk An image-only folder change (replaced, added, or deleted cover/artist images, with no audio files touched) was detected by the scanner but never reached the artwork queue, so clients kept seeing the old coverArt hash until something else forced a re-resolution. Phase 1 now diffs each changed folder's image list and imagesUpdatedAt against the previously persisted folder row, and at the end of the phase bulk-enqueues re-resolution for the affected entities: albums with tracks in the folder or its direct children (covering disc subfolder layouts), and, when an artist-pattern image is involved, artists with albums under the folder's subtree, mirroring the artist resolver's upward search. The artist mapping mirrors the resolver's sole-album-artist album selection. New repository helpers keep the mapping set-based and light: folder GetAllIDs, media_file GetAlbumIDsByFolder (distinct, indexed by folder_id), and album GetSoleAlbumArtistIDs. * refactor: simplify the image-change artwork enqueue after review Load the previous folder image state through the existing GetFolderUpdateInfo bulk pre-pass instead of a per-folder SELECT inside the persist transaction, and skip the diff for new folders, whose artwork the scanner already enqueues inline. Move the artist-image classification into core/artwork (IsArtistImageFile) so the scanner shares the resolver's ArtistArtPriority token grammar instead of re-parsing it (the copy mistreated image-folder as a filename glob). Move the folder-subtree query into the folder repository (GetSubtreeIDs) with LIKE escaping and expression-tree batching, share the sole-album-artist predicate between the resolver and the album repository (model.SoleAlbumArtistFilter), extract a chunked single-column query helper, and deduplicate the ArtworkQueueItem literals behind scanArtworkItem. * fix(persistence): keep slash-form paths in GetSubtreeIDs subtree predicates The scanner hands GetSubtreeIDs io/fs slash-form paths, but filepath.Clean rewrites them with backslashes on Windows while folder.path is stored with forward slashes, so the descendant predicates matched nothing and nested artist folders were never re-enqueued there. Normalize with path.Clean, like HasAudioOutsideFolders does, and cover a nested path in the repo test. * refactor(persistence): move the sole-album-artist rule into the album repository SQLizer filters belong in the persistence package, not model. The rule becomes an unexported filter shared by GetSoleAlbumArtistIDs and a new GetBySoleAlbumArtist repository method, which the artist artwork resolver now calls instead of building the squirrel filter itself. * perf(scanner): resolve image-change artists in one query over album.folder_ids The artist half of the image-change enqueue walked folder subtree IDs, then media_file rows, then album rows, marshalling thousands of bound IDs through the driver on each hop. Matching albums by their own folder_ids instead is one statement, and folder_ids is the same source the artist resolver uses to compute an artist's folders. Benchmarked against a copy of the production DB (97k tracks, 10k folders, 7k albums): 87ms +/-196% -> 17.4ms +/-8%, 7.1MB -> 172KB, 103k -> 1.5k allocs. The subtree predicate becomes a shared folderSubtreeFilter, so Folder GetSubtreeIDs and Album GetSoleAlbumArtistIDs are no longer needed. * fix(scanner): persist ancestor folders discovered by a quick scan A quick scan skipped any new folder with no files of its own, so an artist folder holding only album subfolders never got a row. Adding artist.jpg to it later then produced no artwork enqueue: the entry was new, so the image diff was skipped, and it has no tracks, so nothing was enqueued inline either. Skip only genuinely empty new folders, matching what a full scan already persists. This also fixes artist artwork resolving as absent for artists first imported by a quick scan, since the resolver's folder climb needs that row. Also normalizes the selective-scan preload paths with path.Clean, so its descendant predicates match the stored slash-form paths on Windows. * fix(persistence): chunk subtree paths and match artist globs by basename Two regressions from earlier commits on this branch. Collapsing the subtree query into a single statement dropped the chunking the old GetSubtreeIDs had: each path expands into 3 OR terms and SQLite rejects an expression tree deeper than 1000, measured at 166 paths. A library with more artist-image folders than that (the prod copy has 158) would fail the whole collect, dropping the album items with it, so the scanner now keeps them when the artist query fails. The artist-image classifier compared whole tokens after stripping album/, so a directory-bearing glob like images/artist.* never matched the basenames the scanner has. Match on path.Base, which is what album/artist.* already reduced to; the resolver climbs parent folders, so an exact prefix is not knowable here and a conservative match is the right failure direction. * refactor(persistence): halve the repository surface this PR adds Research on the four new repository methods found two were avoidable. GetAlbumIDsByFolder now expands the changed folders to their direct children in its own subquery, so Folder.GetAllIDs has no callers and is deleted, one round trip per scan disappears, and the previously unchunked id/parent_id IN lists are covered by the existing chunking. GetBySoleAlbumArtist becomes an exported SoleAlbumArtistFilter, matching the ParticipantIDFilter precedent for sharing a Sqlizer with core/, so the rule still lives in persistence but AlbumRepository gains nothing and the mock shim that ignored the artist filter is gone. Also drops queryAllSliceChunked, now callerless, in favour of the file-local slices.Chunk convention used by the sibling folder queries. Rejected on measurement: matching the album path by album.folder_ids is exactly equivalent (13975 pairs, zero difference) but has no index, so it scans every album and runs 5-200x slower than the media_file route. * refactor(persistence): stop reading the deprecated album_artist_id column Both artist lookups this PR touches now go through participation, matching the precedent in core/archiver.go and share_repository.go. SoleAlbumArtistFilter uses ParticipantIDFilter, which is also faster: the album_artists unique constraint is a covering index for it, while the old column needed album_artist_album_id plus a row fetch. GetSoleAlbumArtistIDsInSubtrees reads the sole artist out of the participants JSON it already parses for the sole-artist check, rather than joining back to album_artists, which measured ~1.6x slower on a prod-sized copy. Verified equivalent on that copy: 6828 sole-artist albums and 1088 subtree artists resolve identically via the column, the join and the JSON. The tests now set a deliberately wrong album_artist_id so they fail if either query starts reading it again. * docs: trim comments that carry rationale belonging in commit messages Five comments had grown past the budget with benchmark numbers, rejected alternatives, and a duplicate of the constant's own explanation. * refactor(scanner): move the image-change enqueue into phase_1_folders The three functions were methods on phaseFolders, so they belong with the type; phase_1_image_changes.go also read like a fifth phase, which it wasn't. * refactor(scanner): extract the image-change collector into its own type phaseFolders no longer owns the per-library map and the mapping methods; it records into a collector and asks it to enqueue once. The collector keeps the library alongside the folders, so enqueue needs only ctx and the datastore. * refactor(scanner): simplify enqueue method by removing redundant datastore parameter Signed-off-by: Deluan * docs(scanner): drop the stale zero-value claim on imageChangeCollector The collector now takes its datastore at construction, so the zero value is no longer usable. * fix(scanner): pin the persist stage to concurrency 1 and guard the collector The stage relied on go-pipeline defaulting to one worker; stating it at the stage makes the constraint visible where someone would change it. The collector takes a mutex too, so the type is safe on its own terms rather than by configuration. --------- Signed-off-by: Deluan --- core/artwork/resolve.go | 26 +++-- core/artwork/resolve_test.go | 25 +++++ model/album.go | 3 + model/folder.go | 6 +- model/mediafile.go | 3 + persistence/album_repository.go | 33 ++++++ persistence/album_repository_test.go | 75 +++++++++++++- persistence/folder_repository.go | 49 +++++++-- persistence/folder_repository_test.go | 37 +++++++ persistence/mediafile_repository.go | 20 ++++ persistence/mediafile_repository_test.go | 48 +++++++++ scanner/folder_entry.go | 51 ++++++--- scanner/folder_entry_test.go | 6 +- scanner/image_changes.go | 112 ++++++++++++++++++++ scanner/phase_1_folders.go | 34 +++--- scanner/scanner_test.go | 125 +++++++++++++++++++++++ 16 files changed, 602 insertions(+), 51 deletions(-) create mode 100644 scanner/image_changes.go diff --git a/core/artwork/resolve.go b/core/artwork/resolve.go index 518b6e100..d25f76460 100644 --- a/core/artwork/resolve.go +++ b/core/artwork/resolve.go @@ -9,13 +9,14 @@ import ( "io/fs" "net/url" "os" + "path" "strings" - "github.com/Masterminds/squirrel" "github.com/navidrome/navidrome/conf" "github.com/navidrome/navidrome/core/agents" "github.com/navidrome/navidrome/core/ffmpeg" "github.com/navidrome/navidrome/model" + "github.com/navidrome/navidrome/persistence" ) // resolution is one attempted acquisition outcome for an entity. @@ -248,12 +249,7 @@ func (r *resolver) resolveArtist(ctx context.Context, artistID string) (resoluti } // Only consider albums where the artist is the sole album artist. - als, err := r.ds.Album(ctx).GetAll(model.QueryOptions{ - Filters: squirrel.And{ - squirrel.Eq{"album_artist_id": artistID}, - squirrel.Eq{"json_array_length(participants, '$.albumartist')": 1}, - }, - }) + als, err := r.ds.Album(ctx).GetAll(model.QueryOptions{Filters: persistence.SoleAlbumArtistFilter(artistID)}) if err != nil { return resolution{}, err } @@ -525,6 +521,22 @@ func resolveFolderFile(ctx context.Context, lib libraryView, imgFiles []string, return resolveFolderSource(lib, fromExternalFile(ctx, lib.FS, imgFiles, pattern)) } +// IsArtistImageFile reports whether a file name matches any file-glob token of ArtistArtPriority. +// Basename-only on purpose: the chain climbs parent folders, so a token's prefix is not fixed. +func IsArtistImageFile(name string) bool { + name = strings.ToLower(name) + for pattern := range strings.SplitSeq(strings.ToLower(conf.Server.ArtistArtPriority), ",") { + pattern = strings.TrimSpace(pattern) + if pattern == "" || pattern == externalCandidate || pattern == "image-folder" { + continue + } + if ok, _ := path.Match(path.Base(pattern), name); ok { + return true + } + } + return false +} + func resolveArtistImageFolder(ar *model.Artist) (resolution, bool) { folder := conf.Server.ArtistImageFolder if folder == "" { diff --git a/core/artwork/resolve_test.go b/core/artwork/resolve_test.go index 25fb727de..8b4c11c8c 100644 --- a/core/artwork/resolve_test.go +++ b/core/artwork/resolve_test.go @@ -21,6 +21,31 @@ import ( . "github.com/onsi/gomega" ) +var _ = Describe("IsArtistImageFile", func() { + BeforeEach(func() { + DeferCleanup(configtest.SetupConfig()) + }) + + It("matches bare and album/-prefixed glob tokens, case-insensitively", func() { + conf.Server.ArtistArtPriority = "artist.*, album/artistfolder.*, external" + Expect(IsArtistImageFile("Artist.jpg")).To(BeTrue()) + Expect(IsArtistImageFile("artistfolder.png")).To(BeTrue()) + Expect(IsArtistImageFile("cover.jpg")).To(BeFalse()) + }) + + It("matches a directory-bearing glob by its basename", func() { + conf.Server.ArtistArtPriority = "images/artist.*, external" + Expect(IsArtistImageFile("artist.jpg")).To(BeTrue()) + Expect(IsArtistImageFile("cover.jpg")).To(BeFalse()) + }) + + It("does not treat non-file tokens as globs", func() { + conf.Server.ArtistArtPriority = "image-folder, external" + Expect(IsArtistImageFile("image-folder")).To(BeFalse()) + Expect(IsArtistImageFile("external")).To(BeFalse()) + }) +}) + var _ = Describe("resolveItem", func() { var ( ctx context.Context diff --git a/model/album.go b/model/album.go index f27ca12c0..5a436fec0 100644 --- a/model/album.go +++ b/model/album.go @@ -144,6 +144,9 @@ type AlbumRepository interface { Get(id string) (*Album, error) GetAll(...QueryOptions) (Albums, error) GetAllIDs(...QueryOptions) ([]string, error) + // GetSoleAlbumArtistIDsInSubtrees returns the sole album artists of the albums with folders in + // any of the given library-relative subtrees. + GetSoleAlbumArtistIDsInSubtrees(lib Library, paths ...string) ([]string, error) GetCursor(...QueryOptions) (AlbumCursor, error) GetYears(libraryIDs ...int) ([]int, error) diff --git a/model/folder.go b/model/folder.go index 81800c072..5207a9db0 100644 --- a/model/folder.go +++ b/model/folder.go @@ -76,8 +76,10 @@ func NewFolder(lib Library, folderPath string) *Folder { type FolderCursor iter.Seq2[Folder, error] type FolderUpdateInfo struct { - UpdatedAt time.Time - Hash string + UpdatedAt time.Time + Hash string + ImageFiles []string + ImagesUpdatedAt time.Time } type FolderRepository interface { diff --git a/model/mediafile.go b/model/mediafile.go index 888425d07..f4e767272 100644 --- a/model/mediafile.go +++ b/model/mediafile.go @@ -555,6 +555,9 @@ type MediaFileRepository interface { GetCursor(options ...QueryOptions) (MediaFileCursor, error) // GetAllIDs returns just the media_file IDs for the same row set as GetAll. GetAllIDs(options ...QueryOptions) ([]string, error) + // GetAlbumIDsByFolder returns the distinct IDs of albums with non-missing tracks in the given + // folders or their direct children. + GetAlbumIDsByFolder(lib Library, folderIDs ...string) ([]string, error) // GetCursorWithArtwork streams like GetCursor, hydrated, so callers that render images don't // pay the scanner's per-row cost; it uses the same id pre-pass as the other cursors. GetCursorWithArtwork(options ...QueryOptions) (MediaFileCursor, error) diff --git a/persistence/album_repository.go b/persistence/album_repository.go index 486099fc5..2f0621b73 100644 --- a/persistence/album_repository.go +++ b/persistence/album_repository.go @@ -270,6 +270,39 @@ func (r *albumRepository) GetAllIDs(options ...model.QueryOptions) ([]string, er return ids, err } +// soleAlbumArtistFilter matches albums with exactly one album artist. The artist artwork +// resolver and the scanner's image-change enqueue must select the same albums. +var soleAlbumArtistFilter = Eq{"json_array_length(participants, '$.albumartist')": 1} + +// SoleAlbumArtistFilter matches the albums where the given artist is the only album artist. +// Matches by album-artist participation, not the deprecated album_artist_id column. +func SoleAlbumArtistFilter(artistID string) Sqlizer { + return And{ParticipantIDFilter("album", artistID, model.RoleAlbumArtist), soleAlbumArtistFilter} +} + +// GetSoleAlbumArtistIDsInSubtrees matches albums by their own folder_ids, which is the resolver's +// notion of an album's folders. +func (r *albumRepository) GetSoleAlbumArtistIDsInSubtrees(lib model.Library, paths ...string) ([]string, error) { + if len(paths) == 0 { + return nil, nil + } + ids := []string{} + // Repeated IDs across chunks are fine: the queue upserts by PK. + for chunk := range slices.Chunk(paths, subtreePathChunkSize) { + inSubtree := Exists("json_each(album.folder_ids) je join folder on folder.id = je.value", + folderSubtreeFilter(lib, chunk)) + // Sole album artist, so participants[0] is the only one. + sq := Select("distinct json_extract(participants, '$.albumartist[0].id')").From("album"). + Where(And{soleAlbumArtistFilter, inSubtree}) + var chunkIDs []string + if err := r.queryAllSlice(sq, &chunkIDs); err != nil { + return nil, err + } + ids = append(ids, chunkIDs...) + } + return ids, nil +} + func (r *albumRepository) GetCursor(options ...model.QueryOptions) (model.AlbumCursor, error) { ids, err := r.GetAllIDs(options...) if err != nil { diff --git a/persistence/album_repository_test.go b/persistence/album_repository_test.go index 061083949..526642aa6 100644 --- a/persistence/album_repository_test.go +++ b/persistence/album_repository_test.go @@ -1,6 +1,7 @@ package persistence import ( + "context" "errors" "fmt" "sort" @@ -20,9 +21,10 @@ import ( var _ = Describe("AlbumRepository", func() { var albumRepo *albumRepository + var ctx context.Context BeforeEach(func() { - ctx := request.WithUser(GinkgoT().Context(), model.User{ID: "userid", UserName: "johndoe"}) + ctx = request.WithUser(GinkgoT().Context(), model.User{ID: "userid", UserName: "johndoe"}) albumRepo = NewAlbumRepository(ctx, GetDBXBuilder()).(*albumRepository) }) @@ -96,6 +98,77 @@ var _ = Describe("AlbumRepository", func() { }) }) + Describe("GetSoleAlbumArtistIDsInSubtrees", func() { + It("returns the sole album artists of albums with folders in the subtree", func() { + folderRepo := newFolderRepository(ctx, GetDBXBuilder()) + lib, err := NewLibraryRepository(ctx, GetDBXBuilder()).Get(1) + Expect(err).ToNot(HaveOccurred()) + inTree := model.NewFolder(*lib, "SubtreeAlbums/Artist") + outTree := model.NewFolder(*lib, "OtherTree/Artist") + Expect(folderRepo.Put(inTree)).To(Succeed()) + Expect(folderRepo.Put(outTree)).To(Succeed()) + + // album_artist_id is deliberately wrong: the artist must come from participants + inAl := model.Album{ID: "subtree-in-al", Name: "In", LibraryID: 1, AlbumArtistID: "999", FolderIDs: []string{inTree.ID}, + Participants: model.Participants{model.RoleAlbumArtist: []model.Participant{{Artist: artistKraftwerk}}}} + outAl := model.Album{ID: "subtree-out-al", Name: "Out", LibraryID: 1, AlbumArtistID: "3", FolderIDs: []string{outTree.ID}, + Participants: model.Participants{model.RoleAlbumArtist: []model.Participant{{Artist: artistBeatles}}}} + duoAl := model.Album{ID: "subtree-duo-al", Name: "Duo", LibraryID: 1, AlbumArtistID: "5", FolderIDs: []string{inTree.ID}, + Participants: model.Participants{model.RoleAlbumArtist: []model.Participant{{Artist: artistPunctuation}, {Artist: artistBeatles}}}} + for _, al := range []model.Album{inAl, outAl, duoAl} { + Expect(albumRepo.Put(&al)).To(Succeed()) + } + DeferCleanup(func() { + _, _ = GetDBXBuilder().NewQuery("DELETE FROM album WHERE id LIKE 'subtree-%'").Execute() + _, _ = GetDBXBuilder().NewQuery("DELETE FROM folder WHERE path LIKE 'SubtreeAlbums%' OR path LIKE 'OtherTree%'").Execute() + }) + + ids, err := albumRepo.GetSoleAlbumArtistIDsInSubtrees(*lib, "SubtreeAlbums") + Expect(err).ToNot(HaveOccurred()) + Expect(ids).To(ConsistOf("2")) // sole artist in the subtree; the duo and the outside album are excluded + }) + + It("stays under SQLite's expression tree depth limit with many paths", func() { + lib, err := NewLibraryRepository(ctx, GetDBXBuilder()).Get(1) + Expect(err).ToNot(HaveOccurred()) + paths := make([]string, 200) + for i := range paths { + paths[i] = fmt.Sprintf("DepthProbe/Folder%d", i) + } + + _, err = albumRepo.GetSoleAlbumArtistIDsInSubtrees(*lib, paths...) + Expect(err).ToNot(HaveOccurred()) + }) + + It("returns nothing when given no paths", func() { + lib, err := NewLibraryRepository(ctx, GetDBXBuilder()).Get(1) + Expect(err).ToNot(HaveOccurred()) + ids, err := albumRepo.GetSoleAlbumArtistIDsInSubtrees(*lib) + Expect(err).ToNot(HaveOccurred()) + Expect(ids).To(BeEmpty()) + }) + }) + + Describe("SoleAlbumArtistFilter", func() { + It("matches only albums where the artist is the sole album artist", func() { + // album_artist_id is deliberately wrong: matching must come from participation + sole := model.Album{ID: "sole-artist-al", Name: "Sole", LibraryID: 1, AlbumArtistID: "999", + Participants: model.Participants{model.RoleAlbumArtist: []model.Participant{{Artist: artistKraftwerk}}}} + duo := model.Album{ID: "duo-artist-al", Name: "Duo", LibraryID: 1, AlbumArtistID: "999", + Participants: model.Participants{model.RoleAlbumArtist: []model.Participant{{Artist: artistKraftwerk}, {Artist: artistBeatles}}}} + Expect(albumRepo.Put(&sole)).To(Succeed()) + Expect(albumRepo.Put(&duo)).To(Succeed()) + DeferCleanup(func() { + _, _ = GetDBXBuilder().NewQuery("DELETE FROM album WHERE id IN ('sole-artist-al', 'duo-artist-al')").Execute() + }) + + als, err := albumRepo.GetAll(model.QueryOptions{Filters: SoleAlbumArtistFilter("2")}) + Expect(err).ToNot(HaveOccurred()) + Expect(als).To(HaveLen(1)) + Expect(als[0].ID).To(Equal(sole.ID)) + }) + }) + Describe("GetAll", func() { var GetAll = func(opts ...model.QueryOptions) (model.Albums, error) { albums, err := albumRepo.GetAll(opts...) diff --git a/persistence/folder_repository.go b/persistence/folder_repository.go index 5da395a74..a4b73d9d6 100644 --- a/persistence/folder_repository.go +++ b/persistence/folder_repository.go @@ -6,9 +6,7 @@ import ( "fmt" "iter" "maps" - "os" "path" - "path/filepath" "slices" "strings" "time" @@ -147,9 +145,8 @@ func (r folderRepository) getFolderUpdateInfoBatch(lib model.Library, targetPath pathConditions := make(Or, 0, len(targetPaths)*2) for _, targetPath := range targetPaths { - // Clean the path to normalize it. Paths stored in the folder table do not have leading/trailing slashes. - cleanPath := strings.TrimPrefix(targetPath, string(os.PathSeparator)) - cleanPath = filepath.Clean(cleanPath) + // Slash-form like the stored paths; filepath.Clean would backslash them on Windows. + cleanPath := path.Clean(strings.TrimPrefix(targetPath, "/")) // Include the target folder itself by ID folderIDs = append(folderIDs, model.FolderID(lib, cleanPath)) @@ -172,11 +169,13 @@ func (r folderRepository) getFolderUpdateInfoBatch(lib model.Library, targetPath // queryFolderUpdateInfo executes the query and returns the result map func (r folderRepository) queryFolderUpdateInfo(where And) (map[string]model.FolderUpdateInfo, error) { - sq := r.newSelect().Columns("id", "updated_at", "hash").Where(where) + sq := r.newSelect().Columns("id", "updated_at", "hash", "image_files", "images_updated_at").Where(where) var res []struct { - ID string - UpdatedAt time.Time - Hash string + ID string + UpdatedAt time.Time + Hash string + ImageFiles string + ImagesUpdatedAt time.Time } err := r.queryAll(sq, &res) if err != nil { @@ -184,11 +183,41 @@ func (r folderRepository) queryFolderUpdateInfo(where And) (map[string]model.Fol } m := make(map[string]model.FolderUpdateInfo, len(res)) for _, f := range res { - m[f.ID] = model.FolderUpdateInfo{UpdatedAt: f.UpdatedAt, Hash: f.Hash} + info := model.FolderUpdateInfo{UpdatedAt: f.UpdatedAt, Hash: f.Hash, ImagesUpdatedAt: f.ImagesUpdatedAt} + if f.ImageFiles != "" { + if err := json.Unmarshal([]byte(f.ImageFiles), &info.ImageFiles); err != nil { + return nil, fmt.Errorf("parsing folder image_files: %w", err) + } + } + m[f.ID] = info } return m, nil } +// subtreePathChunkSize bounds how many paths one folderSubtreeFilter may expand into: each adds +// 3 OR terms, and SQLite rejects an expression tree deeper than 1000 (measured: 166 paths). +const subtreePathChunkSize = 100 + +// folderSubtreeFilter matches the folders at the given library-relative paths and all their +// descendants. A path of "" or "." selects the whole library, so it drops the path conditions. +func folderSubtreeFilter(lib model.Library, paths []string) Sqlizer { + conds := make(Or, 0, len(paths)*3) + for _, p := range paths { + // Paths are io/fs slash-form; filepath.Clean would backslash them on Windows. + cleanPath := path.Clean(strings.TrimPrefix(p, "/")) + if cleanPath == "." { + return And{Eq{"folder.library_id": lib.ID}, Eq{"folder.missing": false}} + } + conds = append(conds, + Eq{"folder.id": model.FolderID(lib, cleanPath)}, + // Direct children have path = cleanPath; deeper descendants match the prefix + Eq{"folder.path": cleanPath}, + Expr(`folder.path LIKE ? ESCAPE '\'`, escapeLikePrefix(cleanPath)+"/%"), + ) + } + return And{Eq{"folder.library_id": lib.ID}, Eq{"folder.missing": false}, conds} +} + // HasAudioOutsideFolders reports whether any folder in parent's subtree // (including parent itself) contains audio files and is not one of the given // folder IDs. LIKE wildcards in the parent path are escaped, so it is always diff --git a/persistence/folder_repository_test.go b/persistence/folder_repository_test.go index 8cd45f16b..b429cecb9 100644 --- a/persistence/folder_repository_test.go +++ b/persistence/folder_repository_test.go @@ -9,6 +9,7 @@ import ( "github.com/navidrome/navidrome/model" "github.com/navidrome/navidrome/model/request" "github.com/navidrome/navidrome/tests" + "github.com/navidrome/navidrome/utils/slice" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/pocketbase/dbx" @@ -43,6 +44,42 @@ var _ = Describe("FolderRepository", func() { _, _ = conn.NewQuery(fmt.Sprintf("DELETE FROM library WHERE id = %d", otherLib.ID)).Execute() }) + Describe("folderSubtreeFilter", func() { + var parent, child, grandchild, other *model.Folder + + matching := func(paths ...string) []string { + GinkgoHelper() + folders, err := repo.GetAll(model.QueryOptions{Filters: folderSubtreeFilter(testLib, paths)}) + Expect(err).ToNot(HaveOccurred()) + return slice.Map(folders, func(f model.Folder) string { return f.ID }) + } + + BeforeEach(func() { + parent = model.NewFolder(testLib, "TestSubtree") + child = model.NewFolder(testLib, "TestSubtree/Child") + grandchild = model.NewFolder(testLib, "TestSubtree/Child/Grandchild") + other = model.NewFolder(testLib, "TestSubtreeOther") + for _, f := range []*model.Folder{parent, child, grandchild, other} { + Expect(repo.Put(f)).To(Succeed()) + } + DeferCleanup(func() { + _, _ = conn.NewQuery("DELETE FROM folder WHERE name LIKE 'TestSubtree%' OR path LIKE 'TestSubtree%'").Execute() + }) + }) + + It("matches a folder and all its descendants", func() { + Expect(matching("TestSubtree")).To(ConsistOf(parent.ID, child.ID, grandchild.ID)) + }) + + It("matches the descendants of a nested slash-form path", func() { + Expect(matching("TestSubtree/Child")).To(ConsistOf(child.ID, grandchild.ID)) + }) + + It("matches the whole library for the root path", func() { + Expect(matching(".")).To(ContainElements(parent.ID, child.ID, grandchild.ID, other.ID)) + }) + }) + Describe("GetFolderUpdateInfo", func() { Context("with no target paths", func() { It("returns all folders in the library", func() { diff --git a/persistence/mediafile_repository.go b/persistence/mediafile_repository.go index 5f0addc5e..8146cba2f 100644 --- a/persistence/mediafile_repository.go +++ b/persistence/mediafile_repository.go @@ -318,6 +318,26 @@ func (r *mediaFileRepository) GetAllIDs(options ...model.QueryOptions) ([]string return ids, err } +func (r *mediaFileRepository) GetAlbumIDsByFolder(lib model.Library, folderIDs ...string) ([]string, error) { + ids := []string{} + for chunk := range slices.Chunk(folderIDs, 200) { + // A folder's own cover also covers albums whose tracks sit in its disc subfolders. + inFolders := Select("f.id").From("folder f").Where(And{ + Eq{"f.library_id": lib.ID}, + Eq{"f.missing": false}, + Or{Eq{"f.id": chunk}, Eq{"f.parent_id": chunk}}, + }) + sq := Select("distinct album_id").From("media_file"). + Where(And{Eq{"missing": false}, ConcatExpr("folder_id IN (", inFolders, ")")}) + var chunkIDs []string + if err := r.queryAllSlice(sq, &chunkIDs); err != nil { + return nil, err + } + ids = append(ids, chunkIDs...) + } + return ids, nil +} + // GetCursorWithArtwork streams the same rows as GetCursor, hydrated, via an id pre-pass. func (r *mediaFileRepository) GetCursorWithArtwork(options ...model.QueryOptions) (model.MediaFileCursor, error) { ids, err := r.GetAllIDs(options...) diff --git a/persistence/mediafile_repository_test.go b/persistence/mediafile_repository_test.go index ba89b5def..c1a91c5a5 100644 --- a/persistence/mediafile_repository_test.go +++ b/persistence/mediafile_repository_test.go @@ -30,6 +30,54 @@ var _ = Describe("MediaRepository", func() { mr = NewMediaFileRepository(ctx, GetDBXBuilder()) }) + Describe("GetAlbumIDsByFolder", func() { + var lib model.Library + var albumRoot, disc1, sibling *model.Folder + + BeforeEach(func() { + ctx := request.WithUser(log.NewContext(context.TODO()), model.User{ID: "userid"}) + libPtr, err := NewLibraryRepository(ctx, GetDBXBuilder()).Get(1) + Expect(err).ToNot(HaveOccurred()) + lib = *libPtr + + folderRepo := newFolderRepository(ctx, GetDBXBuilder()) + albumRoot = model.NewFolder(lib, "ByFolder/Album") + disc1 = model.NewFolder(lib, "ByFolder/Album/CD1") + sibling = model.NewFolder(lib, "ByFolder/Other") + for _, f := range []*model.Folder{albumRoot, disc1, sibling} { + Expect(folderRepo.Put(f)).To(Succeed()) + } + // Tracks live in the disc subfolder; the sibling album is the negative control. + Expect(mr.Put(&model.MediaFile{ID: "fol-mf-1", LibraryID: 1, AlbumID: "fol-al-1", FolderID: disc1.ID, Path: "t/1.mp3"})).To(Succeed()) + Expect(mr.Put(&model.MediaFile{ID: "fol-mf-2", LibraryID: 1, AlbumID: "fol-al-1", FolderID: disc1.ID, Path: "t/2.mp3"})).To(Succeed()) + Expect(mr.Put(&model.MediaFile{ID: "fol-mf-3", LibraryID: 1, AlbumID: "fol-al-2", FolderID: sibling.ID, Path: "t/3.mp3"})).To(Succeed()) + Expect(mr.Put(&model.MediaFile{ID: "fol-mf-4", LibraryID: 1, AlbumID: "fol-al-3", FolderID: disc1.ID, Path: "t/4.mp3", Missing: true})).To(Succeed()) + DeferCleanup(func() { + _, _ = GetDBXBuilder().NewQuery("DELETE FROM media_file WHERE id LIKE 'fol-mf-%'").Execute() + _, _ = GetDBXBuilder().NewQuery("DELETE FROM folder WHERE path LIKE 'ByFolder%' OR name = 'ByFolder'").Execute() + }) + }) + + It("returns the distinct album IDs of non-missing tracks in the folder", func() { + ids, err := mr.GetAlbumIDsByFolder(lib, disc1.ID) + Expect(err).ToNot(HaveOccurred()) + Expect(ids).To(ConsistOf("fol-al-1")) + }) + + It("also matches albums whose tracks are in a direct child of the folder", func() { + // A cover in the album root must reach the album whose tracks sit in CD1 + ids, err := mr.GetAlbumIDsByFolder(lib, albumRoot.ID) + Expect(err).ToNot(HaveOccurred()) + Expect(ids).To(ConsistOf("fol-al-1")) + }) + + It("does not match albums outside the folder", func() { + ids, err := mr.GetAlbumIDsByFolder(lib, albumRoot.ID) + Expect(err).ToNot(HaveOccurred()) + Expect(ids).ToNot(ContainElement("fol-al-2")) + }) + }) + Describe("GetCursor", func() { It("yields the same media files as GetAll", func() { opts := model.QueryOptions{Sort: "title"} diff --git a/scanner/folder_entry.go b/scanner/folder_entry.go index c7cc88ee1..0e893d6e6 100644 --- a/scanner/folder_entry.go +++ b/scanner/folder_entry.go @@ -10,21 +10,24 @@ import ( "slices" "time" + "github.com/navidrome/navidrome/core/artwork" "github.com/navidrome/navidrome/core/playlists" "github.com/navidrome/navidrome/model" "github.com/navidrome/navidrome/utils/chrono" ) -func newFolderEntry(job *scanJob, id, path string, updTime time.Time, hash string) *folderEntry { +func newFolderEntry(job *scanJob, id, path string, info model.FolderUpdateInfo) *folderEntry { f := &folderEntry{ - id: id, - job: job, - path: path, - audioFiles: make(map[string]fs.DirEntry), - imageFiles: make(map[string]fs.DirEntry), - albumIDMap: make(map[string]string), - updTime: updTime, - prevHash: hash, + id: id, + job: job, + path: path, + audioFiles: make(map[string]fs.DirEntry), + imageFiles: make(map[string]fs.DirEntry), + albumIDMap: make(map[string]string), + updTime: info.UpdatedAt, + prevHash: info.Hash, + prevImageFiles: info.ImageFiles, + prevImagesUpdatedAt: info.ImagesUpdatedAt, } return f } @@ -42,12 +45,15 @@ type folderEntry struct { numSubFolders int imagesUpdatedAt time.Time prevHash string // Previous hash from DB - tracks model.MediaFiles - albums model.Albums - albumIDMap map[string]string - artists model.Artists - tags model.TagList - missingTracks []*model.MediaFile + // Previous image state from DB, to detect image-only changes + prevImageFiles []string + prevImagesUpdatedAt time.Time + tracks model.MediaFiles + albums model.Albums + albumIDMap map[string]string + artists model.Artists + tags model.TagList + missingTracks []*model.MediaFile } func (f *folderEntry) hasNoFiles() bool { @@ -69,6 +75,21 @@ func (f *folderEntry) isOutdated() bool { return f.prevHash != f.hash() } +// imagesChanged reports whether the folder's image files differ from the previously persisted +// state, and whether an artist image is involved (present in the old or the new list). +func (f *folderEntry) imagesChanged() (changed, artistImage bool) { + newNames := slices.Sorted(maps.Keys(f.imageFiles)) + prevNames := slices.Sorted(slices.Values(f.prevImageFiles)) + // Both empty also skips the timestamp check, which is noise for image-less folders. + if len(prevNames) == 0 && len(newNames) == 0 { + return false, false + } + if slices.Equal(prevNames, newNames) && f.prevImagesUpdatedAt.Equal(f.imagesUpdatedAt) { + return false, false + } + return true, slices.ContainsFunc(slices.Concat(prevNames, newNames), artwork.IsArtistImageFile) +} + func (f *folderEntry) toFolder() *model.Folder { folder := model.NewFolder(f.job.lib, f.path) folder.NumAudioFiles = len(f.audioFiles) diff --git a/scanner/folder_entry_test.go b/scanner/folder_entry_test.go index 0328c6653..e8e354b38 100644 --- a/scanner/folder_entry_test.go +++ b/scanner/folder_entry_test.go @@ -41,7 +41,7 @@ var _ = Describe("folder_entry", func() { Hash: "previous-hash", } - entry := newFolderEntry(job, folderID, path, updateInfo.UpdatedAt, updateInfo.Hash) + entry := newFolderEntry(job, folderID, path, updateInfo) Expect(entry.id).To(Equal(folderID)) Expect(entry.job).To(Equal(job)) @@ -76,7 +76,7 @@ var _ = Describe("folder_entry", func() { BeforeEach(func() { folderID := model.FolderID(lib, path) - entry = newFolderEntry(job, folderID, path, time.Time{}, "") + entry = newFolderEntry(job, folderID, path, model.FolderUpdateInfo{}) }) Describe("hasNoFiles", func() { @@ -457,7 +457,7 @@ var _ = Describe("folder_entry", func() { // Create new folder entry folderPath := "music/rock/album" folderID := model.FolderID(lib, folderPath) - entry := newFolderEntry(job, folderID, folderPath, time.Time{}, "") + entry := newFolderEntry(job, folderID, folderPath, model.FolderUpdateInfo{}) // Initially new and has no files Expect(entry.isNew()).To(BeTrue()) diff --git a/scanner/image_changes.go b/scanner/image_changes.go new file mode 100644 index 000000000..a9c4de365 --- /dev/null +++ b/scanner/image_changes.go @@ -0,0 +1,112 @@ +package scanner + +import ( + "context" + "sync" + + "github.com/navidrome/navidrome/consts" + "github.com/navidrome/navidrome/log" + "github.com/navidrome/navidrome/model" +) + +// imageChangedFolder records a folder whose image files changed during the scan, so the +// affected albums/artists can be re-enqueued for artwork resolution at the end of phase 1. +type imageChangedFolder struct { + id string + path string + artistImage bool +} + +// imageChangeCollector gathers those folders per library while phase 1 persists them, then turns +// them into artwork queue items once. +type imageChangeCollector struct { + libs map[int]model.Library + folders map[int][]imageChangedFolder + ds model.DataStore + mutex sync.Mutex +} + +func (c *imageChangeCollector) record(lib model.Library, folder imageChangedFolder) { + c.mutex.Lock() + defer c.mutex.Unlock() + if c.folders == nil { + c.folders = map[int][]imageChangedFolder{} + c.libs = map[int]model.Library{} + } + c.libs[lib.ID] = lib + c.folders[lib.ID] = append(c.folders[lib.ID], folder) +} + +// enqueue is best-effort: failures are logged and never fail the scan. +func (c *imageChangeCollector) enqueue(ctx context.Context) { + c.mutex.Lock() + foldersMap, libsMap := c.folders, c.libs + c.folders, c.libs = nil, nil + c.mutex.Unlock() + + for libID, folders := range foldersMap { + lib := libsMap[libID] + items, err := c.queueItems(ctx, lib, folders) + if err != nil { + log.Warn(ctx, "Scanner: could not map image changes to artwork items", "lib", lib.Name, err) + continue + } + if len(items) == 0 { + continue + } + if err := c.ds.ArtworkQueue(ctx).Enqueue(items...); err != nil { + log.Warn(ctx, "Scanner: could not enqueue artwork for image changes", "lib", lib.Name, err) + continue + } + log.Debug(ctx, "Scanner: Enqueued artwork resolution for image changes", "lib", lib.Name, + "changedFolders", len(folders), "items", len(items)) + } +} + +func (c *imageChangeCollector) queueItems(ctx context.Context, lib model.Library, + folders []imageChangedFolder, +) ([]model.ArtworkQueueItem, error) { + folderIDs := make([]string, len(folders)) + var artistFolderPaths []string + for i, f := range folders { + folderIDs[i] = f.id + if f.artistImage { + artistFolderPaths = append(artistFolderPaths, f.path) + } + } + + var items []model.ArtworkQueueItem + + albumIDs, err := c.ds.MediaFile(ctx).GetAlbumIDsByFolder(lib, folderIDs...) + if err != nil { + return nil, err + } + for _, id := range albumIDs { + items = append(items, scanArtworkItem(model.KindAlbumArtwork, id)) + } + + if len(artistFolderPaths) == 0 { + return items, nil + } + // The resolver climbs to the library root, so the subtree below the folder is the affected set. + // A failure here must not discard the album items already collected. + artistIDs, err := c.ds.Album(ctx).GetSoleAlbumArtistIDsInSubtrees(lib, artistFolderPaths...) + if err != nil { + log.Warn(ctx, "Scanner: could not map image changes to artists", "lib", lib.Name, err) + return items, nil + } + for _, id := range artistIDs { + if id == "" || id == consts.UnknownArtistID || id == consts.VariousArtistsID { + continue + } + items = append(items, scanArtworkItem(model.KindArtistArtwork, id)) + } + return items, nil +} + +func scanArtworkItem(kind model.Kind, id string) model.ArtworkQueueItem { + return model.ArtworkQueueItem{ + ItemKind: kind.Prefix(), ItemID: id, ImageType: model.ImageTypePrimary, + Priority: model.ArtworkPriorityScan, + } +} diff --git a/scanner/phase_1_folders.go b/scanner/phase_1_folders.go index e853948e6..82e91d5ad 100644 --- a/scanner/phase_1_folders.go +++ b/scanner/phase_1_folders.go @@ -45,7 +45,7 @@ func createPhaseFolders(ctx context.Context, state *scanState, ds model.DataStor jobs = append(jobs, job) } - return &phaseFolders{jobs: jobs, ctx: ctx, ds: ds, state: state} + return &phaseFolders{jobs: jobs, ctx: ctx, ds: ds, state: state, imageChanges: &imageChangeCollector{ds: ds}} } type scanJob struct { @@ -105,7 +105,7 @@ func (j *scanJob) popLastUpdate(folderID string) model.FolderUpdateInfo { func (j *scanJob) createFolderEntry(path string) *folderEntry { id := model.FolderID(j.lib, path) info := j.popLastUpdate(id) - return newFolderEntry(j, id, path, info.UpdatedAt, info.Hash) + return newFolderEntry(j, id, path, info) } // phaseFolders represents the first phase of the scanning process, which is responsible @@ -125,6 +125,7 @@ type phaseFolders struct { ctx context.Context state *scanState prevAlbumPIDConf string + imageChanges *imageChangeCollector } func (p *phaseFolders) description() string { @@ -169,8 +170,10 @@ func (p *phaseFolders) producer() ppl.Producer[*folderEntry] { // Check if folder is outdated if folder.isOutdated() { if !p.state.fullScan { - if folder.hasNoFiles() && folder.isNew() { - log.Trace(p.ctx, "Scanner: Skipping new folder with no files", "folder", folder.path, "lib", job.lib.Name) + // Ancestor folders need a row even with no files of their own: artwork + // resolution climbs them, and an image added later needs a state to diff. + if folder.isEmpty() && folder.isNew() { + log.Trace(p.ctx, "Scanner: Skipping new empty folder", "folder", folder.path, "lib", job.lib.Name) continue } log.Debug(p.ctx, "Scanner: Detected changes in folder", "folder", folder.path, "lastUpdate", folder.modTime, "lib", job.lib.Name) @@ -197,7 +200,8 @@ func (p *phaseFolders) measure(entry *folderEntry) func() time.Duration { func (p *phaseFolders) stages() []ppl.Stage[*folderEntry] { return []ppl.Stage[*folderEntry]{ ppl.NewStage(p.processFolder, ppl.Name("process folder"), ppl.Concurrency(conf.Server.DevScannerThreads)), - ppl.NewStage(p.persistChanges, ppl.Name("persist changes")), + // persistChanges is not reentrant, so it always has to run with concurrency=1 + ppl.NewStage(p.persistChanges, ppl.Name("persist changes"), ppl.Concurrency(1)), ppl.NewStage(p.logFolder, ppl.Name("log results")), } } @@ -339,6 +343,15 @@ func (p *phaseFolders) persistChanges(entry *folderEntry) (*folderEntry, error) albumRepo := tx.Album(p.ctx) mfRepo := tx.MediaFile(p.ctx) + // A new folder's albums/artists are enqueued below; only pre-existing folders need the diff. + if !entry.isNew() { + if changed, artistImage := entry.imagesChanged(); changed { + p.imageChanges.record(entry.job.lib, imageChangedFolder{ + id: entry.id, path: entry.path, artistImage: artistImage, + }) + } + } + // Save folder to DB folder := entry.toFolder() err := folderRepo.Put(folder) @@ -368,10 +381,7 @@ func (p *phaseFolders) persistChanges(entry *folderEntry) (*folderEntry, error) return err } if entry.artists[i].Name != consts.UnknownArtist && entry.artists[i].Name != consts.VariousArtists { - queueItems = append(queueItems, model.ArtworkQueueItem{ - ItemKind: model.KindArtistArtwork.Prefix(), ItemID: entry.artists[i].ID, ImageType: model.ImageTypePrimary, - Priority: model.ArtworkPriorityScan, - }) + queueItems = append(queueItems, scanArtworkItem(model.KindArtistArtwork, entry.artists[i].ID)) } } @@ -383,10 +393,7 @@ func (p *phaseFolders) persistChanges(entry *folderEntry) (*folderEntry, error) return err } if entry.albums[i].Name != consts.UnknownAlbum { - queueItems = append(queueItems, model.ArtworkQueueItem{ - ItemKind: model.KindAlbumArtwork.Prefix(), ItemID: entry.albums[i].ID, ImageType: model.ImageTypePrimary, - Priority: model.ArtworkPriorityScan, - }) + queueItems = append(queueItems, scanArtworkItem(model.KindAlbumArtwork, entry.albums[i].ID)) } } @@ -518,6 +525,7 @@ func (p *phaseFolders) finalize(err error) error { } return nil }, "scanner: finalize phaseFolders") + p.imageChanges.enqueue(p.ctx) return errors.Join(err, errF) } diff --git a/scanner/scanner_test.go b/scanner/scanner_test.go index 6098cbd66..00f91699d 100644 --- a/scanner/scanner_test.go +++ b/scanner/scanner_test.go @@ -278,6 +278,131 @@ var _ = Describe("Scanner", Ordered, func() { }) }) + Context("Library with image files", func() { + var fsys storagetest.FakeFS + image := func(data string) *fstest.MapFile { return &fstest.MapFile{Data: []byte(data)} } + + albumID := func(name string) string { + GinkgoHelper() + albums, err := ds.Album(ctx).GetAll(model.QueryOptions{Filters: squirrel.Eq{"album.name": name}}) + Expect(err).ToNot(HaveOccurred()) + Expect(albums).To(HaveLen(1)) + return albums[0].ID + } + artistID := func(name string) string { + GinkgoHelper() + artists, err := ds.Artist(ctx).GetAll(model.QueryOptions{Filters: squirrel.Eq{"artist.name": name}}) + Expect(err).ToNot(HaveOccurred()) + Expect(artists).To(HaveLen(1)) + return artists[0].ID + } + queuedItems := func() []model.ArtworkQueueItem { + GinkgoHelper() + queued, err := ds.ArtworkQueue(ctx).DequeueBatch(1000) + Expect(err).ToNot(HaveOccurred()) + return queued + } + queueItemFor := func(kind, id string) OmegaMatcher { + return ContainElement(SatisfyAll( + HaveField("ItemKind", kind), + HaveField("ItemID", id), + HaveField("Priority", model.ArtworkPriorityScan), + )) + } + + BeforeEach(func() { + revolver := template(_t{"albumartist": "The Beatles", "album": "Revolver", "year": 1966}) + wall := template(_t{"albumartist": "Pink Floyd", "album": "The Wall", "year": 1979}) + fsys = createFS(fstest.MapFS{ + "The Beatles/artist.jpg": image("beatles-artist-v1"), + "The Beatles/Revolver/cover.jpg": image("revolver-cover-v1"), + "The Beatles/Revolver/01 - Taxman.mp3": revolver(track(1, "Taxman")), + "Pink Floyd/The Wall/cover.jpg": image("wall-cover-v1"), + "Pink Floyd/The Wall/CD1/01 - In the Flesh.mp3": wall(track(1, "In the Flesh?")), + "Pink Floyd/The Wall/CD2/01 - Hey You.mp3": wall(track(1, "Hey You")), + }) + Expect(runScanner(ctx, true)).To(Succeed()) + resolveQueuedArtwork() + }) + + It("re-enqueues only the album whose cover was replaced in place", func() { + fsys.Add("The Beatles/Revolver/cover.jpg", image("revolver-cover-v2")) + + Expect(runScanner(ctx, false)).To(Succeed()) + + queued := queuedItems() + Expect(queued).To(queueItemFor("al", albumID("Revolver"))) + Expect(queued).ToNot(ContainElement(HaveField("ItemID", albumID("The Wall")))) + Expect(queued).ToNot(ContainElement(HaveField("ItemKind", "ar"))) + }) + + It("re-enqueues the album when the cover above its disc folders changes", func() { + fsys.Add("Pink Floyd/The Wall/cover.jpg", image("wall-cover-v2")) + + Expect(runScanner(ctx, false)).To(Succeed()) + + Expect(queuedItems()).To(queueItemFor("al", albumID("The Wall"))) + }) + + It("re-enqueues the album when its cover is removed", func() { + fsys.Remove("The Beatles/Revolver/cover.jpg") + + Expect(runScanner(ctx, false)).To(Succeed()) + + Expect(queuedItems()).To(queueItemFor("al", albumID("Revolver"))) + }) + + It("enqueues the artist when an artist image is added to their folder", func() { + fsys.Add("Pink Floyd/artist.jpg", image("floyd-artist-v1")) + + Expect(runScanner(ctx, false)).To(Succeed()) + + queued := queuedItems() + Expect(queued).To(queueItemFor("ar", artistID("Pink Floyd"))) + Expect(queued).ToNot(ContainElement(HaveField("ItemID", artistID("The Beatles")))) + }) + + It("re-enqueues the artist when their artist image is replaced in place", func() { + fsys.Add("The Beatles/artist.jpg", image("beatles-artist-v2")) + + Expect(runScanner(ctx, false)).To(Succeed()) + + Expect(queuedItems()).To(queueItemFor("ar", artistID("The Beatles"))) + }) + + It("enqueues every artist under the folder when a shared artist image is added", func() { + fsys.Add("artist.png", image("shared-artist-v1")) + + Expect(runScanner(ctx, false)).To(Succeed()) + + queued := queuedItems() + Expect(queued).To(queueItemFor("ar", artistID("The Beatles"))) + Expect(queued).To(queueItemFor("ar", artistID("Pink Floyd"))) + }) + + It("enqueues the artist when an image lands in a folder first seen by a quick scan", func() { + // A quick scan must persist an artist folder that holds only subfolders, or the + // artist.jpg added later has no previous state to diff against. + kraftwerk := template(_t{"albumartist": "Kraftwerk", "album": "Autobahn", "year": 1974}) + files := fsys.MapFS + files["Kraftwerk/Autobahn/01 - Autobahn.mp3"] = kraftwerk(track(1, "Autobahn")) + fsys.SetFiles(files) + Expect(runScanner(ctx, false)).To(Succeed()) + resolveQueuedArtwork() + + fsys.Add("Kraftwerk/artist.jpg", image("kraftwerk-artist-v1")) + Expect(runScanner(ctx, false)).To(Succeed()) + + Expect(queuedItems()).To(queueItemFor("ar", artistID("Kraftwerk"))) + }) + + It("does not enqueue anything on a repeat full scan with no image changes", func() { + Expect(runScanner(ctx, true)).To(Succeed()) + + Expect(queuedItems()).To(BeEmpty()) + }) + }) + Context("Artist with atomic non-ASCII letters, 'GØGGS'", func() { BeforeEach(func() { goggs := template(_t{"albumartist": "GØGGS", "album": "Pre Strike Sweep", "year": 2018}) From ea1e2b95a70074a8e4e98074736f13f0b5ea72a5 Mon Sep 17 00:00:00 2001 From: hotorcelexo Date: Sun, 16 Aug 2026 21:14:41 +0300 Subject: [PATCH 10/27] fix(db): keep album created_at in the driver's timestamp format when copying (#5867) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(db): keep album created_at in the driver's timestamp format when copying Signed-off-by: IgorPolyakov * fix(db): move created_at renormalize migration after merged migrations The migration was versioned 20260813140000, which is older than 20260815015320 (already merged). goose.UpContext runs without WithAllowMissing, so any database that already applied the newer migration would fail with "found 1 missing migrations" and db.Init would log.Fatal on startup. --------- Signed-off-by: IgorPolyakov Co-authored-by: Deluan Quintão --- ...816180040_renormalize_album_created_at.sql | 11 ++++++++ persistence/album_repository.go | 5 +++- persistence/album_repository_test.go | 26 +++++++++++++++++++ 3 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 db/migrations/20260816180040_renormalize_album_created_at.sql diff --git a/db/migrations/20260816180040_renormalize_album_created_at.sql b/db/migrations/20260816180040_renormalize_album_created_at.sql new file mode 100644 index 000000000..9af7b2e5a --- /dev/null +++ b/db/migrations/20260816180040_renormalize_album_created_at.sql @@ -0,0 +1,11 @@ +-- +goose Up + +-- Repairs album.created_at values stored in RFC3339 T-format by CopyAttributes. +-- These values sort incorrectly in "Recently Added", which compares timestamps as raw strings. + +UPDATE album SET created_at = replace(replace(created_at, 'T', ' '), 'Z', '+00:00') +WHERE created_at LIKE '%T%'; + +-- +goose Down + +SELECT 1; diff --git a/persistence/album_repository.go b/persistence/album_repository.go index 2f0621b73..5d7aad22e 100644 --- a/persistence/album_repository.go +++ b/persistence/album_repository.go @@ -329,8 +329,11 @@ func (r *albumRepository) GetYears(libraryIDs ...int) ([]int, error) { } func (r *albumRepository) CopyAttributes(fromID, toID string, columns ...string) error { + // Cast values to text so go-sqlite3 does not decode datetime columns as time.Time + // and reformat them as RFC3339 when written back. + sel := slice.Map(columns, func(c string) string { return fmt.Sprintf("cast(%[1]s as text) as %[1]s", c) }) var from dbx.NullStringMap - err := r.queryOne(Select(columns...).From(r.tableName).Where(Eq{"id": fromID}), &from) + err := r.queryOne(Select(sel...).From(r.tableName).Where(Eq{"id": fromID}), &from) if err != nil { return fmt.Errorf("getting album to copy fields from: %w", err) } diff --git a/persistence/album_repository_test.go b/persistence/album_repository_test.go index 526642aa6..f6768768d 100644 --- a/persistence/album_repository_test.go +++ b/persistence/album_repository_test.go @@ -19,6 +19,16 @@ import ( . "github.com/onsi/gomega" ) +// rawColumn returns a column exactly as stored, bypassing go-sqlite3's decoding of +// `datetime` columns into time.Time. +func rawColumn(r sqlRepository, id, column string) string { + var res struct{ Value string } + sel := squirrel.Select("cast(" + column + " as text) as value"). + From(r.tableName).Where(squirrel.Eq{"id": id}) + ExpectWithOffset(1, r.queryOne(sel, &res)).To(Succeed()) + return res.Value +} + var _ = Describe("AlbumRepository", func() { var albumRepo *albumRepository var ctx context.Context @@ -69,6 +79,22 @@ var _ = Describe("AlbumRepository", func() { Expect(err).ToNot(HaveOccurred()) Expect(got.CreatedAt).To(BeTemporally("~", dstTime, time.Second)) }) + It("returns not found and leaves destination untouched when source does not exist", func() { + err := albumRepo.CopyAttributes("copy-missing", "copy-dst", "created_at") + Expect(errors.Is(err, model.ErrNotFound)).To(BeTrue()) + got, getErr := albumRepo.Get("copy-dst") + Expect(getErr).ToNot(HaveOccurred()) + Expect(got.CreatedAt).To(BeTemporally("~", dstTime, time.Second)) + }) + It("keeps the copied created_at in the driver's space-separated format", func() { + // Copying through a Go string would rewrite it as RFC3339 ("2020-01-02T03:04:05Z"), + // which string-sorts above every space-format timestamp and pins the album to the + // top of "Recently Added". + Expect(albumRepo.CopyAttributes("copy-src", "copy-dst", "created_at")).To(Succeed()) + Expect(rawColumn(albumRepo.sqlRepository, "copy-dst", "created_at")). + To(Equal(rawColumn(albumRepo.sqlRepository, "copy-src", "created_at"))) + Expect(rawColumn(albumRepo.sqlRepository, "copy-dst", "created_at")).ToNot(ContainSubstring("T")) + }) }) Describe("GetCursor", func() { From 65751d7665e3f1a85c96f661a10227a77bb655d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Deluan=20Quint=C3=A3o?= Date: Mon, 17 Aug 2026 15:47:41 -0400 Subject: [PATCH 11/27] fix(playlists): chunk track deletes to stay under the SQLite variable limit (#5977) PlaylistTrackRepository.Delete built a single IN clause with one bind variable per track, so removing more tracks than SQLITE_MAX_VARIABLE_NUMBER (32766) failed with "too many SQL variables". Clients that sync a large playlist by adding the desired tracks and then removing the stale ones would get the add committed and the removal rejected, leaving the playlist with both sets of tracks and growing it on every sync. Delete now works in chunks of 200, the same size addTracks already uses, and renumbers once after the last chunk. Both callers already run inside a transaction, so the delete stays atomic. --- persistence/playlist_track_repository.go | 11 ++-- persistence/playlist_track_repository_test.go | 50 +++++++++++++++++++ 2 files changed, 58 insertions(+), 3 deletions(-) diff --git a/persistence/playlist_track_repository.go b/persistence/playlist_track_repository.go index c1f6fcf69..a5e1975fd 100644 --- a/persistence/playlist_track_repository.go +++ b/persistence/playlist_track_repository.go @@ -2,6 +2,7 @@ package persistence import ( "database/sql" + "slices" . "github.com/Masterminds/squirrel" "github.com/deluan/rest" @@ -224,10 +225,14 @@ func (r *playlistTrackRepository) AddDiscs(discs []model.DiscID) (int, error) { return r.addMediaFileIds(clauses) } +// deleteChunkSize keeps each DELETE under SQLITE_MAX_VARIABLE_NUMBER, matching addTracks. +const deleteChunkSize = 200 + func (r *playlistTrackRepository) Delete(ids ...string) error { - err := r.delete(And{Eq{"playlist_id": r.playlistId}, Eq{"id": ids}}) - if err != nil { - return err + for chunk := range slices.Chunk(ids, deleteChunkSize) { + if err := r.delete(And{Eq{"playlist_id": r.playlistId}, Eq{"id": chunk}}); err != nil { + return err + } } return r.playlistRepo.renumber(r.playlistId) diff --git a/persistence/playlist_track_repository_test.go b/persistence/playlist_track_repository_test.go index a5c67b92c..88ddbff48 100644 --- a/persistence/playlist_track_repository_test.go +++ b/persistence/playlist_track_repository_test.go @@ -1,6 +1,8 @@ package persistence import ( + "strconv" + "github.com/navidrome/navidrome/log" "github.com/navidrome/navidrome/model" "github.com/navidrome/navidrome/model/request" @@ -8,6 +10,9 @@ import ( . "github.com/onsi/gomega" ) +// sqliteMaxVariables is SQLITE_MAX_VARIABLE_NUMBER as compiled into the driver +const sqliteMaxVariables = 32766 + var _ = Describe("PlaylistTrackRepository", func() { var repo model.PlaylistTrackRepository @@ -72,4 +77,49 @@ var _ = Describe("PlaylistTrackRepository", func() { To(Equal([]string{songRadioactivity.ID})) }) }) + + Describe("Delete", func() { + var tracks model.PlaylistTrackRepository + const numTracks = deleteChunkSize*2 + 1 + + positionsUpTo := func(n int) []string { + positions := make([]string, 0, n) + for i := 1; i <= n; i++ { + positions = append(positions, strconv.Itoa(i)) + } + return positions + } + + BeforeEach(func() { + ctx := log.NewContext(GinkgoT().Context()) + ctx = request.WithUser(ctx, model.User{ID: "userid", UserName: "userid", IsAdmin: true}) + plsRepo := NewPlaylistRepository(ctx, GetDBXBuilder()) + + pls := model.Playlist{Name: "Chunked Delete", OwnerID: "userid", OwnerName: "userid"} + Expect(plsRepo.Put(&pls)).To(Succeed()) + DeferCleanup(func() { Expect(plsRepo.Delete(pls.ID)).To(Succeed()) }) + + tracks = plsRepo.Tracks(pls.ID, false) + songIds := make([]string, numTracks) + for i := range songIds { + songIds[i] = songDayInALife.ID + } + Expect(tracks.Add(songIds)).To(Equal(numTracks)) + }) + + It("removes positions spanning several chunks, and renumbers what is left", func() { + Expect(tracks.Delete(positionsUpTo(numTracks - 1)...)).To(Succeed()) + + Expect(tracks.CountAll()).To(Equal(int64(1))) + remaining, err := tracks.GetAll(model.QueryOptions{Sort: "id"}) + Expect(err).ToNot(HaveOccurred()) + Expect(remaining[0].ID).To(Equal("1"), "the surviving track must be renumbered to position 1") + }) + + It("accepts more ids than SQLite allows as bind variables", func() { + Expect(tracks.Delete(positionsUpTo(sqliteMaxVariables + 100)...)).To(Succeed()) + + Expect(tracks.CountAll()).To(BeZero()) + }) + }) }) From 4c0ab074a36f0450a4ac618e8a74ae8d18eeb4e7 Mon Sep 17 00:00:00 2001 From: Deluan Date: Mon, 17 Aug 2026 20:36:13 -0400 Subject: [PATCH 12/27] chore(deps): update module dependencies in `go.mod` and `go.sum` to latest versions Signed-off-by: Deluan --- go.mod | 58 ++++++++++++------------ go.sum | 136 ++++++++++++++++++++++++++++----------------------------- 2 files changed, 96 insertions(+), 98 deletions(-) diff --git a/go.mod b/go.mod index b72f4035c..65a01a9b5 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/bmatcuk/doublestar/v4 v4.10.0 github.com/deluan/rest v0.0.0-20211102003136-6260bc399cbf github.com/deluan/sanitize v0.0.0-20241120162836-fdfd8fdfaa55 - github.com/dexterlb/mpvipc v0.0.0-20241005113212-7cdefca0e933 + github.com/dexterlb/mpvipc v0.0.0-20260722094525-0cf47d745b36 github.com/djherbis/atime v1.1.0 github.com/djherbis/fscache v0.10.2-0.20231127215153-442a07e326c4 github.com/djherbis/stream v1.4.0 @@ -26,7 +26,7 @@ require ( github.com/go-chi/jwtauth/v5 v5.4.0 github.com/go-viper/encoding/ini v0.1.1 github.com/go-viper/mapstructure/v2 v2.5.0 - github.com/gohugoio/hashstructure v0.6.0 + github.com/gohugoio/hashstructure v1.0.0 github.com/google/go-pipeline v0.0.0-20230411140531-6cbedfc1d3fc github.com/google/uuid v1.6.0 github.com/google/wire v0.7.0 @@ -35,37 +35,37 @@ require ( github.com/jellydator/ttlcache/v3 v3.4.1 github.com/kardianos/service v1.3.0 github.com/kr/pretty v0.3.1 - github.com/lestrrat-go/jwx/v3 v3.1.1 - github.com/mattn/go-sqlite3 v1.14.48 + github.com/lestrrat-go/jwx/v3 v3.2.0 + github.com/mattn/go-sqlite3 v1.14.50 github.com/microcosm-cc/bluemonday v1.0.27 github.com/mileusna/useragent v1.3.5 - github.com/onsi/ginkgo/v2 v2.32.0 + github.com/onsi/ginkgo/v2 v2.32.1 github.com/onsi/gomega v1.42.1 github.com/pelletier/go-toml/v2 v2.4.3 github.com/pmezard/go-difflib v1.0.0 github.com/pocketbase/dbx v1.12.0 - github.com/pressly/goose/v3 v3.27.2 - github.com/prometheus/client_golang v1.23.2 + github.com/pressly/goose/v3 v3.27.3 + github.com/prometheus/client_golang v1.24.1 github.com/rjeczalik/notify v0.9.3 github.com/robfig/cron/v3 v3.0.1 github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 - github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 - github.com/sirupsen/logrus v1.9.4 + github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 + github.com/sirupsen/logrus v1.10.0 github.com/spf13/cobra v1.10.2 github.com/spf13/viper v1.21.0 - github.com/stretchr/testify v1.11.1 + github.com/stretchr/testify v1.12.0 github.com/tetratelabs/wazero v1.12.0 github.com/unrolled/secure v1.17.0 github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342 github.com/zeebo/xxh3 v1.1.0 go.senan.xyz/taglib v0.11.1 go.uber.org/goleak v1.3.0 - golang.org/x/image v0.44.0 - golang.org/x/net v0.57.0 + golang.org/x/image v0.45.0 + golang.org/x/net v0.58.0 golang.org/x/sync v0.22.0 golang.org/x/sys v0.47.0 golang.org/x/term v0.45.0 - golang.org/x/text v0.40.0 + golang.org/x/text v0.41.0 golang.org/x/time v0.15.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -79,22 +79,21 @@ require ( github.com/cespare/reflex v0.3.2 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/creack/pty v1.1.24 // indirect - github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1 // indirect github.com/dylibso/observe-sdk/go v0.0.0-20240828172851-9145d8ad07e1 // indirect - github.com/ebitengine/purego v0.10.1 // indirect + github.com/ebitengine/purego v0.10.2 // indirect github.com/fsnotify/fsnotify v1.10.1 // indirect - github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/logr v1.4.4 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/gobwas/glob v0.2.3 // indirect github.com/goccy/go-json v0.10.6 // indirect github.com/goccy/go-yaml v1.19.2 // indirect github.com/google/go-cmp v0.7.0 // indirect - github.com/google/pprof v0.0.0-20260709232956-b9395ee17fa0 // indirect + github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 // indirect github.com/google/subcommands v1.2.0 // indirect github.com/gorilla/css v1.0.1 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect - github.com/ianlancetaylor/demangle v0.0.0-20260505044615-1ff4bf46051f // indirect + github.com/ianlancetaylor/demangle v0.0.0-20260724033716-83e58baca724 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/klauspost/cpuid/v2 v2.4.0 // indirect @@ -114,13 +113,13 @@ require ( github.com/ogier/pflag v0.0.1 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_model v0.6.2 // indirect - github.com/prometheus/common v0.67.5 // indirect - github.com/prometheus/procfs v0.20.1 // indirect - github.com/rogpeppe/go-internal v1.15.0 // indirect + github.com/prometheus/common v0.70.1 // indirect + github.com/prometheus/procfs v0.21.1 // indirect + github.com/rogpeppe/go-internal v1.16.0 // indirect github.com/sagikazarmark/locafero v0.12.0 // indirect github.com/sanity-io/litter v1.5.8 // indirect github.com/segmentio/asm v1.2.1 // indirect - github.com/sethvargo/go-retry v0.3.0 // indirect + github.com/sethvargo/go-retry v0.4.0 // indirect github.com/sosodev/duration v1.3.1 // indirect github.com/spf13/afero v1.15.0 // indirect github.com/spf13/cast v1.10.0 // indirect @@ -129,15 +128,14 @@ require ( github.com/subosito/gotenv v1.6.0 // indirect github.com/tetratelabs/wabin v0.0.0-20230304001439-f6f874872834 // indirect github.com/valyala/fastjson v1.6.10 // indirect - go.opentelemetry.io/proto/otlp v1.10.0 // indirect + go.opentelemetry.io/proto/otlp v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect - go.yaml.in/yaml/v2 v2.4.3 // indirect - go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/crypto v0.54.0 // indirect - golang.org/x/mod v0.38.0 // indirect - golang.org/x/telemetry v0.0.0-20260708182218-49f421fb7959 // indirect - golang.org/x/tools v0.48.0 // indirect - google.golang.org/protobuf v1.36.11 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect + golang.org/x/crypto v0.55.0 // indirect + golang.org/x/mod v0.40.0 // indirect + golang.org/x/telemetry v0.0.0-20260811182544-a038080d80e5 // indirect + golang.org/x/tools v0.49.0 // indirect + google.golang.org/protobuf v1.36.12 // indirect gopkg.in/ini.v1 v1.67.3 // indirect gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect ) diff --git a/go.sum b/go.sum index 29983a27d..c11f4bb20 100644 --- a/go.sum +++ b/go.sum @@ -27,8 +27,6 @@ github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfv github.com/davecgh/go-spew v0.0.0-20161028175848-04cdfd42973b/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1 h1:5RVFMOWjMyRy8cARdy79nAmgYw3hK/4HUq48LQ6Wwqo= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40= github.com/deluan/go-taglib v0.0.0-20260720134629-a133b9719ea3 h1:j7eSXqgtjhlNfwnMEzRdXnJGZTEw4I7J9TeQAll83bU= @@ -37,8 +35,8 @@ github.com/deluan/rest v0.0.0-20211102003136-6260bc399cbf h1:tb246l2Zmpt/GpF9EcH github.com/deluan/rest v0.0.0-20211102003136-6260bc399cbf/go.mod h1:tSgDythFsl0QgS/PFWfIZqcJKnkADWneY80jaVRlqK8= github.com/deluan/sanitize v0.0.0-20241120162836-fdfd8fdfaa55 h1:wSCnggTs2f2ji6nFwQmfwgINcmSMj0xF0oHnoyRSPe4= github.com/deluan/sanitize v0.0.0-20241120162836-fdfd8fdfaa55/go.mod h1:ZNCLJfehvEf34B7BbLKjgpsL9lyW7q938w/GY1XgV4E= -github.com/dexterlb/mpvipc v0.0.0-20241005113212-7cdefca0e933 h1:r4hxcT6GBIA/j8Ox4OXI5MNgMKfR+9plcAWYi1OnmOg= -github.com/dexterlb/mpvipc v0.0.0-20241005113212-7cdefca0e933/go.mod h1:RkQWLNITKkXHLP7LXxZSgEq+uFWU25M5qW7qfEhL9Wc= +github.com/dexterlb/mpvipc v0.0.0-20260722094525-0cf47d745b36 h1:KtPfdSST6e0vJbMzMmVqPa5mO1u8vMBlybRCW2ieXpA= +github.com/dexterlb/mpvipc v0.0.0-20260722094525-0cf47d745b36/go.mod h1:RkQWLNITKkXHLP7LXxZSgEq+uFWU25M5qW7qfEhL9Wc= github.com/djherbis/atime v1.1.0 h1:rgwVbP/5by8BvvjBNrbh64Qz33idKT3pSnMSJsxhi0g= github.com/djherbis/atime v1.1.0/go.mod h1:28OF6Y8s3NQWwacXc5eZTsEsiMzp7LF8MbXE+XJPdBE= github.com/djherbis/fscache v0.10.2-0.20231127215153-442a07e326c4 h1:wdZllsLrDJtYfHiAKogB4PNHSDeO+v+5S3eqSWHGDlc= @@ -53,8 +51,8 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/dylibso/observe-sdk/go v0.0.0-20240828172851-9145d8ad07e1 h1:idfl8M8rPW93NehFw5H1qqH8yG158t5POr+LX9avbJY= github.com/dylibso/observe-sdk/go v0.0.0-20240828172851-9145d8ad07e1/go.mod h1:C8DzXehI4zAbrdlbtOByKX6pfivJTBiV9Jjqv56Yd9Q= -github.com/ebitengine/purego v0.10.1 h1:dewVBCBT2GaMu1SrNTYxQhgQBethzfhiwvZiLGP/qyY= -github.com/ebitengine/purego v0.10.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= +github.com/ebitengine/purego v0.10.2 h1:W809HbnvzAxgdm+aOvlSekrM16wGCdT/e76+9tS7gzE= +github.com/ebitengine/purego v0.10.2/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= github.com/extism/go-sdk v1.7.1 h1:lWJos6uY+tRFdlIHR+SJjwFDApY7OypS/2nMhiVQ9Sw= github.com/extism/go-sdk v1.7.1/go.mod h1:IT+Xdg5AZM9hVtpFUA+uZCJMge/hbvshl8bwzLtFyKA= github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= @@ -79,8 +77,8 @@ github.com/go-chi/httprate v0.16.0 h1:8V5DH9j6pSK6UQoBsTpvMyFxycqaKEIToyPKzHJjUa github.com/go-chi/httprate v0.16.0/go.mod h1:A8lo+qRhk+s9LiuP5saS7XCGDXRXMcrueq0NfIuCa/I= github.com/go-chi/jwtauth/v5 v5.4.0 h1:Ieh0xMJsFvqylqJ02/mQHKzbbKO9DYNBh4DPKCwTwYI= github.com/go-chi/jwtauth/v5 v5.4.0/go.mod h1:w6yjqUUXz1b8+oiJel64Sz1KJwduQM6qUA5QNzO5+bQ= -github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= -github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= +github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.10.0 h1:Q+1LV8DkHJvSYAdR83XzuhDaTykuDx0l6fkXxoWCWfw= github.com/go-sql-driver/mysql v1.10.0/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk= @@ -96,15 +94,15 @@ github.com/goccy/go-json v0.10.6 h1:p8HrPJzOakx/mn/bQtjgNjdTcN+/S6FcG2CTtQOrHVU= github.com/goccy/go-json v0.10.6/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= -github.com/gohugoio/hashstructure v0.6.0 h1:7wMB/2CfXoThFYhdWRGv3u3rUM761Cq29CxUW+NltUg= -github.com/gohugoio/hashstructure v0.6.0/go.mod h1:lapVLk9XidheHG1IQ4ZSbyYrXcaILU1ZEP/+vno5rBQ= +github.com/gohugoio/hashstructure v1.0.0 h1:vWYuyzs1n0LdI0F54TJQeYAiB44fHX7H9hCp9X6gHKg= +github.com/gohugoio/hashstructure v1.0.0/go.mod h1:FSbTK4QwxucJ2bC4Lvrs9a6x0DbQDXNoyBO+h4nlCgE= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/go-pipeline v0.0.0-20230411140531-6cbedfc1d3fc h1:hd+uUVsB1vdxohPneMrhGH2YfQuH5hRIK9u4/XCeUtw= github.com/google/go-pipeline v0.0.0-20230411140531-6cbedfc1d3fc/go.mod h1:SL66SJVysrh7YbDCP9tH30b8a9o/N2HeiQNUm85EKhc= -github.com/google/pprof v0.0.0-20260709232956-b9395ee17fa0 h1:du0WGc8xSKq/++e0cglxhS/mXVqsR7+c7jLEi5Vqduw= -github.com/google/pprof v0.0.0-20260709232956-b9395ee17fa0/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk= github.com/google/subcommands v1.2.0 h1:vWQspBTo2nEqTUFita5/KeEWlUL8kQObDFbub/EN9oE= github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -122,8 +120,8 @@ github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= -github.com/ianlancetaylor/demangle v0.0.0-20260505044615-1ff4bf46051f h1:NW3E2QSchEk63/fjeEvWOa2cE02FSv9ox//VE/N4c8g= -github.com/ianlancetaylor/demangle v0.0.0-20260505044615-1ff4bf46051f/go.mod h1:gx7rwoVhcfuVKG5uya9Hs3Sxj7EIvldVofAWIUtGouw= +github.com/ianlancetaylor/demangle v0.0.0-20260724033716-83e58baca724 h1:QixF8Mcbe87ET7pK/fPbBJ9GXFddmEY8yYMepzMzo30= +github.com/ianlancetaylor/demangle v0.0.0-20260724033716-83e58baca724/go.mod h1:gx7rwoVhcfuVKG5uya9Hs3Sxj7EIvldVofAWIUtGouw= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jellydator/ttlcache/v3 v3.4.1 h1:bOdXmXiycyK6E6Qjyuj5vl+/vU3SCOoDs8a86NbHjAQ= @@ -136,8 +134,8 @@ github.com/kardianos/service v1.3.0 h1:/LGy+xPP2TM+GLTiCZ2di7cy0Jd/qrawlTUfqKYFd github.com/kardianos/service v1.3.0/go.mod h1:E4V9ufUuY82F7Ztlu1eN9VXWIQxg8NoLQlmFe0MtrXc= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= -github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= -github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk= +github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -161,16 +159,16 @@ github.com/lestrrat-go/httpcc v1.0.1 h1:ydWCStUeJLkpYyjLDHihupbn2tYmZ7m22BGkcvZZ github.com/lestrrat-go/httpcc v1.0.1/go.mod h1:qiltp3Mt56+55GPVCbTdM9MlqhvzyuL6W/NMDA8vA5E= github.com/lestrrat-go/httprc/v3 v3.0.6 h1:4FpLQ18KK/ypPbVU3NLWJNRvH3kcYiqKqWfKGqNWxxI= github.com/lestrrat-go/httprc/v3 v3.0.6/go.mod h1:mSMtkZW92Z98M5YoNNztbRGxbXHql7tSitCvaxvo9l0= -github.com/lestrrat-go/jwx/v3 v3.1.1 h1:yd9AdPmZ4INnQ7k42IrzXYpnEG803+SrQ6hdMvzHJzw= -github.com/lestrrat-go/jwx/v3 v3.1.1/go.mod h1:uw/MN2M/Xiu4FhwcIwH11Zsh9JWx9SWzgALl7/uIEkU= +github.com/lestrrat-go/jwx/v3 v3.2.0 h1:Jb3zBASTSZXz7gzzSAfYqxXF8KejvKC4xWoePLQqXCA= +github.com/lestrrat-go/jwx/v3 v3.2.0/go.mod h1:38vQ8iWKq3qRSbilbzvzdQPuywhowwuR03lhkYskyrw= github.com/lestrrat-go/option/v2 v2.0.0 h1:XxrcaJESE1fokHy3FpaQ/cXW8ZsIdWcdFzzLOcID3Ss= github.com/lestrrat-go/option/v2 v2.0.0/go.mod h1:oSySsmzMoR0iRzCDCaUfsCzxQHUEuhOViQObyy7S6Vg= github.com/maruel/natural v1.3.0 h1:VsmCsBmEyrR46RomtgHs5hbKADGRVtliHTyCOLFBpsg= github.com/maruel/natural v1.3.0/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= -github.com/mattn/go-isatty v0.0.21 h1:xYae+lCNBP7QuW4PUnNG61ffM4hVIfm+zUzDuSzYLGs= -github.com/mattn/go-isatty v0.0.21/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4= -github.com/mattn/go-sqlite3 v1.14.48 h1:7XHIgl0a8HwOaiK4E47ozLkST78rR9+OtNGx27D/TFs= -github.com/mattn/go-sqlite3 v1.14.48/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w= +github.com/mattn/go-isatty v0.0.23 h1:cYwCQTQf3HB6xUC+BtyCLZNr7IzbOmoZbmssVNzSyiQ= +github.com/mattn/go-isatty v0.0.23/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= +github.com/mattn/go-sqlite3 v1.14.50 h1:dmdFvo1XG4MPzA4IkAmE9upVz/Nj31uRoM5+jC8hYbY= +github.com/mattn/go-sqlite3 v1.14.50/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w= github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6BbAxPY= github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg= github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE= @@ -187,8 +185,8 @@ github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOF github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/ogier/pflag v0.0.1 h1:RW6JSWSu/RkSatfcLtogGfFgpim5p7ARQ10ECk5O750= github.com/ogier/pflag v0.0.1/go.mod h1:zkFki7tvTa0tafRvTBIZTvzYyAu6kQhPZFnshFFPE+g= -github.com/onsi/ginkgo/v2 v2.32.0 h1:Hw7s2pVrQo/8Yz5N77qdnpHaoc+c6cC9WIV1Jce+J6E= -github.com/onsi/ginkgo/v2 v2.32.0/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44= +github.com/onsi/ginkgo/v2 v2.32.1 h1:6tlvcDm/3sE8lGJbZ4+d4mO3RLy24/tQWOFzVSQNIfw= +github.com/onsi/ginkgo/v2 v2.32.1/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44= github.com/onsi/gomega v1.42.1 h1:iN1rCUX+44NZ1Dc97MPoeFYbFR0vh8zxoxMFwKdyZ6I= github.com/onsi/gomega v1.42.1/go.mod h1:REff/hsDsodHoKlWsP2mAPhu1+5/6hVYNf9rIEBpeSg= github.com/pelletier/go-toml/v2 v2.4.3 h1:GTRvJQutkOSftxIFD5xw9aepkYNuPWmVJpffdDPYVpY= @@ -201,16 +199,16 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pocketbase/dbx v1.12.0 h1:/oLErM+A0b4xI0PWTGPqSDVjzix48PqI/bng2l0PzoA= github.com/pocketbase/dbx v1.12.0/go.mod h1:xXRCIAKTHMgUCyCKZm55pUOdvFziJjQfXaWKhu2vhMs= -github.com/pressly/goose/v3 v3.27.2 h1:FjKNzcmMdGrQlSIu5alMSmakQtJFBgtw+A0bb1p/LC8= -github.com/pressly/goose/v3 v3.27.2/go.mod h1:qWW+/8dkVtJYjJrbIpwD5xxnEJTUKvxkQ9JKQp9LaIM= -github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= -github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/pressly/goose/v3 v3.27.3 h1:pIglVHjw99r4e/hDHHwbl9vfOsDMqUokfkXo6+n/RxA= +github.com/pressly/goose/v3 v3.27.3/go.mod h1:Dag+xpV6o20HR2LFY1j0q6MDwc3f7vPUFDA77R+0yGY= +github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59uiuyR2bPlnU= +github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= -github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4= -github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw= -github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc= -github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo= +github.com/prometheus/common v0.70.1 h1:1HvjP4D5oL3t8RsPlwxA9onvvStjtIHYE5XuuwOi/PY= +github.com/prometheus/common v0.70.1/go.mod h1:VdFUQDMZK3VLkurFUVhia6uys/0suUp86TJz5qbJRhc= +github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI= +github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rjeczalik/notify v0.9.3 h1:6rJAzHTGKXGj76sbRgDiDcYj/HniypXmSJo1SWakZeY= @@ -218,8 +216,8 @@ github.com/rjeczalik/notify v0.9.3/go.mod h1:gF3zSOrafR9DQEWSE8TjfI9NkooDxbyT4Ug github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/rogpeppe/go-internal v1.15.0 h1:D0RCU5rMAp+SpgkiNdrjfJ+LX4J1M32V2NeCY7EJ6hc= -github.com/rogpeppe/go-internal v1.15.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs= +github.com/rogpeppe/go-internal v1.16.0 h1:O9DK+vNMDVGLr2BeZqmpLeMjiMNkuXfcqntWbZV6S5g= +github.com/rogpeppe/go-internal v1.16.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 h1:OkMGxebDjyw0ULyrTYWeN0UNCCkmCWfjPnIA2W6oviI= github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06/go.mod h1:+ePHsJ1keEjQtpvf9HHw0f4ZeJ0TLRsxhunSI2hYJSs= @@ -227,15 +225,15 @@ github.com/sagikazarmark/locafero v0.12.0 h1:/NQhBAkUb4+fH1jivKHWusDYFjMOOKU88ee github.com/sagikazarmark/locafero v0.12.0/go.mod h1:sZh36u/YSZ918v0Io+U9ogLYQJ9tLLBmM4eneO6WwsI= github.com/sanity-io/litter v1.5.8 h1:uM/2lKrWdGbRXDrIq08Lh9XtVYoeGtcQxk9rtQ7+rYg= github.com/sanity-io/litter v1.5.8/go.mod h1:9gzJgR2i4ZpjZHsKvUXIRQVk7P+yM3e+jAF7bU2UI5U= -github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ= -github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 h1:1EYB5IzjZawrrnELUi78f9fPu57HuXjmddZPjrls/28= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.3/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= github.com/segmentio/asm v1.2.1 h1:DTNbBqs57ioxAD4PrArqftgypG4/qNpXoJx8TVXxPR0= github.com/segmentio/asm v1.2.1/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= -github.com/sethvargo/go-retry v0.3.0 h1:EEt31A35QhrcRZtrYFDTBg91cqZVnFL2navjDrah2SE= -github.com/sethvargo/go-retry v0.3.0/go.mod h1:mNX17F0C/HguQMyMyJxcnU471gOZGxCLyYaFyAZraas= +github.com/sethvargo/go-retry v0.4.0 h1:9qy1OoIAxBL+gBYnkTnTnWle5wlfsXQlwRzIbbpdqPw= +github.com/sethvargo/go-retry v0.4.0/go.mod h1:tvsjdKG6xfiCx4LSiUZ06kcv38xvdVQwv8R6/VnnVWg= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= -github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= +github.com/sirupsen/logrus v1.10.0 h1:T8MxJJXVZkfcC5zSRMRAg2F8+lxjmUCGGWPzFxO+Msc= +github.com/sirupsen/logrus v1.10.0/go.mod h1:FXZFonkDAnFozmO+5hGAFvB0Yg9/j2SIhA/QuIkP180= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= @@ -267,8 +265,9 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/stretchr/testify v1.12.0 h1:K6Mr6jO9JICuend/5xzTM03ydSV3vdNRYAdPSukj8uI= +github.com/stretchr/testify v1.12.0/go.mod h1:bOYBZb5qJ00vPzWfIqBUZPaxK8jWiXc6d3ErP4Ca9Gw= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/tetratelabs/wabin v0.0.0-20230304001439-f6f874872834 h1:ZF+QBjOI+tILZjBaFj3HgFonKXUcwgJ4djLb6i42S3Q= @@ -293,27 +292,28 @@ github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= -go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g= -go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk= +go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk= +go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= -go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= -go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= +go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= -golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= -golang.org/x/image v0.44.0 h1:+tDekMZED9+LrtB3G5xzRggpVh9CARjZqROla3R3R+I= -golang.org/x/image v0.44.0/go.mod h1:V8K3KE9KKKE+pLpQDOeN18w9oacNSvy1tDOirTu4xtY= -golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk= -golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40= +golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= +golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= +golang.org/x/image v0.45.0 h1:FMb1nTbH5H9vF55SriQHgFw5GnNL9Jg6L25BwXKzhB0= +golang.org/x/image v0.45.0/go.mod h1:n62x/7RqlwXDvGsSU4u6IUTUf6KghUZ9Bt7cG/T9Fx4= +golang.org/x/mod v0.40.0 h1:hUv+3cXcdRHz08UmSiOob7sadHig73uo5bkXxQ/tvUs= +golang.org/x/mod v0.40.0/go.mod h1:0/weTWkPWGBikyTWAX3dkjVztMmBA5hM0DH6BElSupE= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= -golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20180926160741-c2ed4eda69e7/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -322,26 +322,26 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/telemetry v0.0.0-20260708182218-49f421fb7959 h1:RJhm5l6Fo4rmEIcndxDllNhhf/fAx8qIm4t6A7vpm2A= -golang.org/x/telemetry v0.0.0-20260708182218-49f421fb7959/go.mod h1:LV7u5Oco+Z/g6XI7PqN+EUUUGGkEcmB1uj2ceI0fOVg= +golang.org/x/telemetry v0.0.0-20260811182544-a038080d80e5 h1:ZUSxONxc981v7AW7QUg+I9WwZzSTTJ019ENBYr5pV/Q= +golang.org/x/telemetry v0.0.0-20260811182544-a038080d80e5/go.mod h1:LVehoXe41cL5SCVQilsV7Gg6BNG+Js6P9PhSbYTIUkQ= golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= -golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE= -golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk= +golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI= +golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= -google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/ini.v1 v1.67.3 h1:iM9Lhz5MRSGhHVGGwCuzG9KO8PoirCXj/m/qTmOJJQw= gopkg.in/ini.v1 v1.67.3/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU= @@ -350,11 +350,11 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -modernc.org/libc v1.73.4 h1:+ra4Ui8ngyt8HDcO1FTDPWlkAh6yOdaO2yAoh8MddQA= -modernc.org/libc v1.73.4/go.mod h1:DXZ3eO8qMCNn2SnmTNCiC71nJ9Rcq3PsnpU6Vc4rWK8= +modernc.org/libc v1.74.3 h1:a4J+Z8aVaxPyjyxRAdJzw246PqpcFGvVPnfT/AuM5Ws= +modernc.org/libc v1.74.3/go.mod h1:4H7h/MJ8wnjL8RAbp9v3OXgnk22X7MouHIhDbvP3gj4= modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= -modernc.org/sqlite v1.53.0 h1:20WG8N9q4ji/dEqGk4uiI0c6OPjSeLTNYGFCc3+7c1M= -modernc.org/sqlite v1.53.0/go.mod h1:xoEpOIpGrgT48H5iiyt/YXPCZPEzlfmfFwtk8Lklw8s= +modernc.org/sqlite v1.54.0 h1:JCxR4qwkJvOaqAoYcgDoO25Nc+ROg6EJ2LfBVzdrgog= +modernc.org/sqlite v1.54.0/go.mod h1:4ntCLuNmnH8+GNqjka1wNg7KJd5/Hi5FYp8K+XQ7GZw= From 4b1218eec0068ba8b119a970385aea5b0605660b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Deluan=20Quint=C3=A3o?= Date: Tue, 18 Aug 2026 09:32:26 -0400 Subject: [PATCH 13/27] feat(ui): show translation completion percentage in the language selector (#5979) The language selector now shows how complete each translation is, so users can see at a glance which languages are lagging behind English. The native API's translation resource gained a termCount field holding the number of non-empty terms in each language file; the UI divides that by the term count of the bundled English file to get the percentage. The percentage is wrapped in a Unicode left-to-right isolate, otherwise it renders as "(%61)" beside right-to-left names such as Arabic and Persian. Sorting runs on the plain language name, before the percentage is appended. This also fixes prepareLanguage() mutating the bundled English translations: for the English locale it received the shared en object and aliased albumSong and playlistTrack onto it, growing en by 94 keys at runtime. That inflated the denominator and made every language read about 14 points low. The aliases now go on the merged copy instead. --- server/nativeapi/translations.go | 36 +++++++-- server/nativeapi/translations_test.go | 12 +++ ui/src/i18n/provider.js | 11 +-- ui/src/i18n/provider.test.js | 19 +++++ ui/src/i18n/useGetLanguageChoices.jsx | 31 +++++++- ui/src/i18n/useGetLanguageChoices.test.jsx | 92 ++++++++++++++++++++++ 6 files changed, 184 insertions(+), 17 deletions(-) create mode 100644 ui/src/i18n/provider.test.js create mode 100644 ui/src/i18n/useGetLanguageChoices.test.jsx diff --git a/server/nativeapi/translations.go b/server/nativeapi/translations.go index 685713083..39d071279 100644 --- a/server/nativeapi/translations.go +++ b/server/nativeapi/translations.go @@ -17,9 +17,10 @@ import ( ) type translation struct { - ID string `json:"id"` - Name string `json:"name"` - Data string `json:"data"` + ID string `json:"id"` + Name string `json:"name"` + Data string `json:"data"` + TermCount int `json:"termCount"` } func newTranslationRepository(context.Context) rest.Repository { @@ -97,27 +98,46 @@ func loadTranslation(fsys fs.FS, fileName string) (translation translation, err // Load translation from json file file, err := fsys.Open(filePath) if err != nil { - return + return translation, err } data, err := io.ReadAll(file) if err != nil { - return + return translation, err } var out map[string]any if err = json.Unmarshal(data, &out); err != nil { - return + return translation, err } // Compress JSON buf := new(bytes.Buffer) if err = json.Compact(buf, data); err != nil { - return + return translation, err } translation.Data = buf.String() translation.Name = out["languageName"].(string) translation.ID = id - return + translation.TermCount = countTranslatedTerms(out) + return translation, nil +} + +// countTranslatedTerms counts non-empty leaf values, matching the UI's notion of a translated term +func countTranslatedTerms(obj map[string]any) int { + count := 0 + for _, v := range obj { + switch v := v.(type) { + case map[string]any: + count += countTranslatedTerms(v) + case string: + if v != "" { + count++ + } + default: + count++ + } + } + return count } var _ rest.Repository = (*translationRepository)(nil) diff --git a/server/nativeapi/translations_test.go b/server/nativeapi/translations_test.go index 6c834070c..77c088f6f 100644 --- a/server/nativeapi/translations_test.go +++ b/server/nativeapi/translations_test.go @@ -6,6 +6,7 @@ import ( "io/fs" "os" "path/filepath" + "testing/fstest" "github.com/navidrome/navidrome/consts" "github.com/navidrome/navidrome/resources" @@ -45,5 +46,16 @@ var _ = Describe("Translations", func() { var out map[string]any Expect(json.Unmarshal([]byte(tr.Data), &out)).To(BeNil()) }) + + It("counts only non-empty leaf terms", func() { + fsys := fstest.MapFS{ + "i18n/test.json": &fstest.MapFile{ + Data: []byte(`{"languageName":"Test","a":"x","b":"","nested":{"c":"y","d":""}}`), + }, + } + tr, err := loadTranslation(fsys, "test.json") + Expect(err).To(BeNil()) + Expect(tr.TermCount).To(Equal(3)) + }) }) }) diff --git a/ui/src/i18n/provider.js b/ui/src/i18n/provider.js index f17a5b4ac..58f1782bb 100644 --- a/ui/src/i18n/provider.js +++ b/ui/src/i18n/provider.js @@ -42,13 +42,14 @@ const removeEmpty = (obj) => { const prepareLanguage = (lang) => { removeEmpty(lang) + // Aliases below go on the merged copy: mutating `en` would corrupt the completion baseline + const merged = deepmerge(en, lang) // Make "albumSong" and "playlistTrack" resource use the same translations as "song" - lang.resources.albumSong = lang.resources.song - lang.resources.playlistTrack = lang.resources.song + merged.resources.albumSong = merged.resources.song + merged.resources.playlistTrack = merged.resources.song // ra.boolean.null should always be empty - lang.ra.boolean.null = '' - // Fallback to english translations - return deepmerge(en, lang) + merged.ra.boolean.null = '' + return merged } export default polyglotI18nProvider((locale) => { diff --git a/ui/src/i18n/provider.test.js b/ui/src/i18n/provider.test.js new file mode 100644 index 000000000..82b593788 --- /dev/null +++ b/ui/src/i18n/provider.test.js @@ -0,0 +1,19 @@ +import { describe, it, expect, vi } from 'vitest' +import en from './en.json' + +vi.mock('../dataProvider', () => ({ default: { getOne: vi.fn() } })) + +const countLeaves = (obj) => + Object.values(obj).reduce( + (sum, v) => + sum + (typeof v === 'object' && v !== null ? countLeaves(v) : v ? 1 : 0), + 0, + ) + +describe('i18n provider', () => { + it('does not mutate the bundled English translations', async () => { + const before = countLeaves(en) + await import('./provider') + expect(countLeaves(en)).toEqual(before) + }) +}) diff --git a/ui/src/i18n/useGetLanguageChoices.jsx b/ui/src/i18n/useGetLanguageChoices.jsx index 0c708691f..38f95e422 100644 --- a/ui/src/i18n/useGetLanguageChoices.jsx +++ b/ui/src/i18n/useGetLanguageChoices.jsx @@ -1,5 +1,22 @@ // React Hook to get a list of all languages available. English is hardcoded import { useGetList } from 'react-admin' +import en from './en.json' + +const countLeaves = (obj) => + Object.values(obj).reduce( + (sum, v) => + sum + (typeof v === 'object' && v !== null ? countLeaves(v) : v ? 1 : 0), + 0, + ) + +const enTermCount = countLeaves(en) + +const withPercentage = ({ id, name, termCount }) => { + if (!termCount) return { id, name } + const pct = Math.min(100, Math.round((100 * termCount) / enTermCount)) + // Isolate the percentage, or it renders as "(%61)" next to a right-to-left name + return { id, name: `${name} ⁦(${pct}%)⁩` } +} const useGetLanguageChoices = () => { const { ids, data, loaded, loading } = useGetList( @@ -9,13 +26,19 @@ const useGetLanguageChoices = () => { {}, ) - const choices = [{ id: 'en', name: 'English' }] + const languages = [{ id: 'en', name: 'English', termCount: enTermCount }] if (loaded) { - ids.forEach((id) => choices.push({ id: id, name: data[id].name })) + ids.forEach((id) => + languages.push({ + id, + name: data[id].name, + termCount: data[id].termCount, + }), + ) } - choices.sort((a, b) => a.name.localeCompare(b.name)) + languages.sort((a, b) => a.name.localeCompare(b.name)) - return { choices, loaded, loading } + return { choices: languages.map(withPercentage), loaded, loading } } export default useGetLanguageChoices diff --git a/ui/src/i18n/useGetLanguageChoices.test.jsx b/ui/src/i18n/useGetLanguageChoices.test.jsx new file mode 100644 index 000000000..b8214868e --- /dev/null +++ b/ui/src/i18n/useGetLanguageChoices.test.jsx @@ -0,0 +1,92 @@ +import { describe, it, expect, vi } from 'vitest' +import { renderHook } from '@testing-library/react-hooks' +import { useGetList } from 'react-admin' +import en from './en.json' +import useGetLanguageChoices from './useGetLanguageChoices' + +vi.mock('react-admin', () => ({ + useGetList: vi.fn(), +})) + +const countLeaves = (obj) => + Object.values(obj).reduce( + (sum, v) => + sum + (typeof v === 'object' && v !== null ? countLeaves(v) : v ? 1 : 0), + 0, + ) +const enTermCount = countLeaves(en) + +// The percentage is wrapped in a left-to-right isolate, so it reads the same +// next to right-to-left language names +const label = (pct) => `⁦(${pct}%)⁩` + +const mockLanguages = (languages) => { + const data = {} + languages.forEach((l) => (data[l.id] = l)) + useGetList.mockReturnValue({ + ids: languages.map((l) => l.id), + data, + loaded: true, + loading: false, + }) +} + +const choiceFor = (id) => { + const { result } = renderHook(() => useGetLanguageChoices()) + return result.current.choices.find((c) => c.id === id) +} + +describe('useGetLanguageChoices', () => { + it('appends the completion percentage to incomplete languages', () => { + const termCount = Math.round(enTermCount * 0.62) + mockLanguages([{ id: 'cs', name: 'Čeština', termCount }]) + + const pct = Math.round((100 * termCount) / enTermCount) + expect(choiceFor('cs').name).toEqual(`Čeština ${label(pct)}`) + }) + + it('shows 100% for a complete language', () => { + mockLanguages([{ id: 'de', name: 'Deutsch', termCount: enTermCount }]) + + expect(choiceFor('de').name).toEqual(`Deutsch ${label(100)}`) + }) + + it('caps the percentage at 100 when a language has extra terms', () => { + mockLanguages([ + { id: 'pt', name: 'Português', termCount: enTermCount + 20 }, + ]) + + expect(choiceFor('pt').name).toEqual(`Português ${label(100)}`) + }) + + it('isolates the percentage next to a right-to-left name', () => { + const termCount = Math.round(enTermCount * 0.61) + mockLanguages([{ id: 'ar', name: 'العربية', termCount }]) + + const pct = Math.round((100 * termCount) / enTermCount) + expect(choiceFor('ar').name).toEqual(`العربية ⁦(${pct}%)⁩`) + }) + + it('omits the percentage when the server does not send a term count', () => { + mockLanguages([{ id: 'fr', name: 'Français' }]) + + expect(choiceFor('fr').name).toEqual('Français') + }) + + it('shows 100% for the bundled English', () => { + mockLanguages([]) + + expect(choiceFor('en').name).toEqual(`English ${label(100)}`) + }) + + it('sorts by language name, ignoring the percentage', () => { + mockLanguages([ + { id: 'no', name: 'Norsk', termCount: enTermCount }, + { id: 'da', name: 'Dansk', termCount: 1 }, + ]) + + const { result } = renderHook(() => useGetLanguageChoices()) + + expect(result.current.choices.map((c) => c.id)).toEqual(['da', 'en', 'no']) + }) +}) From 2e03766a9df84cb24303bd2ebc808402ad7c1a45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Deluan=20Quint=C3=A3o?= Date: Tue, 18 Aug 2026 20:58:55 -0400 Subject: [PATCH 14/27] fix(playlist): preserve smart playlist song count on re-import (#5907) (#5908) * fix(playlist): preserve smart playlist counters on re-import (#5907) * perf(playlist): skip re-importing unchanged NSP files (#5907) * feat(playlist): also store content hash for M3U imports (unused for now) * fix(playlist): return stored record when skipping unchanged NSP import Skipping before copying the stored identity broke the ImportFile(sync=false) contract: callers received an ID-less playlist and the requested Sync change was silently dropped. * refactor(playlist): hash imports once at the caller; protect smart counters in Put Move content hashing out of both parsers into the code that owns the file (parsePlaylist and ImportFile), removing the NSP double-buffer and the duplicated hashing idiom. Put now drops song_count/duration/size for smart playlists (PostMapArgs), disarming the counter-zeroing trap for all callers. * fix(playlist): invalidate imported hash when rules are edited via API Without this, a rules edit through the REST API kept the stored file hash, so every scan skipped the unchanged file and never restored the file-backed rules while sync was on. * test(playlist): verify smart counters survive a re-import, end to end The existing Put test seeds the stored counters with a raw SQL update, so it pins the guard in PostMapArgs but not the pipeline around it. This test drives the counters through a real evaluation instead: it saves a smart playlist, reads it with GetWithTracks to populate song_count/duration/size, then saves the playlist the way the scanner rebuilds it after parsing the .nsp file, with the counters back at zero. Both routes fail without the guard, and the new one covers the exact sequence reported in #5907. Test taken from #5970, which diagnosed the same root cause independently. Co-authored-by: Junker der Provinz <133605895+junkerderprovinz@users.noreply.github.com> * test(playlist): build the service with artwork.NewUploader The artwork pipeline in #5847 replaced core.NewImageUploadService() with artwork.NewUploader(ds) and updated every call site it could see. The five call sites this branch adds were written against the old constructor, so the merge applied cleanly but left the package uncompilable. * fix(db): re-stamp the imported_hash migration after the master merge Master gained three migrations while this branch was open, the newest being 20260816180040. The original 20260808200333 stamp now sorts before them, so any database already upgraded past that point would skip this migration entirely and never get the imported_hash column. Same SQL, current timestamp. * refactor(playlist): hash imported playlists with xxh3 and the id encoding ImportedHash is a change detector, not a security boundary, so it does not need a cryptographic digest. xxh3 is already a direct dependency and is used the same way to fingerprint files in the artwork image store. Encoding the 128-bit digest with id.Encode stores it in the same 22-char base62 form as every other id in the schema, down from 64 hex chars. No migration is needed: the imported_hash column has not shipped in a release, so no database holds a value in the old format. * refactor(playlist): extract the imported-playlist fingerprint helper Both import paths encoded the hash inline, so how a playlist file is fingerprinted lived in two places. A third import path that encoded it differently would silently never match the stored value, turning the unchanged-file skip into a no-op. --------- Co-authored-by: Junker der Provinz <133605895+junkerderprovinz@users.noreply.github.com> --- core/playlists/import.go | 32 +++- core/playlists/import_test.go | 143 ++++++++++++++++++ core/playlists/rest_adapter.go | 1 + core/playlists/rest_adapter_test.go | 19 +++ ...60818002312_add_playlist_imported_hash.sql | 5 + model/playlist.go | 1 + persistence/playlist_repository.go | 4 + persistence/playlist_repository_test.go | 27 ++++ persistence/smart_playlist_repository_test.go | 35 +++++ 9 files changed, 264 insertions(+), 3 deletions(-) create mode 100644 db/migrations/20260818002312_add_playlist_imported_hash.sql diff --git a/core/playlists/import.go b/core/playlists/import.go index bafb870cd..e41f61bd1 100644 --- a/core/playlists/import.go +++ b/core/playlists/import.go @@ -12,8 +12,10 @@ import ( "github.com/navidrome/navidrome/conf" "github.com/navidrome/navidrome/log" "github.com/navidrome/navidrome/model" + "github.com/navidrome/navidrome/model/id" "github.com/navidrome/navidrome/model/request" "github.com/navidrome/navidrome/utils/ioutils" + "github.com/zeebo/xxh3" "golang.org/x/text/unicode/norm" ) @@ -57,10 +59,12 @@ func (s *playlists) ImportFile(ctx context.Context, absolutePath string, sync bo } defer file.Close() - reader := ioutils.UTF8Reader(file) + hasher := xxh3.New() + reader := io.TeeReader(ioutils.UTF8Reader(file), hasher) if err := s.parseM3U(ctx, pls, nil, reader); err != nil { return nil, err } + pls.ImportedHash = fingerprint(hasher) if err := s.updatePlaylist(ctx, pls, sync); err != nil { return nil, err } @@ -138,7 +142,9 @@ func (s *playlists) parsePlaylist(ctx context.Context, playlistFile string, fold } defer file.Close() - reader := ioutils.UTF8Reader(file) + // Hash the bytes the parser consumes, giving every imported playlist a content fingerprint + hasher := xxh3.New() + reader := io.TeeReader(ioutils.UTF8Reader(file), hasher) extension := strings.ToLower(filepath.Ext(playlistFile)) switch extension { case ".nsp": @@ -146,7 +152,15 @@ func (s *playlists) parsePlaylist(ctx context.Context, playlistFile string, fold default: err = s.parseM3U(ctx, pls, folder, reader) } - return pls, err + if err != nil { + return pls, err + } + pls.ImportedHash = fingerprint(hasher) + return pls, nil +} + +func fingerprint(h *xxh3.Hasher) string { + return id.Encode(h.Sum128().Bytes()) } // findByPathNormalized looks up a playlist by path, trying both NFC and NFD Unicode @@ -179,6 +193,12 @@ func (s *playlists) updatePlaylist(ctx context.Context, newPls *model.Playlist, } if err == nil { + // Only smart playlists skip on an unchanged file; M3U must re-run so newly-added tracks resolve. + if !forceSync && newPls.IsSmartPlaylist() && newPls.ImportedHash != "" && newPls.ImportedHash == pls.ImportedHash { + log.Trace(ctx, "Playlist file unchanged since last import, skipping", "playlist", pls.Name, "path", pls.Path) + *newPls = *pls // callers must see the stored record, so e.g. ImportFile can still flip Sync + return nil + } log.Info(ctx, "Updating synced playlist", "playlist", pls.Name, "path", newPls.Path) newPls.ID = pls.ID newPls.Name = pls.Name @@ -187,6 +207,12 @@ func (s *playlists) updatePlaylist(ctx context.Context, newPls *model.Playlist, newPls.Public = pls.Public newPls.UploadedImage = pls.UploadedImage // Preserve manual upload newPls.EvaluatedAt = nil // force re-evaluation on next read + if newPls.IsSmartPlaylist() { + // Tracks aren't materialized at parse time; carry the stored counters so callers see real values + newPls.SongCount = pls.SongCount + newPls.Duration = pls.Duration + newPls.Size = pls.Size + } } else { log.Info(ctx, "Adding synced playlist", "playlist", newPls.Name, "path", newPls.Path, "owner", owner.UserName) newPls.OwnerID = owner.ID diff --git a/core/playlists/import_test.go b/core/playlists/import_test.go index 2a84d56b2..445561266 100644 --- a/core/playlists/import_test.go +++ b/core/playlists/import_test.go @@ -15,10 +15,12 @@ import ( "github.com/navidrome/navidrome/core/playlists" "github.com/navidrome/navidrome/model" "github.com/navidrome/navidrome/model/criteria" + "github.com/navidrome/navidrome/model/id" "github.com/navidrome/navidrome/model/request" "github.com/navidrome/navidrome/tests" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/zeebo/xxh3" "golang.org/x/text/unicode/norm" ) @@ -307,6 +309,33 @@ var _ = Describe("Playlists - Import", func() { Expect(pls.ID).To(BeEmpty()) }) + It("stores a content hash but re-imports unchanged M3U playlists", func() { + tmpDir := GinkgoT().TempDir() + mockLibRepo.SetData([]model.Library{{ID: 1, Path: tmpDir}}) + ds.MockedMediaFile = &mockedMediaFileFromListRepo{data: []string{"test.mp3"}} + ps = playlists.NewPlaylists(ds, artwork.NewUploader(ds)) + + plsFile := filepath.Join(tmpDir, "test.m3u") + Expect(os.WriteFile(plsFile, []byte("test.mp3\n"), 0600)).To(Succeed()) + plsFolder := &model.Folder{ID: "1", LibraryID: 1, LibraryPath: tmpDir, Path: "", Name: ""} + + first, err := ps.ImportFromFolder(ctx, plsFolder, "test.m3u") + Expect(err).ToNot(HaveOccurred()) + Expect(first.ImportedHash).ToNot(BeEmpty()) + + // Re-import with a matching stored hash: M3U must still be re-imported, not skipped. + existingPls := &model.Playlist{ + ID: "m3u-id", Name: "Test", Path: plsFile, Sync: true, + OwnerID: "123", ImportedHash: first.ImportedHash, + } + mockPlsRepo.PathMap = map[string]*model.Playlist{plsFile: existingPls} + mockPlsRepo.Last = nil + + _, err = ps.ImportFromFolder(ctx, plsFolder, "test.m3u") + Expect(err).ToNot(HaveOccurred()) + Expect(mockPlsRepo.Last).ToNot(BeNil()) + }) + It("clears ExternalImageURL on re-scan when directive is removed", func() { tmpDir := GinkgoT().TempDir() mockLibRepo.SetData([]model.Library{{ID: 1, Path: tmpDir}}) @@ -371,6 +400,85 @@ var _ = Describe("Playlists - Import", func() { Expect(pls.Name).To(Equal("Recently Played")) Expect(pls.Public).To(BeTrue()) // Should be true since server default is true }) + + It("preserves counters when re-importing an existing smart playlist", func() { + tmpDir := GinkgoT().TempDir() + mockLibRepo.SetData([]model.Library{{ID: 1, Path: tmpDir}}) + ps = playlists.NewPlaylists(ds, artwork.NewUploader(ds)) + + nsp := `{"name":"My Smart","all":[{"is":{"loved":true}}],"sort":"title","order":"asc"}` + plsFile := filepath.Join(tmpDir, "smart.nsp") + Expect(os.WriteFile(plsFile, []byte(nsp), 0600)).To(Succeed()) + + existingPls := &model.Playlist{ + ID: "smart-id", + Name: "My Smart", + Path: plsFile, + Sync: true, + OwnerID: "123", + SongCount: 42, + Duration: 123.4, + Size: 5000, + } + mockPlsRepo.PathMap = map[string]*model.Playlist{plsFile: existingPls} + + plsFolder := &model.Folder{ID: "1", LibraryID: 1, LibraryPath: tmpDir, Path: "", Name: ""} + _, err := ps.ImportFromFolder(ctx, plsFolder, "smart.nsp") + + Expect(err).ToNot(HaveOccurred()) + Expect(mockPlsRepo.Last).ToNot(BeNil()) + Expect(mockPlsRepo.Last.IsSmartPlaylist()).To(BeTrue()) + Expect(mockPlsRepo.Last.SongCount).To(Equal(42)) + Expect(mockPlsRepo.Last.Duration).To(Equal(float32(123.4))) + Expect(mockPlsRepo.Last.Size).To(Equal(int64(5000))) + }) + + It("skips re-import when the smart playlist file content is unchanged", func() { + tmpDir := GinkgoT().TempDir() + mockLibRepo.SetData([]model.Library{{ID: 1, Path: tmpDir}}) + ps = playlists.NewPlaylists(ds, artwork.NewUploader(ds)) + + nsp := `{"name":"My Smart","all":[{"is":{"loved":true}}]}` + plsFile := filepath.Join(tmpDir, "smart.nsp") + Expect(os.WriteFile(plsFile, []byte(nsp), 0600)).To(Succeed()) + + existingPls := &model.Playlist{ + ID: "smart-id", Name: "My Smart", Path: plsFile, Sync: true, + OwnerID: "123", SongCount: 42, + ImportedHash: hashOf(nsp), + } + mockPlsRepo.PathMap = map[string]*model.Playlist{plsFile: existingPls} + + plsFolder := &model.Folder{ID: "1", LibraryID: 1, LibraryPath: tmpDir, Path: "", Name: ""} + _, err := ps.ImportFromFolder(ctx, plsFolder, "smart.nsp") + + Expect(err).ToNot(HaveOccurred()) + Expect(mockPlsRepo.Last).To(BeNil()) // Put never called: nothing re-written + }) + + It("re-imports when the smart playlist file content changed", func() { + tmpDir := GinkgoT().TempDir() + mockLibRepo.SetData([]model.Library{{ID: 1, Path: tmpDir}}) + ps = playlists.NewPlaylists(ds, artwork.NewUploader(ds)) + + nsp := `{"name":"My Smart","all":[{"is":{"loved":true}}]}` + plsFile := filepath.Join(tmpDir, "smart.nsp") + Expect(os.WriteFile(plsFile, []byte(nsp), 0600)).To(Succeed()) + + existingPls := &model.Playlist{ + ID: "smart-id", Name: "My Smart", Path: plsFile, Sync: true, + OwnerID: "123", SongCount: 42, + ImportedHash: hashOf("old content"), + } + mockPlsRepo.PathMap = map[string]*model.Playlist{plsFile: existingPls} + + plsFolder := &model.Folder{ID: "1", LibraryID: 1, LibraryPath: tmpDir, Path: "", Name: ""} + _, err := ps.ImportFromFolder(ctx, plsFolder, "smart.nsp") + + Expect(err).ToNot(HaveOccurred()) + Expect(mockPlsRepo.Last).ToNot(BeNil()) // Put called: file changed + Expect(mockPlsRepo.Last.ImportedHash).To(Equal(hashOf(nsp))) + }) }) DescribeTable("Playlist filename Unicode normalization (regression fix-playlist-filename-normalization)", @@ -760,6 +868,37 @@ var _ = Describe("Playlists - Import", func() { Expect(pls.ID).To(Equal("existing-id")) Expect(pls.Sync).To(BeTrue()) }) + + It("unsyncs a synced smart playlist with sync=false even when content is unchanged", func() { + tmpDir := GinkgoT().TempDir() + mockLibRepo.SetData([]model.Library{{ID: 1, Path: tmpDir}}) + + mockFolderRepo := &mockFolderRepoForImport{ + folder: &model.Folder{ + ID: "1", LibraryID: 1, LibraryPath: tmpDir, Path: "", Name: "", + }, + } + ds.MockedFolder = mockFolderRepo + ps = playlists.NewPlaylists(ds, artwork.NewUploader(ds)) + + nsp := `{"name":"My Smart","all":[{"is":{"loved":true}}]}` + plsFile := filepath.Join(tmpDir, "smart.nsp") + Expect(os.WriteFile(plsFile, []byte(nsp), 0600)).To(Succeed()) + + existingPls := &model.Playlist{ + ID: "smart-id", Name: "My Smart", Path: plsFile, Sync: true, + OwnerID: "123", SongCount: 42, + ImportedHash: hashOf(nsp), + } + mockPlsRepo.PathMap = map[string]*model.Playlist{plsFile: existingPls} + + pls, err := ps.ImportFile(ctx, plsFile, false) + Expect(err).ToNot(HaveOccurred()) + Expect(pls.ID).To(Equal("smart-id")) + Expect(pls.Sync).To(BeFalse()) + Expect(mockPlsRepo.Last).ToNot(BeNil()) + Expect(mockPlsRepo.Last.Sync).To(BeFalse()) + }) }) Describe("ImportM3U", func() { @@ -1083,3 +1222,7 @@ func (m *mockFolderRepoForImport) GetByPath(_ model.Library, _ string) (*model.F } return nil, model.ErrNotFound } + +func hashOf(content string) string { + return id.Encode(xxh3.Hash128([]byte(content)).Bytes()) +} diff --git a/core/playlists/rest_adapter.go b/core/playlists/rest_adapter.go index f34524e27..ca7cfb0cc 100644 --- a/core/playlists/rest_adapter.go +++ b/core/playlists/rest_adapter.go @@ -136,6 +136,7 @@ func (s *playlists) applyContentUpdate(ctx context.Context, current, entity *mod if rulesChanged { current.Rules = entity.Rules current.EvaluatedAt = nil // force re-evaluation on next read + current.ImportedHash = "" // rules no longer match the source file; next scan must re-import it } if sent("sync") && current.Path != "" && current.Sync != entity.Sync { current.Sync = entity.Sync diff --git a/core/playlists/rest_adapter_test.go b/core/playlists/rest_adapter_test.go index 7ae376b07..fbfab350c 100644 --- a/core/playlists/rest_adapter_test.go +++ b/core/playlists/rest_adapter_test.go @@ -160,6 +160,25 @@ var _ = Describe("REST Adapter", func() { Expect(mockPlsRepo.Last.Rules).To(Equal(newRules)) }) + It("invalidates the imported hash when rules change, so the next scan re-syncs the file", func() { + mockPlsRepo.Data["smart-1"] = &model.Playlist{ + ID: "smart-1", + Name: "Smart Playlist", + OwnerID: "user-1", + Path: "/music/smart.nsp", + Sync: true, + ImportedHash: hashOf("file content"), + Rules: &criteria.Criteria{Expression: criteria.Contains{"title": "old"}}, + } + ctx = request.WithUser(ctx, model.User{ID: "user-1", IsAdmin: false}) + repo = ps.NewRepository(ctx).(rest.Persistable) + newRules := &criteria.Criteria{Expression: criteria.Contains{"title": "new"}} + pls := &model.Playlist{Rules: newRules} + err := repo.Update("smart-1", pls, "rules") + Expect(err).ToNot(HaveOccurred()) + Expect(mockPlsRepo.Last.ImportedHash).To(BeEmpty()) + }) + It("allows toggling sync for file-backed playlists", func() { originalTime := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) mockPlsRepo.Data["file-pls"] = &model.Playlist{ diff --git a/db/migrations/20260818002312_add_playlist_imported_hash.sql b/db/migrations/20260818002312_add_playlist_imported_hash.sql new file mode 100644 index 000000000..aab729153 --- /dev/null +++ b/db/migrations/20260818002312_add_playlist_imported_hash.sql @@ -0,0 +1,5 @@ +-- +goose Up +alter table playlist add imported_hash varchar default '' not null; + +-- +goose Down +alter table playlist drop column imported_hash; diff --git a/model/playlist.go b/model/playlist.go index 9aa54bf13..55b94a640 100644 --- a/model/playlist.go +++ b/model/playlist.go @@ -31,6 +31,7 @@ type Playlist struct { ExternalImageURL string `structs:"external_image_url" json:"externalImageUrl,omitempty"` CreatedAt time.Time `structs:"created_at" json:"createdAt"` UpdatedAt time.Time `structs:"updated_at" json:"updatedAt"` + ImportedHash string `structs:"imported_hash" json:"-"` // SmartPlaylist attributes Rules *criteria.Criteria `structs:"rules" json:"rules"` diff --git a/persistence/playlist_repository.go b/persistence/playlist_repository.go index 081dab3fa..cf54c6d5a 100644 --- a/persistence/playlist_repository.go +++ b/persistence/playlist_repository.go @@ -39,6 +39,10 @@ func (p dbPlaylist) PostMapArgs(args map[string]any) error { if err != nil { return fmt.Errorf("invalid criteria expression: %w", err) } + // Smart playlist counters are owned by refreshCounters (evaluation), never by callers + delete(args, "song_count") + delete(args, "duration") + delete(args, "size") return nil } delete(args, "rules") diff --git a/persistence/playlist_repository_test.go b/persistence/playlist_repository_test.go index fc2d4ae3f..9697e6fff 100644 --- a/persistence/playlist_repository_test.go +++ b/persistence/playlist_repository_test.go @@ -7,6 +7,7 @@ import ( "github.com/deluan/rest" "github.com/navidrome/navidrome/log" "github.com/navidrome/navidrome/model" + "github.com/navidrome/navidrome/model/criteria" "github.com/navidrome/navidrome/model/request" "github.com/navidrome/navidrome/utils/slice" . "github.com/onsi/ginkgo/v2" @@ -230,6 +231,32 @@ var _ = Describe("PlaylistRepository", func() { }) }) + Describe("Put", func() { + It("does not overwrite counters when saving a smart playlist", func() { + pls := model.Playlist{Name: "Smart Counters", OwnerID: "userid", Rules: &criteria.Criteria{ + Expression: criteria.All{criteria.Contains{"title": "love"}}, + }} + Expect(repo.Put(&pls)).To(Succeed()) + DeferCleanup(func() { Expect(repo.Delete(pls.ID)).To(Succeed()) }) + + // Simulate a previous evaluation having stored the counters + _, err := GetDBXBuilder().NewQuery("update playlist set song_count = 42, duration = 123, size = 456 where id = {:id}"). + Bind(dbx.Params{"id": pls.ID}).Execute() + Expect(err).ToNot(HaveOccurred()) + + pls.SongCount = 0 + pls.Duration = 0 + pls.Size = 0 + Expect(repo.Put(&pls)).To(Succeed()) + + saved, err := repo.Get(pls.ID) + Expect(err).ToNot(HaveOccurred()) + Expect(saved.SongCount).To(Equal(42)) + Expect(saved.Duration).To(Equal(float32(123))) + Expect(saved.Size).To(Equal(int64(456))) + }) + }) + It("Put/Exists/Delete", func() { By("saves the playlist to the DB") newPls := model.Playlist{Name: "Great!", OwnerID: "userid"} diff --git a/persistence/smart_playlist_repository_test.go b/persistence/smart_playlist_repository_test.go index e62436890..ddc155fab 100644 --- a/persistence/smart_playlist_repository_test.go +++ b/persistence/smart_playlist_repository_test.go @@ -58,6 +58,41 @@ var _ = Describe("PlaylistRepository - Smart Playlists", func() { }) }) + Context("re-imported from disk", func() { + // The scanner re-imports every playlist in a touched folder, and a freshly parsed + // .nsp carries no counters — saving it must not wipe the ones already evaluated. + It("keeps the stored counters when a freshly parsed playlist is saved over it", func() { + rules = &criteria.Criteria{ + Expression: criteria.All{ + criteria.Contains{"title": "Antenna"}, + }, + } + pls := model.Playlist{Name: "Smart", OwnerID: "userid", Rules: rules, Path: "/music/smart.nsp", Sync: true} + Expect(repo.Put(&pls)).To(Succeed()) + DeferCleanup(func() { _ = repo.Delete(pls.ID) }) + + evaluated, err := repo.GetWithTracks(pls.ID, true, false) + Expect(err).ToNot(HaveOccurred()) + Expect(evaluated.SongCount).To(BeNumerically(">", 0)) + + stored, err := repo.Get(pls.ID) + Expect(err).ToNot(HaveOccurred()) + Expect(stored.SongCount).To(Equal(evaluated.SongCount)) + + reimported := model.Playlist{ + ID: pls.ID, Name: pls.Name, OwnerID: "userid", Rules: rules, + Path: pls.Path, Sync: true, + } + Expect(repo.Put(&reimported)).To(Succeed()) + + afterImport, err := repo.Get(pls.ID) + Expect(err).ToNot(HaveOccurred()) + Expect(afterImport.SongCount).To(Equal(stored.SongCount)) + Expect(afterImport.Duration).To(Equal(stored.Duration)) + Expect(afterImport.Size).To(Equal(stored.Size)) + }) + }) + Context("child smart playlists", func() { BeforeEach(func() { DeferCleanup(configtest.SetupConfig()) From 7a11ca69bbfd3465ef4ce2af52966a97c51de77e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Deluan=20Quint=C3=A3o?= Date: Wed, 19 Aug 2026 08:36:44 -0400 Subject: [PATCH 15/27] fix(jellyfin): honor the Filters, SortBy and MaxHeight params clients actually send (#5981) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(jellyfin): honor Filters=IsFavorite on /Artists and /Artists/AlbumArtists listArtistsByRole hand-built its itemsQuery and never set favOnly, so the favorites filter was silently dropped on both artist routes while /Items honored it. Finamp's home screen asks for favorite artists once per load and was served the entire artist list instead: 10,298 artists, 6.15 MB, 2.7s on a real library, and the wrong data on screen. Extract the favOnly parsing that parseItemsQuery already did into parseFavOnly and use it in both places. listArtists now adds the starred predicate to notMissing rather than replacing it, matching listAlbums and listSongs, so a favorite artist whose files are gone stays excluded. * fix(jellyfin): map SortBy=Runtime to duration for albums and songs sortColumnsByType had no runtime/runtimeticks key for any type, so Finamp's "Duration" sort silently misbehaved in two different ways. Albums: Finamp sends a bare SortBy=Runtime. Nothing matched, opts.Sort stayed empty, and applyOptions skips OrderBy entirely when Sort is empty — so the query ran with no ORDER BY at all and Ascending and Descending returned identical lists. Songs: Finamp sends SortBy=Runtime,AlbumArtist,Album,SortName. applySort takes the first *recognized* key, so Runtime was skipped and the list came back sorted by album artist while looking correct. Both repos already accept a duration sort (mediafile_repository maps it explicitly; album_repository falls through to the column name), so no migration is needed. Sorting 97k songs by duration costs a temp B-tree (~114ms on a prod-sized copy) — the same cost the Subsonic and UI duration sorts already pay, and correct where the previous behaviour was merely fast. * fix(jellyfin): apply the played/unplayed filters and MaxHeight image bound Filters was matched with a substring test for IsFavorite, so every other token Jellyfin defines was silently dropped and the response kept rows it should have excluded. Finamp sends Filters=IsUnplayed in normal use. Replace the bool with a parsed itemFilters carrying nullable favorite and played flags, so isFavorite=false and isPlayed=false are real filters rather than indistinguishable from an absent param. Standalone params are read first and the Filters list overrides them, the precedence real Jellyfin has. IsFavoriteOrLikes now maps to favorites deliberately instead of by substring accident; Likes, Dislikes, IsFolder, IsNotFolder and IsResumable have no Navidrome equivalent and are dropped rather than half-applied. The negative cases match NULL as well, since annotations are LEFT JOINed and an untouched item has no row. getItemImage read only maxwidth, so a client sending just MaxHeight got the full-size original: measured against a real cover, maxHeight=100 returned 82,570 bytes where maxWidth=100 returned 3,316. Use the tighter of the two bounds. * refactor(jellyfin): share the plain-param parser between /Items and /Artists listArtistsByRole hand-listed the itemsQuery fields it happened to need, which is exactly how the favorites filter went missing: the literal has been amended in four of the five commits that touched it. Extract listParams for the fields that come straight from query params so both paths read one parser, and the next supported param reaches every list path instead of only /Items. Also from the cleanup pass: collapse imageSize to a single clamped comparison and read its bounds through req.Params like the rest of the package, which drops the strconv import; build the artist and playlist filter lists with the flat append shape the album and song paths already use, instead of re-wrapping opts.Filters into a nested And per predicate; drop a nil guard in listPlaylists that no caller can reach, since both paths into queryItemsOfType build QueryOptions without Filters. applySort now logs when no SortBy key resolves at all — a miss inside a fallback list is normal, but none matching means a silently ignored sort, the failure mode that hid the Runtime bug. Its doc comment records why the remaining keys cannot simply be joined. Folds three duplicated test bodies into the tables that already parameterize them, and covers the artist-parent album branch, which reaches notMissing through filter.AlbumsByArtistID rather than the default branch. * docs(jellyfin): correct how applySort describes Jellyfin's SortBy semantics The comment claimed SortBy is a comma-separated fallback list. It is not: RequestHelpers.GetOrderBy (10.10) builds one (ItemSortBy, SortOrder) pair per key, so Jellyfin orders by every key in turn. Navidrome applies only the first recognized one, which is a real divergence — secondary keys never break ties — not the intended reading of the parameter. The assertion that the keys cannot be joined was also wrong. buildSortOrder does split its input on commas; what it maps is the whole string, so joining raw Jellyfin key names misses the mappings. Mapping each key first and joining the results would work, which makes multi-key sorting a real option rather than a blocked one. Documenting the current behaviour as a known divergence until then. * fix(jellyfin): order by every recognized SortBy key, not just the first Jellyfin orders by each SortBy key in turn, so "DatePlayed,SortName" means break ties by name. Navidrome applied only the first recognized key and dropped the rest, which is 28% of the sort traffic on a real server (23 of 82 requests in 12h carry 2-5 keys). Most were harmless because the primary key dominates, but PremiereDate,Album,ParentIndexNumber,IndexNumber,SortName came back unordered within a year. The keys cannot simply be joined: sortMapping keyed on the whole Sort string, so a joined value missed every mapping and fell through to raw column names. Make it resolve a comma list per part, but only when every part is a known key — the four existing callers that pass raw column lists (core/matcher, core/lyrics, core/maintenance, subsonic/browsing) all carry a part that is not a mapping key, several with their own direction, so they keep falling through exactly as before. Verified each one. applySort now collects every recognized key, skipping duplicates so ParentIndexNumber,IndexNumber does not repeat a column. random stays alone: the repo matches it by exact string equality, so joining it would both break that path and emit a bare 'random' column into the ORDER BY. Verified against a prod-sized copy: every multi-key combination seen in real traffic returns 200, and a secondary key now changes the order within a tied year for songs. Albums are unchanged there, because their max_year mapping already ended in ", name". * fix(persistence): resolve sort mappings exactly once Making sortMapping resolve a comma list per part broke an invariant it had been relying on: idempotence. sanitizeSort mapped the sort key up front and applyOptions then ran buildSortOrder over the result, so sortMapping was already being handed its own output. That was harmless only while a mapped value could never look like a key list. media_file's rated_at maps to "rating, rated_at", and both parts are keys, so the second pass expanded it to "rating, rating, rated_at". Found by round-tripping every mapping in all four repositories; it was the only collision, and the duplicate sort key was benign in SQL, but any future mapping of that shape would silently change meaning. sanitizeSort now validates without resolving, leaving buildSortOrder as the single mapping point. The generated SQL is unchanged — the whole suite passes apart from the two specs that asserted the old return value, which are updated and joined by a round-trip guard covering exactly the rated_at shape. Also use the paren-aware splitFunc that buildSortOrder already uses, so an expression carrying commas inside its parentheses cannot be split apart. * refactor(jellyfin,persistence): flatten the sort resolution paths Cleanup pass over the branch, no behavior change. sortMapping loses the len(parts)>1 guard, which existed only to pick between two identical toSnakeCase exits; the single-key case now falls through the same loop. lookupSortMapping hands back the snake_case form it had to derive so the fallback stops recomputing it — toSnakeCase is two regexps, and on a miss it was running twice per call. sanitizeSort now asks lookupSortMapping instead of probing the map itself, so "is this a known sort key" has one answer; the two had already drifted, since sanitizeSort tried one casing where the resolver tries three. applySort folds the nested random branch into the skip condition and the two trailing length tests into one switch. setSortMappings documents the invariant the comma-list rule depends on, where someone adding a mapping will read it. The README line describing SortBy still said only the first key applied, which the commit before last made false. Tests: the twelve near-identical sorting specs become one DescribeTable of (itemType, SortBy, want) triples, 124 lines to 36, and the applyOptions round-trip assertion collapses to the buildSortOrder call its sibling uses. * fix(jellyfin): keep annotation filters out of search, resolve sorts per part Two findings from the Codex review on #5981. The played/unplayed filters turned working requests into 500s when combined with SearchTerm. Search runs a two-phase FTS query whose first phase selects rowids with no annotation join, so a starred or play_count predicate there is "no such column", not a filter. Measured against master: MusicAlbum with SearchTerm and Filters=IsUnplayed went 200 -> 500, likewise IsPlayed and the Audio equivalents. listAlbums and listSongs now skip those predicates on the search path, matching what listArtists already did. That also clears the same 500 master already had for Filters=IsFavorite with SearchTerm. sortMapping resolved a comma list only while every part was a known key, so a list mixing a plain column with a mapped key kept neither: MusicAlbum SortBy=Runtime,SortName arrives as "duration, name", and duration is a plain album column, so name stayed raw instead of expanding to order_album_name. Albums whose name differs from its sort form — 1,366 of 6,987 on a real library — then ordered by the wrong secondary key, and PreferSortTags was ignored. Each part is now resolved on its own, which is what setSortMappings already documents for a single field. Verified every in-tree caller that passes a raw column list still produces its original ORDER BY. Codex also asked for the artist search path to apply the same filters. It would 500 for the reason above, and wrapping the library scope in a compound filter makes requestedLibraryIDs stop recognizing it, silently widening the search past the requested ParentId. * fix(jellyfin): honor the first SortOrder value for a multi-key sort applySort compared the whole SortOrder string with "Descending", so a per-key list like SortOrder=Descending,Ascending failed the match and every key, including the primary, sorted ascending — the exact opposite of the request. Take the first comma-separated value, which Jellyfin also uses for any key past the end of the SortOrder list. True per-key directions can't be expressed through the single opts.Sort string and are left out; no observed client sends a SortOrder list. --- persistence/sql_base_repository.go | 36 ++++- persistence/sql_base_repository_test.go | 59 +++++++- persistence/sql_restful.go | 12 +- server/jellyfin/README.md | 6 +- server/jellyfin/browsing.go | 19 +-- server/jellyfin/browsing_test.go | 23 +++ server/jellyfin/images.go | 15 +- server/jellyfin/images_test.go | 33 +++- server/jellyfin/items.go | 149 +++++++++++++------ server/jellyfin/items_test.go | 190 +++++++++++++++--------- 10 files changed, 394 insertions(+), 148 deletions(-) diff --git a/persistence/sql_base_repository.go b/persistence/sql_base_repository.go index 428ba7a7b..c61bca1a6 100644 --- a/persistence/sql_base_repository.go +++ b/persistence/sql_base_repository.go @@ -97,6 +97,8 @@ func (r *sqlRepository) registerModel(instance any, filters map[string]filterFun } // setSortMappings sets the mappings for the sort fields. If the sort field is not in the map, it will be used as is. +// This applies per comma-separated part, so a key added here also defines that bare name wherever a +// caller uses it inside a sort list. // // If PreferSortTags is enabled, it will map the order fields to the corresponding sort expression, // which gives precedence to sort tags. @@ -147,17 +149,37 @@ func (r sqlRepository) applyOptions(sq SelectBuilder, options ...model.QueryOpti // TODO Change all sortMappings to have a consistent case func (r sqlRepository) sortMapping(sort string) string { - if mapping, ok := r.sortMappings[sort]; ok { + if mapping, _, ok := r.lookupSortMapping(sort); ok { return mapping } - if mapping, ok := r.sortMappings[toCamelCase(sort)]; ok { - return mapping + // Each part of a comma list is resolved on its own, so a mix of mapped keys and plain columns + // keeps the mappings the recognized parts have. + parts := strings.FieldsFunc(sort, splitFunc(',')) + mapped := make([]string, 0, len(parts)) + for _, part := range parts { + part = strings.TrimSpace(part) + if partMapping, _, ok := r.lookupSortMapping(part); ok { + part = partMapping + } else { + part = toSnakeCase(part) + } + mapped = append(mapped, part) } - sort = toSnakeCase(sort) - if mapping, ok := r.sortMappings[sort]; ok { - return mapping + return strings.Join(mapped, ", ") +} + +// lookupSortMapping also returns the snake_case form when it had to derive one, so a caller's +// fallback doesn't recompute it: toSnakeCase runs two regexps. +func (r sqlRepository) lookupSortMapping(sort string) (mapping, snakeCased string, ok bool) { + if mapping, ok = r.sortMappings[sort]; ok { + return mapping, sort, true } - return sort + if mapping, ok = r.sortMappings[toCamelCase(sort)]; ok { + return mapping, "", true + } + snakeCased = toSnakeCase(sort) + mapping, ok = r.sortMappings[snakeCased] + return mapping, snakeCased, ok } func (r sqlRepository) buildSortOrder(sort, order string) string { diff --git a/persistence/sql_base_repository_test.go b/persistence/sql_base_repository_test.go index 9c6c6007f..0f76eb6ab 100644 --- a/persistence/sql_base_repository_test.go +++ b/persistence/sql_base_repository_test.go @@ -92,14 +92,28 @@ var _ = Describe("sqlRepository", func() { Expect(sort).To(BeEmpty()) }) - It("returns the mapped value when sort key exists", func() { + // Validation only: buildSortOrder resolves the mapping, so mapping here too would hand + // sortMapping its own output and re-map values whose parts are themselves keys. + It("accepts a known sort key without resolving it", func() { sort, _ := r.sanitizeSort("sort1", "") - Expect(sort).To(Equal("mappedSort1")) + Expect(sort).To(Equal("sort1")) }) It("is case insensitive", func() { sort, _ := r.sanitizeSort("Sort1", "") - Expect(sort).To(Equal("mappedSort1")) + Expect(sort).To(Equal("sort1")) + }) + + It("still resolves the mapping by the time the SQL is built", func() { + Expect(r.buildSortOrder("sort1", "asc")).To(Equal("mappedSort1 asc")) + }) + + // A mapping whose parts are themselves keys (media_file rated_at = "rating, rated_at") + // must survive the round trip through sanitizeSort and buildSortOrder unduplicated. + It("does not re-map a value whose parts are also keys", func() { + r.sortMappings = map[string]string{"rating": "rating", "rated_at": "rating, rated_at"} + sort, _ := r.sanitizeSort("rated_at", "") + Expect(r.buildSortOrder(sort, "asc")).To(Equal("rating asc, rated_at asc")) }) It("returns the field if it is a valid field", func() { @@ -135,6 +149,45 @@ var _ = Describe("sqlRepository", func() { }) }) + Describe("sortMapping", func() { + BeforeEach(func() { + r.sortMappings = map[string]string{ + "name": "order_album_name, order_album_artist_name", + "recently_added": "album.created_at, album.id", + } + }) + It("maps a single key", func() { + Expect(r.sortMapping("recently_added")).To(Equal("album.created_at, album.id")) + }) + It("maps every part of a comma list when all of them are known keys", func() { + Expect(r.sortMapping("recently_added, name")). + To(Equal("album.created_at, album.id, order_album_name, order_album_artist_name")) + }) + It("resolves the known parts of a mixed list and leaves the rest as columns", func() { + Expect(r.sortMapping("recently_added, play_count")). + To(Equal("album.created_at, album.id, play_count")) + }) + // Jellyfin's MusicAlbum SortBy=Runtime,SortName arrives as "duration, name"; duration is a + // plain album column while name is mapped, and the mapping must survive the mix. + It("keeps a mapping when an earlier part is a plain column", func() { + Expect(r.sortMapping("duration, name")). + To(Equal("duration, order_album_name, order_album_artist_name")) + }) + It("leaves a raw column list with directions untouched", func() { + Expect(r.sortMapping("starred desc, rating desc")).To(Equal("starred desc, rating desc")) + }) + It("does not split an expression on a comma inside its parentheses", func() { + Expect(r.sortMapping("coalesce(name, ''), title")).To(Equal("coalesce(name, ''), title")) + Expect(r.sortMapping("coalesce(nullif(a,''), b) desc, c")).To(Equal("coalesce(nullif(a,''), b) desc, c")) + }) + It("keeps a mapping whose value nests commas inside parentheses", func() { + r.sortMappings["max_year"] = "coalesce(nullif(original_date,''), cast(max_year as text)), release_date" + Expect(r.sortMapping("max_year, name")).To(Equal( + "coalesce(nullif(original_date,''), cast(max_year as text)), release_date, " + + "order_album_name, order_album_artist_name")) + }) + }) + Describe("buildSortOrder", func() { BeforeEach(func() { r.sortMappings = map[string]string{} diff --git a/persistence/sql_restful.go b/persistence/sql_restful.go index 1dcabcec6..b1cfd2379 100644 --- a/persistence/sql_restful.go +++ b/persistence/sql_restful.go @@ -69,13 +69,11 @@ func (r *sqlRepository) parseRestOptions(ctx context.Context, options ...rest.Qu func (r sqlRepository) sanitizeSort(sort, order string) (string, string) { if sort != "" { sort = toSnakeCase(sort) - if mapped, ok := r.sortMappings[sort]; ok { - sort = mapped - } else { - if !r.isFieldWhiteListed(sort) { - log.Warn(r.ctx, "Ignoring sort not whitelisted", "sort", sort, "table", r.tableName) - sort = "" - } + // Validate only: buildSortOrder resolves the mapping later, and mapping here as well would + // feed sortMapping its own output. + if _, _, known := r.lookupSortMapping(sort); !known && !r.isFieldWhiteListed(sort) { + log.Warn(r.ctx, "Ignoring sort not whitelisted", "sort", sort, "table", r.tableName) + sort = "" } } if order != "" { diff --git a/server/jellyfin/README.md b/server/jellyfin/README.md index 5759e6cc0..dc3219dfa 100644 --- a/server/jellyfin/README.md +++ b/server/jellyfin/README.md @@ -104,7 +104,11 @@ album's tracks — Feishin fetches them this way instead of `ParentId`); `GenreI genre's albums or tracks — Finamp's genre screen sends it the same way; `/Artists/AlbumArtists` and `MusicArtist` queries accept it too, matching artists credited on an album of that genre); `SearchTerm`; -favorites-only (`Filters=IsFavorite` or the standalone `isFavorite=true`); `SortBy`/`SortOrder`; +`Filters` (`IsFavorite`, `IsFavoriteOrLikes`, `IsPlayed`, `IsUnplayed`) and the standalone +`isFavorite`/`isPlayed` booleans it can also be expressed as — `Filters` wins when both are sent, as +in Jellyfin; `Likes`, `Dislikes`, `IsFolder`, `IsNotFolder` and `IsResumable` have no Navidrome +equivalent and are ignored; `SortBy`/`SortOrder` (every recognized key is applied in order, so secondary keys break ties; +unrecognized keys are skipped, and `Random` always sorts alone); `StartIndex`/`Limit`; and `Ids` (batch fetch by id). `Recursive=false` with a library `ParentId` returns direct children only (no tracks — no track is a library's direct child). diff --git a/server/jellyfin/browsing.go b/server/jellyfin/browsing.go index 9acd8d5a8..25c44e748 100644 --- a/server/jellyfin/browsing.go +++ b/server/jellyfin/browsing.go @@ -24,10 +24,6 @@ func (api *Router) getAlbumArtists(w http.ResponseWriter, r *http.Request) { // when accessible (like queryItems) or all accessible libraries otherwise. func (api *Router) listArtistsByRole(w http.ResponseWriter, r *http.Request, role model.Role) { ctx := r.Context() - p := req.Params(r) - opts := model.QueryOptions{Offset: p.IntOr("startindex", 0), Max: p.IntOr("limit", 0)} - applySort(&opts, "MusicArtist", p.StringOr("sortby", ""), p.StringOr("sortorder", "")) - scopeIDs, _, ok := parentIDScope(ctx, r) if !ok { http.Error(w, "Not Found", http.StatusNotFound) @@ -38,14 +34,13 @@ func (api *Router) listArtistsByRole(w http.ResponseWriter, r *http.Request, rol http.Error(w, "Not Found", http.StatusNotFound) return } - // Only the fields listArtists reads; /Artists has no favorites filter, so favOnly stays false. - // Finamp's artist tab sends GenreIds when a genre filter is active. - q := itemsQuery{ - scopeIDs: scopeIDs, - genreIds: genreIds, - search: searchTerm(p), - fields: dto.ParseFields(p.Strings("fields")...), - } + // This route resolves its own scope, so it shares only the plain query params with /Items. + q := listParams(req.Params(r)) + q.scopeIDs = scopeIDs + q.genreIds = genreIds + + opts := model.QueryOptions{Offset: q.offset, Max: q.limit} + applySort(&opts, "MusicArtist", q.sortBy, q.sortOrder) if q.search != "" { opts.Max = clampLimit(opts.Max, defaultSearchLimit, maxSearchLimit) } diff --git a/server/jellyfin/browsing_test.go b/server/jellyfin/browsing_test.go index 2825c03e8..657c7a42c 100644 --- a/server/jellyfin/browsing_test.go +++ b/server/jellyfin/browsing_test.go @@ -156,6 +156,29 @@ var _ = Describe("Browsing", func() { Expect(sql).NotTo(ContainSubstring("library_artist.library_id")) }) + DescribeTable("restricts to favorites", + func(url string, handler func(*Router) http.HandlerFunc) { + artistRepo := ds.Artist(context.Background()).(*tests.MockArtistRepo) + artistRepo.SetData(model.Artists{{ID: testID("ar1"), Name: "Artist"}}) + w := httptest.NewRecorder() + r := httptest.NewRequest("GET", url, nil).WithContext(ctxUser(model.Libraries{{ID: 1}})) + invoke(handler(api), w, r) + Expect(w.Code).To(Equal(http.StatusOK)) + sql, args, err := artistRepo.Options.Filters.ToSql() + Expect(err).NotTo(HaveOccurred()) + Expect(sql).To(ContainSubstring("starred")) + // listArtists always ANDs notMissing, favorites filter or not. + Expect(sql).To(ContainSubstring("missing")) + Expect(args).To(ContainElement(true)) + }, + Entry("Filters=IsFavorite", "/Artists?Filters=IsFavorite", + func(a *Router) http.HandlerFunc { return a.getArtists }), + Entry("isFavorite=true", "/Artists?isFavorite=true", + func(a *Router) http.HandlerFunc { return a.getArtists }), + Entry("on /Artists/AlbumArtists", "/Artists/AlbumArtists?Filters=IsFavorite", + func(a *Router) http.HandlerFunc { return a.getAlbumArtists }), + ) + It("404s a malformed ParentId instead of listing every library's artists", func() { artistRepo := ds.Artist(context.Background()).(*tests.MockArtistRepo) artistRepo.SetData(model.Artists{{ID: testID("ar1"), Name: "Artist"}}) diff --git a/server/jellyfin/images.go b/server/jellyfin/images.go index 65f02d7a0..fcf69c039 100644 --- a/server/jellyfin/images.go +++ b/server/jellyfin/images.go @@ -11,7 +11,6 @@ import ( _ "image/png" "io" "net/http" - "strconv" "github.com/dustin/go-humanize" "github.com/navidrome/navidrome/conf" @@ -20,9 +19,20 @@ import ( "github.com/navidrome/navidrome/model" "github.com/navidrome/navidrome/model/request" "github.com/navidrome/navidrome/server/imghttp" + "github.com/navidrome/navidrome/utils/req" _ "golang.org/x/image/webp" ) +// imageSize picks the tighter of Jellyfin's two bounds, because Navidrome resizes on a single +// dimension: reading only MaxWidth serves the full-size original to a client that sent MaxHeight. +func imageSize(maxWidth, maxHeight int) int { + w, h := max(maxWidth, 0), max(maxHeight, 0) + if w == 0 || h == 0 { + return max(w, h) + } + return min(w, h) +} + func (api *Router) getItemImage(w http.ResponseWriter, r *http.Request) { // Public endpoint, like real Jellyfin's image routes: clients fetch cover URLs without credentials // and item ids are unguessable, so resolution runs elevated to bypass the visibility filter. @@ -31,7 +41,8 @@ func (api *Router) getItemImage(w http.ResponseWriter, r *http.Request) { if !ok { return } - size, _ := strconv.Atoi(r.URL.Query().Get("maxwidth")) + p := req.Params(r) + size := imageSize(p.IntOr("maxwidth", 0), p.IntOr("maxheight", 0)) artID := api.resolveArtworkID(ctx, itemId) img, err := api.artwork.GetOrPlaceholder(ctx, artID, size, false) diff --git a/server/jellyfin/images_test.go b/server/jellyfin/images_test.go index 0e0d6220d..2f32e6ec9 100644 --- a/server/jellyfin/images_test.go +++ b/server/jellyfin/images_test.go @@ -30,14 +30,16 @@ import ( type fakeArtwork struct { artwork.Artwork - recvId string - recvCtx context.Context - data []byte - hash string + recvId string + recvSize int + recvCtx context.Context + data []byte + hash string } func (f *fakeArtwork) GetOrPlaceholder(ctx context.Context, id string, size int, square bool) (*artwork.Image, error) { f.recvId = id + f.recvSize = size f.recvCtx = ctx data := f.data if data == nil { @@ -61,6 +63,29 @@ func newImageRequest(itemId string) (*httptest.ResponseRecorder, *http.Request) } var _ = Describe("Images", func() { + // Real Jellyfin fits the image inside either bound, so a client that sends only MaxHeight must + // still get a resized image rather than the full-size original. + DescribeTable("derives the requested size from MaxWidth or MaxHeight", + func(query string, wantSize int) { + ds := &tests.MockDataStore{} + ds.Album(context.Background()).(*tests.MockAlbumRepo).SetData(model.Albums{{ID: testID("a1"), Name: "One"}}) + fa := &fakeArtwork{} + api := &Router{ds: ds, artwork: fa} + + w, r := newImageRequest(dto.EncodeID(testID("a1"))) + r.URL.RawQuery = query + api.getItemImage(w, r) + + Expect(w.Code).To(Equal(http.StatusOK)) + Expect(fa.recvSize).To(Equal(wantSize)) + }, + Entry("MaxWidth only", "maxwidth=300", 300), + Entry("MaxHeight only", "maxheight=300", 300), + Entry("both, smaller bound wins", "maxwidth=200&maxheight=300", 200), + Entry("both, smaller bound wins regardless of order", "maxwidth=300&maxheight=200", 200), + Entry("neither", "", 0), + ) + It("streams album artwork", func() { ds := &tests.MockDataStore{} ds.Album(context.Background()).(*tests.MockAlbumRepo).SetData(model.Albums{{ID: testID("a1"), Name: "One"}}) diff --git a/server/jellyfin/items.go b/server/jellyfin/items.go index de0da88eb..eb5cd2a1a 100644 --- a/server/jellyfin/items.go +++ b/server/jellyfin/items.go @@ -31,6 +31,51 @@ func searchTerm(p *req.Values) string { return strings.TrimSpace(p.StringOr("searchterm", "")) } +// itemFilters is the parsed Filters=... list together with the standalone isFavorite/isPlayed params +// clients may send instead. A nil field means the client asked for no filtering on that dimension. +type itemFilters struct { + favorite *bool + played *bool +} + +// parseItemFilters reads the standalone params first and lets the Filters list win, matching real +// Jellyfin. Tokens with no Navidrome equivalent (Likes, IsFolder, IsResumable) are dropped. +func parseItemFilters(p *req.Values) itemFilters { + f := itemFilters{favorite: p.BoolPtr("isfavorite"), played: p.BoolPtr("isplayed")} + for token := range strings.SplitSeq(p.StringOr("filters", ""), ",") { + switch strings.TrimSpace(token) { + case "IsFavorite", "IsFavoriteOrLikes": + f.favorite = new(true) + case "IsPlayed": + f.played = new(true) + case "IsUnplayed": + f.played = new(false) + } + } + return f +} + +// predicates renders the filters as annotation-column conditions. The negative cases have to match +// NULL as well: annotations are LEFT JOINed, so an item nobody has touched has no row at all. +func (f itemFilters) predicates() []squirrel.Sqlizer { + var out []squirrel.Sqlizer + if f.favorite != nil { + if *f.favorite { + out = append(out, squirrel.Eq{"starred": true}) + } else { + out = append(out, squirrel.Or{squirrel.Eq{"starred": nil}, squirrel.Eq{"starred": false}}) + } + } + if f.played != nil { + if *f.played { + out = append(out, squirrel.Gt{"play_count": 0}) + } else { + out = append(out, squirrel.Or{squirrel.Eq{"play_count": nil}, squirrel.Eq{"play_count": 0}}) + } + } + return out +} + func (api *Router) getItems(w http.ResponseWriter, r *http.Request) { res, err := api.queryItems(r.Context(), r) if err != nil { @@ -214,7 +259,7 @@ type itemsQuery struct { sortOrder string offset int limit int - favOnly bool + filters itemFilters // parentId scopes the query. entityParent is the same id only when it names an entity (an artist // for MusicAlbum, an album for Audio) rather than a library. parentId string @@ -231,6 +276,19 @@ type itemsQuery struct { studioIds []string } +// listParams reads the itemsQuery fields that come straight from query params. +func listParams(p *req.Values) itemsQuery { + return itemsQuery{ + fields: dto.ParseFields(p.Strings("fields")...), + search: searchTerm(p), + sortBy: p.StringOr("sortby", ""), + sortOrder: p.StringOr("sortorder", ""), + offset: p.IntOr("startindex", 0), + limit: p.IntOr("limit", 0), + filters: parseItemFilters(p), + } +} + // parseItemsQuery also resolves the entity types (inferring them from the parent when // IncludeItemTypes is absent) and the library scope. Query keys are read lowercase because // normalizeQueryKeys folded them (Jellyfin binds case-insensitively). A non-empty id param that @@ -261,24 +319,14 @@ func (api *Router) parseItemsQuery(ctx context.Context, r *http.Request) (itemsQ if !ok { return itemsQuery{}, model.ErrNotFound } - q := itemsQuery{ - fields: dto.ParseFields(p.Strings("fields")...), - ids: ids, - rawTypes: p.StringOr("includeitemtypes", ""), - search: searchTerm(p), - sortBy: p.StringOr("sortby", ""), - sortOrder: p.StringOr("sortorder", ""), - offset: p.IntOr("startindex", 0), - limit: p.IntOr("limit", 0), - // Clients express "favorites only" two ways: Filters=IsFavorite and the standalone - // isFavorite=true param (Finamp's "Favourite tracks" widget uses the latter). - favOnly: strings.Contains(p.StringOr("filters", ""), "IsFavorite") || p.BoolOr("isfavorite", false), - parentId: parentId, - genreIds: genreIds, - albumIds: albumIds, - years: parseYears(r), - studioIds: studioIds, - } + q := listParams(p) + q.ids = ids + q.rawTypes = p.StringOr("includeitemtypes", "") + q.parentId = parentId + q.genreIds = genreIds + q.albumIds = albumIds + q.years = parseYears(r) + q.studioIds = studioIds // An artist's page filters by artist, not ParentId: Finamp sends ParentId= for scoping // plus AlbumArtistIds/ArtistIds/contributingArtistIds for the artist. albumArtistScope := firstNonEmpty(p.StringOr("albumartistids", ""), p.StringOr("artistids", "")) @@ -621,8 +669,10 @@ func (api *Router) listAlbums(ctx context.Context, opts model.QueryOptions, q it if len(q.studioIds) > 0 { filters = append(filters, filter.ByStudioID(q.studioIds)) } - if q.favOnly { - filters = append(filters, filter.ByStarred().Filters) + // Not on the search path: its first FTS phase selects rowids with no annotation join, so a + // starred/play_count predicate there is "no such column" rather than a filter. + if q.search == "" { + filters = append(filters, q.filters.predicates()...) } opts.Filters = filters opts = filter.ApplyLibraryFilter(opts, q.scopeIDs) @@ -668,8 +718,10 @@ func (api *Router) listSongs(ctx context.Context, opts model.QueryOptions, q ite if len(q.studioIds) > 0 { filters = append(filters, filter.ByStudioID(q.studioIds)) } - if q.favOnly { - filters = append(filters, filter.ByStarred().Filters) + // Not on the search path: its first FTS phase selects rowids with no annotation join, so a + // starred/play_count predicate there is "no such column" rather than a filter. + if q.search == "" { + filters = append(filters, q.filters.predicates()...) } opts.Filters = filters opts = filter.ApplyLibraryFilter(opts, q.scopeIDs) @@ -720,14 +772,12 @@ func (api *Router) listArtists(ctx context.Context, opts model.QueryOptions, q i return materialized(result(slice.Map(artists, toItem), total, opts.Offset)), nil } - if q.favOnly { - opts.Filters = filter.ArtistsByStarred().Filters - } else { - opts.Filters = notMissing - } + filters := squirrel.And{notMissing} + filters = append(filters, q.filters.predicates()...) if len(q.genreIds) > 0 { - opts.Filters = squirrel.And{opts.Filters, filter.ArtistsByGenreID(q.genreIds)} + filters = append(filters, filter.ArtistsByGenreID(q.genreIds)) } + opts.Filters = filters opts = filter.ArtistsByRole(opts, role) opts = filter.ApplyArtistLibraryFilter(opts, q.scopeIDs) total, _ := repo.CountAll(model.QueryOptions{Filters: opts.Filters}) @@ -752,13 +802,8 @@ func (api *Router) listGenres(ctx context.Context, opts model.QueryOptions) (ite // listPlaylists lists playlists visible to the current user. Visibility (public or owned) is // enforced by playlistRepository, not scopeIDs. func (api *Router) listPlaylists(ctx context.Context, opts model.QueryOptions, q itemsQuery) (itemsResult, error) { - if q.favOnly { - starred := squirrel.Eq{"starred": true} - if opts.Filters == nil { - opts.Filters = starred - } else { - opts.Filters = squirrel.And{opts.Filters, starred} - } + if preds := q.filters.predicates(); len(preds) > 0 { + opts.Filters = squirrel.And(preds) } repo := api.ds.Playlist(ctx) total, err := repo.CountAll(model.QueryOptions{Filters: opts.Filters}) @@ -908,18 +953,32 @@ func result(items []dto.BaseItemDto, total, start int) dto.QueryResult { return dto.QueryResult{Items: items, TotalRecordCount: total, StartIndex: start} } -// applySort translates Jellyfin's SortBy/SortOrder into a valid model.QueryOptions sort key for the -// item type. Clients send SortBy as a comma-separated fallback list (e.g. "DateCreated,SortName"); -// this uses the first recognized key. An unrecognized SortBy is left untouched (the repo's default), -// not passed through raw where it could produce an invalid ORDER BY. +// applySort keeps every recognized SortBy key, so secondary keys break ties as Jellyfin intends. +// Unrecognized keys are skipped, not passed through raw where they could make an invalid ORDER BY. func applySort(opts *model.QueryOptions, itemType, sortBy, order string) { + var cols []string for key := range strings.SplitSeq(sortBy, ",") { - if col, ok := sortColumn(itemType, strings.TrimSpace(key)); ok { - opts.Sort = col + col, ok := sortColumn(itemType, strings.TrimSpace(key)) + // The repo matches random by exact string equality, so it can only ever sort alone. + if !ok || slices.Contains(cols, col) || (col == "random" && len(cols) > 0) { + continue + } + cols = append(cols, col) + if col == "random" { break } } - if strings.EqualFold(order, "Descending") { + switch { + case len(cols) > 0: + opts.Sort = strings.Join(cols, ", ") + case sortBy != "": + log.Debug("Jellyfin API: no usable SortBy key, falling back to the default order", + "itemType", itemType, "sortBy", sortBy) + } + // Jellyfin allows a per-key SortOrder list, which one Order can't express; honor the first value + // for every key, as Jellyfin does for keys past the end of the list. + first, _, _ := strings.Cut(order, ",") + if strings.EqualFold(first, "Descending") { opts.Order = "desc" } } @@ -941,6 +1000,8 @@ var sortColumnsByType = map[string]map[string]string{ "dateplayed": "play_date", "communityrating": "rating", "random": "random", + "runtime": "duration", + "runtimeticks": "duration", // Finamp's "Latest Releases" sorts by PremiereDate; "year" matches songs' ProductionYear. "premieredate": "year", "productionyear": "year", @@ -964,6 +1025,8 @@ var sortColumnsByType = map[string]map[string]string{ "playcount": "play_count", "dateplayed": "play_date", "communityrating": "rating", + "runtime": "duration", + "runtimeticks": "duration", "premieredate": "max_year", "productionyear": "max_year", }, "MusicGenre": { diff --git a/server/jellyfin/items_test.go b/server/jellyfin/items_test.go index b227ab719..17180c36a 100644 --- a/server/jellyfin/items_test.go +++ b/server/jellyfin/items_test.go @@ -327,17 +327,75 @@ var _ = Describe("Items", func() { Expect(albumRepo.Options.Max).To(Equal(3)) }) - It("applies a starred filter when Filters=IsFavorite", func() { - albumRepo := ds.Album(context.Background()).(*tests.MockAlbumRepo) - albumRepo.SetData(model.Albums{{ID: testID("a1"), Name: "One"}}) - w := httptest.NewRecorder() - r := httptest.NewRequest("GET", "/Items?IncludeItemTypes=MusicAlbum&Filters=IsFavorite", nil).WithContext(ctxUser()) - invoke(api.getItems, w, r) - Expect(w.Code).To(Equal(http.StatusOK)) - sql, _, err := albumRepo.Options.Filters.ToSql() - Expect(err).NotTo(HaveOccurred()) - Expect(sql).To(ContainSubstring("starred")) - }) + DescribeTable("translates the Filters list and its standalone equivalents", + func(query string, wantSQL, notWantSQL []string) { + albumRepo := ds.Album(context.Background()).(*tests.MockAlbumRepo) + albumRepo.SetData(model.Albums{{ID: testID("a1"), Name: "One"}}) + w := httptest.NewRecorder() + r := httptest.NewRequest("GET", "/Items?IncludeItemTypes=MusicAlbum&"+query, nil).WithContext(ctxUser()) + invoke(api.getItems, w, r) + Expect(w.Code).To(Equal(http.StatusOK)) + sql, _, err := albumRepo.Options.Filters.ToSql() + Expect(err).NotTo(HaveOccurred()) + for _, want := range wantSQL { + Expect(sql).To(ContainSubstring(want)) + } + for _, not := range notWantSQL { + Expect(sql).NotTo(ContainSubstring(not)) + } + }, + Entry("IsFavorite", "Filters=IsFavorite", []string{"starred"}, nil), + Entry("IsFavorite,IsUnplayed combined", "Filters=IsFavorite,IsUnplayed", + []string{"starred", "play_count"}, nil), + Entry("IsUnplayed", "Filters=IsUnplayed", []string{"play_count"}, []string{"starred"}), + Entry("IsPlayed", "Filters=IsPlayed", []string{"play_count"}, []string{"starred"}), + Entry("IsFavoriteOrLikes is treated as favorites", "Filters=IsFavoriteOrLikes", []string{"starred"}, nil), + Entry("isPlayed=false", "isPlayed=false", []string{"play_count"}, nil), + Entry("isFavorite=false still filters", "isFavorite=false", []string{"starred"}, nil), + // Jellyfin builds the query from the standalone params, then applies Filters over the top. + Entry("Filters wins over the standalone param", "isFavorite=false&Filters=IsFavorite", + []string{"starred = "}, nil), + // No Navidrome equivalent: these must be dropped, not half-applied. + Entry("Likes is ignored", "Filters=Likes", nil, []string{"starred", "play_count"}), + Entry("IsResumable is ignored", "Filters=IsResumable", nil, []string{"starred", "play_count"}), + // The artist-parent branch gets notMissing from filter.AlbumsByArtistID, not the default + // branch, so favorites must not be the only predicate left on it. + Entry("keeps missing excluded under an artist parent", + "Filters=IsFavorite&ArtistIds="+dto.EncodeID(testID("ar1")), + []string{"starred", "missing"}, nil), + ) + + // Search runs a two-phase FTS query whose first phase has no annotation join, so an + // annotation predicate there is "no such column: starred" -> 500. + DescribeTable("does not push annotation filters into a search", + func(itemType, filters string) { + ds.Album(context.Background()).(*tests.MockAlbumRepo).SetData(model.Albums{{ID: testID("a1"), Name: "One"}}) + ds.MediaFile(context.Background()).(*tests.MockMediaFileRepo).SetData(model.MediaFiles{{ID: testID("s1"), Title: "Song"}}) + w := httptest.NewRecorder() + r := httptest.NewRequest("GET", + "/Items?IncludeItemTypes="+itemType+"&SearchTerm=one&Filters="+filters, nil).WithContext(ctxUser()) + invoke(api.getItems, w, r) + Expect(w.Code).To(Equal(http.StatusOK)) + var opts model.QueryOptions + if itemType == "MusicAlbum" { + opts = ds.Album(context.Background()).(*tests.MockAlbumRepo).Options + } else { + opts = ds.MediaFile(context.Background()).(*tests.MockMediaFileRepo).Options + } + if opts.Filters == nil { + return + } + sql, _, err := opts.Filters.ToSql() + Expect(err).NotTo(HaveOccurred()) + Expect(sql).NotTo(ContainSubstring("starred")) + Expect(sql).NotTo(ContainSubstring("play_count")) + }, + Entry("albums, IsFavorite", "MusicAlbum", "IsFavorite"), + Entry("albums, IsUnplayed", "MusicAlbum", "IsUnplayed"), + Entry("albums, IsPlayed", "MusicAlbum", "IsPlayed"), + Entry("songs, IsFavorite", "Audio", "IsFavorite"), + Entry("songs, IsUnplayed", "Audio", "IsUnplayed"), + ) It("forwards SearchTerm to the repo's Search method", func() { albumRepo := ds.Album(context.Background()).(*tests.MockAlbumRepo) @@ -601,65 +659,59 @@ var _ = Describe("Items", func() { }) Describe("sorting", func() { - It("maps SortBy=PlayCount to the play_count column", func() { - albumRepo := ds.Album(context.Background()).(*tests.MockAlbumRepo) - albumRepo.SetData(model.Albums{{ID: testID("a1"), Name: "One"}}) - w := httptest.NewRecorder() - r := httptest.NewRequest("GET", "/Items?IncludeItemTypes=MusicAlbum&SortBy=PlayCount", nil).WithContext(ctxUser()) - invoke(api.getItems, w, r) - Expect(w.Code).To(Equal(http.StatusOK)) - Expect(albumRepo.Options.Sort).To(Equal("play_count")) - }) + DescribeTable("translates SortBy into the repo's sort keys", + func(itemType, sortBy, want string) { + albumRepo := ds.Album(context.Background()).(*tests.MockAlbumRepo) + albumRepo.SetData(model.Albums{{ID: testID("a1"), Name: "One"}}) + mfRepo := ds.MediaFile(context.Background()).(*tests.MockMediaFileRepo) + mfRepo.SetData(model.MediaFiles{{ID: testID("s1"), Title: "Song"}}) + w := httptest.NewRecorder() + r := httptest.NewRequest("GET", "/Items?IncludeItemTypes="+itemType+"&SortBy="+sortBy, nil).WithContext(ctxUser()) + invoke(api.getItems, w, r) + Expect(w.Code).To(Equal(http.StatusOK)) + got := mfRepo.Options.Sort + if itemType == "MusicAlbum" { + got = albumRepo.Options.Sort + } + Expect(got).To(Equal(want)) + }, + Entry("PlayCount", "MusicAlbum", "PlayCount", "play_count"), + Entry("DatePlayed", "Audio", "DatePlayed", "play_date"), + Entry("Runtime on albums", "MusicAlbum", "Runtime", "duration"), + Entry("RunTimeTicks alias", "MusicAlbum", "RunTimeTicks", "duration"), + // Finamp leads its track sort with Runtime: unless that resolves, the first recognized + // key is AlbumArtist and the list looks sorted while being sorted by the wrong thing. + Entry("Finamp's Runtime-led track sort", "Audio", "Runtime,AlbumArtist,Album,SortName", + "duration, album_artist, album, title"), + Entry("every recognized key, in order", "MusicAlbum", "DateCreated,SortName", "recently_added, name"), + Entry("a key repeating a column is dropped", "Audio", + "PremiereDate,Album,ParentIndexNumber,IndexNumber,SortName", "year, album, title"), + // random is matched by exact string equality in the repo, so it can never share a sort. + Entry("Random stays alone", "MusicAlbum", "Random,SortName", "random"), + Entry("unrecognized keys are skipped", "Audio", "Runtime,Nonsense,SortName", "duration, title"), + Entry("only the last key recognized", "Audio", "Unknown1,Unknown2,SortName", "title"), + Entry("Finamp's album view is disc+track", "Audio", "ParentIndexNumber,IndexNumber,SortName", "album, title"), + Entry("nothing recognized leaves the repo default", "MusicAlbum", "SeriesSortName", ""), + ) - It("maps SortBy=DatePlayed to the play_date column", func() { - mfRepo := ds.MediaFile(context.Background()).(*tests.MockMediaFileRepo) - mfRepo.SetData(model.MediaFiles{{ID: testID("s1"), Title: "Song"}}) - w := httptest.NewRecorder() - r := httptest.NewRequest("GET", "/Items?IncludeItemTypes=Audio&SortBy=DatePlayed", nil).WithContext(ctxUser()) - invoke(api.getItems, w, r) - Expect(w.Code).To(Equal(http.StatusOK)) - Expect(mfRepo.Options.Sort).To(Equal("play_date")) - }) - - It("uses the first recognized key in a comma-separated SortBy list", func() { - albumRepo := ds.Album(context.Background()).(*tests.MockAlbumRepo) - albumRepo.SetData(model.Albums{{ID: testID("a1"), Name: "One"}}) - w := httptest.NewRecorder() - r := httptest.NewRequest("GET", "/Items?IncludeItemTypes=MusicAlbum&SortBy=DateCreated,SortName", nil).WithContext(ctxUser()) - invoke(api.getItems, w, r) - Expect(w.Code).To(Equal(http.StatusOK)) - Expect(albumRepo.Options.Sort).To(Equal("recently_added")) - }) - - It("skips unrecognized keys in a comma-separated SortBy list to find one that is", func() { - mfRepo := ds.MediaFile(context.Background()).(*tests.MockMediaFileRepo) - mfRepo.SetData(model.MediaFiles{{ID: testID("s1"), Title: "Song"}}) - w := httptest.NewRecorder() - r := httptest.NewRequest("GET", "/Items?IncludeItemTypes=Audio&SortBy=Unknown1,Unknown2,SortName", nil).WithContext(ctxUser()) - invoke(api.getItems, w, r) - Expect(w.Code).To(Equal(http.StatusOK)) - Expect(mfRepo.Options.Sort).To(Equal("title")) - }) - - It("maps Finamp's album view SortBy (ParentIndexNumber,IndexNumber) to disc+track order", func() { - mfRepo := ds.MediaFile(context.Background()).(*tests.MockMediaFileRepo) - mfRepo.SetData(model.MediaFiles{{ID: testID("s1"), Title: "Song"}}) - w := httptest.NewRecorder() - r := httptest.NewRequest("GET", "/Items?IncludeItemTypes=Audio&SortBy=ParentIndexNumber,IndexNumber,SortName", nil).WithContext(ctxUser()) - invoke(api.getItems, w, r) - Expect(w.Code).To(Equal(http.StatusOK)) - Expect(mfRepo.Options.Sort).To(Equal("album")) - }) - - It("leaves Sort at the repo default when no SortBy key is recognized", func() { - albumRepo := ds.Album(context.Background()).(*tests.MockAlbumRepo) - albumRepo.SetData(model.Albums{{ID: testID("a1"), Name: "One"}}) - w := httptest.NewRecorder() - r := httptest.NewRequest("GET", "/Items?IncludeItemTypes=MusicAlbum&SortBy=SeriesSortName", nil).WithContext(ctxUser()) - invoke(api.getItems, w, r) - Expect(w.Code).To(Equal(http.StatusOK)) - Expect(albumRepo.Options.Sort).To(Equal("")) - }) + // Jellyfin allows a per-key SortOrder list; we cannot express that through one Order, so + // we honor the first value for all keys, matching Jellyfin's fallback for extra keys. + DescribeTable("reads the first SortOrder value for the whole sort", + func(sortOrder, want string) { + albumRepo := ds.Album(context.Background()).(*tests.MockAlbumRepo) + albumRepo.SetData(model.Albums{{ID: testID("a1"), Name: "One"}}) + w := httptest.NewRecorder() + r := httptest.NewRequest("GET", + "/Items?IncludeItemTypes=MusicAlbum&SortBy=Runtime,SortName&SortOrder="+sortOrder, nil).WithContext(ctxUser()) + invoke(api.getItems, w, r) + Expect(w.Code).To(Equal(http.StatusOK)) + Expect(albumRepo.Options.Order).To(Equal(want)) + }, + Entry("ascending", "Ascending", ""), + Entry("descending", "Descending", "desc"), + Entry("descending leading a list", "Descending,Ascending", "desc"), + Entry("ascending leading a list", "Ascending,Descending", ""), + ) }) Describe("library scoping", func() { From 1f3034f0229777c2023e861a4d9a92aaca1a1207 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Deluan=20Quint=C3=A3o?= Date: Wed, 19 Aug 2026 08:47:53 -0400 Subject: [PATCH 16/27] fix(playlist): block track edits on synced playlists across all APIs (#5984) * fix(playlist): block track edits on synced playlists across all APIs A synced playlist's tracks come from its source file, so any track edit made through the UI or an API was silently reverted on the next scan. Track mutations funnel through two service guards, checkTracksEditable (incremental edits) and Create (wholesale replace, used by Subsonic createPlaylist and Jellyfin's replace path), which each duplicated the smart-playlist check. Both now consult a shared model.Playlist.TracksEditable() predicate, so the native, Subsonic, and Jellyfin paths are all locked: track edits return ErrNotAuthorized (403, or Subsonic error 50) instead of being accepted and lost. Metadata-only edits (name, comment, public, the sync flag itself) still go through checkWritable and are unaffected. In the UI, a synced playlist's track list becomes read-only, mirroring how smart playlists already behave. * fix(playlist): return 409 Conflict for non-editable playlist track edits The previous commit rejected track edits on smart and synced playlists with ErrNotAuthorized (403). That conflates two different things: a 403 says the caller lacks permission, but a synced or smart playlist's tracks are immutable for everyone, including the owner and admins. It is a property of the resource, not the caller. Introduce ErrPlaylistNotEditable and return it from both track-edit guards. The Native and Jellyfin APIs now map it to 409 Conflict; Subsonic maps it to error 50, the closest code it has (it has no read-only concept). The Native track handlers previously mapped this rejection inconsistently (400 on add, 500 on remove, 403 on reorder) through a new shared writePlaylistError helper. Genuine authorization failures (non-owner, non-admin) still return ErrNotAuthorized. * fix(playlist): surface synced read-only state in picker, Jellyfin, and OpenSubsonic Follow-up to the track-edit lock: the read-only state was enforced but not advertised consistently, so clients still offered edits that the server rejects. - UI: the Add to Playlist picker filtered targets by isWritable only, offering synced playlists that then 409 on add. It now filters with canChangeTracks. - Jellyfin: addToPlaylist/removeFromPlaylist hard-coded every error to 404, so a locked playlist reported "not found" instead of 409. They now return 409 for ErrPlaylistNotEditable while keeping the deliberate anti-probing 404 for every other error (a non-owner never reaches ErrPlaylistNotEditable, so 409 leaks nothing). - OpenSubsonic: buildOSPlaylist marked only smart playlists readonly; owned synced playlists advertised readonly=false. Readonly now also covers !TracksEditable(), matching the existing smart-playlist treatment. * fix(jellyfin): report CanEdit from playlist editability in permission probes getPlaylistUsers and getPlaylistUser returned CanEdit: true unconditionally, so Finamp (which probes this before showing edit controls) offered track editing on synced/smart playlists whose add/remove requests now return 409. Both handlers now fetch the playlist and set CanEdit from TracksEditable(), keeping the deliberate non-owner looseness (CanEdit stays true for a normal playlist a non-owner views) and mapping any lookup error to 404 like the sibling probes. * fix(playlist): check ownership before editability when replacing tracks Create checked TracksEditable() before ownership, so a non-owner replacing another user's public smart/synced playlist (Jellyfin updatePlaylist with a non-empty Ids list) received a 409 read-only conflict instead of a 403 authorization failure. The incremental guards check ownership first via checkWritable; Create now matches that order. Subsonic is unaffected (both errors map to code 50). Owners of their own smart/synced playlists still get the read-only conflict. * fix(jellyfin): return 403 for locked playlists, matching Jellyfin Jellyfin itself refuses edits on its file-backed playlists with Forbid() (403): PlaylistsController gates every mutation on OwnerUserId == caller or a share with CanEdit, and playlists imported from .m3u files satisfy neither. Its CanEdit is an ACL field, not a read-only marker, and Jellyfin core has no server-managed playlist type at all. Our Jellyfin routes exist to imitate that API, so ErrPlaylistNotEditable now maps to 403 there instead of 409. The native API keeps 409 (a resource-state conflict is the accurate REST answer where we define the contract) and Subsonic keeps error 50, its closest code. * chore(playlist): trim comments added by this branch Several comments ran to three or four lines and carried rationale that belongs in the commit history rather than the code: what Jellyfin does with its own file-backed playlists, and restatements of the expressions directly below them. Each block is now one or two lines covering only the non-obvious why. --- core/playlists/playlists.go | 13 +++-- core/playlists/playlists_test.go | 46 +++++++++++++-- model/errors.go | 3 + model/playlist.go | 5 ++ model/playlist_test.go | 15 +++++ server/jellyfin/playlists.go | 49 ++++++++++++---- server/jellyfin/playlists_test.go | 63 ++++++++++++++++++++- server/nativeapi/playlists.go | 30 ++++++---- server/nativeapi/playlists_test.go | 14 +++++ server/subsonic/api.go | 3 +- server/subsonic/playlists.go | 2 +- server/subsonic/playlists_test.go | 9 +++ ui/src/common/playlistUtils.js | 2 +- ui/src/common/playlistUtils.test.js | 6 ++ ui/src/dialogs/SelectPlaylistInput.jsx | 5 +- ui/src/dialogs/SelectPlaylistInput.test.jsx | 9 +++ 16 files changed, 232 insertions(+), 42 deletions(-) diff --git a/core/playlists/playlists.go b/core/playlists/playlists.go index 656bde05e..9ae8f09cb 100644 --- a/core/playlists/playlists.go +++ b/core/playlists/playlists.go @@ -130,12 +130,13 @@ func (s *playlists) Create(ctx context.Context, playlistId string, name string, if err != nil { return err } - if pls.IsSmartPlaylist() { - return model.ErrNotAuthorized - } + // Ownership first: a non-owner must get ErrNotAuthorized, not a read-only conflict. if !usr.IsAdmin && pls.OwnerID != usr.ID { return model.ErrNotAuthorized } + if !pls.TracksEditable() { + return model.ErrPlaylistNotEditable + } } else { pls = &model.Playlist{Name: name} pls.OwnerID = usr.ID @@ -230,14 +231,14 @@ func (s *playlists) checkWritable(ctx context.Context, id string) (*model.Playli return pls, nil } -// checkTracksEditable verifies the user can modify tracks (ownership + not smart playlist). +// checkTracksEditable verifies the user owns the playlist and its tracks are editable. func (s *playlists) checkTracksEditable(ctx context.Context, playlistID string) (*model.Playlist, error) { pls, err := s.checkWritable(ctx, playlistID) if err != nil { return nil, err } - if pls.IsSmartPlaylist() { - return nil, model.ErrNotAuthorized + if !pls.TracksEditable() { + return nil, model.ErrPlaylistNotEditable } return pls, nil } diff --git a/core/playlists/playlists_test.go b/core/playlists/playlists_test.go index e8134b2ef..dd182b3f1 100644 --- a/core/playlists/playlists_test.go +++ b/core/playlists/playlists_test.go @@ -102,6 +102,8 @@ var _ = Describe("Playlists", func() { "pls-2": {ID: "pls-2", Name: "Other's", OwnerID: "other-user"}, "pls-smart": {ID: "pls-smart", Name: "Smart", OwnerID: "user-1", Rules: &criteria.Criteria{Expression: criteria.Contains{"title": "test"}}}, + "pls-synced": {ID: "pls-synced", Name: "Synced", OwnerID: "user-1", Sync: true}, + "pls-synced-other": {ID: "pls-synced-other", Name: "Other's Synced", OwnerID: "other-user", Sync: true, Public: true}, } ps = playlists.NewPlaylists(ds, artwork.NewUploader(ds)) }) @@ -145,6 +147,18 @@ var _ = Describe("Playlists", func() { It("denies replacing tracks on a smart playlist", func() { ctx = request.WithUser(ctx, model.User{ID: "user-1", IsAdmin: false}) _, err := ps.Create(ctx, "pls-smart", "", []string{"song-1"}) + Expect(err).To(MatchError(model.ErrPlaylistNotEditable)) + }) + + It("denies replacing tracks on a synced playlist", func() { + ctx = request.WithUser(ctx, model.User{ID: "user-1", IsAdmin: false}) + _, err := ps.Create(ctx, "pls-synced", "", []string{"song-1"}) + Expect(err).To(MatchError(model.ErrPlaylistNotEditable)) + }) + + It("denies a non-owner with authorization, not a conflict, on a public synced playlist", func() { + ctx = request.WithUser(ctx, model.User{ID: "user-1", IsAdmin: false}) + _, err := ps.Create(ctx, "pls-synced-other", "", []string{"song-1"}) Expect(err).To(MatchError(model.ErrNotAuthorized)) }) }) @@ -159,6 +173,7 @@ var _ = Describe("Playlists", func() { "pls-other": {ID: "pls-other", Name: "Other's", OwnerID: "other-user"}, "pls-smart": {ID: "pls-smart", Name: "Smart", OwnerID: "user-1", Rules: &criteria.Criteria{Expression: criteria.Contains{"title": "test"}}}, + "pls-synced": {ID: "pls-synced", Name: "Synced", OwnerID: "user-1", Sync: true}, } mockPlsRepo.TracksRepo = mockTracks ps = playlists.NewPlaylists(ds, artwork.NewUploader(ds)) @@ -191,13 +206,13 @@ var _ = Describe("Playlists", func() { It("denies adding tracks to a smart playlist", func() { ctx = request.WithUser(ctx, model.User{ID: "user-1", IsAdmin: false}) err := ps.Update(ctx, "pls-smart", nil, nil, nil, []string{"song-1"}, nil) - Expect(err).To(MatchError(model.ErrNotAuthorized)) + Expect(err).To(MatchError(model.ErrPlaylistNotEditable)) }) It("denies removing tracks from a smart playlist", func() { ctx = request.WithUser(ctx, model.User{ID: "user-1", IsAdmin: false}) err := ps.Update(ctx, "pls-smart", nil, nil, nil, nil, []int{0}) - Expect(err).To(MatchError(model.ErrNotAuthorized)) + Expect(err).To(MatchError(model.ErrPlaylistNotEditable)) }) It("allows metadata updates on a smart playlist", func() { @@ -205,6 +220,18 @@ var _ = Describe("Playlists", func() { err := ps.Update(ctx, "pls-smart", new("Updated Smart"), nil, nil, nil, nil) Expect(err).ToNot(HaveOccurred()) }) + + It("denies adding tracks to a synced playlist", func() { + ctx = request.WithUser(ctx, model.User{ID: "user-1", IsAdmin: false}) + err := ps.Update(ctx, "pls-synced", nil, nil, nil, []string{"song-1"}, nil) + Expect(err).To(MatchError(model.ErrPlaylistNotEditable)) + }) + + It("allows metadata updates on a synced playlist", func() { + ctx = request.WithUser(ctx, model.User{ID: "user-1", IsAdmin: false}) + err := ps.Update(ctx, "pls-synced", new("Renamed Synced"), nil, nil, nil, nil) + Expect(err).ToNot(HaveOccurred()) + }) }) Describe("AddTracks", func() { @@ -216,7 +243,8 @@ var _ = Describe("Playlists", func() { "pls-1": {ID: "pls-1", Name: "My Playlist", OwnerID: "user-1"}, "pls-smart": {ID: "pls-smart", Name: "Smart", OwnerID: "user-1", Rules: &criteria.Criteria{Expression: criteria.Contains{"title": "test"}}}, - "pls-other": {ID: "pls-other", Name: "Other's", OwnerID: "other-user"}, + "pls-other": {ID: "pls-other", Name: "Other's", OwnerID: "other-user"}, + "pls-synced": {ID: "pls-synced", Name: "Synced", OwnerID: "user-1", Sync: true}, } mockPlsRepo.TracksRepo = mockTracks ps = playlists.NewPlaylists(ds, artwork.NewUploader(ds)) @@ -246,7 +274,13 @@ var _ = Describe("Playlists", func() { It("denies editing smart playlists", func() { ctx = request.WithUser(ctx, model.User{ID: "user-1", IsAdmin: false}) _, err := ps.AddTracks(ctx, "pls-smart", []string{"song-1"}) - Expect(err).To(MatchError(model.ErrNotAuthorized)) + Expect(err).To(MatchError(model.ErrPlaylistNotEditable)) + }) + + It("denies editing synced playlists", func() { + ctx = request.WithUser(ctx, model.User{ID: "user-1", IsAdmin: false}) + _, err := ps.AddTracks(ctx, "pls-synced", []string{"song-1"}) + Expect(err).To(MatchError(model.ErrPlaylistNotEditable)) }) It("returns error when playlist not found", func() { @@ -280,7 +314,7 @@ var _ = Describe("Playlists", func() { It("denies on smart playlist", func() { ctx = request.WithUser(ctx, model.User{ID: "user-1", IsAdmin: false}) err := ps.RemoveTracks(ctx, "pls-smart", []string{"track-1"}) - Expect(err).To(MatchError(model.ErrNotAuthorized)) + Expect(err).To(MatchError(model.ErrPlaylistNotEditable)) }) It("denies non-owner", func() { @@ -314,7 +348,7 @@ var _ = Describe("Playlists", func() { It("denies on smart playlist", func() { ctx = request.WithUser(ctx, model.User{ID: "user-1", IsAdmin: false}) err := ps.ReorderTrack(ctx, "pls-smart", 1, 3) - Expect(err).To(MatchError(model.ErrNotAuthorized)) + Expect(err).To(MatchError(model.ErrPlaylistNotEditable)) }) }) diff --git a/model/errors.go b/model/errors.go index 41029d316..0e2543378 100644 --- a/model/errors.go +++ b/model/errors.go @@ -9,4 +9,7 @@ var ( ErrExpired = errors.New("access expired") ErrNotAvailable = errors.New("functionality not available") ErrValidation = errors.New("validation error") + + // ErrPlaylistNotEditable: tracks are server-managed, so nobody can edit them (not an ACL failure). + ErrPlaylistNotEditable = errors.New("playlist tracks are not editable") ) diff --git a/model/playlist.go b/model/playlist.go index 55b94a640..306401271 100644 --- a/model/playlist.go +++ b/model/playlist.go @@ -42,6 +42,11 @@ func (pls Playlist) IsSmartPlaylist() bool { return pls.Rules != nil && pls.Rules.Expression != nil } +// TracksEditable reports whether the track list is user-owned rather than server-managed. +func (pls Playlist) TracksEditable() bool { + return !pls.IsSmartPlaylist() && !pls.Sync +} + // RefreshDelay returns the playlist's own refresh window when set, falling // back to the global SmartPlaylistRefreshDelay. func (pls Playlist) RefreshDelay() time.Duration { diff --git a/model/playlist_test.go b/model/playlist_test.go index d936129ce..d98c85716 100644 --- a/model/playlist_test.go +++ b/model/playlist_test.go @@ -73,4 +73,19 @@ var _ = Describe("Playlist", func() { Expect(pls.RefreshDelay()).To(Equal(5 * time.Second)) }) }) + + Describe("TracksEditable", func() { + It("is true for a plain playlist", func() { + Expect(model.Playlist{}.TracksEditable()).To(BeTrue()) + }) + + It("is false for a smart playlist", func() { + pls := model.Playlist{Rules: &criteria.Criteria{Expression: criteria.Is{"loved": true}}} + Expect(pls.TracksEditable()).To(BeFalse()) + }) + + It("is false for a synced playlist", func() { + Expect(model.Playlist{Sync: true}.TracksEditable()).To(BeFalse()) + }) + }) }) diff --git a/server/jellyfin/playlists.go b/server/jellyfin/playlists.go index 805b369c8..ac790a821 100644 --- a/server/jellyfin/playlists.go +++ b/server/jellyfin/playlists.go @@ -29,11 +29,11 @@ func playlistsFolder() dto.BaseItemDto { } } -// playlistError maps core/playlists write errors to HTTP status: ownership -> 403, missing/invisible -// -> 404 (never revealing another user's private playlist), else -> 500. +// playlistError maps core/playlists write errors to HTTP status: ownership or locked -> 403, +// missing/invisible -> 404 (never revealing another user's private playlist), else -> 500. func (api *Router) playlistError(w http.ResponseWriter, r *http.Request, err error) { switch { - case errors.Is(err, model.ErrNotAuthorized): + case errors.Is(err, model.ErrNotAuthorized), errors.Is(err, model.ErrPlaylistNotEditable): http.Error(w, "Forbidden", http.StatusForbidden) case errors.Is(err, model.ErrNotFound): http.Error(w, "Not Found", http.StatusNotFound) @@ -262,7 +262,7 @@ func (api *Router) songIDs(ctx context.Context, opts model.QueryOptions) []strin } // addToPlaylist appends items by id, expanding containers into tracks (see expandContainerIDs). -// AddTracks enforces ownership; any error maps to 404. +// AddTracks enforces ownership; a locked playlist maps to 403, any other error to 404. func (api *Router) addToPlaylist(w http.ResponseWriter, r *http.Request) { ctx := r.Context() id, ok := itemIDParam(w, r, "playlistId") @@ -276,6 +276,10 @@ func (api *Router) addToPlaylist(w http.ResponseWriter, r *http.Request) { } ids := api.expandContainerIDs(ctx, decoded) if _, err := api.playlists.AddTracks(ctx, id, ids); err != nil { + if errors.Is(err, model.ErrPlaylistNotEditable) { + http.Error(w, "Forbidden", http.StatusForbidden) + return + } http.Error(w, "Not Found", http.StatusNotFound) return } @@ -284,7 +288,7 @@ func (api *Router) addToPlaylist(w http.ResponseWriter, r *http.Request) { // removeFromPlaylist removes entries by entryIds — playlist-entry ids (PlaylistItemId), not media // file ids, since RemoveTracks deletes playlist_tracks rows by that id. RemoveTracks enforces -// ownership; any error maps to 404. +// ownership; a locked playlist maps to 403, any other error to 404. func (api *Router) removeFromPlaylist(w http.ResponseWriter, r *http.Request) { ctx := r.Context() id, ok := itemIDParam(w, r, "playlistId") @@ -302,21 +306,44 @@ func (api *Router) removeFromPlaylist(w http.ResponseWriter, r *http.Request) { ids = append(ids, entry) } if err := api.playlists.RemoveTracks(ctx, id, ids); err != nil { + if errors.Is(err, model.ErrPlaylistNotEditable) { + http.Error(w, "Forbidden", http.StatusForbidden) + return + } http.Error(w, "Not Found", http.StatusNotFound) return } w.WriteHeader(http.StatusNoContent) } -// getPlaylistUsers and getPlaylistUser answer client probes (e.g. Finamp) made before allowing -// edits. Navidrome has no per-playlist ACL, so every user is reported CanEdit; ownership is still -// enforced by AddTracks/RemoveTracks. +// Clients probe these before offering edits. Navidrome has no per-playlist ACL, so CanEdit carries +// only editability; ownership is enforced on write, and a lookup error 404s to prevent probing. func (api *Router) getPlaylistUsers(w http.ResponseWriter, r *http.Request) { - u, _ := request.UserFrom(r.Context()) - api.ok(w, r, []dto.PlaylistUserPermissions{{UserId: dto.EncodeID(u.ID), CanEdit: true}}) + ctx := r.Context() + id, ok := itemIDParam(w, r, "playlistId") + if !ok { + return + } + pls, err := api.playlists.Get(ctx, id) + if err != nil { + http.Error(w, "Not Found", http.StatusNotFound) + return + } + u, _ := request.UserFrom(ctx) + api.ok(w, r, []dto.PlaylistUserPermissions{{UserId: dto.EncodeID(u.ID), CanEdit: pls.TracksEditable()}}) } func (api *Router) getPlaylistUser(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + id, ok := itemIDParam(w, r, "playlistId") + if !ok { + return + } + pls, err := api.playlists.Get(ctx, id) + if err != nil { + http.Error(w, "Not Found", http.StatusNotFound) + return + } userId := chi.URLParam(r, "userId") - api.ok(w, r, dto.PlaylistUserPermissions{UserId: userId, CanEdit: true}) + api.ok(w, r, dto.PlaylistUserPermissions{UserId: userId, CanEdit: pls.TracksEditable()}) } diff --git a/server/jellyfin/playlists_test.go b/server/jellyfin/playlists_test.go index 3599cc784..270f5fe08 100644 --- a/server/jellyfin/playlists_test.go +++ b/server/jellyfin/playlists_test.go @@ -381,6 +381,15 @@ var _ = Describe("Playlists", func() { Expect(w.Code).To(Equal(http.StatusNotFound)) }) + It("returns 403 when the playlist is not editable (synced/smart), like Jellyfin", func() { + fp.addErr = model.ErrPlaylistNotEditable + w := httptest.NewRecorder() + r := httptest.NewRequest("POST", "/Playlists/"+dto.EncodeID(testID("pl1"))+"/Items?ids="+dto.EncodeID(testID("s1")), nil).WithContext(context.Background()) + r = withChiURLParam(r, "playlistId", dto.EncodeID(testID("pl1"))) + invoke(api.addToPlaylist, w, r) + Expect(w.Code).To(Equal(http.StatusForbidden)) + }) + It("passes no ids (not a spurious empty string) when the ids param is absent", func() { w := httptest.NewRecorder() r := httptest.NewRequest("POST", "/Playlists/"+dto.EncodeID(testID("pl1"))+"/Items", nil).WithContext(context.Background()) @@ -430,6 +439,15 @@ var _ = Describe("Playlists", func() { Expect(w.Code).To(Equal(http.StatusNotFound)) }) + It("returns 403 when the playlist is not editable (synced/smart), like Jellyfin", func() { + fp.removeErr = model.ErrPlaylistNotEditable + w := httptest.NewRecorder() + r := httptest.NewRequest("DELETE", "/Playlists/"+dto.EncodeID(testID("pl1"))+"/Items?entryIds="+dto.EncodePlaylistEntryID("1"), nil).WithContext(context.Background()) + r = withChiURLParam(r, "playlistId", dto.EncodeID(testID("pl1"))) + invoke(api.removeFromPlaylist, w, r) + Expect(w.Code).To(Equal(http.StatusForbidden)) + }) + It("passes no ids (not a spurious empty string) when the entryIds param is absent", func() { w := httptest.NewRecorder() r := httptest.NewRequest("DELETE", "/Playlists/"+dto.EncodeID(testID("pl1"))+"/Items", nil).WithContext(context.Background()) @@ -442,7 +460,8 @@ var _ = Describe("Playlists", func() { }) Describe("getPlaylistUsers", func() { - It("returns the current user with CanEdit true", func() { + It("returns the current user with CanEdit true for an editable playlist", func() { + fp.getByIDPls = &model.Playlist{ID: testID("pl1")} w := httptest.NewRecorder() ctx := request.WithUser(context.Background(), model.User{ID: testID("u1"), UserName: "alice"}) r := httptest.NewRequest("GET", "/Playlists/"+testID("pl1")+"/Users", nil).WithContext(ctx) @@ -453,21 +472,59 @@ var _ = Describe("Playlists", func() { Expect(json.Unmarshal(w.Body.Bytes(), &res)).To(Succeed()) Expect(res).To(Equal([]dto.PlaylistUserPermissions{{UserId: dto.EncodeID(testID("u1")), CanEdit: true}})) }) + + It("reports CanEdit false for a synced playlist", func() { + fp.getByIDPls = &model.Playlist{ID: testID("pl1"), Sync: true} + w := httptest.NewRecorder() + ctx := request.WithUser(context.Background(), model.User{ID: testID("u1"), UserName: "alice"}) + r := httptest.NewRequest("GET", "/Playlists/"+testID("pl1")+"/Users", nil).WithContext(ctx) + r = withChiURLParam(r, "playlistId", dto.EncodeID(testID("pl1"))) + api.getPlaylistUsers(w, r) + Expect(w.Code).To(Equal(http.StatusOK)) + var res []dto.PlaylistUserPermissions + Expect(json.Unmarshal(w.Body.Bytes(), &res)).To(Succeed()) + Expect(res[0].CanEdit).To(BeFalse()) + }) + + It("returns 404 when the playlist is not visible", func() { + fp.getByIDErr = model.ErrNotFound + w := httptest.NewRecorder() + ctx := request.WithUser(context.Background(), model.User{ID: testID("u1"), UserName: "alice"}) + r := httptest.NewRequest("GET", "/Playlists/"+testID("pl1")+"/Users", nil).WithContext(ctx) + r = withChiURLParam(r, "playlistId", dto.EncodeID(testID("pl1"))) + api.getPlaylistUsers(w, r) + Expect(w.Code).To(Equal(http.StatusNotFound)) + }) }) Describe("getPlaylistUser", func() { - It("returns CanEdit true for the requested user", func() { + requestUser := func() *httptest.ResponseRecorder { w := httptest.NewRecorder() r := httptest.NewRequest("GET", "/Playlists/"+testID("pl1")+"/Users/"+testID("u1"), nil).WithContext(context.Background()) rctx := chi.NewRouteContext() - rctx.URLParams.Add("playlistId", testID("pl1")) + rctx.URLParams.Add("playlistId", dto.EncodeID(testID("pl1"))) rctx.URLParams.Add("userId", testID("u1")) r = r.WithContext(context.WithValue(r.Context(), chi.RouteCtxKey, rctx)) api.getPlaylistUser(w, r) + return w + } + + It("returns CanEdit true for an editable playlist", func() { + fp.getByIDPls = &model.Playlist{ID: testID("pl1")} + w := requestUser() Expect(w.Code).To(Equal(http.StatusOK)) var res dto.PlaylistUserPermissions Expect(json.Unmarshal(w.Body.Bytes(), &res)).To(Succeed()) Expect(res).To(Equal(dto.PlaylistUserPermissions{UserId: testID("u1"), CanEdit: true})) }) + + It("reports CanEdit false for a synced playlist", func() { + fp.getByIDPls = &model.Playlist{ID: testID("pl1"), Sync: true} + w := requestUser() + Expect(w.Code).To(Equal(http.StatusOK)) + var res dto.PlaylistUserPermissions + Expect(json.Unmarshal(w.Body.Bytes(), &res)).To(Succeed()) + Expect(res.CanEdit).To(BeFalse()) + }) }) }) diff --git a/server/nativeapi/playlists.go b/server/nativeapi/playlists.go index 90b2f9e94..d215eb9dd 100644 --- a/server/nativeapi/playlists.go +++ b/server/nativeapi/playlists.go @@ -20,6 +20,20 @@ import ( type restHandler = func(rest.RepositoryConstructor, ...rest.Logger) http.HandlerFunc +// writePlaylistError maps a playlist service error to an HTTP status, or defaultStatus if unknown. +func writePlaylistError(w http.ResponseWriter, err error, defaultStatus int) { + switch { + case errors.Is(err, model.ErrNotFound): + http.Error(w, err.Error(), http.StatusNotFound) + case errors.Is(err, model.ErrNotAuthorized): + http.Error(w, err.Error(), http.StatusForbidden) + case errors.Is(err, model.ErrPlaylistNotEditable): + http.Error(w, err.Error(), http.StatusConflict) + default: + http.Error(w, err.Error(), defaultStatus) + } +} + func playlistTracksHandler(pls playlists.Playlists, handler restHandler, refreshSmartPlaylist func(*http.Request) bool) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { plsId := chi.URLParam(r, "playlistId") @@ -111,7 +125,7 @@ func deleteFromPlaylist(pls playlists.Playlists) http.HandlerFunc { } if err != nil { log.Error(r.Context(), "Error deleting tracks from playlist", "playlistId", playlistId, "ids", ids, err) - http.Error(w, err.Error(), http.StatusInternalServerError) + writePlaylistError(w, err, http.StatusInternalServerError) return } writeDeleteManyResponse(w, r, ids) @@ -138,22 +152,22 @@ func addToPlaylist(pls playlists.Playlists) http.HandlerFunc { } count, c := 0, 0 if c, err = pls.AddTracks(ctx, playlistId, payload.Ids); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) + writePlaylistError(w, err, http.StatusBadRequest) return } count += c if c, err = pls.AddAlbums(ctx, playlistId, payload.AlbumIds); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) + writePlaylistError(w, err, http.StatusBadRequest) return } count += c if c, err = pls.AddArtists(ctx, playlistId, payload.ArtistIds); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) + writePlaylistError(w, err, http.StatusBadRequest) return } count += c if c, err = pls.AddDiscs(ctx, playlistId, payload.Discs); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) + writePlaylistError(w, err, http.StatusBadRequest) return } count += c @@ -192,12 +206,8 @@ func reorderItem(pls playlists.Playlists) http.HandlerFunc { return } err = pls.ReorderTrack(ctx, playlistId, id, newPos) - if errors.Is(err, model.ErrNotAuthorized) { - http.Error(w, err.Error(), http.StatusForbidden) - return - } if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) + writePlaylistError(w, err, http.StatusBadRequest) return } diff --git a/server/nativeapi/playlists_test.go b/server/nativeapi/playlists_test.go index 9bf502687..74ef58cab 100644 --- a/server/nativeapi/playlists_test.go +++ b/server/nativeapi/playlists_test.go @@ -183,6 +183,20 @@ var _ = Describe("Playlist Tracks Endpoint", func() { }) }) +var _ = Describe("writePlaylistError", func() { + DescribeTable("maps a service error to an HTTP status", + func(err error, expected int) { + w := httptest.NewRecorder() + writePlaylistError(w, err, http.StatusBadRequest) + Expect(w.Code).To(Equal(expected)) + }, + Entry("not found -> 404", model.ErrNotFound, http.StatusNotFound), + Entry("not authorized -> 403", model.ErrNotAuthorized, http.StatusForbidden), + Entry("not editable -> 409", model.ErrPlaylistNotEditable, http.StatusConflict), + Entry("unrecognized -> default", model.ErrValidation, http.StatusBadRequest), + ) +}) + type mockPlaylistTrackRepo struct { model.PlaylistTrackRepository tracks model.PlaylistTracks diff --git a/server/subsonic/api.go b/server/subsonic/api.go index 82e404228..029046c39 100644 --- a/server/subsonic/api.go +++ b/server/subsonic/api.go @@ -304,7 +304,8 @@ func mapToSubsonicError(err error) subError { err = newError(responses.ErrorGeneric, err.Error()) case errors.Is(err, model.ErrNotFound), errors.Is(err, rest.ErrNotFound): err = newError(responses.ErrorDataNotFound, "data not found") - case errors.Is(err, model.ErrNotAuthorized), errors.Is(err, rest.ErrPermissionDenied): + case errors.Is(err, model.ErrNotAuthorized), errors.Is(err, rest.ErrPermissionDenied), + errors.Is(err, model.ErrPlaylistNotEditable): // Subsonic has no code for "read-only resource" err = newError(responses.ErrorAuthorizationFail) case errors.Is(err, stream.ErrTooManyTranscodes): err = newError(responses.ErrorGeneric, "too many concurrent transcodes, please retry shortly") diff --git a/server/subsonic/playlists.go b/server/subsonic/playlists.go index bd53528d9..774a9c430 100644 --- a/server/subsonic/playlists.go +++ b/server/subsonic/playlists.go @@ -172,7 +172,7 @@ func buildOSPlaylist(ctx context.Context, p model.Playlist) *responses.OpenSubso } } else { user, ok := request.UserFrom(ctx) - pls.Readonly = !ok || p.OwnerID != user.ID + pls.Readonly = !ok || p.OwnerID != user.ID || !p.TracksEditable() } return &pls diff --git a/server/subsonic/playlists_test.go b/server/subsonic/playlists_test.go index a7c9e2ec7..f18f33b47 100644 --- a/server/subsonic/playlists_test.go +++ b/server/subsonic/playlists_test.go @@ -111,6 +111,15 @@ var _ = Describe("buildPlaylist", func() { Expect(result.Public).To(BeTrue()) Expect(result.Readonly).To(BeFalse()) }) + + It("is read-only for a synced playlist even as owner", func() { + ctx = request.WithUser(ctx, model.User{ID: "1234", UserName: "admin"}) + playlist.Sync = true + + result := router.buildPlaylist(ctx, playlist) + + Expect(result.Readonly).To(BeTrue()) + }) }) Context("when minimal clients list is empty", func() { diff --git a/ui/src/common/playlistUtils.js b/ui/src/common/playlistUtils.js index 74a01d47a..2d0c745ff 100644 --- a/ui/src/common/playlistUtils.js +++ b/ui/src/common/playlistUtils.js @@ -12,4 +12,4 @@ export const isReadOnly = (ownerId) => { export const isSmartPlaylist = (pls) => !!pls.rules export const canChangeTracks = (pls) => - isWritable(pls.ownerId) && !isSmartPlaylist(pls) + isWritable(pls.ownerId) && !isSmartPlaylist(pls) && !pls.sync diff --git a/ui/src/common/playlistUtils.test.js b/ui/src/common/playlistUtils.test.js index 2c671ecf5..345b6045d 100644 --- a/ui/src/common/playlistUtils.test.js +++ b/ui/src/common/playlistUtils.test.js @@ -74,5 +74,11 @@ describe('playlistUtils', () => { const playlist = { ownerId: 'user1', rules: [] } expect(canChangeTracks(playlist)).toBe(false) }) + + it('returns false if playlist is synced', () => { + localStorage.setItem('userId', 'user1') + const playlist = { ownerId: 'user1', sync: true } + expect(canChangeTracks(playlist)).toBe(false) + }) }) }) diff --git a/ui/src/dialogs/SelectPlaylistInput.jsx b/ui/src/dialogs/SelectPlaylistInput.jsx index 847107523..2f040a7f7 100644 --- a/ui/src/dialogs/SelectPlaylistInput.jsx +++ b/ui/src/dialogs/SelectPlaylistInput.jsx @@ -16,7 +16,7 @@ import { import AddIcon from '@material-ui/icons/Add' import { useGetList, useTranslate } from 'react-admin' import PropTypes from 'prop-types' -import { isWritable } from '../common' +import { canChangeTracks } from '../common' import { makeStyles } from '@material-ui/core' const useStyles = makeStyles((theme) => ({ @@ -268,8 +268,7 @@ export const SelectPlaylistInput = ({ onChange }) => { ) const options = - ids && - ids.map((id) => data[id]).filter((option) => isWritable(option.ownerId)) + ids && ids.map((id) => data[id]).filter((option) => canChangeTracks(option)) // Filter playlists based on search text const filteredOptions = diff --git a/ui/src/dialogs/SelectPlaylistInput.test.jsx b/ui/src/dialogs/SelectPlaylistInput.test.jsx index 4ffcdf0b6..753d12c03 100644 --- a/ui/src/dialogs/SelectPlaylistInput.test.jsx +++ b/ui/src/dialogs/SelectPlaylistInput.test.jsx @@ -16,6 +16,7 @@ const mockPlaylists = [ { id: 'playlist-2', name: 'Jazz Collection', ownerId: 'admin' }, { id: 'playlist-3', name: 'Electronic Beats', ownerId: 'admin' }, { id: 'playlist-4', name: 'Chill Vibes', ownerId: 'user2' }, // Not writable by admin + { id: 'playlist-5', name: 'Synced List', ownerId: 'admin', sync: true }, ] const mockIndexedData = { @@ -27,6 +28,12 @@ const mockIndexedData = { ownerId: 'admin', }, 'playlist-4': { id: 'playlist-4', name: 'Chill Vibes', ownerId: 'user2' }, + 'playlist-5': { + id: 'playlist-5', + name: 'Synced List', + ownerId: 'admin', + sync: true, + }, } const createTestComponent = ( @@ -89,6 +96,8 @@ describe('SelectPlaylistInput', () => { // Should not show playlists not owned by admin (not writable) expect(screen.queryByText('Chill Vibes')).not.toBeInTheDocument() + // Should not show synced playlists (their tracks are not editable) + expect(screen.queryByText('Synced List')).not.toBeInTheDocument() }) it('should filter playlists based on search input', async () => { From 2dab4b40483a6feb3be2007afa4a1ac83c2d0504 Mon Sep 17 00:00:00 2001 From: Deluan Date: Wed, 19 Aug 2026 08:49:15 -0400 Subject: [PATCH 17/27] chore: remove redundant comment Signed-off-by: Deluan --- model/errors.go | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/model/errors.go b/model/errors.go index 0e2543378..9e56f9501 100644 --- a/model/errors.go +++ b/model/errors.go @@ -3,13 +3,11 @@ package model import "errors" var ( - ErrNotFound = errors.New("data not found") - ErrInvalidAuth = errors.New("invalid authentication") - ErrNotAuthorized = errors.New("not authorized") - ErrExpired = errors.New("access expired") - ErrNotAvailable = errors.New("functionality not available") - ErrValidation = errors.New("validation error") - - // ErrPlaylistNotEditable: tracks are server-managed, so nobody can edit them (not an ACL failure). + ErrNotFound = errors.New("data not found") + ErrInvalidAuth = errors.New("invalid authentication") + ErrNotAuthorized = errors.New("not authorized") + ErrExpired = errors.New("access expired") + ErrNotAvailable = errors.New("functionality not available") + ErrValidation = errors.New("validation error") ErrPlaylistNotEditable = errors.New("playlist tracks are not editable") ) From 6d8a3e48eeacf4dda8edd6841f714f3ee064a371 Mon Sep 17 00:00:00 2001 From: Deluan Date: Wed, 19 Aug 2026 09:19:47 -0400 Subject: [PATCH 18/27] refactor: replace `md5` with `xxh3` for faster and more efficient hashing Signed-off-by: Deluan --- .golangci.yml | 3 +++ core/artwork/housekeeping.go | 6 ++---- core/artwork/housekeeping_test.go | 2 +- model/mediafile.go | 4 ++-- model/mediafile_test.go | 12 ++++-------- model/participants.go | 4 ++-- model/tag.go | 4 ++-- persistence/sql_base_repository.go | 6 +++--- 8 files changed, 19 insertions(+), 22 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 0804138dc..7df4f8ec3 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -27,6 +27,9 @@ linters: disable: - staticcheck settings: + errcheck: + exclude-functions: + - (*github.com/zeebo/xxh3.Hasher).Write gocritic: disable-all: true enabled-checks: diff --git a/core/artwork/housekeeping.go b/core/artwork/housekeeping.go index ae1fc0a0d..f2996d044 100644 --- a/core/artwork/housekeeping.go +++ b/core/artwork/housekeeping.go @@ -2,8 +2,6 @@ package artwork import ( "context" - "crypto/md5" - "encoding/hex" "fmt" "slices" "strconv" @@ -16,6 +14,7 @@ import ( "github.com/navidrome/navidrome/log" "github.com/navidrome/navidrome/model" "github.com/navidrome/navidrome/utils/slice" + "github.com/zeebo/xxh3" ) // StaleAbsentAge is how long an absent state is trusted before a recheck retries it. @@ -66,8 +65,7 @@ func FingerprintInputs() []FingerprintInput { func ConfigFingerprint() string { values := slice.Map(FingerprintInputs(), func(i FingerprintInput) string { return i.Value }) raw := fmt.Sprintf("%s|%d", strings.Join(values, "|"), artworkEpoch) - sum := md5.Sum([]byte(raw)) //nolint:gosec // fingerprint, not security-sensitive - return hex.EncodeToString(sum[:]) + return fmt.Sprintf("%016x", xxh3.Hash([]byte(raw))) } // backfill enqueues artwork resolution for every entity when the config fingerprint changed. diff --git a/core/artwork/housekeeping_test.go b/core/artwork/housekeeping_test.go index 32a7688b4..4ea15ab04 100644 --- a/core/artwork/housekeeping_test.go +++ b/core/artwork/housekeeping_test.go @@ -135,7 +135,7 @@ var _ = Describe("Housekeeping", func() { conf.Server.EnableExternalServices = true conf.Server.EnableM3UExternalAlbumArt = false - Expect(ConfigFingerprint()).To(Equal("7e537a22febc07d3d5ca40546e88da54")) + Expect(ConfigFingerprint()).To(Equal("7b538a83a870c16d")) }) It("reports the config inputs it hashes, so a change can be traced to a setting", func() { diff --git a/model/mediafile.go b/model/mediafile.go index f4e767272..99aee591e 100644 --- a/model/mediafile.go +++ b/model/mediafile.go @@ -2,7 +2,6 @@ package model import ( "cmp" - "crypto/md5" "encoding/json" "fmt" "iter" @@ -20,6 +19,7 @@ import ( "github.com/navidrome/navidrome/utils/gg" "github.com/navidrome/navidrome/utils/number" "github.com/navidrome/navidrome/utils/slice" + "github.com/zeebo/xxh3" ) type MediaFile struct { @@ -232,7 +232,7 @@ func (mf MediaFile) Hash() string { ZeroNil: true, } hash, _ := hashstructure.Hash(mf, opts) - sum := md5.New() + sum := xxh3.New() sum.Write(fmt.Appendf(nil, "%d", hash)) sum.Write(mf.Tags.Hash()) sum.Write(mf.Participants.Hash()) diff --git a/model/mediafile_test.go b/model/mediafile_test.go index 3f306f1a7..097e3ca54 100644 --- a/model/mediafile_test.go +++ b/model/mediafile_test.go @@ -715,14 +715,10 @@ var _ = Describe("MediaFile.Movements", func() { }) var _ = Describe("MediaFile.Hash", func() { - // Guards the upgrade guarantee: converting BPM/BitDepth from int to *int must not change hashes, - // or every file would be spuriously re-imported on the next scan. - // Golden hashes were captured at 46221d516 when those fields were plain ints. - It("keeps hashes identical to the pre-pointer-conversion values", func() { - // Golden hashes computed at 46221d516, when BPM/BitDepth were plain ints — pinning - // them guarantees the pointer conversion cannot trigger a full-library re-import. - Expect(MediaFile{Title: "Song"}.Hash()).To(Equal("1d856ced42cb96db39e354a4bac9a622")) - Expect(MediaFile{Title: "Song", BPM: new(120), BitDepth: new(16)}.Hash()).To(Equal("b2b0b1d1dd7fd767093588e4af3a0689")) + // Pins the hash formula: an accidental change spuriously re-imports every file on the next scan. + It("hashes to a stable value", func() { + Expect(MediaFile{Title: "Song"}.Hash()).To(Equal("05fdf70bb0cbe090")) + Expect(MediaFile{Title: "Song", BPM: new(120), BitDepth: new(16)}.Hash()).To(Equal("b5daf6ac1009a538")) }) It("changes the hash when a pointer field has a value", func() { base := MediaFile{Title: "Song"} diff --git a/model/participants.go b/model/participants.go index afbda10de..b5390d22c 100644 --- a/model/participants.go +++ b/model/participants.go @@ -2,12 +2,12 @@ package model import ( "cmp" - "crypto/md5" "fmt" "slices" "strings" "github.com/navidrome/navidrome/utils/slice" + "github.com/zeebo/xxh3" ) var ( @@ -193,7 +193,7 @@ func (p Participants) Hash() []byte { flattened = append(flattened, role.String()+":"+strings.Join(ids, "/")) } slices.Sort(flattened) - sum := md5.New() + sum := xxh3.New() sum.Write([]byte(strings.Join(flattened, "|"))) return sum.Sum(nil) } diff --git a/model/tag.go b/model/tag.go index 1bc011495..bb4fce181 100644 --- a/model/tag.go +++ b/model/tag.go @@ -2,13 +2,13 @@ package model import ( "cmp" - "crypto/md5" "fmt" "slices" "strings" "github.com/navidrome/navidrome/model/id" "github.com/navidrome/navidrome/utils/slice" + "github.com/zeebo/xxh3" ) type Tag struct { @@ -117,7 +117,7 @@ func (t Tags) Hash() []byte { } ids := t.IDs() slices.Sort(ids) - sum := md5.New() + sum := xxh3.New() sum.Write([]byte(strings.Join(ids, "|"))) return sum.Sum(nil) } diff --git a/persistence/sql_base_repository.go b/persistence/sql_base_repository.go index c61bca1a6..d4cf9b456 100644 --- a/persistence/sql_base_repository.go +++ b/persistence/sql_base_repository.go @@ -2,7 +2,6 @@ package persistence import ( "context" - "crypto/md5" "database/sql" "errors" "fmt" @@ -23,6 +22,7 @@ import ( "github.com/navidrome/navidrome/utils/hasher" "github.com/navidrome/navidrome/utils/slice" "github.com/pocketbase/dbx" + "github.com/zeebo/xxh3" ) // sqlRepository is the base repository for all SQL repositories. It provides common functions to interact with the DB. @@ -298,8 +298,8 @@ func (r sqlRepository) visibleLibraryIDs() ([]int, error) { func (r sqlRepository) seedKey() string { // Seed keys must be all lowercase, or else SQLite3 will encode it, making it not match the seed // used in the query. Hashing the user ID and converting it to a hex string will do the trick - userIDHash := md5.Sum([]byte(loggedUser(r.ctx).ID)) - return fmt.Sprintf("%s|%x", r.tableName, userIDHash) + userIDHash := xxh3.Hash([]byte(loggedUser(r.ctx).ID)) + return fmt.Sprintf("%s|%016x", r.tableName, userIDHash) } func (r sqlRepository) resetSeededRandom(options []model.QueryOptions) { From bd6b7a66863ce4771cd8e3450e355cf512b31363 Mon Sep 17 00:00:00 2001 From: Deluan Date: Wed, 19 Aug 2026 10:35:15 -0400 Subject: [PATCH 19/27] test: increase timeout for cache availability checks to 10 seconds --- core/artwork/e2e/acquire_serve_test.go | 2 +- core/artwork/e2e/resolution_harness_test.go | 2 +- core/stream/media_streamer_test.go | 9 +++++---- server/subsonic/e2e/subsonic_artwork_test.go | 2 +- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/core/artwork/e2e/acquire_serve_test.go b/core/artwork/e2e/acquire_serve_test.go index 0a9363b5d..34dfb2cac 100644 --- a/core/artwork/e2e/acquire_serve_test.go +++ b/core/artwork/e2e/acquire_serve_test.go @@ -106,7 +106,7 @@ var _ = Describe("Acquisition → serve loop", func() { func(context.Context, cache.Item) (io.Reader, error) { return nil, errors.New("resize not exercised in e2e") }) - Eventually(func() bool { return imgCache.Available(ctx) }).Should(BeTrue()) + Eventually(func() bool { return imgCache.Available(ctx) }, 10*time.Second).Should(BeTrue()) svc = artwork.NewArtwork(ds, imgCache, store, ffm) worker = artwork.NewWorker(ds, store, agents.GetAgents(ds, nil), ffm, events.NoopBroker(), imgCache) diff --git a/core/artwork/e2e/resolution_harness_test.go b/core/artwork/e2e/resolution_harness_test.go index 1168393df..fff62ce69 100644 --- a/core/artwork/e2e/resolution_harness_test.go +++ b/core/artwork/e2e/resolution_harness_test.go @@ -117,7 +117,7 @@ func setupResolutionHarness() { func(context.Context, cache.Item) (io.Reader, error) { return nil, fmt.Errorf("resize not exercised in e2e") }) - Eventually(func() bool { return imgCache.Available(rctx) }).Should(BeTrue()) + Eventually(func() bool { return imgCache.Available(rctx) }, 10*time.Second).Should(BeTrue()) rsvc = artwork.NewArtwork(rds, imgCache, rstore, ffm) rworker = artwork.NewWorker(rds, rstore, agents.GetAgents(rds, nil), ffm, events.NoopBroker(), imgCache) diff --git a/core/stream/media_streamer_test.go b/core/stream/media_streamer_test.go index f5ca16d3f..fb1c59a60 100644 --- a/core/stream/media_streamer_test.go +++ b/core/stream/media_streamer_test.go @@ -5,6 +5,7 @@ import ( "errors" "io" "os" + "time" "github.com/navidrome/navidrome/conf" "github.com/navidrome/navidrome/conf/configtest" @@ -33,7 +34,7 @@ var _ = Describe("MediaStreamer", func() { {ID: "123", Path: "tests/fixtures/test.mp3", Suffix: "mp3", BitRate: 128, Duration: 257.0}, }) testCache := stream.NewTranscodingCache() - Eventually(func() bool { return testCache.Available(context.TODO()) }).Should(BeTrue()) + Eventually(func() bool { return testCache.Available(context.TODO()) }, 10*time.Second).Should(BeTrue()) streamer = stream.NewMediaStreamer(ds, ffmpeg, testCache) }) AfterEach(func() { @@ -75,7 +76,7 @@ var _ = Describe("MediaStreamer", func() { conf.Server.Transcoding.MaxConcurrent = 1 conf.Server.Transcoding.MaxConcurrentPerUser = 0 tightCache := stream.NewTranscodingCache() - Eventually(func() bool { return tightCache.Available(context.TODO()) }).Should(BeTrue()) + Eventually(func() bool { return tightCache.Available(context.TODO()) }, 10*time.Second).Should(BeTrue()) tightStreamer := stream.NewMediaStreamer(ds, blockingFFmpeg, tightCache) userCtx := request.WithUsername(ctx, "alice") @@ -92,7 +93,7 @@ var _ = Describe("MediaStreamer", func() { conf.Server.Transcoding.MaxConcurrent = 1 conf.Server.Transcoding.MaxConcurrentPerUser = 0 tightCache := stream.NewTranscodingCache() - Eventually(func() bool { return tightCache.Available(context.TODO()) }).Should(BeTrue()) + Eventually(func() bool { return tightCache.Available(context.TODO()) }, 10*time.Second).Should(BeTrue()) tightStreamer := stream.NewMediaStreamer(ds, ffmpeg, tightCache) userCtx := request.WithUsername(ctx, "alice") @@ -112,7 +113,7 @@ var _ = Describe("MediaStreamer", func() { conf.Server.Transcoding.MaxConcurrent = 1 conf.Server.Transcoding.MaxConcurrentPerUser = 0 tightCache := stream.NewTranscodingCache() - Eventually(func() bool { return tightCache.Available(context.TODO()) }).Should(BeTrue()) + Eventually(func() bool { return tightCache.Available(context.TODO()) }, 10*time.Second).Should(BeTrue()) tightStreamer := stream.NewMediaStreamer(ds, ffmpeg, tightCache) userCtx := request.WithUsername(ctx, "alice") diff --git a/server/subsonic/e2e/subsonic_artwork_test.go b/server/subsonic/e2e/subsonic_artwork_test.go index 619619810..9324ea9e3 100644 --- a/server/subsonic/e2e/subsonic_artwork_test.go +++ b/server/subsonic/e2e/subsonic_artwork_test.go @@ -313,7 +313,7 @@ func newDummyImageCache(ctx context.Context) cache.FileCache { func(context.Context, cache.Item) (io.Reader, error) { return nil, errors.New("resize not exercised in subsonic artwork e2e") }) - Eventually(func() bool { return c.Available(ctx) }).Should(BeTrue()) + Eventually(func() bool { return c.Available(ctx) }, 10*time.Second).Should(BeTrue()) return c } From c362519f76d1922a6c3e1345e5ec91dada6275ed Mon Sep 17 00:00:00 2001 From: Junker der Provinz <133605895+junkerderprovinz@users.noreply.github.com> Date: Wed, 19 Aug 2026 17:50:32 +0200 Subject: [PATCH 20/27] test: unskip path-separator tests on Windows (#5381) (#5916) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test: unskip AbsolutePath and i18n path-separator tests on Windows (#5381) Signed-off-by: junkerderprovinz * test: unskip metadata folder-PID test on Windows via path.Dir (#5381) Signed-off-by: junkerderprovinz * test(storage): make relative-folder assertion cross-platform and unskip on Windows (#5381) Signed-off-by: junkerderprovinz * fix(persistence): normalize folder-update-info paths with forward slashes on Windows (#5381) Signed-off-by: junkerderprovinz * review: drop folder-PID change, trim storage_test comment (#5381) Revert model/metadata/persistent_ids.go to master: switching the `folder` PID attribute from filepath.Dir to path.Dir would change the persistent IDs of existing Windows libraries and needs a migration path, so it is out of scope for this PR. The matching test unskip is reverted with it, leaving #TBD-path-sep-metadata open in #5381. Trim the core/storage/storage_test.go comment to two lines. Signed-off-by: junkerderprovinz --------- Signed-off-by: junkerderprovinz Co-authored-by: Deluan Quintão --- core/common_test.go | 4 ++-- core/storage/storage_test.go | 10 ++++++---- persistence/folder_repository_test.go | 5 ----- server/nativeapi/translations_test.go | 5 ++--- 4 files changed, 10 insertions(+), 14 deletions(-) diff --git a/core/common_test.go b/core/common_test.go index 0d6e3a299..f57ea8d38 100644 --- a/core/common_test.go +++ b/core/common_test.go @@ -2,6 +2,7 @@ package core import ( "context" + "path/filepath" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -41,10 +42,9 @@ var _ = Describe("common.go", func() { }) It("returns the absolute path when library exists", func() { - tests.SkipOnWindows("path separator bug (#TBD-path-sep-core)") ctx := context.Background() abs := AbsolutePath(ctx, ds, libId, path) - Expect(abs).To(Equal("/library/root/music/file.mp3")) + Expect(abs).To(Equal(filepath.FromSlash("/library/root/music/file.mp3"))) }) It("returns the original path if library not found", func() { diff --git a/core/storage/storage_test.go b/core/storage/storage_test.go index 336b5a7a9..03904d086 100644 --- a/core/storage/storage_test.go +++ b/core/storage/storage_test.go @@ -7,7 +7,6 @@ import ( "runtime" "testing" - "github.com/navidrome/navidrome/tests" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) @@ -56,13 +55,16 @@ var _ = Describe("Storage", func() { Expect(s.(*fakeLocalStorage).u.Path).To(Equal("/tmp")) }) It("should return a file implementation for a relative folder", func() { - tests.SkipOnWindows("path separator bug (#TBD-path-sep-storage)") s, err := For("tmp") Expect(err).ToNot(HaveOccurred()) cwd, _ := os.Getwd() Expect(s).To(BeAssignableToTypeOf(&fakeLocalStorage{})) - Expect(s.(*fakeLocalStorage).u.Scheme).To(Equal("file")) - Expect(s.(*fakeLocalStorage).u.Path).To(Equal(filepath.Join(cwd, "tmp"))) + u := s.(*fakeLocalStorage).u + Expect(u.Scheme).To(Equal("file")) + // On Windows the drive letter lands in u.Host, so re-join it with + // u.Path (as newLocalStorage does) to keep the assertion OS-independent. + got := filepath.Join(u.Host, filepath.FromSlash(u.Path)) + Expect(got).To(Equal(filepath.Join(cwd, "tmp"))) }) It("should return error if schema is unregistered", func() { _, err := For("webdav:///tmp") diff --git a/persistence/folder_repository_test.go b/persistence/folder_repository_test.go index b429cecb9..545d8ad49 100644 --- a/persistence/folder_repository_test.go +++ b/persistence/folder_repository_test.go @@ -8,7 +8,6 @@ import ( "github.com/navidrome/navidrome/log" "github.com/navidrome/navidrome/model" "github.com/navidrome/navidrome/model/request" - "github.com/navidrome/navidrome/tests" "github.com/navidrome/navidrome/utils/slice" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -137,7 +136,6 @@ var _ = Describe("FolderRepository", func() { }) It("includes all child folders when querying parent", func() { - tests.SkipOnWindows("path storage (#TBD-path-sep-persistence)") // Create a parent folder with multiple children parent := model.NewFolder(testLib, "TestParent/Music") child1 := model.NewFolder(testLib, "TestParent/Music/Rock/Queen") @@ -159,7 +157,6 @@ var _ = Describe("FolderRepository", func() { }) It("excludes children from other libraries", func() { - tests.SkipOnWindows("path storage (#TBD-path-sep-persistence)") // Create parent in testLib parent := model.NewFolder(testLib, "TestIsolation/Parent") child := model.NewFolder(testLib, "TestIsolation/Parent/Child") @@ -185,7 +182,6 @@ var _ = Describe("FolderRepository", func() { }) It("excludes missing children when querying parent", func() { - tests.SkipOnWindows("path storage (#TBD-path-sep-persistence)") // Create parent and children, mark one as missing parent := model.NewFolder(testLib, "TestMissingChild/Parent") child1 := model.NewFolder(testLib, "TestMissingChild/Parent/Child1") @@ -206,7 +202,6 @@ var _ = Describe("FolderRepository", func() { }) It("handles mix of existing and non-existing target paths", func() { - tests.SkipOnWindows("path storage (#TBD-path-sep-persistence)") // Create folders for one path but not the other existingParent := model.NewFolder(testLib, "TestMixed/Exists") existingChild := model.NewFolder(testLib, "TestMixed/Exists/Child") diff --git a/server/nativeapi/translations_test.go b/server/nativeapi/translations_test.go index 77c088f6f..1c641da77 100644 --- a/server/nativeapi/translations_test.go +++ b/server/nativeapi/translations_test.go @@ -5,12 +5,12 @@ import ( "io" "io/fs" "os" + "path" "path/filepath" "testing/fstest" "github.com/navidrome/navidrome/consts" "github.com/navidrome/navidrome/resources" - "github.com/navidrome/navidrome/tests" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) @@ -18,13 +18,12 @@ import ( var _ = Describe("Translations", func() { Describe("I18n files", func() { It("contains only valid json language files", func() { - tests.SkipOnWindows("path separator bug (#TBD-path-sep-nativeapi)") fsys := resources.FS() dir, _ := fsys.Open(consts.I18nFolder) files, _ := dir.(fs.ReadDirFile).ReadDir(-1) for _, f := range files { name := filepath.Base(f.Name()) - filePath := filepath.Join(consts.I18nFolder, name) + filePath := path.Join(consts.I18nFolder, name) file, _ := fsys.Open(filePath) data, _ := io.ReadAll(file) var out map[string]any From dff9e47c2ea53e040ddac2fcc36f4563e53053be Mon Sep 17 00:00:00 2001 From: Junker der Provinz <133605895+junkerderprovinz@users.noreply.github.com> Date: Wed, 19 Aug 2026 19:11:00 +0200 Subject: [PATCH 21/27] fix(scanner): detect in-place playlist edits via the folder content hash (#5914) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(scanner): detect in-place playlist edits via the folder content hash Signed-off-by: junkerderprovinz * docs(scanner): clarify the playlist entries in the folder hash Shorten the comment on the playlist loop, and record why the playlist count stays in the hash header: it is redundant with the loop for change detection, but removing it changes the hashed byte stream for every folder, including folders without playlists, which would mark every folder outdated on the first scan after upgrade. Signed-off-by: junkerderprovinz * test(scanner): pin filename and size into the playlist hash assertions The playlist size test called time.Now() twice, so the modtime differed too and carried the assertion — dropping info.Size() from the hash left the suite green. It now shares one baseTime. A new rename test swaps the map key with count, size and modtime held constant, so dropping the filename from the hash fails. Both mutations were verified to fail before this change and pass after. --------- Signed-off-by: junkerderprovinz Co-authored-by: Deluan --- scanner/folder_entry.go | 22 +++++++--- scanner/folder_entry_test.go | 80 ++++++++++++++++++++++++++++++++---- scanner/phase_1_folders.go | 4 +- scanner/walk_dir_tree.go | 4 +- 4 files changed, 93 insertions(+), 17 deletions(-) diff --git a/scanner/folder_entry.go b/scanner/folder_entry.go index 0e893d6e6..e7eef223c 100644 --- a/scanner/folder_entry.go +++ b/scanner/folder_entry.go @@ -23,6 +23,7 @@ func newFolderEntry(job *scanJob, id, path string, info model.FolderUpdateInfo) path: path, audioFiles: make(map[string]fs.DirEntry), imageFiles: make(map[string]fs.DirEntry), + playlistFiles: make(map[string]fs.DirEntry), albumIDMap: make(map[string]string), updTime: info.UpdatedAt, prevHash: info.Hash, @@ -41,7 +42,7 @@ type folderEntry struct { updTime time.Time // from DB audioFiles map[string]fs.DirEntry imageFiles map[string]fs.DirEntry - numPlaylists int + playlistFiles map[string]fs.DirEntry numSubFolders int imagesUpdatedAt time.Time prevHash string // Previous hash from DB @@ -57,7 +58,7 @@ type folderEntry struct { } func (f *folderEntry) hasNoFiles() bool { - return len(f.audioFiles) == 0 && len(f.imageFiles) == 0 && f.numPlaylists == 0 + return len(f.audioFiles) == 0 && len(f.imageFiles) == 0 && len(f.playlistFiles) == 0 } func (f *folderEntry) isEmpty() bool { @@ -94,7 +95,7 @@ func (f *folderEntry) toFolder() *model.Folder { folder := model.NewFolder(f.job.lib, f.path) folder.NumAudioFiles = len(f.audioFiles) if playlists.InPath(*folder) { - folder.NumPlaylists = f.numPlaylists + folder.NumPlaylists = len(f.playlistFiles) } folder.ImageFiles = slices.Collect(maps.Keys(f.imageFiles)) folder.ImagesUpdatedAt = f.imagesUpdatedAt @@ -108,16 +109,18 @@ func (f *folderEntry) hash() string { h, "%s:%d:%d:%s", f.modTime.UTC(), - f.numPlaylists, + len(f.playlistFiles), // redundant with the loop below, but dropping it re-hashes every folder f.numSubFolders, f.imagesUpdatedAt.UTC(), ) - // Sort the keys of audio and image files to ensure consistent hashing + // Sort the keys of audio, image and playlist files to ensure consistent hashing audioKeys := slices.Collect(maps.Keys(f.audioFiles)) slices.Sort(audioKeys) imageKeys := slices.Collect(maps.Keys(f.imageFiles)) slices.Sort(imageKeys) + playlistKeys := slices.Collect(maps.Keys(f.playlistFiles)) + slices.Sort(playlistKeys) // Include audio files with their size and modtime for _, key := range audioKeys { @@ -135,5 +138,14 @@ func (f *folderEntry) hash() string { } } + // Include playlist files, so a content edit is detected even when the folder's + // mtime is preserved (rsync -a) or the playlist is not the newest file. + for _, key := range playlistKeys { + _, _ = io.WriteString(h, key) + if info, err := f.playlistFiles[key].Info(); err == nil { + _, _ = fmt.Fprintf(h, ":%d:%s", info.Size(), info.ModTime().UTC().String()) + } + } + return hex.EncodeToString(h.Sum(nil)) } diff --git a/scanner/folder_entry_test.go b/scanner/folder_entry_test.go index e8e354b38..4493f9309 100644 --- a/scanner/folder_entry_test.go +++ b/scanner/folder_entry_test.go @@ -48,6 +48,7 @@ var _ = Describe("folder_entry", func() { Expect(entry.path).To(Equal(path)) Expect(entry.audioFiles).To(BeEmpty()) Expect(entry.imageFiles).To(BeEmpty()) + Expect(entry.playlistFiles).To(BeEmpty()) Expect(entry.albumIDMap).To(BeEmpty()) Expect(entry.updTime).To(Equal(updateInfo.UpdatedAt)) Expect(entry.prevHash).To(Equal(updateInfo.Hash)) @@ -95,7 +96,7 @@ var _ = Describe("folder_entry", func() { }) It("returns false when folder has playlists", func() { - entry.numPlaylists = 1 + entry.playlistFiles["list.m3u"] = &fakeDirEntry{name: "list.m3u"} Expect(entry.hasNoFiles()).To(BeFalse()) }) @@ -107,7 +108,7 @@ var _ = Describe("folder_entry", func() { It("returns false when folder has multiple types of content", func() { entry.audioFiles["test.mp3"] = &fakeDirEntry{name: "test.mp3"} entry.imageFiles["cover.jpg"] = &fakeDirEntry{name: "cover.jpg"} - entry.numPlaylists = 2 + entry.playlistFiles["list.m3u"] = &fakeDirEntry{name: "list.m3u"} entry.numSubFolders = 3 Expect(entry.hasNoFiles()).To(BeFalse()) }) @@ -149,7 +150,11 @@ var _ = Describe("folder_entry", func() { "cover.jpg": &fakeDirEntry{name: "cover.jpg"}, "folder.png": &fakeDirEntry{name: "folder.png"}, } - entry.numPlaylists = 3 + entry.playlistFiles = map[string]fs.DirEntry{ + "list1.m3u": &fakeDirEntry{name: "list1.m3u"}, + "list2.m3u": &fakeDirEntry{name: "list2.m3u"}, + "list3.m3u": &fakeDirEntry{name: "list3.m3u"}, + } entry.imagesUpdatedAt = time.Now() }) @@ -200,7 +205,10 @@ var _ = Describe("folder_entry", func() { "z.jpg": &fakeDirEntry{name: "z.jpg"}, "x.png": &fakeDirEntry{name: "x.png"}, } - entry.numPlaylists = 2 + entry.playlistFiles = map[string]fs.DirEntry{ + "q.m3u": &fakeDirEntry{name: "q.m3u"}, + "p.m3u": &fakeDirEntry{name: "p.m3u"}, + } entry.numSubFolders = 3 hash1 := entry.hash() @@ -214,6 +222,10 @@ var _ = Describe("folder_entry", func() { "x.png": &fakeDirEntry{name: "x.png"}, "z.jpg": &fakeDirEntry{name: "z.jpg"}, } + entry.playlistFiles = map[string]fs.DirEntry{ + "p.m3u": &fakeDirEntry{name: "p.m3u"}, + "q.m3u": &fakeDirEntry{name: "q.m3u"}, + } hash2 := entry.hash() Expect(hash1).To(Equal(hash2)) @@ -252,10 +264,10 @@ var _ = Describe("folder_entry", func() { Expect(hash1).ToNot(Equal(hash2)) }) - It("produces different hash when playlist count changes", func() { + It("produces different hash when playlist files change", func() { hash1 := entry.hash() - entry.numPlaylists = 5 + entry.playlistFiles["new.m3u"] = &fakeDirEntry{name: "new.m3u"} hash2 := entry.hash() Expect(hash1).ToNot(Equal(hash2)) @@ -377,6 +389,58 @@ var _ = Describe("folder_entry", func() { Expect(hash1).ToNot(Equal(hash2)) }) + It("produces different hash when playlist file size changes", func() { + baseTime := time.Now() + entry.playlistFiles["list.m3u"] = &fakeDirEntry{ + name: "list.m3u", + fileInfo: &fakeFileInfo{name: "list.m3u", size: 1000, modTime: baseTime}, + } + hash1 := entry.hash() + + entry.playlistFiles["list.m3u"] = &fakeDirEntry{ + name: "list.m3u", + fileInfo: &fakeFileInfo{name: "list.m3u", size: 2000, modTime: baseTime}, + } + hash2 := entry.hash() + + Expect(hash1).ToNot(Equal(hash2)) + }) + + It("produces different hash when playlist file modification time changes", func() { + baseTime := time.Now() + entry.playlistFiles["list.m3u"] = &fakeDirEntry{ + name: "list.m3u", + fileInfo: &fakeFileInfo{name: "list.m3u", size: 1000, modTime: baseTime}, + } + hash1 := entry.hash() + + entry.playlistFiles["list.m3u"] = &fakeDirEntry{ + name: "list.m3u", + fileInfo: &fakeFileInfo{name: "list.m3u", size: 1000, modTime: baseTime.Add(1 * time.Hour)}, + } + hash2 := entry.hash() + + Expect(hash1).ToNot(Equal(hash2)) + }) + + It("produces different hash when a playlist is renamed", func() { + baseTime := time.Now() + entry.playlistFiles["old.m3u"] = &fakeDirEntry{ + name: "old.m3u", + fileInfo: &fakeFileInfo{name: "old.m3u", size: 1000, modTime: baseTime}, + } + hash1 := entry.hash() + + delete(entry.playlistFiles, "old.m3u") + entry.playlistFiles["new.m3u"] = &fakeDirEntry{ + name: "new.m3u", + fileInfo: &fakeFileInfo{name: "new.m3u", size: 1000, modTime: baseTime}, + } + hash2 := entry.hash() + + Expect(hash1).ToNot(Equal(hash2)) + }) + It("produces valid hex-encoded hash", func() { hash := entry.hash() Expect(hash).To(HaveLen(32)) // MD5 hash should be 32 hex characters @@ -421,7 +485,7 @@ var _ = Describe("folder_entry", func() { }) It("returns true when hash has changed", func() { - entry.numPlaylists = 10 // Change something to change the hash + entry.playlistFiles["list.m3u"] = &fakeDirEntry{name: "list.m3u"} // Change something to change the hash Expect(entry.isOutdated()).To(BeTrue()) }) @@ -445,7 +509,7 @@ var _ = Describe("folder_entry", func() { It("returns true when full scan condition is not met but hash changed", func() { entry.updTime = entry.job.lib.LastScanStartedAt.Add(1 * time.Hour) - entry.numPlaylists = 10 // Change hash + entry.playlistFiles["list.m3u"] = &fakeDirEntry{name: "list.m3u"} // Change hash Expect(entry.isOutdated()).To(BeTrue()) }) }) diff --git a/scanner/phase_1_folders.go b/scanner/phase_1_folders.go index 82e91d5ad..6107b3316 100644 --- a/scanner/phase_1_folders.go +++ b/scanner/phase_1_folders.go @@ -165,7 +165,7 @@ func (p *phaseFolders) producer() ppl.Producer[*folderEntry] { log.Trace(p.ctx, "Scanner: Checking folder state", " folder", folder.path, "_updTime", folder.updTime, "_modTime", folder.modTime, "_lastScanStartedAt", folder.job.lib.LastScanStartedAt, "numAudioFiles", len(folder.audioFiles), "numImageFiles", len(folder.imageFiles), - "numPlaylists", folder.numPlaylists, "numSubfolders", folder.numSubFolders) + "numPlaylists", len(folder.playlistFiles), "numSubfolders", folder.numSubFolders) // Check if folder is outdated if folder.isOutdated() { @@ -492,7 +492,7 @@ func (p *phaseFolders) logFolder(entry *folderEntry) (*folderEntry, error) { logCall = log.Trace } logCall(p.ctx, "Scanner: Completed processing folder", - "audioCount", len(entry.audioFiles), "imageCount", len(entry.imageFiles), "plsCount", entry.numPlaylists, + "audioCount", len(entry.audioFiles), "imageCount", len(entry.imageFiles), "plsCount", len(entry.playlistFiles), "elapsed", entry.elapsed.Elapsed(), "tracksMissing", len(entry.missingTracks), "tracksImported", len(entry.tracks), "library", entry.job.lib.Name, consts.Zwsp+"folder", entry.path) return entry, nil diff --git a/scanner/walk_dir_tree.go b/scanner/walk_dir_tree.go index 887344b1b..20f6d4213 100644 --- a/scanner/walk_dir_tree.go +++ b/scanner/walk_dir_tree.go @@ -82,7 +82,7 @@ func walkFolder(ctx context.Context, job *scanJob, currentFolder string, checker dir := path.Clean(currentFolder) log.Trace(ctx, "Scanner: Found directory", " path", dir, "audioFiles", maps.Keys(folder.audioFiles), - "images", maps.Keys(folder.imageFiles), "playlists", folder.numPlaylists, "imagesUpdatedAt", folder.imagesUpdatedAt, + "images", maps.Keys(folder.imageFiles), "playlists", len(folder.playlistFiles), "imagesUpdatedAt", folder.imagesUpdatedAt, "updTime", folder.updTime, "modTime", folder.modTime, "numChildren", len(children)) folder.path = dir folder.elapsed.Start() @@ -157,7 +157,7 @@ func loadDir(ctx context.Context, job *scanJob, dirPath string, checker *IgnoreC case model.IsAudioFile(name): folder.audioFiles[entry.Name()] = entry case model.IsValidPlaylist(name): - folder.numPlaylists++ + folder.playlistFiles[entry.Name()] = entry case model.IsImageFile(name): folder.imageFiles[entry.Name()] = entry folder.imagesUpdatedAt = utils.TimeNewest(folder.imagesUpdatedAt, fileInfo.ModTime(), folder.modTime) From fd4b3256e4c3e2c6eeeb90c2e3034149b536f873 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Deluan=20Quint=C3=A3o?= Date: Wed, 19 Aug 2026 16:20:38 -0400 Subject: [PATCH 22/27] perf(artwork): compute the blurhash DCT separably (#5989) The cosine basis factors into cosX[i][x] * cosY[j][y], so the pixel loop does not need to visit every (i,j) pair. Each row now collapses to xComp dot products, folded over yComp once per row: w*h*xComp + h*xComp*yComp multiply-accumulates instead of w*h*xComp*yComp. Encoding is ~60% faster at every input size, and ~80% faster at the 128px size the artwork pipeline actually feeds it (263us -> 53us). Hashes are byte-identical, so the existing golden-value specs cover the rewrite. --- core/artwork/blurhash/blurhash.go | 35 +++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/core/artwork/blurhash/blurhash.go b/core/artwork/blurhash/blurhash.go index ca5bf7026..29e1ccfda 100644 --- a/core/artwork/blurhash/blurhash.go +++ b/core/artwork/blurhash/blurhash.go @@ -52,6 +52,12 @@ func Encode(img image.Image) (string, error) { lin := srgbToLinearTable() factors := make([][3]float64, xComp*yComp) + linR := make([]float64, w) + linG := make([]float64, w) + linB := make([]float64, w) + rowR := make([]float64, xComp) + rowG := make([]float64, xComp) + rowB := make([]float64, xComp) for y := range h { row := src.pix[y*src.stride:] for x := range w { @@ -60,15 +66,26 @@ func Encode(img image.Image) (string, error) { if src.straight { r, g, b = premultiply(r, g, b, row[p+3]) } - lr, lg, lb := lin[r], lin[g], lin[b] - for j := range yComp { - for i := range xComp { - basis := cosX[i][x] * cosY[j][y] - f := &factors[j*xComp+i] - f[0] += basis * lr - f[1] += basis * lg - f[2] += basis * lb - } + linR[x], linG[x], linB[x] = lin[r], lin[g], lin[b] + } + // The basis is separable, so a row costs xComp dot products plus one fold over yComp, + // rather than xComp*yComp multiply-accumulates per pixel. + for i := range xComp { + var sr, sg, sb float64 + for x, c := range cosX[i] { + sr += c * linR[x] + sg += c * linG[x] + sb += c * linB[x] + } + rowR[i], rowG[i], rowB[i] = sr, sg, sb + } + for j := range yComp { + cy := cosY[j][y] + for i := range xComp { + f := &factors[j*xComp+i] + f[0] += cy * rowR[i] + f[1] += cy * rowG[i] + f[2] += cy * rowB[i] } } } From 881073c1833e66c47839927dc3c1577e39471c25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Deluan=20Quint=C3=A3o?= Date: Wed, 19 Aug 2026 16:25:12 -0400 Subject: [PATCH 23/27] feat(cli): let artwork explain/refresh accept an id without its kind (#5988) * feat(cmd): make artwork explain/refresh accept an id without its kind The kind can now come from the id itself: a full artwork id (al-) carries it in the prefix, and a bare id is resolved across tables via GetEntityByID. The explicit leader still works. * fix(cmd): keep refreshing resolvable ids when others fail to resolve resolveArtworkTargets now collects a self-describing id it cannot resolve as a failure instead of aborting, so refresh reports and skips the bad ones and still queues the rest, matching refreshItems per-item behavior. explain stays strict and rejects any unresolved input. --- cmd/artwork.go | 110 +++++++++++++++++++++++++++++++-------- cmd/artwork_test.go | 83 ++++++++++++++++++++++++++--- model/get_entity.go | 37 +++++++++---- model/get_entity_test.go | 22 ++++++++ 4 files changed, 212 insertions(+), 40 deletions(-) diff --git a/cmd/artwork.go b/cmd/artwork.go index e9c910f11..aeaec0e43 100644 --- a/cmd/artwork.go +++ b/cmd/artwork.go @@ -59,31 +59,27 @@ var artworkCmd = &cobra.Command{ } var artworkExplainCmd = &cobra.Command{ - Use: "explain ", + Use: "explain [] ", Short: "Explain why an item's artwork resolved the way it did", Long: "Explain why an item's artwork resolved the way it did.\n\n" + + "The item can be given as a bare id, a full artwork id (e.g. al-), or a pair.\n" + " is one of: " + kindPrefixes(explainKinds) + ".\n" + "A disc artwork id is the album id and the disc number, joined by a colon: :2", - Args: cobra.ExactArgs(2), + Args: cobra.RangeArgs(1, 2), Run: func(cmd *cobra.Command, args []string) { - kind, err := parseArtworkKind(args[0], explainKinds) - if err != nil { - log.Fatal(cmd.Context(), err) - } - runExplain(cmd.Context(), kind, args[1]) + runExplain(cmd.Context(), args) }, } var artworkRefreshCmd = &cobra.Command{ - Use: "refresh ...", + Use: "refresh [] ...", Short: "Clear an item's artwork state and re-resolve it", - Args: cobra.MinimumNArgs(2), + Long: "Clear an item's artwork state and re-resolve it.\n\n" + + "Each item can be given as a bare id, a full artwork id (e.g. al-), or a shared\n" + + " ... leader. is one of: " + kindPrefixes(artwork.RefreshableKinds) + ".", + Args: cobra.MinimumNArgs(1), Run: func(cmd *cobra.Command, args []string) { - kind, err := parseArtworkKind(args[0], artwork.RefreshableKinds) - if err != nil { - log.Fatal(cmd.Context(), err) - } - runRefresh(cmd.Context(), kind, args[1:]) + runRefresh(cmd.Context(), args) }, } @@ -499,19 +495,28 @@ func printReprocessPreview(out io.Writer, kinds []model.Kind, matched []int64, t } } -func runRefresh(ctx context.Context, kind model.Kind, ids []string) { +func runRefresh(ctx context.Context, args []string) { defer db.Init(ctx)() ds, ctx := getAdminContext(ctx) - if failed := refreshItems(ctx, ds, kind, ids, os.Stdout); failed > 0 { - log.Fatal(ctx, "Failed to refresh artwork", "kind", kind, "failed", failed, "total", len(ids)) + targets, failures, err := resolveArtworkTargets(ctx, ds, args, artwork.RefreshableKinds) + if err != nil { + log.Fatal(ctx, err) + } + for _, f := range failures { + log.Error(ctx, "Skipping unresolved item", f) + } + failed := refreshItems(ctx, ds, targets, os.Stdout) + len(failures) + if failed > 0 { + log.Fatal(ctx, "Failed to refresh artwork", "failed", failed, "total", len(targets)+len(failures)) } } -// refreshItems keeps going after a failure — the ids are independent — and returns how many failed. -func refreshItems(ctx context.Context, ds model.DataStore, kind model.Kind, ids []string, out io.Writer) int { +// refreshItems keeps going after a failure — the items are independent — and returns how many failed. +func refreshItems(ctx context.Context, ds model.DataStore, targets []model.ArtworkID, out io.Writer) int { var failed int - for _, id := range ids { + for _, t := range targets { + kind, id := t.Kind, t.ID // artwork.Refresh would happily queue an id that does not exist, orphaning a queue row. if _, err := artworkItemName(ctx, ds, kind, id); err != nil { log.Error(ctx, "Item not found", "kind", kind, "id", id, err) @@ -544,7 +549,56 @@ func parseArtworkKind(s string, valid []model.Kind) (model.Kind, error) { if ok && slices.Contains(valid, kind) { return kind, nil } - return kind, fmt.Errorf("invalid kind %q, expected one of: %s", s, kindPrefixes(valid)) + return kind, invalidKindErr(s, valid) +} + +func invalidKindErr(s string, valid []model.Kind) error { + return fmt.Errorf("invalid kind %q, expected one of: %s", s, kindPrefixes(valid)) +} + +// resolveArtworkTargets resolves explain/refresh positional args into artwork ids, accepting a +// shared " ..." leader or self-describing args (a bare id, or a full artwork id). A +// self-describing arg that cannot be resolved is returned as a failure rather than aborting the +// batch, so refresh can process the resolvable ids; a malformed leader is a usage error. +func resolveArtworkTargets(ctx context.Context, ds model.DataStore, args []string, valid []model.Kind) ([]model.ArtworkID, []error, error) { + if kind, ok := model.ParseKind(args[0]); ok && len(args) > 1 { + if !slices.Contains(valid, kind) { + return nil, nil, invalidKindErr(args[0], valid) + } + return slice.Map(args[1:], func(id string) model.ArtworkID { + return model.ArtworkID{Kind: kind, ID: id} + }), nil, nil + } + var targets []model.ArtworkID + var failures []error + for _, arg := range args { + target, err := artworkKindAndID(ctx, ds, arg) + if err == nil && !slices.Contains(valid, target.Kind) { + err = invalidKindErr(target.Kind.Prefix(), valid) + } + if err != nil { + failures = append(failures, err) + continue + } + targets = append(targets, target) + } + return targets, failures, nil +} + +// artworkKindAndID resolves one self-describing argument: a full artwork id (al-) takes its kind +// from the prefix, a bare id is looked up. Entity ids never start with "-", so no collision. +func artworkKindAndID(ctx context.Context, ds model.DataStore, arg string) (model.ArtworkID, error) { + if artID, err := model.ParseArtworkID(arg); err == nil && artID.ID != "" { + return model.ArtworkID{Kind: artID.Kind, ID: artID.ID}, nil + } + kind, err := model.GetEntityKindByID(ctx, ds, arg) + if errors.Is(err, model.ErrNotFound) { + return model.ArtworkID{}, fmt.Errorf("could not determine kind for %q; pass an explicit ", arg) + } + if err != nil { + return model.ArtworkID{}, err + } + return model.ArtworkID{Kind: kind, ID: arg}, nil } // explainAgents accounts for every configured agent: one the CLI cannot construct (a plugin, or a @@ -721,10 +775,22 @@ func formatTime(t time.Time) string { return t.Format(time.RFC3339) } -func runExplain(ctx context.Context, kind model.Kind, id string) { +func runExplain(ctx context.Context, args []string) { defer db.Init(ctx)() ds, ctx := getAdminContext(ctx) + targets, failures, err := resolveArtworkTargets(ctx, ds, args, explainKinds) + if err != nil { + log.Fatal(ctx, err) + } + if len(failures) > 0 { + log.Fatal(ctx, failures[0]) + } + if len(targets) != 1 { + log.Fatal(ctx, "explain takes a single item; pass one id or a pair") + } + kind, id := targets[0].Kind, targets[0].ID + name, err := artworkItemName(ctx, ds, kind, id) if err != nil { log.Fatal(ctx, "Item not found", "kind", kind, "id", id, err) diff --git a/cmd/artwork_test.go b/cmd/artwork_test.go index b28d8dac5..8b50ba775 100644 --- a/cmd/artwork_test.go +++ b/cmd/artwork_test.go @@ -54,6 +54,74 @@ var _ = Describe("parseArtworkKind", func() { }) }) +var _ = Describe("resolveArtworkTargets", func() { + var ds *tests.MockDataStore + ctx := context.Background() + + BeforeEach(func() { + artists := tests.CreateMockArtistRepo() + artists.SetData(model.Artists{{ID: "artist1"}}) + ds = &tests.MockDataStore{MockedArtist: artists} + }) + + It("accepts the explicit leader shared by every id", func() { + targets, failures, err := resolveArtworkTargets(ctx, ds, []string{"al", "x", "y"}, explainKinds) + Expect(err).ToNot(HaveOccurred()) + Expect(failures).To(BeEmpty()) + Expect(targets).To(Equal([]model.ArtworkID{ + {Kind: model.KindAlbumArtwork, ID: "x"}, {Kind: model.KindAlbumArtwork, ID: "y"}})) + }) + + It("rejects an explicit kind the command does not accept as a usage error", func() { + _, _, err := resolveArtworkTargets(ctx, ds, []string{"dc", "x"}, artwork.RefreshableKinds) + Expect(err).To(MatchError(ContainSubstring("invalid kind"))) + }) + + It("resolves a bare id by looking it up across tables", func() { + targets, failures, err := resolveArtworkTargets(ctx, ds, []string{"artist1"}, explainKinds) + Expect(err).ToNot(HaveOccurred()) + Expect(failures).To(BeEmpty()) + Expect(targets).To(Equal([]model.ArtworkID{{Kind: model.KindArtistArtwork, ID: "artist1"}})) + }) + + It("reads the kind from a full artwork id prefix without a database lookup", func() { + targets, _, err := resolveArtworkTargets(ctx, ds, []string{"al-realalbum"}, explainKinds) + Expect(err).ToNot(HaveOccurred()) + Expect(targets).To(Equal([]model.ArtworkID{{Kind: model.KindAlbumArtwork, ID: "realalbum"}})) + }) + + It("strips the hash suffix from a full artwork id", func() { + targets, _, err := resolveArtworkTargets(ctx, ds, []string{"al-realalbum_0123456789abcdef"}, explainKinds) + Expect(err).ToNot(HaveOccurred()) + Expect(targets).To(Equal([]model.ArtworkID{{Kind: model.KindAlbumArtwork, ID: "realalbum"}})) + }) + + It("collects a self-describing arg whose kind the command does not accept", func() { + targets, failures, err := resolveArtworkTargets(ctx, ds, []string{"dc-realalbum:2"}, artwork.RefreshableKinds) + Expect(err).ToNot(HaveOccurred()) + Expect(targets).To(BeEmpty()) + Expect(failures).To(HaveLen(1)) + Expect(failures[0]).To(MatchError(ContainSubstring("invalid kind"))) + }) + + It("collects an id that matches nothing and has no kind prefix", func() { + targets, failures, err := resolveArtworkTargets(ctx, ds, []string{"nope"}, explainKinds) + Expect(err).ToNot(HaveOccurred()) + Expect(targets).To(BeEmpty()) + Expect(failures).To(HaveLen(1)) + Expect(failures[0]).To(MatchError(ContainSubstring("could not determine kind"))) + }) + + It("resolves the valid ids and collects the unresolvable ones", func() { + targets, failures, err := resolveArtworkTargets(ctx, ds, []string{"artist1", "nope", "al-realalbum"}, explainKinds) + Expect(err).ToNot(HaveOccurred()) + Expect(targets).To(Equal([]model.ArtworkID{ + {Kind: model.KindArtistArtwork, ID: "artist1"}, {Kind: model.KindAlbumArtwork, ID: "realalbum"}})) + Expect(failures).To(HaveLen(1)) + Expect(failures[0]).To(MatchError(ContainSubstring("could not determine kind"))) + }) +}) + var _ = Describe("explainResult", func() { It("reports the winning source", func() { steps := []artwork.TraceStep{{Candidate: "folder", Outcome: "hit", Detail: "/music/a.jpg"}} @@ -337,10 +405,10 @@ var _ = Describe("discArtworkName", func() { }) var _ = Describe("artwork refresh command", func() { - It("requires at least a kind and one id", func() { - Expect(artworkRefreshCmd.Args(artworkRefreshCmd, []string{"ar"})).To(HaveOccurred()) + It("requires at least one argument", func() { + Expect(artworkRefreshCmd.Args(artworkRefreshCmd, []string{})).To(HaveOccurred()) + Expect(artworkRefreshCmd.Args(artworkRefreshCmd, []string{"id1"})).ToNot(HaveOccurred()) Expect(artworkRefreshCmd.Args(artworkRefreshCmd, []string{"ar", "id1"})).ToNot(HaveOccurred()) - Expect(artworkRefreshCmd.Args(artworkRefreshCmd, []string{"ar", "id1", "id2"})).ToNot(HaveOccurred()) }) }) @@ -873,7 +941,8 @@ var _ = Describe("refreshItems", func() { Expect(art.PutItemArtwork(&model.ItemArtwork{ItemKind: model.KindAlbumArtwork.Prefix(), ItemID: "al-1", ImageType: model.ImageTypePrimary, Hash: "abc123"})).To(Succeed()) - Expect(refreshItems(ctx, ds, model.KindAlbumArtwork, []string{"al-1", "al-3"}, &out)).To(BeZero()) + Expect(refreshItems(ctx, ds, []model.ArtworkID{ + {Kind: model.KindAlbumArtwork, ID: "al-1"}, {Kind: model.KindAlbumArtwork, ID: "al-3"}}, &out)).To(BeZero()) _, err := art.GetItemArtwork(model.KindAlbumArtwork, "al-1", model.ImageTypePrimary) Expect(err).To(MatchError(model.ErrNotFound)) @@ -884,7 +953,7 @@ var _ = Describe("refreshItems", func() { }) It("skips an id that does not exist instead of queuing it", func() { - Expect(refreshItems(ctx, ds, model.KindAlbumArtwork, []string{"al-2"}, &out)).To(Equal(1)) + Expect(refreshItems(ctx, ds, []model.ArtworkID{{Kind: model.KindAlbumArtwork, ID: "al-2"}}, &out)).To(Equal(1)) _, err := queue.Get(model.KindAlbumArtwork, "al-2", model.ImageTypePrimary) Expect(err).To(MatchError(model.ErrNotFound), "a typo must not leave an orphan queue row") @@ -892,8 +961,8 @@ var _ = Describe("refreshItems", func() { }) It("continues past a failing id and counts the failures", func() { - Expect(refreshItems(ctx, ds, model.KindAlbumArtwork, - []string{"al-1", "al-2", "al-3"}, &out)).To(Equal(1)) + Expect(refreshItems(ctx, ds, []model.ArtworkID{{Kind: model.KindAlbumArtwork, ID: "al-1"}, + {Kind: model.KindAlbumArtwork, ID: "al-2"}, {Kind: model.KindAlbumArtwork, ID: "al-3"}}, &out)).To(Equal(1)) Expect(out.String()).To(Equal("al/al-1: queued\nal/al-3: queued\n"), "the ids after a failure are still refreshed") diff --git a/model/get_entity.go b/model/get_entity.go index 3e1a78d1d..e5d41f1be 100644 --- a/model/get_entity.go +++ b/model/get_entity.go @@ -7,21 +7,36 @@ import ( // TODO: Should the type be encoded in the ID? func GetEntityByID(ctx context.Context, ds DataStore, id string) (any, error) { - getters := []func() (any, error){ - func() (any, error) { return ds.Artist(ctx).Get(id) }, - func() (any, error) { return ds.Album(ctx).Get(id) }, - func() (any, error) { return ds.Playlist(ctx).Get(id) }, - func() (any, error) { return ds.MediaFile(ctx).Get(id) }, - func() (any, error) { return ds.Radio(ctx).Get(id) }, + entity, _, err := getEntity(ctx, ds, id) + return entity, err +} + +// GetEntityKindByID resolves a bare entity id to its artwork Kind, searching the same tables as +// GetEntityByID. It reports ErrNotFound when no entity owns the id. +func GetEntityKindByID(ctx context.Context, ds DataStore, id string) (Kind, error) { + _, kind, err := getEntity(ctx, ds, id) + return kind, err +} + +func getEntity(ctx context.Context, ds DataStore, id string) (any, Kind, error) { + getters := []struct { + kind Kind + get func() (any, error) + }{ + {KindArtistArtwork, func() (any, error) { return ds.Artist(ctx).Get(id) }}, + {KindAlbumArtwork, func() (any, error) { return ds.Album(ctx).Get(id) }}, + {KindPlaylistArtwork, func() (any, error) { return ds.Playlist(ctx).Get(id) }}, + {KindMediaFileArtwork, func() (any, error) { return ds.MediaFile(ctx).Get(id) }}, + {KindRadioArtwork, func() (any, error) { return ds.Radio(ctx).Get(id) }}, } - for _, get := range getters { - entity, err := get() + for _, g := range getters { + entity, err := g.get() if err == nil { - return entity, nil + return entity, g.kind, nil } if !errors.Is(err, ErrNotFound) { - return nil, err + return nil, Kind{}, err } } - return nil, ErrNotFound + return nil, Kind{}, ErrNotFound } diff --git a/model/get_entity_test.go b/model/get_entity_test.go index f8a4c9e8e..4e589406a 100644 --- a/model/get_entity_test.go +++ b/model/get_entity_test.go @@ -38,3 +38,25 @@ var _ = Describe("GetEntityByID", func() { Expect(err).ToNot(MatchError(model.ErrNotFound)) }) }) + +var _ = Describe("GetEntityKindByID", func() { + var ds *tests.MockDataStore + var ctx context.Context + + BeforeEach(func() { + ds = &tests.MockDataStore{} + ctx = GinkgoT().Context() + }) + + It("returns the artwork kind for the matching id", func() { + ds.Album(ctx).(*tests.MockAlbumRepo).SetData(model.Albums{{ID: "a1"}}) + kind, err := model.GetEntityKindByID(ctx, ds, "a1") + Expect(err).ToNot(HaveOccurred()) + Expect(kind).To(Equal(model.KindAlbumArtwork)) + }) + + It("returns ErrNotFound when no entity matches", func() { + _, err := model.GetEntityKindByID(ctx, ds, "missing") + Expect(err).To(MatchError(model.ErrNotFound)) + }) +}) From 3d3c3ed60186deb95461cd80b052e564fb9d2055 Mon Sep 17 00:00:00 2001 From: Daniel Barrientos Anariba <69573860+danielbanariba@users.noreply.github.com> Date: Thu, 20 Aug 2026 06:53:11 -0600 Subject: [PATCH 24/27] ci: lint with the golangci-lint version the Makefile declares (#5994) * ci: lint with the golangci-lint version the Makefile declares The workflow asked for `version: latest` while the Makefile pins `GOLANGCI_LINT_VERSION ?= v2.12.0`, so `make lint` and CI ran different linters. golangci-lint v2.13.0 started reporting G404 on three existing `rand.Shuffle` calls, which turned every PR red without a line of Go changing. Read the version from the Makefile instead of resolving `latest`, so the two stay in step and a new release cannot break unchanged code. * chore: re-run CI --- .github/workflows/pipeline.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml index 8e6e8126a..b91c19505 100644 --- a/.github/workflows/pipeline.yml +++ b/.github/workflows/pipeline.yml @@ -68,10 +68,16 @@ jobs: with: go-version-file: go.mod + # Keep CI on the same version `make lint` installs, so a clean local run + # cannot turn red in CI just because a new golangci-lint was released. + - name: Resolve golangci-lint version + id: golangci-version + run: echo "version=$(grep '^GOLANGCI_LINT_VERSION' Makefile | cut -d ' ' -f 3)" >> "$GITHUB_OUTPUT" + - name: golangci-lint uses: golangci/golangci-lint-action@v9 with: - version: latest + version: ${{ steps.golangci-version.outputs.version }} problem-matchers: true args: --timeout 2m From fc1c1366dcfb18c6ac885fe271d04ec6fede8c2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Deluan=20Quint=C3=A3o?= Date: Thu, 20 Aug 2026 09:04:09 -0400 Subject: [PATCH 25/27] fix(cli): write `pls -p` playlist output to stdout (#5996) The export path used the `println` builtin, which writes to stderr, so `navidrome pls -p X > playlist.m3u8` produced an empty file while the M3U body was interleaved with the startup logs on stderr. `println` also appended a newline that `ToM3U8` already provides, so the piped output had a stray trailing blank line that `-o file` did not. Both destinations are now byte-identical. The stdout/file choice moved into a `writePlaylist` helper shared by `pls -p` and `pls export -p`, which both had the same bug. It takes the destination as an `io.Writer`, matching the existing convention in cmd/artwork.go. --- cmd/pls.go | 17 ++++++++++------- cmd/pls_test.go | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 7 deletions(-) create mode 100644 cmd/pls_test.go diff --git a/cmd/pls.go b/cmd/pls.go index 184ca6fe7..93b411483 100644 --- a/cmd/pls.go +++ b/cmd/pls.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "os" "path/filepath" "strconv" @@ -141,14 +142,16 @@ func findPlaylist(ctx context.Context, ds model.DataStore, nameOrID string) *mod func runExporter(ctx context.Context) { ds, ctx := getAdminContext(ctx) playlist := findPlaylist(ctx, ds, playlistID) - pls := playlist.ToM3U8() - if outputFile == "-" || outputFile == "" { - println(pls) + writePlaylist(playlist.ToM3U8(), os.Stdout, outputFile) +} + +func writePlaylist(m3u string, out io.Writer, file string) { + if file == "" || file == "-" { + fmt.Fprint(out, m3u) return } - err := os.WriteFile(outputFile, []byte(pls), 0600) - if err != nil { - log.Fatal("Error writing to the output file", "file", outputFile, err) + if err := os.WriteFile(file, []byte(m3u), 0600); err != nil { + log.Fatal("Error writing to the output file", "file", file, err) } } @@ -157,7 +160,7 @@ func runExport(ctx context.Context) { if playlistID != "" && outputFile == "" { playlist := findPlaylist(ctx, ds, playlistID) - println(playlist.ToM3U8()) + writePlaylist(playlist.ToM3U8(), os.Stdout, outputFile) return } diff --git a/cmd/pls_test.go b/cmd/pls_test.go new file mode 100644 index 000000000..f3e8c7edd --- /dev/null +++ b/cmd/pls_test.go @@ -0,0 +1,35 @@ +package cmd + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("writePlaylist", func() { + const m3u = "#EXTM3U\n#PLAYLIST:DJ Wave\n#EXTINF:364,Bel Canto - Dreaming Girl\n" + plsFile := filepath.Join(os.TempDir(), fmt.Sprintf("navidrome-pls-%d.m3u8", os.Getpid())) + + BeforeEach(func() { + DeferCleanup(func() { _ = os.Remove(plsFile) }) + }) + + DescribeTable("writes the playlist to exactly one destination", + func(file, wantStream, wantFile string) { + var out strings.Builder + + writePlaylist(m3u, &out, file) + + written, _ := os.ReadFile(plsFile) + Expect(out.String()).To(Equal(wantStream)) + Expect(string(written)).To(Equal(wantFile)) + }, + Entry("no file name writes to the stream", "", m3u, ""), + Entry("a dash writes to the stream", "-", m3u, ""), + Entry("a path writes to the file", plsFile, "", m3u), + ) +}) From 17db7d40770bb30edffc675323e4329a9303288a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Deluan=20Quint=C3=A3o?= Date: Thu, 20 Aug 2026 10:02:04 -0400 Subject: [PATCH 26/27] ci: pull base images through mirror.gcr.io instead of ECR Public (#5997) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI builds were failing at random with: buildx failed with: toomanyrequests: Rate exceeded The 429 comes from public.ecr.aws, not Docker Hub. AWS caps unauthenticated ECR Public pulls at 1 per second per source IP (authenticated: 10/s). The build matrix starts 11 jobs at once and each resolves 4 base images, so roughly 44 anonymous pulls land in a couple of seconds — from GitHub runner IPs that are shared with every other GitHub customer. Measured against public.ecr.aws with an anonymous token, 60 requests at concurrency 30 returned 31x 429 in 0.48s. The same probe against mirror.gcr.io (Google's Docker Hub pull-through cache) returned zero errors: 750 manifest requests up to ~141 req/s, plus 120 layer blob requests at concurrency 60. Google publishes no rate limit for it, so this is measured headroom, not a contract — but it is roughly 10x the pipeline's peak rate, and cached pulls do not count against Docker Hub's limits either. Authenticating to ECR Public was the alternative. It was rejected because 10 pulls/s is still under the ~44-pull burst, it needs an AWS account plus a secret, and secrets never reach fork pull requests — so forks would keep failing. The mirror fixes forks too. Verified buildkit honours the mirror block by routing a build through a local logging registry: all 6 requests (manifests and blobs) hit the mirror, none went to Docker Hub directly. Confirmed mirror.gcr.io answers 200 for buildkit's "?ns=docker.io" query form on all four images, for both GET and HEAD. Confirmed buildkit falls back to Docker Hub when the mirror is unreachable — the build still succeeds, but the resolve takes ~30s instead of ~0.3s, so a mirror outage means slow builds, not broken ones. The existing Docker Hub login covers that fallback on main-repo runs. msitools.dockerfile is only used by the local `make docker-msi` target, but is switched over too so no ECR Public reference is left behind. --- .github/actions/prepare-docker/action.yml | 5 +++++ Dockerfile | 10 +++++----- release/wix/msitools.dockerfile | 2 +- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/.github/actions/prepare-docker/action.yml b/.github/actions/prepare-docker/action.yml index b8cde4aaf..6cb54dbdb 100644 --- a/.github/actions/prepare-docker/action.yml +++ b/.github/actions/prepare-docker/action.yml @@ -68,6 +68,11 @@ runs: - name: Set up Docker Buildx id: buildx uses: docker/setup-buildx-action@v4 + with: + # Runner IPs are shared, so anonymous base image pulls get rate-limited. + buildkitd-config-inline: | + [registry."docker.io"] + mirrors = ["mirror.gcr.io"] - name: Extract metadata for Docker image id: meta diff --git a/Dockerfile b/Dockerfile index df5df52ab..847c19bf7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM --platform=$BUILDPLATFORM ghcr.io/crazy-max/osxcross:14.5-debian AS osxcros ######################################################################################################################## ### Build xx (original image: tonistiigi/xx) -FROM --platform=$BUILDPLATFORM public.ecr.aws/docker/library/alpine:3.20 AS xx-build +FROM --platform=$BUILDPLATFORM alpine:3.20 AS xx-build # v1.9.0 ENV XX_VERSION=a5592eab7a57895e8d385394ff12241bc65ecd50 @@ -26,7 +26,7 @@ COPY --from=xx-build /out/ /usr/bin/ ######################################################################################################################## ### Build Navidrome UI -FROM --platform=$BUILDPLATFORM public.ecr.aws/docker/library/node:lts-alpine AS ui +FROM --platform=$BUILDPLATFORM node:lts-alpine AS ui WORKDIR /app # Install node dependencies @@ -43,7 +43,7 @@ COPY --from=ui /build /build ######################################################################################################################## ### Build Navidrome binary for Docker image (dynamic musl, enables native libwebp via dlopen) -FROM --platform=$BUILDPLATFORM public.ecr.aws/docker/library/golang:1.26-alpine AS build-alpine +FROM --platform=$BUILDPLATFORM golang:1.26-alpine AS build-alpine COPY --from=xx / / ARG TARGETPLATFORM @@ -85,7 +85,7 @@ EOT ######################################################################################################################## ### Build Navidrome binary for standalone distribution (static glibc, cross-compiled) -FROM --platform=$BUILDPLATFORM public.ecr.aws/docker/library/golang:1.26-trixie AS base +FROM --platform=$BUILDPLATFORM golang:1.26-trixie AS base RUN apt-get update && apt-get install -y clang lld COPY --from=xx / / WORKDIR /workspace @@ -154,7 +154,7 @@ COPY --from=build /out / ######################################################################################################################## ### Build Final Image -FROM public.ecr.aws/docker/library/alpine:3.20 AS final +FROM alpine:3.20 AS final LABEL maintainer="deluan@navidrome.org" LABEL org.opencontainers.image.source="https://github.com/navidrome/navidrome" diff --git a/release/wix/msitools.dockerfile b/release/wix/msitools.dockerfile index 38364eb47..90249c1ce 100644 --- a/release/wix/msitools.dockerfile +++ b/release/wix/msitools.dockerfile @@ -1,3 +1,3 @@ -FROM public.ecr.aws/docker/library/alpine +FROM alpine RUN apk update && apk add jq msitools WORKDIR /workspace \ No newline at end of file From 0a55cc8cafdde9ab92aa0a207507dc2165ff1caf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Deluan=20Quint=C3=A3o?= Date: Thu, 20 Aug 2026 22:48:26 -0400 Subject: [PATCH 27/27] docs(plugins): document how a metadata agent signals "not found" (#6001) A MetadataAgent plugin reports "I have no data for this item" by returning an empty response with a nil error. Any error it returns instead is treated as a plugin fault and retried with backoff. That rule was not documented anywhere, so an author naturally returns an error for a missing item, and Navidrome then retries every item the plugin's source does not cover. This is not hypothetical: the artist-nfo-metadata plugin returned an error for every artist without an artist.nfo, which kept those artists in the artwork retry queue for hours and tripped the artwork circuit breaker for the plugin as a whole. Document the rule on the capability interface, which ndpgen copies into the Go PDK, and in the MetadataAgent section of the plugin README. --- plugins/README.md | 8 ++++++++ plugins/capabilities/metadata_agent.go | 3 +++ plugins/pdk/go/metadata/metadata.go | 3 +++ plugins/pdk/go/metadata/metadata_stub.go | 3 +++ 4 files changed, 17 insertions(+) diff --git a/plugins/README.md b/plugins/README.md index b04e12bd9..7042b8c45 100644 --- a/plugins/README.md +++ b/plugins/README.md @@ -174,6 +174,14 @@ Capabilities define what your plugin can do. They're automatically detected base Provides artist and album metadata. All methods are **optional** — implement only the ones your data source supports. +> **Returning "not found".** When you have no data for an item, return an empty response and no +> error. In the Go PDK that is `return nil, nil`. Navidrome reads it as a definitive "not found" +> and stops asking. +> +> Return an error only when the plugin itself failed, such as an unreachable API or a broken host +> call. Navidrome retries failed calls with backoff. A plugin that errors on "no data" makes +> Navidrome retry every item it has no data for. + | Function | Input | Output | Description | |-----------------------------------|----------------------------|----------------------------------|--------------------------| | `nd_get_artist_mbid` | `{id, name}` | `{mbid}` | Get MusicBrainz ID | diff --git a/plugins/capabilities/metadata_agent.go b/plugins/capabilities/metadata_agent.go index f856562c6..72cb1622f 100644 --- a/plugins/capabilities/metadata_agent.go +++ b/plugins/capabilities/metadata_agent.go @@ -9,6 +9,9 @@ import "github.com/navidrome/navidrome/plugins/types" // Plugins implementing this capability can choose which methods to implement. // Each method is optional - plugins only need to provide the functionality they support. // +// To say "no data for this item", return a nil response and a nil error. Return an error only when +// the plugin itself failed, because Navidrome retries failed calls with backoff. +// //nd:capability name=metadata type MetadataAgent interface { // GetArtistMBID retrieves the MusicBrainz ID for an artist. diff --git a/plugins/pdk/go/metadata/metadata.go b/plugins/pdk/go/metadata/metadata.go index c561c2893..bb0ae9620 100644 --- a/plugins/pdk/go/metadata/metadata.go +++ b/plugins/pdk/go/metadata/metadata.go @@ -186,6 +186,9 @@ type TopSongsResponse struct { // // Plugins implementing this capability can choose which methods to implement. // Each method is optional - plugins only need to provide the functionality they support. +// +// To say "no data for this item", return a nil response and a nil error. Return an error only when +// the plugin itself failed, because Navidrome retries failed calls with backoff. type Metadata interface{} // ArtistMBIDProvider provides the GetArtistMBID function. diff --git a/plugins/pdk/go/metadata/metadata_stub.go b/plugins/pdk/go/metadata/metadata_stub.go index e72cca103..572eba4da 100644 --- a/plugins/pdk/go/metadata/metadata_stub.go +++ b/plugins/pdk/go/metadata/metadata_stub.go @@ -184,6 +184,9 @@ type TopSongsResponse struct { // // Plugins implementing this capability can choose which methods to implement. // Each method is optional - plugins only need to provide the functionality they support. +// +// To say "no data for this item", return a nil response and a nil error. Return an error only when +// the plugin itself failed, because Navidrome retries failed calls with backoff. type Metadata interface{} // ArtistMBIDProvider provides the GetArtistMBID function.