nik/navidrome
1
0
Fork 0
mirror of https://github.com/navidrome/navidrome.git synced 2026-08-31 07:30:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into go127-bugfixes

Browse Source
This commit is contained in:
Deluan Quintão 2026-08-30 16:27:16 -04:00 committed by GitHub
commit 7973e78048
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
164 changed files with 5128 additions and 1026 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.devcontainer/devcontainer.json
View File

@ -4,7 +4,7 @@
"dockerfile": "Dockerfile",
"args": {
// Update the VARIANT arg to pick a version of Go: 1, 1.15, 1.14
"VARIANT": "1.26",
"VARIANT": "1.27",
// Options
"INSTALL_NODE": "true",
"NODE_VERSION": "v24"

5
.github/actions/prepare-docker/action.yml vendored
View File

@ -68,6 +68,11 @@ runs:
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@v4
with:
# Runner IPs are shared, so anonymous base image pulls get rate-limited.
buildkitd-config-inline: |
[registry."docker.io"]
mirrors = ["mirror.gcr.io"]
- name: Extract metadata for Docker image
id: meta

18
.github/workflows/pipeline.yml vendored
View File

@ -68,10 +68,16 @@ jobs:
with:
go-version-file: go.mod
# Keep CI on the same version `make lint` installs, so a clean local run
# cannot turn red in CI just because a new golangci-lint was released.
- name: Resolve golangci-lint version
id: golangci-version
run: echo "version=$(grep '^GOLANGCI_LINT_VERSION' Makefile | cut -d ' ' -f 3)" >> "$GITHUB_OUTPUT"
- name: golangci-lint
uses: golangci/golangci-lint-action@v9
with:
version: latest
version: ${{ steps.golangci-version.outputs.version }}
problem-matchers: true
args: --timeout 2m
@ -125,12 +131,22 @@ jobs:
uses: actions/checkout@v7
- uses: actions/setup-go@v6
id: setup-go
with:
go-version-file: go.mod
- name: Download dependencies
run: go mod download
# Without this, the plugins suite recompiles every test plugin WASM module,
# which dominates the job runtime under -race.
- name: Cache the plugins test suite WASM compilation cache
uses: actions/cache@v6
with:
path: plugins/testdata/.wazero-cache
key: wazero-${{ runner.os }}-go${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('plugins/testdata/*/*.go', 'plugins/testdata/*/go.*', 'plugins/pdk/go/**/*.go', 'plugins/pdk/go/go.*') }}
restore-keys: wazero-${{ runner.os }}-
- name: Test
run: go test -shuffle=on -tags netgo,sqlite_fts5 -race ./... -v

5
.gitignore vendored
View File

@ -43,4 +43,7 @@ go.work*
.playwright-mcp/
# Temp benchmark files
zz_*_test.go
zz_*_test.go
# wazero compilation cache for the plugins test suite
/plugins/testdata/.wazero-cache/

45
Dockerfile
View File

@ -2,7 +2,7 @@ FROM --platform=$BUILDPLATFORM ghcr.io/crazy-max/osxcross:14.5-debian AS osxcros
########################################################################################################################
### Build xx (original image: tonistiigi/xx)
FROM --platform=$BUILDPLATFORM public.ecr.aws/docker/library/alpine:3.20 AS xx-build
FROM --platform=$BUILDPLATFORM alpine:3.22 AS xx-build
# v1.9.0
ENV XX_VERSION=a5592eab7a57895e8d385394ff12241bc65ecd50
@ -26,7 +26,7 @@ COPY --from=xx-build /out/ /usr/bin/
########################################################################################################################
### Build Navidrome UI
FROM --platform=$BUILDPLATFORM public.ecr.aws/docker/library/node:lts-alpine AS ui
FROM --platform=$BUILDPLATFORM node:lts-alpine AS ui
WORKDIR /app
# Install node dependencies
@ -43,7 +43,7 @@ COPY --from=ui /build /build
########################################################################################################################
### Build Navidrome binary for Docker image (dynamic musl, enables native libwebp via dlopen)
FROM --platform=$BUILDPLATFORM public.ecr.aws/docker/library/golang:1.26-alpine AS build-alpine
FROM --platform=$BUILDPLATFORM golang:1.27-alpine AS build-alpine
COPY --from=xx / /
ARG TARGETPLATFORM
@ -85,7 +85,7 @@ EOT
########################################################################################################################
### Build Navidrome binary for standalone distribution (static glibc, cross-compiled)
FROM --platform=$BUILDPLATFORM public.ecr.aws/docker/library/golang:1.26-trixie AS base
FROM --platform=$BUILDPLATFORM golang:1.27-trixie AS base
RUN apt-get update && apt-get install -y clang lld
COPY --from=xx / /
WORKDIR /workspace
@ -152,19 +152,52 @@ RUN xx-verify --static /out/navidrome*
FROM scratch AS binary
COPY --from=build /out /
########################################################################################################################
### Build no-op stubs for mpv's video-output libraries
# mpv links libEGL/libgbm for video output only; Navidrome drives it headless, for audio.
# Real mesa pulls in LLVM + gallium (+218MB uncompressed), so ship stubs it never calls.
FROM --platform=$BUILDPLATFORM alpine:3.22 AS mpv-stubs
COPY --from=xx / /
RUN apk add --no-cache clang lld binutils mesa-egl mesa-gbm
ARG TARGETPLATFORM
RUN xx-apk add --no-cache musl-dev
RUN <<EOT
set -e
mkdir -p /out
for so in libEGL.so.1 libgbm.so.1; do
readelf -sW /usr/lib/$so \
| awk '$5 == "GLOBAL" && $7 != "UND" { print $8 }' \
| sed 's/@.*//' \
| grep -vE '^(_init|_fini|_edata|_end|__bss_start|_GLOBAL_OFFSET_TABLE_)$' \
| sort -u \
| awk '{ print "void " $1 "(void) {}" }' > /tmp/stub.c
test -s /tmp/stub.c
xx-clang -shared -nostdlib -fPIC -Wl,-soname,$so -o /out/$so /tmp/stub.c
xx-verify /out/$so
done
EOT
########################################################################################################################
### Build Final Image
FROM public.ecr.aws/docker/library/alpine:3.20 AS final
FROM alpine:3.22 AS final
LABEL maintainer="deluan@navidrome.org"
LABEL org.opencontainers.image.source="https://github.com/navidrome/navidrome"
# Install runtime dependencies
# - libwebp + symlinks: enables native WebP encoding via purego/dlopen
# The mesa/LLVM stack mpv pulls in for video output is dropped in this same layer,
# otherwise the deleted bytes still ship in the image.
RUN apk add -U --no-cache ffmpeg mpv sqlite libwebp libwebpdemux libwebpmux && \
for lib in libwebp libwebpdemux libwebpmux; do \
target=$(ls /usr/lib/$lib.so.* 2>/dev/null | head -1) && \
[ -n "$target" ] && ln -sf "$target" /usr/lib/$lib.so; \
done
done && \
rm -rf /usr/lib/gallium-pipe /usr/lib/dri \
/usr/lib/libEGL.so* /usr/lib/libgbm.so* /usr/lib/libgallium*.so /usr/lib/libLLVM.so* \
/usr/lib/libGL.so* /usr/lib/libGLESv2.so* /usr/lib/libglapi.so*
COPY --from=mpv-stubs /out/ /usr/lib/
RUN mpv --no-video --ao=null --version > /dev/null
# Copy navidrome binary (musl build for Docker, enables native libwebp)
COPY --from=build-alpine /out/navidrome /app/

2
Makefile
View File

@ -20,7 +20,7 @@ IMAGE_PLATFORMS ?= $(shell echo $(SUPPORTED_PLATFORMS) | tr ',' '\n' | grep "lin
PLATFORMS ?= $(SUPPORTED_PLATFORMS)
DOCKER_TAG ?= deluan/navidrome:develop
GOLANGCI_LINT_VERSION ?= v2.12.0
GOLANGCI_LINT_VERSION ?= v2.13.2
UI_SRC_FILES := $(shell find ui -type f -not -path "ui/build/*" -not -path "ui/node_modules/*")

6
adapters/deezer/deezer.go
View File

@ -5,7 +5,6 @@ import (
"context"
"errors"
"fmt"
"net/http"
"slices"
"strings"
@ -15,6 +14,7 @@ import (
"github.com/navidrome/navidrome/log"
"github.com/navidrome/navidrome/model"
"github.com/navidrome/navidrome/utils/cache"
"github.com/navidrome/navidrome/utils/httpclient"
"github.com/navidrome/navidrome/utils/slice"
)
@ -36,9 +36,7 @@ func deezerConstructor(dataStore model.DataStore) agents.Interface {
dataStore: dataStore,
languages: conf.Server.Deezer.Languages,
}
httpClient := &http.Client{
Timeout: consts.DefaultHttpClientTimeOut,
}
httpClient := httpclient.New(consts.DefaultHttpClientTimeOut)
cachedHttpClient := cache.NewHTTPClient(httpClient, consts.DefaultHttpClientTimeOut)
agent.client = newClient(cachedHttpClient)
return agent

8
adapters/lastfm/agent.go
View File

@ -18,6 +18,7 @@ import (
"github.com/navidrome/navidrome/log"
"github.com/navidrome/navidrome/model"
"github.com/navidrome/navidrome/utils/cache"
"github.com/navidrome/navidrome/utils/httpclient"
"golang.org/x/net/html"
)
@ -59,9 +60,7 @@ func lastFMConstructor(ds model.DataStore) *lastfmAgent {
secret: conf.Server.LastFM.Secret,
sessionKeys: &agents.SessionKeys{DataStore: ds, KeyName: sessionKeyProperty},
}
hc := &http.Client{
Timeout: consts.DefaultHttpClientTimeOut,
}
hc := httpclient.New(consts.DefaultHttpClientTimeOut)
chc := cache.NewHTTPClient(hc, consts.DefaultHttpClientTimeOut)
l.httpClient = chc
l.client = newClient(l.apiKey, l.secret, chc)
@ -406,7 +405,8 @@ func (l *lastfmAgent) Scrobble(ctx context.Context, userId string, s scrobbler.S
log.Warn(ctx, "Last.fm client.scrobble returned error", "track", s.Title, err)
return errors.Join(err, scrobbler.ErrRetryLater)
}
if lfErr.Code == 11 || lfErr.Code == 16 {
// 11: service offline; 16: temporarily unavailable. Rate limiting is mapped by the client.
if lfErr.Code == 11 || lfErr.Code == 16 || errors.Is(err, scrobbler.ErrRetryLater) {
return errors.Join(err, scrobbler.ErrRetryLater)
}
return errors.Join(err, scrobbler.ErrUnrecoverable)

19
adapters/lastfm/agent_test.go
View File

@ -100,6 +100,15 @@ var _ = Describe("lastfmAgent", func() {
Expect(httpClient.RequestCount).To(Equal(1))
Expect(httpClient.SavedRequest.URL.Query().Get("artist")).To(Equal("U2"))
})
It("returns ErrRetryLater on error 29 (rate limit exceeded)", func() {
httpClient.Res = http.Response{
Body: io.NopCloser(bytes.NewBufferString(`{"error":29,"message":"Rate limit exceeded"}`)),
StatusCode: 200,
}
_, err := agent.GetArtistBiography(ctx, "123", "U2", "")
Expect(errors.Is(err, agents.ErrRetryLater)).To(BeTrue())
})
})
Describe("Language Fallback", func() {
@ -497,6 +506,16 @@ var _ = Describe("lastfmAgent", func() {
Expect(err).To(MatchError(scrobbler.ErrRetryLater))
})
It("returns ErrRetryLater on error 29 (rate limit exceeded)", func() {
httpClient.Res = http.Response{
Body: io.NopCloser(bytes.NewBufferString(`{"error":29,"message":"Rate limit exceeded"}`)),
StatusCode: 200,
}
err := agent.Scrobble(ctx, "user-1", scrobbler.Scrobble{MediaFile: *track, TimeStamp: time.Now()})
Expect(errors.Is(err, scrobbler.ErrRetryLater)).To(BeTrue())
})
It("returns ErrRetryLater on http errors", func() {
httpClient.Res = http.Response{
Body: io.NopCloser(bytes.NewBufferString(`internal server error`)),

5
adapters/lastfm/auth_router.go
View File

@ -18,6 +18,7 @@ import (
"github.com/navidrome/navidrome/model"
"github.com/navidrome/navidrome/model/request"
"github.com/navidrome/navidrome/server"
"github.com/navidrome/navidrome/utils/httpclient"
"github.com/navidrome/navidrome/utils/req"
)
@ -41,9 +42,7 @@ func NewRouter(ds model.DataStore) *Router {
sessionKeys: &agents.SessionKeys{DataStore: ds, KeyName: sessionKeyProperty},
}
r.Handler = r.routes()
hc := &http.Client{
Timeout: consts.DefaultHttpClientTimeOut,
}
hc := httpclient.New(consts.DefaultHttpClientTimeOut)
r.client = newClient(r.apiKey, r.secret, hc)
return r
}

9
adapters/lastfm/auth_router_test.go
View File

@ -214,5 +214,14 @@ var _ = Describe("auth_router", func() {
_, err = verifyLinkToken(nonExpiringToken)
Expect(err).To(MatchError("link token missing expiration"))
})
It("rejects a Jellyfin access token", func() {
usr := &model.User{ID: "u1", UserName: "johndoe"}
tokenStr, err := auth.CreateAPIToken(usr, auth.AudienceJellyfin)
Expect(err).ToNot(HaveOccurred())
_, err = verifyLinkToken(tokenStr)
Expect(err).To(HaveOccurred())
})
})
})

11
adapters/lastfm/client.go
View File

@ -5,6 +5,7 @@ import (
"crypto/md5"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"net/http"
"net/url"
@ -14,11 +15,15 @@ import (
"strings"
"time"
"github.com/navidrome/navidrome/core/agents"
"github.com/navidrome/navidrome/log"
)
const (
apiBaseUrl = "https://ws.audioscrobbler.com/2.0/"
// errCodeRateLimit is Last.fm's "rate limit exceeded"; it arrives in the body, with HTTP 200
// and no rate-limit headers, so the body code is the only signal.
errCodeRateLimit = 29
)
type lastFMError struct {
@ -225,7 +230,11 @@ func (c *client) makeRequest(ctx context.Context, method string, params url.Valu
return nil, jsonErr
}
if response.Error != 0 {
return &response, &lastFMError{Code: response.Error, Message: response.Message}
var err error = &lastFMError{Code: response.Error, Message: response.Message}
if response.Error == errCodeRateLimit {
err = errors.Join(err, &agents.RetryLaterError{})
}
return &response, err
}
return &response, nil

6
adapters/listenbrainz/agent.go
View File

@ -3,7 +3,6 @@ package listenbrainz
import (
"context"
"errors"
"net/http"
"github.com/navidrome/navidrome/conf"
"github.com/navidrome/navidrome/consts"
@ -12,6 +11,7 @@ import (
"github.com/navidrome/navidrome/log"
"github.com/navidrome/navidrome/model"
"github.com/navidrome/navidrome/utils/cache"
"github.com/navidrome/navidrome/utils/httpclient"
"github.com/navidrome/navidrome/utils/slice"
)
@ -33,9 +33,7 @@ func listenBrainzConstructor(ds model.DataStore) *listenBrainzAgent {
sessionKeys: &agents.SessionKeys{DataStore: ds, KeyName: sessionKeyProperty},
baseURL: conf.Server.ListenBrainz.BaseURL,
}
hc := &http.Client{
Timeout: consts.DefaultHttpClientTimeOut,
}
hc := httpclient.New(consts.DefaultHttpClientTimeOut)
chc := cache.NewHTTPClient(hc, consts.DefaultHttpClientTimeOut)
l.client = newClient(l.baseURL, chc)
return l

13
adapters/listenbrainz/agent_test.go
View File

@ -164,6 +164,19 @@ var _ = Describe("listenBrainzAgent", func() {
err := agent.Scrobble(ctx, "user-1", sc)
Expect(err).To(MatchError(scrobbler.ErrUnrecoverable))
})
It("keeps a 429 scrobble for retry and carries the delay", func() {
httpClient.Res = http.Response{
StatusCode: 429,
Header: http.Header{"X-Ratelimit-Reset-In": []string{"7"}},
Body: io.NopCloser(bytes.NewBufferString(`{"code":429,"error":"rate limited"}`)),
}
err := agent.Scrobble(ctx, "user-1", scrobbler.Scrobble{MediaFile: *track, TimeStamp: time.Now()})
Expect(errors.Is(err, scrobbler.ErrRetryLater)).To(BeTrue())
retry, ok := errors.AsType[*agents.RetryLaterError](err)
Expect(ok).To(BeTrue())
Expect(retry.RetryIn).To(Equal(7 * time.Second))
})
})
Describe("GetArtistUrl", func() {

5
adapters/listenbrainz/auth_router.go
View File

@ -16,6 +16,7 @@ import (
"github.com/navidrome/navidrome/model"
"github.com/navidrome/navidrome/model/request"
"github.com/navidrome/navidrome/server"
"github.com/navidrome/navidrome/utils/httpclient"
)
type sessionKeysRepo interface {
@ -37,9 +38,7 @@ func NewRouter(ds model.DataStore) *Router {
sessionKeys: &agents.SessionKeys{DataStore: ds, KeyName: sessionKeyProperty},
}
r.Handler = r.routes()
hc := &http.Client{
Timeout: consts.DefaultHttpClientTimeOut,
}
hc := httpclient.New(consts.DefaultHttpClientTimeOut)
r.client = newClient(conf.Server.ListenBrainz.BaseURL, hc)
return r
}

17
adapters/listenbrainz/client.go
View File

@ -13,6 +13,7 @@ import (
"slices"
"github.com/navidrome/navidrome/conf"
"github.com/navidrome/navidrome/core/agents"
"github.com/navidrome/navidrome/log"
)
@ -21,6 +22,12 @@ const (
labsBase = "https://labs.api.listenbrainz.org/"
)
// retryLaterErr reads the wait ListenBrainz asked for. It sends X-RateLimit-Reset-In
// (delta-seconds) on every response, including the 429, and never Retry-After.
func retryLaterErr(h http.Header) *agents.RetryLaterError {
return &agents.RetryLaterError{RetryIn: agents.ParseRetryIn(h.Get("X-RateLimit-Reset-In"))}
}
var (
ErrorNotFound = errors.New("listenbrainz: not found")
)
@ -174,6 +181,9 @@ func (c *client) makeAuthenticatedRequest(ctx context.Context, method string, en
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusTooManyRequests {
return nil, retryLaterErr(resp.Header)
}
decoder := json.NewDecoder(resp.Body)
var response listenBrainzResponse
@ -185,6 +195,10 @@ func (c *client) makeAuthenticatedRequest(ctx context.Context, method string, en
return nil, jsonErr
}
if response.Code != 0 && response.Code != 200 {
// LB also reports rate limiting as a body code, not only as an HTTP status.
if response.Code == http.StatusTooManyRequests {
return &response, retryLaterErr(resp.Header)
}
return &response, &listenBrainzError{Code: response.Code, Message: response.Error}
}
@ -211,6 +225,9 @@ func (c *client) makeGenericRequest(ctx context.Context, method string, endpoint
// On a 200 code, there is no code. Decode using using error message if it exists
if resp.StatusCode != 200 {
defer resp.Body.Close()
if resp.StatusCode == http.StatusTooManyRequests {
return nil, retryLaterErr(resp.Header)
}
decoder := json.NewDecoder(resp.Body)
var lbzError lbzHttpError

73
adapters/listenbrainz/client_test.go
View File

@ -4,13 +4,17 @@ import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"os"
"strings"
"time"
"github.com/navidrome/navidrome/conf"
"github.com/navidrome/navidrome/conf/configtest"
"github.com/navidrome/navidrome/core/agents"
"github.com/navidrome/navidrome/tests"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
@ -461,4 +465,73 @@ var _ = Describe("client", func() {
}))
})
})
Describe("rate limiting", func() {
It("returns RetryLaterError with the header delay on 429", func() {
httpClient.Res = http.Response{
StatusCode: 429,
Header: http.Header{"X-Ratelimit-Reset-In": []string{"3"}},
Body: io.NopCloser(strings.NewReader(`{"code":429,"error":"You have exceeded your rate limit."}`)),
}
_, err := client.validateToken(context.Background(), "token")
Expect(errors.Is(err, agents.ErrRetryLater)).To(BeTrue())
retry, ok := errors.AsType[*agents.RetryLaterError](err)
Expect(ok).To(BeTrue())
Expect(retry.RetryIn).To(Equal(3 * time.Second))
})
It("returns RetryLaterError with zero delay when no header is present", func() {
httpClient.Res = http.Response{
StatusCode: 429,
Body: io.NopCloser(strings.NewReader(`{"code":429,"error":"rate limited"}`)),
}
_, err := client.validateToken(context.Background(), "token")
Expect(errors.Is(err, agents.ErrRetryLater)).To(BeTrue())
retry, _ := errors.AsType[*agents.RetryLaterError](err)
Expect(retry.RetryIn).To(BeZero())
})
DescribeTable("caps absurd header values at one hour",
func(header string) {
httpClient.Res = http.Response{
StatusCode: 429,
Header: http.Header{"X-Ratelimit-Reset-In": []string{header}},
Body: io.NopCloser(strings.NewReader(`{"code":429,"error":"rate limited"}`)),
}
_, err := client.validateToken(context.Background(), "token")
retry, _ := errors.AsType[*agents.RetryLaterError](err)
Expect(retry.RetryIn).To(Equal(time.Hour))
},
Entry("a large value", "999999"),
Entry("a huge value", "99999999999"),
// Scaling this to nanoseconds before capping wraps past 2^64, landing on ~0.29s.
Entry("a value that overflows int64 nanoseconds", "18446744074"),
)
It("maps a body-level 429 sent with a non-429 status", func() {
httpClient.Res = http.Response{
StatusCode: 200,
Header: http.Header{"X-Ratelimit-Reset-In": []string{"7"}},
Body: io.NopCloser(strings.NewReader(`{"code":429,"error":"You have exceeded your rate limit."}`)),
}
_, err := client.validateToken(context.Background(), "token")
Expect(errors.Is(err, agents.ErrRetryLater)).To(BeTrue())
retry, ok := errors.AsType[*agents.RetryLaterError](err)
Expect(ok).To(BeTrue())
Expect(retry.RetryIn).To(Equal(7 * time.Second))
})
It("returns RetryLaterError on a 429 from makeGenericRequest", func() {
httpClient.Res = http.Response{
StatusCode: 429,
Header: http.Header{"X-Ratelimit-Reset-In": []string{"5"}},
Body: io.NopCloser(strings.NewReader(`{"code":429,"error":"rate limited"}`)),
}
_, err := client.getArtistUrl(context.Background(), "1")
Expect(errors.Is(err, agents.ErrRetryLater)).To(BeTrue())
retry, ok := errors.AsType[*agents.RetryLaterError](err)
Expect(ok).To(BeTrue())
Expect(retry.RetryIn).To(Equal(5 * time.Second))
})
})
})

402
cmd/artwork.go
View File

@ -26,29 +26,41 @@ import (
var explainLive bool
// Only one subcommand runs per invocation, so reprocess and cancel bind the same flag targets.
var (
reprocessKinds []string
reprocessSources []string
reprocessAll bool
reprocessDryRun bool
reprocessYes bool
artworkKinds []string
artworkSources []string
artworkPriorities []string
artworkAll bool
artworkDryRun bool
artworkYes bool
)
func init() {
artworkExplainCmd.Flags().BoolVar(&explainLive, "live", false,
"perform real external lookups instead of reporting what would be tried; "+
"also initializes plugin agents, which may open external connections")
artworkReprocessCmd.Flags().StringSliceVar(&reprocessKinds, "kind", nil,
"walk the chain again now, performing real external lookups, instead of reporting the "+
"stored trace of the last resolution; also initializes plugin agents, which may open "+
"external connections")
artworkReprocessCmd.Flags().StringSliceVar(&artworkKinds, "kind", nil,
"kinds to reprocess ("+kindPrefixes(artwork.RecheckKinds)+"); repeatable")
artworkReprocessCmd.Flags().StringSliceVar(&reprocessSources, "source", nil,
artworkReprocessCmd.Flags().StringSliceVar(&artworkSources, "source", nil,
"only items currently resolved from these sources (e.g. folder, external:deezer, absent)")
artworkReprocessCmd.Flags().BoolVar(&reprocessAll, "all", false, "reprocess every kind")
artworkReprocessCmd.Flags().BoolVar(&reprocessDryRun, "dry-run", false,
artworkReprocessCmd.Flags().BoolVar(&artworkAll, "all", false, "reprocess every kind")
artworkReprocessCmd.Flags().BoolVar(&artworkDryRun, "dry-run", false,
"report what would be queued and exit without queueing")
artworkReprocessCmd.Flags().BoolVarP(&reprocessYes, "yes", "y", false, "skip the confirmation prompt")
artworkReprocessCmd.Flags().BoolVarP(&artworkYes, "yes", "y", false, "skip the confirmation prompt")
artworkCancelCmd.Flags().StringSliceVar(&artworkKinds, "kind", nil,
"kinds to cancel ("+kindPrefixes(artwork.RefreshableKinds)+"); repeatable")
artworkCancelCmd.Flags().StringSliceVar(&artworkPriorities, "priority", nil,
"only rows queued at these priorities ("+priorityNames()+"); repeatable")
artworkCancelCmd.Flags().BoolVar(&artworkAll, "all", false, "cancel every kind at every priority")
artworkCancelCmd.Flags().BoolVar(&artworkDryRun, "dry-run", false,
"report what would be cancelled and exit without cancelling")
artworkCancelCmd.Flags().BoolVarP(&artworkYes, "yes", "y", false, "skip the confirmation prompt")
artworkCmd.AddCommand(artworkExplainCmd)
artworkCmd.AddCommand(artworkRefreshCmd)
artworkCmd.AddCommand(artworkReprocessCmd)
artworkCmd.AddCommand(artworkCancelCmd)
artworkCmd.AddCommand(artworkStatusCmd)
rootCmd.AddCommand(artworkCmd)
}
@ -92,6 +104,22 @@ var artworkReprocessCmd = &cobra.Command{
},
}
var artworkCancelCmd = &cobra.Command{
Use: "cancel",
Short: "Cancel pending artwork work in bulk, by kind and/or queue priority",
Long: "Cancel pending artwork work in bulk, by kind and/or queue priority.\n\n" +
"Only the queue is touched: resolved artwork and the state behind `artwork explain` are\n" +
"left alone, and the trace of why a cancelled item last failed goes with its queue row.\n\n" +
"Work already picked up is not interrupted, and an item with no artwork yet can be\n" +
"queued again by the hourly re-check. The selection is applied again when you confirm,\n" +
"so anything queued after the preview is cancelled too. Use it to call off a bulk\n" +
"backfill, not to stop the worker.",
Args: cobra.NoArgs,
Run: func(cmd *cobra.Command, args []string) {
runCancel(cmd.Context())
},
}
var artworkStatusCmd = &cobra.Command{
Use: "status",
Short: "Report the artwork queue, where artwork resolves from, and the backfill state",
@ -132,9 +160,11 @@ type statusReport struct {
current string
}
func (r statusReport) queueTotal() int64 {
func (r statusReport) queueTotal() int64 { return queueTotal(r.queue) }
func queueTotal(stats []model.ArtworkQueueStat) int64 {
var n int64
for _, s := range r.queue {
for _, s := range stats {
n += s.Count
}
return n
@ -154,7 +184,7 @@ func collectStatus(ctx context.Context, ds model.DataStore) (statusReport, error
q := ds.ArtworkQueue(ctx)
var rep statusReport
var err error
if rep.queue, err = q.CountByKindAndPriority(); err != nil {
if rep.queue, err = q.CountQueued(nil, nil); err != nil {
return rep, fmt.Errorf("breaking the artwork queue down by kind: %w", err)
}
@ -194,11 +224,7 @@ func formatStatus(rep statusReport) string {
if len(rep.queue) == 0 {
fmt.Fprintln(w, " (empty)")
} else {
fmt.Fprintln(w, " KIND\tPRIORITY\tITEMS")
for _, s := range rep.queue {
fmt.Fprintf(w, " %s\t%s\t%d\n", kindName(s.ItemKind), priorityName(s.Priority), s.Count)
}
fmt.Fprintf(w, " TOTAL\t\t%d\n", rep.queueTotal())
printQueueStats(w, rep.queue, rep.queueTotal(), "ITEMS", " ")
}
fmt.Fprintln(w, "\nSources")
@ -212,7 +238,8 @@ func formatStatus(rep statusReport) string {
for _, a := range rep.absent {
fmt.Fprintf(w, " %s\t%d\t%d\n", a.kind, a.Total, a.Stale)
}
fmt.Fprintf(w, " (rechecked once the last attempt is older than %gh)\n", artwork.StaleAbsentAge.Hours())
fmt.Fprintf(w, " (eligible once the last attempt is older than %gh; re-queued %d per kind per hour, oldest first)\n",
artwork.StaleAbsentAge.Hours(), artwork.StaleAbsentRecheckBatch)
fmt.Fprintln(w, "\nBackfill")
fmt.Fprintf(w, " State:\t%s\n", backfillState(rep))
@ -245,6 +272,15 @@ func backfillState(rep statusReport) string {
return "up to date"
}
// printQueueStats writes the shared queue breakdown; the caller owns the tab writer and flushes it.
func printQueueStats(w io.Writer, stats []model.ArtworkQueueStat, total int64, countHeader, indent string) {
fmt.Fprintf(w, "%sKIND\tPRIORITY\t%s\n", indent, countHeader)
for _, s := range stats {
fmt.Fprintf(w, "%s%s\t%s\t%d\n", indent, kindName(s.ItemKind), priorityName(s.Priority), s.Count)
}
fmt.Fprintf(w, "%sTOTAL\t\t%d\n", indent, total)
}
func kindName(prefix string) string {
if k, ok := model.ParseKind(prefix); ok {
return k.String()
@ -252,22 +288,44 @@ func kindName(prefix string) string {
return prefix
}
type artworkPriority struct {
name string
value int
}
// knownPriorities is the one listing behind both the name and the parse, so they cannot drift.
var knownPriorities = []artworkPriority{
{"bump", model.ArtworkPriorityBump},
{"scan", model.ArtworkPriorityScan},
{"backfill", model.ArtworkPriorityBackfill},
{"recheck", model.ArtworkPriorityRecheck},
}
// priorityName falls back to the number: a row written by a newer version still has to print.
func priorityName(p int) string {
switch p {
case model.ArtworkPriorityRecheck:
return "recheck"
case model.ArtworkPriorityBackfill:
return "backfill"
case model.ArtworkPriorityScan:
return "scan"
case model.ArtworkPriorityBump:
return "bump"
for _, ap := range knownPriorities {
if ap.value == p {
return ap.name
}
}
return strconv.Itoa(p)
}
func priorityNames() string {
return strings.Join(slice.Map(knownPriorities, func(ap artworkPriority) string { return ap.name }), ", ")
}
func parseArtworkPriority(s string) (int, error) {
for _, ap := range knownPriorities {
if ap.name == s {
return ap.value, nil
}
}
return 0, fmt.Errorf("invalid priority %q, expected one of: %s", s, priorityNames())
}
func runReprocess(ctx context.Context) {
kinds, err := selectedKinds(reprocessKinds, reprocessSources, reprocessAll)
kinds, err := selectedKinds(artworkKinds, artworkSources, artworkAll)
if err != nil {
log.Fatal(ctx, err)
}
@ -281,11 +339,11 @@ func runReprocess(ctx context.Context) {
if needsImageAgents(kinds) {
mgr := loadPluginAgents(ctx, false)
defer func() { _ = mgr.Stop() }()
imageAgents = imageAgentCount(ds, mgr)
imageAgents = artwork.NewImageAgentCount(agents.GetAgents(ds, mgr))
}
if err := reprocessArtwork(ctx, ds, kinds, repositorySources(reprocessSources), imageAgents,
reprocessDryRun, reprocessConfirm(reprocessYes, os.Stdin), os.Stdout); err != nil {
if err := reprocessArtwork(ctx, ds, kinds, repositorySources(artworkSources), imageAgents,
artworkDryRun, confirmUnlessYes(artworkYes, os.Stdin, "re-resolve"), os.Stdout); err != nil {
log.Fatal(ctx, err)
}
}
@ -298,16 +356,9 @@ func selectedKinds(kinds, sources []string, all bool) ([]model.Kind, error) {
if len(kinds) == 0 {
return nil, fmt.Errorf("no selector given: pass --kind, --source or --all")
}
out := make([]model.Kind, 0, len(kinds))
for _, k := range kinds {
kind, err := parseArtworkKind(k, artwork.RecheckKinds)
if err != nil {
return nil, err
}
out = append(out, kind)
}
// A repeated kind would be counted twice, overstating the cost the operator confirms.
return slice.Unique(out), nil
return parseAll(kinds, func(s string) (model.Kind, error) {
return parseArtworkKind(s, artwork.RecheckKinds)
})
}
// absentSource is how the stored empty source — resolved, no image — is spelled on the CLI.
@ -326,11 +377,11 @@ func displaySource(s string) string { return cmp.Or(s, absentSource) }
type confirmFunc func(out io.Writer, total, external int64) bool
func reprocessConfirm(yes bool, in io.Reader) confirmFunc {
func confirmUnlessYes(yes bool, in io.Reader, verb string) confirmFunc {
if yes {
return func(io.Writer, int64, int64) bool { return true }
}
return promptConfirm(in)
return promptConfirm(in, verb)
}
// externalEstimate claims no bound: a local hit ends the walk before any agent is asked, and the
@ -346,11 +397,6 @@ func externalLookupLine(n int64) string {
return fmt.Sprintf("External lookups: %s.", externalEstimate(n))
}
func imageAgentCount(ds model.DataStore, mgr *plugins.Manager) artwork.ImageAgentCount {
ag := agents.GetAgents(ds, mgr)
return artwork.ImageAgentCount{Artist: len(ag.ArtistImageAgents()), Album: len(ag.AlbumImageAgents())}
}
// loadPluginAgents loads the plugins named in Agents, so the CLI resolves through the same agents a
// running server would. A load failure is reported, not fatal: the built-in agents still answer.
func loadPluginAgents(ctx context.Context, runInit bool) *plugins.Manager {
@ -378,13 +424,13 @@ func configuredAgents() []string {
return names
}
func promptConfirm(in io.Reader) confirmFunc {
func promptConfirm(in io.Reader, verb string) confirmFunc {
return func(out io.Writer, total, external int64) bool {
var cost string
if external > 0 {
cost = fmt.Sprintf(" %s", externalLookupLine(external))
}
fmt.Fprintf(out, "\nThis will re-resolve %d items.%s Continue? [y/N] ", total, cost)
fmt.Fprintf(out, "\nThis will %s %d items.%s Continue? [y/N] ", verb, total, cost)
var answer string
if _, err := fmt.Fscanln(in, &answer); err != nil {
return false
@ -476,6 +522,90 @@ func reprocessArtwork(ctx context.Context, ds model.DataStore, kinds []model.Kin
return nil
}
func runCancel(ctx context.Context) {
kinds, priorities, err := cancelSelection(artworkKinds, artworkPriorities, artworkAll)
if err != nil {
log.Fatal(ctx, err)
}
defer db.Init(ctx)()
ds, ctx := getAdminContext(ctx)
if err := cancelArtwork(ctx, ds, kinds, priorities, artworkDryRun,
confirmUnlessYes(artworkYes, os.Stdin, "cancel"), os.Stdout); err != nil {
log.Fatal(ctx, err)
}
}
// cancelSelection leaves --all as the empty filter the repository reads as "every one", so a row
// whose kind this build does not know still gets cancelled.
func cancelSelection(kinds, priorities []string, all bool) ([]model.Kind, []int, error) {
if all {
return nil, nil, nil
}
if len(kinds) == 0 && len(priorities) == 0 {
return nil, nil, fmt.Errorf("no selector given: pass --kind, --priority or --all")
}
// RefreshableKinds, not RecheckKinds: media files are queued, so --kind must reach them.
outKinds, err := parseAll(kinds, func(s string) (model.Kind, error) {
return parseArtworkKind(s, artwork.RefreshableKinds)
})
if err != nil {
return nil, nil, err
}
outPriorities, err := parseAll(priorities, parseArtworkPriority)
if err != nil {
return nil, nil, err
}
return outKinds, outPriorities, nil
}
// parseAll drops repeats: a doubled selector would overstate the total the operator confirms.
func parseAll[T comparable](values []string, parse func(string) (T, error)) ([]T, error) {
out := make([]T, 0, len(values))
for _, v := range values {
parsed, err := parse(v)
if err != nil {
return nil, err
}
out = append(out, parsed)
}
return slice.Unique(out), nil
}
func cancelArtwork(ctx context.Context, ds model.DataStore, kinds []model.Kind, priorities []int,
dryRun bool, confirm confirmFunc, out io.Writer) error {
q := ds.ArtworkQueue(ctx)
matched, err := q.CountQueued(kinds, priorities)
if err != nil {
return fmt.Errorf("counting queued artwork: %w", err)
}
total := queueTotal(matched)
w := newTabWriter(out)
printQueueStats(w, matched, total, "MATCHED", "")
w.Flush()
switch {
case total == 0:
fmt.Fprintln(out, "\nNothing matches this selection.")
return nil
case dryRun:
fmt.Fprintln(out, "\nDry run: nothing was cancelled.")
return nil
case !confirm(out, total, 0):
fmt.Fprintln(out, "Aborted: nothing was cancelled.")
return nil
}
cancelled, err := q.PurgeQueued(kinds, priorities)
if err != nil {
return fmt.Errorf("cancelling queued artwork: %w", err)
}
// Count and delete are separate statements, so a drain in between makes these two differ.
fmt.Fprintf(out, "Cancelled %d of %d matched items.\n", cancelled, total)
return nil
}
// printReprocessPreview also states the external estimate, which --dry-run must show because it
// skips the prompt that would otherwise carry it.
func printReprocessPreview(out io.Writer, kinds []model.Kind, matched []int64, total, external int64, sources []string) {
@ -518,7 +648,7 @@ func refreshItems(ctx context.Context, ds model.DataStore, targets []model.Artwo
for _, t := range targets {
kind, id := t.Kind, t.ID
// artwork.Refresh would happily queue an id that does not exist, orphaning a queue row.
if _, err := artworkItemName(ctx, ds, kind, id); err != nil {
if _, err := artwork.ItemName(ctx, ds, kind, id); err != nil {
log.Error(ctx, "Item not found", "kind", kind, "id", id, err)
failed++
continue
@ -541,7 +671,7 @@ var explainKinds = []model.Kind{
}
func kindPrefixes(kinds []model.Kind) string {
return strings.Join(slice.Map(kinds, func(k model.Kind) string { return k.Prefix() }), ", ")
return strings.Join(model.KindPrefixes(kinds), ", ")
}
func parseArtworkKind(s string, valid []model.Kind) (model.Kind, error) {
@ -640,10 +770,6 @@ func explainResult(source string, steps []artwork.TraceStep) string {
if s.Outcome == artwork.OutcomeHit {
break
}
if s.Outcome == artwork.OutcomeWouldTry {
return "resolved from " + source +
" (offline: a higher-priority external candidate was not tried; re-run with --live)"
}
// An external winner discards the earlier error, so the resolver settles it with no retry.
if s.Outcome == artwork.OutcomeError && strings.HasPrefix(s.Candidate, artwork.ExternalPrefix) &&
!strings.HasPrefix(source, artwork.ExternalPrefix) {
@ -655,14 +781,12 @@ func explainResult(source string, steps []artwork.TraceStep) string {
}
for _, s := range steps {
switch {
case s.Outcome == artwork.OutcomeWouldTry:
return "indeterminate (external agents not called; re-run with --live)"
case s.Outcome == artwork.OutcomeError && strings.HasPrefix(s.Candidate, artwork.ExternalPrefix):
return "indeterminate (an external lookup failed; the item may resolve on a later attempt)"
// The worker treats an unreadable local candidate exactly as it treats a failed external one:
// it retries instead of settling absent, so the verdict must not read as a clean miss.
case s.Outcome == artwork.OutcomeUnreadable:
return "indeterminate (a candidate exists but could not be read; the worker retries rather than settling absent)"
// A stage error or an unreadable candidate means a source was found but not processed; the
// worker retries rather than settling absent, so neither reads as a clean miss.
case s.Outcome == artwork.OutcomeError, s.Outcome == artwork.OutcomeUnreadable:
return "indeterminate (a candidate was found but could not be processed; the worker retries rather than settling absent)"
}
}
return "not resolved"
@ -684,22 +808,55 @@ func explainConfig(kind model.Kind) (name, value string) {
}
type explainReport struct {
kind model.Kind
id string
name string
stored *model.ItemArtwork
queued *model.ArtworkQueueItem
agents string
kind model.Kind
id string
name string
stored *model.ItemArtwork
queued *model.ArtworkQueueItem
agents string
// steps is the chain walk: recorded when the item was resolved, or performed just now when walked.
steps []artwork.TraceStep
source string
walked bool
resolveErr error
}
// explainChainOrigin says whether the operator is reading history or a walk performed just now,
// since the two can disagree after a config change.
func explainChainOrigin(rep explainReport) string {
if rep.walked {
return "walked now"
}
if rep.stored != nil {
return "recorded " + formatTime(rep.stored.AttemptedAt)
}
return "not recorded"
}
// writeSteps prints the trace rows. An empty last cell would end tabwriter's column block and
// break the alignment, so a missing detail is rendered as a dash.
func writeSteps(w io.Writer, indent string, steps []artwork.TraceStep) {
for _, s := range steps {
fmt.Fprintf(w, "%s%s\t%s\t%s\n", indent, s.Candidate, s.Outcome, cmp.Or(s.Detail, "-"))
}
}
// writeStepTable prints a secondary trace, and nothing at all when there is none to show.
func writeStepTable(w io.Writer, title string, steps []artwork.TraceStep) {
if len(steps) == 0 {
return
}
// No tab on the title: it closes the preceding column block, so these rows align among themselves.
fmt.Fprintf(w, " %s:\n", title)
writeSteps(w, " ", steps)
}
func formatExplain(rep explainReport) string {
var sb strings.Builder
w := newTabWriter(&sb)
explainable := artwork.Explainable(rep.kind)
stateful := artwork.KeepsState(rep.kind)
unrecorded := !rep.walked && rep.stored == nil
fmt.Fprintln(w, "Item")
fmt.Fprintf(w, " Kind:\t%s (%s)\n", rep.kind, rep.kind.Prefix())
@ -732,6 +889,12 @@ func formatExplain(rep explainReport) string {
fmt.Fprintf(w, " Attempts:\t%d\n", rep.queued.Attempts)
fmt.Fprintf(w, " Retry at:\t%s\n", formatTime(rep.queued.RetryAt))
}
if rep.queued != nil {
writeStepTable(w, "Last attempt failed", artwork.DecodeTrace(rep.queued.Trace, ""))
}
if rep.stored != nil {
writeStepTable(w, "Gave up after", artwork.DecodeTrace(rep.stored.LastFailure, ""))
}
fmt.Fprintln(w, "\nConfig")
if setting, value := explainConfig(rep.kind); setting == "" {
@ -743,15 +906,22 @@ func formatExplain(rep explainReport) string {
}
}
fmt.Fprintln(w, "\nChain")
if !explainable {
fmt.Fprintf(w, "\nChain (%s)\n", explainChainOrigin(rep))
switch {
case !explainable:
fmt.Fprintf(w, " (%s artwork does not walk a priority chain)\n", rep.kind)
} else {
case unrecorded:
fmt.Fprintln(w, " (no resolution recorded yet; re-run with --live to walk the chain now)")
case !rep.walked && len(rep.steps) == 0 && rep.stored.Hash != "":
// A stored image with no chain can only predate trace recording: a recorded resolution that
// found an image always records its winning candidate.
fmt.Fprintln(w, " (this item was resolved before traces were recorded; re-run with --live)")
case !rep.walked && len(rep.steps) == 0:
// Absent with no chain: an empty priority list walked nothing, or a pre-tracing absent row.
fmt.Fprintln(w, " (no candidates were recorded; re-run with --live to walk the chain now)")
default:
fmt.Fprintln(w, " CANDIDATE\tOUTCOME\tDETAIL")
for _, s := range rep.steps {
// A row with an empty last cell would end tabwriter's column block, breaking alignment.
fmt.Fprintf(w, " %s\t%s\t%s\n", s.Candidate, s.Outcome, cmp.Or(s.Detail, "-"))
}
writeSteps(w, " ", rep.steps)
}
fmt.Fprintln(w, "\nResult")
@ -760,6 +930,8 @@ func formatExplain(rep explainReport) string {
fmt.Fprintf(w, " resolution failed: %s\n", rep.resolveErr)
case !explainable:
fmt.Fprintln(w, " not evaluated (no chain was walked; see Stored above)")
case unrecorded:
fmt.Fprintln(w, " not evaluated (nothing recorded; re-run with --live to walk the chain now)")
default:
fmt.Fprintf(w, " %s\n", explainResult(rep.source, rep.steps))
}
@ -791,7 +963,7 @@ func runExplain(ctx context.Context, args []string) {
}
kind, id := targets[0].Kind, targets[0].ID
name, err := artworkItemName(ctx, ds, kind, id)
name, err := artwork.ItemName(ctx, ds, kind, id)
if err != nil {
log.Fatal(ctx, "Item not found", "kind", kind, "id", id, err)
}
@ -807,6 +979,8 @@ func runExplain(ctx context.Context, args []string) {
}
}
// Disc artwork keeps no row, so it has no stored trace and can only be explained by walking now.
rep.walked = explainLive || !artwork.KeepsState(kind)
if artwork.Explainable(kind) {
// Only artist and album reach an agent, and the load must precede the resolver, which reads
// the same manager.
@ -815,71 +989,19 @@ func runExplain(ctx context.Context, args []string) {
defer func() { _ = mgr.Stop() }()
rep.agents = explainAgents(conf.Server.Agents, availableImageAgents(ds, mgr, kind))
}
trace := &artwork.ChainTrace{}
rep.source, rep.resolveErr = CreateArtworkResolver(trace, explainLive).Resolve(ctx, kind, id)
rep.steps = trace.Steps()
switch {
case rep.walked:
trace := &artwork.ChainTrace{}
rep.source, rep.resolveErr = CreateArtworkResolver(trace, explainLive).Resolve(ctx, kind, id)
rep.steps = trace.Steps()
case rep.stored != nil:
rep.steps = artwork.DecodeTrace(rep.stored.Trace, rep.stored.SourcePath)
rep.source = rep.stored.Source
}
}
fmt.Print(formatExplain(rep))
// The steps taken before a failed walk are the diagnosis, so report them before exiting.
if rep.resolveErr != nil {
log.Fatal(ctx, "Failed to resolve artwork", "kind", kind, "id", id, rep.resolveErr)
}
}
// artworkItemName looks the entity up under its own kind, so a mismatched kind/id pair is
// reported as not found instead of silently explaining another entity's artwork.
func artworkItemName(ctx context.Context, ds model.DataStore, kind model.Kind, id string) (string, error) {
switch kind {
case model.KindArtistArtwork:
ar, err := ds.Artist(ctx).Get(id)
if err != nil {
return "", err
}
return ar.Name, nil
case model.KindAlbumArtwork:
al, err := ds.Album(ctx).Get(id)
if err != nil {
return "", err
}
return al.Name, nil
case model.KindPlaylistArtwork:
pls, err := ds.Playlist(ctx).Get(id)
if err != nil {
return "", err
}
return pls.Name, nil
case model.KindRadioArtwork:
rd, err := ds.Radio(ctx).Get(id)
if err != nil {
return "", err
}
return rd.Name, nil
case model.KindMediaFileArtwork:
mf, err := ds.MediaFile(ctx).Get(id)
if err != nil {
return "", err
}
return mf.Title, nil
case model.KindDiscArtwork:
return discArtworkName(ctx, ds, id)
}
return "", fmt.Errorf("unsupported kind %q", kind.Prefix())
}
func discArtworkName(ctx context.Context, ds model.DataStore, id string) (string, error) {
albumID, discNumber, err := model.ParseDiscArtworkID(id)
if err != nil {
return "", err
}
al, err := ds.Album(ctx).Get(albumID)
if err != nil {
return "", err
}
name := fmt.Sprintf("%s (disc %d)", al.Name, discNumber)
// The subtitle is itself a DiscArtPriority candidate, so name it where the chain can be read against it.
if subtitle := strings.TrimSpace(al.Discs[discNumber]); subtitle != "" {
name += ": " + subtitle
}
return name, nil
}

311
cmd/artwork_test.go
View File

@ -3,6 +3,7 @@ package cmd
import (
"context"
"errors"
"fmt"
"io"
"strings"
"time"
@ -145,6 +146,15 @@ var _ = Describe("explainResult", func() {
"the worker retries an unreadable candidate instead of settling absent, so this is not a clean miss")
})
It("reports indeterminate when a processing stage errored after a candidate was found", func() {
steps := []artwork.TraceStep{
{Candidate: "cover.*", Outcome: "hit", Detail: "/music/cover.jpg"},
{Candidate: "store", Outcome: "error", Detail: "disk full"},
}
Expect(explainResult("", steps)).To(ContainSubstring("indeterminate"),
"a stage error is a processing failure the worker retries, not a definitive miss")
})
It("does not qualify a hit that an earlier unreadable candidate preceded", func() {
// chainState.try stamps only the external error onto a hit and drops the local one, so the
// worker settles this as found; warning about it would be a false alarm.
@ -164,34 +174,6 @@ var _ = Describe("explainResult", func() {
"a failed network call is not evidence that the item has no artwork")
})
It("qualifies a win a skipped higher-priority external candidate could have taken", func() {
steps := []artwork.TraceStep{
{Candidate: "external:deezer", Outcome: "would-try"},
{Candidate: "artist.*", Outcome: "hit", Detail: "/music/artist.jpg"},
}
res := explainResult("artist.*", steps)
Expect(res).To(ContainSubstring("resolved from artist.*"))
Expect(res).To(ContainSubstring("--live"),
"offline, the winner is only the winner because the external tier was skipped")
})
It("does not qualify a win that no skipped candidate outranked", func() {
steps := []artwork.TraceStep{
{Candidate: "artist.*", Outcome: "hit"},
{Candidate: "external:deezer", Outcome: "would-try"},
}
Expect(explainResult("artist.*", steps)).To(Equal("resolved from artist.*"))
})
It("reports indeterminate when external agents were never called", func() {
steps := []artwork.TraceStep{
{Candidate: "artist.*", Outcome: "miss"},
{Candidate: "external:deezer", Outcome: "would-try"},
}
Expect(explainResult("", steps)).To(ContainSubstring("indeterminate"),
"an offline run must not claim an item is unresolvable when external agents were skipped")
})
It("qualifies a win a failed higher-priority external lookup could have taken", func() {
steps := []artwork.TraceStep{
{Candidate: "external:deezer", Outcome: "error", Detail: "context deadline exceeded"},
@ -252,9 +234,10 @@ var _ = Describe("formatExplain", func() {
id: "ar-1",
name: "Radiohead",
agents: "lastfm,spotify",
walked: true,
steps: []artwork.TraceStep{
{Candidate: "upload", Outcome: "skipped", Detail: "no uploaded image"},
{Candidate: "external:deezer", Outcome: "would-try"},
{Candidate: "external:deezer", Outcome: "error", Detail: "context deadline exceeded"},
},
source: "",
}
@ -267,7 +250,6 @@ var _ = Describe("formatExplain", func() {
Expect(out).To(ContainSubstring("ArtistArtPriority"))
Expect(out).To(ContainSubstring("lastfm,spotify"))
Expect(out).To(ContainSubstring("external:deezer"))
Expect(out).To(ContainSubstring("would-try"))
Expect(out).To(ContainSubstring("indeterminate"))
})
@ -304,7 +286,7 @@ var _ = Describe("formatExplain", func() {
out := formatExplain(rep)
Expect(out).To(ContainSubstring("resolution failed: no such directory"))
Expect(out).ToNot(ContainSubstring("indeterminate"))
Expect(out).To(ContainSubstring("would-try"), "the steps taken before the failure still print")
Expect(out).To(ContainSubstring("external:deezer"), "the steps taken before the failure still print")
})
It("says a kind that does not walk a chain has no chain, without an empty table", func() {
@ -329,6 +311,7 @@ var _ = Describe("formatExplain", func() {
kind: model.KindDiscArtwork, id: "al-1:2", name: "OK Computer (disc 2)",
steps: []artwork.TraceStep{{Candidate: "cover.jpg", Outcome: "hit", Detail: "/music/cover.jpg"}},
source: "folder",
walked: true,
}
out := formatExplain(rep)
@ -341,10 +324,74 @@ var _ = Describe("formatExplain", func() {
Expect(out).To(ContainSubstring("resolved from folder"))
})
Context("stored traces", func() {
BeforeEach(func() {
rep.walked = false
rep.steps = nil
})
It("labels a recorded chain with when it was recorded, not as a walk done now", func() {
attempted := time.Date(2026, 8, 13, 10, 0, 0, 0, time.UTC)
rep.stored = &model.ItemArtwork{Source: "folder", Hash: "abc", AttemptedAt: attempted}
rep.steps = []artwork.TraceStep{{Candidate: "artist.*", Outcome: "hit", Detail: "/music/artist.jpg"}}
rep.source = "folder"
out := formatExplain(rep)
Expect(out).To(ContainSubstring("Chain (recorded 2026-08-13T10:00:00Z)"))
Expect(out).To(ContainSubstring("/music/artist.jpg"))
Expect(out).To(ContainSubstring("resolved from folder"))
})
It("says so when the item has never been resolved", func() {
out := formatExplain(rep)
Expect(out).To(ContainSubstring("no resolution recorded yet"))
Expect(out).To(ContainSubstring("--live"))
Expect(out).ToNot(ContainSubstring("not resolved"),
"nothing was recorded, which is not the same as resolving to nothing")
})
It("distinguishes a row written before traces existed from one with an empty chain", func() {
rep.stored = &model.ItemArtwork{Source: "folder", Hash: "abc", AttemptedAt: time.Now()}
Expect(formatExplain(rep)).To(ContainSubstring("resolved before traces were recorded"))
})
It("does not call an absent row with an empty recorded chain a pre-tracing row", func() {
// An empty priority list records a real but empty chain and resolves absent; that is not a
// legacy row, so it must not be reported as resolved before tracing existed.
rep.stored = &model.ItemArtwork{Source: "", Hash: "", AttemptedAt: time.Now()}
out := formatExplain(rep)
Expect(out).ToNot(ContainSubstring("resolved before traces were recorded"))
Expect(out).To(ContainSubstring("no candidates were recorded"))
Expect(out).To(ContainSubstring("not resolved"), "the Result still reports the absence plainly")
})
It("prints why the last attempt failed and why it gave up", func() {
rep.queued = &model.ArtworkQueueItem{Priority: model.ArtworkPriorityScan, Attempts: 3,
Trace: `[{"c":"decode","o":"error","d":"bad header"}]`}
rep.stored = &model.ItemArtwork{Source: "folder", Hash: "abc", AttemptedAt: time.Now(),
LastFailure: `[{"c":"read","o":"error","d":"i/o timeout"}]`}
out := formatExplain(rep)
Expect(out).To(ContainSubstring("Last attempt failed"))
Expect(out).To(ContainSubstring("bad header"))
Expect(out).To(ContainSubstring("Gave up after"))
Expect(out).To(ContainSubstring("i/o timeout"))
})
It("omits the failure tables when there is no failure to report", func() {
out := formatExplain(rep)
Expect(out).ToNot(ContainSubstring("Last attempt failed"))
Expect(out).ToNot(ContainSubstring("Gave up after"))
})
})
It("reports the setting that governs media file artwork", func() {
conf.Server.EnableMediaFileCoverArt = false
rep = explainReport{
kind: model.KindMediaFileArtwork, id: "mf-1", name: "Airbag",
walked: true,
steps: []artwork.TraceStep{
{Candidate: "embedded", Outcome: "skipped", Detail: "EnableMediaFileCoverArt is off"},
},
@ -377,33 +424,6 @@ var _ = Describe("explainConfig", func() {
)
})
var _ = Describe("discArtworkName", func() {
var ds *tests.MockDataStore
BeforeEach(func() {
albumRepo := tests.CreateMockAlbumRepo()
albumRepo.SetData(model.Albums{{ID: "al-1", Name: "Sandinista!", Discs: model.Discs{2: "Side Three"}}})
ds = &tests.MockDataStore{MockedAlbum: albumRepo}
})
It("names the album, the disc and its subtitle", func() {
name, err := artworkItemName(context.Background(), ds, model.KindDiscArtwork, "al-1:2")
Expect(err).ToNot(HaveOccurred())
Expect(name).To(Equal("Sandinista! (disc 2): Side Three"))
})
It("omits the subtitle when the disc has none", func() {
name, err := artworkItemName(context.Background(), ds, model.KindDiscArtwork, "al-1:1")
Expect(err).ToNot(HaveOccurred())
Expect(name).To(Equal("Sandinista! (disc 1)"))
})
It("rejects an id that is not <albumID>:<disc>", func() {
_, err := artworkItemName(context.Background(), ds, model.KindDiscArtwork, "al-1")
Expect(err).To(HaveOccurred())
})
})
var _ = Describe("artwork refresh command", func() {
It("requires at least one argument", func() {
Expect(artworkRefreshCmd.Args(artworkRefreshCmd, []string{})).To(HaveOccurred())
@ -503,36 +523,36 @@ var _ = Describe("promptConfirm", func() {
BeforeEach(func() { out.Reset() })
It("states the external cost and accepts an explicit yes", func() {
Expect(promptConfirm(strings.NewReader("y\n"))(&out, 42, 7)).To(BeTrue())
Expect(promptConfirm(strings.NewReader("y\n"), "re-resolve")(&out, 42, 7)).To(BeTrue())
Expect(out.String()).To(ContainSubstring("re-resolve 42 items"))
Expect(out.String()).To(ContainSubstring("External lookups: ~7 estimated"))
})
It("defaults to no on anything else", func() {
Expect(promptConfirm(strings.NewReader("\n"))(&out, 1, 1)).To(BeFalse())
Expect(promptConfirm(strings.NewReader("nope\n"))(&out, 1, 1)).To(BeFalse())
Expect(promptConfirm(strings.NewReader(""))(&out, 1, 1)).To(BeFalse())
Expect(promptConfirm(strings.NewReader("\n"), "re-resolve")(&out, 1, 1)).To(BeFalse())
Expect(promptConfirm(strings.NewReader("nope\n"), "re-resolve")(&out, 1, 1)).To(BeFalse())
Expect(promptConfirm(strings.NewReader(""), "re-resolve")(&out, 1, 1)).To(BeFalse())
})
It("drops the external clause when no lookup will be made", func() {
Expect(promptConfirm(strings.NewReader("y\n"))(&out, 3, 0)).To(BeTrue())
Expect(out.String()).To(ContainSubstring("re-resolve 3 items."))
Expect(promptConfirm(strings.NewReader("y\n"), "cancel")(&out, 3, 0)).To(BeTrue())
Expect(out.String()).To(ContainSubstring("cancel 3 items."))
Expect(out.String()).ToNot(ContainSubstring("External lookups"))
})
})
var _ = Describe("reprocessConfirm", func() {
var _ = Describe("confirmUnlessYes", func() {
var out strings.Builder
BeforeEach(func() { out.Reset() })
It("prompts when --yes was not given", func() {
Expect(reprocessConfirm(false, strings.NewReader("n\n"))(&out, 5, 5)).To(BeFalse())
Expect(confirmUnlessYes(false, strings.NewReader("n\n"), "re-resolve")(&out, 5, 5)).To(BeFalse())
Expect(out.String()).To(ContainSubstring("Continue?"))
})
It("bypasses the prompt only for --yes", func() {
Expect(reprocessConfirm(true, strings.NewReader(""))(&out, 5, 5)).To(BeTrue())
Expect(confirmUnlessYes(true, strings.NewReader(""), "re-resolve")(&out, 5, 5)).To(BeTrue())
Expect(out.String()).To(BeEmpty(), "--yes must not print a prompt it never reads")
})
})
@ -772,7 +792,7 @@ var _ = Describe("collectStatus", func() {
ImageType: model.ImageTypePrimary, Source: source, Hash: hash, AttemptedAt: attempted})).To(Succeed())
}
put(model.KindArtistArtwork, "ar-1", "external:deezer", "h1", time.Now())
put(model.KindArtistArtwork, "ar-2", "", "", time.Now().Add(-48*time.Hour))
put(model.KindArtistArtwork, "ar-2", "", "", time.Now().Add(-artwork.StaleAbsentAge-time.Hour))
put(model.KindArtistArtwork, "ar-3", "", "", time.Now())
put(model.KindAlbumArtwork, "al-1", "folder", "h2", time.Now())
Expect(queue.Enqueue(model.ArtworkQueueItem{ItemKind: "ar", ItemID: "ar-9",
@ -863,8 +883,9 @@ var _ = Describe("formatStatus", func() {
Expect(absent).To(MatchRegexp(`artist\s+2\s+1`))
})
It("states the recheck window the absent counts are bucketed against", func() {
Expect(formatStatus(rep)).To(ContainSubstring("24h"))
It("states the recheck window and the drip rate the absent counts are bucketed against", func() {
Expect(formatStatus(rep)).To(ContainSubstring(fmt.Sprintf("%gh", artwork.StaleAbsentAge.Hours())))
Expect(formatStatus(rep)).To(ContainSubstring("100 per kind per hour"))
})
It("leads with the queued backlog, which is the finding, not with the fingerprint verdict", func() {
@ -1013,3 +1034,153 @@ var _ = Describe("configuredAgents", func() {
Expect(configuredAgents()).To(BeEmpty())
})
})
var _ = Describe("parseArtworkPriority", func() {
It("accepts every name status prints", func() {
for _, p := range []int{model.ArtworkPriorityRecheck, model.ArtworkPriorityBackfill,
model.ArtworkPriorityScan, model.ArtworkPriorityBump} {
Expect(parseArtworkPriority(priorityName(p))).To(Equal(p))
}
})
It("rejects an unknown name and lists the valid ones", func() {
_, err := parseArtworkPriority("urgent")
Expect(err).To(MatchError(ContainSubstring(`invalid priority "urgent"`)))
Expect(err).To(MatchError(ContainSubstring("backfill")))
})
// Accepting the raw numbers would make the help text a lie and let a typo like 11 select nothing.
It("rejects the numeric form", func() {
_, err := parseArtworkPriority("10")
Expect(err).To(HaveOccurred())
})
})
var _ = Describe("artwork cancel selection", func() {
It("errors when no selector is given", func() {
_, _, err := cancelSelection(nil, nil, false)
Expect(err).To(MatchError(ContainSubstring("no selector given")))
})
// Empty, not an enumeration of the known kinds: --all must also take a queue row whose kind
// this build does not recognise.
It("selects with no filter at all for --all", func() {
kinds, priorities, err := cancelSelection(nil, nil, true)
Expect(err).ToNot(HaveOccurred())
Expect(kinds).To(BeEmpty())
Expect(priorities).To(BeEmpty())
})
// The queue holds media file rows, so --all must reach them.
It("accepts media file artwork, which reprocess does not", func() {
kinds, _, err := cancelSelection([]string{"mf"}, nil, false)
Expect(err).ToNot(HaveOccurred())
Expect(kinds).To(Equal([]model.Kind{model.KindMediaFileArtwork}))
})
It("treats a priority filter on its own as a complete selection", func() {
kinds, priorities, err := cancelSelection(nil, []string{"backfill"}, false)
Expect(err).ToNot(HaveOccurred())
Expect(kinds).To(BeEmpty(), "no kind filter means every kind")
Expect(priorities).To(Equal([]int{model.ArtworkPriorityBackfill}))
})
It("returns only the named kinds and priorities", func() {
kinds, priorities, err := cancelSelection([]string{"ar", "al"}, []string{"backfill", "scan"}, false)
Expect(err).ToNot(HaveOccurred())
Expect(kinds).To(Equal([]model.Kind{model.KindArtistArtwork, model.KindAlbumArtwork}))
Expect(priorities).To(Equal([]int{model.ArtworkPriorityBackfill, model.ArtworkPriorityScan}))
})
It("counts a repeated kind and a repeated priority once", func() {
kinds, priorities, err := cancelSelection([]string{"ar", "ar"}, []string{"bump", "bump"}, false)
Expect(err).ToNot(HaveOccurred())
Expect(kinds).To(HaveLen(1))
Expect(priorities).To(HaveLen(1))
})
It("rejects an unknown kind", func() {
_, _, err := cancelSelection([]string{"zz"}, nil, false)
Expect(err).To(MatchError(ContainSubstring(`invalid kind "zz"`)))
})
It("rejects a kind that is never queued", func() {
_, _, err := cancelSelection([]string{"dc"}, nil, false)
Expect(err).To(MatchError(ContainSubstring("invalid kind")))
})
It("rejects an unknown priority", func() {
_, _, err := cancelSelection(nil, []string{"urgent"}, false)
Expect(err).To(MatchError(ContainSubstring("invalid priority")))
})
})
var _ = Describe("cancelArtwork", func() {
var ds *tests.MockDataStore
var queue *tests.MockArtworkQueueRepo
var out strings.Builder
ctx := context.Background()
accept := func(io.Writer, int64, int64) bool { return true }
decline := func(io.Writer, int64, int64) bool { return false }
BeforeEach(func() {
ds = &tests.MockDataStore{}
queue = ds.ArtworkQueue(ctx).(*tests.MockArtworkQueueRepo)
out.Reset()
Expect(queue.Enqueue(
model.ArtworkQueueItem{ItemKind: "ar", ItemID: "ar-1", ImageType: model.ImageTypePrimary,
Priority: model.ArtworkPriorityBackfill},
model.ArtworkQueueItem{ItemKind: "ar", ItemID: "ar-2", ImageType: model.ImageTypePrimary,
Priority: model.ArtworkPriorityBump},
model.ArtworkQueueItem{ItemKind: "al", ItemID: "al-1", ImageType: model.ImageTypePrimary,
Priority: model.ArtworkPriorityBackfill},
)).To(Succeed())
})
It("previews the per-kind breakdown and cancels nothing on a dry run", func() {
Expect(cancelArtwork(ctx, ds, []model.Kind{model.KindArtistArtwork}, nil, true, accept, &out)).To(Succeed())
Expect(out.String()).To(ContainSubstring("artist"))
Expect(out.String()).To(ContainSubstring("backfill"))
Expect(out.String()).To(ContainSubstring("TOTAL"))
Expect(out.String()).To(ContainSubstring("Dry run"))
Expect(queue.Count()).To(BeNumerically("==", 3))
})
It("cancels nothing when the operator declines", func() {
Expect(cancelArtwork(ctx, ds, nil, nil, false, decline, &out)).To(Succeed())
Expect(out.String()).To(ContainSubstring("Aborted"))
Expect(queue.Count()).To(BeNumerically("==", 3))
})
It("deletes the selected rows and leaves the rest queued", func() {
Expect(cancelArtwork(ctx, ds, nil, []int{model.ArtworkPriorityBackfill}, false, accept, &out)).To(Succeed())
Expect(queue.Count()).To(BeNumerically("==", 1))
_, err := queue.Get(model.KindArtistArtwork, "ar-2", model.ImageTypePrimary)
Expect(err).ToNot(HaveOccurred(), "a non-matching priority must stay queued")
Expect(out.String()).To(ContainSubstring("Cancelled 2 of 2 matched items."))
})
It("cancels every kind and priority when neither filter is given", func() {
Expect(cancelArtwork(ctx, ds, nil, nil, false, accept, &out)).To(Succeed())
Expect(queue.Count()).To(BeZero())
})
It("stops at a selection that matches nothing instead of prompting", func() {
refuse := func(io.Writer, int64, int64) bool {
Fail("must not prompt when nothing matches")
return false
}
Expect(cancelArtwork(ctx, ds, []model.Kind{model.KindPlaylistArtwork}, nil, false, refuse, &out)).To(Succeed())
Expect(out.String()).To(ContainSubstring("Nothing matches this selection."))
Expect(queue.Count()).To(BeNumerically("==", 3))
})
It("reports a queue read failure instead of reporting nothing to cancel", func() {
queue.Err = errors.New("read failed")
Expect(cancelArtwork(ctx, ds, nil, nil, false, accept, &out)).To(MatchError(ContainSubstring("read failed")))
})
})

17
cmd/pls.go
View File

@ -6,6 +6,7 @@ import (
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"strconv"
@ -141,14 +142,16 @@ func findPlaylist(ctx context.Context, ds model.DataStore, nameOrID string) *mod
func runExporter(ctx context.Context) {
ds, ctx := getAdminContext(ctx)
playlist := findPlaylist(ctx, ds, playlistID)
pls := playlist.ToM3U8()
if outputFile == "-" || outputFile == "" {
println(pls)
writePlaylist(playlist.ToM3U8(), os.Stdout, outputFile)
}
func writePlaylist(m3u string, out io.Writer, file string) {
if file == "" || file == "-" {
fmt.Fprint(out, m3u)
return
}
err := os.WriteFile(outputFile, []byte(pls), 0600)
if err != nil {
log.Fatal("Error writing to the output file", "file", outputFile, err)
if err := os.WriteFile(file, []byte(m3u), 0600); err != nil {
log.Fatal("Error writing to the output file", "file", file, err)
}
}
@ -157,7 +160,7 @@ func runExport(ctx context.Context) {
if playlistID != "" && outputFile == "" {
playlist := findPlaylist(ctx, ds, playlistID)
println(playlist.ToM3U8())
writePlaylist(playlist.ToM3U8(), os.Stdout, outputFile)
return
}

35
cmd/pls_test.go Normal file
View File

@ -0,0 +1,35 @@
package cmd
import (
"fmt"
"os"
"path/filepath"
"strings"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
var _ = Describe("writePlaylist", func() {
const m3u = "#EXTM3U\n#PLAYLIST:DJ Wave\n#EXTINF:364,Bel Canto - Dreaming Girl\n"
plsFile := filepath.Join(os.TempDir(), fmt.Sprintf("navidrome-pls-%d.m3u8", os.Getpid()))
BeforeEach(func() {
DeferCleanup(func() { _ = os.Remove(plsFile) })
})
DescribeTable("writes the playlist to exactly one destination",
func(file, wantStream, wantFile string) {
var out strings.Builder
writePlaylist(m3u, &out, file)
written, _ := os.ReadFile(plsFile)
Expect(out.String()).To(Equal(wantStream))
Expect(string(written)).To(Equal(wantFile))
},
Entry("no file name writes to the stream", "", m3u, ""),
Entry("a dash writes to the stream", "-", m3u, ""),
Entry("a path writes to the file", plsFile, "", m3u),
)
})

9
cmd/wire_gen.go
View File

@ -76,7 +76,10 @@ func CreateNativeAPIRouter(ctx context.Context) *nativeapi.Router {
library := core.NewLibrary(dataStore, modelScanner, watcher, broker, manager)
user := core.NewUser(dataStore, manager)
maintenance := core.NewMaintenance(dataStore)
router := nativeapi.New(dataStore, share, playlistsPlaylists, insights, library, user, maintenance, manager, uploader)
agentsAgents := agents.GetAgents(dataStore, manager)
matcherMatcher := matcher.New(dataStore)
provider := external.NewProvider(dataStore, agentsAgents, matcherMatcher, broker)
router := nativeapi.New(dataStore, share, playlistsPlaylists, insights, library, user, maintenance, manager, uploader, provider)
return router
}
@ -97,7 +100,7 @@ func CreateSubsonicAPIRouter(ctx context.Context) *subsonic.Router {
manager := plugins.GetManager(dataStore, broker, metricsMetrics)
agentsAgents := agents.GetAgents(dataStore, manager)
matcherMatcher := matcher.New(dataStore)
provider := external.NewProvider(dataStore, agentsAgents, matcherMatcher)
provider := external.NewProvider(dataStore, agentsAgents, matcherMatcher, broker)
uploader := artwork.NewUploader(dataStore)
playlistsPlaylists := playlists.NewPlaylists(dataStore, uploader)
modelScanner := scanner.New(ctx, dataStore, broker, playlistsPlaylists, metricsMetrics)
@ -129,7 +132,7 @@ func CreateJellyfinAPIRouter(ctx context.Context) *jellyfin.Router {
playlistsPlaylists := playlists.NewPlaylists(dataStore, uploader)
agentsAgents := agents.GetAgents(dataStore, manager)
matcherMatcher := matcher.New(dataStore)
provider := external.NewProvider(dataStore, agentsAgents, matcherMatcher)
provider := external.NewProvider(dataStore, agentsAgents, matcherMatcher, broker)
sonicSonic := sonic.New(dataStore, manager, matcherMatcher)
lyricsLyrics := lyrics.NewLyrics(dataStore, manager)
router := jellyfin.New(dataStore, artworkArtwork, mediaStreamer, transcodeDecider, players, playTracker, playlistsPlaylists, provider, sonicSonic, lyricsLyrics, broker)

2
conf/configuration.go
View File

@ -73,6 +73,7 @@ type configOptions struct {
Matcher matcherOptions `json:",omitzero"`
RecentlyAddedByModTime bool
PreferSortTags bool
EnableNaturalSorting bool
IgnoredArticles string
IndexGroups string
FFmpegPath string
@ -973,6 +974,7 @@ func setViperDefaults() {
viper.SetDefault("matcher.fuzzythreshold", 85)
viper.SetDefault("recentlyaddedbymodtime", false)
viper.SetDefault("prefersorttags", false)
viper.SetDefault("enablenaturalsorting", false)
viper.SetDefault("ignoredarticles", "The El La Los Las Le Les Os As O A")
viper.SetDefault("indexgroups", "A B C D E F G H I J K L M N O P Q R S T U V W X-Z(XYZ) [Unknown]([)")
viper.SetDefault("ffmpegpath", "")

2
consts/consts.go
View File

@ -201,7 +201,7 @@ var (
}
)
var HTTPUserAgent = "Navidrome" + "/" + Version
var HTTPUserAgent = "Navidrome/" + Version + " - https://github.com/navidrome"
var (
VariousArtists = "Various Artists"

154
core/agents/agents.go
View File

@ -1,9 +1,13 @@
package agents
import (
"cmp"
"context"
"errors"
"maps"
"slices"
"strings"
"sync"
"time"
"github.com/navidrome/navidrome/conf"
@ -22,11 +26,43 @@ type PluginLoader interface {
LoadMediaAgent(name string) (Interface, bool)
}
// agentCooldown is the default cooldown duration for an agent that returns a RetryLaterError without a specific
// RetryIn duration.
const agentCooldown = time.Minute
// errUnsupported marks an agent that does not implement the requested method: it never ran,
// so it neither answered nor throttled.
var errUnsupported = errors.New("agent does not support this method")
// Agents is a meta-agent that aggregates multiple built-in and plugin agents. It tries each enabled agent in order
// until one returns valid data.
type Agents struct {
ds model.DataStore
pluginLoader PluginLoader
cooldowns cooldowns
}
// cooldowns remembers, across dispatches, which agents asked to be left alone and until when.
type cooldowns struct {
mu sync.RWMutex
until map[string]time.Time
}
func (c *cooldowns) active(name string) bool {
c.mu.RLock()
defer c.mu.RUnlock()
return time.Now().Before(c.until[name])
}
// park keeps whichever deadline is later, so a call still in flight when a longer cooldown
// starts cannot cut it short when it finally answers.
func (c *cooldowns) park(name string, d time.Duration) {
until := time.Now().Add(d)
c.mu.Lock()
defer c.mu.Unlock()
if until.After(c.until[name]) {
c.until[name] = until
}
}
// GetAgents returns the singleton instance of Agents
@ -41,6 +77,7 @@ func createAgents(ds model.DataStore, pluginLoader PluginLoader) *Agents {
return &Agents{
ds: ds,
pluginLoader: pluginLoader,
cooldowns: cooldowns{until: map[string]time.Time{}},
}
}
@ -90,12 +127,19 @@ func (a *Agents) getEnabledAgentNames() []enabledAgent {
} else if isPlugin {
validAgents = append(validAgents, enabledAgent{name: name, isPlugin: true})
} else {
log.Debug("Unknown agent ignored", "name", name)
log.Debug("Unknown agent ignored", "name", name, "available", availableAgentNames(availablePlugins))
}
}
return validAgents
}
// availableAgentNames returns every name accepted by the Agents config option.
func availableAgentNames(plugins []string) []string {
names := append(slices.Collect(maps.Keys(Map)), plugins...)
slices.Sort(names)
return names
}
func (a *Agents) getAgent(ea enabledAgent) Interface {
if ea.isPlugin {
// Try to load WASM plugin agent (if plugin loader is available)
@ -171,7 +215,7 @@ func (a *Agents) GetArtistMBID(ctx context.Context, id string, name string) (str
return callAgentMethod(ctx, a, "GetArtistMBID", func(ag Interface) (string, error) {
retriever, ok := ag.(ArtistMBIDRetriever)
if !ok {
return "", ErrNotFound
return "", errUnsupported
}
return retriever.GetArtistMBID(ctx, id, name)
})
@ -188,7 +232,7 @@ func (a *Agents) GetArtistURL(ctx context.Context, id, name, mbid string) (strin
return callAgentMethod(ctx, a, "GetArtistURL", func(ag Interface) (string, error) {
retriever, ok := ag.(ArtistURLRetriever)
if !ok {
return "", ErrNotFound
return "", errUnsupported
}
return retriever.GetArtistURL(ctx, id, name, mbid)
})
@ -205,7 +249,7 @@ func (a *Agents) GetArtistBiography(ctx context.Context, id, name, mbid string)
return callAgentMethod(ctx, a, "GetArtistBiography", func(ag Interface) (string, error) {
retriever, ok := ag.(ArtistBiographyRetriever)
if !ok {
return "", ErrNotFound
return "", errUnsupported
}
return retriever.GetArtistBiography(ctx, id, name, mbid)
})
@ -224,7 +268,11 @@ func (a *Agents) GetSimilarArtists(ctx context.Context, id, name, mbid string, l
overLimit := int(float64(limit) * conf.Server.DevExternalArtistFetchMultiplier)
start := time.Now()
attempts := newAttempts(&a.cooldowns)
for _, enabledAgent := range a.getEnabledAgentNames() {
if attempts.skip(enabledAgent.name) {
continue
}
ag := a.getAgent(enabledAgent)
if ag == nil {
continue
@ -237,6 +285,7 @@ func (a *Agents) GetSimilarArtists(ctx context.Context, id, name, mbid string, l
continue
}
similar, err := retriever.GetSimilarArtists(ctx, id, name, mbid, overLimit)
attempts.record(enabledAgent.name, err)
if len(similar) > 0 && err == nil {
if log.IsGreaterOrEqualTo(log.LevelTrace) {
log.Debug(ctx, "Got Similar Artists", "agent", ag.AgentName(), "artist", name, "similar", similar, "elapsed", time.Since(start))
@ -246,7 +295,7 @@ func (a *Agents) GetSimilarArtists(ctx context.Context, id, name, mbid string, l
return similar, err
}
}
return nil, ErrNotFound
return nil, attempts.noResultErr()
}
func (a *Agents) GetArtistImages(ctx context.Context, id, name, mbid string) ([]ExternalImage, error) {
@ -260,7 +309,7 @@ func (a *Agents) GetArtistImages(ctx context.Context, id, name, mbid string) ([]
return callAgentSliceMethod(ctx, a, "GetArtistImages", func(ag Interface) ([]ExternalImage, error) {
retriever, ok := ag.(ArtistImageRetriever)
if !ok {
return nil, ErrNotFound
return nil, errUnsupported
}
return retriever.GetArtistImages(ctx, id, name, mbid)
})
@ -281,7 +330,7 @@ func (a *Agents) GetArtistTopSongs(ctx context.Context, id, artistName, mbid str
return callAgentSliceMethod(ctx, a, "GetArtistTopSongs", func(ag Interface) ([]Song, error) {
retriever, ok := ag.(ArtistTopSongsRetriever)
if !ok {
return nil, ErrNotFound
return nil, errUnsupported
}
return retriever.GetArtistTopSongs(ctx, id, artistName, mbid, overLimit)
})
@ -295,7 +344,7 @@ func (a *Agents) GetAlbumInfo(ctx context.Context, name, artist, mbid string) (*
return callAgentMethod(ctx, a, "GetAlbumInfo", func(ag Interface) (*AlbumInfo, error) {
retriever, ok := ag.(AlbumInfoRetriever)
if !ok {
return nil, ErrNotFound
return nil, errUnsupported
}
return retriever.GetAlbumInfo(ctx, name, artist, mbid)
})
@ -309,7 +358,7 @@ func (a *Agents) GetAlbumImages(ctx context.Context, name, artist, mbid string)
return callAgentSliceMethod(ctx, a, "GetAlbumImages", func(ag Interface) ([]ExternalImage, error) {
retriever, ok := ag.(AlbumImageRetriever)
if !ok {
return nil, ErrNotFound
return nil, errUnsupported
}
return retriever.GetAlbumImages(ctx, name, artist, mbid)
})
@ -320,7 +369,7 @@ func (a *Agents) GetSimilarSongsByTrack(ctx context.Context, id, name, artist, m
return callAgentSliceMethod(ctx, a, "GetSimilarSongsByTrack", func(ag Interface) ([]Song, error) {
retriever, ok := ag.(SimilarSongsByTrackRetriever)
if !ok {
return nil, ErrNotFound
return nil, errUnsupported
}
return retriever.GetSimilarSongsByTrack(ctx, id, name, artist, mbid, count)
})
@ -331,7 +380,7 @@ func (a *Agents) GetSimilarSongsByAlbum(ctx context.Context, id, name, artist, m
return callAgentSliceMethod(ctx, a, "GetSimilarSongsByAlbum", func(ag Interface) ([]Song, error) {
retriever, ok := ag.(SimilarSongsByAlbumRetriever)
if !ok {
return nil, ErrNotFound
return nil, errUnsupported
}
return retriever.GetSimilarSongsByAlbum(ctx, id, name, artist, mbid, count)
})
@ -349,16 +398,61 @@ func (a *Agents) GetSimilarSongsByArtist(ctx context.Context, id, name, mbid str
return callAgentSliceMethod(ctx, a, "GetSimilarSongsByArtist", func(ag Interface) ([]Song, error) {
retriever, ok := ag.(SimilarSongsByArtistRetriever)
if !ok {
return nil, ErrNotFound
return nil, errUnsupported
}
return retriever.GetSimilarSongsByArtist(ctx, id, name, mbid, count)
})
}
func callAgentMethod[T comparable](ctx context.Context, agents *Agents, methodName string, fn func(Interface) (T, error)) (T, error) {
// agentAttempts tallies what the enabled agents did in one dispatch.
type agentAttempts struct {
cooldowns *cooldowns
throttled bool
answered bool
}
func newAttempts(c *cooldowns) agentAttempts {
return agentAttempts{cooldowns: c}
}
// skip reports whether name is still cooling down, counting it as throttled for this dispatch.
func (t *agentAttempts) skip(name string) bool {
if !t.cooldowns.active(name) {
return false
}
t.throttled = true
return true
}
// record files one agent's outcome, parking it when it asked to be retried later.
func (t *agentAttempts) record(name string, err error) {
switch retry, isRetryLater := errors.AsType[*RetryLaterError](err); {
case errors.Is(err, errUnsupported):
case isRetryLater:
t.cooldowns.park(name, cmp.Or(retry.RetryIn, agentCooldown))
t.throttled = true
default:
t.answered = true
}
}
// noResultErr tells a retryable empty dispatch (nobody answered) from a definitive miss.
func (t *agentAttempts) noResultErr() error {
if t.throttled && !t.answered {
return ErrRetryLater
}
return ErrNotFound
}
// callAgent tries each enabled agent in order until found reports a usable result.
func callAgent[T any](ctx context.Context, agents *Agents, methodName string, fn func(Interface) (T, error), found func(T) bool) (T, error) {
var zero T
start := time.Now()
attempts := newAttempts(&agents.cooldowns)
for _, enabledAgent := range agents.getEnabledAgentNames() {
if attempts.skip(enabledAgent.name) {
continue
}
ag := agents.getAgent(enabledAgent)
if ag == nil {
continue
@ -367,41 +461,29 @@ func callAgentMethod[T comparable](ctx context.Context, agents *Agents, methodNa
break
}
result, err := fn(ag)
attempts.record(enabledAgent.name, err)
if err != nil {
log.Trace(ctx, "Agent method call error", "method", methodName, "agent", ag.AgentName(), "error", err)
continue
}
if result != zero {
if found(result) {
log.Debug(ctx, "Got result", "method", methodName, "agent", ag.AgentName(), "elapsed", time.Since(start))
return result, nil
}
}
return zero, ErrNotFound
return zero, attempts.noResultErr()
}
func callAgentMethod[T comparable](ctx context.Context, agents *Agents, methodName string, fn func(Interface) (T, error)) (T, error) {
return callAgent(ctx, agents, methodName, fn, func(result T) bool {
var zero T
return result != zero
})
}
func callAgentSliceMethod[T any](ctx context.Context, agents *Agents, methodName string, fn func(Interface) ([]T, error)) ([]T, error) {
start := time.Now()
for _, enabledAgent := range agents.getEnabledAgentNames() {
ag := agents.getAgent(enabledAgent)
if ag == nil {
continue
}
if utils.IsCtxDone(ctx) {
break
}
results, err := fn(ag)
if err != nil {
log.Trace(ctx, "Agent method call error", "method", methodName, "agent", ag.AgentName(), "error", err)
continue
}
if len(results) > 0 {
log.Debug(ctx, "Got results", "method", methodName, "agent", ag.AgentName(), "count", len(results), "elapsed", time.Since(start))
return results, nil
}
}
return nil, ErrNotFound
return callAgent(ctx, agents, methodName, fn, func(results []T) bool { return len(results) > 0 })
}
var _ Interface = (*Agents)(nil)

143
core/agents/agents_test.go
View File

@ -3,6 +3,8 @@ package agents
import (
"context"
"errors"
"slices"
"time"
"github.com/navidrome/navidrome/conf/configtest"
"github.com/navidrome/navidrome/consts"
@ -14,6 +16,29 @@ import (
. "github.com/onsi/gomega"
)
var _ = Describe("cooldowns", func() {
// Calls to one agent overlap, so a short cooldown can land after a long one started.
It("keeps the longer deadline when a shorter park lands after it", func() {
c := cooldowns{until: map[string]time.Time{}}
c.park("fake", time.Hour)
c.park("fake", time.Millisecond)
time.Sleep(10 * time.Millisecond)
Expect(c.active("fake")).To(BeTrue())
})
It("extends the deadline when the later park is longer", func() {
c := cooldowns{until: map[string]time.Time{}}
c.park("fake", time.Millisecond)
c.park("fake", time.Hour)
time.Sleep(10 * time.Millisecond)
Expect(c.active("fake")).To(BeTrue())
})
})
var _ = Describe("Agents", func() {
var ctx context.Context
var cancel context.CancelFunc
@ -67,6 +92,22 @@ var _ = Describe("Agents", func() {
Expect(ags).ToNot(ContainElement("disabled"))
})
Describe("availableAgentNames", func() {
It("combines built-in agents with the given plugins", func() {
names := availableAgentNames([]string{"apple-music"})
Expect(names).To(ContainElements("apple-music", LocalAgentName, "fake", "empty"))
})
It("returns the names sorted", func() {
names := availableAgentNames([]string{"zz-plugin", "aa-plugin"})
Expect(slices.IsSorted(names)).To(BeTrue())
})
It("works when there are no plugins", func() {
Expect(availableAgentNames(nil)).To(ContainElement(LocalAgentName))
})
})
Describe("GetArtistMBID", func() {
It("returns on first match", func() {
Expect(ag.GetArtistMBID(ctx, "123", "test")).To(Equal("mbid"))
@ -160,6 +201,102 @@ var _ = Describe("Agents", func() {
})
})
Describe("cooldown", func() {
It("skips an agent that returned RetryLaterError until the deadline", func() {
mock.Err = &RetryLaterError{RetryIn: time.Hour}
_, err := ag.GetArtistBiography(ctx, "id", "name", "mbid")
Expect(errors.Is(err, ErrRetryLater)).To(BeTrue())
// Immediately after: agent is skipped, not called
mock.Err = nil
calls := mock.Calls
_, err = ag.GetArtistBiography(ctx, "id", "name", "mbid")
Expect(mock.Calls).To(Equal(calls))
Expect(errors.Is(err, ErrRetryLater)).To(BeTrue())
})
// Providers that throttle without saying for how long (Last.fm sends no delay at all)
// must still be parked, or the aggregate keeps calling them on every request.
It("parks an agent that asked to be retried without a delay", func() {
mock.Err = ErrRetryLater
_, err := ag.GetArtistBiography(ctx, "id", "name", "mbid")
Expect(errors.Is(err, ErrRetryLater)).To(BeTrue())
mock.Err = nil
calls := mock.Calls
_, err = ag.GetArtistBiography(ctx, "id", "name", "mbid")
Expect(mock.Calls).To(Equal(calls), "the default cooldown must outlast the request")
Expect(errors.Is(err, ErrRetryLater)).To(BeTrue())
})
It("calls the agent again once the cooldown expires", func() {
mock.Err = &RetryLaterError{RetryIn: 10 * time.Millisecond}
_, err := ag.GetArtistBiography(ctx, "id", "name", "mbid")
Expect(errors.Is(err, ErrRetryLater)).To(BeTrue())
mock.Err = nil
Eventually(func() (string, error) {
return ag.GetArtistBiography(ctx, "id", "name", "mbid")
}, 5*time.Second, 10*time.Millisecond).Should(Equal("bio"))
})
It("returns ErrNotFound, not ErrRetryLater, when agents failed for other reasons", func() {
mock.Err = errors.New("boom")
_, err := ag.GetArtistBiography(ctx, "id", "name", "mbid")
Expect(errors.Is(err, ErrNotFound)).To(BeTrue())
Expect(errors.Is(err, ErrRetryLater)).To(BeFalse())
})
// ErrRetryLater tells the caller "nobody answered, do not cache this". A definitive
// answer from any other agent is an answer, throttled peer or not.
It("returns ErrNotFound when another agent answered with a definitive miss", func() {
other := &mockAgent{Err: ErrNotFound}
Register("fake2", func(model.DataStore) Interface { return other })
conf.Server.Agents = "fake,fake2"
ag = createAgents(ds, nil)
mock.Err = &RetryLaterError{RetryIn: time.Hour}
_, err := ag.GetArtistBiography(ctx, "id", "name", "mbid")
Expect(errors.Is(err, ErrNotFound)).To(BeTrue())
Expect(errors.Is(err, ErrRetryLater)).To(BeFalse())
// The cooldown was still recorded for the throttled agent
calls := mock.Calls
_, _ = ag.GetArtistBiography(ctx, "id", "name", "mbid")
Expect(mock.Calls).To(Equal(calls))
})
It("returns ErrNotFound when another agent answered with an empty slice", func() {
empty := &testImageAgent{Name: "emptyImages"}
Register("emptyImages", func(model.DataStore) Interface { return empty })
conf.Server.Agents = "fake,emptyImages"
ag = createAgents(ds, nil)
mock.Err = &RetryLaterError{RetryIn: time.Hour}
_, err := ag.GetArtistImages(ctx, "123", "test", "mb123")
Expect(errors.Is(err, ErrNotFound)).To(BeTrue())
Expect(errors.Is(err, ErrRetryLater)).To(BeFalse())
})
It("returns ErrRetryLater from GetSimilarArtists when only cooling agents remain", func() {
mock.Err = &RetryLaterError{RetryIn: time.Hour}
_, err := ag.GetSimilarArtists(ctx, "123", "test", "mb123", 2)
Expect(errors.Is(err, ErrRetryLater)).To(BeTrue())
})
It("returns ErrNotFound from GetSimilarArtists when another agent answered", func() {
other := &mockAgent{Err: ErrNotFound}
Register("fake2", func(model.DataStore) Interface { return other })
conf.Server.Agents = "fake,fake2"
ag = createAgents(ds, nil)
mock.Err = &RetryLaterError{RetryIn: time.Hour}
_, err := ag.GetSimilarArtists(ctx, "123", "test", "mb123", 2)
Expect(errors.Is(err, ErrNotFound)).To(BeTrue())
Expect(errors.Is(err, ErrRetryLater)).To(BeFalse())
})
})
Describe("GetArtistImages", func() {
It("returns on first match", func() {
Expect(ag.GetArtistImages(ctx, "123", "test", "mb123")).To(Equal([]ExternalImage{{
@ -423,8 +560,9 @@ var _ = Describe("Agents", func() {
})
type mockAgent struct {
Args []any
Err error
Args []any
Err error
Calls int
}
func (a *mockAgent) AgentName() string {
@ -449,6 +587,7 @@ func (a *mockAgent) GetArtistURL(_ context.Context, id, name, mbid string) (stri
func (a *mockAgent) GetArtistBiography(_ context.Context, id, name, mbid string) (string, error) {
a.Args = []any{id, name, mbid}
a.Calls++
if a.Err != nil {
return "", a.Err
}

51
core/agents/interfaces.go
View File

@ -3,6 +3,9 @@ package agents
import (
"context"
"errors"
"fmt"
"strconv"
"time"
"github.com/gohugoio/hashstructure"
"github.com/navidrome/navidrome/model"
@ -52,11 +55,49 @@ func (s Song) Equals(other Song) bool {
return h1 == h2
}
var (
// ErrNotFound means the provider answered and had nothing. Return the underlying error
// for a fault instead, or callers that back off on faults will treat it as definitive.
ErrNotFound = errors.New("not found")
)
// ErrNotFound means the provider answered and had nothing. Return the underlying error
// for a fault instead, or callers that back off on faults will treat it as definitive.
var ErrNotFound = errors.New("not found")
// ErrRetryLater is the zero-delay RetryLaterError: the provider is temporarily unavailable
// or throttling us, but did not say for how long. Both errors.Is(err, ErrRetryLater) and
// errors.AsType[*RetryLaterError] match it and every delay-carrying variant.
// Treat it as immutable; build a new RetryLaterError to name a delay.
var ErrRetryLater = &RetryLaterError{}
// RetryLaterError asks callers to back off, optionally for the delay the provider requested.
type RetryLaterError struct {
RetryIn time.Duration
}
func (e *RetryLaterError) Error() string {
if e.RetryIn > 0 {
return fmt.Sprintf("retry later (in %s)", e.RetryIn)
}
return "retry later"
}
func (e *RetryLaterError) Is(target error) bool {
_, ok := target.(*RetryLaterError)
return ok
}
// MaxRetryIn caps a delay parsed from a provider, so a bogus value cannot park it indefinitely.
const MaxRetryIn = time.Hour
const maxRetryInSeconds = int(MaxRetryIn / time.Second)
// ParseRetryIn reads a provider's delay given in seconds, from a header or a plugin token.
// Anything unparseable or non-positive means unspecified.
func ParseRetryIn(seconds string) time.Duration {
// Clamp in seconds: scaling first would wrap a huge value past int64 nanoseconds,
// turning "wait an age" into a fraction of a second. Parse at a fixed width so the
// cap holds on the 32-bit targets we ship, where a plain Atoi would overflow first.
secs, err := strconv.ParseInt(seconds, 10, 64)
if err != nil || secs <= 0 {
return 0
}
return time.Duration(min(secs, int64(maxRetryInSeconds))) * time.Second
}
// AlbumInfoRetriever provides album info (no images)
type AlbumInfoRetriever interface {

47
core/agents/interfaces_test.go
View File

@ -1,27 +1,42 @@
package agents
package agents_test
import (
"errors"
"fmt"
"time"
"github.com/navidrome/navidrome/core/agents"
"github.com/navidrome/navidrome/core/scrobbler"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
var _ = Describe("Song.Equals", func() {
base := Song{ID: "1", Name: "S", Artists: []Artist{{ID: "x", Name: "A"}}}
It("true for identical songs incl Artists", func() {
Expect(base.Equals(base)).To(BeTrue())
var _ = Describe("RetryLaterError", func() {
It("matches the ErrRetryLater sentinel via errors.Is", func() {
err := &agents.RetryLaterError{RetryIn: 30 * time.Second}
Expect(errors.Is(err, agents.ErrRetryLater)).To(BeTrue())
})
It("false when Artists differ", func() {
other := base
other.Artists = []Artist{{ID: "y", Name: "B"}}
Expect(base.Equals(other)).To(BeFalse())
It("matches through errors.Join and wrapping", func() {
err := fmt.Errorf("calling LB: %w", errors.Join(errors.New("http 429"), &agents.RetryLaterError{}))
Expect(errors.Is(err, agents.ErrRetryLater)).To(BeTrue())
})
It("false when a scalar differs", func() {
other := base
other.Name = "T"
Expect(base.Equals(other)).To(BeFalse())
It("exposes the delay through the wrapped error", func() {
err := errors.Join(errors.New("http 429"), &agents.RetryLaterError{RetryIn: 42 * time.Second})
retry, ok := errors.AsType[*agents.RetryLaterError](err)
Expect(ok).To(BeTrue())
Expect(retry.RetryIn).To(Equal(42 * time.Second))
})
It("true when both have empty Artists and equal scalars", func() {
a := Song{ID: "1", Name: "S"}
Expect(a.Equals(a)).To(BeTrue())
It("matches the sentinel too, reporting no delay", func() {
retry, ok := errors.AsType[*agents.RetryLaterError](agents.ErrRetryLater)
Expect(ok).To(BeTrue())
Expect(retry.RetryIn).To(BeZero())
})
It("is the same sentinel as scrobbler.ErrRetryLater", func() {
Expect(errors.Is(scrobbler.ErrRetryLater, agents.ErrRetryLater)).To(BeTrue())
Expect(errors.Is(&agents.RetryLaterError{}, scrobbler.ErrRetryLater)).To(BeTrue())
})
})

27
core/agents/song_test.go Normal file
View File

@ -0,0 +1,27 @@
package agents
import (
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
var _ = Describe("Song.Equals", func() {
base := Song{ID: "1", Name: "S", Artists: []Artist{{ID: "x", Name: "A"}}}
It("true for identical songs incl Artists", func() {
Expect(base.Equals(base)).To(BeTrue())
})
It("false when Artists differ", func() {
other := base
other.Artists = []Artist{{ID: "y", Name: "B"}}
Expect(base.Equals(other)).To(BeFalse())
})
It("false when a scalar differs", func() {
other := base
other.Name = "T"
Expect(base.Equals(other)).To(BeFalse())
})
It("true when both have empty Artists and equal scalars", func() {
a := Song{ID: "1", Name: "S"}
Expect(a.Equals(a)).To(BeTrue())
})
})

42
core/artwork/agent_images.go
View File

@ -2,6 +2,7 @@ package artwork
import (
"context"
"errors"
"io"
"net/url"
@ -41,24 +42,38 @@ func bestImageURL(imgs []agents.ExternalImage) *url.URL {
return best
}
// fetchArtistImage tries each enabled artist-image agent in order. extErr is true only when no
// longerRetry keeps whichever external failure asks for the longer wait, so one provider's
// short delay cannot shorten another's.
func longerRetry(a, b error) error {
if a == nil {
return b
}
var ra, rb *agents.RetryLaterError
if errors.As(b, &rb) && (!errors.As(a, &ra) || rb.RetryIn > ra.RetryIn) {
return b
}
return a
}
// fetchArtistImage tries each enabled artist-image agent in order. The error is non-nil only when no
// agent succeeded and at least one failed transiently.
func fetchArtistImage(ctx context.Context, ag *agents.Agents, gate gateFunc, ar model.Artist) (r io.ReadCloser, agentName string, extErr bool) {
func fetchArtistImage(ctx context.Context, ag *agents.Agents, gate gateFunc, ar model.Artist) (io.ReadCloser, string, error) {
// Synthetic artists would otherwise get an unrelated agent result assigned to them.
switch ar.ID {
case consts.UnknownArtistID, consts.VariousArtistsID:
traceFrom(ctx).add(TraceStep{Candidate: externalCandidate, Outcome: OutcomeSkipped, Detail: "synthetic artist"})
return nil, "", false
return nil, "", nil
}
name := externalName(ar.Name)
imageAgents := ag.ArtistImageAgents()
if len(imageAgents) == 0 {
traceFrom(ctx).add(TraceStep{Candidate: externalCandidate, Outcome: OutcomeSkipped,
Detail: "no enabled agent provides artist images"})
return nil, "", false
return nil, "", nil
}
var extErr error
for _, a := range imageAgents {
reader, _, err := gate(a.Name, func() (io.ReadCloser, string, error) {
reader, path, err := gate(a.Name, func() (io.ReadCloser, string, error) {
imgs, err := a.Retriever.GetArtistImages(ctx, ar.ID, name, ar.MbzArtistID)
if err != nil {
return nil, "", err
@ -69,11 +84,12 @@ func fetchArtistImage(ctx context.Context, ag *agents.Agents, gate gateFunc, ar
}
return fromURL(ctx, u)
})
recordAgent(ctx, a.Name, reader, path, err)
if reader != nil {
return reader, a.Name, false
return reader, a.Name, nil
}
if isTransientExternal(err) {
extErr = true
extErr = longerRetry(extErr, err)
log.Debug(ctx, "Artwork: External artist-image lookup failed", "agent", a.Name, "artist", ar.Name, err)
}
}
@ -81,16 +97,17 @@ func fetchArtistImage(ctx context.Context, ag *agents.Agents, gate gateFunc, ar
}
// fetchAlbumImage is the album counterpart of fetchArtistImage.
func fetchAlbumImage(ctx context.Context, ag *agents.Agents, gate gateFunc, al model.Album) (r io.ReadCloser, agentName string, extErr bool) {
func fetchAlbumImage(ctx context.Context, ag *agents.Agents, gate gateFunc, al model.Album) (io.ReadCloser, string, error) {
name, artist := externalName(al.Name), externalName(al.AlbumArtist)
imageAgents := ag.AlbumImageAgents()
if len(imageAgents) == 0 {
traceFrom(ctx).add(TraceStep{Candidate: externalCandidate, Outcome: OutcomeSkipped,
Detail: "no enabled agent provides album images"})
return nil, "", false
return nil, "", nil
}
var extErr error
for _, a := range imageAgents {
reader, _, err := gate(a.Name, func() (io.ReadCloser, string, error) {
reader, path, err := gate(a.Name, func() (io.ReadCloser, string, error) {
imgs, err := a.Retriever.GetAlbumImages(ctx, name, artist, al.MbzAlbumID)
if err != nil {
return nil, "", err
@ -101,11 +118,12 @@ func fetchAlbumImage(ctx context.Context, ag *agents.Agents, gate gateFunc, al m
}
return fromURL(ctx, u)
})
recordAgent(ctx, a.Name, reader, path, err)
if reader != nil {
return reader, a.Name, false
return reader, a.Name, nil
}
if isTransientExternal(err) {
extErr = true
extErr = longerRetry(extErr, err)
log.Debug(ctx, "Artwork: External album-image lookup failed", "agent", a.Name, "album", al.Name, err)
}
}

65
core/artwork/agent_images_test.go
View File

@ -2,11 +2,13 @@ package artwork
import (
"context"
"errors"
"io"
"net/http"
"net/http/httptest"
"strings"
"sync"
"time"
"github.com/navidrome/navidrome/conf"
"github.com/navidrome/navidrome/conf/configtest"
@ -153,11 +155,11 @@ var _ = Describe("agent images", func() {
a := &fakeImageAgent{name: "agentA", imgs: []agents.ExternalImage{img("/a", 100)}}
ag := imageAgents(a)
r, name, extErr := fetchArtistImage(ctx, ag, passthroughGate, model.Artist{ID: "ar1", Name: "Artist"})
r, name, err := fetchArtistImage(ctx, ag, passthroughGate, model.Artist{ID: "ar1", Name: "Artist"})
Expect(r).ToNot(BeNil())
defer r.Close()
Expect(name).To(Equal("agentA"))
Expect(extErr).To(BeFalse())
Expect(err).ToNot(HaveOccurred())
})
It("skips the external lookup for synthetic artists", func() {
@ -165,10 +167,10 @@ var _ = Describe("agent images", func() {
ag := imageAgents(a)
for _, id := range []string{consts.UnknownArtistID, consts.VariousArtistsID} {
r, name, extErr := fetchArtistImage(ctx, ag, passthroughGate, model.Artist{ID: id, Name: "Various Artists"})
r, name, err := fetchArtistImage(ctx, ag, passthroughGate, model.Artist{ID: id, Name: "Various Artists"})
Expect(r).To(BeNil())
Expect(name).To(BeEmpty())
Expect(extErr).To(BeFalse())
Expect(err).ToNot(HaveOccurred())
}
Expect(a.artistCalls).To(Equal(0), "synthetic artists never reach the agents")
})
@ -177,9 +179,9 @@ var _ = Describe("agent images", func() {
ag := imageAgents()
t := &ChainTrace{}
r, _, extErr := fetchArtistImage(withTrace(ctx, t), ag, passthroughGate, model.Artist{ID: "ar1"})
r, _, err := fetchArtistImage(withTrace(ctx, t), ag, passthroughGate, model.Artist{ID: "ar1"})
Expect(r).To(BeNil())
Expect(extErr).To(BeFalse())
Expect(err).ToNot(HaveOccurred())
Expect(t.Steps()).To(Equal([]TraceStep{{Candidate: "external", Outcome: OutcomeSkipped,
Detail: "no enabled agent provides artist images"}}),
"a configured external token must never be silently absent from the chain")
@ -211,11 +213,11 @@ var _ = Describe("agent images", func() {
b := &fakeImageAgent{name: "agentB", imgs: []agents.ExternalImage{img("/b", 50)}}
ag := imageAgents(a, b)
r, name, extErr := fetchArtistImage(ctx, ag, passthroughGate, model.Artist{ID: "ar1"})
r, name, err := fetchArtistImage(ctx, ag, passthroughGate, model.Artist{ID: "ar1"})
Expect(r).ToNot(BeNil())
defer r.Close()
Expect(name).To(Equal("agentB"))
Expect(extErr).To(BeFalse(), "a later hit clears an earlier agent's error")
Expect(err).ToNot(HaveOccurred(), "a later hit clears an earlier agent's error")
Expect(a.artistCalls).To(Equal(1))
Expect(b.artistCalls).To(Equal(1))
})
@ -225,20 +227,43 @@ var _ = Describe("agent images", func() {
b := &fakeImageAgent{name: "agentB", err: agents.ErrNotFound}
ag := imageAgents(a, b)
r, name, extErr := fetchArtistImage(ctx, ag, passthroughGate, model.Artist{ID: "ar1"})
r, name, err := fetchArtistImage(ctx, ag, passthroughGate, model.Artist{ID: "ar1"})
Expect(r).To(BeNil())
Expect(name).To(BeEmpty())
Expect(extErr).To(BeFalse(), "not-found is definitive, never a transient failure")
Expect(err).ToNot(HaveOccurred(), "not-found is definitive, never a transient failure")
})
It("reports extErr when one agent fails transiently and the rest find nothing", func() {
It("reports an error when one agent fails transiently and the rest find nothing", func() {
a := &fakeImageAgent{name: "agentA", err: agents.ErrNotFound}
b := &fakeImageAgent{name: "agentB", err: context.DeadlineExceeded}
ag := imageAgents(a, b)
r, _, extErr := fetchArtistImage(ctx, ag, passthroughGate, model.Artist{ID: "ar1"})
r, _, err := fetchArtistImage(ctx, ag, passthroughGate, model.Artist{ID: "ar1"})
Expect(r).To(BeNil())
Expect(extErr).To(BeTrue())
Expect(err).To(HaveOccurred())
})
// The worker reschedules on this delay, so it is only honored if the agent loop
// returns it. Two throttled agents: the longest wait is the one that must survive.
It("returns the longest retry delay the providers asked for", func() {
a := &fakeImageAgent{name: "agentA", err: &agents.RetryLaterError{RetryIn: 10 * time.Second}}
b := &fakeImageAgent{name: "agentB", err: &agents.RetryLaterError{RetryIn: 5 * time.Second}}
ag := imageAgents(a, b)
r, _, err := fetchArtistImage(ctx, ag, passthroughGate, model.Artist{ID: "ar1"})
Expect(r).To(BeNil())
retry, ok := errors.AsType[*agents.RetryLaterError](err)
Expect(ok).To(BeTrue())
Expect(retry.RetryIn).To(Equal(10 * time.Second))
})
It("returns no delay when the provider did not ask for one", func() {
ag := imageAgents(&fakeImageAgent{name: "agentA", err: errors.New("boom")})
_, _, err := fetchArtistImage(ctx, ag, passthroughGate, model.Artist{ID: "ar1"})
Expect(err).To(HaveOccurred())
_, ok := errors.AsType[*agents.RetryLaterError](err)
Expect(ok).To(BeFalse(), "a plain failure must not look like a throttle")
})
})
@ -247,11 +272,11 @@ var _ = Describe("agent images", func() {
a := &fakeImageAgent{name: "agentA", imgs: []agents.ExternalImage{img("/a", 100)}}
ag := imageAgents(a)
r, name, extErr := fetchAlbumImage(ctx, ag, passthroughGate, model.Album{Name: "Album", AlbumArtist: "Artist"})
r, name, err := fetchAlbumImage(ctx, ag, passthroughGate, model.Album{Name: "Album", AlbumArtist: "Artist"})
Expect(r).ToNot(BeNil())
defer r.Close()
Expect(name).To(Equal("agentA"))
Expect(extErr).To(BeFalse())
Expect(err).ToNot(HaveOccurred())
Expect(a.albumCalls).To(Equal(1))
})
@ -259,21 +284,21 @@ var _ = Describe("agent images", func() {
ag := imageAgents()
t := &ChainTrace{}
r, _, extErr := fetchAlbumImage(withTrace(ctx, t), ag, passthroughGate, model.Album{Name: "Album"})
r, _, err := fetchAlbumImage(withTrace(ctx, t), ag, passthroughGate, model.Album{Name: "Album"})
Expect(r).To(BeNil())
Expect(extErr).To(BeFalse())
Expect(err).ToNot(HaveOccurred())
Expect(t.Steps()).To(Equal([]TraceStep{{Candidate: "external", Outcome: OutcomeSkipped,
Detail: "no enabled agent provides album images"}}),
"a configured external token must never be silently absent from the chain")
})
It("reports extErr when the only agent fails transiently", func() {
It("reports an error when the only agent fails transiently", func() {
a := &fakeImageAgent{name: "agentA", err: context.DeadlineExceeded}
ag := imageAgents(a)
r, _, extErr := fetchAlbumImage(ctx, ag, passthroughGate, model.Album{Name: "Album"})
r, _, err := fetchAlbumImage(ctx, ag, passthroughGate, model.Album{Name: "Album"})
Expect(r).To(BeNil())
Expect(extErr).To(BeTrue())
Expect(err).To(HaveOccurred())
})
})

13
core/artwork/artwork.go
View File

@ -393,16 +393,15 @@ type TracingResolver struct {
trace *ChainTrace
}
// NewTracingResolver builds a TracingResolver that records its priority-chain walk. With live
// false the external tier is reported but never called.
// NewTracingResolver builds a TracingResolver that records its priority-chain walk. Without live
// it gets no agents at all, so neither a chain nor any fallback added later can reach a provider;
// with it, one item is at most one call per agent, so the rate limiter and breaker are bypassed.
func NewTracingResolver(ds model.DataStore, ag *agents.Agents, ffm ffmpeg.FFmpeg, t *ChainTrace, live bool) *TracingResolver {
gate := offlineGate(t)
inner := newLocalResolver(ds, ffm)
if live {
// A diagnostic must show the provider's real answer, and one item is at most one call
// per agent, so --live deliberately bypasses the rate limiter and circuit breaker.
gate = tracingGate(t, passthroughGate)
inner = newResolver(ds, ag, ffm, passthroughGate)
}
return &TracingResolver{inner: newResolver(ds, ag, ffm, gate), trace: t}
return &TracingResolver{inner: inner, trace: t}
}
// Resolve walks kind's sources for id, recording the walk, and reports the winning source

100
core/artwork/housekeeping.go
View File

@ -18,7 +18,11 @@ import (
)
// StaleAbsentAge is how long an absent state is trusted before a recheck retries it.
const StaleAbsentAge = 24 * time.Hour
const StaleAbsentAge = 30 * 24 * time.Hour
// StaleAbsentRecheckBatch caps how many absent states each hourly tick re-queues per kind,
// oldest first, so external agents see a flat drip instead of a daily burst.
const StaleAbsentRecheckBatch = 100
// RecheckKinds omits media files: they resolve embedded-only, at scan or on view.
var RecheckKinds = []model.Kind{
@ -68,18 +72,27 @@ func ConfigFingerprint() string {
return fmt.Sprintf("%016x", xxh3.Hash([]byte(raw)))
}
// backfillSummary is what a backfill enqueued. MaxExternalLookups is an upper estimate for one
// attempt per item, not a bound: a local hit ends the walk, and a retry asks the agents again.
type backfillSummary struct {
Ran bool
PerKind map[string]int64
Items int64
MaxExternalLookups int64
}
// backfill enqueues artwork resolution for every entity when the config fingerprint changed.
func backfill(ctx context.Context, ds model.DataStore) (bool, error) {
func backfill(ctx context.Context, ds model.DataStore, agentCount func() ImageAgentCount) (backfillSummary, error) {
start := time.Now()
ctx = auth.WithAdminUser(ctx, ds)
current := ConfigFingerprint()
props := ds.Property(ctx)
stored, err := props.DefaultGet(consts.ArtConfFingerprintPropertyKey, "")
if err != nil {
return false, err
return backfillSummary{}, err
}
if stored == current {
return false, nil
return backfillSummary{}, nil
}
// Artists first: few entities, most external-dependent, so they get a queue headstart.
@ -92,21 +105,31 @@ func backfill(ctx context.Context, ds model.DataStore) (bool, error) {
{model.KindPlaylistArtwork, func() ([]string, error) { return ds.Playlist(ctx).GetAllIDs() }},
{model.KindRadioArtwork, func() ([]string, error) { return ds.Radio(ctx).GetAllIDs() }},
}
// Counted here, not by the caller: building the agent list constructs every enabled agent, and
// an unchanged fingerprint returns above without ever needing the number.
agents := agentCount()
summary := backfillSummary{Ran: true, PerKind: map[string]int64{}}
for _, k := range kinds {
ids, err := k.fetch()
if err != nil {
return false, err
return backfillSummary{}, err
}
if err := enqueueBackfillKind(ctx, ds, k.kind, ids); err != nil {
return false, err
return backfillSummary{}, err
}
n := int64(len(ids))
summary.PerKind[k.kind.Prefix()] = n
summary.Items += n
summary.MaxExternalLookups += n * ExternalLookupsPerItem(k.kind, agents)
}
if err := props.Put(consts.ArtConfFingerprintPropertyKey, current); err != nil {
return false, err
return backfillSummary{}, err
}
log.Info(ctx, "Artwork: Config fingerprint changed, backfill enqueued", "elapsed", time.Since(start))
return true, nil
log.Info(ctx, "Artwork: Config fingerprint changed, backfill enqueued", "items", summary.Items,
"byKind", summary.PerKind, "maxExternalLookups", summary.MaxExternalLookups,
"elapsed", time.Since(start))
return summary, nil
}
func enqueueBackfillKind(ctx context.Context, ds model.DataStore, kind model.Kind, ids []string) error {
@ -125,7 +148,7 @@ func enqueueStaleAbsentAll(ctx context.Context, ds model.DataStore) error {
cutoff := time.Now().Add(-StaleAbsentAge)
queue := ds.ArtworkQueue(ctx)
for _, kind := range RecheckKinds {
if _, err := queue.EnqueueStaleAbsent(kind, cutoff); err != nil {
if _, err := queue.EnqueueStaleAbsent(kind, cutoff, StaleAbsentRecheckBatch); err != nil {
return err
}
}
@ -143,6 +166,63 @@ func enqueueMissingAll(ctx context.Context, ds model.DataStore) error {
return nil
}
// ItemName resolves a kind+id to the entity's display name, and errors when the item
// does not exist. Callers use it to reject ids that would otherwise orphan a queue row.
func ItemName(ctx context.Context, ds model.DataStore, kind model.Kind, id string) (string, error) {
switch kind {
case model.KindArtistArtwork:
ar, err := ds.Artist(ctx).Get(id)
if err != nil {
return "", err
}
return ar.Name, nil
case model.KindAlbumArtwork:
al, err := ds.Album(ctx).Get(id)
if err != nil {
return "", err
}
return al.Name, nil
case model.KindPlaylistArtwork:
pls, err := ds.Playlist(ctx).Get(id)
if err != nil {
return "", err
}
return pls.Name, nil
case model.KindRadioArtwork:
rd, err := ds.Radio(ctx).Get(id)
if err != nil {
return "", err
}
return rd.Name, nil
case model.KindMediaFileArtwork:
mf, err := ds.MediaFile(ctx).Get(id)
if err != nil {
return "", err
}
return mf.Title, nil
case model.KindDiscArtwork:
return discArtworkName(ctx, ds, id)
}
return "", fmt.Errorf("unsupported kind %q", kind.Prefix())
}
func discArtworkName(ctx context.Context, ds model.DataStore, id string) (string, error) {
albumID, discNumber, err := model.ParseDiscArtworkID(id)
if err != nil {
return "", err
}
al, err := ds.Album(ctx).Get(albumID)
if err != nil {
return "", err
}
name := fmt.Sprintf("%s (disc %d)", al.Name, discNumber)
// The subtitle is itself a DiscArtPriority candidate, so name it where the chain can be read against it.
if subtitle := strings.TrimSpace(al.Discs[discNumber]); subtitle != "" {
name += ": " + subtitle
}
return name, nil
}
// Refresh drops an item's resolved artwork state and re-queues it at Bump priority.
func Refresh(ctx context.Context, ds model.DataStore, kind model.Kind, id string) error {
if err := ds.Artwork(ctx).DeleteForItems(kind, []string{id}); err != nil {

111
core/artwork/housekeeping_test.go
View File

@ -2,6 +2,7 @@ package artwork
import (
"context"
"fmt"
"slices"
"time"
@ -38,6 +39,8 @@ func adminUserRepo() *tests.MockedUserRepo {
return repo
}
func noAgents() ImageAgentCount { return ImageAgentCount{} }
// orderTrackingQueueRepo records the item kind of each Enqueue call, so tests can
// assert phase ordering (artists-first) that same-priority timestamps can't guarantee.
type orderTrackingQueueRepo struct {
@ -163,9 +166,14 @@ var _ = Describe("Housekeeping", func() {
seedEntities()
Expect(propRepo.Put(consts.ArtConfFingerprintPropertyKey, ConfigFingerprint())).To(Succeed())
did, err := backfill(ctx, ds)
counted := false
s, err := backfill(ctx, ds, func() ImageAgentCount {
counted = true
return ImageAgentCount{Artist: 3, Album: 2}
})
Expect(err).ToNot(HaveOccurred())
Expect(did).To(BeFalse())
Expect(s).To(Equal(backfillSummary{}))
Expect(counted).To(BeFalse(), "building the agent list constructs every agent; an unchanged fingerprint must not pay for it")
count, err := queueRepo.Count()
Expect(err).ToNot(HaveOccurred())
@ -175,9 +183,9 @@ var _ = Describe("Housekeeping", func() {
It("runs the backfill when no fingerprint was ever stored", func() {
seedEntities()
did, err := backfill(ctx, ds)
s, err := backfill(ctx, ds, noAgents)
Expect(err).ToNot(HaveOccurred())
Expect(did).To(BeTrue())
Expect(s.Ran).To(BeTrue())
count, err := queueRepo.Count()
Expect(err).ToNot(HaveOccurred())
@ -196,9 +204,9 @@ var _ = Describe("Housekeeping", func() {
tracks: &tests.MockPlaylistTrackRepo{},
}
did, err := backfill(ctx, vds)
s, err := backfill(ctx, vds, noAgents)
Expect(err).ToNot(HaveOccurred())
Expect(did).To(BeTrue())
Expect(s.Ran).To(BeTrue())
Expect(findQueued(queueRepo.MockArtworkQueueRepo, "pl", "plPrivate")).ToNot(BeNil())
})
@ -206,9 +214,9 @@ var _ = Describe("Housekeeping", func() {
seedEntities()
Expect(propRepo.Put(consts.ArtConfFingerprintPropertyKey, "stale-fingerprint")).To(Succeed())
did, err := backfill(ctx, ds)
s, err := backfill(ctx, ds, noAgents)
Expect(err).ToNot(HaveOccurred())
Expect(did).To(BeTrue())
Expect(s.Ran).To(BeTrue())
Expect(queueRepo.callKinds).ToNot(BeEmpty())
firstOther := slices.IndexFunc(queueRepo.callKinds, func(k string) bool { return k != "ar" })
@ -223,6 +231,22 @@ var _ = Describe("Housekeeping", func() {
Expect(it.ItemKind).To(BeElementOf("ar", "al", "pl", "ra"))
}
})
It("reports what it enqueued, per kind and as an external-lookup ceiling", func() {
conf.Server.ArtistArtPriority = "artist.*, external"
conf.Server.CoverArtPriority = "cover.*, external"
conf.Server.EnableM3UExternalAlbumArt = false
seedEntities()
s, err := backfill(ctx, ds, func() ImageAgentCount { return ImageAgentCount{Artist: 3, Album: 2} })
Expect(err).ToNot(HaveOccurred())
Expect(s.Ran).To(BeTrue())
Expect(s.PerKind).To(Equal(map[string]int64{"ar": 2, "al": 1, "pl": 1, "ra": 1}))
Expect(s.Items).To(Equal(int64(5)))
// 2 artists x 3 agents, 1 album x 2, 1 playlist grid x 2, and radios never fetch.
Expect(s.MaxExternalLookups).To(Equal(int64(6 + 2 + PlaylistGridSamples*2)))
})
})
Describe("EnqueueStaleAbsentAll", func() {
@ -235,8 +259,8 @@ var _ = Describe("Housekeeping", func() {
})
It("enqueues only absent entries older than the recheck window, across all kinds", func() {
old := time.Now().Add(-48 * time.Hour)
recent := time.Now().Add(-time.Hour)
old := time.Now().Add(-StaleAbsentAge - time.Hour)
recent := time.Now().Add(-StaleAbsentAge + time.Hour)
artRepo.ItemData["ar-stale"] = model.ItemArtwork{ItemKind: "ar", ItemID: "ar1", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: old}
artRepo.ItemData["al-stale"] = model.ItemArtwork{ItemKind: "al", ItemID: "al1", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: old}
@ -259,6 +283,20 @@ var _ = Describe("Housekeeping", func() {
Expect(findQueued(queueRepo.MockArtworkQueueRepo, "ar", "ar2")).To(BeNil())
Expect(findQueued(queueRepo.MockArtworkQueueRepo, "al", "al2")).To(BeNil())
})
It("caps each tick at the recheck batch, oldest attempts first", func() {
for i := range StaleAbsentRecheckBatch + 1 {
id := fmt.Sprintf("ar%d", i)
artRepo.ItemData[id] = model.ItemArtwork{ItemKind: "ar", ItemID: id, ImageType: model.ImageTypePrimary,
Hash: "", AttemptedAt: time.Now().Add(-StaleAbsentAge - time.Duration(i+1)*time.Minute)}
}
Expect(enqueueStaleAbsentAll(ctx, ds)).To(Succeed())
Expect(queueRepo.Data).To(HaveLen(StaleAbsentRecheckBatch))
// ar0 has the newest attempted_at of the cohort, so it is the one left out.
Expect(findQueued(queueRepo.MockArtworkQueueRepo, "ar", "ar0")).To(BeNil())
})
})
Describe("EnqueueMissingAll", func() {
@ -294,3 +332,56 @@ var _ = Describe("Housekeeping", func() {
})
})
})
var _ = Describe("ItemName", func() {
var ds *tests.MockDataStore
var ctx context.Context
BeforeEach(func() {
ctx = context.Background()
albumRepo := tests.CreateMockAlbumRepo()
albumRepo.SetData(model.Albums{
{ID: "al-1", Name: "Kid A"},
{ID: "al-2", Name: "Sandinista!", Discs: model.Discs{2: "Side Three"}},
})
ds = &tests.MockDataStore{MockedAlbum: albumRepo}
Expect(ds.Artist(ctx).(*tests.MockArtistRepo).Put(&model.Artist{ID: "ar-1", Name: "Radiohead"})).To(Succeed())
})
It("returns the album name", func() {
Expect(ItemName(ctx, ds, model.KindAlbumArtwork, "al-1")).To(Equal("Kid A"))
})
It("returns the artist name", func() {
Expect(ItemName(ctx, ds, model.KindArtistArtwork, "ar-1")).To(Equal("Radiohead"))
})
It("errors for an unknown album", func() {
_, err := ItemName(ctx, ds, model.KindAlbumArtwork, "nope")
Expect(err).To(MatchError(model.ErrNotFound))
})
It("errors for an unsupported kind", func() {
// model.Kind is a struct with unexported fields, so the zero value is the only
// unsupported Kind constructible from outside package model.
_, err := ItemName(ctx, ds, model.Kind{}, "al-1")
Expect(err).To(HaveOccurred())
})
Context("disc artwork", func() {
It("names the album, the disc and its subtitle", func() {
Expect(ItemName(ctx, ds, model.KindDiscArtwork, "al-2:2")).
To(Equal("Sandinista! (disc 2): Side Three"))
})
It("omits the subtitle when the disc has none", func() {
Expect(ItemName(ctx, ds, model.KindDiscArtwork, "al-2:1")).
To(Equal("Sandinista! (disc 1)"))
})
It("rejects an id that is not <albumID>:<disc>", func() {
_, err := ItemName(ctx, ds, model.KindDiscArtwork, "al-2")
Expect(err).To(HaveOccurred())
})
})
})

53
core/artwork/processor.go
View File

@ -2,6 +2,7 @@ package artwork
import (
"bytes"
"cmp"
"context"
"encoding/base64"
"errors"
@ -15,6 +16,7 @@ import (
"github.com/navidrome/navidrome/conf"
"github.com/navidrome/navidrome/consts"
"github.com/navidrome/navidrome/core/agents"
"github.com/navidrome/navidrome/core/artwork/blurhash"
"github.com/navidrome/navidrome/core/artwork/dominant"
"github.com/navidrome/navidrome/core/artwork/thumbhash"
@ -79,7 +81,7 @@ type processor struct {
// acquire resolves one queue item end to end: find an image, hash/decode/
// blurhash it, place its bytes, and persist the resulting state.
func (p *processor) acquire(ctx context.Context, item model.ArtworkQueueItem) (out outcome, got *acquired) {
func (p *processor) acquire(ctx context.Context, item model.ArtworkQueueItem) (out outcome, got *acquired, retryIn time.Duration) {
repo := p.ds.Artwork(ctx)
start := time.Now()
defer func() {
@ -89,25 +91,38 @@ func (p *processor) acquire(ctx context.Context, item model.ArtworkQueueItem) (o
res, err := p.resolver.resolve(ctx, item)
if err != nil {
traceStage(ctx, "resolve", err)
log.Warn(ctx, "Artwork: Could not resolve item", "kind", item.ItemKind, "id", item.ItemID, err)
return outcomeFailed, nil
return outcomeFailed, nil, 0
}
if retry, ok := errors.AsType[*agents.RetryLaterError](res.extErr); ok {
retryIn = retry.RetryIn
}
if res.reader == nil {
if res.extError || res.localError {
if res.extErr != nil || res.localError {
// A fault is not a definitive "no image": never settle absent, keep serving old state.
// A chainless resolver (playlist/radio) records no step, so leave a fallback or explain is blank.
if t := traceFrom(ctx); len(t.Steps()) == 0 {
outcome := OutcomeError
if res.localError {
outcome = OutcomeUnreadable
}
t.add(TraceStep{Candidate: cmp.Or(res.source, "source"), Outcome: outcome})
}
log.Debug(ctx, "Artwork: No image, but a source faulted; keeping previous state",
"kind", item.ItemKind, "id", item.ItemID, "extError", res.extError, "localError", res.localError)
return outcomeFailed, nil
"kind", item.ItemKind, "id", item.ItemID, "extErr", res.extErr, "localError", res.localError)
return outcomeFailed, nil, retryIn
}
return writeAbsent(ctx, repo, item), nil
return writeAbsent(ctx, repo, item), nil, 0
}
defer res.reader.Close()
readStart := time.Now()
data, err := readCapped(res.reader)
if err != nil {
traceStage(ctx, "read", err)
log.Warn(ctx, "Artwork: Failed to read resolved image", "kind", item.ItemKind, "id", item.ItemID, "source", res.source, err)
return outcomeFailed, nil
return outcomeFailed, nil, retryIn
}
log.Debug(ctx, "Artwork: Read resolved image", "kind", item.ItemKind, "id", item.ItemID,
"source", res.source, "bytes", len(data), "elapsed", time.Since(readStart))
@ -115,8 +130,9 @@ func (p *processor) acquire(ctx context.Context, item model.ArtworkQueueItem) (o
hashStart := time.Now()
hash, err := hashImage(bytes.NewReader(data))
if err != nil {
traceStage(ctx, "hash", err)
log.Warn(ctx, "Artwork: Failed to hash image", "kind", item.ItemKind, "id", item.ItemID, err)
return outcomeFailed, nil
return outcomeFailed, nil, retryIn
}
log.Trace(ctx, "Artwork: Hashed image", "kind", item.ItemKind, "id", item.ItemID,
"hash", hash, "bytes", len(data), "elapsed", time.Since(hashStart))
@ -138,34 +154,37 @@ func (p *processor) acquire(ctx context.Context, item model.ArtworkQueueItem) (o
art, err = undecodedArtwork(hash), nil
}
if err != nil {
traceStage(ctx, "decode", err)
log.Warn(ctx, "Artwork: Failed to decode resolved image", "kind", item.ItemKind, "id", item.ItemID, err)
return outcomeFailed, nil
return outcomeFailed, nil, retryIn
}
log.Debug(ctx, "Artwork: Decoded new image", "kind", item.ItemKind, "id", item.ItemID, "hash", hash,
"width", art.Width, "height", art.Height, "mime", art.Mime, "elapsed", time.Since(decodeStart))
default:
traceStage(ctx, "lookup", err)
log.Warn(ctx, "Artwork: Failed to look up image hash", "kind", item.ItemKind, "id", item.ItemID, err)
return outcomeFailed, nil
return outcomeFailed, nil, retryIn
}
art.SizeBytes = int64(len(data))
ia, err := p.persist(repo, item, art, res, data)
ia, err := p.persist(ctx, repo, item, art, res, data)
if err != nil {
traceStage(ctx, "store", err)
log.Warn(ctx, "Artwork: Failed to persist resolved image", "kind", item.ItemKind, "id", item.ItemID, err)
return outcomeFailed, nil
return outcomeFailed, nil, retryIn
}
got = &acquired{ia: ia, mime: art.Mime, data: data}
if res.extError {
if res.extErr != nil {
log.Debug(ctx, "Artwork: Serving a lower-priority source after an external failure",
"kind", item.ItemKind, "id", item.ItemID, "source", res.source)
return outcomeFoundStale, got
return outcomeFoundStale, got, retryIn
}
return outcomeFound, got
return outcomeFound, got, retryIn
}
// persist places the bytes and commits the rows referencing them, excluding Prune for that
// window only so a slow resolution can never hold it off.
func (p *processor) persist(repo model.ArtworkRepository, item model.ArtworkQueueItem,
func (p *processor) persist(ctx context.Context, repo model.ArtworkRepository, item model.ArtworkQueueItem,
art *model.Artwork, res resolution, data []byte,
) (*model.ItemArtwork, error) {
if p.pruneLock != nil {
@ -188,6 +207,7 @@ func (p *processor) persist(repo model.ArtworkRepository, item model.ArtworkQueu
SourcePath: sourcePath,
RefMtime: refMtime,
AttemptedAt: time.Now(),
Trace: traceFrom(ctx).encode(sourcePath),
}
// PutItemArtwork stamps UpdatedAt on ia, so the returned struct matches the persisted row.
if err := repo.PutItemArtwork(ia); err != nil {
@ -203,6 +223,7 @@ func writeAbsent(ctx context.Context, repo model.ArtworkRepository, item model.A
ItemID: item.ItemID,
ImageType: item.ImageType,
AttemptedAt: time.Now(),
Trace: traceFrom(ctx).encode(""),
})
if err != nil {
log.Warn(ctx, "Artwork: Failed to persist absent state", "kind", item.ItemKind, "id", item.ItemID, err)

84
core/artwork/processor_test.go
View File

@ -90,7 +90,7 @@ var _ = Describe("processor.acquire", func() {
{ID: "al1", Name: "Album", FolderIDs: []string{"f1"}},
})
out, _ := proc.acquire(ctx, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al1"})
out, _, _ := proc.acquire(ctx, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al1"})
Expect(out).To(Equal(outcomeFound))
ia, err := artRepo.GetItemArtwork(model.KindAlbumArtwork, "al1", model.ImageTypePrimary)
@ -127,7 +127,7 @@ var _ = Describe("processor.acquire", func() {
{ID: "alL1", Name: "Album", FolderIDs: []string{"f1"}},
})
out, _ := proc.acquire(ctx, model.ArtworkQueueItem{ItemKind: "al", ItemID: "alL1"})
out, _, _ := proc.acquire(ctx, model.ArtworkQueueItem{ItemKind: "al", ItemID: "alL1"})
Expect(out).To(Equal(outcomeFound))
Expect(lock.locks).To(BeNumerically(">", 0), "the write window must exclude prune")
Expect(lock.held()).To(BeFalse(), "the window must close before acquire returns")
@ -141,7 +141,7 @@ var _ = Describe("processor.acquire", func() {
{ID: "alL2", Name: "Album", FolderIDs: []string{"f1"}},
})
out, _ := proc.acquire(ctx, model.ArtworkQueueItem{ItemKind: "al", ItemID: "alL2"})
out, _, _ := proc.acquire(ctx, model.ArtworkQueueItem{ItemKind: "al", ItemID: "alL2"})
Expect(out).To(Equal(outcomeAbsent))
Expect(lock.locks).To(BeZero())
})
@ -153,7 +153,7 @@ var _ = Describe("processor.acquire", func() {
})
folderRepo.result = nil
out, _ := proc.acquire(ctx, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al2"})
out, _, _ := proc.acquire(ctx, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al2"})
Expect(out).To(Equal(outcomeFound))
ia, err := artRepo.GetItemArtwork(model.KindAlbumArtwork, "al2", model.ImageTypePrimary)
@ -176,7 +176,7 @@ var _ = Describe("processor.acquire", func() {
{ID: "al3", Name: "Album"},
})
out, _ := proc.acquire(ctx, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al3"})
out, _, _ := proc.acquire(ctx, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al3"})
Expect(out).To(Equal(outcomeAbsent))
ia, err := artRepo.GetItemArtwork(model.KindAlbumArtwork, "al3", model.ImageTypePrimary)
@ -197,7 +197,7 @@ var _ = Describe("processor.acquire", func() {
{ID: "al-io", Name: "Album", FolderIDs: []string{"f1"}},
})
out, _ := proc.acquire(ctx, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al-io"})
out, _, _ := proc.acquire(ctx, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al-io"})
Expect(out).To(Equal(outcomeFailed))
_, err := artRepo.GetItemArtwork(model.KindAlbumArtwork, "al-io", model.ImageTypePrimary)
@ -222,13 +222,42 @@ var _ = Describe("processor.acquire", func() {
DeferCleanup(func() { _ = os.Chmod(upload, 0o600) })
radioRepo.Data["ra-io"] = &model.Radio{ID: "ra-io", Name: "Station", UploadedImage: "ra-io.jpg"}
out, _ := proc.acquire(ctx, model.ArtworkQueueItem{ItemKind: "ra", ItemID: "ra-io"})
out, _, _ := proc.acquire(ctx, model.ArtworkQueueItem{ItemKind: "ra", ItemID: "ra-io"})
Expect(out).To(Equal(outcomeFailed))
_, err := artRepo.GetItemArtwork(model.KindRadioArtwork, "ra-io", model.ImageTypePrimary)
Expect(err).To(MatchError(model.ErrNotFound), "an unreadable upload must not be recorded as absent")
})
// Playlist/radio resolvers walk no chain, so a fault records no step; without a fallback,
// explain would show a give-up with an empty "Gave up after" table.
It("chainless fault: records a fallback trace step naming the faulted source", func() {
if runtime.GOOS == "windows" {
// os.Open under a non-directory maps to a not-exist error on Windows, so no localError.
Skip("cannot provoke an open fault via a non-directory parent on Windows")
}
radioRepo := tests.CreateMockedRadioRepo()
radioRepo.Data = map[string]*model.Radio{}
ds.MockedRadio = radioRepo
dir := GinkgoT().TempDir()
conf.Server.DataFolder = conf.NewDir(dir)
upload := model.UploadedImagePath(consts.EntityRadio, "ra-tr.jpg")
// A plain file where the upload's parent should be makes os.Open fault with ENOTDIR,
// deterministically and regardless of the test user's privileges.
Expect(os.MkdirAll(filepath.Dir(filepath.Dir(upload)), 0o755)).To(Succeed())
Expect(os.WriteFile(filepath.Dir(upload), []byte("x"), 0o600)).To(Succeed())
radioRepo.Data["ra-tr"] = &model.Radio{ID: "ra-tr", Name: "Station", UploadedImage: "ra-tr.jpg"}
trace := &ChainTrace{}
out, _, _ := proc.acquire(withTrace(ctx, trace), model.ArtworkQueueItem{ItemKind: "ra", ItemID: "ra-tr"})
Expect(out).To(Equal(outcomeFailed))
steps := trace.Steps()
Expect(steps).To(HaveLen(1), "a radio fault must leave one step so explain is not blank")
Expect(steps[0].Candidate).To(Equal("upload"))
Expect(steps[0].Outcome).To(Equal(OutcomeUnreadable))
})
It("failed-on-extError: leaves the item's state untouched", func() {
conf.Server.CoverArtPriority = "external"
ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{
@ -236,13 +265,26 @@ var _ = Describe("processor.acquire", func() {
})
imageAgents(&fakeImageAgent{name: "failAgent", err: errors.New("agent timed out")})
out, _ := proc.acquire(ctx, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al4"})
out, _, retryIn := proc.acquire(ctx, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al4"})
Expect(out).To(Equal(outcomeFailed))
Expect(retryIn).To(BeZero(), "a plain failure asks for no particular delay")
_, err := artRepo.GetItemArtwork(model.KindAlbumArtwork, "al4", model.ImageTypePrimary)
Expect(err).To(MatchError(model.ErrNotFound))
})
It("failed-on-extError: reports the delay a throttled provider asked for", func() {
conf.Server.CoverArtPriority = "external"
ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{
{ID: "al4r", Name: "Album"},
})
imageAgents(&fakeImageAgent{name: "throttled", err: &agents.RetryLaterError{RetryIn: 42 * time.Second}})
out, _, retryIn := proc.acquire(ctx, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al4r"})
Expect(out).To(Equal(outcomeFailed))
Expect(retryIn).To(Equal(42 * time.Second))
})
It("found-stale: a fallback hit after a transient external failure persists state and returns outcomeFoundStale", func() {
conf.Server.CoverArtPriority = "external, cover.jpg"
folderRepo.result = []model.Folder{{
@ -254,7 +296,7 @@ var _ = Describe("processor.acquire", func() {
})
imageAgents(&fakeImageAgent{name: "failAgent", err: errors.New("agent timed out")})
out, _ := proc.acquire(ctx, model.ArtworkQueueItem{ItemKind: "al", ItemID: "alstale"})
out, _, _ := proc.acquire(ctx, model.ArtworkQueueItem{ItemKind: "al", ItemID: "alstale"})
Expect(out).To(Equal(outcomeFoundStale))
ia, err := artRepo.GetItemArtwork(model.KindAlbumArtwork, "alstale", model.ImageTypePrimary)
@ -271,7 +313,7 @@ var _ = Describe("processor.acquire", func() {
ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{{ID: "alU", Name: "Album", FolderIDs: []string{"f1"}}})
folderRepo.result = []model.Folder{{Path: "album", ImageFiles: []string{"cover.jpg"}}}
out, _ := proc.acquire(ctx, model.ArtworkQueueItem{ItemKind: "al", ItemID: "alU"})
out, _, _ := proc.acquire(ctx, model.ArtworkQueueItem{ItemKind: "al", ItemID: "alU"})
Expect(out).To(Equal(outcomeFound))
ia, err := artRepo.GetItemArtwork(model.KindAlbumArtwork, "alU", model.ImageTypePrimary)
@ -291,7 +333,7 @@ var _ = Describe("processor.acquire", func() {
ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{{ID: "alE", Name: "Album", FolderIDs: []string{"f1"}}})
folderRepo.result = []model.Folder{{Path: "album", ImageFiles: []string{"cover.jpg"}}}
out, _ := proc.acquire(ctx, model.ArtworkQueueItem{ItemKind: "al", ItemID: "alE"})
out, _, _ := proc.acquire(ctx, model.ArtworkQueueItem{ItemKind: "al", ItemID: "alE"})
Expect(out).To(Equal(outcomeFailed))
_, err := artRepo.GetItemArtwork(model.KindAlbumArtwork, "alE", model.ImageTypePrimary)
@ -309,7 +351,7 @@ var _ = Describe("processor.acquire", func() {
ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{{ID: "alX", Name: "Album"}})
imageAgents(&fakeImageAgent{name: "deezerFake", imgs: []agents.ExternalImage{{URL: srv.URL, Size: 500}}})
out, _ := proc.acquire(ctx, model.ArtworkQueueItem{ItemKind: "al", ItemID: "alX"})
out, _, _ := proc.acquire(ctx, model.ArtworkQueueItem{ItemKind: "al", ItemID: "alX"})
Expect(out).To(Equal(outcomeFailed))
_, err := artRepo.GetItemArtwork(model.KindAlbumArtwork, "alX", model.ImageTypePrimary)
@ -328,7 +370,7 @@ var _ = Describe("processor.acquire", func() {
ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{{ID: "alext", Name: "Album"}})
imageAgents(&fakeImageAgent{name: "deezerFake", imgs: []agents.ExternalImage{{URL: srv.URL, Size: 500}}})
out, _ := proc.acquire(ctx, model.ArtworkQueueItem{ItemKind: "al", ItemID: "alext"})
out, _, _ := proc.acquire(ctx, model.ArtworkQueueItem{ItemKind: "al", ItemID: "alext"})
Expect(out).To(Equal(outcomeFound))
ia, err := artRepo.GetItemArtwork(model.KindAlbumArtwork, "alext", model.ImageTypePrimary)
@ -353,7 +395,7 @@ var _ = Describe("processor.acquire", func() {
{ID: "al6", Name: "Album B", FolderIDs: []string{"f1"}},
})
out1, _ := proc.acquire(ctx, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al5"})
out1, _, _ := proc.acquire(ctx, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al5"})
Expect(out1).To(Equal(outcomeFound))
ia1, err := artRepo.GetItemArtwork(model.KindAlbumArtwork, "al5", model.ImageTypePrimary)
Expect(err).ToNot(HaveOccurred())
@ -363,7 +405,7 @@ var _ = Describe("processor.acquire", func() {
poisoned.BlurHash = "SENTINEL"
artRepo.Data[ia1.Hash] = poisoned
out2, _ := proc.acquire(ctx, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al6"})
out2, _, _ := proc.acquire(ctx, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al6"})
Expect(out2).To(Equal(outcomeFound))
ia2, err := artRepo.GetItemArtwork(model.KindAlbumArtwork, "al6", model.ImageTypePrimary)
Expect(err).ToNot(HaveOccurred())
@ -393,7 +435,7 @@ var _ = Describe("processor.acquire", func() {
})
folderRepo.result = []model.Folder{{Path: "album-a", ImageFiles: []string{"cover.jpg"}}}
outN, _ := proc.acquire(ctx, model.ArtworkQueueItem{ItemKind: "al", ItemID: "alA"})
outN, _, _ := proc.acquire(ctx, model.ArtworkQueueItem{ItemKind: "al", ItemID: "alA"})
Expect(outN).To(Equal(outcomeFound))
iaA, err := artRepo.GetItemArtwork(model.KindAlbumArtwork, "alA", model.ImageTypePrimary)
Expect(err).ToNot(HaveOccurred())
@ -407,7 +449,7 @@ var _ = Describe("processor.acquire", func() {
artRepo.Data[iaA.Hash] = poisoned
folderRepo.result = []model.Folder{{Path: "album-b", ImageFiles: []string{"cover.jpg"}}}
outN, _ = proc.acquire(ctx, model.ArtworkQueueItem{ItemKind: "al", ItemID: "alB"})
outN, _, _ = proc.acquire(ctx, model.ArtworkQueueItem{ItemKind: "al", ItemID: "alB"})
Expect(outN).To(Equal(outcomeFound))
iaB, err := artRepo.GetItemArtwork(model.KindAlbumArtwork, "alB", model.ImageTypePrimary)
Expect(err).ToNot(HaveOccurred())
@ -438,7 +480,7 @@ var _ = Describe("processor.acquire", func() {
radioRepo.Data = map[string]*model.Radio{"ra1": {ID: "ra1", Name: "Radio", UploadedImage: "ra1_test.jpg"}}
ds.MockedRadio = radioRepo
out, _ := proc.acquire(ctx, model.ArtworkQueueItem{ItemKind: "ra", ItemID: "ra1"})
out, _, _ := proc.acquire(ctx, model.ArtworkQueueItem{ItemKind: "ra", ItemID: "ra1"})
Expect(out).To(Equal(outcomeFailed))
_, err := artRepo.GetItemArtwork(model.KindRadioArtwork, "ra1", model.ImageTypePrimary)
@ -459,7 +501,7 @@ var _ = Describe("processor.acquire", func() {
radioRepo.Data = map[string]*model.Radio{"big": {ID: "big", Name: "Radio", UploadedImage: "big_test.jpg"}}
ds.MockedRadio = radioRepo
out, _ := proc.acquire(ctx, model.ArtworkQueueItem{ItemKind: "ra", ItemID: "big"})
out, _, _ := proc.acquire(ctx, model.ArtworkQueueItem{ItemKind: "ra", ItemID: "big"})
Expect(out).To(Equal(outcomeFailed))
_, err = artRepo.GetItemArtwork(model.KindRadioArtwork, "big", model.ImageTypePrimary)
@ -525,7 +567,7 @@ var _ = Describe("processor.acquire", func() {
Expect(err).ToNot(HaveOccurred())
Expect(artRepo.PutImage(&model.Artwork{Hash: hash, Mime: "application/octet-stream"})).To(Succeed())
out, _ := proc.acquire(ctx, model.ArtworkQueueItem{ItemKind: "al", ItemID: "alM"})
out, _, _ := proc.acquire(ctx, model.ArtworkQueueItem{ItemKind: "al", ItemID: "alM"})
Expect(out).To(Equal(outcomeFound))
upgraded, err := artRepo.GetImage(hash)
@ -545,7 +587,7 @@ var _ = Describe("processor.acquire", func() {
Expect(os.WriteFile(blockedRoot, []byte("x"), 0600)).To(Succeed())
proc.store = NewImageStore(blockedRoot)
out, _ := proc.acquire(ctx, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al7"})
out, _, _ := proc.acquire(ctx, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al7"})
Expect(out).To(Equal(outcomeFailed))
_, err := artRepo.GetItemArtwork(model.KindAlbumArtwork, "al7", model.ImageTypePrimary)

82
core/artwork/resolve.go
View File

@ -25,9 +25,9 @@ type resolution struct {
source string // model.ItemArtwork.Source value: "folder", "embedded", "external", "upload", "generated"
sourcePath string // backing library/upload file (folder/upload: the image; embedded: the audio file); "" otherwise
refMtime int64 // sourcePath mtime (unix-nanoseconds) at resolution; 0 when no sourcePath
// external source errored/timed out. With no reader it forces failed (never absent);
// on a hit a higher-priority external step failed—serve this, but retry later.
extError bool
// a faulted external source, carrying the provider's requested delay when it named one.
// With no reader it forces failed (never absent); on a hit, serve this but retry later.
extErr error
// a local source that should have been readable wasn't. With no reader it forces failed,
// so a transient I/O fault never records absent.
localError bool
@ -36,14 +36,15 @@ type resolution struct {
// chainState carries what a priority walk has seen so far. A hit takes extErr with it so a
// transient external failure still retries; localErr is dropped, as the scanner re-lists changes.
type chainState struct {
extErr, localErr bool
trace *ChainTrace // nil unless the CLI asked for a trace
extErr error
localErr bool
trace *ChainTrace // nil only where no caller attached one
}
// try stamps the accumulated external failure onto a hit, and records the miss otherwise.
func (c *chainState) try(candidate string, res resolution, ok bool) (resolution, bool) {
if ok {
res.extError = c.extErr
res.extErr = c.extErr
c.record(candidate, OutcomeHit, res.sourcePath)
return res, true
}
@ -62,7 +63,7 @@ func (c *chainState) record(candidate string, out Outcome, detail string) {
// exhausted is the outcome when no source in the chain yielded an image.
func (c *chainState) exhausted() resolution {
return resolution{extError: c.extErr, localError: c.localErr}
return resolution{extErr: c.extErr, localError: c.localErr}
}
// externalSource holds the agents to ask and the rate limiter/circuit breaker to ask them through.
@ -137,6 +138,15 @@ func MayFetchExternal(kind model.Kind) bool {
// ImageAgentCount is how many enabled agents provide artist and album images.
type ImageAgentCount struct{ Artist, Album int }
// NewImageAgentCount counts what an external step would consult, so an estimate and the gate that
// guards it cannot disagree about which agents exist.
func NewImageAgentCount(ag *agents.Agents) ImageAgentCount {
if ag == nil {
return ImageAgentCount{}
}
return ImageAgentCount{Artist: len(ag.ArtistImageAgents()), Album: len(ag.AlbumImageAgents())}
}
// ExternalLookupsPerItem reports what resolving one item of this kind can cost: every image agent is
// tried, and a zero count still bills one, so agents the caller cannot see never read as free.
func ExternalLookupsPerItem(kind model.Kind, agents ImageAgentCount) int64 {
@ -172,16 +182,16 @@ func chainFetchesExternal(priority string) bool {
// Album and artist fetches stop here when the resolver is local-only, rather than at each point in
// the chain walk; resolvePlaylist gates the third network path, the m3u image URL, itself.
func (r *resolver) fetchExternalAlbum(ctx context.Context, al model.Album) (io.ReadCloser, string, bool) {
func (r *resolver) fetchExternalAlbum(ctx context.Context, al model.Album) (io.ReadCloser, string, error) {
if r.ext == nil {
return nil, "", false
return nil, "", nil
}
return fetchAlbumImage(ctx, r.ext.agents, r.ext.gate, al)
}
func (r *resolver) fetchExternalArtist(ctx context.Context, ar model.Artist) (io.ReadCloser, string, bool) {
func (r *resolver) fetchExternalArtist(ctx context.Context, ar model.Artist) (io.ReadCloser, string, error) {
if r.ext == nil {
return nil, "", false
return nil, "", nil
}
return fetchArtistImage(ctx, r.ext.agents, r.ext.gate, ar)
}
@ -214,10 +224,10 @@ func (r *resolver) resolveAlbum(ctx context.Context, albumID string) (resolution
return res, nil
}
case pattern == externalCandidate:
if rd, name, isErr := r.fetchExternalAlbum(ctx, *al); rd != nil {
if rd, name, err := r.fetchExternalAlbum(ctx, *al); rd != nil {
return resolution{reader: rd, source: ExternalPrefix + name}, nil
} else if isErr {
chain.extErr = true
} else if err != nil {
chain.extErr = longerRetry(chain.extErr, err)
}
case len(imgFiles) > 0:
res, ok := resolveFolderFile(ctx, lib, imgFiles, pattern)
@ -276,10 +286,10 @@ func (r *resolver) resolveArtist(ctx context.Context, artistID string) (resoluti
}
switch {
case pattern == externalCandidate:
if rd, name, isErr := r.fetchExternalArtist(ctx, *ar); rd != nil {
if rd, name, err := r.fetchExternalArtist(ctx, *ar); rd != nil {
return resolution{reader: rd, source: ExternalPrefix + name}, nil
} else if isErr {
chain.extErr = true
} else if err != nil {
chain.extErr = longerRetry(chain.extErr, err)
}
case pattern == "image-folder":
res, ok := resolveArtistImageFolder(ar)
@ -323,7 +333,7 @@ func (r *resolver) resolvePlaylist(ctx context.Context, playlistID string) (reso
return resolution{}, err
}
var extErr bool
var extErr error
for _, src := range []struct{ path, source string }{
{pl.UploadedImagePath(), "upload"},
{findPlaylistSidecarPath(ctx, pl.Path), "folder"},
@ -354,10 +364,13 @@ func (r *resolver) resolvePlaylist(ctx context.Context, playlistID string) (reso
}
if remoteImg != nil && conf.Server.EnableM3UExternalAlbumArt {
sf := func() (io.ReadCloser, string, error) { return fromURL(ctx, remoteImg) }
if res, ok, isErr := resolveExternalStep(r.ext.gate, "m3u", sf); ok {
if res, ok, err := resolveExternalStep(r.ext.gate, "m3u", sf); ok {
return res, nil
} else if isErr {
extErr = true
} else if err != nil {
extErr = longerRetry(extErr, err)
// Record it here with its detail: once album sampling adds its own steps, the processor's
// empty-trace fallback no longer fires, and the error that forced the retry would be lost.
traceFrom(ctx).add(TraceStep{Candidate: ExternalPrefix + "m3u", Outcome: OutcomeError, Detail: err.Error()})
}
}
@ -377,8 +390,8 @@ func (r *resolver) resolvePlaylist(ctx context.Context, playlistID string) (reso
}
continue
}
if res.extError {
extErr = true
if res.extErr != nil {
extErr = longerRetry(extErr, res.extErr)
}
if res.reader == nil {
continue
@ -397,7 +410,7 @@ func (r *resolver) resolvePlaylist(ctx context.Context, playlistID string) (reso
if tileErr != nil {
return resolution{}, fmt.Errorf("resolvePlaylist: sampled album art failed: %w", tileErr)
}
return resolution{extError: extErr}, nil
return resolution{extErr: extErr}, nil
}
// Grow to 4 tiles by repeating what we have.
switch len(tiles) {
@ -408,9 +421,9 @@ func (r *resolver) resolvePlaylist(ctx context.Context, playlistID string) (reso
}
grid, err := assembleTiles(tiles)
if err != nil {
return resolution{extError: extErr}, nil //nolint:nilerr // encode failure is a soft "no image", not a resolution error
return resolution{extErr: extErr}, nil //nolint:nilerr // encode failure is a soft "no image", not a resolution error
}
return resolution{reader: grid, source: "generated", extError: extErr}, nil
return resolution{reader: grid, source: "generated", extErr: extErr}, nil
}
// resolveRadio serves only an uploaded image; there is no fallback.
@ -461,14 +474,17 @@ func (r *resolver) resolveDisc(ctx context.Context, id string) (resolution, erro
return dr.selectImage(ctx, r.ffmpeg, conf.Server.DiscArtPriority, &chain)
}
// resolveExternalStep runs a single external sourceFunc through the named gate. extErr excludes
// a not-found, which is a definitive "no" rather than a failure.
func resolveExternalStep(gate gateFunc, name string, sf sourceFunc) (res resolution, ok bool, extErr bool) {
// resolveExternalStep runs a single external sourceFunc through the named gate. A not-found is a
// definitive "no", returned as (_, false, nil); any other error is a failure the caller records.
func resolveExternalStep(gate gateFunc, name string, sf sourceFunc) (resolution, bool, error) {
r, path, err := gate(name, sf)
if r != nil {
return resolution{reader: r, source: externalCandidate, sourcePath: path}, true, false
return resolution{reader: r, source: externalCandidate, sourcePath: path}, true, nil
}
return resolution{}, false, err != nil && !errors.Is(err, model.ErrNotFound)
if errors.Is(err, model.ErrNotFound) {
return resolution{}, false, nil
}
return resolution{}, false, err
}
// classifyPlaylistImage splits a playlist ExternalImageURL into a local filesystem path or a
@ -561,7 +577,9 @@ func resolveLocalFile(path, source string) (resolution, bool) {
}
f, err := os.Open(path)
if err != nil {
return resolution{localError: !errors.Is(err, fs.ErrNotExist)}, false
// Carry the source label even on a fault, so a resolver with no chain (playlist/radio) can
// still name what faulted in the trace.
return resolution{source: source, localError: !errors.Is(err, fs.ErrNotExist)}, false
}
return resolution{reader: f, source: source, sourcePath: path, refMtime: mtimeOf(path)}, true
}

75
core/artwork/resolve_test.go
View File

@ -100,7 +100,7 @@ var _ = Describe("resolveItem", func() {
Expect(res.source).To(Equal("embedded"))
Expect(filepath.ToSlash(res.sourcePath)).To(HaveSuffix("tests/fixtures/artist/an-album/test.mp3"))
Expect(res.refMtime).To(BeNumerically(">", 0))
Expect(res.extError).To(BeFalse())
Expect(res.extErr).ToNot(HaveOccurred())
})
It("resolves absent when the track has no cover art", func() {
@ -111,7 +111,7 @@ var _ = Describe("resolveItem", func() {
res, err := newResolver(ds, ag, ffm, nil).resolve(ctx, model.ArtworkQueueItem{ItemKind: "mf", ItemID: "mf2"})
Expect(err).ToNot(HaveOccurred())
Expect(res.reader).To(BeNil())
Expect(res.extError).To(BeFalse())
Expect(res.extErr).ToNot(HaveOccurred())
})
It("resolves absent when media file cover art is disabled", func() {
@ -154,7 +154,7 @@ var _ = Describe("resolveItem", func() {
Expect(res.source).To(Equal("folder"))
Expect(filepath.ToSlash(res.sourcePath)).To(HaveSuffix("tests/fixtures/artist/an-album/cover.jpg"))
Expect(res.refMtime).To(BeNumerically(">", 0))
Expect(res.extError).To(BeFalse())
Expect(res.extErr).ToNot(HaveOccurred())
})
It("falls back to embedded art when no folder image matches", func() {
@ -172,7 +172,7 @@ var _ = Describe("resolveItem", func() {
Expect(res.refMtime).To(BeNumerically(">", 0))
})
It("sets extError when the external source errors without being not-found", func() {
It("sets extErr when the external source errors without being not-found", func() {
conf.Server.CoverArtPriority = "external"
ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{
{ID: "al3", Name: "Album"},
@ -182,10 +182,10 @@ var _ = Describe("resolveItem", func() {
res, err := newResolver(ds, ag, ffm, nil).resolve(ctx, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al3"})
Expect(err).ToNot(HaveOccurred())
Expect(res.reader).To(BeNil())
Expect(res.extError).To(BeTrue())
Expect(res.extErr).To(HaveOccurred())
})
It("does not set extError when the external source reports not-found", func() {
It("does not set extErr when the external source reports not-found", func() {
conf.Server.CoverArtPriority = "external"
ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{
{ID: "al4", Name: "Album"},
@ -195,10 +195,10 @@ var _ = Describe("resolveItem", func() {
res, err := newResolver(ds, ag, ffm, nil).resolve(ctx, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al4"})
Expect(err).ToNot(HaveOccurred())
Expect(res.reader).To(BeNil())
Expect(res.extError).To(BeFalse())
Expect(res.extErr).ToNot(HaveOccurred())
})
It("carries extError onto a fallback folder hit after a transient external failure", func() {
It("carries extErr onto a fallback folder hit after a transient external failure", func() {
conf.Server.CoverArtPriority = "external, cover.jpg"
folderRepo.result = []model.Folder{{
Path: "tests/fixtures/artist/an-album",
@ -214,10 +214,10 @@ var _ = Describe("resolveItem", func() {
Expect(res.reader).ToNot(BeNil())
defer res.reader.Close()
Expect(res.source).To(Equal("folder"))
Expect(res.extError).To(BeTrue())
Expect(res.extErr).To(HaveOccurred())
})
It("does not carry extError onto a fallback folder hit after a definitive external not-found", func() {
It("does not carry extErr onto a fallback folder hit after a definitive external not-found", func() {
conf.Server.CoverArtPriority = "external, cover.jpg"
folderRepo.result = []model.Folder{{
Path: "tests/fixtures/artist/an-album",
@ -233,7 +233,7 @@ var _ = Describe("resolveItem", func() {
Expect(res.reader).ToNot(BeNil())
defer res.reader.Close()
Expect(res.source).To(Equal("folder"))
Expect(res.extError).To(BeFalse())
Expect(res.extErr).ToNot(HaveOccurred())
})
It("routes the external step through the injected gate, keyed by agent name", func() {
@ -250,7 +250,7 @@ var _ = Describe("resolveItem", func() {
res, err := newResolver(ds, ag, ffm, gate).resolve(ctx, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al5"})
Expect(err).ToNot(HaveOccurred())
Expect(res.extError).To(BeTrue())
Expect(res.extErr).To(HaveOccurred())
Expect(gatedNames).To(Equal([]string{"failAgent"}))
})
})
@ -298,7 +298,7 @@ var _ = Describe("resolveItem", func() {
Expect(filepath.ToSlash(res.sourcePath)).To(HaveSuffix("tests/fixtures/artist/an-album/artist.png"))
})
It("sets extError when the external source errors without being not-found", func() {
It("sets extErr when the external source errors without being not-found", func() {
conf.Server.ArtistArtPriority = "external"
artistRepo := tests.CreateMockArtistRepo()
artistRepo.SetData(model.Artists{{ID: "ar3", Name: "Artist"}})
@ -308,10 +308,10 @@ var _ = Describe("resolveItem", func() {
res, err := newResolver(ds, ag, ffm, nil).resolve(ctx, model.ArtworkQueueItem{ItemKind: "ar", ItemID: "ar3"})
Expect(err).ToNot(HaveOccurred())
Expect(res.reader).To(BeNil())
Expect(res.extError).To(BeTrue())
Expect(res.extErr).To(HaveOccurred())
})
It("does not set extError when the external source reports not-found", func() {
It("does not set extErr when the external source reports not-found", func() {
conf.Server.ArtistArtPriority = "external"
artistRepo := tests.CreateMockArtistRepo()
artistRepo.SetData(model.Artists{{ID: "ar4", Name: "Artist"}})
@ -321,7 +321,7 @@ var _ = Describe("resolveItem", func() {
res, err := newResolver(ds, ag, ffm, nil).resolve(ctx, model.ArtworkQueueItem{ItemKind: "ar", ItemID: "ar4"})
Expect(err).ToNot(HaveOccurred())
Expect(res.reader).To(BeNil())
Expect(res.extError).To(BeFalse())
Expect(res.extErr).ToNot(HaveOccurred())
})
It("routes the external step through the injected gate, keyed by agent name", func() {
@ -338,7 +338,7 @@ var _ = Describe("resolveItem", func() {
res, err := newResolver(ds, ag, ffm, gate).resolve(ctx, model.ArtworkQueueItem{ItemKind: "ar", ItemID: "ar5"})
Expect(err).ToNot(HaveOccurred())
Expect(res.extError).To(BeTrue())
Expect(res.extErr).To(HaveOccurred())
Expect(gatedNames).To(Equal([]string{"failAgent"}))
})
})
@ -516,10 +516,41 @@ var _ = Describe("resolveItem", func() {
res, err := newResolver(ds, ag, ffm, gate).resolve(ctx, model.ArtworkQueueItem{ItemKind: "pl", ItemID: "ple"})
Expect(err).ToNot(HaveOccurred())
Expect(res.reader).To(BeNil())
Expect(res.extError).To(BeTrue())
Expect(res.extErr).To(HaveOccurred())
Expect(gatedNames).To(Equal([]string{"m3u"}), "the playlist URL fetch is gated under \"m3u\"")
})
It("records the m3u failure in the trace even when album sampling adds its own steps", func() {
conf.Server.EnableM3UExternalAlbumArt = true
folderRepo.result = nil // the sampled album yields no tile, so the m3u failure is what forced the retry
plRepo := tests.CreateMockPlaylistRepo()
plRepo.SetData(model.Playlists{{ID: "plm3u", Name: "Playlist", ExternalImageURL: "http://example.com/cover.jpg"}})
plRepo.TracksRepo = &tests.MockPlaylistTrackRepo{AlbumIDs: []string{"t1"}}
ds.MockedPlaylist = plRepo
gate := func(string, func() (io.ReadCloser, string, error)) (io.ReadCloser, string, error) {
return nil, "", errors.New("network down")
}
trace := &ChainTrace{}
res, err := newResolver(ds, ag, ffm, gate).resolve(withTrace(ctx, trace),
model.ArtworkQueueItem{ItemKind: "pl", ItemID: "plm3u"})
Expect(err).ToNot(HaveOccurred())
Expect(res.extErr).To(HaveOccurred())
steps := trace.Steps()
var m3u *TraceStep
for i := range steps {
if steps[i].Candidate == ExternalPrefix+"m3u" && steps[i].Outcome == OutcomeError {
m3u = &steps[i]
}
}
Expect(m3u).ToNot(BeNil(), "the m3u fetch error must be traced at its source, not left to the empty-trace fallback")
Expect(m3u.Detail).To(Equal("network down"),
"the trace must carry the underlying error so explain can tell a timeout from an HTTP error")
})
It("treats a missing local ExternalImageURL as a definitive miss, not extError", func() {
folderRepo.result = nil // no grid tiles, so the local-file miss is what surfaces
@ -531,7 +562,7 @@ var _ = Describe("resolveItem", func() {
res, err := newResolver(ds, ag, ffm, nil).resolve(ctx, model.ArtworkQueueItem{ItemKind: "pl", ItemID: "plm"})
Expect(err).ToNot(HaveOccurred())
Expect(res.reader).To(BeNil())
Expect(res.extError).To(BeFalse())
Expect(res.extErr).ToNot(HaveOccurred())
})
It("treats an ExternalImageURL 404 as a definitive miss and falls through to the grid", func() {
@ -551,7 +582,7 @@ var _ = Describe("resolveItem", func() {
Expect(res.reader).ToNot(BeNil())
defer res.reader.Close()
Expect(res.source).To(Equal("generated"))
Expect(res.extError).To(BeFalse())
Expect(res.extErr).ToNot(HaveOccurred())
})
// A local resolver holds no agents: reaching the external branch would panic, not degrade.
@ -563,7 +594,7 @@ var _ = Describe("resolveItem", func() {
res, err := newLocalResolver(ds, ffm).resolve(ctx, model.ArtworkQueueItem{ItemKind: "al", ItemID: "alx"})
Expect(err).ToNot(HaveOccurred())
Expect(res.reader).To(BeNil())
Expect(res.extError).To(BeFalse(), "a skipped step is not a failed one")
Expect(res.extErr).ToNot(HaveOccurred(), "a skipped step is not a failed one")
})
// The worker resolving the same playlist is asserted alongside, so this cannot pass vacuously.
@ -611,7 +642,7 @@ var _ = Describe("resolveItem", func() {
res, err := newResolver(ds, ag, ffm, nil).resolve(ctx, model.ArtworkQueueItem{ItemKind: "pl", ItemID: "pl500"})
Expect(err).ToNot(HaveOccurred())
Expect(res.reader).To(BeNil())
Expect(res.extError).To(BeTrue())
Expect(res.extErr).To(HaveOccurred())
})
It("yields an empty resolution when no album has art", func() {

5
core/artwork/sources.go
View File

@ -14,10 +14,10 @@ import (
"strings"
"time"
"github.com/navidrome/navidrome/consts"
"github.com/navidrome/navidrome/core/ffmpeg"
"github.com/navidrome/navidrome/log"
"github.com/navidrome/navidrome/model"
"github.com/navidrome/navidrome/utils/httpclient"
"go.senan.xyz/taglib"
)
@ -151,9 +151,8 @@ type readCloser struct {
}
func fromURL(ctx context.Context, imageUrl *url.URL) (io.ReadCloser, string, error) {
hc := http.Client{Timeout: 5 * time.Second}
hc := httpclient.New(5 * time.Second)
req, _ := http.NewRequestWithContext(ctx, http.MethodGet, imageUrl.String(), nil)
req.Header.Set("User-Agent", consts.HTTPUserAgent)
resp, err := hc.Do(req) //nolint:gosec
if err != nil {
return nil, "", err

107
core/artwork/trace.go
View File

@ -2,10 +2,12 @@ package artwork
import (
"context"
"errors"
"encoding/json"
"io"
"slices"
"sync"
"github.com/navidrome/navidrome/utils/str"
)
// Outcome is what the priority chain observed for one candidate; the CLI renders and branches on these.
@ -16,7 +18,6 @@ const (
OutcomeMiss Outcome = "miss"
OutcomeUnreadable Outcome = "unreadable"
OutcomeSkipped Outcome = "skipped"
OutcomeWouldTry Outcome = "would-try"
OutcomeError Outcome = "error"
)
@ -34,8 +35,8 @@ type TraceStep struct {
Detail string
}
// ChainTrace collects the walk of a single resolution. The artwork worker never attaches
// one; only the CLI does, so resolution stays allocation-free in the hot path.
// ChainTrace collects the walk of a single resolution: the worker attaches one per queue
// item so it can be stored, and the CLI attaches one per explain.
type ChainTrace struct {
mu sync.Mutex
steps []TraceStep
@ -59,6 +60,65 @@ func (t *ChainTrace) Steps() []TraceStep {
return slices.Clone(t.steps)
}
// maxTraceDetail bounds a stored Detail, which on the failure paths is an error string of
// unknown length. Past ~1kB a row spills to an overflow page, slowing every scan of the table.
const maxTraceDetail = 200
// storedStep is the persisted shape of a TraceStep. The keys are single letters because a trace
// is written for every item, and the encoded length is repeated across the whole library.
type storedStep struct {
C string `json:"c"`
O Outcome `json:"o"`
D string `json:"d,omitempty"`
}
// encode serializes the trace for storage, without the copy Steps would make for a caller
// that only wants to write it.
func (t *ChainTrace) encode(sourcePath string) string {
if t == nil {
return encodeSteps(nil, sourcePath)
}
t.mu.Lock()
defer t.mu.Unlock()
return encodeSteps(t.steps, sourcePath)
}
// encodeSteps writes the stored form. A hit's Detail is the winning source's path, which the
// same row already stores as source_path, so it is dropped and DecodeTrace puts it back.
func encodeSteps(steps []TraceStep, sourcePath string) string {
out := make([]storedStep, 0, len(steps))
for _, s := range steps {
d := s.Detail
if s.Outcome == OutcomeHit && d == sourcePath {
d = ""
}
out = append(out, storedStep{C: s.Candidate, O: s.Outcome, D: str.TruncateRunes(d, maxTraceDetail, "...")})
}
b, _ := json.Marshal(out) // []storedStep is all strings, so this cannot fail
return string(b)
}
// DecodeTrace reverses the stored form. A trace that will not parse is reported as no trace at all,
// since a diagnostic command must not fail on a bad row.
func DecodeTrace(encoded, sourcePath string) []TraceStep {
if encoded == "" {
return nil
}
var stored []storedStep
if err := json.Unmarshal([]byte(encoded), &stored); err != nil {
return nil
}
steps := make([]TraceStep, 0, len(stored))
for _, s := range stored {
d := s.D
if d == "" && s.O == OutcomeHit {
d = sourcePath
}
steps = append(steps, TraceStep{Candidate: s.C, Outcome: s.O, Detail: d})
}
return steps
}
type traceCtxKey struct{}
func withTrace(ctx context.Context, t *ChainTrace) context.Context {
@ -70,30 +130,23 @@ func traceFrom(ctx context.Context) *ChainTrace {
return t
}
var errOfflineSkipped = errors.New("artwork: external lookup skipped (offline)")
// tracingGate records each external agent's outcome without changing what the gate returns.
func tracingGate(t *ChainTrace, inner gateFunc) gateFunc {
return func(name string, f func() (io.ReadCloser, string, error)) (io.ReadCloser, string, error) {
r, path, err := inner(name, f)
candidate := ExternalPrefix + name
switch {
case r != nil:
t.add(TraceStep{Candidate: candidate, Outcome: OutcomeHit, Detail: path})
case isTransientExternal(err):
t.add(TraceStep{Candidate: candidate, Outcome: OutcomeError, Detail: err.Error()})
default:
t.add(TraceStep{Candidate: candidate, Outcome: OutcomeMiss})
}
return r, path, err
// recordAgent files what one external agent answered. The agent loops call this rather than a
// gate wrapper, because only they hold the context that carries the trace.
func recordAgent(ctx context.Context, name string, r io.ReadCloser, path string, err error) {
t := traceFrom(ctx)
candidate := ExternalPrefix + name
switch {
case r != nil:
t.add(TraceStep{Candidate: candidate, Outcome: OutcomeHit, Detail: path})
case isTransientExternal(err):
t.add(TraceStep{Candidate: candidate, Outcome: OutcomeError, Detail: err.Error()})
default:
t.add(TraceStep{Candidate: candidate, Outcome: OutcomeMiss})
}
}
// offlineGate reports which agents would be asked without asking them, so a diagnostic
// command cannot add load to a provider that is already rate-limiting us.
func offlineGate(t *ChainTrace) gateFunc {
return func(name string, _ func() (io.ReadCloser, string, error)) (io.ReadCloser, string, error) {
t.add(TraceStep{Candidate: ExternalPrefix + name, Outcome: OutcomeWouldTry})
return nil, "", errOfflineSkipped
}
// traceStage records a failure from the stages that run after the priority chain has already
// picked a winner: most ways an item can fail are here, not in the chain walk.
func traceStage(ctx context.Context, stage string, err error) {
traceFrom(ctx).add(TraceStep{Candidate: stage, Outcome: OutcomeError, Detail: err.Error()})
}

170
core/artwork/trace_test.go
View File

@ -24,13 +24,63 @@ var _ = Describe("trace vocabulary", func() {
// what `artwork explain` tells an operator, so it must be made deliberately.
It("pins the wire values the CLI reads", func() {
Expect([]Outcome{
OutcomeHit, OutcomeMiss, OutcomeUnreadable, OutcomeSkipped, OutcomeWouldTry, OutcomeError,
}).To(Equal([]Outcome{"hit", "miss", "unreadable", "skipped", "would-try", "error"}))
OutcomeHit, OutcomeMiss, OutcomeUnreadable, OutcomeSkipped, OutcomeError,
}).To(Equal([]Outcome{"hit", "miss", "unreadable", "skipped", "error"}))
Expect(externalCandidate).To(Equal("external"))
Expect(ExternalPrefix).To(Equal("external:"))
})
})
var _ = Describe("encodeSteps/DecodeTrace", func() {
It("round-trips a trace", func() {
steps := []TraceStep{
{Candidate: "cover.png", Outcome: OutcomeMiss},
{Candidate: "cover.*", Outcome: OutcomeHit, Detail: "/music/a/cover.jpg"},
}
Expect(DecodeTrace(encodeSteps(steps, ""), "")).To(Equal(steps))
})
It("encodes an empty trace as an empty JSON array", func() {
Expect(encodeSteps(nil, "")).To(Equal("[]"))
Expect(DecodeTrace("[]", "")).To(BeEmpty())
})
It("tolerates a row written before the column existed", func() {
Expect(DecodeTrace("", "")).To(BeEmpty())
})
// The hit detail repeats source_path byte for byte, and that column is on the same row.
It("drops a hit detail that repeats sourcePath, and restores it on read", func() {
path := "/music/artist/album/cover.jpg"
steps := []TraceStep{{Candidate: "cover.*", Outcome: OutcomeHit, Detail: path}}
encoded := encodeSteps(steps, path)
Expect(encoded).NotTo(ContainSubstring(path))
Expect(DecodeTrace(encoded, path)).To(Equal(steps))
})
It("keeps a detail that differs from sourcePath", func() {
steps := []TraceStep{{Candidate: "external:deezer", Outcome: OutcomeHit, Detail: "https://cdn/x.jpg"}}
Expect(DecodeTrace(encodeSteps(steps, "/music/a/cover.jpg"), "/music/a/cover.jpg")).To(Equal(steps))
})
// A row past ~1kB spills to an overflow page on these WITHOUT ROWID tables, which would
// slow every scan; Detail is an error string on the failure paths, so it needs a bound.
It("bounds a detail so one long error cannot inflate the row", func() {
steps := []TraceStep{{Candidate: "decode", Outcome: OutcomeError, Detail: strings.Repeat("x", 5000)}}
got := DecodeTrace(encodeSteps(steps, ""), "")
Expect(len(got[0].Detail)).To(BeNumerically("<=", 210))
Expect(got[0].Detail).To(HaveSuffix("..."))
Expect(got[0].Candidate).To(Equal("decode"), "truncating the detail must not disturb the step")
})
It("only restores sourcePath onto a detail-less hit", func() {
steps := []TraceStep{{Candidate: "cover.*", Outcome: OutcomeMiss}}
Expect(DecodeTrace(encodeSteps(steps, "/music/a/cover.jpg"), "/music/a/cover.jpg")).To(Equal(steps))
})
})
var _ = Describe("chainTrace", func() {
It("returns nil when no trace is attached", func() {
Expect(traceFrom(context.Background())).To(BeNil())
@ -113,63 +163,41 @@ var _ = Describe("chainState tracing", func() {
})
})
var _ = Describe("external gate tracing", func() {
hit := func() (io.ReadCloser, string, error) {
return io.NopCloser(strings.NewReader("x")), "http://img", nil
}
miss := func() (io.ReadCloser, string, error) { return nil, "", agents.ErrNotFound }
boom := func() (io.ReadCloser, string, error) { return nil, "", errors.New("returned status 429") }
var _ = Describe("external agent tracing", func() {
var (
t *ChainTrace
ctx context.Context
body io.ReadCloser
)
BeforeEach(func() {
t = &ChainTrace{}
ctx = withTrace(context.Background(), t)
body = io.NopCloser(strings.NewReader("x"))
})
It("records a hit with the image path", func() {
t := &ChainTrace{}
g := tracingGate(t, passthroughGate)
r, _, err := g("deezer", hit)
Expect(err).ToNot(HaveOccurred())
Expect(r).ToNot(BeNil())
recordAgent(ctx, "deezer", body, "http://img", nil)
Expect(t.Steps()).To(Equal([]TraceStep{
{Candidate: "external:deezer", Outcome: OutcomeHit, Detail: "http://img"},
}))
})
It("records a miss for a not-found", func() {
t := &ChainTrace{}
_, _, _ = tracingGate(t, passthroughGate)("deezer", miss)
recordAgent(ctx, "deezer", nil, "", agents.ErrNotFound)
Expect(t.Steps()[0].Outcome).To(Equal(OutcomeMiss))
})
It("records a miss for a model not-found", func() {
t := &ChainTrace{}
notFound := func() (io.ReadCloser, string, error) { return nil, "", model.ErrNotFound }
_, _, _ = tracingGate(t, passthroughGate)("deezer", notFound)
recordAgent(ctx, "deezer", nil, "", model.ErrNotFound)
Expect(t.Steps()[0].Outcome).To(Equal(OutcomeMiss),
"both not-found flavours are definitive answers, not faults")
})
It("records an error with its reason", func() {
t := &ChainTrace{}
_, _, _ = tracingGate(t, passthroughGate)("apple-music", boom)
recordAgent(ctx, "apple-music", nil, "", errors.New("returned status 429"))
Expect(t.Steps()[0].Outcome).To(Equal(OutcomeError))
Expect(t.Steps()[0].Detail).To(ContainSubstring("429"))
})
It("never calls the agent in offline mode", func() {
t := &ChainTrace{}
called := false
counting := func() (io.ReadCloser, string, error) {
called = true
return hit()
}
_, _, err := offlineGate(t)("deezer", counting)
Expect(called).To(BeFalse(), "offline mode must not perform external requests")
Expect(err).To(MatchError(errOfflineSkipped))
Expect(t.Steps()).To(Equal([]TraceStep{
{Candidate: "external:deezer", Outcome: OutcomeWouldTry},
}))
})
})
var _ = Describe("resolveAlbum tracing", func() {
@ -409,51 +437,51 @@ var _ = Describe("NewTracingResolver", func() {
t = &ChainTrace{}
})
Context("offline", func() {
Context("resolving", func() {
var fake *fakeImageAgent
BeforeEach(func() {
fake = &fakeImageAgent{name: "offline-probe"}
// Misses, so the chain falls through to the local tier and both are traced.
fake = &fakeImageAgent{name: "probe", err: agents.ErrNotFound}
albumRepo.SetData(model.Albums{{
ID: "al1", Name: "Album", EmbedArtPath: "tests/fixtures/artist/an-album/test.mp3", FolderIDs: []string{"f1"},
}})
artistRepo.SetData(model.Artists{{ID: "ar1", Name: "Artist"}})
})
It("reports the external tier without asking any agent", func() {
source, err := NewTracingResolver(ds, imageAgents(fake), ffm, t, false).Resolve(context.Background(), model.KindAlbumArtwork, "al1")
It("asks the agents and records what each answered", func() {
source, err := NewTracingResolver(ds, imageAgents(fake), ffm, t, true).Resolve(context.Background(), model.KindAlbumArtwork, "al1")
Expect(err).ToNot(HaveOccurred())
Expect(source).To(Equal("embedded"))
Expect(fake.albumCalls).To(BeZero(), "offline mode must not add load to an external provider")
Expect(t.Steps()).To(ContainElement(TraceStep{Candidate: "external:offline-probe", Outcome: OutcomeWouldTry}))
Expect(fake.albumCalls).To(Equal(1))
Expect(t.Steps()).To(ContainElement(TraceStep{Candidate: "external:probe", Outcome: OutcomeMiss}))
})
It("records the local chain steps too", func() {
_, err := NewTracingResolver(ds, imageAgents(fake), ffm, t, false).Resolve(context.Background(), model.KindAlbumArtwork, "al1")
_, err := NewTracingResolver(ds, imageAgents(fake), ffm, t, true).Resolve(context.Background(), model.KindAlbumArtwork, "al1")
Expect(err).ToNot(HaveOccurred())
last := t.Steps()[len(t.Steps())-1]
Expect(last.Candidate).To(Equal("embedded"), "the local chain must be traced, not just the external gate")
Expect(last.Candidate).To(Equal("embedded"), "the local chain must be traced, not just the external tier")
Expect(last.Outcome).To(Equal(OutcomeHit))
})
It("never persists artwork state", func() {
_, err := NewTracingResolver(ds, imageAgents(fake), ffm, t, false).Resolve(context.Background(), model.KindAlbumArtwork, "al1")
_, err := NewTracingResolver(ds, imageAgents(fake), ffm, t, true).Resolve(context.Background(), model.KindAlbumArtwork, "al1")
Expect(err).ToNot(HaveOccurred())
Expect(artworkRepo.ItemData).To(BeEmpty(),
"an offline resolution carries extError, which must never be recorded as a real provider failure")
"explain is read-only; a diagnostic walk must never become the stored answer")
Expect(queueRepo.Data).To(BeEmpty())
})
It("resolves an artist without persisting anything", func() {
source, err := NewTracingResolver(ds, imageAgents(fake), ffm, t, false).Resolve(context.Background(), model.KindArtistArtwork, "ar1")
source, err := NewTracingResolver(ds, imageAgents(fake), ffm, t, true).Resolve(context.Background(), model.KindArtistArtwork, "ar1")
Expect(err).ToNot(HaveOccurred())
Expect(source).To(BeEmpty())
Expect(fake.artistCalls).To(BeZero())
Expect(t.Steps()).To(ContainElement(TraceStep{Candidate: "external:offline-probe", Outcome: OutcomeWouldTry}))
Expect(fake.artistCalls).To(Equal(1))
Expect(artworkRepo.ItemData).To(BeEmpty())
Expect(queueRepo.Data).To(BeEmpty())
})
@ -465,31 +493,39 @@ var _ = Describe("NewTracingResolver", func() {
ID: "al2", Name: "Album", EmbedArtPath: "tests/fixtures/artist/an-album/no-such-file.mp3", FolderIDs: []string{"f1"},
}})
source, err := NewTracingResolver(ds, imageAgents(fake), ffm, t, false).Resolve(context.Background(), model.KindAlbumArtwork, "al2")
source, err := NewTracingResolver(ds, imageAgents(fake), ffm, t, true).Resolve(context.Background(), model.KindAlbumArtwork, "al2")
Expect(err).ToNot(HaveOccurred())
Expect(source).To(Equal("embedded"))
Expect(ffm.IsClosed()).To(BeTrue(), "nothing downstream closes it, so a leak is one file handle per invocation")
})
// Serving falls back disc -> album and track -> disc -> album. The resolver does not, but
// if it ever did, an explain without --live would start calling providers uninvited.
It("cannot reach a provider without live, whatever the chain does", func() {
conf.Server.DiscArtPriority = "external, cover.*"
conf.Server.CoverArtPriority = "external, cover.*"
conf.Server.EnableMediaFileCoverArt = true
mfRepo := tests.CreateMockMediaFileRepo()
mfRepo.SetData(model.MediaFiles{{ID: "mf1", LibraryID: 0, HasCoverArt: true,
Path: "tests/fixtures/artist/an-album/test.mp3"}})
ds.MockedMediaFile = mfRepo
offline := NewTracingResolver(ds, imageAgents(fake), ffm, t, false)
_, err := offline.Resolve(context.Background(), model.KindDiscArtwork, "al1:1")
Expect(err).ToNot(HaveOccurred())
_, err = offline.Resolve(context.Background(), model.KindMediaFileArtwork, "mf1")
Expect(err).ToNot(HaveOccurred())
Expect(fake.albumCalls).To(BeZero())
Expect(fake.artistCalls).To(BeZero())
})
It("propagates a lookup error", func() {
_, err := NewTracingResolver(ds, imageAgents(fake), ffm, t, false).Resolve(context.Background(), model.KindAlbumArtwork, "nope")
_, err := NewTracingResolver(ds, imageAgents(fake), ffm, t, true).Resolve(context.Background(), model.KindAlbumArtwork, "nope")
Expect(err).To(MatchError(model.ErrNotFound))
})
})
It("asks the agents when live is true", func() {
fake := &fakeImageAgent{name: "live-probe", err: agents.ErrNotFound}
albumRepo.SetData(model.Albums{{
ID: "al1", Name: "Album", EmbedArtPath: "tests/fixtures/artist/an-album/test.mp3", FolderIDs: []string{"f1"},
}})
_, err := NewTracingResolver(ds, imageAgents(fake), ffm, t, true).Resolve(context.Background(), model.KindAlbumArtwork, "al1")
Expect(err).ToNot(HaveOccurred())
Expect(fake.albumCalls).To(Equal(1))
Expect(t.Steps()).To(ContainElement(TraceStep{Candidate: "external:live-probe", Outcome: OutcomeMiss}))
})
})
var _ = Describe("resolveDisc tracing", func() {

39
core/artwork/worker.go
View File

@ -40,6 +40,7 @@ type drainPool struct {
// independently, and pruneMu serializes prune against the store-write window.
type Worker struct {
proc *processor
agents *agents.Agents
cache cache.FileCache
ffmpeg ffmpeg.FFmpeg
broker events.Broker
@ -54,6 +55,7 @@ type Worker struct {
func NewWorker(ds model.DataStore, store *ImageStore, ag *agents.Agents, ffmpeg ffmpeg.FFmpeg, broker events.Broker, imgCache cache.FileCache) *Worker {
w := &Worker{
proc: &processor{ds: ds, store: store},
agents: ag,
cache: imgCache,
ffmpeg: ffmpeg,
broker: broker,
@ -132,12 +134,14 @@ func (w *Worker) RunPrune(ctx context.Context) error {
}
// Backfill enqueues every entity for re-resolution when the artwork config fingerprint changed,
// artists first. It reports whether anything was enqueued.
// artists first. It reports whether the backfill ran.
func (w *Worker) Backfill(ctx context.Context) (bool, error) {
return backfill(ctx, w.proc.ds)
s, err := backfill(ctx, w.proc.ds, func() ImageAgentCount { return NewImageAgentCount(w.agents) })
return s.Ran, err
}
// EnqueueStaleAbsentAll requeues known-absent entries older than StaleAbsentAge.
// EnqueueStaleAbsentAll requeues known-absent entries older than StaleAbsentAge, at most
// StaleAbsentRecheckBatch per kind, oldest first.
func (w *Worker) EnqueueStaleAbsentAll(ctx context.Context) error {
return enqueueStaleAbsentAll(ctx, w.proc.ds)
}
@ -235,7 +239,9 @@ func (w *Worker) broadcastRefresh(ctx context.Context, found []model.ArtworkQueu
func (w *Worker) process(ctx context.Context, item model.ArtworkQueueItem) (outcome, *acquired) {
item.ImageType = cmp.Or(item.ImageType, model.ImageTypePrimary)
out, got := w.proc.acquire(ctx, item)
trace := &ChainTrace{}
ctx = withTrace(ctx, trace)
out, got, retryIn := w.proc.acquire(ctx, item)
queue := w.proc.ds.ArtworkQueue(ctx)
switch out {
@ -246,11 +252,12 @@ func (w *Worker) process(ctx context.Context, item model.ArtworkQueueItem) (outc
log.Warn(ctx, "Artwork: Could not delete processed queue item", "kind", item.ItemKind, "id", item.ItemID, err)
}
case outcomeFoundStale, outcomeFailed:
retryAt := time.Now().Add(backoff(item.Attempts))
retryAt := time.Now().Add(retryDelay(item.Attempts, retryIn))
encoded := trace.encode("")
if retryAt.Before(item.EnqueuedAt.Add(giveUpAfter)) {
// A mid-flight re-enqueue reset retry_at; stale backoff must not stomp its
// fresh, immediate eligibility.
if err := queue.MarkFailedIfUnchanged(item.ItemKind, item.ItemID, item.ImageType, item.RetryAt, retryAt); err != nil {
if err := queue.MarkFailedIfUnchanged(item.ItemKind, item.ItemID, item.ImageType, item.RetryAt, retryAt, encoded); err != nil {
log.Warn(ctx, "Artwork: Could not reschedule failed queue item", "kind", item.ItemKind, "id", item.ItemID, err)
}
log.Debug(ctx, "Artwork: Rescheduled item", "kind", item.ItemKind, "id", item.ItemID,
@ -265,6 +272,9 @@ func (w *Worker) process(ctx context.Context, item model.ArtworkQueueItem) (outc
writeAbsent(ctx, w.proc.ds.Artwork(ctx), item)
settled = "recorded absent"
}
// The queue row is about to go, taking the only record of the failure with it. This write is
// unconditional (not CAS-guarded) — safe only because the drain resolves each item serially.
w.recordGiveUp(ctx, item, encoded)
log.Info(ctx, "Artwork: Retry budget exhausted, giving up", "kind", item.ItemKind, "id", item.ItemID,
"outcome", out, "attempts", item.Attempts+1, "budget", giveUpAfter, "settled", settled)
if err := queue.DeleteIfUnchanged(item.ItemKind, item.ItemID, item.ImageType, item.RetryAt); err != nil {
@ -274,6 +284,18 @@ func (w *Worker) process(ctx context.Context, item model.ArtworkQueueItem) (outc
return out, got
}
// recordGiveUp keeps the last failure on the state row after the queue row is deleted. An item
// that never resolved has no row to update, and creating one would settle it absent.
func (w *Worker) recordGiveUp(ctx context.Context, item model.ArtworkQueueItem, trace string) {
kind, ok := model.ParseKind(item.ItemKind)
if !ok {
return
}
if err := w.proc.ds.Artwork(ctx).PutLastFailure(kind, item.ItemID, item.ImageType, trace); err != nil {
log.Warn(ctx, "Artwork: Could not record the last failure", "kind", item.ItemKind, "id", item.ItemID, err)
}
}
func (w *Worker) hasResolvedArtwork(ctx context.Context, item model.ArtworkQueueItem) bool {
kind, ok := model.ParseKind(item.ItemKind)
if !ok {
@ -319,3 +341,8 @@ func backoffFor(attempts int, jitter float64) time.Duration {
func backoff(attempts int) time.Duration {
return backoffFor(attempts, rand.Float64()*0.8-0.4) //nolint:gosec // retry jitter, not security-sensitive
}
// retryDelay is how long a failed item waits: our backoff, unless the provider asked for longer.
func retryDelay(attempts int, hint time.Duration) time.Duration {
return max(backoff(attempts), hint)
}

2
core/artwork/worker_soak_test.go
View File

@ -95,7 +95,7 @@ var _ = Describe("Worker soak", func() {
start := time.Now()
for i := range soakCycles {
it := items[i%len(items)]
out, _ := proc.acquire(context.Background(), it)
out, _, _ := proc.acquire(context.Background(), it)
// Read-back exercises the surfaces a caller would use after acquisition.
if out == outcomeFound {

128
core/artwork/worker_test.go
View File

@ -95,6 +95,17 @@ func (f *fakeEventBroker) getEvents() []events.Event {
var _ events.Broker = (*fakeEventBroker)(nil)
// expireQueued ages a row past the retry budget, so the next drain settles it instead of retrying.
func expireQueued(q *tests.MockArtworkQueueRepo, id string) {
GinkgoHelper()
for k, v := range q.Data {
if v.ItemID == id {
v.EnqueuedAt = time.Now().Add(-(giveUpAfter + time.Hour))
q.Data[k] = v
}
}
}
func findQueued(q *tests.MockArtworkQueueRepo, kind, id string) *model.ArtworkQueueItem {
for _, it := range q.Data {
if it.ItemKind == kind && it.ItemID == id {
@ -234,6 +245,23 @@ var _ = Describe("Worker", func() {
Expect(err).To(MatchError(model.ErrNotFound), "a timeout must never settle on absent")
})
It("reschedules past the provider's requested delay when it exceeds the backoff", func() {
conf.Server.CoverArtPriority = "external"
ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{{ID: "al9", Name: "Album"}})
// Well above backoff(0)'s jittered ceiling, so only the hint can produce this retry_at.
const askedFor = 90 * time.Minute
imageAgents(&fakeImageAgent{name: "throttledAgent", err: &agents.RetryLaterError{RetryIn: askedFor}})
Expect(queueRepo.Enqueue(model.ArtworkQueueItem{ItemKind: "al", ItemID: "al9"})).To(Succeed())
n, err := w.drain(ctx, 2)
Expect(err).ToNot(HaveOccurred())
Expect(n).To(Equal(1))
it := findQueued(queueRepo, "al", "al9")
Expect(it).ToNot(BeNil())
Expect(it.RetryAt).To(BeTemporally("~", time.Now().Add(askedFor), time.Minute))
})
It("reschedules a found-stale item via MarkFailed while keeping its served state", func() {
conf.Server.CoverArtPriority = "external, cover.jpg"
folderRepo.result = []model.Folder{{
@ -318,12 +346,7 @@ var _ = Describe("Worker", func() {
w = NewWorker(ds, store, ag, ffm, broker, imgCache)
Expect(queueRepo.Enqueue(model.ArtworkQueueItem{ItemKind: "al", ItemID: "al9"})).To(Succeed())
// Age the row past the retry budget.
for k, v := range queueRepo.Data {
if v.ItemID == "al9" {
v.EnqueuedAt = time.Now().Add(-(giveUpAfter + time.Hour))
queueRepo.Data[k] = v
}
}
expireQueued(queueRepo, "al9")
n, err := w.drain(ctx, 1)
Expect(err).ToNot(HaveOccurred())
@ -345,12 +368,7 @@ var _ = Describe("Worker", func() {
imageAgents(&fakeImageAgent{name: "failAgent", err: errors.New("agent timed out")})
w = NewWorker(ds, store, ag, ffm, broker, imgCache)
Expect(queueRepo.Enqueue(model.ArtworkQueueItem{ItemKind: "al", ItemID: "al10"})).To(Succeed())
for k, v := range queueRepo.Data {
if v.ItemID == "al10" {
v.EnqueuedAt = time.Now().Add(-(giveUpAfter + time.Hour))
queueRepo.Data[k] = v
}
}
expireQueued(queueRepo, "al10")
n, err := w.drain(ctx, 1)
Expect(err).ToNot(HaveOccurred())
@ -362,6 +380,67 @@ var _ = Describe("Worker", func() {
Expect(ia.Hash).To(Equal("cafebabe"), "a persistent outage must not discard served art")
})
It("records on the queue row why the last attempt failed", func() {
conf.Server.CoverArtPriority = "external"
ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{{ID: "al11", Name: "Album"}})
imageAgents(&fakeImageAgent{name: "failAgent", err: errors.New("agent timed out")})
w = NewWorker(ds, store, ag, ffm, broker, imgCache)
Expect(queueRepo.Enqueue(model.ArtworkQueueItem{ItemKind: "al", ItemID: "al11"})).To(Succeed())
_, err := w.drain(ctx, 1)
Expect(err).ToNot(HaveOccurred())
it := findQueued(queueRepo, "al", "al11")
Expect(it).ToNot(BeNil())
Expect(DecodeTrace(it.Trace, "")).To(ContainElement(SatisfyAll(
HaveField("Candidate", "external:failAgent"),
HaveField("Outcome", OutcomeError),
HaveField("Detail", ContainSubstring("agent timed out")),
)), "a retrying row must say why it is retrying")
})
// The give-up path settles absent before recording, so the row exists by the time the
// failure is written. Recording first would silently lose it for every unresolved item.
It("keeps the failure for an item that never resolved at all", func() {
conf.Server.CoverArtPriority = "external"
ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{{ID: "al13", Name: "Album"}})
imageAgents(&fakeImageAgent{name: "failAgent", err: errors.New("agent timed out")})
w = NewWorker(ds, store, ag, ffm, broker, imgCache)
Expect(queueRepo.Enqueue(model.ArtworkQueueItem{ItemKind: "al", ItemID: "al13"})).To(Succeed())
expireQueued(queueRepo, "al13")
_, err := w.drain(ctx, 1)
Expect(err).ToNot(HaveOccurred())
ia, err := artRepo.GetItemArtwork(model.KindAlbumArtwork, "al13", model.ImageTypePrimary)
Expect(err).ToNot(HaveOccurred(), "settling absent must create the row the failure is written to")
Expect(ia.Hash).To(BeEmpty())
Expect(DecodeTrace(ia.LastFailure, "")).ToNot(BeEmpty())
})
It("keeps the failure on the state row after the queue row is deleted", func() {
conf.Server.CoverArtPriority = "external"
ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{{ID: "al12", Name: "Album"}})
Expect(artRepo.PutItemArtwork(&model.ItemArtwork{
ItemKind: "al", ItemID: "al12", ImageType: model.ImageTypePrimary,
Hash: "cafebabe", Source: "external:lastfm",
})).To(Succeed())
imageAgents(&fakeImageAgent{name: "failAgent", err: errors.New("agent timed out")})
w = NewWorker(ds, store, ag, ffm, broker, imgCache)
Expect(queueRepo.Enqueue(model.ArtworkQueueItem{ItemKind: "al", ItemID: "al12"})).To(Succeed())
expireQueued(queueRepo, "al12")
_, err := w.drain(ctx, 1)
Expect(err).ToNot(HaveOccurred())
Expect(findQueued(queueRepo, "al", "al12")).To(BeNil())
ia, err := artRepo.GetItemArtwork(model.KindAlbumArtwork, "al12", model.ImageTypePrimary)
Expect(err).ToNot(HaveOccurred())
Expect(DecodeTrace(ia.LastFailure, "")).ToNot(BeEmpty(),
"the queue row is gone, so this is the only remaining record of the failure")
Expect(ia.Hash).To(Equal("cafebabe"), "recording the failure must not disturb the served art")
})
// Media files are excluded from RecheckKinds, so an absent row here would never be
// revisited: a transient read error would look permanent.
It("does not settle absent on exhaustion for a kind with no recheck path", func() {
@ -371,12 +450,7 @@ var _ = Describe("Worker", func() {
{ID: "mfX", LibraryID: 0, Path: "tests/fixtures/artist/an-album/gone.mp3", HasCoverArt: true},
})
Expect(queueRepo.Enqueue(model.ArtworkQueueItem{ItemKind: "mf", ItemID: "mfX"})).To(Succeed())
for k, v := range queueRepo.Data {
if v.ItemID == "mfX" {
v.EnqueuedAt = time.Now().Add(-(giveUpAfter + time.Hour))
queueRepo.Data[k] = v
}
}
expireQueued(queueRepo, "mfX")
n, err := w.drain(ctx, 1)
Expect(err).ToNot(HaveOccurred())
@ -386,6 +460,8 @@ var _ = Describe("Worker", func() {
_, err = artRepo.GetItemArtwork(model.KindMediaFileArtwork, "mfX", model.ImageTypePrimary)
Expect(err).To(MatchError(model.ErrNotFound),
"no row leaves the track unresolved, so a later view can still recover it")
// Known gap: with no row and no absent settle, there is nowhere to keep the failure.
// Creating one here would write an empty hash, which every reader treats as absent.
})
It("resolves a private playlist under an admin context instead of failing forever", func() {
@ -852,3 +928,19 @@ var _ = Describe("backoff", func() {
}
})
})
var _ = Describe("retryDelay", func() {
It("uses the backoff schedule when the provider asked for nothing", func() {
d := retryDelay(0, 0)
Expect(d).To(BeNumerically(">=", 3*time.Second))
Expect(d).To(BeNumerically("<=", 7*time.Second))
})
It("waits the provider's delay when it is longer than the backoff", func() {
Expect(retryDelay(0, time.Hour)).To(Equal(time.Hour))
})
It("keeps the backoff when it is longer than the provider's delay", func() {
Expect(retryDelay(4, time.Second)).To(BeNumerically(">=", 3*time.Second))
})
})

59
core/auth/auth.go
View File

@ -4,6 +4,8 @@ import (
"cmp"
"context"
"crypto/sha256"
"errors"
"slices"
"sync"
"time"
@ -26,6 +28,13 @@ var (
PublicTokenAuth *jwtauth.JWTAuth
)
// Audiences a session token can be scoped to. A token with no audience is accepted anywhere.
const (
AudienceJellyfin = "jellyfin"
AudienceSubsonic = "subsonic"
AudienceNative = "native"
)
// Init creates the JWTAuth objects from the secrets stored in the DB.
// Missing or undecryptable secrets are regenerated and stored.
func Init(ds model.DataStore) {
@ -66,15 +75,20 @@ func CreateExpiringPublicToken(exp time.Time, claims Claims) (string, error) {
return token, err
}
func CreateToken(u *model.User) (string, error) {
claims := Claims{
func userClaims(u *model.User, audience []string) Claims {
return Claims{
Issuer: consts.JWTIssuer,
Subject: u.UserName,
IssuedAt: time.Now(),
UserID: u.ID,
IsAdmin: u.IsAdmin,
Epoch: u.TokenEpoch,
Audience: audience,
}
token, _, err := TokenAuth.Encode(claims.ToMap())
}
func CreateToken(u *model.User) (string, error) {
token, _, err := TokenAuth.Encode(userClaims(u, nil).ToMap())
if err != nil {
return "", err
}
@ -82,10 +96,20 @@ func CreateToken(u *model.User) (string, error) {
return TouchToken(token)
}
// CreateAPIToken mints a non-expiring token scoped to one API, matching how Jellyfin
// clients expect tokens to behave. Revocation is by token epoch, not expiry.
func CreateAPIToken(u *model.User, audience string) (string, error) {
_, token, err := TokenAuth.Encode(userClaims(u, []string{audience}).ToMap())
return token, err
}
func TouchToken(token jwt.Token) (string, error) {
claims := ClaimsFromToken(token).
WithExpiresAt(time.Now().UTC().Add(conf.Server.SessionTimeout))
_, newToken, err := TokenAuth.Encode(claims.ToMap())
return TouchClaims(ClaimsFromToken(token))
}
func TouchClaims(c Claims) (string, error) {
c = c.WithExpiresAt(time.Now().UTC().Add(conf.Server.SessionTimeout))
_, newToken, err := TokenAuth.Encode(c.ToMap())
return newToken, err
}
@ -106,6 +130,29 @@ func ValidatePublic(tokenStr string) (Claims, error) {
return ClaimsFromToken(token), nil
}
var (
ErrTokenRevoked = errors.New("token revoked")
ErrWrongAudience = errors.New("token not valid for this API")
ErrWrongUser = errors.New("token issued for a different user")
)
// CheckClaims gates a session token against the user it names. Callers must have already
// verified the signature; this adds revocation and API scoping on top.
func CheckClaims(c Claims, usr model.User, audience string) error {
// Usernames can be reused: deleting a user and recreating the name yields a new random id
// at epoch 0, which an old token would otherwise match.
if c.UserID != "" && c.UserID != usr.ID {
return ErrWrongUser
}
if c.Epoch != usr.TokenEpoch {
return ErrTokenRevoked
}
if len(c.Audience) > 0 && !slices.Contains(c.Audience, audience) {
return ErrWrongAudience
}
return nil
}
func WithAdminUser(ctx context.Context, ds model.DataStore) context.Context {
u, err := ds.User(ctx).FindFirstAdmin()
if err != nil {

109
core/auth/auth_test.go
View File

@ -151,4 +151,113 @@ var _ = Describe("Auth", func() {
Expect(decodedClaims.ExpiresAt.Sub(yesterday)).To(BeNumerically(">=", oneDay))
})
})
Describe("CreateAPIToken", func() {
var usr *model.User
BeforeEach(func() {
usr = &model.User{ID: "123", UserName: "johndoe", TokenEpoch: 4}
})
It("does not expire", func() {
tokenStr, err := auth.CreateAPIToken(usr, auth.AudienceJellyfin)
Expect(err).ToNot(HaveOccurred())
claims, err := auth.Validate(tokenStr)
Expect(err).ToNot(HaveOccurred())
Expect(claims.ExpiresAt.IsZero()).To(BeTrue())
})
It("carries the audience and the user's epoch", func() {
tokenStr, err := auth.CreateAPIToken(usr, auth.AudienceJellyfin)
Expect(err).ToNot(HaveOccurred())
claims, err := auth.Validate(tokenStr)
Expect(err).ToNot(HaveOccurred())
Expect(claims.Audience).To(Equal([]string{"jellyfin"}))
Expect(claims.Epoch).To(Equal(4))
Expect(claims.Subject).To(Equal("johndoe"))
Expect(claims.UserID).To(Equal("123"))
})
})
Describe("CreateToken with an epoch", func() {
It("carries the epoch and still expires", func() {
usr := &model.User{ID: "123", UserName: "johndoe", TokenEpoch: 9}
tokenStr, err := auth.CreateToken(usr)
Expect(err).ToNot(HaveOccurred())
claims, err := auth.Validate(tokenStr)
Expect(err).ToNot(HaveOccurred())
Expect(claims.Epoch).To(Equal(9))
Expect(claims.Audience).To(BeEmpty())
Expect(claims.ExpiresAt).To(BeTemporally(">", time.Now()))
})
})
Describe("TouchClaims", func() {
It("preserves custom claims and refreshes the expiry", func() {
tokenStr, err := auth.TouchClaims(auth.Claims{Subject: "johndoe", UserID: "123", Epoch: 5})
Expect(err).ToNot(HaveOccurred())
claims, err := auth.Validate(tokenStr)
Expect(err).ToNot(HaveOccurred())
Expect(claims.Epoch).To(Equal(5))
Expect(claims.Subject).To(Equal("johndoe"))
Expect(claims.ExpiresAt).To(BeTemporally(">", time.Now()))
})
})
Describe("CheckClaims", func() {
usr := model.User{ID: "123", UserName: "johndoe", TokenEpoch: 2}
It("accepts a matching epoch and audience", func() {
c := auth.Claims{Epoch: 2, Audience: []string{auth.AudienceJellyfin}}
Expect(auth.CheckClaims(c, usr, auth.AudienceJellyfin)).To(Succeed())
})
It("accepts a token with no audience on any API", func() {
c := auth.Claims{Epoch: 2}
Expect(auth.CheckClaims(c, usr, auth.AudienceNative)).To(Succeed())
Expect(auth.CheckClaims(c, usr, auth.AudienceJellyfin)).To(Succeed())
Expect(auth.CheckClaims(c, usr, auth.AudienceSubsonic)).To(Succeed())
})
It("rejects a stale epoch", func() {
c := auth.Claims{Epoch: 1, Audience: []string{auth.AudienceJellyfin}}
Expect(auth.CheckClaims(c, usr, auth.AudienceJellyfin)).To(MatchError(auth.ErrTokenRevoked))
})
It("rejects a token minted for another API", func() {
c := auth.Claims{Epoch: 2, Audience: []string{auth.AudienceJellyfin}}
Expect(auth.CheckClaims(c, usr, auth.AudienceNative)).To(MatchError(auth.ErrWrongAudience))
Expect(auth.CheckClaims(c, usr, auth.AudienceSubsonic)).To(MatchError(auth.ErrWrongAudience))
})
It("accepts a multi-audience token that includes this API", func() {
c := auth.Claims{Epoch: 2, Audience: []string{"other", auth.AudienceNative}}
Expect(auth.CheckClaims(c, usr, auth.AudienceNative)).To(Succeed())
})
It("accepts a pre-upgrade token against a never-bumped user", func() {
fresh := model.User{ID: "456", UserName: "newbie"}
Expect(auth.CheckClaims(auth.Claims{}, fresh, auth.AudienceNative)).To(Succeed())
})
It("accepts a token whose user id matches", func() {
c := auth.Claims{UserID: "123", Epoch: 2}
Expect(auth.CheckClaims(c, usr, auth.AudienceNative)).To(Succeed())
})
It("rejects a token for a deleted user recreated under the same name", func() {
recreated := model.User{ID: "new-random-id", UserName: "johndoe"}
c := auth.Claims{UserID: "123", Audience: []string{auth.AudienceJellyfin}}
Expect(auth.CheckClaims(c, recreated, auth.AudienceJellyfin)).To(MatchError(auth.ErrWrongUser))
})
It("accepts a token that carries no user id", func() {
fresh := model.User{ID: "456", UserName: "newbie"}
Expect(auth.CheckClaims(auth.Claims{}, fresh, auth.AudienceNative)).To(Succeed())
})
})
})

32
core/auth/claims.go
View File

@ -11,7 +11,8 @@ import (
type Claims struct {
// Standard JWT claims
Issuer string
Subject string // username for session tokens
Subject string // username for session tokens
Audience []string // which API may accept this token; empty means any
IssuedAt time.Time
ExpiresAt time.Time
@ -22,6 +23,7 @@ type Claims struct {
Format string // "f" - audio format
BitRate int // "b" - audio bitrate
ShareID string // "sid" - share ID for share stream tokens
Epoch int // "ep" - the user's token_epoch at mint time
}
// ToMap converts Claims to a map[string]any for use with TokenAuth.Encode().
@ -34,6 +36,9 @@ func (c Claims) ToMap() map[string]any {
if c.Subject != "" {
m[jwt.SubjectKey] = c.Subject
}
if len(c.Audience) > 0 {
m[jwt.AudienceKey] = c.Audience
}
if !c.IssuedAt.IsZero() {
m[jwt.IssuedAtKey] = c.IssuedAt.UTC().Unix()
}
@ -58,6 +63,9 @@ func (c Claims) ToMap() map[string]any {
if c.ShareID != "" {
m["sid"] = c.ShareID
}
if c.Epoch != 0 {
m["ep"] = c.Epoch
}
return m
}
@ -73,6 +81,7 @@ func ClaimsFromToken(token jwt.Token) Claims {
c.Subject, _ = token.Subject()
c.IssuedAt, _ = token.IssuedAt()
c.ExpiresAt, _ = token.Expiration()
c.Audience, _ = token.Audience()
var uid string
if err := token.Get("uid", &uid); err == nil {
@ -90,15 +99,24 @@ func ClaimsFromToken(token jwt.Token) Claims {
if err := token.Get("f", &f); err == nil {
c.Format = f
}
if err := token.Get("b", &c.BitRate); err != nil {
var bf float64
if err := token.Get("b", &bf); err == nil {
c.BitRate = int(bf)
}
}
c.BitRate = intClaim(token, "b")
var sid string
if err := token.Get("sid", &sid); err == nil {
c.ShareID = sid
}
c.Epoch = intClaim(token, "ep")
return c
}
// intClaim reads a numeric claim, which a parsed token may decode as either int or float64.
func intClaim(token jwt.Token, key string) int {
var i int
if err := token.Get(key, &i); err == nil {
return i
}
var f float64
if err := token.Get(key, &f); err == nil {
return int(f)
}
return 0
}

40
core/auth/claims_test.go
View File

@ -105,4 +105,44 @@ var _ = Describe("Claims", func() {
})
})
Describe("Audience and Epoch claims", func() {
It("omits both when zero", func() {
m := auth.Claims{ID: "artwork-id"}.ToMap()
Expect(m).ToNot(HaveKey("aud"))
Expect(m).ToNot(HaveKey("ep"))
})
It("includes them when set", func() {
m := auth.Claims{Subject: "u", Epoch: 3, Audience: []string{"jellyfin"}}.ToMap()
Expect(m).To(HaveKeyWithValue("ep", 3))
Expect(m).To(HaveKeyWithValue("aud", []string{"jellyfin"}))
})
It("round-trips through a signed token", func() {
tokenAuth := jwtauth.New("HS256", []byte("test-secret"), nil)
_, tokenStr, err := tokenAuth.Encode(auth.Claims{
Subject: "u", Epoch: 7, Audience: []string{"jellyfin"},
}.ToMap())
Expect(err).ToNot(HaveOccurred())
token, err := jwtauth.VerifyToken(tokenAuth, tokenStr)
Expect(err).ToNot(HaveOccurred())
claims := auth.ClaimsFromToken(token)
Expect(claims.Epoch).To(Equal(7))
Expect(claims.Audience).To(Equal([]string{"jellyfin"}))
})
It("reads a token that has neither claim", func() {
tokenAuth := jwtauth.New("HS256", []byte("test-secret"), nil)
_, tokenStr, err := tokenAuth.Encode(auth.Claims{Subject: "u"}.ToMap())
Expect(err).ToNot(HaveOccurred())
token, err := jwtauth.VerifyToken(tokenAuth, tokenStr)
Expect(err).ToNot(HaveOccurred())
claims := auth.ClaimsFromToken(token)
Expect(claims.Epoch).To(BeZero())
Expect(claims.Audience).To(BeEmpty())
})
})
})

110
core/external/provider.go vendored
View File

@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
"slices"
"sort"
"strings"
"time"
@ -14,6 +15,7 @@ import (
"github.com/navidrome/navidrome/core/matcher"
"github.com/navidrome/navidrome/log"
"github.com/navidrome/navidrome/model"
"github.com/navidrome/navidrome/server/events"
"github.com/navidrome/navidrome/utils"
. "github.com/navidrome/navidrome/utils/gg"
"github.com/navidrome/navidrome/utils/slice"
@ -33,12 +35,14 @@ type Provider interface {
UpdateArtistInfo(ctx context.Context, id string, count int, includeNotPresent bool) (*model.Artist, error)
SimilarSongs(ctx context.Context, id string, count int) (model.MediaFiles, error)
TopSongs(ctx context.Context, artist, artistId string, count int) (model.MediaFiles, error)
RefreshInfo(ctx context.Context, kind model.Kind, id string) error
}
type provider struct {
ds model.DataStore
ag Agents
matcher *matcher.Matcher
broker events.Broker
artistQueue refreshQueue[auxArtist]
albumQueue refreshQueue[auxAlbum]
}
@ -83,13 +87,17 @@ type Agents interface {
agents.SimilarSongsByArtistRetriever
}
func NewProvider(ds model.DataStore, agents Agents, m *matcher.Matcher) Provider {
e := &provider{ds: ds, ag: agents, matcher: m}
func NewProvider(ds model.DataStore, agents Agents, m *matcher.Matcher, broker events.Broker) Provider {
e := &provider{ds: ds, ag: agents, matcher: m, broker: broker}
e.artistQueue = newRefreshQueue(context.TODO(), e.populateArtistInfo)
e.albumQueue = newRefreshQueue(context.TODO(), e.populateAlbumInfo)
return e
}
func (e *provider) broadcastRefresh(ctx context.Context, resource, id string) {
e.broker.SendBroadcastMessage(ctx, (&events.RefreshResource{}).With(resource, id))
}
func (e *provider) getAlbum(ctx context.Context, id string) (auxAlbum, error) {
var entity any
entity, err := model.GetEntityByID(ctx, e.ds, id)
@ -140,7 +148,8 @@ func (e *provider) populateAlbumInfo(ctx context.Context, album auxAlbum) (auxAl
start := time.Now()
albumName := album.Name()
info, err := e.ag.GetAlbumInfo(ctx, albumName, album.AlbumArtist, album.MbzAlbumID)
if errors.Is(err, agents.ErrNotFound) {
// Throttled joins not-found: no answer to store, and an unstamped timestamp retries next call.
if errors.Is(err, agents.ErrNotFound) || errors.Is(err, agents.ErrRetryLater) {
return album, nil
}
if err != nil {
@ -179,6 +188,7 @@ func (e *provider) populateAlbumInfo(ctx context.Context, album auxAlbum) (auxAl
"elapsed", time.Since(start), err)
} else {
log.Trace(ctx, "AlbumInfo collected", "album", album, "elapsed", time.Since(start))
e.broadcastRefresh(ctx, "album", album.ID)
}
return album, nil
@ -244,38 +254,81 @@ func (e *provider) populateArtistInfo(ctx context.Context, artist auxArtist) (au
start := time.Now()
// Get MBID first, if it is not yet available
artistName := artist.Name()
var mbidErr error
if artist.MbzArtistID == "" {
mbid, err := e.ag.GetArtistMBID(ctx, artist.ID, artistName)
mbidErr = err
if mbid != "" && err == nil {
artist.MbzArtistID = mbid
}
}
// Call all registered agents and collect information
// Call all registered agents and collect information. The group carries no context, so a
// returned error does not cancel the siblings; only throttling is reported back.
g := errgroup.Group{}
g.SetLimit(2)
g.Go(func() error { _ = e.callGetImage(ctx, e.ag, &artist); return nil })
g.Go(func() error { e.callGetBiography(ctx, e.ag, &artist); return nil })
g.Go(func() error { e.callGetURL(ctx, e.ag, &artist); return nil })
g.Go(func() error { e.callGetSimilarArtists(ctx, e.ag, &artist, maxSimilarArtists, true); return nil })
_ = g.Wait()
g.Go(func() error { return retryLaterOnly(e.callGetImage(ctx, e.ag, &artist)) })
g.Go(func() error { return retryLaterOnly(e.callGetBiography(ctx, e.ag, &artist)) })
g.Go(func() error { return retryLaterOnly(e.callGetURL(ctx, e.ag, &artist)) })
g.Go(func() error {
return retryLaterOnly(e.callGetSimilarArtists(ctx, e.ag, &artist, maxSimilarArtists, true))
})
throttled := errors.Is(g.Wait(), agents.ErrRetryLater) || errors.Is(mbidErr, agents.ErrRetryLater)
if utils.IsCtxDone(ctx) {
log.Warn(ctx, "ArtistInfo update canceled", "id", artist.ID, "name", artistName, "elapsed", time.Since(start), ctx.Err())
return artist, ctx.Err()
}
artist.ExternalInfoUpdatedAt = new(time.Now())
// A throttled round keeps the previous timestamp, so the next call retries instead of
// serving an empty cache entry for the whole TTL.
if !throttled {
artist.ExternalInfoUpdatedAt = new(time.Now())
}
err := e.ds.Artist(ctx).UpdateExternalInfo(&artist.Artist)
if err != nil {
log.Error(ctx, "Error trying to update artist external information", "id", artist.ID, "name", artistName,
"elapsed", time.Since(start), err)
} else {
log.Trace(ctx, "ArtistInfo collected", "artist", artist, "elapsed", time.Since(start))
e.broadcastRefresh(ctx, "artist", artist.ID)
}
return artist, nil
}
// infoKinds are the kinds RefreshInfo can act on. Callers check this instead of restating
// the set, so the switch below stays the only place that has to know how each kind loads.
var infoKinds = []model.Kind{model.KindArtistArtwork, model.KindAlbumArtwork}
// HasInfo reports whether a kind has external info to refresh.
func HasInfo(kind model.Kind) bool { return slices.Contains(infoKinds, kind) }
// RefreshInfo re-fetches external info for one item, ignoring the TTL. It is synchronous:
// callers that must not block are responsible for detaching it.
func (e *provider) RefreshInfo(ctx context.Context, kind model.Kind, id string) error {
ctx, cancel := context.WithTimeout(ctx, refreshTimeout)
defer cancel()
switch kind {
case model.KindArtistArtwork:
artist, err := e.getArtist(ctx, id)
if err != nil {
return err
}
_, err = e.populateArtistInfo(ctx, artist)
return err
case model.KindAlbumArtwork:
album, err := e.getAlbum(ctx, id)
if err != nil {
return err
}
_, err = e.populateAlbumInfo(ctx, album)
return err
default:
return model.ErrNotFound
}
}
func (e *provider) TopSongs(ctx context.Context, artistName, id string, count int) (model.MediaFiles, error) {
artist, err := e.findArtist(ctx, artistName, id)
if err != nil {
@ -291,8 +344,9 @@ func (e *provider) TopSongs(ctx context.Context, artistName, id string, count in
songs, err := e.getMatchingTopSongs(ctx, e.ag, artist, count)
if err != nil {
switch {
case errors.Is(err, agents.ErrNotFound):
log.Trace(ctx, "TopSongs not found", "name", artistName)
// Throttled is not an answer, but the caller keeps the empty 200 it got before.
case errors.Is(err, agents.ErrNotFound), errors.Is(err, agents.ErrRetryLater):
log.Trace(ctx, "TopSongs not found", "name", artistName, err)
return nil, model.ErrNotFound
case errors.Is(err, context.Canceled):
log.Debug(ctx, "TopSongs call canceled", err)
@ -342,22 +396,33 @@ func (e *provider) getMatchingTopSongs(ctx context.Context, agent agents.ArtistT
return mfs, nil
}
func (e *provider) callGetURL(ctx context.Context, agent agents.ArtistURLRetriever, artist *auxArtist) {
artisURL, err := agent.GetArtistURL(ctx, artist.ID, artist.Name(), artist.MbzArtistID)
if err != nil {
return
// retryLaterOnly discards every failure the caller does not act on, so errgroup's
// first-error slot is reserved for the throttling signal.
func retryLaterOnly(err error) error {
if errors.Is(err, agents.ErrRetryLater) {
return err
}
artist.ExternalUrl = artisURL
return nil
}
func (e *provider) callGetBiography(ctx context.Context, agent agents.ArtistBiographyRetriever, artist *auxArtist) {
func (e *provider) callGetURL(ctx context.Context, agent agents.ArtistURLRetriever, artist *auxArtist) error {
artisURL, err := agent.GetArtistURL(ctx, artist.ID, artist.Name(), artist.MbzArtistID)
if err != nil {
return err
}
artist.ExternalUrl = artisURL
return nil
}
func (e *provider) callGetBiography(ctx context.Context, agent agents.ArtistBiographyRetriever, artist *auxArtist) error {
bio, err := agent.GetArtistBiography(ctx, artist.ID, artist.Name(), artist.MbzArtistID)
if err != nil {
return
return err
}
bio = str.SanitizeText(bio)
bio = strings.ReplaceAll(bio, "\n", " ")
artist.Biography = strings.ReplaceAll(bio, "<a ", "<a target='_blank' ")
return nil
}
// callGetImage populates artist's image URLs. A transient agent failure is
@ -385,19 +450,20 @@ func (e *provider) callGetImage(ctx context.Context, agent agents.ArtistImageRet
}
func (e *provider) callGetSimilarArtists(ctx context.Context, agent agents.ArtistSimilarRetriever, artist *auxArtist,
limit int, includeNotPresent bool) {
limit int, includeNotPresent bool) error {
artistName := artist.Name()
similar, err := agent.GetSimilarArtists(ctx, artist.ID, artistName, artist.MbzArtistID, limit)
if len(similar) == 0 || err != nil {
return
return err
}
start := time.Now()
sa, err := e.mapSimilarArtists(ctx, similar, limit, includeNotPresent)
log.Debug(ctx, "Mapped Similar Artists", "artist", artistName, "numSimilar", len(sa), "elapsed", time.Since(start))
if err != nil {
return
return err
}
artist.SimilarArtists = sa
return nil
}
func (e *provider) mapSimilarArtists(ctx context.Context, similar []agents.Artist, limit int, includeNotPresent bool) (model.Artists, error) {

156
core/external/provider_refreshinfo_test.go vendored Normal file
View File

@ -0,0 +1,156 @@
package external_test
import (
"context"
"slices"
"sync"
"time"
"github.com/navidrome/navidrome/core/agents"
"github.com/navidrome/navidrome/core/external"
"github.com/navidrome/navidrome/core/matcher"
"github.com/navidrome/navidrome/model"
"github.com/navidrome/navidrome/server/events"
"github.com/navidrome/navidrome/tests"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/stretchr/testify/mock"
)
type fakeBroker struct {
events.Broker
mu sync.Mutex
events []events.Event
}
func (f *fakeBroker) SendBroadcastMessage(_ context.Context, e events.Event) {
f.mu.Lock()
defer f.mu.Unlock()
f.events = append(f.events, e)
}
func (f *fakeBroker) sent() []events.Event {
f.mu.Lock()
defer f.mu.Unlock()
return slices.Clone(f.events)
}
var _ = Describe("Provider - RefreshInfo", func() {
var (
ctx context.Context
p external.Provider
ds *tests.MockDataStore
ag *mockAgents
broker *fakeBroker
mockArtistRepo *tests.MockArtistRepo
mockAlbumRepo *tests.MockAlbumRepo
)
expectArtistAgents := func() {
ag.On("GetArtistMBID", mock.Anything, mock.Anything, mock.Anything).Return("mbid-1", nil)
ag.On("GetArtistImages", mock.Anything, mock.Anything, mock.Anything, mock.Anything).
Return([]agents.ExternalImage{}, nil)
ag.On("GetArtistBiography", mock.Anything, mock.Anything, mock.Anything, mock.Anything).
Return("Fresh Bio", nil)
ag.On("GetArtistURL", mock.Anything, mock.Anything, mock.Anything, mock.Anything).
Return("http://artist.url", nil)
ag.On("GetSimilarArtists", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).
Return([]agents.Artist{}, nil)
}
expectAlbumAgents := func() {
ag.On("GetAlbumInfo", mock.Anything, mock.Anything, mock.Anything, mock.Anything).
Return(&agents.AlbumInfo{URL: "http://album.url", Description: "Fresh Notes"}, nil)
ag.On("GetAlbumImages", mock.Anything, mock.Anything, mock.Anything, mock.Anything).
Return([]agents.ExternalImage{}, nil)
}
BeforeEach(func() {
ctx = GinkgoT().Context()
ds = new(tests.MockDataStore)
ag = new(mockAgents)
broker = &fakeBroker{}
p = external.NewProvider(ds, ag, matcher.New(ds), broker)
mockArtistRepo = ds.Artist(ctx).(*tests.MockArtistRepo)
mockAlbumRepo = ds.Album(ctx).(*tests.MockAlbumRepo)
})
It("repopulates an artist even when its info is fresh", func() {
fresh := time.Now()
mockArtistRepo.SetData(model.Artists{{
ID: "ar-1", Name: "Test Artist", Biography: "stale", ExternalInfoUpdatedAt: &fresh,
}})
expectArtistAgents()
Expect(p.RefreshInfo(ctx, model.KindArtistArtwork, "ar-1")).To(Succeed())
saved, err := mockArtistRepo.Get("ar-1")
Expect(err).ToNot(HaveOccurred())
Expect(saved.Biography).To(Equal("Fresh Bio"))
})
It("repopulates an album even when its info is fresh", func() {
fresh := time.Now()
mockAlbumRepo.SetData(model.Albums{{
ID: "al-1", Name: "Test Album", AlbumArtist: "Test Artist",
Description: "stale", ExternalInfoUpdatedAt: &fresh,
}})
expectAlbumAgents()
Expect(p.RefreshInfo(ctx, model.KindAlbumArtwork, "al-1")).To(Succeed())
saved, err := mockAlbumRepo.Get("al-1")
Expect(err).ToNot(HaveOccurred())
Expect(saved.Description).To(Equal("Fresh Notes"))
})
It("returns ErrNotFound for an unknown id", func() {
Expect(p.RefreshInfo(ctx, model.KindArtistArtwork, "nope")).To(MatchError(model.ErrNotFound))
})
It("returns ErrNotFound for a kind with no external info", func() {
Expect(p.RefreshInfo(ctx, model.KindPlaylistArtwork, "pl-1")).To(MatchError(model.ErrNotFound))
})
It("broadcasts a RefreshResource naming the artist", func() {
mockArtistRepo.SetData(model.Artists{{ID: "ar-1", Name: "Test Artist"}})
expectArtistAgents()
Expect(p.RefreshInfo(ctx, model.KindArtistArtwork, "ar-1")).To(Succeed())
sent := broker.sent()
Expect(sent).To(HaveLen(1))
rr, ok := sent[0].(*events.RefreshResource)
Expect(ok).To(BeTrue())
Expect(rr.Data(rr)).To(ContainSubstring("ar-1"))
Expect(rr.Data(rr)).To(ContainSubstring("artist"))
})
It("broadcasts a RefreshResource naming the album", func() {
mockAlbumRepo.SetData(model.Albums{{ID: "al-1", Name: "Test Album", AlbumArtist: "Test Artist"}})
expectAlbumAgents()
Expect(p.RefreshInfo(ctx, model.KindAlbumArtwork, "al-1")).To(Succeed())
sent := broker.sent()
Expect(sent).To(HaveLen(1))
Expect(sent[0].Data(sent[0])).To(ContainSubstring("album"))
Expect(sent[0].Data(sent[0])).To(ContainSubstring("al-1"))
})
It("does not broadcast when the artist cannot be loaded", func() {
mockArtistRepo.SetData(model.Artists{{ID: "ar-1", Name: "Test Artist"}})
expectArtistAgents()
mockArtistRepo.SetError(true)
_ = p.RefreshInfo(ctx, model.KindArtistArtwork, "ar-1")
Expect(broker.sent()).To(BeEmpty())
})
It("reports which kinds have external info", func() {
Expect(external.HasInfo(model.KindArtistArtwork)).To(BeTrue())
Expect(external.HasInfo(model.KindAlbumArtwork)).To(BeTrue())
Expect(external.HasInfo(model.KindPlaylistArtwork)).To(BeFalse())
})
})

3
core/external/provider_similarsongs.go vendored
View File

@ -167,6 +167,7 @@ func (e *provider) seedMix(ctx context.Context, count int, sample func() (model.
if len(matched) == 0 {
matched = seeds
}
//nolint:gosec // shuffle order is not a security decision
rand.Shuffle(len(matched), func(i, j int) { matched[i], matched[j] = matched[j], matched[i] })
if len(matched) > count {
matched = matched[:count]
@ -239,7 +240,7 @@ func (e *provider) similarSongsFallback(ctx context.Context, id string, count in
return nil, err
}
e.callGetSimilarArtists(ctx, e.ag, &artist, 15, false)
_ = e.callGetSimilarArtists(ctx, e.ag, &artist, 15, false)
if utils.IsCtxDone(ctx) {
log.Warn(ctx, "SimilarSongs call canceled", ctx.Err())
return nil, ctx.Err()

2
core/external/provider_similarsongs_test.go vendored
View File

@ -61,7 +61,7 @@ var _ = Describe("Provider - SimilarSongs", func() {
similarAgent: mockSimilarAgent,
}
provider = NewProvider(ds, agentsCombined, matcher.New(ds))
provider = NewProvider(ds, agentsCombined, matcher.New(ds), &fakeBroker{})
})
// Resolves track-1 through the GetEntityByID probe order and on to its artist. Left permissive:

17
core/external/provider_topsongs_test.go vendored
View File

@ -45,7 +45,7 @@ var _ = Describe("Provider - TopSongs", func() {
ag = new(mockAgents)
p = NewProvider(ds, ag, matcher.New(ds))
p = NewProvider(ds, ag, matcher.New(ds), &fakeBroker{})
})
It("returns top songs for a known artist", func() {
@ -232,6 +232,21 @@ var _ = Describe("Provider - TopSongs", func() {
ag.AssertExpectations(GinkgoT())
})
// This endpoint answered with an empty list before retry-later existed; it must keep doing so.
It("returns an empty list, not a client error, when the agents are throttled", func() {
artist1 := model.Artist{ID: "artist-1", Name: "Artist One", MbzArtistID: "mbid-artist-1"}
artistRepo.On("GetAll", mock.AnythingOfType("model.QueryOptions")).Return(model.Artists{artist1}, nil).Once()
ag.On("GetArtistTopSongs", ctx, "artist-1", "Artist One", "mbid-artist-1", 5).
Return(nil, agents.ErrRetryLater).Once()
songs, err := p.TopSongs(ctx, "Artist One", "", 5)
Expect(songs).To(BeEmpty())
Expect(err).To(MatchError(model.ErrNotFound), "the handler renders this as an empty 200")
Expect(err).ToNot(MatchError(agents.ErrRetryLater))
ag.AssertExpectations(GinkgoT())
})
It("returns fewer songs if count is less than available top songs", func() {
// Mock finding the artist
artist1 := model.Artist{ID: "artist-1", Name: "Artist One", MbzArtistID: "mbid-artist-1"}

24
core/external/provider_updatealbuminfo_test.go vendored
View File

@ -34,7 +34,7 @@ var _ = Describe("Provider - UpdateAlbumInfo", func() {
ctx = GinkgoT().Context()
ds = new(tests.MockDataStore)
ag = new(mockAgents)
p = external.NewProvider(ds, ag, matcher.New(ds))
p = external.NewProvider(ds, ag, matcher.New(ds), &fakeBroker{})
mockAlbumRepo = ds.Album(ctx).(*tests.MockAlbumRepo)
conf.Server.DevAlbumInfoTimeToLive = 1 * time.Hour
})
@ -164,4 +164,26 @@ var _ = Describe("Provider - UpdateAlbumInfo", func() {
ag.AssertExpectations(GinkgoT())
})
It("returns the original album, unstamped, when the agents are throttled", func() {
originalAlbum := &model.Album{
ID: "al-throttled",
Name: "Throttled Album",
AlbumArtist: "Throttled Artist",
MbzAlbumID: "mbid-throttled",
}
mockAlbumRepo.SetData(model.Albums{*originalAlbum})
ag.On("GetAlbumInfo", ctx, "Throttled Album", "Throttled Artist", "mbid-throttled").
Return(nil, agents.ErrRetryLater)
updatedAlbum, err := p.UpdateAlbumInfo(ctx, "al-throttled")
Expect(err).NotTo(HaveOccurred())
Expect(updatedAlbum).NotTo(BeNil())
Expect(*updatedAlbum).To(Equal(*originalAlbum))
Expect(updatedAlbum.ExternalInfoUpdatedAt).To(BeNil())
ag.AssertExpectations(GinkgoT())
})
})

21
core/external/provider_updateartistinfo_test.go vendored
View File

@ -37,7 +37,7 @@ var _ = Describe("Provider - UpdateArtistInfo", func() {
ctx = GinkgoT().Context()
ds = new(tests.MockDataStore)
ag = new(mockAgents)
p = external.NewProvider(ds, ag, matcher.New(ds))
p = external.NewProvider(ds, ag, matcher.New(ds), &fakeBroker{})
mockArtistRepo = ds.Artist(ctx).(*tests.MockArtistRepo)
})
@ -104,6 +104,25 @@ var _ = Describe("Provider - UpdateArtistInfo", func() {
ag.AssertExpectations(GinkgoT())
})
// Stamping a throttled round would cache the empty result for the whole TTL.
It("does not stamp ExternalInfoUpdatedAt when the agents are throttled", func() {
originalArtist := &model.Artist{ID: "ar-throttled", Name: "Throttled Artist"}
mockArtistRepo.SetData(model.Artists{*originalArtist})
ag.On("GetArtistMBID", ctx, "ar-throttled", "Throttled Artist").Return("", agents.ErrRetryLater).Once()
ag.On("GetArtistImages", ctx, "ar-throttled", "Throttled Artist", "").Return(nil, agents.ErrRetryLater).Once()
ag.On("GetArtistBiography", ctx, "ar-throttled", "Throttled Artist", "").Return("", agents.ErrRetryLater).Once()
ag.On("GetArtistURL", ctx, "ar-throttled", "Throttled Artist", "").Return("", agents.ErrRetryLater).Once()
ag.On("GetSimilarArtists", ctx, "ar-throttled", "Throttled Artist", "", 100).Return(nil, agents.ErrRetryLater).Once()
updatedArtist, err := p.UpdateArtistInfo(ctx, "ar-throttled", 10, false)
Expect(err).ToNot(HaveOccurred())
Expect(updatedArtist).NotTo(BeNil())
Expect(updatedArtist.ExternalInfoUpdatedAt).To(BeNil())
ag.AssertExpectations(GinkgoT())
})
It("preserves decoded plain text in biography storage", func() {
originalArtist := &model.Artist{
ID: "ar-encoded-bio",

5
core/metrics/insights.go
View File

@ -26,6 +26,7 @@ import (
"github.com/navidrome/navidrome/model/request"
"github.com/navidrome/navidrome/plugins"
"github.com/navidrome/navidrome/server/events"
"github.com/navidrome/navidrome/utils/httpclient"
"github.com/navidrome/navidrome/utils/singleton"
)
@ -95,9 +96,7 @@ func (c *insightsCollector) sendInsights(ctx context.Context) {
log.Trace(ctx, "No users found, skipping Insights data collection")
return
}
hc := &http.Client{
Timeout: consts.DefaultHttpClientTimeOut,
}
hc := httpclient.New(consts.DefaultHttpClientTimeOut)
data := c.collect(ctx)
if data == nil {
return

1
core/playback/queue.go
View File

@ -100,6 +100,7 @@ func (pd *Queue) Shuffle() {
backupID = current.ID
}
//nolint:gosec // shuffle order is not a security decision
rand.Shuffle(len(pd.Items), func(i, j int) { pd.Items[i], pd.Items[j] = pd.Items[j], pd.Items[i] })
var err error

47
core/scrobbler/buffered_scrobbler.go
View File

@ -5,6 +5,7 @@ import (
"errors"
"time"
"github.com/navidrome/navidrome/core/agents"
"github.com/navidrome/navidrome/log"
"github.com/navidrome/navidrome/model"
"github.com/navidrome/navidrome/model/request"
@ -126,42 +127,52 @@ func (b *bufferedScrobbler) run(ctx context.Context) {
timer.Stop()
defer timer.Stop()
failures := 0
backingOff := false
for {
if b.processQueue(ctx) {
failures = 0
timer.Stop()
} else {
timer.Reset(backoffDelay(failures))
if failures < maxRetryShift {
failures++
// While a backoff window is open the timer is already armed for the rest of it, so a
// wake (a new play enqueued) must not drain: that is the hammering this avoids.
if !backingOff {
if ok, retryIn := b.processQueue(ctx); ok {
failures = 0
timer.Stop()
} else {
timer.Reset(max(backoffDelay(failures), retryIn))
backingOff = true
if failures < maxRetryShift {
failures++
}
}
}
select {
case <-b.wakeSignal:
case <-timer.C:
backingOff = false
case <-ctx.Done():
return
}
}
}
func (b *bufferedScrobbler) processQueue(ctx context.Context) bool {
func (b *bufferedScrobbler) processQueue(ctx context.Context) (bool, time.Duration) {
buffer := b.ds.ScrobbleBuffer(ctx)
userIds, err := buffer.UserIDs(b.service)
if err != nil {
log.Error(ctx, "Error retrieving userIds from scrobble buffer", "scrobbler", b.service, err)
return false
return false, 0
}
result := true
var retryIn time.Duration
for _, userId := range userIds {
if !b.processUserQueue(ctx, userId) {
ok, d := b.processUserQueue(ctx, userId)
if !ok {
result = false
retryIn = max(retryIn, d)
}
}
return result
return result, retryIn
}
func (b *bufferedScrobbler) processUserQueue(ctx context.Context, userId string) bool {
func (b *bufferedScrobbler) processUserQueue(ctx context.Context, userId string) (bool, time.Duration) {
// Scrobbles are drained on a background context that no longer carries the
// request's authenticated user. Restore it from the buffered userId so that
// scrobblers relying on the user in the context (e.g. plugins) still get it.
@ -175,25 +186,25 @@ func (b *bufferedScrobbler) processUserQueue(ctx context.Context, userId string)
entry, err := buffer.Next(b.service, userId)
if err != nil {
log.Error(ctx, "Error reading from scrobble buffer", "scrobbler", b.service, err)
return false
return false, 0
}
if entry == nil {
return true
return true, 0
}
s, ok := b.loader()
if !ok {
log.Warn(ctx, "Scrobbler not available, will retry later", "scrobbler", b.service)
return false
return false, 0
}
log.Debug(ctx, "Sending scrobble", "scrobbler", b.service, "track", entry.Title, "artist", entry.Artist)
err = s.Scrobble(ctx, entry.UserID, Scrobble{
MediaFile: entry.MediaFile,
TimeStamp: entry.PlayTime,
})
if errors.Is(err, ErrRetryLater) {
if retry, ok := errors.AsType[*agents.RetryLaterError](err); ok {
log.Warn(ctx, "Could not send scrobble. Will be retried", "userId", entry.UserID,
"track", entry.Title, "artist", entry.Artist, "scrobbler", b.service, err)
return false
return false, retry.RetryIn
}
if err != nil {
log.Error(ctx, "Error sending scrobble to service. Discarding", "scrobbler", b.service,
@ -203,7 +214,7 @@ func (b *bufferedScrobbler) processUserQueue(ctx context.Context, userId string)
if err != nil {
log.Error(ctx, "Error removing entry from scrobble buffer", "userId", entry.UserID,
"track", entry.Title, "artist", entry.Artist, "scrobbler", b.service, err)
return false
return false, 0
}
}
}

122
core/scrobbler/buffered_scrobbler_test.go
View File

@ -2,11 +2,13 @@ package scrobbler
import (
"context"
"errors"
"sync/atomic"
"testing"
"testing/synctest"
"time"
"github.com/navidrome/navidrome/core/agents"
"github.com/navidrome/navidrome/model"
"github.com/navidrome/navidrome/tests"
. "github.com/onsi/ginkgo/v2"
@ -158,19 +160,128 @@ func TestBufferedScrobblerBackoffSchedule(t *testing.T) {
g.Expect(flaky.count.Load()).To(Equal(want), "retry did not fire after the %s backoff", gap)
}
// Once the service recovers, waking the loop drains the buffered entry.
// Once the service recovers, the buffered entry drains when the open
// backoff window closes (a wake alone must not drain it early).
flaky.succeed()
bs.sendWakeSignal()
synctest.Wait()
g.Expect(buffer.Length()).To(Equal(int64(1)), "wake during backoff drained early")
time.Sleep(80 * time.Second)
synctest.Wait()
g.Expect(buffer.Length()).To(Equal(int64(0)))
})
}
func TestBufferedScrobblerBackoffWindow(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
buffer := tests.CreateMockedScrobbleBufferRepo()
userRepo := tests.CreateMockUserRepo()
_ = userRepo.Put(&model.User{ID: "user1", UserName: "alice"})
ds := &tests.MockDataStore{MockedScrobbleBuffer: buffer, MockedUser: userRepo}
scr := &fakeScrobbler{Authorized: true}
scr.SetError(errors.Join(errors.New("boom"), ErrRetryLater))
bs := newBufferedScrobbler(ds, scr, "test")
defer bs.Stop()
// First enqueue: one immediate attempt, then a 5s window opens.
_ = bs.Scrobble(context.Background(), "user1", Scrobble{MediaFile: model.MediaFile{ID: "1"}, TimeStamp: time.Now()})
synctest.Wait()
if got := scr.ScrobbleAttempts(); got != 1 {
t.Fatalf("expected 1 attempt after first enqueue, got %d", got)
}
// A wake inside the window must NOT trigger an early attempt.
time.Sleep(1 * time.Second)
_ = bs.Scrobble(context.Background(), "user1", Scrobble{MediaFile: model.MediaFile{ID: "2"}, TimeStamp: time.Now()})
synctest.Wait()
if got := scr.ScrobbleAttempts(); got != 1 {
t.Fatalf("wake during backoff drained early: %d attempts", got)
}
// When the 5s window closes, the retry happens.
time.Sleep(4100 * time.Millisecond)
synctest.Wait()
if got := scr.ScrobbleAttempts(); got != 2 {
t.Fatalf("expected retry after window, got %d attempts", got)
}
})
}
func TestBufferedScrobblerHonorsServerDelay(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
buffer := tests.CreateMockedScrobbleBufferRepo()
userRepo := tests.CreateMockUserRepo()
_ = userRepo.Put(&model.User{ID: "user1", UserName: "alice"})
ds := &tests.MockDataStore{MockedScrobbleBuffer: buffer, MockedUser: userRepo}
scr := &fakeScrobbler{Authorized: true}
scr.SetError(errors.Join(errors.New("429"), &agents.RetryLaterError{RetryIn: 30 * time.Second}))
bs := newBufferedScrobbler(ds, scr, "test")
defer bs.Stop()
_ = bs.Scrobble(context.Background(), "user1", Scrobble{MediaFile: model.MediaFile{ID: "1"}, TimeStamp: time.Now()})
synctest.Wait()
if got := scr.ScrobbleAttempts(); got != 1 {
t.Fatalf("expected 1 attempt, got %d", got)
}
// The 5s exponential floor is overridden by the 30s server delay.
time.Sleep(20 * time.Second)
synctest.Wait()
if got := scr.ScrobbleAttempts(); got != 1 {
t.Fatalf("retried before server delay elapsed: %d attempts", got)
}
time.Sleep(10100 * time.Millisecond)
synctest.Wait()
if got := scr.ScrobbleAttempts(); got != 2 {
t.Fatalf("expected retry after server delay, got %d attempts", got)
}
})
}
// The drain visits users in an arbitrary order, so the longest delay must win regardless
// of which user was seen last.
func TestBufferedScrobblerTakesTheLongestServerDelayAcrossUsers(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
buffer := tests.CreateMockedScrobbleBufferRepo()
userRepo := tests.CreateMockUserRepo()
_ = userRepo.Put(&model.User{ID: "user1", UserName: "alice"})
_ = userRepo.Put(&model.User{ID: "user2", UserName: "bob"})
ds := &tests.MockDataStore{MockedScrobbleBuffer: buffer, MockedUser: userRepo}
scr := &recoveringScrobbler{delays: map[string]time.Duration{
"user1": 10 * time.Second,
"user2": 45 * time.Second,
}}
bs := newBufferedScrobbler(ds, scr, "test")
defer bs.Stop()
// user2 is enqueued directly so both are buffered before the first drain wakes.
_ = buffer.Enqueue("test", "user2", "2", time.Now())
_ = bs.Scrobble(context.Background(), "user1", Scrobble{MediaFile: model.MediaFile{ID: "1"}, TimeStamp: time.Now()})
synctest.Wait()
if got := scr.count.Load(); got != 2 {
t.Fatalf("expected both users drained, got %d attempts", got)
}
time.Sleep(30 * time.Second)
synctest.Wait()
if got := scr.count.Load(); got != 2 {
t.Fatalf("retried on the shorter delay: %d attempts", got)
}
time.Sleep(15100 * time.Millisecond)
synctest.Wait()
if got := scr.count.Load(); got != 4 {
t.Fatalf("expected a retry after the longest delay, got %d attempts", got)
}
})
}
// recoveringScrobbler is a race-safe Scrobbler whose error can be toggled while
// the buffered scrobbler's goroutine is draining, to exercise retry then recovery.
// With delays set, it instead fails every scrobble asking for that user's delay.
type recoveringScrobbler struct {
err atomic.Pointer[error]
count atomic.Int32
err atomic.Pointer[error]
count atomic.Int32
delays map[string]time.Duration
}
func (f *recoveringScrobbler) fail(err error) { f.err.Store(&err) }
@ -182,8 +293,11 @@ func (f *recoveringScrobbler) NowPlaying(context.Context, string, *model.MediaFi
return nil
}
func (f *recoveringScrobbler) Scrobble(_ context.Context, _ string, _ Scrobble) error {
func (f *recoveringScrobbler) Scrobble(_ context.Context, userId string, _ Scrobble) error {
f.count.Add(1)
if f.delays != nil {
return errors.Join(errors.New("429"), &agents.RetryLaterError{RetryIn: f.delays[userId]})
}
if e := f.err.Load(); e != nil {
return *e
}

4
core/scrobbler/interfaces.go
View File

@ -5,6 +5,7 @@ import (
"errors"
"time"
"github.com/navidrome/navidrome/core/agents"
"github.com/navidrome/navidrome/model"
)
@ -15,7 +16,8 @@ type Scrobble struct {
var (
ErrNotAuthorized = errors.New("not authorized")
ErrRetryLater = errors.New("retry later")
// ErrRetryLater is an alias of agents.ErrRetryLater so adapters and plugins share one identity.
ErrRetryLater = agents.ErrRetryLater
ErrUnrecoverable = errors.New("unrecoverable")
)

37
core/scrobbler/play_tracker_test.go
View File

@ -293,7 +293,7 @@ var _ = Describe("PlayTracker", func() {
})
It("increments play counts even if it cannot scrobble", func() {
fake.Error = errors.New("error")
fake.SetError(errors.New("error"))
err := tracker.Submit(ctx, []Submission{{TrackID: "123", Timestamp: time.Now()}})
@ -1414,7 +1414,25 @@ type fakeScrobbler struct {
position atomic.Int32
LastScrobble atomic.Pointer[Scrobble]
LastPlaybackReport atomic.Pointer[PlaybackSession]
Error error
err atomic.Pointer[error]
scrobbleAttempts atomic.Int32
}
// SetError sets the error returned by IsAuthorized/NowPlaying/Scrobble/PlaybackReport.
func (f *fakeScrobbler) SetError(err error) {
f.err.Store(&err)
}
func (f *fakeScrobbler) getError() error {
if e := f.err.Load(); e != nil {
return *e
}
return nil
}
// ScrobbleAttempts returns how many times Scrobble was called.
func (f *fakeScrobbler) ScrobbleAttempts() int32 {
return f.scrobbleAttempts.Load()
}
func (f *fakeScrobbler) GetNowPlayingCalled() bool {
@ -1440,13 +1458,13 @@ func (f *fakeScrobbler) GetTrack() *model.MediaFile {
}
func (f *fakeScrobbler) IsAuthorized(ctx context.Context, userId string) bool {
return f.Error == nil && f.Authorized
return f.getError() == nil && f.Authorized
}
func (f *fakeScrobbler) NowPlaying(ctx context.Context, userId string, track *model.MediaFile, position int) error {
f.nowPlayingCalled.Store(true)
if f.Error != nil {
return f.Error
if err := f.getError(); err != nil {
return err
}
f.userID.Store(&userId)
// Capture username from context (this is what plugin scrobblers do)
@ -1478,16 +1496,17 @@ func (f *fakeScrobbler) Scrobble(ctx context.Context, userId string, s Scrobble)
}
f.LastScrobble.Store(&s)
f.ScrobbleCalled.Store(true)
if f.Error != nil {
return f.Error
f.scrobbleAttempts.Add(1)
if err := f.getError(); err != nil {
return err
}
return nil
}
func (f *fakeScrobbler) PlaybackReport(ctx context.Context, info PlaybackSession) error {
f.PlaybackReportCalled.Store(true)
if f.Error != nil {
return f.Error
if err := f.getError(); err != nil {
return err
}
f.userID.Store(new(info.UserId))
f.LastPlaybackReport.Store(&info)

3
core/share.go
View File

@ -123,8 +123,7 @@ func (r *shareRepositoryWrapper) Save(entity any) (string, error) {
s.Contents = str.TruncateRunes(s.Contents, 30, "...")
id, err = r.Persistable.Save(s)
return id, err
return r.Persistable.Save(s)
}
func (r *shareRepositoryWrapper) Update(id string, entity any, _ ...string) error {

18
core/storage/local/deviceid_unix.go Normal file
View File

@ -0,0 +1,18 @@
//go:build !windows
package local
import (
"io/fs"
"syscall"
)
// deviceID identifies the filesystem a file lives on, used to key birth time support per mount.
// It is returned opaquely because its width varies by platform, and it is only used as a map key.
func deviceID(fi fs.FileInfo) (any, bool) {
st, ok := fi.Sys().(*syscall.Stat_t)
if !ok {
return nil, false
}
return st.Dev, true
}

8
core/storage/local/deviceid_windows.go Normal file
View File

@ -0,0 +1,8 @@
//go:build windows
package local
import "io/fs"
// deviceID has no Windows equivalent, and none is needed: birth time comes straight from FileInfo.
func deviceID(fs.FileInfo) (any, bool) { return nil, false }

40
core/storage/local/local.go
View File

@ -6,6 +6,7 @@ import (
"net/url"
"os"
"path/filepath"
"sync"
"sync/atomic"
"time"
@ -61,6 +62,8 @@ type localFS struct {
fs.FS
extractor Extractor
root string
// devices whose statx never reports a birth time (NFS, rclone/FUSE), so we ask each only once
noBirthTime sync.Map
}
// ResolveSymlink implements storage.SymlinkResolverFS. It resolves the whole chain at the
@ -84,7 +87,11 @@ func (lfs *localFS) ReadTags(path ...string) (map[string]metadata.Info, error) {
if err != nil {
return nil, err
}
v.FileInfo = localFileInfo{info}
v.FileInfo = localFileInfo{
FileInfo: info,
path: filepath.Join(lfs.root, filepath.FromSlash(path)),
noBirthTime: &lfs.noBirthTime,
}
res[path] = v
}
}
@ -95,15 +102,46 @@ func (lfs *localFS) ReadTags(path ...string) (map[string]metadata.Info, error) {
// with metadata.FileInfo
type localFileInfo struct {
fs.FileInfo
path string
noBirthTime *sync.Map
}
func (lfi localFileInfo) BirthTime() time.Time {
if ts := times.Get(lfi.FileInfo); ts.HasBirthTime() {
return ts.BirthTime()
}
if bt, ok := lfi.statxBirthTime(); ok {
return bt
}
return time.Now()
}
// statxBirthTime reads the birth time from the path, which on Linux is the only way to get it.
// Filesystems that never report one are remembered per device, so a scan asks each only once.
func (lfi localFileInfo) statxBirthTime() (time.Time, bool) {
if lfi.path == "" {
return time.Time{}, false
}
dev, hasDev := deviceID(lfi.FileInfo)
memo := lfi.noBirthTime
if hasDev && memo != nil {
if _, skip := memo.Load(dev); skip {
return time.Time{}, false
}
}
ts, err := times.Stat(lfi.path)
if err != nil {
return time.Time{}, false
}
if ts.HasBirthTime() {
return ts.BirthTime(), true
}
if hasDev && memo != nil {
memo.Store(dev, struct{}{})
}
return time.Time{}, false
}
func init() {
storage.Register(storage.LocalSchemaID, newLocalStorage)
}

33
core/storage/local/local_test.go
View File

@ -6,8 +6,10 @@ import (
"os"
"path/filepath"
"runtime"
"sync"
"time"
"github.com/djherbis/times"
"github.com/navidrome/navidrome/conf"
"github.com/navidrome/navidrome/conf/configtest"
"github.com/navidrome/navidrome/consts"
@ -440,6 +442,37 @@ var _ = Describe("LocalStorage", func() {
// Should be around the current time (within last few minutes)
Expect(birthTime).To(BeTemporally("~", time.Now(), 5*time.Minute))
})
It("reads the birth time from the path, not the time of the call", func() {
// On Linux, birth time is only available via statx(2) on the path.
lfi := localFileInfo{FileInfo: fileInfo, path: testFile}
time.Sleep(300 * time.Millisecond)
Expect(lfi.BirthTime()).To(BeTemporally("<", time.Now().Add(-200*time.Millisecond)))
})
It("does not remember filesystems that do report a birth time", func() {
memo := &sync.Map{}
lfi := localFileInfo{FileInfo: fileInfo, path: testFile, noBirthTime: memo}
lfi.BirthTime()
count := 0
memo.Range(func(_, _ any) bool { count++; return true })
Expect(count).To(BeZero())
})
It("skips statx on filesystems already known to have none", func() {
if times.Get(fileInfo).HasBirthTime() {
Skip("this platform reports birth time from FileInfo, so statx is never called")
}
dev, ok := deviceID(fileInfo)
Expect(ok).To(BeTrue())
memo := &sync.Map{}
memo.Store(dev, struct{}{})
lfi := localFileInfo{FileInfo: fileInfo, path: testFile, noBirthTime: memo}
time.Sleep(300 * time.Millisecond)
Expect(lfi.BirthTime()).To(BeTemporally("~", time.Now(), 100*time.Millisecond))
})
})
It("should delegate all other FileInfo methods", func() {

6
core/stream/media_streamer.go
View File

@ -152,8 +152,9 @@ func (s *Stream) EstimatedContentLength() int {
// Serve writes the stream to the HTTP response. For seekable streams it uses http.ServeContent
// (supporting range requests). For non-seekable streams it writes directly and logs any errors.
// Returns the number of bytes written and an error only when io.Copy fails with 0 bytes written
// Returns the number of bytes written and an error only when it fails with 0 bytes written
// (meaning the HTTP 200 status has not been flushed yet and the caller can still send an error response).
// Once bytes are on the wire it panics with http.ErrAbortHandler instead, aborting the response.
// Empty output (0 bytes, no error) is logged but not treated as an error.
func (s *Stream) Serve(ctx context.Context, w http.ResponseWriter, r *http.Request) (int64, error) {
if s.Seekable() {
@ -183,7 +184,8 @@ func (s *Stream) Serve(ctx context.Context, w http.ResponseWriter, r *http.Reque
w.Header().Del("Content-Length")
return 0, fmt.Errorf("sending transcoded file: %w", err)
}
return c, nil
// The 200 is already sent, so dropping the connection is the only way to say "truncated".
panic(http.ErrAbortHandler)
}
if c == 0 {
log.Error(ctx, "Transcoding returned empty output, ffmpeg may have failed. "+

51
core/stream/media_streamer_test.go
View File

@ -1,12 +1,18 @@
package stream_test
import (
"bytes"
"context"
"errors"
"io"
"net/http"
"net/http/httptest"
"os"
"testing/iotest"
"time"
"github.com/go-chi/chi/v5/middleware"
"github.com/navidrome/navidrome/conf"
"github.com/navidrome/navidrome/conf/configtest"
"github.com/navidrome/navidrome/core/stream"
@ -140,4 +146,49 @@ var _ = Describe("MediaStreamer", func() {
Expect(s.Seekable()).To(BeTrue())
})
})
Context("Serve", func() {
var mf *model.MediaFile
BeforeEach(func() {
var err error
mf, err = ds.MediaFile(ctx).Get("123")
Expect(err).ToNot(HaveOccurred())
})
It("keeps empty output a non-error, so callers still reply 200 with an empty body", func() {
s := stream.NewStream(mf, "mp3", 128, io.NopCloser(bytes.NewReader(nil)))
w := httptest.NewRecorder()
r := httptest.NewRequest(http.MethodGet, "/", nil)
n, err := s.Serve(ctx, w, r)
Expect(err).ToNot(HaveOccurred())
Expect(n).To(BeZero())
Expect(w.Code).To(Equal(http.StatusOK))
})
It("aborts the response when the source fails after sending data", func() {
src := io.NopCloser(io.MultiReader(
bytes.NewReader(bytes.Repeat([]byte("a"), 64*1024)),
iotest.ErrReader(errors.New("transcoder died")),
))
server := httptest.NewServer(serveHandler(stream.NewStream(mf, "mp3", 128, src)))
DeferCleanup(server.Close)
resp, err := http.Get(server.URL)
Expect(err).ToNot(HaveOccurred())
defer resp.Body.Close()
// A client-side read failure is the only observable proof the response was aborted.
_, err = io.ReadAll(resp.Body)
Expect(err).To(HaveOccurred())
})
})
})
// Serve runs behind the real server's Recoverer, which must let ErrAbortHandler through.
func serveHandler(s *stream.Stream) http.Handler {
return middleware.Recoverer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = s.Serve(r.Context(), w, r)
}))
}

10
core/stream/token_test.go
View File

@ -232,6 +232,16 @@ var _ = Describe("Token", func() {
_, err := svc.ResolveRequestFromToken(ctx, token, mf, 0)
Expect(err).To(MatchError(ErrTokenStale))
})
It("rejects a Jellyfin access token", func() {
mf := &model.MediaFile{ID: "song-1", UpdatedAt: sourceTime}
usr := &model.User{ID: "u1", UserName: "johndoe"}
tokenStr, err := auth.CreateAPIToken(usr, auth.AudienceJellyfin)
Expect(err).ToNot(HaveOccurred())
_, err = svc.ResolveRequestFromToken(ctx, tokenStr, mf, 0)
Expect(err).To(MatchError(ErrTokenInvalid))
})
})
Describe("paramsFromToken", func() {

1
db/backup_test.go
View File

@ -68,6 +68,7 @@ var _ = Describe("database backups", func() {
timesShuffled = make([]time.Time, len(timesDecreasingChronologically))
copy(timesShuffled, timesDecreasingChronologically)
//nolint:gosec // shuffle order is not a security decision
rand.Shuffle(len(timesShuffled), func(i, j int) {
timesShuffled[i], timesShuffled[j] = timesShuffled[j], timesShuffled[i]
})

10
db/db.go
View File

@ -13,10 +13,15 @@ import (
_ "github.com/navidrome/navidrome/db/migrations"
"github.com/navidrome/navidrome/log"
"github.com/navidrome/navidrome/utils/hasher"
"github.com/navidrome/navidrome/utils/natural"
"github.com/navidrome/navidrome/utils/singleton"
"github.com/pressly/goose/v3"
)
// NaturalCollation sorts embedded numbers by value. It is registered on every
// connection, but only referenced when conf.Server.EnableNaturalSorting is on.
const NaturalCollation = "NATSORT"
var (
Dialect = "sqlite3"
Driver = Dialect + "_custom"
@ -32,7 +37,10 @@ func Db() *sql.DB {
return singleton.GetInstance(func() *sql.DB {
sql.Register(Driver, &sqlite3.SQLiteDriver{
ConnectHook: func(conn *sqlite3.SQLiteConn) error {
return conn.RegisterFunc("SEEDEDRAND", hasher.HashFunc(), false)
if err := conn.RegisterFunc("SEEDEDRAND", hasher.HashFunc(), false); err != nil {
return err
}
return conn.RegisterCollation(NaturalCollation, natural.CompareFold)
},
})
Path = conf.Server.DbPath

9
db/migrations/20260819204637_add_artwork_trace_columns.sql Normal file
View File

@ -0,0 +1,9 @@
-- +goose Up
ALTER TABLE item_artwork ADD COLUMN trace jsonb NOT NULL DEFAULT '[]';
ALTER TABLE item_artwork ADD COLUMN last_failure jsonb NOT NULL DEFAULT '[]';
ALTER TABLE artwork_queue ADD COLUMN trace jsonb NOT NULL DEFAULT '[]';
-- +goose Down
ALTER TABLE artwork_queue DROP COLUMN trace;
ALTER TABLE item_artwork DROP COLUMN last_failure;
ALTER TABLE item_artwork DROP COLUMN trace;

7
db/migrations/20260822062750_add_user_token_epoch.sql Normal file
View File

@ -0,0 +1,7 @@
-- +goose Up
ALTER TABLE user ADD COLUMN token_epoch INTEGER NOT NULL DEFAULT 0;
-- +goose Down
ALTER TABLE user DROP COLUMN token_epoch;

18
go.mod
View File

@ -1,6 +1,6 @@
module github.com/navidrome/navidrome
go 1.26
go 1.27
// Fork to implement raw tags support
replace go.senan.xyz/taglib => github.com/deluan/go-taglib v0.0.0-20260720134629-a133b9719ea3
@ -13,14 +13,14 @@ require (
github.com/deluan/sanitize v0.0.0-20241120162836-fdfd8fdfaa55
github.com/dexterlb/mpvipc v0.0.0-20260722094525-0cf47d745b36
github.com/djherbis/atime v1.1.0
github.com/djherbis/fscache v0.10.2-0.20231127215153-442a07e326c4
github.com/djherbis/stream v1.4.0
github.com/djherbis/fscache v0.10.2-0.20260829235704-6d85d5878c22
github.com/djherbis/stream v1.5.1
github.com/djherbis/times v1.6.0
github.com/dustin/go-humanize v1.0.1
github.com/extism/go-sdk v1.7.1
github.com/fatih/structs v1.1.0
github.com/gen2brain/webp v0.6.4
github.com/go-chi/chi/v5 v5.3.1
github.com/go-chi/chi/v5 v5.3.2
github.com/go-chi/cors v1.2.2
github.com/go-chi/httprate v0.16.0
github.com/go-chi/jwtauth/v5 v5.4.0
@ -40,7 +40,7 @@ require (
github.com/microcosm-cc/bluemonday v1.0.27
github.com/mileusna/useragent v1.3.5
github.com/onsi/ginkgo/v2 v2.32.1
github.com/onsi/gomega v1.42.1
github.com/onsi/gomega v1.43.0
github.com/pelletier/go-toml/v2 v2.4.3
github.com/pmezard/go-difflib v1.0.0
github.com/pocketbase/dbx v1.12.0
@ -50,10 +50,10 @@ require (
github.com/robfig/cron/v3 v3.0.1
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06
github.com/santhosh-tekuri/jsonschema/v6 v6.0.3
github.com/sirupsen/logrus v1.10.0
github.com/sirupsen/logrus v1.10.2
github.com/spf13/cobra v1.10.2
github.com/spf13/viper v1.21.0
github.com/stretchr/testify v1.12.0
github.com/stretchr/testify v1.12.1
github.com/tetratelabs/wazero v1.12.0
github.com/unrolled/secure v1.17.0
github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342
@ -89,7 +89,7 @@ require (
github.com/goccy/go-json v0.10.6 // indirect
github.com/goccy/go-yaml v1.19.2 // indirect
github.com/google/go-cmp v0.7.0 // indirect
github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 // indirect
github.com/google/pprof v0.0.0-20260825171938-4d453200e7d9 // indirect
github.com/google/subcommands v1.2.0 // indirect
github.com/gorilla/css v1.0.1 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
@ -101,7 +101,7 @@ require (
github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect
github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect
github.com/lestrrat-go/blackmagic v1.0.4 // indirect
github.com/lestrrat-go/dsig v1.3.0 // indirect
github.com/lestrrat-go/dsig v1.4.0 // indirect
github.com/lestrrat-go/dsig-secp256k1 v1.0.0 // indirect
github.com/lestrrat-go/httpcc v1.0.1 // indirect
github.com/lestrrat-go/httprc/v3 v3.0.6 // indirect

32
go.sum
View File

@ -39,10 +39,10 @@ github.com/dexterlb/mpvipc v0.0.0-20260722094525-0cf47d745b36 h1:KtPfdSST6e0vJbM
github.com/dexterlb/mpvipc v0.0.0-20260722094525-0cf47d745b36/go.mod h1:RkQWLNITKkXHLP7LXxZSgEq+uFWU25M5qW7qfEhL9Wc=
github.com/djherbis/atime v1.1.0 h1:rgwVbP/5by8BvvjBNrbh64Qz33idKT3pSnMSJsxhi0g=
github.com/djherbis/atime v1.1.0/go.mod h1:28OF6Y8s3NQWwacXc5eZTsEsiMzp7LF8MbXE+XJPdBE=
github.com/djherbis/fscache v0.10.2-0.20231127215153-442a07e326c4 h1:wdZllsLrDJtYfHiAKogB4PNHSDeO+v+5S3eqSWHGDlc=
github.com/djherbis/fscache v0.10.2-0.20231127215153-442a07e326c4/go.mod h1:dHWjlanKIxaHVH1xJOTb4kzP800XdcXlgJ6JYlR2DPU=
github.com/djherbis/stream v1.4.0 h1:aVD46WZUiq5kJk55yxJAyw6Kuera6kmC3i2vEQyW/AE=
github.com/djherbis/stream v1.4.0/go.mod h1:cqjC1ZRq3FFwkGmUtHwcldbnW8f0Q4YuVsGW1eAFtOk=
github.com/djherbis/fscache v0.10.2-0.20260829235704-6d85d5878c22 h1:GNKxzBirvK9arfVRGVebhFYBp3tnOZG3nlIog6N5/6I=
github.com/djherbis/fscache v0.10.2-0.20260829235704-6d85d5878c22/go.mod h1:Bbk9SqpJcg/saiPfG6byM1G4G/LQndknrsLVOQ+VJqY=
github.com/djherbis/stream v1.5.1 h1:9AoCl0bnm7imWT2uUORqT8aLuTh+YllyynYpolpjIqY=
github.com/djherbis/stream v1.5.1/go.mod h1:cqjC1ZRq3FFwkGmUtHwcldbnW8f0Q4YuVsGW1eAFtOk=
github.com/djherbis/times v1.6.0 h1:w2ctJ92J8fBvWPxugmXIv7Nz7Q3iDMKNx9v5ocVH20c=
github.com/djherbis/times v1.6.0/go.mod h1:gOHeRAz2h+VJNZ5Gmc/o7iD9k4wW7NMVqieYCY99oc0=
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
@ -69,8 +69,8 @@ github.com/gkampitakis/go-diff v1.3.2 h1:Qyn0J9XJSDTgnsgHRdz9Zp24RaJeKMUHg2+PDZZ
github.com/gkampitakis/go-diff v1.3.2/go.mod h1:LLgOrpqleQe26cte8s36HTWcTmMEur6OPYerdAAS9tk=
github.com/gkampitakis/go-snaps v0.5.15 h1:amyJrvM1D33cPHwVrjo9jQxX8g/7E2wYdZ+01KS3zGE=
github.com/gkampitakis/go-snaps v0.5.15/go.mod h1:HNpx/9GoKisdhw9AFOBT1N7DBs9DiHo/hGheFGBZ+mc=
github.com/go-chi/chi/v5 v5.3.1 h1:3j4HZLGZQ3JpMCrPJF/Jl3mYJfWLKBfNJ6quurUGCf8=
github.com/go-chi/chi/v5 v5.3.1/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto=
github.com/go-chi/chi/v5 v5.3.2 h1:5YQkICvTCSZ25hoRsyJazN0scjzKGiu4VAUc7H1o1nY=
github.com/go-chi/chi/v5 v5.3.2/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto=
github.com/go-chi/cors v1.2.2 h1:Jmey33TE+b+rB7fT8MUy1u0I4L+NARQlK6LhzKPSyQE=
github.com/go-chi/cors v1.2.2/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58=
github.com/go-chi/httprate v0.16.0 h1:8V5DH9j6pSK6UQoBsTpvMyFxycqaKEIToyPKzHJjUa8=
@ -101,8 +101,8 @@ github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/go-pipeline v0.0.0-20230411140531-6cbedfc1d3fc h1:hd+uUVsB1vdxohPneMrhGH2YfQuH5hRIK9u4/XCeUtw=
github.com/google/go-pipeline v0.0.0-20230411140531-6cbedfc1d3fc/go.mod h1:SL66SJVysrh7YbDCP9tH30b8a9o/N2HeiQNUm85EKhc=
github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo=
github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk=
github.com/google/pprof v0.0.0-20260825171938-4d453200e7d9 h1:dl4UZiszMU+NKHirOiCKTC+hRuNAQ0moHPxSg6WcU1o=
github.com/google/pprof v0.0.0-20260825171938-4d453200e7d9/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk=
github.com/google/subcommands v1.2.0 h1:vWQspBTo2nEqTUFita5/KeEWlUL8kQObDFbub/EN9oE=
github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
@ -151,8 +151,8 @@ github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 h1:P6pPBnrTSX3DEVR4fDembhR
github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0/go.mod h1:vmVJ0l/dxyfGW6FmdpVm2joNMFikkuWg0EoCKLGUMNw=
github.com/lestrrat-go/blackmagic v1.0.4 h1:IwQibdnf8l2KoO+qC3uT4OaTWsW7tuRQXy9TRN9QanA=
github.com/lestrrat-go/blackmagic v1.0.4/go.mod h1:6AWFyKNNj0zEXQYfTMPfZrAXUWUfTIZ5ECEUEJaijtw=
github.com/lestrrat-go/dsig v1.3.0 h1:phjMOCXvYzhuIgn7Voe2rex8z166vGfxRxmqM25P9/Q=
github.com/lestrrat-go/dsig v1.3.0/go.mod h1:RD2eOaidyPvpc7IJQoO3Qq52RWdy8ZcJs8lrOnoa1Kc=
github.com/lestrrat-go/dsig v1.4.0 h1:g7LUjK8cT74A5DzBXJI5HzsJuLhoYN0Wzj4nuOMIrH8=
github.com/lestrrat-go/dsig v1.4.0/go.mod h1:I8Nddg/vN2cUl/h8N7SRRApLnNNeyZPIqLYpvpOtGGo=
github.com/lestrrat-go/dsig-secp256k1 v1.0.0 h1:JpDe4Aybfl0soBvoVwjqDbp+9S1Y2OM7gcrVVMFPOzY=
github.com/lestrrat-go/dsig-secp256k1 v1.0.0/go.mod h1:CxUgAhssb8FToqbL8NjSPoGQlnO4w3LG1P0qPWQm/NU=
github.com/lestrrat-go/httpcc v1.0.1 h1:ydWCStUeJLkpYyjLDHihupbn2tYmZ7m22BGkcvZZrIE=
@ -187,8 +187,8 @@ github.com/ogier/pflag v0.0.1 h1:RW6JSWSu/RkSatfcLtogGfFgpim5p7ARQ10ECk5O750=
github.com/ogier/pflag v0.0.1/go.mod h1:zkFki7tvTa0tafRvTBIZTvzYyAu6kQhPZFnshFFPE+g=
github.com/onsi/ginkgo/v2 v2.32.1 h1:6tlvcDm/3sE8lGJbZ4+d4mO3RLy24/tQWOFzVSQNIfw=
github.com/onsi/ginkgo/v2 v2.32.1/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44=
github.com/onsi/gomega v1.42.1 h1:iN1rCUX+44NZ1Dc97MPoeFYbFR0vh8zxoxMFwKdyZ6I=
github.com/onsi/gomega v1.42.1/go.mod h1:REff/hsDsodHoKlWsP2mAPhu1+5/6hVYNf9rIEBpeSg=
github.com/onsi/gomega v1.43.0 h1:VlG/1FxqNxhSO+lq/OHBNaaqwiBK/mO8JbVkX9Y+FeU=
github.com/onsi/gomega v1.43.0/go.mod h1:REff/hsDsodHoKlWsP2mAPhu1+5/6hVYNf9rIEBpeSg=
github.com/pelletier/go-toml/v2 v2.4.3 h1:GTRvJQutkOSftxIFD5xw9aepkYNuPWmVJpffdDPYVpY=
github.com/pelletier/go-toml/v2 v2.4.3/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
@ -232,8 +232,8 @@ github.com/segmentio/asm v1.2.1/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr
github.com/sethvargo/go-retry v0.4.0 h1:9qy1OoIAxBL+gBYnkTnTnWle5wlfsXQlwRzIbbpdqPw=
github.com/sethvargo/go-retry v0.4.0/go.mod h1:tvsjdKG6xfiCx4LSiUZ06kcv38xvdVQwv8R6/VnnVWg=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/sirupsen/logrus v1.10.0 h1:T8MxJJXVZkfcC5zSRMRAg2F8+lxjmUCGGWPzFxO+Msc=
github.com/sirupsen/logrus v1.10.0/go.mod h1:FXZFonkDAnFozmO+5hGAFvB0Yg9/j2SIhA/QuIkP180=
github.com/sirupsen/logrus v1.10.2 h1:G2SED73/qrAu6YwbdxOD6peLkCBI3z7L+ykJFTXJBBo=
github.com/sirupsen/logrus v1.10.2/go.mod h1:SLEg8TqYulVKKfIGHldVp2K2aYz2DKSVBq4g/H5bR7Q=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
@ -266,8 +266,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/stretchr/testify v1.12.0 h1:K6Mr6jO9JICuend/5xzTM03ydSV3vdNRYAdPSukj8uI=
github.com/stretchr/testify v1.12.0/go.mod h1:bOYBZb5qJ00vPzWfIqBUZPaxK8jWiXc6d3ErP4Ca9Gw=
github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
github.com/tetratelabs/wabin v0.0.0-20230304001439-f6f874872834 h1:ZF+QBjOI+tILZjBaFj3HgFonKXUcwgJ4djLb6i42S3Q=

9
log/log.go
View File

@ -47,8 +47,13 @@ var redacted = &Hook{
// External services query params. Values can be JWTs (dots, dashes), so match everything up
// to the next query separator or whitespace, not just word chars. A [\w]+ class would stop
// at a JWT's first '.' and leak its payload and signature.
"([^\\w]api_key=)[^&\\s]+",
// at a JWT's first '.' and leak its payload and signature. Case-insensitive with an
// optional underscore: the API accepts api_key, apikey and ApiKey alike.
"(?i)([^\\w]api_?key=)[^&\\s]+",
// Sensitive request headers, logged as a JSON blob at trace level and never matched by the
// query-param patterns above. Blank the whole value array; values may hold escaped quotes.
`(?i)("(?:Authorization|X-Emby-Token|X-MediaBrowser-Token|X-Nd-Authorization)":\[")[^\]]*("\])`,
},
}

29
log/log_test.go
View File

@ -2,7 +2,9 @@ package log
import (
"context"
"encoding/json"
"errors"
"net/http"
"net/http/httptest"
"testing"
"time"
@ -92,7 +94,7 @@ var _ = Describe("Logger", func() {
SetLogSourceLine(true)
Error("A crash happened")
// NOTE: This assertion breaks if the line number above changes
Expect(hook.LastEntry().Data[" source"]).To(ContainSubstring("/log/log_test.go:93"))
Expect(hook.LastEntry().Data[" source"]).To(ContainSubstring("/log/log_test.go:95"))
Expect(hook.LastEntry().Message).To(Equal("A crash happened"))
})
@ -264,5 +266,30 @@ var _ = Describe("Logger", func() {
msg := "/jellyfin/Audio/abc/universal?static=true&api_key=eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJhZG1pbiJ9.c2ln-X_1&other=1"
Expect(Redact(msg)).To(Equal("/jellyfin/Audio/abc/universal?static=true&api_key=[REDACTED]&other=1"))
})
DescribeTable("redacts every api_key spelling the Jellyfin API accepts",
func(param string) {
msg := "/jellyfin/Audio/abc/File?" + param + "=SECRET&other=1"
Expect(Redact(msg)).To(Equal("/jellyfin/Audio/abc/File?" + param + "=[REDACTED]&other=1"))
},
Entry("api_key", "api_key"),
Entry("apikey", "apikey"),
Entry("ApiKey", "ApiKey"),
Entry("APIKEY", "APIKEY"),
)
It("redacts sensitive request headers in a logged header blob", func() {
h := http.Header{
"Authorization": {`MediaBrowser Client="Finamp", Token="jwt-secret"`},
"X-Emby-Token": {"emby-secret"},
"X-Mediabrowser-Token": {"mb-secret"},
"X-Nd-Authorization": {"Bearer nd-secret"},
"User-Agent": {"Finamp/1.0"},
}
blob, _ := json.Marshal(h)
got := Redact(string(blob))
Expect(got).ToNot(ContainSubstring("secret"))
Expect(got).To(ContainSubstring(`"User-Agent":["Finamp/1.0"]`))
})
})
})

26
model/artwork.go
View File

@ -51,6 +51,10 @@ type ItemArtwork struct {
SourcePath string `structs:"source_path"`
// RefMtime is SourcePath's mtime (unix-nanoseconds) at resolution; 0 when there is no SourcePath.
RefMtime int64 `structs:"ref_mtime"`
// Trace is the encoded walk that produced this state; LastFailure is the walk of the attempt
// that exhausted the retry budget. Both are JSON, read back with artwork.DecodeTrace.
Trace string `structs:"trace"`
LastFailure string `structs:"last_failure"`
// Nullable in the schema, but every insert must set them: these non-pointer fields cannot scan NULL.
AttemptedAt time.Time `structs:"attempted_at"`
UpdatedAt time.Time `structs:"updated_at"`
@ -91,6 +95,8 @@ type ArtworkQueueItem struct {
Attempts int `structs:"attempts"`
RetryAt time.Time `structs:"retry_at"`
EnqueuedAt time.Time `structs:"enqueued_at"`
// Trace is why the last attempt failed. Only Get reads it; the drain projects it away.
Trace string `structs:"trace"`
}
// Queue priorities: higher drains first.
@ -109,6 +115,8 @@ type ArtworkRepository interface {
PurgeOrphans(createdBefore time.Time) (int64, error)
GetItemArtwork(kind Kind, id, imageType string) (*ItemArtwork, error)
PutItemArtwork(ia *ItemArtwork) error
// PutLastFailure records the trace of the attempt that exhausted the retry budget.
PutLastFailure(kind Kind, id, imageType, trace string) error
DeleteForItems(kind Kind, ids []string) error
// GetInfoForItems hydrates a page in one batched query.
GetInfoForItems(kind Kind, ids []string) (map[string]ItemArtworkInfo, error)
@ -126,8 +134,9 @@ type ArtworkQueueRepository interface {
// EnqueuePreservingBackoff upserts like Enqueue but preserves an existing row's retry_at, so a
// request-triggered read-through never resets a failed resolution's backoff.
EnqueuePreservingBackoff(items ...ArtworkQueueItem) error
// EnqueueStaleAbsent inserts queue rows (priority Recheck) for absent states older than cutoff.
EnqueueStaleAbsent(kind Kind, attemptedBefore time.Time) (int64, error)
// EnqueueStaleAbsent inserts queue rows (priority Recheck) for absent states older than cutoff, oldest
// first; limit caps the selection, so already-queued rows use up budget (backpressure when the drain stalls).
EnqueueStaleAbsent(kind Kind, attemptedBefore time.Time, limit int) (int64, error)
// EnqueueAllMissing inserts queue rows for all entities with no item_artwork row, at the given priority.
EnqueueAllMissing(kind Kind, priority int) (int64, error)
// EnqueueIfMissing inserts only for items with no item_artwork row yet.
@ -145,17 +154,20 @@ type ArtworkQueueRepository interface {
DequeueBatch(n int, kinds ...string) ([]ArtworkQueueItem, error)
// MarkFailedIfUnchanged applies the failure backoff only while retry_at still matches
// seenRetryAt, so a concurrent re-enqueue keeps its fresh eligibility.
MarkFailedIfUnchanged(kind, id, imageType string, seenRetryAt, retryAt time.Time) error
MarkFailedIfUnchanged(kind, id, imageType string, seenRetryAt, retryAt time.Time, trace string) error
// DeleteIfUnchanged deletes only while retry_at still matches, sparing a concurrent re-enqueue.
DeleteIfUnchanged(kind, id, imageType string, retryAt time.Time) error
Count() (int64, error)
// CountByKindAndPriority reports the pending queue rows grouped by kind and priority.
CountByKindAndPriority() ([]ArtworkQueueStat, error)
// CountAbsent reports the absent states of a kind, and how many of those EnqueueStaleAbsent
// would pick up at the given cutoff.
// CountQueued reports the pending rows matching the kinds and priorities, grouped by both;
// an empty filter means every one.
CountQueued(kinds []Kind, priorities []int) ([]ArtworkQueueStat, error)
// CountAbsent reports the absent states of a kind, and how many are past the given cutoff,
// eligible for EnqueueStaleAbsent (which drains them limit rows per call).
CountAbsent(kind Kind, attemptedBefore time.Time) (ArtworkAbsentStat, error)
// PurgeDangling removes queue rows whose entity no longer exists.
PurgeDangling() (int64, error)
// PurgeQueued removes pending rows matching the kinds and priorities; an empty filter means every one.
PurgeQueued(kinds []Kind, priorities []int) (int64, error)
}
type ArtworkQueueStat struct {

7
model/artwork_id.go
View File

@ -6,6 +6,8 @@ import (
"strconv"
"strings"
"time"
"github.com/navidrome/navidrome/utils/slice"
)
type Kind struct {
@ -40,6 +42,11 @@ var artworkKindMap = map[string]Kind{
KindRadioArtwork.prefix: KindRadioArtwork,
}
// KindPrefixes leaves the typed Kind domain for the item_kind column, or for a help string.
func KindPrefixes(kinds []Kind) []string {
return slice.Map(kinds, func(k Kind) string { return k.prefix })
}
// ParseKind resolves an item_kind prefix (e.g. "al") to its Kind, reporting whether it was known.
// Use it at string boundaries — URL params, the item_kind column — to enter the typed Kind domain.
func ParseKind(prefix string) (Kind, bool) {

6
model/mediafile_test.go
View File

@ -218,11 +218,11 @@ var _ = Describe("MediaFiles", func() {
{Tags: Tags{"genre": []string{"Alternative", "Rock"}}},
}
})
It("sets the correct Genre, sorted by frequency, then alphabetically", func() {
It("sets the correct Genre, sorted by frequency, then by order of appearance", func() {
album := mfs.ToAlbum()
Expect(album.Tags).To(HaveLen(2))
Expect(album.Tags).To(HaveKeyWithValue(TagGenre, []string{"Rock", "Alternative", "Punk"}))
Expect(album.Tags).To(HaveKeyWithValue(TagMood, []string{"Chill", "Happy"}))
Expect(album.Tags).To(HaveKeyWithValue(TagGenre, []string{"Rock", "Punk", "Alternative"}))
Expect(album.Tags).To(HaveKeyWithValue(TagMood, []string{"Happy", "Chill"}))
})
})
When("we have tags with mismatching case", func() {

49
model/request/request.go
View File

@ -2,6 +2,7 @@ package request
import (
"context"
"sync/atomic"
"github.com/navidrome/navidrome/model"
)
@ -9,15 +10,16 @@ import (
type contextKey string
const (
User = contextKey("user")
Username = contextKey("username")
Client = contextKey("client")
Version = contextKey("version")
Player = contextKey("player")
Transcoding = contextKey("transcoding")
ClientUniqueId = contextKey("clientUniqueId")
ReverseProxyIp = contextKey("reverseProxyIp")
InternalAuth = contextKey("internalAuth") // Used for internal API calls, e.g., from the plugins
User = contextKey("user")
Username = contextKey("username")
Client = contextKey("client")
Version = contextKey("version")
Player = contextKey("player")
Transcoding = contextKey("transcoding")
ClientUniqueId = contextKey("clientUniqueId")
ReverseProxyIp = contextKey("reverseProxyIp")
InternalAuth = contextKey("internalAuth") // Used for internal API calls, e.g., from the plugins
TokenEpochHolder = contextKey("tokenEpochHolder")
)
var allKeys = []contextKey{
@ -125,3 +127,32 @@ func AddValues(ctx, requestCtx context.Context) context.Context {
}
return ctx
}
type tokenEpochHolder struct {
value atomic.Int64
}
// WithTokenEpochHolder installs a slot a handler can use to report a bumped token epoch
// back to middleware that has already returned from the handler's perspective.
func WithTokenEpochHolder(ctx context.Context) context.Context {
h := &tokenEpochHolder{}
h.value.Store(-1)
return context.WithValue(ctx, TokenEpochHolder, h)
}
func SetTokenEpoch(ctx context.Context, epoch int) {
if h, ok := ctx.Value(TokenEpochHolder).(*tokenEpochHolder); ok {
h.value.Store(int64(epoch))
}
}
func TokenEpochFrom(ctx context.Context) (int, bool) {
h, ok := ctx.Value(TokenEpochHolder).(*tokenEpochHolder)
if !ok {
return 0, false
}
if v := h.value.Load(); v >= 0 {
return int(v), true
}
return 0, false
}

17
model/request/request_suite_test.go Normal file
View File

@ -0,0 +1,17 @@
package request
import (
"testing"
"github.com/navidrome/navidrome/log"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
// tests.Init is not used here: the tests package imports model/request, so importing it
// back would create an import cycle.
func TestRequest(t *testing.T) {
log.SetLevel(log.LevelFatal)
RegisterFailHandler(Fail)
RunSpecs(t, "Request Suite")
}

40
model/request/request_test.go Normal file
View File

@ -0,0 +1,40 @@
package request
import (
"context"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
var _ = Describe("Token epoch holder", func() {
It("reports nothing when unset", func() {
ctx := WithTokenEpochHolder(context.TODO())
_, ok := TokenEpochFrom(ctx)
Expect(ok).To(BeFalse())
})
It("round-trips a value set by the handler", func() {
ctx := WithTokenEpochHolder(context.TODO())
SetTokenEpoch(ctx, 7)
epoch, ok := TokenEpochFrom(ctx)
Expect(ok).To(BeTrue())
Expect(epoch).To(Equal(7))
})
It("survives being wrapped in a derived context", func() {
ctx := WithTokenEpochHolder(context.TODO())
SetTokenEpoch(context.WithValue(ctx, contextKey("unrelated"), 1), 3)
epoch, ok := TokenEpochFrom(ctx)
Expect(ok).To(BeTrue())
Expect(epoch).To(Equal(3))
})
It("is a no-op with no holder installed", func() {
Expect(func() { SetTokenEpoch(context.TODO(), 5) }).ToNot(Panic())
_, ok := TokenEpochFrom(context.TODO())
Expect(ok).To(BeFalse())
})
})

8
model/tag.go
View File

@ -24,13 +24,17 @@ type TagList []Tag
func (l TagList) GroupByFrequency() Tags {
grouped := map[string]map[string]int{}
values := map[string]string{}
for _, t := range l {
firstSeen := map[string]int{}
for i, t := range l {
if m, ok := grouped[string(t.TagName)]; !ok {
grouped[string(t.TagName)] = map[string]int{t.ID: 1}
} else {
m[t.ID]++
}
values[t.ID] = t.TagValue
if _, ok := firstSeen[t.ID]; !ok {
firstSeen[t.ID] = i
}
}
tags := Tags{}
@ -42,7 +46,7 @@ func (l TagList) GroupByFrequency() Tags {
slices.SortFunc(idList, func(a, b string) int {
return cmp.Or(
cmp.Compare(counts[b], counts[a]),
cmp.Compare(values[a], values[b]),
cmp.Compare(firstSeen[a], firstSeen[b]),
)
})
tags[TagName(name)] = slice.Map(idList, func(id string) string { return values[id] })

4
model/tag_test.go
View File

@ -93,7 +93,7 @@ var _ = Describe("Tag", func() {
Expect(groupedTags).To(HaveKeyWithValue(TagName("artist"), []string{"The Beatles", "The Rolling Stones"}))
})
It("should sort tags by name when frequency is the same", func() {
It("should keep the order the values appeared in when frequency is the same", func() {
tagList := TagList{
NewTag("genre", "Jazz"),
NewTag("genre", "Rock"),
@ -103,7 +103,7 @@ var _ = Describe("Tag", func() {
groupedTags := tagList.GroupByFrequency()
Expect(groupedTags).To(HaveKeyWithValue(TagName("genre"), []string{"Alternative", "Jazz", "Pop", "Rock"}))
Expect(groupedTags).To(HaveKeyWithValue(TagName("genre"), []string{"Jazz", "Rock", "Alternative", "Pop"}))
})
It("should normalize casing", func() {
tagList := TagList{

2
model/user.go
View File

@ -22,6 +22,8 @@ type User struct {
// This is only available on the backend, and it is never sent over the wire
Password string `structs:"-" json:"-"`
// Bumped on password change to invalidate every issued token for this user.
TokenEpoch int `structs:"-" json:"-"`
// This is used to set or change a password when calling Put. If it is empty, the password is not changed.
// It is received from the UI with the name "password"
NewPassword string `structs:"password,omitempty" json:"password,omitempty"` //nolint:gosec

2
persistence/album_repository.go
View File

@ -113,7 +113,7 @@ func NewAlbumRepository(ctx context.Context, db dbx.Builder) model.AlbumReposito
"artist": "compilation, order_album_artist_name, order_album_name",
"album_artist": "compilation, order_album_artist_name, order_album_name",
// TODO Rename this to just year (or date)
"max_year": "coalesce(nullif(original_date,''), cast(max_year as text)), release_date, name",
"max_year": "coalesce(nullif(original_date,''), cast(max_year as text)), release_date, " + naturalSort("album.name"),
"random": "random",
"recently_added": recentlyAddedSort(),
"starred_at": "starred, starred_at",

39
persistence/album_repository_test.go
View File

@ -10,6 +10,7 @@ import (
"github.com/Masterminds/squirrel"
"github.com/deluan/rest"
"github.com/navidrome/navidrome/conf"
"github.com/navidrome/navidrome/conf/configtest"
"github.com/navidrome/navidrome/consts"
"github.com/navidrome/navidrome/model"
"github.com/navidrome/navidrome/model/id"
@ -38,6 +39,44 @@ var _ = Describe("AlbumRepository", func() {
albumRepo = NewAlbumRepository(ctx, GetDBXBuilder()).(*albumRepository)
})
Describe("natural sorting", func() {
var ids []string
BeforeEach(func() {
DeferCleanup(configtest.SetupConfig())
ids = nil
for _, n := range []string{"foo 1", "foo 10", "foo 2", "foo 20", "foo 3"} {
aid := "nat-" + n
ids = append(ids, aid)
Expect(albumRepo.Put(&model.Album{
ID: aid, LibraryID: 1, Name: n, OrderAlbumName: n,
})).To(Succeed())
}
DeferCleanup(func() {
_, _ = albumRepo.executeSQL(squirrel.Delete("album").Where(squirrel.Eq{"id": ids}))
})
})
DescribeTable("sorts albums by name",
func(naturalSorting, preferSortTags bool, expected []string) {
conf.Server.EnableNaturalSorting = naturalSorting
conf.Server.PreferSortTags = preferSortTags
albumRepo = NewAlbumRepository(ctx, GetDBXBuilder()).(*albumRepository)
albums, err := albumRepo.GetAll(model.QueryOptions{
Sort: "name", Filters: squirrel.Eq{"album.id": ids},
})
Expect(err).ToNot(HaveOccurred())
Expect(slice.Map(albums, func(a model.Album) string { return a.Name })).To(Equal(expected))
},
Entry("lexicographically by default", false, false,
[]string{"foo 1", "foo 10", "foo 2", "foo 20", "foo 3"}),
Entry("by number value when natural sorting is enabled", true, false,
[]string{"foo 1", "foo 2", "foo 3", "foo 10", "foo 20"}),
Entry("by number value with sort tags preferred too", true, true,
[]string{"foo 1", "foo 2", "foo 3", "foo 10", "foo 20"}),
)
})
Describe("Get", func() {
var Get = func(id string) (*model.Album, error) {
album, err := albumRepo.Get(id)

60
persistence/artwork_queue_repository.go
View File

@ -18,6 +18,7 @@ import (
const enqueueChunkSize = 100
// Every insert writes these, in this order; the INSERT..SELECT forms must project them to match.
// DequeueBatch also selects exactly these, to leave the drain's rows free of the trace it never reads.
var enqueueColumns = []string{"item_kind", "item_id", "image_type", "priority", "attempts", "retry_at", "enqueued_at"}
type artworkQueueRepository struct {
@ -42,11 +43,12 @@ func (r *artworkQueueRepository) Get(kind model.Kind, id, imageType string) (*mo
return &res, nil
}
// Enqueue also resets enqueued_at, so a fresh request does not inherit an old row's spent retry budget.
// Enqueue starts a fresh lifecycle: it resets enqueued_at (so a fresh request does not inherit an old
// row's spent retry budget) and clears trace (so explain does not show a prior failure at attempts 0).
func (r *artworkQueueRepository) Enqueue(items ...model.ArtworkQueueItem) error {
return r.enqueue(`ON CONFLICT (item_kind, item_id, image_type) DO UPDATE SET
priority = MAX(priority, excluded.priority), retry_at = excluded.retry_at,
attempts = 0, enqueued_at = excluded.enqueued_at`, items)
attempts = 0, enqueued_at = excluded.enqueued_at, trace = '[]'`, items)
}
func (r *artworkQueueRepository) EnqueuePreservingBackoff(items ...model.ArtworkQueueItem) error {
@ -54,11 +56,12 @@ func (r *artworkQueueRepository) EnqueuePreservingBackoff(items ...model.Artwork
priority = MAX(priority, excluded.priority)`, items)
}
func (r *artworkQueueRepository) EnqueueStaleAbsent(kind model.Kind, attemptedBefore time.Time) (int64, error) {
func (r *artworkQueueRepository) EnqueueStaleAbsent(kind model.Kind, attemptedBefore time.Time, limit int) (int64, error) {
now := time.Now()
return r.insertIfNotQueued("", `SELECT item_kind, item_id, image_type, ?, 0, ?, ?
FROM `+itemArtworkTable+` WHERE item_kind = ? AND hash = '' AND attempted_at < ?`,
model.ArtworkPriorityRecheck, now, now, kind.Prefix(), attemptedBefore)
FROM `+itemArtworkTable+` WHERE item_kind = ? AND hash = '' AND attempted_at < ?
ORDER BY attempted_at LIMIT ?`,
model.ArtworkPriorityRecheck, now, now, kind.Prefix(), attemptedBefore, limit)
}
func (r *artworkQueueRepository) EnqueueAllMissing(kind model.Kind, priority int) (int64, error) {
@ -159,7 +162,7 @@ func (r *artworkQueueRepository) enqueue(conflict string, items []model.ArtworkQ
}
func (r *artworkQueueRepository) DequeueBatch(n int, kinds ...string) ([]model.ArtworkQueueItem, error) {
sel := Select("*").From(r.tableName).
sel := Select(enqueueColumns...).From(r.tableName).
Where(LtOrEq{"retry_at": time.Now()}).
OrderBy("priority DESC", "enqueued_at ASC").
Limit(uint64(n))
@ -171,10 +174,11 @@ func (r *artworkQueueRepository) DequeueBatch(n int, kinds ...string) ([]model.A
return res, err
}
func (r *artworkQueueRepository) MarkFailedIfUnchanged(kind, id, imageType string, seenRetryAt, retryAt time.Time) error {
func (r *artworkQueueRepository) MarkFailedIfUnchanged(kind, id, imageType string, seenRetryAt, retryAt time.Time, trace string) error {
upd := Update(r.tableName).
Set("attempts", Expr("attempts + 1")).
Set("retry_at", retryAt).
Set("trace", trace).
Where(Eq{"item_kind": kind, "item_id": id, "image_type": imageType, "retry_at": seenRetryAt})
_, err := r.executeSQL(upd)
return err
@ -188,20 +192,46 @@ func (r *artworkQueueRepository) PurgeDangling() (int64, error) {
return purgeDangling(r.sqlRepository)
}
// artworkQueueFilter returns no conditions for an empty filter, so an unfiltered DELETE keeps
// SQLite's truncate path. It ignores retry_at: a backing-off row is pending work too.
func artworkQueueFilter(kinds []model.Kind, priorities []int) And {
var f And
if len(kinds) > 0 {
f = append(f, Eq{"item_kind": model.KindPrefixes(kinds)})
}
if len(priorities) > 0 {
f = append(f, Eq{"priority": priorities})
}
return f
}
// CountQueued shares its filter with PurgeQueued, so a preview cannot count rows the delete misses.
func (r *artworkQueueRepository) CountQueued(kinds []model.Kind, priorities []int) ([]model.ArtworkQueueStat, error) {
sel := Select("item_kind", "priority", "count(*) as count").From(r.tableName).
GroupBy("item_kind", "priority").OrderBy("item_kind", "priority desc")
if f := artworkQueueFilter(kinds, priorities); len(f) > 0 {
sel = sel.Where(f)
}
var res []model.ArtworkQueueStat
err := r.queryAll(sel, &res)
return res, err
}
func (r *artworkQueueRepository) PurgeQueued(kinds []model.Kind, priorities []int) (int64, error) {
del := Delete(r.tableName)
if f := artworkQueueFilter(kinds, priorities); len(f) > 0 {
del = del.Where(f)
}
return r.executeSQL(del)
}
func (r *artworkQueueRepository) Count() (int64, error) {
var res struct{ Count int64 }
err := r.queryOne(Select("count(*) as count").From(r.tableName), &res)
return res.Count, err
}
func (r *artworkQueueRepository) CountByKindAndPriority() ([]model.ArtworkQueueStat, error) {
var res []model.ArtworkQueueStat
err := r.queryAll(Select("item_kind", "priority", "count(*) as count").From(r.tableName).
GroupBy("item_kind", "priority").OrderBy("item_kind", "priority desc"), &res)
return res, err
}
// CountAbsent matches EnqueueStaleAbsent on hash, so the stale count is what a recheck would queue.
// CountAbsent matches EnqueueStaleAbsent on hash, so the stale count is the pool a recheck drains from.
func (r *artworkQueueRepository) CountAbsent(kind model.Kind, attemptedBefore time.Time) (model.ArtworkAbsentStat, error) {
var res model.ArtworkAbsentStat
err := r.queryOne(Select("count(*) as total").

117
persistence/artwork_queue_repository_test.go
View File

@ -127,19 +127,42 @@ var _ = Describe("ArtworkQueueRepository", func() {
Expect(repo.Enqueue(item("al", "m1", model.ArtworkPriorityScan))).To(Succeed())
future := time.Now().Add(48 * time.Hour)
Expect(repo.MarkFailedIfUnchanged("al", "m1", model.ImageTypePrimary, original, future)).To(Succeed())
Expect(repo.MarkFailedIfUnchanged("al", "m1", model.ImageTypePrimary, original, future, "[]")).To(Succeed())
got, _ = repo.DequeueBatch(10)
Expect(got).To(HaveLen(1), "the fresh re-enqueue stays immediately eligible")
Expect(got[0].Attempts).To(BeZero(), "re-enqueue clears attempts, and the stale failure must not bump them")
current := got[0].RetryAt
Expect(repo.MarkFailedIfUnchanged("al", "m1", model.ImageTypePrimary, current, future)).To(Succeed())
Expect(repo.MarkFailedIfUnchanged("al", "m1", model.ImageTypePrimary, current, future, `[{"c":"read","o":"error"}]`)).To(Succeed())
got, _ = repo.DequeueBatch(10)
Expect(got).To(BeEmpty(), "backed-off row is hidden until the future retry_at")
all, _ := repo.Count()
Expect(all).To(Equal(int64(1)))
})
It("Enqueue clears a prior lifecycle's failure trace; EnqueuePreservingBackoff keeps it", func() {
// Fail an attempt so the queue row carries a failure trace.
Expect(repo.Enqueue(item("al", "t1", model.ArtworkPriorityScan))).To(Succeed())
backOff("al", "t1", time.Now().Add(-time.Hour))
got, _ := repo.DequeueBatch(10)
Expect(got).To(HaveLen(1))
future := time.Now().Add(48 * time.Hour)
Expect(repo.MarkFailedIfUnchanged("al", "t1", model.ImageTypePrimary, got[0].RetryAt, future, `[{"c":"read","o":"error"}]`)).To(Succeed())
// A continuation of the same lifecycle must retain the trace.
Expect(repo.EnqueuePreservingBackoff(item("al", "t1", model.ArtworkPriorityBump))).To(Succeed())
kept, err := repo.Get(model.KindAlbumArtwork, "t1", model.ImageTypePrimary)
Expect(err).ToNot(HaveOccurred())
Expect(kept.Trace).To(Equal(`[{"c":"read","o":"error"}]`))
// A fresh Enqueue resets attempts to 0, so the stale failure trace must be cleared with it.
Expect(repo.Enqueue(item("al", "t1", model.ArtworkPriorityScan))).To(Succeed())
fresh, err := repo.Get(model.KindAlbumArtwork, "t1", model.ImageTypePrimary)
Expect(err).ToNot(HaveOccurred())
Expect(fresh.Attempts).To(BeZero())
Expect(fresh.Trace).To(Equal("[]"), "a fresh lifecycle has no last-attempt trace")
})
It("Enqueue restarts the retry budget an existing row had spent", func() {
Expect(repo.Enqueue(item("al", "e1", model.ArtworkPriorityScan))).To(Succeed())
backOff("al", "e1", time.Now().Add(-time.Hour))
@ -221,7 +244,7 @@ var _ = Describe("ArtworkQueueRepository", func() {
Expect(awRepo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "fresh1", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: time.Now()})).To(Succeed())
Expect(awRepo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "found1", ImageType: model.ImageTypePrimary, Hash: "hX", AttemptedAt: old})).To(Succeed())
n, err := repo.EnqueueStaleAbsent(model.KindArtistArtwork, time.Now().Add(-24*time.Hour))
n, err := repo.EnqueueStaleAbsent(model.KindArtistArtwork, time.Now().Add(-24*time.Hour), 100)
Expect(err).ToNot(HaveOccurred())
Expect(n).To(Equal(int64(1)))
@ -232,6 +255,23 @@ var _ = Describe("ArtworkQueueRepository", func() {
Expect(items[0].Priority).To(Equal(model.ArtworkPriorityRecheck))
})
It("enqueues only the oldest stale absent states up to the limit", func() {
awRepo := NewArtworkRepository(context.Background(), GetDBXBuilder())
now := time.Now()
Expect(awRepo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "oldest", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: now.Add(-72 * time.Hour)})).To(Succeed())
Expect(awRepo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "older", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: now.Add(-60 * time.Hour)})).To(Succeed())
Expect(awRepo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "old", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: now.Add(-48 * time.Hour)})).To(Succeed())
n, err := repo.EnqueueStaleAbsent(model.KindArtistArtwork, now.Add(-24*time.Hour), 2)
Expect(err).ToNot(HaveOccurred())
Expect(n).To(Equal(int64(2)))
items, err := repo.DequeueBatch(10)
Expect(err).ToNot(HaveOccurred())
ids := slice.Map(items, func(it model.ArtworkQueueItem) string { return it.ItemID })
Expect(ids).To(ConsistOf("oldest", "older"))
})
It("enqueues entities that have no item_artwork row at all", func() {
awRepo := NewArtworkRepository(context.Background(), GetDBXBuilder())
Expect(awRepo.PutItemArtwork(&model.ItemArtwork{ItemKind: "al", ItemID: albumSgtPeppers.ID, ImageType: model.ImageTypePrimary, Hash: "hX", AttemptedAt: time.Now()})).To(Succeed())
@ -388,7 +428,7 @@ var _ = Describe("ArtworkQueueRepository", func() {
Expect(repo.Enqueue(item("ar", "a3", model.ArtworkPriorityBump))).To(Succeed())
Expect(repo.Enqueue(item("al", "b1", model.ArtworkPriorityScan))).To(Succeed())
Expect(repo.CountByKindAndPriority()).To(ConsistOf(
Expect(repo.CountQueued(nil, nil)).To(ConsistOf(
model.ArtworkQueueStat{ItemKind: "ar", Priority: model.ArtworkPriorityBackfill, Count: 2},
model.ArtworkQueueStat{ItemKind: "ar", Priority: model.ArtworkPriorityBump, Count: 1},
model.ArtworkQueueStat{ItemKind: "al", Priority: model.ArtworkPriorityScan, Count: 1},
@ -396,7 +436,7 @@ var _ = Describe("ArtworkQueueRepository", func() {
})
It("reports an empty queue as no rows", func() {
Expect(repo.CountByKindAndPriority()).To(BeEmpty())
Expect(repo.CountQueued(nil, nil)).To(BeEmpty())
})
It("counts absent states and how many are due for recheck", func() {
@ -419,4 +459,71 @@ var _ = Describe("ArtworkQueueRepository", func() {
Expect(repo.CountAbsent(model.KindRadioArtwork, time.Now())).To(Equal(model.ArtworkAbsentStat{}))
})
})
Describe("PurgeQueued", func() {
queuedIDs := func() []string {
GinkgoHelper()
got, err := repo.DequeueBatch(100)
Expect(err).ToNot(HaveOccurred())
return slice.Map(got, func(it model.ArtworkQueueItem) string { return it.ItemID })
}
BeforeEach(func() {
Expect(repo.Enqueue(
item("ar", "ar-backfill", model.ArtworkPriorityBackfill),
item("ar", "ar-bump", model.ArtworkPriorityBump),
item("al", "al-backfill", model.ArtworkPriorityBackfill),
item("mf", "mf-scan", model.ArtworkPriorityScan),
)).To(Succeed())
})
// CountQueued feeds the preview and PurgeQueued does the delete; they share one filter, so
// every selection must count exactly what it deletes.
DescribeTable("selects the same rows to count and to delete",
func(kinds []model.Kind, priorities []int, deleted int, remaining []string) {
counted, err := repo.CountQueued(kinds, priorities)
Expect(err).ToNot(HaveOccurred())
var total int64
for _, s := range counted {
total += s.Count
}
Expect(total).To(BeNumerically("==", deleted), "the preview must match the delete")
Expect(repo.PurgeQueued(kinds, priorities)).To(BeNumerically("==", deleted))
Expect(queuedIDs()).To(ConsistOf(remaining))
},
Entry("only the given kinds", []model.Kind{model.KindArtistArtwork}, nil,
2, []string{"al-backfill", "mf-scan"}),
Entry("only the given priorities", nil, []int{model.ArtworkPriorityBackfill},
2, []string{"ar-bump", "mf-scan"}),
Entry("the intersection of both", []model.Kind{model.KindArtistArtwork}, []int{model.ArtworkPriorityBackfill},
1, []string{"ar-bump", "al-backfill", "mf-scan"}),
Entry("everything, when neither filter is given", nil, nil,
4, []string{}),
Entry("several kinds and priorities at once",
[]model.Kind{model.KindArtistArtwork, model.KindMediaFileArtwork},
[]int{model.ArtworkPriorityBackfill, model.ArtworkPriorityScan},
2, []string{"ar-bump", "al-backfill"}),
Entry("nothing, leaving the queue alone", []model.Kind{model.KindPlaylistArtwork}, nil,
0, []string{"ar-backfill", "ar-bump", "al-backfill", "mf-scan"}),
)
It("deletes a row that is still backing off", func() {
backOff("ar", "ar-bump", time.Now().Add(time.Hour))
Expect(repo.PurgeQueued([]model.Kind{model.KindArtistArtwork}, nil)).To(BeNumerically("==", 2))
Expect(repo.Get(model.KindArtistArtwork, "ar-bump", model.ImageTypePrimary)).
Error().To(MatchError(model.ErrNotFound))
})
// A WHERE clause, even one that matches everything, costs SQLite its truncate optimization
// and turns `artwork cancel --all` into a full scan of the queue.
It("adds no conditions at all for an empty filter", func() {
Expect(artworkQueueFilter(nil, nil)).To(BeEmpty())
Expect(artworkQueueFilter([]model.Kind{model.KindArtistArtwork}, nil)).To(HaveLen(1))
Expect(artworkQueueFilter(nil, []int{model.ArtworkPriorityBump})).To(HaveLen(1))
Expect(artworkQueueFilter([]model.Kind{model.KindArtistArtwork}, []int{model.ArtworkPriorityBump})).
To(HaveLen(2))
})
})
})

10
persistence/artwork_repository.go
View File

@ -134,11 +134,21 @@ func (r *artworkRepository) PutItemArtwork(ia *model.ItemArtwork) error {
}
ins := Insert(itemArtworkTable).SetMap(values).Suffix(`ON CONFLICT (item_kind, item_id, image_type) DO UPDATE SET
hash=excluded.hash, source=excluded.source, source_path=excluded.source_path, ref_mtime=excluded.ref_mtime,
trace=excluded.trace, last_failure=excluded.last_failure,
attempted_at=excluded.attempted_at, updated_at=excluded.updated_at`)
_, err = r.items.executeSQL(ins)
return err
}
// PutLastFailure records why an item exhausted its retry budget. It only updates an existing row:
// inserting one would write an empty hash, which the rest of the system reads as a settled absent.
func (r *artworkRepository) PutLastFailure(kind model.Kind, id, imageType, trace string) error {
upd := Update(itemArtworkTable).Set("last_failure", trace).
Where(Eq{"item_kind": kind.Prefix(), "item_id": id, "image_type": imageType})
_, err := r.items.executeSQL(upd)
return err
}
func (r *artworkRepository) DeleteForItems(kind model.Kind, ids []string) error {
for chunk := range slices.Chunk(ids, artworkBatchSize) {
if err := r.items.delete(Eq{"item_kind": kind.Prefix(), "item_id": chunk}); err != nil {

45
persistence/artwork_repository_test.go
View File

@ -28,6 +28,51 @@ var _ = Describe("ArtworkRepository", func() {
repo = NewArtworkRepository(context.Background(), GetDBXBuilder())
})
Context("resolution traces", func() {
const traceJSON = `[{"c":"cover.*","o":"hit"}]`
It("round-trips the trace with the state row", func() {
Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "al", ItemID: "t1",
ImageType: model.ImageTypePrimary, Hash: "h1", Trace: traceJSON})).To(Succeed())
got, err := repo.GetItemArtwork(model.KindAlbumArtwork, "t1", model.ImageTypePrimary)
Expect(err).ToNot(HaveOccurred())
Expect(got.Trace).To(Equal(traceJSON))
Expect(got.LastFailure).To(BeEmpty())
})
It("replaces the trace when the item is resolved again", func() {
Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "al", ItemID: "t2",
ImageType: model.ImageTypePrimary, Trace: traceJSON})).To(Succeed())
Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "al", ItemID: "t2",
ImageType: model.ImageTypePrimary, Trace: `[{"c":"embedded","o":"hit"}]`})).To(Succeed())
got, _ := repo.GetItemArtwork(model.KindAlbumArtwork, "t2", model.ImageTypePrimary)
Expect(got.Trace).To(Equal(`[{"c":"embedded","o":"hit"}]`))
})
It("records a last failure on an existing row", func() {
Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "al", ItemID: "t3",
ImageType: model.ImageTypePrimary, Hash: "h3"})).To(Succeed())
Expect(repo.PutLastFailure(model.KindAlbumArtwork, "t3", model.ImageTypePrimary,
`[{"c":"decode","o":"error"}]`)).To(Succeed())
got, _ := repo.GetItemArtwork(model.KindAlbumArtwork, "t3", model.ImageTypePrimary)
Expect(got.LastFailure).To(Equal(`[{"c":"decode","o":"error"}]`))
Expect(got.Hash).To(Equal("h3"), "recording a failure must not disturb the served artwork")
})
// Inserting here would write hash='', which every reader treats as a settled absent.
It("never creates a row for an item that has no state", func() {
Expect(repo.PutLastFailure(model.KindAlbumArtwork, "ghost", model.ImageTypePrimary,
`[{"c":"decode","o":"error"}]`)).To(Succeed())
_, err := repo.GetItemArtwork(model.KindAlbumArtwork, "ghost", model.ImageTypePrimary)
Expect(err).To(MatchError(model.ErrNotFound))
})
})
Context("image identity", func() {
It("stores and retrieves an artwork by hash", func() {
a := &model.Artwork{Hash: "abc123", Mime: "image/jpeg", Width: 500, Height: 500, SizeBytes: 1234, BlurHash: "LKO2?U%2Tw=w"}

29
persistence/helpers.go
View File

@ -9,6 +9,8 @@ import (
"github.com/Masterminds/squirrel"
"github.com/fatih/structs"
"github.com/navidrome/navidrome/conf"
"github.com/navidrome/navidrome/db"
)
type PostMapper interface {
@ -82,11 +84,28 @@ func (e existsCond) ToSql() (string, []any, error) {
var sortOrderRegex = regexp.MustCompile(`order_([a-z_]+)`)
// Convert the order_* columns to an expression using sort_* columns. Example:
// sort_album_name -> (coalesce(nullif(sort_album_name,”),order_album_name) collate nocase)
// naturalSort makes a plain text column sort numbers by value, leaving it alone
// otherwise so it keeps its declared collation. Parens guard buildSortOrder's space split.
func naturalSort(col string) string {
if !conf.Server.EnableNaturalSorting {
return col
}
return fmt.Sprintf("(%s collate %s)", col, db.NaturalCollation)
}
// Convert the order_* columns to a collated sort expression, falling back to the
// sort_* column when those are preferred. Example:
// order_album_name -> (coalesce(nullif(sort_album_name,”),order_album_name) collate nocase)
// It finds order column names anywhere in the substring
func mapSortOrder(tableName, order string) string {
order = strings.ToLower(order)
repl := fmt.Sprintf("(coalesce(nullif(%[1]s.sort_$1,''),%[1]s.order_$1) collate nocase)", tableName)
return sortOrderRegex.ReplaceAllString(order, repl)
col := tableName + ".order_$1"
if conf.Server.PreferSortTags {
col = fmt.Sprintf("coalesce(nullif(%[1]s.sort_$1,''),%[1]s.order_$1)", tableName)
}
collation := "nocase"
if conf.Server.EnableNaturalSorting {
collation = db.NaturalCollation
}
repl := fmt.Sprintf("(%s collate %s)", col, collation)
return sortOrderRegex.ReplaceAllString(strings.ToLower(order), repl)
}

55
persistence/helpers_test.go
View File

@ -4,6 +4,8 @@ import (
"time"
"github.com/Masterminds/squirrel"
"github.com/navidrome/navidrome/conf"
"github.com/navidrome/navidrome/conf/configtest"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
@ -85,22 +87,51 @@ var _ = Describe("Helpers", func() {
})
Describe("mapSortOrder", func() {
BeforeEach(func() {
DeferCleanup(configtest.SetupConfig())
})
It("does not change the sort string if there are no order columns", func() {
sort := "album_name asc"
mapped := mapSortOrder("album", sort)
Expect(mapped).To(Equal(sort))
})
It("changes order columns to sort expression", func() {
sort := "ORDER_ALBUM_NAME asc"
mapped := mapSortOrder("album", sort)
Expect(mapped).To(Equal(`(coalesce(nullif(album.sort_album_name,''),album.order_album_name)` +
` collate nocase) asc`))
Expect(mapSortOrder("album", "album_name asc")).To(Equal("album_name asc"))
})
DescribeTable("maps order columns to a collated expression",
func(preferSortTags, naturalSorting bool, expected string) {
conf.Server.PreferSortTags = preferSortTags
conf.Server.EnableNaturalSorting = naturalSorting
Expect(mapSortOrder("album", "ORDER_ALBUM_NAME asc")).To(Equal(expected))
},
Entry("qualified column", false, false,
"(album.order_album_name collate nocase) asc"),
Entry("natural collation", false, true,
"(album.order_album_name collate NATSORT) asc"),
Entry("sort tags preferred", true, false,
`(coalesce(nullif(album.sort_album_name,''),album.order_album_name) collate nocase) asc`),
Entry("sort tags preferred, natural collation", true, true,
`(coalesce(nullif(album.sort_album_name,''),album.order_album_name) collate NATSORT) asc`),
)
It("changes multiple order columns to sort expressions", func() {
conf.Server.PreferSortTags = true
sort := "compilation, order_title asc, order_album_artist_name desc, year desc"
mapped := mapSortOrder("album", sort)
Expect(mapped).To(Equal(`compilation, (coalesce(nullif(album.sort_title,''),album.order_title) collate nocase) asc,` +
` (coalesce(nullif(album.sort_album_artist_name,''),album.order_album_artist_name) collate nocase) desc, year desc`))
Expect(mapSortOrder("album", sort)).To(Equal(
`compilation, (coalesce(nullif(album.sort_title,''),album.order_title) collate nocase) asc,` +
` (coalesce(nullif(album.sort_album_artist_name,''),album.order_album_artist_name) collate nocase) desc, year desc`))
})
})
Describe("naturalSort", func() {
BeforeEach(func() {
DeferCleanup(configtest.SetupConfig())
})
It("leaves the column alone by default, keeping its declared collation", func() {
Expect(naturalSort("media_file.title")).To(Equal("media_file.title"))
})
It("applies the natural collation when enabled", func() {
conf.Server.EnableNaturalSorting = true
Expect(naturalSort("media_file.title")).To(Equal("(media_file.title collate NATSORT)"))
})
})
})

2
persistence/mediafile_repository.go
View File

@ -86,7 +86,7 @@ func NewMediaFileRepository(ctx context.Context, db dbx.Builder) model.MediaFile
"title": "order_title",
"artist": "order_artist_name, order_album_name, release_date, disc_number, track_number",
"album_artist": "order_album_artist_name, order_album_name, release_date, disc_number, track_number",
"album": "order_album_name, album_id, disc_number, track_number, order_artist_name, title",
"album": "order_album_name, album_id, disc_number, track_number, order_artist_name, " + naturalSort("media_file.title"),
"random": "random",
"created_at": "media_file.created_at",
"recently_added": mediaFileRecentlyAddedSort(),

3
persistence/playlist_repository.go
View File

@ -60,7 +60,8 @@ func NewPlaylistRepository(ctx context.Context, db dbx.Builder) model.PlaylistRe
"starred": annotationBoolFilter("starred"),
})
r.setSortMappings(map[string]string{
"owner_name": "owner_name",
"name": naturalSort("playlist.name"),
"owner_name": naturalSort("owner_name"),
})
return r
}

35
persistence/playlist_repository_test.go
View File

@ -5,6 +5,8 @@ import (
"github.com/Masterminds/squirrel"
"github.com/deluan/rest"
"github.com/navidrome/navidrome/conf"
"github.com/navidrome/navidrome/conf/configtest"
"github.com/navidrome/navidrome/log"
"github.com/navidrome/navidrome/model"
"github.com/navidrome/navidrome/model/criteria"
@ -24,6 +26,39 @@ var _ = Describe("PlaylistRepository", func() {
repo = NewPlaylistRepository(ctx, GetDBXBuilder())
})
Describe("natural sorting", func() {
var ids []string
BeforeEach(func() {
DeferCleanup(configtest.SetupConfig())
conf.Server.EnableNaturalSorting = true
ctx := log.NewContext(GinkgoT().Context())
ctx = request.WithUser(ctx, model.User{ID: "userid", UserName: "userid", IsAdmin: true})
repo = NewPlaylistRepository(ctx, GetDBXBuilder())
ids = nil
for _, n := range []string{"mix 1", "mix 10", "mix 2"} {
pls := model.Playlist{Name: n, OwnerID: "userid"}
Expect(repo.Put(&pls)).To(Succeed())
ids = append(ids, pls.ID)
}
DeferCleanup(func() {
for _, id := range ids {
_ = repo.Delete(id)
}
})
})
It("sorts playlist names by number value", func() {
all, err := repo.GetAll(model.QueryOptions{
Sort: "name", Filters: squirrel.Eq{"playlist.id": ids},
})
Expect(err).ToNot(HaveOccurred())
Expect(slice.Map(all, func(p model.Playlist) string { return p.Name })).To(
Equal([]string{"mix 1", "mix 2", "mix 10"}))
})
})
Describe("Count", func() {
It("returns the number of playlists in the DB", func() {
Expect(repo.CountAll()).To(Equal(int64(2)))

2
persistence/playlist_track_repository.go
View File

@ -56,7 +56,7 @@ func (r *playlistRepository) Tracks(playlistId string, refreshSmartPlaylist bool
"id": "playlist_tracks.id",
"artist": "order_artist_name",
"album_artist": "order_album_artist_name",
"album": "order_album_name, album_id, disc_number, track_number, order_artist_name, title",
"album": "order_album_name, album_id, disc_number, track_number, order_artist_name, " + naturalSort("f.title"),
"title": "order_title",
"random": "random()",
// To make sure these fields will be whitelisted

5
persistence/sql_base_repository.go
View File

@ -113,10 +113,9 @@ func (r *sqlRepository) setSortMappings(mappings map[string]string, tableName ..
if len(tableName) > 0 {
tn = tableName[0]
}
if conf.Server.PreferSortTags {
if conf.Server.PreferSortTags || conf.Server.EnableNaturalSorting {
for k, v := range mappings {
v = mapSortOrder(tn, v)
mappings[k] = v
mappings[k] = mapSortOrder(tn, v)
}
}
r.sortMappings = mappings

35
persistence/user_repository.go
View File

@ -18,6 +18,7 @@ import (
"github.com/navidrome/navidrome/model"
"github.com/navidrome/navidrome/model/criteria"
"github.com/navidrome/navidrome/model/id"
"github.com/navidrome/navidrome/model/request"
"github.com/navidrome/navidrome/utils"
"github.com/navidrome/navidrome/utils/slice"
"github.com/pocketbase/dbx"
@ -126,14 +127,30 @@ func (r *userRepository) Put(u *model.User) error {
}
delete(values, "current_password")
// Save/update the user
// The epoch bump rides the password UPDATE: as two statements they can interleave with a
// concurrent change and leave a session valid that the other change should have revoked.
update := Update(r.tableName).Where(Eq{"id": u.ID}).SetMap(values)
count, err := r.executeSQL(update)
if err != nil {
return err
var isNewUser bool
var epoch int
if u.NewPassword != "" {
var res struct{ TokenEpoch int }
err = r.queryOne(update.Set("token_epoch", Expr("token_epoch + 1")).
Suffix("RETURNING token_epoch"), &res)
switch {
case errors.Is(err, model.ErrNotFound):
isNewUser = true
case err != nil:
return err
default:
epoch = res.TokenEpoch
}
} else {
count, err := r.executeSQL(update)
if err != nil {
return err
}
isNewUser = count == 0
}
isNewUser := count == 0
if isNewUser {
values["created_at"] = time.Now()
insert := Insert(r.tableName).SetMap(values)
@ -163,6 +180,12 @@ func (r *userRepository) Put(u *model.User) error {
}
}
// Only the caller's own token can be refreshed in-flight; an admin resetting another
// user must keep their own epoch.
if u.NewPassword != "" && !isNewUser && loggedUser(r.ctx).ID == u.ID {
request.SetTokenEpoch(r.ctx, epoch)
}
return nil
}

Some files were not shown because too many files have changed in this diff Show More