navidrome/model/artwork.go
Deluan 6823bfd436 perf(artwork): drain local and external artwork in separate pools
A first backfill enqueues artists before albums at a single priority, so
the drain took them in that order. Artists resolve through a
rate-limited agent, and gate() waits for its permit while holding a
worker slot, so the whole pool sat asleep in the limiter with every
album queued behind it.

Measured on a 96k-track library (29,115 artists to 6,949 albums, 4:1):
zero albums resolved in seven minutes, and roughly 3.3 hours before the
first album cover would have appeared. Splitting the drain gives each
class its own slots: albums now finish in under eight minutes while
artists trickle at the same 2/s they were always limited to.

The two budgets are carved out of MaxOpenConns so a second pool cannot
take connections the scanner and the UI need. Dequeue filters by kind,
and the drain index leads with item_kind so each pool seeks to its own
work instead of scanning past the other's backlog.
2026-07-25 13:32:07 -04:00

123 lines
5.6 KiB
Go

package model
import "time"
// Artwork is one unique image, identified by the XXH3-64 hash of its bytes.
type Artwork struct {
Hash string `structs:"hash"`
Mime string `structs:"mime"`
Width int `structs:"width"`
Height int `structs:"height"`
SizeBytes int64 `structs:"size_bytes"`
BlurHash string `structs:"blur_hash"`
CreatedAt time.Time `structs:"created_at"`
}
const ImageTypePrimary = "primary"
// ItemImage is per-entity artwork state hydrated at query time; never persisted
// (structs:"-" keeps it out of upserts).
type ItemImage struct {
ImageHash string `structs:"-" json:"imageHash,omitempty"`
ImageAbsent bool `structs:"-" json:"imageAbsent,omitempty"`
BlurHash string `structs:"-" json:"blurHash,omitempty"`
}
// ItemArtwork is an entity's resolved artwork state. Hash=="" means known absent.
type ItemArtwork struct {
ItemKind string `structs:"item_kind"`
ItemID string `structs:"item_id"`
ImageType string `structs:"image_type"`
Hash string `structs:"hash"`
Source string `structs:"source"`
// SourcePath is the backing file (folder/upload: the image; embedded: the audio file); "" otherwise.
SourcePath string `structs:"source_path"`
// RefMtime is SourcePath's mtime (unix-nanoseconds) at resolution; 0 when there is no SourcePath.
RefMtime int64 `structs:"ref_mtime"`
// attempted_at/updated_at are nullable in the schema but always set by PutItemArtwork;
// raw inserts must set them too, since these non-pointer time.Time fields fail to scan NULL.
AttemptedAt time.Time `structs:"attempted_at"`
UpdatedAt time.Time `structs:"updated_at"`
}
// ItemArtworkInfo is the list-hydration projection (item_artwork joined with artwork).
type ItemArtworkInfo struct {
ItemID string
Hash string
BlurHash string
}
// Absent reports a known-absent artwork state (resolved, no image).
func (i ItemArtworkInfo) Absent() bool { return i.Hash == "" }
type ArtworkQueueItem struct {
ItemKind string `structs:"item_kind"`
ItemID string `structs:"item_id"`
ImageType string `structs:"image_type"`
Priority int `structs:"priority"`
Attempts int `structs:"attempts"`
RetryAt time.Time `structs:"retry_at"`
EnqueuedAt time.Time `structs:"enqueued_at"`
}
// Queue priorities: higher drains first.
const (
ArtworkPriorityRecheck = 0
ArtworkPriorityBackfill = 10
ArtworkPriorityScan = 50
ArtworkPriorityBump = 100
)
type ArtworkRepository interface {
// Image identity (artwork table)
GetImage(hash string) (*Artwork, error)
PutImage(a *Artwork) error
GetImages(hashes []string) (map[string]Artwork, error)
// GetOrphanHashes returns hashes referenced by no item_artwork row and older than cutoff.
GetOrphanHashes(createdBefore time.Time) ([]string, error)
// DeleteOrphans deletes the given hashes only if still unreferenced and older than cutoff (atomic re-check).
DeleteOrphans(createdBefore time.Time, hashes []string) error
// Per-item state (item_artwork table)
GetItemArtwork(kind Kind, id, imageType string) (*ItemArtwork, error)
PutItemArtwork(ia *ItemArtwork) error
DeleteForItem(kind Kind, id string) error
// DeleteForItems removes state rows for the given ids of one kind, in chunks.
DeleteForItems(kind Kind, ids []string) error
// GetInfoForItems hydrates a page: one batched query, item_artwork joined to artwork.
GetInfoForItems(kind Kind, ids []string) (map[string]ItemArtworkInfo, error)
// GetAllMimes returns hash -> current mime for every stored artwork, for sweep retention checks.
GetAllMimes() (map[string]string, error)
// PurgeDanglingItemArtwork removes state rows whose entity no longer exists.
PurgeDanglingItemArtwork() (int64, error)
}
type ArtworkQueueRepository interface {
// Enqueue upserts; an existing row keeps the higher of the two priorities and has its
// retry_at reset (a detected change wants immediate re-resolution).
Enqueue(items ...ArtworkQueueItem) error
// EnqueueBump upserts like Enqueue but preserves an existing row's retry_at, so a
// request-triggered read-through never resets a failed resolution's backoff.
EnqueueBump(items ...ArtworkQueueItem) error
// DequeueBatch returns up to n items with retry_at <= now, priority desc, enqueued_at asc.
// Restricted to the given item kinds when any are passed, so a drain pool sees only its own
// work and cannot be held up behind another kind's backlog.
DequeueBatch(n int, kinds ...string) ([]ArtworkQueueItem, error)
// MarkFailed increments attempts and pushes retry_at into the future.
MarkFailed(kind, id, imageType string, retryAt time.Time) error
// MarkFailedIfUnchanged applies the failure backoff only while retry_at still matches
// seenRetryAt; a concurrent re-enqueue (which resets retry_at) keeps its fresh eligibility.
MarkFailedIfUnchanged(kind, id, imageType string, seenRetryAt, retryAt time.Time) error
Delete(kind, id, imageType string) error
// DeleteIfUnchanged deletes the row only if its retry_at still matches retryAt, so a
// concurrent re-enqueue (which resets retry_at) survives instead of being erased.
DeleteIfUnchanged(kind, id, imageType string, retryAt time.Time) error
Count() (int64, error)
// EnqueueStaleAbsent inserts queue rows (priority Recheck) for absent states older than cutoff.
EnqueueStaleAbsent(kind Kind, attemptedBefore time.Time) (int64, error)
// EnqueueMissing inserts queue rows (priority Recheck) for entities of the kind that have no
// item_artwork row at all, so a never-processed entity is eventually resolved even without a scan.
EnqueueMissing(kind Kind) (int64, error)
// PurgeDangling removes queue rows whose entity no longer exists.
PurgeDangling() (int64, error)
}