diff --git a/model/artwork.go b/model/artwork.go index 8fe2bc7fe..6507c24d7 100644 --- a/model/artwork.go +++ b/model/artwork.go @@ -19,11 +19,13 @@ const ImageTypePrimary = "primary" // ItemArtwork is an entity's resolved artwork state. Hash=="" means known absent. type ItemArtwork struct { - ItemKind string `structs:"item_kind"` - ItemID string `structs:"item_id"` - ImageType string `structs:"image_type"` - Hash string `structs:"hash"` - Source string `structs:"source"` + ItemKind string `structs:"item_kind"` + ItemID string `structs:"item_id"` + ImageType string `structs:"image_type"` + Hash string `structs:"hash"` + Source string `structs:"source"` + // attempted_at/updated_at are nullable in the schema but always set by PutItemArtwork; + // raw inserts must set them too, since these non-pointer time.Time fields fail to scan NULL. AttemptedAt time.Time `structs:"attempted_at"` UpdatedAt time.Time `structs:"updated_at"` } @@ -59,13 +61,16 @@ type ArtworkRepository interface { GetImage(hash string) (*Artwork, error) PutImage(a *Artwork) error GetImages(hashes []string) (map[string]Artwork, error) + // GetOrphanHashes returns hashes referenced by no item_artwork row and older than cutoff. GetOrphanHashes(createdBefore time.Time) ([]string, error) DeleteImages(hashes ...string) error // Per-item state (item_artwork table) GetItemArtwork(kind, id, imageType string) (*ItemArtwork, error) PutItemArtwork(ia *ItemArtwork) error DeleteForItem(kind, id string) error + // GetInfoForItems hydrates a page: one batched query, item_artwork joined to artwork. GetInfoForItems(kind string, ids []string) (map[string]ItemArtworkInfo, error) + // EnqueueStaleAbsent inserts queue rows (priority Recheck) for absent states older than cutoff. EnqueueStaleAbsent(kind string, attemptedBefore time.Time) (int64, error) } diff --git a/persistence/artwork_repository.go b/persistence/artwork_repository.go index 839a390ed..4556bc588 100644 --- a/persistence/artwork_repository.go +++ b/persistence/artwork_repository.go @@ -2,6 +2,7 @@ package persistence import ( "context" + "slices" "time" . "github.com/Masterminds/squirrel" @@ -47,16 +48,15 @@ func (r *artworkRepository) PutImage(a *model.Artwork) error { func (r *artworkRepository) GetImages(hashes []string) (map[string]model.Artwork, error) { res := map[string]model.Artwork{} - if len(hashes) == 0 { - return res, nil - } - sel := Select("*").From(r.tableName).Where(Eq{"hash": hashes}) - var all []model.Artwork - if err := r.queryAll(sel, &all); err != nil { - return nil, err - } - for _, a := range all { - res[a.Hash] = a + for chunk := range slices.Chunk(hashes, 200) { + sel := Select("*").From(r.tableName).Where(Eq{"hash": chunk}) + var all []model.Artwork + if err := r.queryAll(sel, &all); err != nil { + return nil, err + } + for _, a := range all { + res[a.Hash] = a + } } return res, nil } @@ -73,10 +73,12 @@ func (r *artworkRepository) GetOrphanHashes(createdBefore time.Time) ([]string, } func (r *artworkRepository) DeleteImages(hashes ...string) error { - if len(hashes) == 0 { - return nil + for chunk := range slices.Chunk(hashes, 200) { + if err := r.delete(Eq{"hash": chunk}); err != nil { + return err + } } - return r.delete(Eq{"hash": hashes}) + return nil } func (r *artworkRepository) GetItemArtwork(kind, id, imageType string) (*model.ItemArtwork, error) { @@ -113,28 +115,27 @@ func (r *artworkRepository) DeleteForItem(kind, id string) error { func (r *artworkRepository) GetInfoForItems(kind string, ids []string) (map[string]model.ItemArtworkInfo, error) { res := map[string]model.ItemArtworkInfo{} - if len(ids) == 0 { - return res, nil - } - sel := Select("ia.item_id", "ia.hash", "COALESCE(a.blur_hash, '') as blur_hash"). - From("item_artwork ia"). - LeftJoin("artwork a ON a.hash = ia.hash"). - Where(And{ - Eq{"ia.item_kind": kind}, - Eq{"ia.image_type": model.ImageTypePrimary}, - Eq{"ia.item_id": ids}, - }) - var rows []struct { - ItemID string - Hash string - BlurHash string - } - if err := r.queryAll(sel, &rows); err != nil { - return nil, err - } - for _, row := range rows { - res[row.ItemID] = model.ItemArtworkInfo{ - ItemID: row.ItemID, Hash: row.Hash, BlurHash: row.BlurHash, Absent: row.Hash == "", + for chunk := range slices.Chunk(ids, 200) { + sel := Select("ia.item_id", "ia.hash", "COALESCE(a.blur_hash, '') as blur_hash"). + From("item_artwork ia"). + LeftJoin("artwork a ON a.hash = ia.hash"). + Where(And{ + Eq{"ia.item_kind": kind}, + Eq{"ia.image_type": model.ImageTypePrimary}, + Eq{"ia.item_id": chunk}, + }) + var rows []struct { + ItemID string + Hash string + BlurHash string + } + if err := r.queryAll(sel, &rows); err != nil { + return nil, err + } + for _, row := range rows { + res[row.ItemID] = model.ItemArtworkInfo{ + ItemID: row.ItemID, Hash: row.Hash, BlurHash: row.BlurHash, Absent: row.Hash == "", + } } } return res, nil diff --git a/persistence/artwork_repository_test.go b/persistence/artwork_repository_test.go index b4da4ebbe..a77fd7038 100644 --- a/persistence/artwork_repository_test.go +++ b/persistence/artwork_repository_test.go @@ -2,6 +2,7 @@ package persistence import ( "context" + "fmt" "time" "github.com/navidrome/navidrome/model" @@ -81,6 +82,20 @@ var _ = Describe("ArtworkRepository", func() { _, err := repo.GetImage("d1") Expect(err).To(MatchError(model.ErrNotFound)) }) + + It("fetches a batch larger than the SQL variable limit", func() { + hashes := make([]string, 0, 250) + for i := 0; i < 250; i++ { + h := fmt.Sprintf("big%03d", i) + Expect(repo.PutImage(&model.Artwork{Hash: h, Mime: "image/jpeg"})).To(Succeed()) + hashes = append(hashes, h) + } + hashes = append(hashes, "absent1", "absent2") + + got, err := repo.GetImages(hashes) + Expect(err).ToNot(HaveOccurred()) + Expect(got).To(HaveLen(250)) + }) }) var _ = Describe("ArtworkRepository item state", func() {