diff --git a/core/artwork/dominant/dominant.go b/core/artwork/dominant/dominant.go new file mode 100644 index 000000000..2bbda8ae9 --- /dev/null +++ b/core/artwork/dominant/dominant.go @@ -0,0 +1,127 @@ +// Package dominant extracts an image's dominant colour, for use as a flat placeholder while the +// real artwork loads. +package dominant + +import ( + "fmt" + "image" + "math" + "sort" +) + +const ( + // 4 bits per channel: coarse enough that near-identical pixels land together, fine enough that + // distinct colours stay apart. + bits = 4 + nBins = 1 << (3 * bits) + // Only the heaviest bins can win, and merging is O(n^2) over whatever survives. + maxBins = 64 + // Oklab distance below which two bins are the same colour to the eye. Merging matters because a + // gradient splits across adjacent bins and would otherwise lose to a smaller flat region. + mergeDist = 0.10 +) + +type bin struct { + r, g, b float64 + n float64 +} + +// Color returns the dominant colour as "#rrggbb", or "" when the image has no pixels. It reports +// presence, not salience: a mostly white sleeve returns white. +func Color(img image.Image) string { + var bins [nBins]bin + total := 0 + eachPixel(img, func(r, g, b uint8) { + i := int(r>>(8-bits))<<(2*bits) | int(g>>(8-bits))<>(8-bits)) + bins[i].r += float64(r) + bins[i].g += float64(g) + bins[i].b += float64(b) + bins[i].n++ + total++ + }) + if total == 0 { + return "" + } + + used := make([]bin, 0, 32) + for i := range bins { + if bins[i].n > 0 { + used = append(used, bins[i]) + } + } + sort.Slice(used, func(i, j int) bool { return used[i].n > used[j].n }) + if len(used) > maxBins { + used = used[:maxBins] + } + + merged := make([]bin, 0, len(used)) + for _, b := range used { + if i := nearest(merged, b); i >= 0 { + merged[i].r += b.r + merged[i].g += b.g + merged[i].b += b.b + merged[i].n += b.n + continue + } + merged = append(merged, b) + } + + best := merged[0] + for _, m := range merged[1:] { + if m.n > best.n { + best = m + } + } + return fmt.Sprintf("#%02x%02x%02x", + uint8(best.r/best.n+0.5), uint8(best.g/best.n+0.5), uint8(best.b/best.n+0.5)) +} + +func nearest(merged []bin, b bin) int { + bl, ba, bb := oklab(b.r/b.n, b.g/b.n, b.b/b.n) + for i, m := range merged { + ml, ma, mb := oklab(m.r/m.n, m.g/m.n, m.b/m.n) + if math.Sqrt((bl-ml)*(bl-ml)+(ba-ma)*(ba-ma)+(bb-mb)*(bb-mb)) < mergeDist { + return i + } + } + return -1 +} + +// eachPixel walks the image, taking the NRGBA fast path the artwork pipeline always hits: both hash +// encoders already read the shared thumbnail in that form. +func eachPixel(img image.Image, fn func(r, g, b uint8)) { + if p, ok := img.(*image.NRGBA); ok { + for y := range p.Rect.Dy() { + row := p.Pix[y*p.Stride : y*p.Stride+p.Rect.Dx()*4] + for x := 0; x < len(row); x += 4 { + fn(row[x], row[x+1], row[x+2]) + } + } + return + } + b := img.Bounds() + for y := b.Min.Y; y < b.Max.Y; y++ { + for x := b.Min.X; x < b.Max.X; x++ { + r, g, bl, _ := img.At(x, y).RGBA() + fn(uint8(r>>8), uint8(g>>8), uint8(bl>>8)) + } + } +} + +func srgbToLinear(v float64) float64 { + v /= 255 + if v <= 0.04045 { + return v / 12.92 + } + return math.Pow((v+0.055)/1.055, 2.4) +} + +func oklab(r, g, b float64) (float64, float64, float64) { + lr, lg, lb := srgbToLinear(r), srgbToLinear(g), srgbToLinear(b) + l := math.Cbrt(0.4122214708*lr + 0.5363325363*lg + 0.0514459929*lb) + m := math.Cbrt(0.2119034982*lr + 0.6806995451*lg + 0.1073969566*lb) + s := math.Cbrt(0.0883024619*lr + 0.2817188376*lg + 0.6299787005*lb) + return 0.2104542553*l + 0.7936177850*m - 0.0040720468*s, + 1.9779984951*l - 2.4285922050*m + 0.4505937099*s, + 0.0259040371*l + 0.7827717662*m - 0.8086757660*s +} diff --git a/core/artwork/dominant/dominant_suite_test.go b/core/artwork/dominant/dominant_suite_test.go new file mode 100644 index 000000000..56dbd6209 --- /dev/null +++ b/core/artwork/dominant/dominant_suite_test.go @@ -0,0 +1,17 @@ +package dominant_test + +import ( + "testing" + + "github.com/navidrome/navidrome/log" + "github.com/navidrome/navidrome/tests" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestDominant(t *testing.T) { + tests.Init(t, false) + log.SetLevel(log.LevelFatal) + RegisterFailHandler(Fail) + RunSpecs(t, "Dominant Suite") +} diff --git a/core/artwork/dominant/dominant_test.go b/core/artwork/dominant/dominant_test.go new file mode 100644 index 000000000..da7fed09c --- /dev/null +++ b/core/artwork/dominant/dominant_test.go @@ -0,0 +1,91 @@ +package dominant_test + +import ( + "image" + "image/color" + + "github.com/navidrome/navidrome/core/artwork/dominant" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// fill paints rect with c onto img. +func fill(img *image.NRGBA, r image.Rectangle, c color.NRGBA) { + for y := r.Min.Y; y < r.Max.Y; y++ { + for x := r.Min.X; x < r.Max.X; x++ { + img.SetNRGBA(x, y, c) + } + } +} + +func newImg(w, h int, c color.NRGBA) *image.NRGBA { + img := image.NewNRGBA(image.Rect(0, 0, w, h)) + fill(img, img.Bounds(), c) + return img +} + +var _ = Describe("Color", func() { + It("returns a solid image's own colour", func() { + Expect(dominant.Color(newImg(20, 20, color.NRGBA{0x33, 0x66, 0x99, 255}))).To(Equal("#336699")) + }) + + It("returns empty for an image with no pixels", func() { + Expect(dominant.Color(image.NewNRGBA(image.Rect(0, 0, 0, 0)))).To(Equal("")) + }) + + // Presence, not salience: this is a placeholder, so the large field wins even though the small + // patch is the more interesting colour. + It("picks the largest area, not the most vivid one", func() { + img := newImg(20, 20, color.NRGBA{0xfa, 0xfa, 0xfa, 255}) + fill(img, image.Rect(0, 0, 4, 4), color.NRGBA{0xff, 0x00, 0x00, 255}) + Expect(dominant.Color(img)).To(Equal("#fafafa")) + }) + + It("reports a near-black cover as near-black", func() { + img := newImg(20, 20, color.NRGBA{0x05, 0x05, 0x05, 255}) + fill(img, image.Rect(0, 0, 5, 5), color.NRGBA{0x00, 0xff, 0x00, 255}) + Expect(dominant.Color(img)).To(Equal("#050505")) + }) + + // A gradient splits across many quantisation bins. Without merging, each slice is smaller than + // the flat block and the block would win despite covering far less of the image. + It("merges a gradient's bins so it beats a smaller flat block", func() { + img := image.NewNRGBA(image.Rect(0, 0, 40, 40)) + for y := range 40 { + for x := range 40 { + // 30 columns of blue gradient == 75% of the image + if x < 30 { + img.SetNRGBA(x, y, color.NRGBA{0x10, 0x20, uint8(0xa0 + x), 255}) + } else { + img.SetNRGBA(x, y, color.NRGBA{0xff, 0xcc, 0x00, 255}) + } + } + } + got := dominant.Color(img) + Expect(got).To(HavePrefix("#1020"), "expected the blue gradient, got "+got) + }) + + It("is deterministic", func() { + img := image.NewNRGBA(image.Rect(0, 0, 30, 30)) + for y := range 30 { + for x := range 30 { + img.SetNRGBA(x, y, color.NRGBA{uint8(x * 7), uint8(y * 5), uint8(x + y), 255}) + } + } + first := dominant.Color(img) + for range 5 { + Expect(dominant.Color(img)).To(Equal(first)) + } + }) + + It("handles images that are not NRGBA", func() { + src := newImg(10, 10, color.NRGBA{0x20, 0x40, 0x60, 255}) + rgba := image.NewRGBA(src.Bounds()) + for y := range 10 { + for x := range 10 { + rgba.Set(x, y, src.At(x, y)) + } + } + Expect(dominant.Color(rgba)).To(Equal("#204060")) + }) +}) diff --git a/core/artwork/processor.go b/core/artwork/processor.go index f8c0c6d80..c3cf15b71 100644 --- a/core/artwork/processor.go +++ b/core/artwork/processor.go @@ -14,6 +14,7 @@ import ( "time" "github.com/navidrome/navidrome/core/artwork/blurhash" + "github.com/navidrome/navidrome/core/artwork/dominant" "github.com/navidrome/navidrome/core/artwork/thumbhash" "github.com/navidrome/navidrome/log" "github.com/navidrome/navidrome/model" @@ -251,12 +252,13 @@ func decodeArtwork(ctx context.Context, hash string, data []byte) (*model.Artwor } return &model.Artwork{ - Hash: hash, - Mime: mimeForFormat(format), - Width: img.Bounds().Dx(), - Height: img.Bounds().Dy(), - BlurHash: bh, - ThumbHash: th, + Hash: hash, + Mime: mimeForFormat(format), + Width: img.Bounds().Dx(), + Height: img.Bounds().Dy(), + BlurHash: bh, + ThumbHash: th, + DominantColor: dominant.Color(thumb), }, nil } diff --git a/core/artwork/processor_test.go b/core/artwork/processor_test.go index d7ef13bfc..d326dde09 100644 --- a/core/artwork/processor_test.go +++ b/core/artwork/processor_test.go @@ -98,6 +98,10 @@ var _ = Describe("processor.acquire", func() { art, err := artRepo.GetImage(ia.Hash) Expect(err).ToNot(HaveOccurred()) + // Every placeholder is derived from the one shared thumbnail, so all three land together. + Expect(art.BlurHash).ToNot(BeEmpty()) + Expect(art.ThumbHash).ToNot(BeEmpty()) + Expect(art.DominantColor).To(MatchRegexp(`^#[0-9a-f]{6}$`)) _, err = store.Open(ia.Hash, art.Mime) Expect(os.IsNotExist(err)).To(BeTrue(), "folder-backed art must not be duplicated into the store") }) diff --git a/db/migrations/20260722023032_add_artwork_tables.sql b/db/migrations/20260722023032_add_artwork_tables.sql index a4935b692..e307d13e8 100644 --- a/db/migrations/20260722023032_add_artwork_tables.sql +++ b/db/migrations/20260722023032_add_artwork_tables.sql @@ -7,6 +7,7 @@ CREATE TABLE artwork ( size_bytes INTEGER NOT NULL DEFAULT 0, blur_hash TEXT NOT NULL DEFAULT '', thumb_hash TEXT NOT NULL DEFAULT '', + dominant_color TEXT NOT NULL DEFAULT '', created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ); diff --git a/model/artwork.go b/model/artwork.go index 0513ebaf8..87b424f33 100644 --- a/model/artwork.go +++ b/model/artwork.go @@ -4,14 +4,16 @@ import "time" // Artwork is one unique image, identified by the XXH3-64 hash of its bytes. type Artwork struct { - Hash string `structs:"hash"` - Mime string `structs:"mime"` - Width int `structs:"width"` - Height int `structs:"height"` - SizeBytes int64 `structs:"size_bytes"` - BlurHash string `structs:"blur_hash"` - ThumbHash string `structs:"thumb_hash"` - CreatedAt time.Time `structs:"created_at"` + Hash string `structs:"hash"` + Mime string `structs:"mime"` + Width int `structs:"width"` + Height int `structs:"height"` + SizeBytes int64 `structs:"size_bytes"` + BlurHash string `structs:"blur_hash"` + ThumbHash string `structs:"thumb_hash"` + // DominantColor is "#rrggbb": a flat placeholder clients can paint before any decode. + DominantColor string `structs:"dominant_color"` + CreatedAt time.Time `structs:"created_at"` } const ImageTypePrimary = "primary" @@ -23,6 +25,8 @@ type ItemImage struct { // BlurHash is Jellyfin's; its mappers read this field directly, so it stays off native JSON. BlurHash string `structs:"-" json:"-"` ThumbHash string `structs:"-" json:"thumbHash,omitempty"` + // DominantColor is the only placeholder needing no decode, so it can paint on the first frame. + DominantColor string `structs:"-" json:"dominantColor,omitempty"` // A thumbhash's own aspect is quantised, so clients need these to shape the placeholder exactly. ImageWidth int `structs:"-" json:"imageWidth,omitempty"` ImageHeight int `structs:"-" json:"imageHeight,omitempty"` @@ -54,12 +58,13 @@ type ItemArtwork struct { // ItemArtworkInfo is the list-hydration projection (item_artwork joined with artwork). type ItemArtworkInfo struct { - ItemID string - Hash string - BlurHash string - ThumbHash string - Width int - Height int + ItemID string + Hash string + BlurHash string + ThumbHash string + DominantColor string + Width int + Height int } // Absent reports a known-absent artwork state (resolved, no image). @@ -68,12 +73,13 @@ func (i ItemArtworkInfo) Absent() bool { return i.Hash == "" } // Image projects the hydration entry onto the entity-facing struct. func (i ItemArtworkInfo) Image() ItemImage { return ItemImage{ - ImageHash: i.Hash, - ImageAbsent: i.Absent(), - BlurHash: i.BlurHash, - ThumbHash: i.ThumbHash, - ImageWidth: i.Width, - ImageHeight: i.Height, + ImageHash: i.Hash, + ImageAbsent: i.Absent(), + BlurHash: i.BlurHash, + ThumbHash: i.ThumbHash, + DominantColor: i.DominantColor, + ImageWidth: i.Width, + ImageHeight: i.Height, } } diff --git a/persistence/artwork_repository.go b/persistence/artwork_repository.go index 6c49e8057..22662b575 100644 --- a/persistence/artwork_repository.go +++ b/persistence/artwork_repository.go @@ -51,7 +51,8 @@ func (r *artworkRepository) PutImage(a *model.Artwork) error { // created_at=excluded.created_at: reacquiring an orphan must reset the prune grace window. ins := Insert(r.tableName).SetMap(values).Suffix(`ON CONFLICT (hash) DO UPDATE SET mime=excluded.mime, width=excluded.width, height=excluded.height, size_bytes=excluded.size_bytes, blur_hash=excluded.blur_hash, - thumb_hash=excluded.thumb_hash, created_at=excluded.created_at`) + thumb_hash=excluded.thumb_hash, dominant_color=excluded.dominant_color, + created_at=excluded.created_at`) _, err = r.executeSQL(ins) return err } @@ -152,6 +153,7 @@ func (r *artworkRepository) GetInfoForItems(kind model.Kind, ids []string) (map[ for chunk := range slices.Chunk(ids, artworkBatchSize) { sel := Select("ia.item_id", "ia.hash", "COALESCE(a.blur_hash, '') as blur_hash", "COALESCE(a.thumb_hash, '') as thumb_hash", + "COALESCE(a.dominant_color, '') as dominant_color", "COALESCE(a.width, 0) as width", "COALESCE(a.height, 0) as height"). From(itemArtworkTable + " ia"). LeftJoin("artwork a ON a.hash = ia.hash"). diff --git a/persistence/artwork_repository_test.go b/persistence/artwork_repository_test.go index c1527fcf8..683dc2d0f 100644 --- a/persistence/artwork_repository_test.go +++ b/persistence/artwork_repository_test.go @@ -41,22 +41,25 @@ var _ = Describe("ArtworkRepository", func() { }) It("round-trips the thumbhash alongside the blurhash", func() { - a := &model.Artwork{Hash: "both1", Mime: "image/jpeg", BlurHash: "LKO2?U%2Tw=w", ThumbHash: "1QcSHQRnh493V4dIh4eXh1h4kJUI"} + a := &model.Artwork{Hash: "both1", Mime: "image/jpeg", BlurHash: "LKO2?U%2Tw=w", + ThumbHash: "1QcSHQRnh493V4dIh4eXh1h4kJUI", DominantColor: "#336699"} Expect(repo.PutImage(a)).To(Succeed()) got, err := repo.GetImage("both1") Expect(err).ToNot(HaveOccurred()) Expect(got.BlurHash).To(Equal("LKO2?U%2Tw=w")) Expect(got.ThumbHash).To(Equal("1QcSHQRnh493V4dIh4eXh1h4kJUI")) + Expect(got.DominantColor).To(Equal("#336699")) }) It("overwrites the thumbhash on re-acquisition", func() { - Expect(repo.PutImage(&model.Artwork{Hash: "th2", Mime: "image/png", ThumbHash: "first"})).To(Succeed()) - Expect(repo.PutImage(&model.Artwork{Hash: "th2", Mime: "image/png", ThumbHash: "second"})).To(Succeed()) + Expect(repo.PutImage(&model.Artwork{Hash: "th2", Mime: "image/png", ThumbHash: "first", DominantColor: "#111111"})).To(Succeed()) + Expect(repo.PutImage(&model.Artwork{Hash: "th2", Mime: "image/png", ThumbHash: "second", DominantColor: "#222222"})).To(Succeed()) got, err := repo.GetImage("th2") Expect(err).ToNot(HaveOccurred()) Expect(got.ThumbHash).To(Equal("second")) + Expect(got.DominantColor).To(Equal("#222222")) }) It("is idempotent on Put (upsert by hash)", func() { @@ -195,7 +198,8 @@ var _ = Describe("ArtworkRepository", func() { }) It("hydrates a page in one batch, including blurhash, dimensions and absence", func() { - Expect(repo.PutImage(&model.Artwork{Hash: "h9", Mime: "image/jpeg", BlurHash: "BH9", Width: 1200, Height: 800})).To(Succeed()) + Expect(repo.PutImage(&model.Artwork{Hash: "h9", Mime: "image/jpeg", BlurHash: "BH9", + DominantColor: "#abcdef", Width: 1200, Height: 800})).To(Succeed()) Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "al", ItemID: "x1", ImageType: model.ImageTypePrimary, Hash: "h9", Source: "folder"})).To(Succeed()) Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "al", ItemID: "x2", ImageType: model.ImageTypePrimary, Hash: "", Source: ""})).To(Succeed()) @@ -204,6 +208,7 @@ var _ = Describe("ArtworkRepository", func() { Expect(info).To(HaveLen(2)) Expect(info["x1"].Hash).To(Equal("h9")) Expect(info["x1"].BlurHash).To(Equal("BH9")) + Expect(info["x1"].DominantColor).To(Equal("#abcdef")) Expect(info["x1"].Width).To(Equal(1200)) Expect(info["x1"].Height).To(Equal(800)) Expect(info["x1"].Absent()).To(BeFalse()) diff --git a/tests/mock_artwork_repo.go b/tests/mock_artwork_repo.go index bec589314..2ace0daba 100644 --- a/tests/mock_artwork_repo.go +++ b/tests/mock_artwork_repo.go @@ -165,6 +165,7 @@ func (m *MockArtworkRepo) GetInfoForItems(kind model.Kind, ids []string) (map[st info := model.ItemArtworkInfo{ItemID: id, Hash: ia.Hash} if a, ok := m.Data[ia.Hash]; ok { info.BlurHash, info.ThumbHash = a.BlurHash, a.ThumbHash + info.DominantColor = a.DominantColor info.Width, info.Height = a.Width, a.Height } res[id] = info