perf(genre): index genre filtering via join tables across all APIs (#5940)

Filtering by genre scanned every media_file/album row and JSON-parsed its
`tags` column (a per-row json_tree(tags) EXISTS) with no usable index, so
Finamp's genre screen took 1.9-6.5s per tap against a ~97k-track library.
Album and album-artist genre queries had the same unindexed shape.

Add normalized media_file_tags and album_tags join tables (genre only for
now, via an indexedTagNames allowlist), populated by a new updateTags in
Put (mirroring updateParticipants) and backfilled in the migration. Genre
filtering across the Jellyfin, Subsonic and native APIs now runs as an
index-backed semi-join through shared TagIDSemiJoin/TagNameSemiJoin helpers
instead of a full scan. On a copy of the production DB the per-request cost
for a typical genre drops from ~330ms to sub-millisecond, adding ~10MB.

db/migrations/20260811023338_add_item_genre_tag_indexes.sql

@@ -0,0 +1,34 @@
+-- +goose Up
+create table if not exists media_file_tags(
+    media_file_id varchar not null
+        references media_file (id) on delete cascade,
+    tag_id varchar not null
+        references tag (id) on delete cascade,
+    constraint media_file_tags unique (media_file_id, tag_id)
+);
+create index if not exists media_file_tags_tag_id on media_file_tags (tag_id);
+
+create table if not exists album_tags(
+    album_id varchar not null
+        references album (id) on delete cascade,
+    tag_id varchar not null
+        references tag (id) on delete cascade,
+    constraint album_tags unique (album_id, tag_id)
+);
+create index if not exists album_tags_tag_id on album_tags (tag_id);
+
+-- Backfill genre rows from the per-row `tags` JSON. json_tree over the "$.genre" subtree yields one
+-- row per node; the "id" key nodes carry the tag ids. Single scan per table, no correlated subquery.
+insert or ignore into media_file_tags (media_file_id, tag_id)
+select mf.id, jt.value
+from media_file mf, json_tree(mf.tags, '$.genre') jt
+where jt.key = 'id' and jt.atom is not null;
+
+insert or ignore into album_tags (album_id, tag_id)
+select al.id, jt.value
+from album al, json_tree(al.tags, '$.genre') jt
+where jt.key = 'id' and jt.atom is not null;
+
+-- +goose Down
+drop table if exists media_file_tags;
+drop table if exists album_tags;

persistence/album_repository.go

@@ -133,7 +133,7 @@ var albumFilters = sync.OnceValue(func() map[string]filterFunc {
 		"starred":         annotationBoolFilter("starred"),
 		"has_rating":      annotationBoolFilter("rating"),
 		"missing":         booleanFilter,
-		"genre_id":        tagIDFilter,
+		"genre_id":        genreFilter(AlbumGenres),
 		"role_total_id":   allRolesFilter,
 		"library_id":      libraryIdFilter,
 	}
@@ -212,7 +212,10 @@ func (r *albumRepository) Put(al *model.Album) error {
 		return err
 	}
 	al.ID = id
-	return r.updateParticipants(al.ID, al.Participants)
+	if err := r.updateParticipants(al.ID, al.Participants); err != nil {
+		return err
+	}
+	return r.updateTags(al.ID, al.Tags)
 }
 
 // TODO Move external metadata to a separated table

persistence/item_tags_test.go

@@ -0,0 +1,108 @@
+package persistence
+
+import (
+	"github.com/deluan/rest"
+	"github.com/navidrome/navidrome/model"
+	"github.com/navidrome/navidrome/model/request"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"github.com/pocketbase/dbx"
+)
+
+var _ = Describe("item genre tag indexes", func() {
+	var conn *dbx.DB
+	var mr model.MediaFileRepository
+	var ar model.AlbumRepository
+	var rock, jazz model.Tag
+
+	BeforeEach(func() {
+		ctx := request.WithUser(GinkgoT().Context(), model.User{ID: "userid"})
+		conn = GetDBXBuilder()
+		mr = NewMediaFileRepository(ctx, conn)
+		ar = NewAlbumRepository(ctx, conn)
+		// Test-only genre values, so they can't collide with the golden fixtures.
+		rock = model.NewTag(model.TagGenre, "GenreIdxRock")
+		jazz = model.NewTag(model.TagGenre, "GenreIdxJazz")
+		// The join tables FK to tag(id); the scanner adds tags before saving items.
+		Expect(NewTagRepository(ctx, conn).Add(1, rock, jazz)).To(Succeed())
+		// The suite shares one golden DB with no per-test restore, so undo the rows we add
+		// (media_file/album deletes cascade to the *_tags join rows; tag deletes cascade too).
+		DeferCleanup(func() {
+			_, _ = conn.NewQuery("DELETE FROM media_file WHERE id LIKE 'mf-%'").Execute()
+			_, _ = conn.NewQuery("DELETE FROM album WHERE id LIKE 'al-%'").Execute()
+			_, _ = conn.NewQuery("DELETE FROM tag WHERE id={:r} OR id={:j}").
+				Bind(dbx.Params{"r": rock.ID, "j": jazz.ID}).Execute()
+		})
+	})
+
+	tagIDsFor := func(table, col, id string) []string {
+		var rows []struct {
+			TagID string `db:"tag_id"`
+		}
+		err := conn.NewQuery("SELECT tag_id FROM " + table + " WHERE " + col + "={:id}").
+			Bind(dbx.Params{"id": id}).All(&rows)
+		Expect(err).ToNot(HaveOccurred())
+		ids := make([]string, len(rows))
+		for i, r := range rows {
+			ids[i] = r.TagID
+		}
+		return ids
+	}
+
+	Describe("media files", func() {
+		It("writes a media_file_tags row for each genre when the track is saved", func() {
+			mf := model.MediaFile{ID: "mf-g1", LibraryID: 1, Path: "/m/g1.mp3", Title: "G1",
+				Tags: model.Tags{model.TagGenre: []string{rock.TagValue, jazz.TagValue}}}
+			Expect(mr.Put(&mf)).To(Succeed())
+			Expect(tagIDsFor("media_file_tags", "media_file_id", "mf-g1")).To(ConsistOf(rock.ID, jazz.ID))
+		})
+
+		It("replaces the rows when the genres change", func() {
+			mf := model.MediaFile{ID: "mf-g2", LibraryID: 1, Path: "/m/g2.mp3", Title: "G2",
+				Tags: model.Tags{model.TagGenre: []string{rock.TagValue}}}
+			Expect(mr.Put(&mf)).To(Succeed())
+			mf.Tags = model.Tags{model.TagGenre: []string{jazz.TagValue}}
+			Expect(mr.Put(&mf)).To(Succeed())
+			Expect(tagIDsFor("media_file_tags", "media_file_id", "mf-g2")).To(ConsistOf(jazz.ID))
+		})
+
+		It("clears the rows when all genres are removed", func() {
+			mf := model.MediaFile{ID: "mf-g3", LibraryID: 1, Path: "/m/g3.mp3", Title: "G3",
+				Tags: model.Tags{model.TagGenre: []string{rock.TagValue}}}
+			Expect(mr.Put(&mf)).To(Succeed())
+			mf.Tags = model.Tags{}
+			Expect(mr.Put(&mf)).To(Succeed())
+			Expect(tagIDsFor("media_file_tags", "media_file_id", "mf-g3")).To(BeEmpty())
+		})
+	})
+
+	Describe("albums", func() {
+		It("writes an album_tags row for each genre when the album is saved", func() {
+			al := model.Album{ID: "al-g1", LibraryID: 1, Name: "AG1",
+				Tags: model.Tags{model.TagGenre: []string{rock.TagValue, jazz.TagValue}}}
+			Expect(ar.Put(&al)).To(Succeed())
+			Expect(tagIDsFor("album_tags", "album_id", "al-g1")).To(ConsistOf(rock.ID, jazz.ID))
+		})
+	})
+
+	// The native (REST) API filters by genre_id; it must resolve through the join table too.
+	Describe("native genre_id filter", func() {
+		It("filters media files by genre_id", func() {
+			mf := model.MediaFile{ID: "mf-nat1", LibraryID: 1, Path: "/m/nat1.mp3", Title: "Nat1",
+				Tags: model.Tags{model.TagGenre: []string{rock.TagValue}}}
+			Expect(mr.Put(&mf)).To(Succeed())
+			res, err := mr.(model.ResourceRepository).ReadAll(rest.QueryOptions{Filters: map[string]any{"genre_id": rock.ID}})
+			Expect(err).ToNot(HaveOccurred())
+			Expect(res.(model.MediaFiles)).To(ContainElement(HaveField("ID", "mf-nat1")))
+		})
+
+		It("filters albums by genre_id", func() {
+			al := model.Album{ID: "al-nat1", LibraryID: 1, Name: "ANat1",
+				Tags: model.Tags{model.TagGenre: []string{rock.TagValue}}}
+			Expect(ar.Put(&al)).To(Succeed())
+			res, err := ar.(model.ResourceRepository).ReadAll(rest.QueryOptions{Filters: map[string]any{"genre_id": rock.ID}})
+			Expect(err).ToNot(HaveOccurred())
+			Expect(res.(model.Albums)).To(ContainElement(HaveField("ID", "al-nat1")))
+		})
+	})
+})

persistence/mediafile_repository.go

@@ -111,7 +111,7 @@ var mediaFileFilter = sync.OnceValue(func() map[string]filterFunc {
 		"title":      fullTextFilter("media_file", "mbz_recording_id", "mbz_release_track_id"),
 		"starred":    annotationBoolFilter("starred"),
 		"has_rating": annotationBoolFilter("rating"),
-		"genre_id":   tagIDFilter,
+		"genre_id":   genreFilter(SongGenres),
 		"missing":    booleanFilter,
 		"artists_id": mediaFileArtistFilter,
 		"library_id": libraryIdFilter,
@@ -181,7 +181,10 @@ func (r *mediaFileRepository) Put(m *model.MediaFile) error {
 		return err
 	}
 	m.ID = id
-	return r.updateParticipants(m.ID, m.Participants)
+	if err := r.updateParticipants(m.ID, m.Participants); err != nil {
+		return err
+	}
+	return r.updateTags(m.ID, m.Tags)
 }
 
 func (r *mediaFileRepository) UpdateProbeData(id string, data string) error {

persistence/sql_tags.go

@@ -48,6 +48,77 @@ func marshalTags(tags model.Tags) string {
 	return string(res)
 }
 
+// indexedTagNames are the tag types materialized into the <table>_tags join tables, so filtering by
+// them is an index-backed semi-join instead of a per-row json_tree(tags) scan. Genre only for now.
+var indexedTagNames = []model.TagName{model.TagGenre}
+
+// updateTags rewrites this item's <table>_tags rows from its in-memory tags, mirroring
+// updateParticipants (delete-then-insert in the same Put; JOIN to tag skips not-yet-saved ids).
+func (r sqlRepository) updateTags(itemID string, tags model.Tags) error {
+	del := Delete(r.tableName + "_tags").Where(Eq{r.tableName + "_id": itemID})
+	if _, err := r.executeSQL(del); err != nil {
+		return err
+	}
+	var tagIDs []string
+	for _, name := range indexedTagNames {
+		for _, value := range tags.Values(name) {
+			tagIDs = append(tagIDs, model.NewTag(name, value).ID)
+		}
+	}
+	if len(tagIDs) == 0 {
+		return nil
+	}
+	idsJSON, err := json.Marshal(tagIDs)
+	if err != nil {
+		return fmt.Errorf("marshaling tag ids: %w", err)
+	}
+	query := fmt.Sprintf(`
+		INSERT INTO %[1]s_tags (%[1]s_id, tag_id)
+		SELECT ?, value FROM json_each(?)
+		JOIN tag ON tag.id = value
+		ON CONFLICT (%[1]s_id, tag_id) DO NOTHING`, r.tableName)
+	_, err = r.executeSQL(Expr(query, itemID, string(idsJSON)))
+	return err
+}
+
+// genreFilterDef builds indexed genre filters for one item type. Callers use the exported SongGenres / AlbumGenres instances.
+type genreFilterDef struct{ idCol, table, joinCol string }
+
+var (
+	SongGenres  = genreFilterDef{"media_file.id", "media_file_tags", "media_file_id"}
+	AlbumGenres = genreFilterDef{"album.id", "album_tags", "album_id"}
+)
+
+// ByID matches items tagged with any of the given genre tag ids (scalar or slice).
+func (g genreFilterDef) ByID(tagIDs any) Sqlizer {
+	sub, args, _ := Select(g.joinCol).From(g.table).Where(Eq{"tag_id": tagIDs}).ToSql()
+	return Expr(g.idCol+" IN ("+sub+")", args...)
+}
+
+// ByName matches by genre name (Subsonic passes a name, not an id), resolved through the tag
+// dictionary, which is uniquely indexed on (tag_name, tag_value).
+func (g genreFilterDef) ByName(genre string) Sqlizer {
+	sub, args, _ := Select("jt." + g.joinCol).From(g.table + " jt").
+		Join("tag on tag.id = jt.tag_id").
+		Where(And{Eq{"tag.tag_name": "genre"}, Like{"tag.tag_value": genre}}).ToSql()
+	return Expr(g.idCol+" IN ("+sub+")", args...)
+}
+
+// AlbumArtistsByGenreID matches album artists of albums tagged with any of the genre ids. It's a
+// two-table join (album_artists ⨝ album_tags), so it doesn't fit the single-table genreFilterDef.
+func AlbumArtistsByGenreID(tagIDs []string) Sqlizer {
+	sub, args, _ := Select("aa.artist_id").From("album_artists aa").
+		Join("album_tags at on at.album_id = aa.album_id").
+		Where(And{Eq{"aa.role": "albumartist"}, Eq{"at.tag_id": tagIDs}}).ToSql()
+	return Expr("artist.id IN ("+sub+")", args...)
+}
+
+func genreFilter(filter genreFilterDef) func(_ string, v any) Sqlizer {
+	return func(_ string, v any) Sqlizer {
+		return filter.ByID(v)
+	}
+}
+
 // tagIDFilter matches rows whose tags JSON contains the tag id(s); a "<name>_id" key maps to "$.<name>".
 func tagIDFilter(name string, idValue any) Sqlizer {
 	name = strings.TrimSuffix(name, "_id")

server/filter/filters.go

@@ -112,7 +112,7 @@ func SongsByGenreAndYearRange(genre string, fromYear, toYear int) Options {
 	options := Options{}
 	ff := And{}
 	if genre != "" {
-		ff = append(ff, filterByGenre(genre))
+		ff = append(ff, persistence.SongGenres.ByName(genre))
 	}
 	if fromYear != 0 {
 		ff = append(ff, GtOrEq{"year": fromYear})
@@ -171,16 +171,17 @@ func ArtistsByRole(opts Options, role model.Role) Options {
 	return opts
 }
 
-func ByGenre(genre string) Options {
-	return addDefaultFilters(Options{
-		Sort:    "name",
-		Filters: filterByGenre(genre),
-	})
+// SongsByGenreID / AlbumsByGenreID (by tag id) and AlbumsByGenre / SongsByGenre (by name, wrapped
+// as Options for Subsonic) delegate to the persistence genre filters, which own the join schema.
+func SongsByGenreID(genreIds []string) Sqlizer  { return persistence.SongGenres.ByID(genreIds) }
+func AlbumsByGenreID(genreIds []string) Sqlizer { return persistence.AlbumGenres.ByID(genreIds) }
+
+func AlbumsByGenre(genre string) Options {
+	return addDefaultFilters(Options{Sort: "name", Filters: persistence.AlbumGenres.ByName(genre)})
 }
 
-// ByGenreID matches items (albums or songs) tagged with any of the given genre tag ids.
-func ByGenreID(genreIds []string) Sqlizer {
-	return genreTagFilter(Eq{"value": genreIds})
+func SongsByGenre(genre string) Options {
+	return addDefaultFilters(Options{Sort: "name", Filters: persistence.SongGenres.ByName(genre)})
 }
 
 // ByAlbumID matches media files belonging to any of the given albums.
@@ -198,14 +199,8 @@ func SongsByYears(years []int) Sqlizer {
 	return Eq{"year": years}
 }
 
-// ArtistsByGenreID matches artists credited as album artist on an album with any of the given
-// genre tag ids. Non-correlated semi-join: the correlated EXISTS form rescans albums per artist row.
 func ArtistsByGenreID(genreIds []string) Sqlizer {
-	return Expr(
-		`artist.id IN (SELECT jt.value FROM album, json_tree(album.participants, '$.albumartist') jt
-			WHERE jt.atom IS NOT NULL AND ?)`,
-		genreTagFilter(Eq{"value": genreIds}),
-	)
+	return persistence.AlbumArtistsByGenreID(genreIds)
 }
 
 // tagIDFilter builds an EXISTS over the given tag role's entries in the tags JSON, matching each
@@ -214,17 +209,11 @@ func tagIDFilter(tagName string, cond Sqlizer) Sqlizer {
 	return persistence.Exists(`json_tree(tags, "$.`+tagName+`")`, And{NotEq{"atom": nil}, cond})
 }
 
-func genreTagFilter(cond Sqlizer) Sqlizer { return tagIDFilter("genre", cond) }
-
 // ByStudioID matches items (albums or songs) whose record-label tag id is in ids.
 func ByStudioID(ids []string) Sqlizer {
 	return tagIDFilter("recordlabel", Eq{"value": ids})
 }
 
-func filterByGenre(genre string) Sqlizer {
-	return genreTagFilter(Like{"value": genre})
-}
-
 func ByRating() Options {
 	return addDefaultFilters(Options{Sort: "rating", Order: "desc", Filters: Gt{"rating": 0}})
 }

server/jellyfin/items.go

@@ -577,7 +577,7 @@ func (api *Router) listAlbums(ctx context.Context, opts model.QueryOptions, q it
 		filters = append(filters, notMissing)
 	}
 	if len(q.genreIds) > 0 {
-		filters = append(filters, filter.ByGenreID(q.genreIds))
+		filters = append(filters, filter.AlbumsByGenreID(q.genreIds))
 	}
 	if len(q.years) > 0 {
 		filters = append(filters, filter.AlbumsByYears(q.years))
@@ -624,7 +624,7 @@ func (api *Router) listSongs(ctx context.Context, opts model.QueryOptions, q ite
 		filters = append(filters, filter.ByAlbumID(q.albumIds))
 	}
 	if len(q.genreIds) > 0 {
-		filters = append(filters, filter.ByGenreID(q.genreIds))
+		filters = append(filters, filter.SongsByGenreID(q.genreIds))
 	}
 	if len(q.years) > 0 {
 		filters = append(filters, filter.SongsByYears(q.years))

server/subsonic/album_lists.go

@@ -46,7 +46,7 @@ func (api *Router) getAlbumList(r *http.Request) (model.Albums, int64, error) {
 		if err != nil {
 			return nil, 0, err
 		}
-		opts = filter.ByGenre(genre)
+		opts = filter.AlbumsByGenre(genre)
 	case "byYear":
 		fromYear, err := p.Int("fromYear")
 		if err != nil {
@@ -267,7 +267,7 @@ func (api *Router) GetSongsByGenre(r *http.Request) (*responses.Subsonic, error)
 	if err != nil {
 		return nil, err
 	}
-	opts := filter.ByGenre(genre)
+	opts := filter.SongsByGenre(genre)
 	opts = filter.ApplyLibraryFilter(opts, musicFolderIds)
 
 	ctx := r.Context()