diff --git a/.gitignore b/.gitignore index 3567a7d90..6459ded9b 100644 --- a/.gitignore +++ b/.gitignore @@ -41,3 +41,6 @@ openspec/ go.work* .worktrees/ .playwright-mcp/ + +# Temp benchmark files +zz_*_test.go \ No newline at end of file diff --git a/db/migrations/20260810143445_reorder_album_artists_unique_constraint.sql b/db/migrations/20260810143445_reorder_album_artists_unique_constraint.sql new file mode 100644 index 000000000..fd1598e93 --- /dev/null +++ b/db/migrations/20260810143445_reorder_album_artists_unique_constraint.sql @@ -0,0 +1,59 @@ +-- +goose Up +-- +goose StatementBegin +-- Lead the unique constraint with artist_id (mirroring media_file_artists) so artist-driven +-- filters and the artist-delete cascade get an index seek; album_id lookups keep album_artists_album_id. +CREATE TABLE album_artists_tmp +( + album_id varchar not null + REFERENCES album (id) + ON DELETE CASCADE, + artist_id varchar not null + REFERENCES artist (id) + ON DELETE CASCADE, + role varchar default '' not null, + sub_role varchar default '' not null, + CONSTRAINT album_artists + UNIQUE (artist_id, album_id, role, sub_role) +); + +INSERT INTO album_artists_tmp(album_id, artist_id, role, sub_role) +SELECT album_id, artist_id, role, sub_role +FROM album_artists; + +DROP TABLE album_artists; +ALTER TABLE album_artists_tmp RENAME TO album_artists; + +CREATE INDEX album_artists_album_id + ON album_artists (album_id); +CREATE INDEX album_artists_role + ON album_artists (role); +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +CREATE TABLE album_artists_tmp +( + album_id varchar not null + REFERENCES album (id) + ON DELETE CASCADE, + artist_id varchar not null + REFERENCES artist (id) + ON DELETE CASCADE, + role varchar default '' not null, + sub_role varchar default '' not null, + CONSTRAINT album_artists + UNIQUE (album_id, artist_id, role, sub_role) +); + +INSERT INTO album_artists_tmp(album_id, artist_id, role, sub_role) +SELECT album_id, artist_id, role, sub_role +FROM album_artists; + +DROP TABLE album_artists; +ALTER TABLE album_artists_tmp RENAME TO album_artists; + +CREATE INDEX album_artists_album_id + ON album_artists (album_id); +CREATE INDEX album_artists_role + ON album_artists (role); +-- +goose StatementEnd diff --git a/persistence/album_repository.go b/persistence/album_repository.go index 1ebb09751..91e90127c 100644 --- a/persistence/album_repository.go +++ b/persistence/album_repository.go @@ -172,24 +172,22 @@ func yearFilter(_ string, value any) Sqlizer { } func artistFilter(_ string, value any) Sqlizer { - return Or{ - Exists("json_tree(participants, '$.albumartist')", Eq{"value": value}), - Exists("json_tree(participants, '$.artist')", Eq{"value": value}), - } + return ParticipantIDFilter("album", value, model.RoleAlbumArtist, model.RoleArtist) } func artistRoleFilter(name string, value any) Sqlizer { roleName := strings.TrimSuffix(strings.TrimPrefix(name, "role_"), "_id") // Check if the role name is valid. If not, return an invalid filter - if _, ok := model.AllRoles[roleName]; !ok { + role, ok := model.AllRoles[roleName] + if !ok { return Gt{"": nil} } - return Exists(fmt.Sprintf("json_tree(participants, '$.%s')", roleName), Eq{"value": value}) + return ParticipantIDFilter("album", value, role) } func allRolesFilter(_ string, value any) Sqlizer { - return Like{"participants": fmt.Sprintf(`%%"%s"%%`, value)} + return ParticipantIDFilter("album", value) } func (r *albumRepository) CountAll(options ...model.QueryOptions) (int64, error) { @@ -214,12 +212,7 @@ func (r *albumRepository) Put(al *model.Album) error { return err } al.ID = id - if len(al.Participants) > 0 { - if err = r.updateParticipants(al.ID, al.Participants); err != nil { - return err - } - } - return nil + return r.updateParticipants(al.ID, al.Participants) } // TODO Move external metadata to a separated table diff --git a/persistence/album_repository_test.go b/persistence/album_repository_test.go index 1ead89529..061083949 100644 --- a/persistence/album_repository_test.go +++ b/persistence/album_repository_test.go @@ -527,19 +527,16 @@ var _ = Describe("AlbumRepository", func() { Describe("artistRoleFilter", func() { DescribeTable("creates correct SQL expressions for artist roles", - func(filterName, artistID, expectedSQL string) { + func(filterName, artistID, expectedRole string) { sqlizer := artistRoleFilter(filterName, artistID) sql, args, err := sqlizer.ToSql() Expect(err).ToNot(HaveOccurred()) - Expect(sql).To(Equal(expectedSQL)) - Expect(args).To(Equal([]any{artistID})) + Expect(sql).To(Equal("album.id IN (SELECT album_id FROM album_artists WHERE artist_id = ? AND role IN (?))")) + Expect(args).To(Equal([]any{artistID, expectedRole})) }, - Entry("artist role", "role_artist_id", "123", - "exists (select 1 from json_tree(participants, '$.artist') where value = ?)"), - Entry("albumartist role", "role_albumartist_id", "456", - "exists (select 1 from json_tree(participants, '$.albumartist') where value = ?)"), - Entry("composer role", "role_composer_id", "789", - "exists (select 1 from json_tree(participants, '$.composer') where value = ?)"), + Entry("artist role", "role_artist_id", "123", "artist"), + Entry("albumartist role", "role_albumartist_id", "456", "albumartist"), + Entry("composer role", "role_composer_id", "789", "composer"), ) It("works with the actual filter map", func() { @@ -553,8 +550,8 @@ var _ = Describe("AlbumRepository", func() { sqlizer := filterFunc(filterName, "test-id") sql, args, err := sqlizer.ToSql() Expect(err).ToNot(HaveOccurred()) - Expect(sql).To(Equal(fmt.Sprintf("exists (select 1 from json_tree(participants, '$.%s') where value = ?)", roleName))) - Expect(args).To(Equal([]any{"test-id"})) + Expect(sql).To(Equal("album.id IN (SELECT album_id FROM album_artists WHERE artist_id = ? AND role IN (?))")) + Expect(args).To(Equal([]any{"test-id", roleName})) } }) @@ -644,6 +641,74 @@ var _ = Describe("AlbumRepository", func() { _, _ = albumRepo.executeSQL(squirrel.Delete("album").Where(squirrel.Eq{"id": album.ID})) }) + It("finds albums through the participant-based filters", func() { + artist := &model.Artist{ID: "filter-artist-1", Name: "Filter Artist", OrderArtistName: "filter artist"} + Expect(createArtistWithLibrary(artistRepo, artist, 1)).To(Succeed()) + + album := &model.Album{ + LibraryID: 1, + ID: "filter-album-1", + Name: "Filter Album", + AlbumArtistID: artist.ID, + AlbumArtist: artist.Name, + Participants: model.Participants{ + model.RoleAlbumArtist: {{Artist: model.Artist{ID: artist.ID, Name: artist.Name}}}, + model.RoleComposer: {{Artist: model.Artist{ID: artist.ID, Name: artist.Name}}}, + }, + } + Expect(albumRepo.Put(album)).To(Succeed()) + + byArtist, err := albumRepo.GetAll(model.QueryOptions{Filters: artistFilter("artist_id", artist.ID)}) + Expect(err).ToNot(HaveOccurred()) + Expect(byArtist).To(HaveLen(1)) + Expect(byArtist[0].ID).To(Equal(album.ID)) + + byComposer, err := albumRepo.GetAll(model.QueryOptions{Filters: artistRoleFilter("role_composer_id", artist.ID)}) + Expect(err).ToNot(HaveOccurred()) + Expect(byComposer).To(HaveLen(1)) + + byLyricist, err := albumRepo.GetAll(model.QueryOptions{Filters: artistRoleFilter("role_lyricist_id", artist.ID)}) + Expect(err).ToNot(HaveOccurred()) + Expect(byLyricist).To(BeEmpty()) + + byAnyRole, err := albumRepo.GetAll(model.QueryOptions{Filters: allRolesFilter("role_total_id", artist.ID)}) + Expect(err).ToNot(HaveOccurred()) + Expect(byAnyRole).To(HaveLen(1)) + + count, err := albumRepo.CountAll(model.QueryOptions{Filters: artistFilter("artist_id", artist.ID)}) + Expect(err).ToNot(HaveOccurred()) + Expect(count).To(Equal(int64(1))) + + _, _ = artistRepo.executeSQL(squirrel.Delete("artist").Where(squirrel.Eq{"id": artist.ID})) + _, _ = albumRepo.executeSQL(squirrel.Delete("album").Where(squirrel.Eq{"id": album.ID})) + }) + + It("clears album_artists rows when saved with empty participants", func() { + artist := &model.Artist{ID: "clear-artist-1", Name: "Clear Artist", OrderArtistName: "clear artist"} + Expect(createArtistWithLibrary(artistRepo, artist, 1)).To(Succeed()) + + album := &model.Album{ + LibraryID: 1, + ID: "clear-album-1", + Name: "Clear Album", + AlbumArtistID: artist.ID, + AlbumArtist: artist.Name, + Participants: model.Participants{ + model.RoleAlbumArtist: {{Artist: model.Artist{ID: artist.ID, Name: artist.Name}}}, + }, + } + DeferCleanup(func() { + _, _ = artistRepo.executeSQL(squirrel.Delete("artist").Where(squirrel.Eq{"id": artist.ID})) + _, _ = albumRepo.executeSQL(squirrel.Delete("album").Where(squirrel.Eq{"id": album.ID})) + }) + Expect(albumRepo.Put(album)).To(Succeed()) + verifyAlbumArtists(album.ID, []albumArtistRecord{{ArtistID: artist.ID, Role: "albumartist", SubRole: ""}}) + + album.Participants = model.Participants{} + Expect(albumRepo.Put(album)).To(Succeed()) + verifyAlbumArtists(album.ID, []albumArtistRecord{}) + }) + It("filters out invalid artist IDs leaving only valid participants in database", func() { // Create two real artists in the database artist1 := &model.Artist{ diff --git a/persistence/mediafile_repository.go b/persistence/mediafile_repository.go index df1472c2a..f372b30f5 100644 --- a/persistence/mediafile_repository.go +++ b/persistence/mediafile_repository.go @@ -113,7 +113,7 @@ var mediaFileFilter = sync.OnceValue(func() map[string]filterFunc { "has_rating": annotationBoolFilter("rating"), "genre_id": tagIDFilter, "missing": booleanFilter, - "artists_id": artistFilter, + "artists_id": mediaFileArtistFilter, "library_id": libraryIdFilter, "path": startsWithFilter("media_file.path"), } @@ -126,6 +126,10 @@ var mediaFileFilter = sync.OnceValue(func() map[string]filterFunc { return filters }) +func mediaFileArtistFilter(_ string, value any) Sqlizer { + return ParticipantIDFilter("media_file", value, model.RoleAlbumArtist, model.RoleArtist) +} + func mediaFileRecentlyAddedSort() string { if conf.Server.RecentlyAddedByModTime { return "media_file.updated_at, media_file.id" diff --git a/persistence/sql_participations.go b/persistence/sql_participations.go index 38b0203fa..746abed01 100644 --- a/persistence/sql_participations.go +++ b/persistence/sql_participations.go @@ -22,6 +22,30 @@ type flatParticipant struct { SubRole string `json:"sub_role,omitempty"` } +// ParticipantIDFilter matches rows of table where the artist participates in any of the given roles +// (any role when empty). Semi-joins _artists; json_tree over the JSON is far slower at scale. +func ParticipantIDFilter(table string, artistID any, roles ...model.Role) Sqlizer { + return participantIDFilter(table, artistID, false, roles) +} + +// NotParticipantIDFilter is the negation of ParticipantIDFilter. +func NotParticipantIDFilter(table string, artistID any, roles ...model.Role) Sqlizer { + return participantIDFilter(table, artistID, true, roles) +} + +func participantIDFilter(table string, artistID any, negate bool, roles []model.Role) Sqlizer { + sel := Select(table + "_id").From(table + "_artists").Where(Eq{"artist_id": artistID}) + if len(roles) > 0 { + sel = sel.Where(Eq{"role": slice.Map(roles, func(r model.Role) string { return r.String() })}) + } + sql, args, _ := sel.ToSql() + op := " IN (" + if negate { + op = " NOT IN (" + } + return Expr(table+".id"+op+sql+")", args...) +} + func marshalParticipants(participants model.Participants) string { dbParticipants := make(map[model.Role][]participant) for role, artists := range participants { diff --git a/server/filter/filters.go b/server/filter/filters.go index 067f7b1f9..62d8f0523 100644 --- a/server/filter/filters.go +++ b/server/filter/filters.go @@ -47,17 +47,13 @@ func AlbumsByArtist() Options { } func AlbumsByArtistID(artistId string) Options { - filters := []Sqlizer{ - persistence.Exists("json_tree(participants, '$.albumartist')", Eq{"value": artistId}), - } + roles := []model.Role{model.RoleAlbumArtist} if conf.Server.Subsonic.ArtistParticipations { - filters = append(filters, - persistence.Exists("json_tree(participants, '$.artist')", Eq{"value": artistId}), - ) + roles = append(roles, model.RoleArtist) } return addDefaultFilters(Options{ Sort: "max_year", - Filters: Or(filters), + Filters: persistence.ParticipantIDFilter("album", artistId, roles...), }) } @@ -68,8 +64,8 @@ func AlbumsByContributingArtistID(artistId string) Options { return addDefaultFilters(Options{ Sort: "max_year", Filters: And{ - persistence.Exists("json_tree(participants, '$.artist')", Eq{"value": artistId}), - persistence.NotExists("json_tree(participants, '$.albumartist')", Eq{"value": artistId}), + persistence.ParticipantIDFilter("album", artistId, model.RoleArtist), + persistence.NotParticipantIDFilter("album", artistId, model.RoleAlbumArtist), }, }) } @@ -104,13 +100,11 @@ func SongsByAlbum(albumId string) Options { } // SongsByArtistID matches media files where the artist participates as album or track artist, in -// album order. Semi-joins media_file_artists; scanning the participants JSON is ~10x slower at scale. +// album order. func SongsByArtistID(artistId string) Options { return addDefaultFilters(Options{ - Sort: "album", - Filters: Expr( - "media_file.id IN (SELECT media_file_id FROM media_file_artists WHERE artist_id = ? AND role IN (?, ?))", - artistId, model.RoleArtist.String(), model.RoleAlbumArtist.String()), + Sort: "album", + Filters: persistence.ParticipantIDFilter("media_file", artistId, model.RoleArtist, model.RoleAlbumArtist), }) } diff --git a/server/jellyfin/items_test.go b/server/jellyfin/items_test.go index 401145461..4eddcc654 100644 --- a/server/jellyfin/items_test.go +++ b/server/jellyfin/items_test.go @@ -219,7 +219,7 @@ var _ = Describe("Items", func() { albumRepo := ds.Album(context.Background()).(*tests.MockAlbumRepo) sql, _, err := albumRepo.Options.Filters.ToSql() Expect(err).NotTo(HaveOccurred()) - Expect(sql).To(ContainSubstring("json_tree")) + Expect(sql).To(ContainSubstring("album_artists")) }) It("lists artists when IncludeItemTypes=MusicArtist", func() { @@ -708,7 +708,7 @@ var _ = Describe("Items", func() { Expect(w.Code).To(Equal(http.StatusOK)) sql, args, err := albumRepo.Options.Filters.ToSql() Expect(err).NotTo(HaveOccurred()) - Expect(sql).NotTo(ContainSubstring("json_tree")) // not treated as an artist-parent filter + Expect(sql).NotTo(ContainSubstring("album_artists")) // not treated as an artist-parent filter Expect(sql).To(ContainSubstring("library_id")) Expect(args).To(ContainElement(2)) }) @@ -724,7 +724,7 @@ var _ = Describe("Items", func() { sql, args, err := albumRepo.Options.Filters.ToSql() Expect(err).NotTo(HaveOccurred()) // Falls back to treating "99" as an (empty-matching) artist-parent id... - Expect(sql).To(ContainSubstring("json_tree")) + Expect(sql).To(ContainSubstring("album_artists")) // ...while still scoping to the user's own accessible libraries. Expect(sql).To(ContainSubstring("library_id")) Expect(args).To(ContainElement(1))