package external import ( "context" "errors" "fmt" "slices" "sort" "strings" "time" "github.com/Masterminds/squirrel" "github.com/navidrome/navidrome/conf" "github.com/navidrome/navidrome/core/agents" "github.com/navidrome/navidrome/core/matcher" "github.com/navidrome/navidrome/log" "github.com/navidrome/navidrome/model" "github.com/navidrome/navidrome/server/events" "github.com/navidrome/navidrome/utils" . "github.com/navidrome/navidrome/utils/gg" "github.com/navidrome/navidrome/utils/slice" "github.com/navidrome/navidrome/utils/str" "golang.org/x/sync/errgroup" ) const ( maxSimilarArtists = 100 refreshDelay = 5 * time.Second refreshTimeout = 15 * time.Second refreshQueueLength = 2000 ) type Provider interface { UpdateAlbumInfo(ctx context.Context, id string) (*model.Album, error) UpdateArtistInfo(ctx context.Context, id string, count int, includeNotPresent bool) (*model.Artist, error) SimilarSongs(ctx context.Context, id string, count int) (model.MediaFiles, error) TopSongs(ctx context.Context, artist, artistId string, count int) (model.MediaFiles, error) RefreshInfo(ctx context.Context, kind model.Kind, id string) error } type provider struct { ds model.DataStore ag Agents matcher *matcher.Matcher broker events.Broker artistQueue refreshQueue[auxArtist] albumQueue refreshQueue[auxAlbum] } type auxAlbum struct { model.Album } // Name returns the appropriate album name for external API calls // based on the DevPreserveUnicodeInExternalCalls configuration option func (a *auxAlbum) Name() string { if conf.Server.DevPreserveUnicodeInExternalCalls { return a.Album.Name } return str.Clear(a.Album.Name) } type auxArtist struct { model.Artist } // Name returns the appropriate artist name for external API calls // based on the DevPreserveUnicodeInExternalCalls configuration option func (a *auxArtist) Name() string { if conf.Server.DevPreserveUnicodeInExternalCalls { return a.Artist.Name } return str.Clear(a.Artist.Name) } type Agents interface { agents.AlbumInfoRetriever agents.AlbumImageRetriever agents.ArtistBiographyRetriever agents.ArtistMBIDRetriever agents.ArtistImageRetriever agents.ArtistSimilarRetriever agents.ArtistTopSongsRetriever agents.ArtistURLRetriever agents.SimilarSongsByTrackRetriever agents.SimilarSongsByAlbumRetriever agents.SimilarSongsByArtistRetriever } func NewProvider(ds model.DataStore, agents Agents, m *matcher.Matcher, broker events.Broker) Provider { e := &provider{ds: ds, ag: agents, matcher: m, broker: broker} e.artistQueue = newRefreshQueue(context.TODO(), e.populateArtistInfo) e.albumQueue = newRefreshQueue(context.TODO(), e.populateAlbumInfo) return e } func (e *provider) broadcastRefresh(ctx context.Context, resource, id string) { e.broker.SendBroadcastMessage(ctx, (&events.RefreshResource{}).With(resource, id)) } func (e *provider) getAlbum(ctx context.Context, id string) (auxAlbum, error) { var entity any entity, err := model.GetEntityByID(ctx, e.ds, id) if err != nil { return auxAlbum{}, err } var album auxAlbum switch v := entity.(type) { case *model.Album: album.Album = *v case *model.MediaFile: return e.getAlbum(ctx, v.AlbumID) default: return auxAlbum{}, model.ErrNotFound } return album, nil } func (e *provider) UpdateAlbumInfo(ctx context.Context, id string) (*model.Album, error) { album, err := e.getAlbum(ctx, id) if err != nil { log.Info(ctx, "Not found", "id", id) return nil, err } updatedAt := V(album.ExternalInfoUpdatedAt) albumName := album.Name() if updatedAt.IsZero() { log.Debug(ctx, "AlbumInfo not cached. Retrieving it now", "updatedAt", updatedAt, "id", id, "name", albumName) album, err = e.populateAlbumInfo(ctx, album) if err != nil { return nil, err } } // If info is expired, trigger a populateAlbumInfo in the background if time.Since(updatedAt) > conf.Server.DevAlbumInfoTimeToLive { log.Debug("Found expired cached AlbumInfo, refreshing in the background", "updatedAt", album.ExternalInfoUpdatedAt, "name", albumName) e.albumQueue.enqueue(&album) } return &album.Album, nil } func (e *provider) populateAlbumInfo(ctx context.Context, album auxAlbum) (auxAlbum, error) { start := time.Now() albumName := album.Name() info, err := e.ag.GetAlbumInfo(ctx, albumName, album.AlbumArtist, album.MbzAlbumID) // Throttled joins not-found: no answer to store, and an unstamped timestamp retries next call. if errors.Is(err, agents.ErrNotFound) || errors.Is(err, agents.ErrRetryLater) { return album, nil } if err != nil { log.Error("Error refreshing AlbumInfo", "id", album.ID, "name", albumName, "artist", album.AlbumArtist, "elapsed", time.Since(start), err) return album, err } album.ExternalInfoUpdatedAt = new(time.Now()) album.ExternalUrl = info.URL if info.Description != "" { album.Description = info.Description } images, err := e.ag.GetAlbumImages(ctx, albumName, album.AlbumArtist, album.MbzAlbumID) if err == nil && len(images) > 0 { sort.Slice(images, func(i, j int) bool { return images[i].Size > images[j].Size }) album.LargeImageUrl = images[0].URL if len(images) >= 2 { album.MediumImageUrl = images[1].URL } if len(images) >= 3 { album.SmallImageUrl = images[2].URL } } err = e.ds.Album(ctx).UpdateExternalInfo(&album.Album) if err != nil { log.Error(ctx, "Error trying to update album external information", "id", album.ID, "name", albumName, "elapsed", time.Since(start), err) } else { log.Trace(ctx, "AlbumInfo collected", "album", album, "elapsed", time.Since(start)) e.broadcastRefresh(ctx, "album", album.ID) } return album, nil } func (e *provider) getArtist(ctx context.Context, id string) (auxArtist, error) { var entity any entity, err := model.GetEntityByID(ctx, e.ds, id) if err != nil { return auxArtist{}, err } var artist auxArtist switch v := entity.(type) { case *model.Artist: artist.Artist = *v case *model.MediaFile: return e.getArtist(ctx, v.ArtistID) case *model.Album: return e.getArtist(ctx, v.AlbumArtistID) default: return auxArtist{}, model.ErrNotFound } return artist, nil } func (e *provider) UpdateArtistInfo(ctx context.Context, id string, similarCount int, includeNotPresent bool) (*model.Artist, error) { artist, err := e.refreshArtistInfo(ctx, id) if err != nil { return nil, err } err = e.loadSimilar(ctx, &artist, similarCount, includeNotPresent) return &artist.Artist, err } func (e *provider) refreshArtistInfo(ctx context.Context, id string) (auxArtist, error) { artist, err := e.getArtist(ctx, id) if err != nil { return auxArtist{}, err } // If we don't have any info, retrieves it now updatedAt := V(artist.ExternalInfoUpdatedAt) artistName := artist.Name() if updatedAt.IsZero() { log.Debug(ctx, "ArtistInfo not cached. Retrieving it now", "updatedAt", updatedAt, "id", id, "name", artistName) artist, err = e.populateArtistInfo(ctx, artist) if err != nil { return auxArtist{}, err } } // If info is expired, trigger a populateArtistInfo in the background if time.Since(updatedAt) > conf.Server.DevArtistInfoTimeToLive { log.Debug("Found expired cached ArtistInfo, refreshing in the background", "updatedAt", updatedAt, "name", artistName) e.artistQueue.enqueue(&artist) } return artist, nil } func (e *provider) populateArtistInfo(ctx context.Context, artist auxArtist) (auxArtist, error) { start := time.Now() // Get MBID first, if it is not yet available artistName := artist.Name() var mbidErr error if artist.MbzArtistID == "" { mbid, err := e.ag.GetArtistMBID(ctx, artist.ID, artistName) mbidErr = err if mbid != "" && err == nil { artist.MbzArtistID = mbid } } // Call all registered agents and collect information. The group carries no context, so a // returned error does not cancel the siblings; only throttling is reported back. g := errgroup.Group{} g.SetLimit(2) g.Go(func() error { return retryLaterOnly(e.callGetImage(ctx, e.ag, &artist)) }) g.Go(func() error { return retryLaterOnly(e.callGetBiography(ctx, e.ag, &artist)) }) g.Go(func() error { return retryLaterOnly(e.callGetURL(ctx, e.ag, &artist)) }) g.Go(func() error { return retryLaterOnly(e.callGetSimilarArtists(ctx, e.ag, &artist, maxSimilarArtists, true)) }) throttled := errors.Is(g.Wait(), agents.ErrRetryLater) || errors.Is(mbidErr, agents.ErrRetryLater) if utils.IsCtxDone(ctx) { log.Warn(ctx, "ArtistInfo update canceled", "id", artist.ID, "name", artistName, "elapsed", time.Since(start), ctx.Err()) return artist, ctx.Err() } // A throttled round keeps the previous timestamp, so the next call retries instead of // serving an empty cache entry for the whole TTL. if !throttled { artist.ExternalInfoUpdatedAt = new(time.Now()) } err := e.ds.Artist(ctx).UpdateExternalInfo(&artist.Artist) if err != nil { log.Error(ctx, "Error trying to update artist external information", "id", artist.ID, "name", artistName, "elapsed", time.Since(start), err) } else { log.Trace(ctx, "ArtistInfo collected", "artist", artist, "elapsed", time.Since(start)) e.broadcastRefresh(ctx, "artist", artist.ID) } return artist, nil } // infoKinds are the kinds RefreshInfo can act on. Callers check this instead of restating // the set, so the switch below stays the only place that has to know how each kind loads. var infoKinds = []model.Kind{model.KindArtistArtwork, model.KindAlbumArtwork} // HasInfo reports whether a kind has external info to refresh. func HasInfo(kind model.Kind) bool { return slices.Contains(infoKinds, kind) } // RefreshInfo re-fetches external info for one item, ignoring the TTL. It is synchronous: // callers that must not block are responsible for detaching it. func (e *provider) RefreshInfo(ctx context.Context, kind model.Kind, id string) error { ctx, cancel := context.WithTimeout(ctx, refreshTimeout) defer cancel() switch kind { case model.KindArtistArtwork: artist, err := e.getArtist(ctx, id) if err != nil { return err } _, err = e.populateArtistInfo(ctx, artist) return err case model.KindAlbumArtwork: album, err := e.getAlbum(ctx, id) if err != nil { return err } _, err = e.populateAlbumInfo(ctx, album) return err default: return model.ErrNotFound } } func (e *provider) TopSongs(ctx context.Context, artistName, id string, count int) (model.MediaFiles, error) { artist, err := e.findArtist(ctx, artistName, id) if err != nil { if errors.Is(err, model.ErrNotFound) { log.Error(ctx, "Artist not found", "name", artistName, "id", id, err) return nil, nil } log.Error(ctx, "Failure occurred when trying to fetch artist", "name", artistName, "id", id, err) return nil, err } songs, err := e.getMatchingTopSongs(ctx, e.ag, artist, count) if err != nil { switch { // Throttled is not an answer, but the caller keeps the empty 200 it got before. case errors.Is(err, agents.ErrNotFound), errors.Is(err, agents.ErrRetryLater): log.Trace(ctx, "TopSongs not found", "name", artistName, err) return nil, model.ErrNotFound case errors.Is(err, context.Canceled): log.Debug(ctx, "TopSongs call canceled", err) default: log.Warn(ctx, "Error getting top songs from agent", "artist", artistName, err) } return nil, err } return songs, nil } func (e *provider) getMatchingTopSongs(ctx context.Context, agent agents.ArtistTopSongsRetriever, artist *auxArtist, count int) (model.MediaFiles, error) { artistName := artist.Name() songs, err := agent.GetArtistTopSongs(ctx, artist.ID, artistName, artist.MbzArtistID, count) if err != nil { return nil, fmt.Errorf("failed to get top songs for artist %s: %w", artistName, err) } // Enrich top songs with the queried artist. A song with no artists, or whose first credit the // agent left unnamed, is attributed to the queried artist. A first credit that already names an // artist is left as-is: it may be a different (e.g. featured) artist, so stamping the queried // MBID onto it would create a false name+MBID pairing. for i := range songs { switch { case len(songs[i].Artists) == 0: songs[i].Artists = []agents.Artist{{Name: artistName, MBID: artist.MbzArtistID}} case songs[i].Artists[0].Name == "": songs[i].Artists[0].Name = artistName if songs[i].Artists[0].MBID == "" { songs[i].Artists[0].MBID = artist.MbzArtistID } } } mfs, err := e.matcher.MatchSongs(ctx, songs, count) if err != nil { return nil, err } if len(mfs) == 0 { log.Debug(ctx, "No matching top songs found", "name", artistName) } else { log.Debug(ctx, "Found matching top songs", "name", artistName, "numSongs", len(mfs)) } return mfs, nil } // retryLaterOnly discards every failure the caller does not act on, so errgroup's // first-error slot is reserved for the throttling signal. func retryLaterOnly(err error) error { if errors.Is(err, agents.ErrRetryLater) { return err } return nil } func (e *provider) callGetURL(ctx context.Context, agent agents.ArtistURLRetriever, artist *auxArtist) error { artisURL, err := agent.GetArtistURL(ctx, artist.ID, artist.Name(), artist.MbzArtistID) if err != nil { return err } artist.ExternalUrl = artisURL return nil } func (e *provider) callGetBiography(ctx context.Context, agent agents.ArtistBiographyRetriever, artist *auxArtist) error { bio, err := agent.GetArtistBiography(ctx, artist.ID, artist.Name(), artist.MbzArtistID) if err != nil { return err } bio = str.SanitizeText(bio) bio = strings.ReplaceAll(bio, "\n", " ") artist.Biography = strings.ReplaceAll(bio, " images[j].Size }) if len(images) >= 1 { artist.LargeImageUrl = images[0].URL } if len(images) >= 2 { artist.MediumImageUrl = images[1].URL } if len(images) >= 3 { artist.SmallImageUrl = images[2].URL } return nil } func (e *provider) callGetSimilarArtists(ctx context.Context, agent agents.ArtistSimilarRetriever, artist *auxArtist, limit int, includeNotPresent bool) error { artistName := artist.Name() similar, err := agent.GetSimilarArtists(ctx, artist.ID, artistName, artist.MbzArtistID, limit) if len(similar) == 0 || err != nil { return err } start := time.Now() sa, err := e.mapSimilarArtists(ctx, similar, limit, includeNotPresent) log.Debug(ctx, "Mapped Similar Artists", "artist", artistName, "numSimilar", len(sa), "elapsed", time.Since(start)) if err != nil { return err } artist.SimilarArtists = sa return nil } func (e *provider) mapSimilarArtists(ctx context.Context, similar []agents.Artist, limit int, includeNotPresent bool) (model.Artists, error) { var result model.Artists var notPresent []string // Load artists by ID (highest priority) idMatches, err := e.loadArtistsByID(ctx, similar) if err != nil { return nil, err } // Load artists by MBID (second priority) mbidMatches, err := e.loadArtistsByMBID(ctx, similar, idMatches) if err != nil { return nil, err } // Load artists by name (lowest priority, fallback) nameMatches, err := e.loadArtistsByName(ctx, similar, idMatches, mbidMatches) if err != nil { return nil, err } count := 0 // Process the similar artists using priority: ID → MBID → Name for _, s := range similar { if count >= limit { break } // Try ID match first if s.ID != "" { if artist, found := idMatches[s.ID]; found { result = append(result, artist) count++ continue } } // Try MBID match second if s.MBID != "" { if artist, found := mbidMatches[s.MBID]; found { result = append(result, artist) count++ continue } } // Fall back to name match if artist, found := nameMatches[s.Name]; found { result = append(result, artist) count++ } else { notPresent = append(notPresent, s.Name) } } // Then fill up with non-present artists if includeNotPresent && count < limit { for _, s := range notPresent { // Let the ID empty to indicate that the artist is not present in the DB sa := model.Artist{Name: s} result = append(result, sa) count++ if count >= limit { break } } } return result, nil } func (e *provider) loadArtistsByID(ctx context.Context, similar []agents.Artist) (map[string]model.Artist, error) { var ids []string for _, s := range similar { if s.ID != "" { ids = append(ids, s.ID) } } matches := map[string]model.Artist{} if len(ids) == 0 { return matches, nil } res, err := e.ds.Artist(ctx).GetAll(model.QueryOptions{ Filters: squirrel.Eq{"artist.id": ids}, }) if err != nil { return matches, err } for _, a := range res { if _, ok := matches[a.ID]; !ok { matches[a.ID] = a } } return matches, nil } func (e *provider) loadArtistsByMBID(ctx context.Context, similar []agents.Artist, idMatches map[string]model.Artist) (map[string]model.Artist, error) { var mbids []string for _, s := range similar { // Skip if already matched by ID if s.ID != "" && idMatches[s.ID].ID != "" { continue } if s.MBID != "" { mbids = append(mbids, s.MBID) } } matches := map[string]model.Artist{} if len(mbids) == 0 { return matches, nil } res, err := e.ds.Artist(ctx).GetAll(model.QueryOptions{ Filters: squirrel.Eq{"mbz_artist_id": mbids}, }) if err != nil { return matches, err } for _, a := range res { if id := a.MbzArtistID; id != "" { if _, ok := matches[id]; !ok { matches[id] = a } } } return matches, nil } func (e *provider) loadArtistsByName(ctx context.Context, similar []agents.Artist, idMatches map[string]model.Artist, mbidMatches map[string]model.Artist) (map[string]model.Artist, error) { var names []string for _, s := range similar { // Skip if already matched by ID or MBID if s.ID != "" && idMatches[s.ID].ID != "" { continue } if s.MBID != "" && mbidMatches[s.MBID].ID != "" { continue } names = append(names, s.Name) } matches := map[string]model.Artist{} if len(names) == 0 { return matches, nil } clauses := slice.Map(names, func(name string) squirrel.Sqlizer { return squirrel.Like{"artist.name": name} }) res, err := e.ds.Artist(ctx).GetAll(model.QueryOptions{ Filters: squirrel.Or(clauses), }) if err != nil { return matches, err } for _, a := range res { if _, ok := matches[a.Name]; !ok { matches[a.Name] = a } } return matches, nil } func (e *provider) findArtist(ctx context.Context, artistName, id string) (*auxArtist, error) { if id != "" { artist, err := e.ds.Artist(ctx).Get(id) if err == nil { return &auxArtist{Artist: *artist}, nil } if errors.Is(err, model.ErrNotFound) { log.Warn(ctx, "Could not find artist by id", "id", id, err) } else { return nil, err } } if artistName == "" { return nil, model.ErrNotFound } artists, err := e.ds.Artist(ctx).GetAll(model.QueryOptions{ Filters: squirrel.Like{"artist.name": artistName}, Max: 1, }) if err != nil { return nil, err } if len(artists) == 0 { return nil, model.ErrNotFound } return &auxArtist{Artist: artists[0]}, nil } func (e *provider) loadSimilar(ctx context.Context, artist *auxArtist, count int, includeNotPresent bool) error { var ids []string for _, sa := range artist.SimilarArtists { if sa.ID == "" { continue } ids = append(ids, sa.ID) } similar, err := e.ds.Artist(ctx).GetAll(model.QueryOptions{ Filters: squirrel.Eq{"artist.id": ids}, }) if err != nil { log.Error("Error loading similar artists", "id", artist.ID, "name", artist.Name(), err) return err } // Use a map and iterate through original array, to keep the same order artistMap := make(map[string]model.Artist) for _, sa := range similar { artistMap[sa.ID] = sa } var loaded model.Artists for _, sa := range artist.SimilarArtists { if len(loaded) >= count { break } la, ok := artistMap[sa.ID] if !ok { if !includeNotPresent { continue } la = sa la.ID = "" } loaded = append(loaded, la) } artist.SimilarArtists = loaded return nil } type refreshQueue[T any] chan<- *T func newRefreshQueue[T any](ctx context.Context, processFn func(context.Context, T) (T, error)) refreshQueue[T] { queue := make(chan *T, refreshQueueLength) go func() { for { select { case <-ctx.Done(): return case <-time.After(refreshDelay): ctx, cancel := context.WithTimeout(ctx, refreshTimeout) select { case item := <-queue: _, _ = processFn(ctx, *item) cancel() case <-ctx.Done(): cancel() } } } }() return queue } func (q *refreshQueue[T]) enqueue(item *T) { select { case *q <- item: default: // It is ok to miss a refresh request } }