perf(scanner): improve playlist importing on large libraries (#6055)

* perf(persistence): avoid a full media_file scan when resolving playlist paths

FindByPaths built one OR-ed equality term per path. On the real media_file
schema SQLite abandons the path index at just two OR-ed terms and falls back to
SCAN media_file, re-testing every term against every row, so the cost grows with
(rows x terms).

Group the candidates by library and emit one IN list per library instead, which
plans as SEARCH media_file USING INDEX media_file_path_nocase. The NOCASE
collation is kept so ASCII case-insensitive matching still works.

This is the dominant cost of M3U playlist import, which resolves every track on
every scan. Measured with a 1000-track playlist against a migrated DB:

  100k media_file rows:   397 -> 51,414 tracks/sec
  500k media_file rows:  78.5 -> 47,174 tracks/sec

The rate no longer degrades as the table grows, which is the expected shape for
an index lookup. Reported in #6043, where an 8 hour scan of a 2M-song library
spent 7h52m in the playlist phase.

* docs(playlists): correct the stale reason for the M3U lookup chunk size

The expression-tree depth ceiling applied to the old OR-per-path query, which
capped a batch at roughly 500 terms. The IN form is bound by SQLite's 32766
variable limit instead, which the 400 candidates per chunk sit far below.
This commit is contained in:
Deluan Quintão 2026-08-30 22:17:25 -04:00 committed by GitHub
parent 9ff0058620
commit dbd26ba2e7
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 42 additions and 9 deletions

View File

@ -25,8 +25,8 @@ func (s *playlists) parseM3U(ctx context.Context, pls *model.Playlist, folder *m
return err
}
var mfs model.MediaFiles
// Chunk size of 100 lines, as each line can generate up to 4 lookup candidates
// (NFC/NFD × raw/lowercase), and SQLite has a max expression tree depth of 1000.
// Chunked so a huge playlist is not held in memory at once. Each line yields up to
// 4 lookup candidates (NFC/NFD × raw/lowercase), far below SQLite's 32766 variables.
for lines := range slice.CollectChunks(slice.LinesFrom(reader), 100) {
filteredLines := make([]string, 0, len(lines))
for _, line := range lines {

View File

@ -4,6 +4,7 @@ import (
"context"
"fmt"
"iter"
"maps"
"slices"
"strconv"
"strings"
@ -355,7 +356,10 @@ func (r *mediaFileRepository) GetCursorWithArtwork(options ...model.QueryOptions
// Library-qualified paths search within the specified library, while unqualified paths
// search across all libraries for backward compatibility.
func (r *mediaFileRepository) FindByPaths(paths []string) (model.MediaFiles, error) {
query := Or{}
// One IN list per library instead of one OR term per path: SQLite abandons the
// path index at just two OR-ed equality terms and scans the whole table.
byLibrary := map[int][]string{}
var unqualified []string
for _, path := range paths {
parts := strings.SplitN(path, ":", 2)
@ -366,17 +370,24 @@ func (r *mediaFileRepository) FindByPaths(paths []string) (model.MediaFiles, err
// Invalid format, skip
continue
}
relativePath := parts[1]
query = append(query, And{
Eq{"path collate nocase": relativePath},
Eq{"library_id": libraryID},
})
byLibrary[libraryID] = append(byLibrary[libraryID], parts[1])
} else {
// Unqualified path: search across all libraries
query = append(query, Eq{"path collate nocase": path})
unqualified = append(unqualified, path)
}
}
query := Or{}
for _, libraryID := range slices.Sorted(maps.Keys(byLibrary)) {
query = append(query, And{
Eq{"path collate nocase": byLibrary[libraryID]},
Eq{"library_id": libraryID},
})
}
if len(unqualified) > 0 {
query = append(query, Eq{"path collate nocase": unqualified})
}
if len(query) == 0 {
return model.MediaFiles{}, nil
}

View File

@ -1055,6 +1055,28 @@ var _ = Describe("MediaRepository", func() {
Expect(results).To(HaveLen(1))
Expect(results[0].ID).To(Equal("otherlib-track"))
})
It("resolves paths from multiple libraries in a single call", func() {
adminMr := NewMediaFileRepository(request.WithUser(GinkgoT().Context(), adminUser), GetDBXBuilder())
results, err := adminMr.FindByPaths([]string{
"1:artist/Album/track.mp3",
fmt.Sprintf("%d:hidden/test.mp3", otherLib.ID),
})
Expect(err).ToNot(HaveOccurred())
Expect(results).To(HaveLen(2))
Expect([]string{results[0].ID, results[1].ID}).To(ConsistOf("findpath-1", "otherlib-track"))
})
It("keeps each path scoped to its own library when several are queried", func() {
adminMr := NewMediaFileRepository(request.WithUser(GinkgoT().Context(), adminUser), GetDBXBuilder())
// Each path exists, but under the other library's ID, so neither must match.
results, err := adminMr.FindByPaths([]string{
fmt.Sprintf("%d:artist/Album/track.mp3", otherLib.ID),
"1:hidden/test.mp3",
})
Expect(err).ToNot(HaveOccurred())
Expect(results).To(BeEmpty())
})
})
})