mirror of
https://github.com/navidrome/navidrome.git
synced 2026-08-01 07:21:17 +00:00
* perf(db): keep query planner statistics trustworthy with full ANALYZE PRAGMA optimize's internal ANALYZE runs with a limited analysis budget (~2000 rows) that writes wrong sqlite_stat1 entries for low-cardinality indexes: on a 96K-track library it claimed (missing, library_id) narrows to ~2000 rows when it matches the whole table. The planner then prefers that index over the sort index and falls back to a full-table temp B-tree sort per request, turning paginated song listings into multi-second queries (reproduced at 5.5s on real hardware; ~90x slower than with correct stats). Every index-creating migration re-triggered the poisoning via the post-migration optimize, and the daily optimizer could re-trigger it on large library changes. Setting analysis_limit on the connection does not help: optimize ignores it. Run a plain full ANALYZE instead: after migrations with schema changes, and in db.Optimize (daily schedule and scan-end). Stats are stored in the database file, so one connection suffices and the per-connection pool loop is gone. The Optimize call at shutdown is removed: stats are maintained at migration/scan/daily points, and an ANALYZE during shutdown only delays it and races container stop timeouts. * perf(db): drop startup PRAGMA optimize that re-poisons planner stats The startup PRAGMA optimize=0x10002 runs SQLite's budget-limited internal ANALYZE (bit 0x02), which writes truncated sqlite_stat1 rows for low-cardinality indexes -- the exact statistics-poisoning this PR set out to eliminate. Because DevOptimizeDB defaults to true, a restart with no pending migrations would re-poison the planner until the next scan or daily Optimize. Remove it: statistics are already refreshed with a full ANALYZE after schema-changing migrations (Init) and via Optimize at scan-end and on the daily schedule, so nothing on the startup path needs to touch them. Also clarify that Optimize is a no-op unless DevOptimizeDB is enabled. * chore(db): remove the DevOptimizeDB flag and skip Optimize on quick scans The flag only gated the optimize/ANALYZE maintenance calls and there is no reason to leave planner statistics unmaintained; the guards are gone along with the flag. The scan-end Optimize now runs only after full scans — quick scans barely move the statistics, and the daily schedule covers drift. * style(scanner): drop redundant comment in runOptimize * chore(persistence): drop the no-op PRAGMA optimize from ScanEnd Mask 0x10000 only selects candidate tables by size change; without the 0x02 action bit optimize does nothing (verified: sqlite_stat1 stays stale after a 100x table growth). The scan-end statistics refresh is db.Optimize's full ANALYZE, and the expression-collation-index concern the old comment guarded against no longer applies. * fix(scanner): run the post-scan ANALYZE in the server process With the external scanner (the default), the scan pipeline runs in a subprocess, so its ANALYZE was invisible to the server: SQLite loads sqlite_stat1 into the process's shared schema cache, and an ANALYZE from another process does not refresh it — verified with the production DSN that even brand-new pool connections keep planning with the old statistics until the server restarts. An in-process ANALYZE, by contrast, is immediately visible to every pooled connection through the same shared cache. Move the full-scan Optimize from the scanner pipeline to the scan controller, which always runs in the server process. * fix(scanner): honor promoted full scans in the optimize gate A quick scan resuming an interrupted full scan is promoted inside the scanner (possibly in a subprocess); mirror the promotion in the controller so the post-scan ANALYZE isn't skipped. * refactor: apply cleanup review findings - drop forceFullRescan's inline ANALYZE: Init already runs a full ANALYZE after any migration batch with schema changes, so upgrades including a full-rescan migration analyzed the whole DB twice - resumingFullScan uses a filtered CountAll instead of fetching and scanning all libraries - document why CallScan (CLI) deliberately skips the post-scan Optimize * perf(db): make planner analysis maintenance resilient Check analysis freshness every 30 minutes and refresh statistics when the last successful run is over 24 hours old or a scan marked them pending. Persist successful analysis state, retry skipped or failed maintenance, coordinate checks with scans, and cover standalone CLI full scans. * perf(db): avoid analyzing routine quick-scan changes Reserve pending analysis for full scans, unscanned libraries, and retry state. Incremental quick scans now rely on the 24-hour freshness window instead of triggering a full ANALYZE at the next maintenance check. * fix(scan): analyze resumed full scans in CLI * fix(db): back off failed analysis retries * feat(db): allow disabling scheduled analysis * test(db): remove redundant analysis coverage * refactor(db): split ANALYZE maintenance into optimize.go and dedupe call sites - move query-planner statistics code from db.go to its own optimize.go (and matching optimize_test.go) - log ANALYZE elapsed time inside Optimize/OptimizeIfNeeded instead of repeating the timing block at every call site - drop the LastDBAnalyzeAttemptAt write on success: it is only read while failures >= 1, and every failure rewrites it first - extract runPostScanAnalysis (cmd) and anyIncludedLibrary (scanner) helpers
173 lines
4.0 KiB
Go
173 lines
4.0 KiB
Go
package db
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"embed"
|
|
"fmt"
|
|
"runtime"
|
|
"time"
|
|
|
|
"github.com/mattn/go-sqlite3"
|
|
"github.com/navidrome/navidrome/conf"
|
|
_ "github.com/navidrome/navidrome/db/migrations"
|
|
"github.com/navidrome/navidrome/log"
|
|
"github.com/navidrome/navidrome/utils/hasher"
|
|
"github.com/navidrome/navidrome/utils/singleton"
|
|
"github.com/pressly/goose/v3"
|
|
)
|
|
|
|
var (
|
|
Dialect = "sqlite3"
|
|
Driver = Dialect + "_custom"
|
|
Path string
|
|
)
|
|
|
|
//go:embed migrations/*.sql
|
|
var embedMigrations embed.FS
|
|
|
|
const migrationsFolder = "migrations"
|
|
|
|
func Db() *sql.DB {
|
|
return singleton.GetInstance(func() *sql.DB {
|
|
sql.Register(Driver, &sqlite3.SQLiteDriver{
|
|
ConnectHook: func(conn *sqlite3.SQLiteConn) error {
|
|
return conn.RegisterFunc("SEEDEDRAND", hasher.HashFunc(), false)
|
|
},
|
|
})
|
|
Path = conf.Server.DbPath
|
|
if Path == ":memory:" {
|
|
Path = "file::memory:?cache=shared&_foreign_keys=on"
|
|
conf.Server.DbPath = Path
|
|
} else {
|
|
conf.Server.DataFolder.MustPath()
|
|
}
|
|
log.Debug("Opening DataBase", "dbPath", Path, "driver", Driver)
|
|
db, err := sql.Open(Driver, Path)
|
|
db.SetMaxOpenConns(max(4, runtime.NumCPU()))
|
|
if err != nil {
|
|
log.Fatal("Error opening database", err)
|
|
}
|
|
return db
|
|
})
|
|
}
|
|
|
|
func Close(ctx context.Context) {
|
|
// Ignore cancellations when closing the DB
|
|
ctx = context.WithoutCancel(ctx)
|
|
|
|
log.Info(ctx, "Closing Database")
|
|
err := Db().Close()
|
|
if err != nil {
|
|
log.Error(ctx, "Error closing Database", err)
|
|
}
|
|
}
|
|
|
|
func Init(ctx context.Context) func() {
|
|
db := Db()
|
|
|
|
// Disable foreign_keys to allow re-creating tables in migrations
|
|
_, err := db.ExecContext(ctx, "PRAGMA foreign_keys=off")
|
|
defer func() {
|
|
_, err := db.ExecContext(ctx, "PRAGMA foreign_keys=on")
|
|
if err != nil {
|
|
log.Error(ctx, "Error re-enabling foreign_keys", err)
|
|
}
|
|
}()
|
|
if err != nil {
|
|
log.Error(ctx, "Error disabling foreign_keys", err)
|
|
}
|
|
|
|
goose.SetBaseFS(embedMigrations)
|
|
err = goose.SetDialect(Dialect)
|
|
if err != nil {
|
|
log.Fatal(ctx, "Invalid DB driver", "driver", Driver, err)
|
|
}
|
|
schemaEmpty := isSchemaEmpty(ctx, db)
|
|
hasSchemaChanges := hasPendingMigrations(ctx, db, migrationsFolder)
|
|
if !schemaEmpty && hasSchemaChanges {
|
|
log.Info(ctx, "Upgrading DB Schema to latest version")
|
|
}
|
|
goose.SetLogger(&logAdapter{ctx: ctx, silent: schemaEmpty})
|
|
err = goose.UpContext(ctx, db, migrationsFolder)
|
|
if err != nil {
|
|
log.Fatal(ctx, "Failed to apply new migrations", err)
|
|
}
|
|
|
|
if hasSchemaChanges {
|
|
log.Debug(ctx, "Running ANALYZE after schema changes")
|
|
err = optimizeAt(ctx, db, time.Now())
|
|
if err != nil {
|
|
log.Error(ctx, "Error running ANALYZE", err)
|
|
}
|
|
}
|
|
|
|
return func() {
|
|
Close(ctx)
|
|
}
|
|
}
|
|
|
|
type statusLogger struct{ numPending int }
|
|
|
|
func (*statusLogger) Fatalf(format string, v ...any) { log.Fatal(fmt.Sprintf(format, v...)) }
|
|
func (l *statusLogger) Printf(format string, v ...any) {
|
|
if len(v) < 1 {
|
|
return
|
|
}
|
|
if v0, ok := v[0].(string); !ok {
|
|
return
|
|
} else if v0 == "Pending" {
|
|
l.numPending++
|
|
}
|
|
}
|
|
|
|
func hasPendingMigrations(ctx context.Context, db *sql.DB, folder string) bool {
|
|
l := &statusLogger{}
|
|
goose.SetLogger(l)
|
|
err := goose.StatusContext(ctx, db, folder)
|
|
if err != nil {
|
|
log.Fatal(ctx, "Failed to check for pending migrations", err)
|
|
}
|
|
return l.numPending > 0
|
|
}
|
|
|
|
func isSchemaEmpty(ctx context.Context, db *sql.DB) bool {
|
|
rows, err := db.QueryContext(ctx, "SELECT name FROM sqlite_master WHERE type='table' AND name='goose_db_version';") // nolint:rowserrcheck
|
|
if err != nil {
|
|
log.Fatal(ctx, "Database could not be opened!", err)
|
|
}
|
|
defer rows.Close()
|
|
return !rows.Next()
|
|
}
|
|
|
|
type logAdapter struct {
|
|
ctx context.Context
|
|
silent bool
|
|
}
|
|
|
|
func (l *logAdapter) Fatal(v ...any) {
|
|
log.Fatal(l.ctx, fmt.Sprint(v...))
|
|
}
|
|
|
|
func (l *logAdapter) Fatalf(format string, v ...any) {
|
|
log.Fatal(l.ctx, fmt.Sprintf(format, v...))
|
|
}
|
|
|
|
func (l *logAdapter) Print(v ...any) {
|
|
if !l.silent {
|
|
log.Info(l.ctx, fmt.Sprint(v...))
|
|
}
|
|
}
|
|
|
|
func (l *logAdapter) Println(v ...any) {
|
|
if !l.silent {
|
|
log.Info(l.ctx, fmt.Sprintln(v...))
|
|
}
|
|
}
|
|
|
|
func (l *logAdapter) Printf(format string, v ...any) {
|
|
if !l.silent {
|
|
log.Info(l.ctx, fmt.Sprintf(format, v...))
|
|
}
|
|
}
|