mirror of
https://github.com/navidrome/navidrome.git
synced 2026-05-03 06:51:16 +00:00
Compare commits
15 Commits
9fbbcfd31e
...
adb2e1b16b
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
adb2e1b16b | ||
|
|
4f7dc105b0 | ||
|
|
e918e049e2 | ||
|
|
1e8d28ff46 | ||
|
|
a128b3cf98 | ||
|
|
290a9fdeaa | ||
|
|
58b5ed86df | ||
|
|
fe1cee0159 | ||
|
|
3dfaa8cca1 | ||
|
|
0a5abfc1b1 | ||
|
|
c501bc6996 | ||
|
|
0c71842b12 | ||
|
|
70f43437b5 | ||
|
|
7add723a85 | ||
|
|
26a0a0cd5f |
18
.github/workflows/pipeline.yml
vendored
18
.github/workflows/pipeline.yml
vendored
@ -217,7 +217,7 @@ jobs:
|
||||
CROSS_TAGLIB_VERSION=${{ env.CROSS_TAGLIB_VERSION }}
|
||||
|
||||
- name: Upload Binaries
|
||||
uses: actions/upload-artifact@v4
|
||||
uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: navidrome-${{ env.PLATFORM }}
|
||||
path: ./output
|
||||
@ -248,7 +248,7 @@ jobs:
|
||||
touch "/tmp/digests/${digest#sha256:}"
|
||||
|
||||
- name: Upload digest
|
||||
uses: actions/upload-artifact@v4
|
||||
uses: actions/upload-artifact@v5
|
||||
if: env.IS_LINUX == 'true' && env.IS_DOCKER_PUSH_CONFIGURED == 'true' && env.IS_ARMV5 == 'false'
|
||||
with:
|
||||
name: digests-${{ env.PLATFORM }}
|
||||
@ -267,7 +267,7 @@ jobs:
|
||||
- uses: actions/checkout@v5
|
||||
|
||||
- name: Download digests
|
||||
uses: actions/download-artifact@v5
|
||||
uses: actions/download-artifact@v6
|
||||
with:
|
||||
path: /tmp/digests
|
||||
pattern: digests-*
|
||||
@ -320,7 +320,7 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
|
||||
- uses: actions/download-artifact@v5
|
||||
- uses: actions/download-artifact@v6
|
||||
with:
|
||||
path: ./binaries
|
||||
pattern: navidrome-windows*
|
||||
@ -339,7 +339,7 @@ jobs:
|
||||
du -h binaries/msi/*.msi
|
||||
|
||||
- name: Upload MSI files
|
||||
uses: actions/upload-artifact@v4
|
||||
uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: navidrome-windows-installers
|
||||
path: binaries/msi/*.msi
|
||||
@ -357,7 +357,7 @@ jobs:
|
||||
fetch-depth: 0
|
||||
fetch-tags: true
|
||||
|
||||
- uses: actions/download-artifact@v5
|
||||
- uses: actions/download-artifact@v6
|
||||
with:
|
||||
path: ./binaries
|
||||
pattern: navidrome-*
|
||||
@ -383,7 +383,7 @@ jobs:
|
||||
rm ./dist/*.tar.gz ./dist/*.zip
|
||||
|
||||
- name: Upload all-packages artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: packages
|
||||
path: dist/navidrome_0*
|
||||
@ -406,13 +406,13 @@ jobs:
|
||||
item: ${{ fromJson(needs.release.outputs.package_list) }}
|
||||
steps:
|
||||
- name: Download all-packages artifact
|
||||
uses: actions/download-artifact@v5
|
||||
uses: actions/download-artifact@v6
|
||||
with:
|
||||
name: packages
|
||||
path: ./dist
|
||||
|
||||
- name: Upload all-packages artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: navidrome_linux_${{ matrix.item }}
|
||||
path: dist/navidrome_0*_linux_${{ matrix.item }}
|
||||
|
||||
@ -98,6 +98,7 @@ type configOptions struct {
|
||||
PID pidOptions `json:",omitzero"`
|
||||
Inspect inspectOptions `json:",omitzero"`
|
||||
Subsonic subsonicOptions `json:",omitzero"`
|
||||
SQLite sqliteOptions `json:",omitzero"`
|
||||
LastFM lastfmOptions `json:",omitzero"`
|
||||
Spotify spotifyOptions `json:",omitzero"`
|
||||
Deezer deezerOptions `json:",omitzero"`
|
||||
|
||||
20
conf/sqlite_options.go
Normal file
20
conf/sqlite_options.go
Normal file
@ -0,0 +1,20 @@
|
||||
package conf
|
||||
|
||||
// sqliteOptions configures SQLite database behavior
|
||||
type sqliteOptions struct {
|
||||
// JournalMode sets the SQLite journal mode (WAL, DELETE, etc)
|
||||
// Default: WAL - provides better concurrency but may not work on network filesystems
|
||||
JournalMode string `json:",omitzero"`
|
||||
|
||||
// BusyTimeout sets how long SQLite should wait for locks to clear (milliseconds)
|
||||
// Default: 5000 - waits up to 5 seconds before returning "database is locked"
|
||||
BusyTimeout int `json:",omitzero"`
|
||||
|
||||
// SyncMode controls how aggressively SQLite writes to disk
|
||||
// Default: NORMAL - good balance of safety and performance
|
||||
SyncMode string `json:",omitzero"`
|
||||
|
||||
// MaxConnections limits concurrent database connections
|
||||
// Default: 0 (uses max(4, runtime.NumCPU()))
|
||||
MaxConnections int `json:",omitzero"`
|
||||
}
|
||||
311
contrib/grafana/sqlite-dashboard.json
Normal file
311
contrib/grafana/sqlite-dashboard.json
Normal file
@ -0,0 +1,311 @@
|
||||
{
|
||||
"annotations": {
|
||||
"list": []
|
||||
},
|
||||
"editable": true,
|
||||
"gnetId": null,
|
||||
"graphTooltip": 0,
|
||||
"id": null,
|
||||
"links": [],
|
||||
"panels": [
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "Prometheus",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"fill": 1,
|
||||
"fillGradient": 0,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"hiddenSeries": false,
|
||||
"id": 2,
|
||||
"legend": {
|
||||
"avg": false,
|
||||
"current": false,
|
||||
"max": false,
|
||||
"min": false,
|
||||
"show": true,
|
||||
"total": false,
|
||||
"values": false
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 1,
|
||||
"nullPointMode": "null",
|
||||
"options": {
|
||||
"alertThreshold": true
|
||||
},
|
||||
"percentage": false,
|
||||
"pluginVersion": "7.4.0",
|
||||
"pointradius": 2,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"stack": false,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "rate(sqlite_lock_wait_duration_seconds_sum[5m])",
|
||||
"interval": "",
|
||||
"legendFormat": "{{operation}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeRegions": [],
|
||||
"timeShift": null,
|
||||
"title": "SQLite Lock Wait Duration (5m rate)",
|
||||
"tooltip": {
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "individual"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"format": "s",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
}
|
||||
],
|
||||
"yaxis": {
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "Prometheus",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"fill": 1,
|
||||
"fillGradient": 0,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 0
|
||||
},
|
||||
"hiddenSeries": false,
|
||||
"id": 4,
|
||||
"legend": {
|
||||
"avg": false,
|
||||
"current": false,
|
||||
"max": false,
|
||||
"min": false,
|
||||
"show": true,
|
||||
"total": false,
|
||||
"values": false
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 1,
|
||||
"nullPointMode": "null",
|
||||
"options": {
|
||||
"alertThreshold": true
|
||||
},
|
||||
"percentage": false,
|
||||
"pluginVersion": "7.4.0",
|
||||
"pointradius": 2,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"stack": false,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "rate(sqlite_lock_errors_total[5m])",
|
||||
"interval": "",
|
||||
"legendFormat": "{{operation}} - {{type}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeRegions": [],
|
||||
"timeShift": null,
|
||||
"title": "SQLite Lock Errors (5m rate)",
|
||||
"tooltip": {
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "individual"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
}
|
||||
],
|
||||
"yaxis": {
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "Prometheus",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"fill": 1,
|
||||
"fillGradient": 0,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 8
|
||||
},
|
||||
"hiddenSeries": false,
|
||||
"id": 6,
|
||||
"legend": {
|
||||
"avg": false,
|
||||
"current": false,
|
||||
"max": false,
|
||||
"min": false,
|
||||
"show": true,
|
||||
"total": false,
|
||||
"values": false
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 1,
|
||||
"nullPointMode": "null",
|
||||
"options": {
|
||||
"alertThreshold": true
|
||||
},
|
||||
"percentage": false,
|
||||
"pluginVersion": "7.4.0",
|
||||
"pointradius": 2,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"stack": false,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "rate(sqlite_operation_retries_total[5m])",
|
||||
"interval": "",
|
||||
"legendFormat": "{{operation}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeRegions": [],
|
||||
"timeShift": null,
|
||||
"title": "SQLite Operation Retries (5m rate)",
|
||||
"tooltip": {
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "individual"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
}
|
||||
],
|
||||
"yaxis": {
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
}
|
||||
],
|
||||
"schemaVersion": 27,
|
||||
"style": "dark",
|
||||
"tags": ["navidrome", "sqlite"],
|
||||
"templating": {
|
||||
"list": []
|
||||
},
|
||||
"time": {
|
||||
"from": "now-6h",
|
||||
"to": "now"
|
||||
},
|
||||
"timepicker": {},
|
||||
"timezone": "",
|
||||
"title": "Navidrome SQLite Monitoring",
|
||||
"version": 1
|
||||
}
|
||||
53
core/metrics/sqlite.go
Normal file
53
core/metrics/sqlite.go
Normal file
@ -0,0 +1,53 @@
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
)
|
||||
|
||||
var (
|
||||
sqliteLockWaitDuration = prometheus.NewHistogramVec(
|
||||
prometheus.HistogramOpts{
|
||||
Name: "sqlite_lock_wait_duration_seconds",
|
||||
Help: "Time spent waiting for SQLite locks to be released",
|
||||
Buckets: []float64{.005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10},
|
||||
},
|
||||
[]string{"operation"},
|
||||
)
|
||||
|
||||
sqliteLockErrors = prometheus.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Name: "sqlite_lock_errors_total",
|
||||
Help: "Number of SQLite lock-related errors",
|
||||
},
|
||||
[]string{"operation", "type"},
|
||||
)
|
||||
|
||||
sqliteRetries = prometheus.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Name: "sqlite_operation_retries_total",
|
||||
Help: "Number of retried SQLite operations",
|
||||
},
|
||||
[]string{"operation"},
|
||||
)
|
||||
)
|
||||
|
||||
func init() {
|
||||
prometheus.MustRegister(sqliteLockWaitDuration)
|
||||
prometheus.MustRegister(sqliteLockErrors)
|
||||
prometheus.MustRegister(sqliteRetries)
|
||||
}
|
||||
|
||||
// ObserveSQLiteLockWait records the duration spent waiting for a lock
|
||||
func ObserveSQLiteLockWait(operation string, duration float64) {
|
||||
sqliteLockWaitDuration.WithLabelValues(operation).Observe(duration)
|
||||
}
|
||||
|
||||
// IncrementSQLiteLockError increments the counter for lock-related errors
|
||||
func IncrementSQLiteLockError(operation, errType string) {
|
||||
sqliteLockErrors.WithLabelValues(operation, errType).Inc()
|
||||
}
|
||||
|
||||
// IncrementSQLiteRetry increments the counter for operation retries
|
||||
func IncrementSQLiteRetry(operation string) {
|
||||
sqliteRetries.WithLabelValues(operation).Inc()
|
||||
}
|
||||
@ -13,6 +13,7 @@ import (
|
||||
"github.com/navidrome/navidrome/model"
|
||||
. "github.com/navidrome/navidrome/utils/gg"
|
||||
"github.com/navidrome/navidrome/utils/slice"
|
||||
"github.com/navidrome/navidrome/utils/str"
|
||||
)
|
||||
|
||||
type Share interface {
|
||||
@ -119,9 +120,8 @@ func (r *shareRepositoryWrapper) Save(entity interface{}) (string, error) {
|
||||
log.Error(r.ctx, "Invalid Resource ID", "id", firstId)
|
||||
return "", model.ErrNotFound
|
||||
}
|
||||
if len(s.Contents) > 30 {
|
||||
s.Contents = s.Contents[:26] + "..."
|
||||
}
|
||||
|
||||
s.Contents = str.TruncateRunes(s.Contents, 30, "...")
|
||||
|
||||
id, err = r.Persistable.Save(s)
|
||||
return id, err
|
||||
|
||||
@ -38,6 +38,38 @@ var _ = Describe("Share", func() {
|
||||
Expect(id).ToNot(BeEmpty())
|
||||
Expect(entity.ID).To(Equal(id))
|
||||
})
|
||||
|
||||
It("does not truncate ASCII labels shorter than 30 characters", func() {
|
||||
_ = ds.MediaFile(ctx).Put(&model.MediaFile{ID: "456", Title: "Example Media File"})
|
||||
entity := &model.Share{Description: "test", ResourceIDs: "456"}
|
||||
_, err := repo.Save(entity)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(entity.Contents).To(Equal("Example Media File"))
|
||||
})
|
||||
|
||||
It("truncates ASCII labels longer than 30 characters", func() {
|
||||
_ = ds.MediaFile(ctx).Put(&model.MediaFile{ID: "789", Title: "Example Media File But The Title Is Really Long For Testing Purposes"})
|
||||
entity := &model.Share{Description: "test", ResourceIDs: "789"}
|
||||
_, err := repo.Save(entity)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(entity.Contents).To(Equal("Example Media File But The ..."))
|
||||
})
|
||||
|
||||
It("does not truncate CJK labels shorter than 30 runes", func() {
|
||||
_ = ds.MediaFile(ctx).Put(&model.MediaFile{ID: "456", Title: "青春コンプレックス"})
|
||||
entity := &model.Share{Description: "test", ResourceIDs: "456"}
|
||||
_, err := repo.Save(entity)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(entity.Contents).To(Equal("青春コンプレックス"))
|
||||
})
|
||||
|
||||
It("truncates CJK labels longer than 30 runes", func() {
|
||||
_ = ds.MediaFile(ctx).Put(&model.MediaFile{ID: "789", Title: "私の中の幻想的世界観及びその顕現を想起させたある現実での出来事に関する一考察"})
|
||||
entity := &model.Share{Description: "test", ResourceIDs: "789"}
|
||||
_, err := repo.Save(entity)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(entity.Contents).To(Equal("私の中の幻想的世界観及びその顕現を想起させたある現実で..."))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Update", func() {
|
||||
|
||||
27
db/db.go
27
db/db.go
@ -41,7 +41,32 @@ func Db() *sql.DB {
|
||||
}
|
||||
log.Debug("Opening DataBase", "dbPath", Path, "driver", Driver)
|
||||
db, err := sql.Open(Driver, Path)
|
||||
db.SetMaxOpenConns(max(4, runtime.NumCPU()))
|
||||
|
||||
maxConns := max(4, runtime.NumCPU())
|
||||
if conf.Server.SQLite.MaxConnections > 0 {
|
||||
maxConns = conf.Server.SQLite.MaxConnections
|
||||
}
|
||||
db.SetMaxOpenConns(maxConns)
|
||||
|
||||
// Configure SQLite PRAGMAs to improve concurrency and reduce "database is locked" errors
|
||||
// WAL allows concurrent readers while a writer is active
|
||||
// busy_timeout tells SQLite how long to wait for a lock before error
|
||||
// Note: some network filesystems (NFS/CIFS) may not fully support WAL
|
||||
if conf.Server.SQLite.JournalMode != "" {
|
||||
if _, err := db.Exec("PRAGMA journal_mode=" + conf.Server.SQLite.JournalMode + ";"); err != nil {
|
||||
log.Error("Error setting PRAGMA journal_mode", "mode", conf.Server.SQLite.JournalMode, err)
|
||||
}
|
||||
}
|
||||
if conf.Server.SQLite.BusyTimeout > 0 {
|
||||
if _, err := db.Exec(fmt.Sprintf("PRAGMA busy_timeout=%d;", conf.Server.SQLite.BusyTimeout)); err != nil {
|
||||
log.Error("Error setting PRAGMA busy_timeout", "timeout", conf.Server.SQLite.BusyTimeout, err)
|
||||
}
|
||||
}
|
||||
if conf.Server.SQLite.SyncMode != "" {
|
||||
if _, err := db.Exec("PRAGMA synchronous=" + conf.Server.SQLite.SyncMode + ";"); err != nil {
|
||||
log.Error("Error setting PRAGMA synchronous", "mode", conf.Server.SQLite.SyncMode, err)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
log.Fatal("Error opening database", err)
|
||||
}
|
||||
|
||||
@ -0,0 +1,9 @@
|
||||
-- +goose Up
|
||||
-- +goose StatementBegin
|
||||
ALTER TABLE playqueue ADD COLUMN position_int integer;
|
||||
UPDATE playqueue SET position_int = CAST(position as INTEGER) ;
|
||||
ALTER TABLE playqueue DROP COLUMN position;
|
||||
ALTER TABLE playqueue RENAME COLUMN position_int TO position;
|
||||
-- +goose StatementEnd
|
||||
|
||||
-- +goose Down
|
||||
25
db/migrations/20251026_add_sqlite_settings.go
Normal file
25
db/migrations/20251026_add_sqlite_settings.go
Normal file
@ -0,0 +1,25 @@
|
||||
-- +migrate Up
|
||||
-- Enable WAL mode and set busy timeout if not already set
|
||||
UPDATE user_property SET value = 'WAL'
|
||||
WHERE name = 'sqlite_journal_mode' AND NOT EXISTS (
|
||||
SELECT 1 FROM user_property WHERE name = 'sqlite_journal_mode'
|
||||
);
|
||||
|
||||
INSERT INTO user_property (name, value)
|
||||
SELECT 'sqlite_busy_timeout', '5000'
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM user_property WHERE name = 'sqlite_busy_timeout'
|
||||
);
|
||||
|
||||
INSERT INTO user_property (name, value)
|
||||
SELECT 'sqlite_sync_mode', 'NORMAL'
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM user_property WHERE name = 'sqlite_sync_mode'
|
||||
);
|
||||
|
||||
-- +migrate Down
|
||||
DELETE FROM user_property WHERE name IN (
|
||||
'sqlite_journal_mode',
|
||||
'sqlite_busy_timeout',
|
||||
'sqlite_sync_mode'
|
||||
);
|
||||
47
docs/config/sqlite.md
Normal file
47
docs/config/sqlite.md
Normal file
@ -0,0 +1,47 @@
|
||||
# SQLite Configuration Options
|
||||
|
||||
The following SQLite-specific configuration options are available under the `SQLite` section:
|
||||
|
||||
## Basic Options
|
||||
|
||||
### JournalMode
|
||||
|
||||
- **Default:** `"WAL"`
|
||||
- **Options:** `"DELETE"`, `"TRUNCATE"`, `"PERSIST"`, `"MEMORY"`, `"WAL"`, `"OFF"`
|
||||
- **Description:** Controls how SQLite manages its journal file. WAL (Write-Ahead Logging) mode generally provides better concurrency and performance but may not work on some network filesystems.
|
||||
|
||||
### BusyTimeout
|
||||
|
||||
- **Default:** `5000` (milliseconds)
|
||||
- **Description:** How long SQLite should wait when the database is locked before returning a "database is locked" error. Higher values allow more concurrency but may impact responsiveness.
|
||||
|
||||
### SyncMode
|
||||
|
||||
- **Default:** `"NORMAL"`
|
||||
- **Options:** `"OFF"`, `"NORMAL"`, `"FULL"`, `"EXTRA"`
|
||||
- **Description:** Controls how aggressively SQLite writes data to disk. NORMAL provides a good balance between safety and performance.
|
||||
|
||||
### MaxConnections
|
||||
|
||||
- **Default:** `0` (uses max(4, number of CPU cores))
|
||||
- **Description:** Maximum number of concurrent database connections. Lower this if you experience "database is locked" errors, especially on network filesystems.
|
||||
|
||||
## Example Configuration
|
||||
|
||||
```toml
|
||||
[SQLite]
|
||||
JournalMode = "WAL" # Enable Write-Ahead Logging for better concurrency
|
||||
BusyTimeout = 5000 # Wait up to 5 seconds for locks to clear
|
||||
SyncMode = "NORMAL" # Good balance of durability and performance
|
||||
MaxConnections = 4 # Limit concurrent connections if needed
|
||||
```
|
||||
|
||||
## Network Filesystem Considerations
|
||||
|
||||
If your database is on a network filesystem (NFS, CIFS, etc.):
|
||||
|
||||
1. Consider moving the database to local storage
|
||||
2. If using network storage is required:
|
||||
- Set `JournalMode = "DELETE"`
|
||||
- Lower `MaxConnections` to reduce contention
|
||||
- Increase `BusyTimeout` for better reliability
|
||||
92
docs/config/sqlite_monitoring.md
Normal file
92
docs/config/sqlite_monitoring.md
Normal file
@ -0,0 +1,92 @@
|
||||
# SQLite Monitoring
|
||||
|
||||
Navidrome provides several Prometheus metrics to monitor SQLite database performance and lock contention:
|
||||
|
||||
## Available Metrics
|
||||
|
||||
### Lock Wait Duration
|
||||
|
||||
- **Metric**: `sqlite_lock_wait_duration_seconds`
|
||||
- **Type**: Histogram
|
||||
- **Labels**: `operation`
|
||||
- **Description**: Time spent waiting for SQLite locks to be released
|
||||
- **Use Case**: Identify operations that are frequently blocked by locks
|
||||
|
||||
### Lock Errors
|
||||
|
||||
- **Metric**: `sqlite_lock_errors_total`
|
||||
- **Type**: Counter
|
||||
- **Labels**: `operation`, `type`
|
||||
- **Description**: Number of SQLite lock-related errors
|
||||
- **Types**:
|
||||
- `retryable`: Temporary lock errors that can be retried
|
||||
- `non_retryable`: Fatal errors that cannot be retried
|
||||
|
||||
### Operation Retries
|
||||
|
||||
- **Metric**: `sqlite_operation_retries_total`
|
||||
- **Type**: Counter
|
||||
- **Labels**: `operation`
|
||||
- **Description**: Number of retried SQLite operations
|
||||
- **Use Case**: Track which operations require frequent retries
|
||||
|
||||
## Grafana Dashboard
|
||||
|
||||
A pre-configured Grafana dashboard is available at `contrib/grafana/sqlite-dashboard.json`.
|
||||
This dashboard provides visualizations for:
|
||||
|
||||
- Lock wait duration trends
|
||||
- Lock error rates by operation
|
||||
- Retry rates by operation
|
||||
|
||||
## Alerting Recommendations
|
||||
|
||||
Consider setting up alerts for:
|
||||
|
||||
1. High lock wait durations (> 1s)
|
||||
2. Increasing error rates
|
||||
3. Frequent retries on specific operations
|
||||
|
||||
Example Prometheus alert rules:
|
||||
|
||||
```yaml
|
||||
groups:
|
||||
- name: SQLiteAlerts
|
||||
rules:
|
||||
- alert: SQLiteLongLockWaits
|
||||
expr: rate(sqlite_lock_wait_duration_seconds_sum[5m]) > 1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
description: "SQLite operations are waiting long for locks"
|
||||
|
||||
- alert: SQLiteHighErrorRate
|
||||
expr: rate(sqlite_lock_errors_total[5m]) > 0.1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
description: "High rate of SQLite lock errors"
|
||||
```
|
||||
|
||||
## Troubleshooting with Metrics
|
||||
|
||||
If you see:
|
||||
|
||||
1. High lock wait durations:
|
||||
|
||||
- Consider adjusting `SQLite.BusyTimeout`
|
||||
- Review concurrent operations
|
||||
- Check if database is on network storage
|
||||
|
||||
2. Many retryable errors:
|
||||
|
||||
- Increase `SQLite.BusyTimeout`
|
||||
- Consider reducing `SQLite.MaxConnections`
|
||||
- Enable WAL mode if not using network storage
|
||||
|
||||
3. High retry rates:
|
||||
- Review operations causing contention
|
||||
- Consider batching updates
|
||||
- Check for long-running transactions
|
||||
13
go.mod
13
go.mod
@ -1,9 +1,13 @@
|
||||
module github.com/navidrome/navidrome
|
||||
|
||||
go 1.25.3
|
||||
go 1.25.4
|
||||
|
||||
// Fork to fix https://github.com/navidrome/navidrome/pull/3254
|
||||
replace github.com/dhowden/tag v0.0.0-20240417053706-3d75831295e8 => github.com/deluan/tag v0.0.0-20241002021117-dfe5e6ea396d
|
||||
replace (
|
||||
// Fork to fix https://github.com/navidrome/navidrome/issues/3254
|
||||
github.com/dhowden/tag v0.0.0-20240417053706-3d75831295e8 => github.com/deluan/tag v0.0.0-20241002021117-dfe5e6ea396d
|
||||
// Using version from main that fixes https://github.com/navidrome/navidrome/issues/4396
|
||||
github.com/tetratelabs/wazero v1.9.0 => github.com/tetratelabs/wazero v0.0.0-20251106165119-514cdb337684
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/Masterminds/squirrel v1.5.4
|
||||
@ -43,7 +47,7 @@ require (
|
||||
github.com/mattn/go-sqlite3 v1.14.32
|
||||
github.com/microcosm-cc/bluemonday v1.0.27
|
||||
github.com/mileusna/useragent v1.3.5
|
||||
github.com/onsi/ginkgo/v2 v2.27.1
|
||||
github.com/onsi/ginkgo/v2 v2.27.2
|
||||
github.com/onsi/gomega v1.38.2
|
||||
github.com/pelletier/go-toml/v2 v2.2.4
|
||||
github.com/pocketbase/dbx v1.11.0
|
||||
@ -124,7 +128,6 @@ require (
|
||||
github.com/stretchr/objx v0.5.2 // indirect
|
||||
github.com/subosito/gotenv v1.6.0 // indirect
|
||||
github.com/zeebo/xxh3 v1.0.2 // indirect
|
||||
go.uber.org/automaxprocs v1.6.0 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.2 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
|
||||
12
go.sum
12
go.sum
@ -186,8 +186,8 @@ github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdh
|
||||
github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
|
||||
github.com/ogier/pflag v0.0.1 h1:RW6JSWSu/RkSatfcLtogGfFgpim5p7ARQ10ECk5O750=
|
||||
github.com/ogier/pflag v0.0.1/go.mod h1:zkFki7tvTa0tafRvTBIZTvzYyAu6kQhPZFnshFFPE+g=
|
||||
github.com/onsi/ginkgo/v2 v2.27.1 h1:0LJC8MpUSQnfnp4n/3W3GdlmJP3ENGF0ZPzjQGLPP7s=
|
||||
github.com/onsi/ginkgo/v2 v2.27.1/go.mod h1:wmy3vCqiBjirARfVhAqFpYt8uvX0yaFe+GudAqqcCqA=
|
||||
github.com/onsi/ginkgo/v2 v2.27.2 h1:LzwLj0b89qtIy6SSASkzlNvX6WktqurSHwkk2ipF/Ns=
|
||||
github.com/onsi/ginkgo/v2 v2.27.2/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo=
|
||||
github.com/onsi/gomega v1.38.2 h1:eZCjf2xjZAqe+LeWvKb5weQ+NcPwX84kqJ0cZNxok2A=
|
||||
github.com/onsi/gomega v1.38.2/go.mod h1:W2MJcYxRGV63b418Ai34Ud0hEdTVXq9NW9+Sx6uXf3k=
|
||||
github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
|
||||
@ -201,8 +201,6 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/pocketbase/dbx v1.11.0 h1:LpZezioMfT3K4tLrqA55wWFw1EtH1pM4tzSVa7kgszU=
|
||||
github.com/pocketbase/dbx v1.11.0/go.mod h1:xXRCIAKTHMgUCyCKZm55pUOdvFziJjQfXaWKhu2vhMs=
|
||||
github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g=
|
||||
github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U=
|
||||
github.com/pressly/goose/v3 v3.26.0 h1:KJakav68jdH0WDvoAcj8+n61WqOIaPGgH0bJWS6jpmM=
|
||||
github.com/pressly/goose/v3 v3.26.0/go.mod h1:4hC1KrritdCxtuFsqgs1R4AU5bWtTAf+cnWvfhf2DNY=
|
||||
github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
|
||||
@ -267,8 +265,8 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
|
||||
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
|
||||
github.com/tetratelabs/wazero v1.9.0 h1:IcZ56OuxrtaEz8UYNRHBrUa9bYeX9oVY93KspZZBf/I=
|
||||
github.com/tetratelabs/wazero v1.9.0/go.mod h1:TSbcXCfFP0L2FGkRPxHphadXPjo1T6W+CseNNY7EkjM=
|
||||
github.com/tetratelabs/wazero v0.0.0-20251106165119-514cdb337684 h1:ugT1JTRsK1Jhn95BWilCugyZ1Svsyxm9xSiflOa2e7E=
|
||||
github.com/tetratelabs/wazero v0.0.0-20251106165119-514cdb337684/go.mod h1:DRm5twOQ5Gr1AoEdSi0CLjDQF1J9ZAuyqFIjl1KKfQU=
|
||||
github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY=
|
||||
github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
|
||||
github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
|
||||
@ -286,8 +284,6 @@ github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
|
||||
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
|
||||
github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
|
||||
github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
|
||||
go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs=
|
||||
go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8=
|
||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
|
||||
|
||||
@ -55,6 +55,7 @@ var _ = Describe("AlbumRepository", func() {
|
||||
It("returns all records sorted", func() {
|
||||
Expect(GetAll(model.QueryOptions{Sort: "name"})).To(Equal(model.Albums{
|
||||
albumAbbeyRoad,
|
||||
albumMultiDisc,
|
||||
albumRadioactivity,
|
||||
albumSgtPeppers,
|
||||
}))
|
||||
@ -64,6 +65,7 @@ var _ = Describe("AlbumRepository", func() {
|
||||
Expect(GetAll(model.QueryOptions{Sort: "name", Order: "desc"})).To(Equal(model.Albums{
|
||||
albumSgtPeppers,
|
||||
albumRadioactivity,
|
||||
albumMultiDisc,
|
||||
albumAbbeyRoad,
|
||||
}))
|
||||
})
|
||||
|
||||
@ -38,7 +38,7 @@ var _ = Describe("MediaRepository", func() {
|
||||
})
|
||||
|
||||
It("counts the number of mediafiles in the DB", func() {
|
||||
Expect(mr.CountAll()).To(Equal(int64(6)))
|
||||
Expect(mr.CountAll()).To(Equal(int64(10)))
|
||||
})
|
||||
|
||||
It("returns songs ordered by lyrics with a specific title/artist", func() {
|
||||
|
||||
@ -69,10 +69,12 @@ var (
|
||||
albumSgtPeppers = al(model.Album{ID: "101", Name: "Sgt Peppers", AlbumArtist: "The Beatles", OrderAlbumName: "sgt peppers", AlbumArtistID: "3", EmbedArtPath: p("/beatles/1/sgt/a day.mp3"), SongCount: 1, MaxYear: 1967})
|
||||
albumAbbeyRoad = al(model.Album{ID: "102", Name: "Abbey Road", AlbumArtist: "The Beatles", OrderAlbumName: "abbey road", AlbumArtistID: "3", EmbedArtPath: p("/beatles/1/come together.mp3"), SongCount: 1, MaxYear: 1969})
|
||||
albumRadioactivity = al(model.Album{ID: "103", Name: "Radioactivity", AlbumArtist: "Kraftwerk", OrderAlbumName: "radioactivity", AlbumArtistID: "2", EmbedArtPath: p("/kraft/radio/radio.mp3"), SongCount: 2})
|
||||
albumMultiDisc = al(model.Album{ID: "104", Name: "Multi Disc Album", AlbumArtist: "Test Artist", OrderAlbumName: "multi disc album", AlbumArtistID: "1", EmbedArtPath: p("/test/multi/disc1/track1.mp3"), SongCount: 4})
|
||||
testAlbums = model.Albums{
|
||||
albumSgtPeppers,
|
||||
albumAbbeyRoad,
|
||||
albumRadioactivity,
|
||||
albumMultiDisc,
|
||||
}
|
||||
)
|
||||
|
||||
@ -94,13 +96,22 @@ var (
|
||||
Lyrics: `[{"lang":"xxx","line":[{"value":"This is a set of lyrics"}],"synced":false}]`,
|
||||
})
|
||||
songAntenna2 = mf(model.MediaFile{ID: "1006", Title: "Antenna", ArtistID: "2", Artist: "Kraftwerk", AlbumID: "103"})
|
||||
testSongs = model.MediaFiles{
|
||||
// Multi-disc album tracks (intentionally out of order to test sorting)
|
||||
songDisc2Track11 = mf(model.MediaFile{ID: "2001", Title: "Disc 2 Track 11", ArtistID: "1", Artist: "Test Artist", AlbumID: "104", Album: "Multi Disc Album", DiscNumber: 2, TrackNumber: 11, Path: p("/test/multi/disc2/track11.mp3"), OrderAlbumName: "multi disc album", OrderArtistName: "test artist"})
|
||||
songDisc1Track01 = mf(model.MediaFile{ID: "2002", Title: "Disc 1 Track 1", ArtistID: "1", Artist: "Test Artist", AlbumID: "104", Album: "Multi Disc Album", DiscNumber: 1, TrackNumber: 1, Path: p("/test/multi/disc1/track1.mp3"), OrderAlbumName: "multi disc album", OrderArtistName: "test artist"})
|
||||
songDisc2Track01 = mf(model.MediaFile{ID: "2003", Title: "Disc 2 Track 1", ArtistID: "1", Artist: "Test Artist", AlbumID: "104", Album: "Multi Disc Album", DiscNumber: 2, TrackNumber: 1, Path: p("/test/multi/disc2/track1.mp3"), OrderAlbumName: "multi disc album", OrderArtistName: "test artist"})
|
||||
songDisc1Track02 = mf(model.MediaFile{ID: "2004", Title: "Disc 1 Track 2", ArtistID: "1", Artist: "Test Artist", AlbumID: "104", Album: "Multi Disc Album", DiscNumber: 1, TrackNumber: 2, Path: p("/test/multi/disc1/track2.mp3"), OrderAlbumName: "multi disc album", OrderArtistName: "test artist"})
|
||||
testSongs = model.MediaFiles{
|
||||
songDayInALife,
|
||||
songComeTogether,
|
||||
songRadioactivity,
|
||||
songAntenna,
|
||||
songAntennaWithLyrics,
|
||||
songAntenna2,
|
||||
songDisc2Track11,
|
||||
songDisc1Track01,
|
||||
songDisc2Track01,
|
||||
songDisc1Track02,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@ -124,7 +124,13 @@ func (r *playlistRepository) Put(p *model.Playlist) error {
|
||||
}
|
||||
pls.UpdatedAt = time.Now()
|
||||
|
||||
id, err := r.put(pls.ID, pls)
|
||||
var id string
|
||||
err := RetryWithBackoff(r.ctx, "playlist_put", func() error {
|
||||
var putErr error
|
||||
id, putErr = r.put(pls.ID, pls)
|
||||
return putErr
|
||||
}, 3, 100*time.Millisecond, 2*time.Second)
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@ -219,4 +219,37 @@ var _ = Describe("PlaylistRepository", func() {
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Playlist Track Sorting", func() {
|
||||
var testPlaylistID string
|
||||
|
||||
AfterEach(func() {
|
||||
if testPlaylistID != "" {
|
||||
Expect(repo.Delete(testPlaylistID)).To(BeNil())
|
||||
testPlaylistID = ""
|
||||
}
|
||||
})
|
||||
|
||||
It("sorts tracks correctly by album (disc and track number)", func() {
|
||||
By("creating a playlist with multi-disc album tracks in arbitrary order")
|
||||
newPls := model.Playlist{Name: "Multi-Disc Test", OwnerID: "userid"}
|
||||
// Add tracks in intentionally scrambled order
|
||||
newPls.AddMediaFilesByID([]string{"2001", "2002", "2003", "2004"})
|
||||
Expect(repo.Put(&newPls)).To(Succeed())
|
||||
testPlaylistID = newPls.ID
|
||||
|
||||
By("retrieving tracks sorted by album")
|
||||
tracksRepo := repo.Tracks(newPls.ID, false)
|
||||
tracks, err := tracksRepo.GetAll(model.QueryOptions{Sort: "album", Order: "asc"})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
By("verifying tracks are sorted by disc number then track number")
|
||||
Expect(tracks).To(HaveLen(4))
|
||||
// Expected order: Disc 1 Track 1, Disc 1 Track 2, Disc 2 Track 1, Disc 2 Track 11
|
||||
Expect(tracks[0].MediaFileID).To(Equal("2002")) // Disc 1, Track 1
|
||||
Expect(tracks[1].MediaFileID).To(Equal("2004")) // Disc 1, Track 2
|
||||
Expect(tracks[2].MediaFileID).To(Equal("2003")) // Disc 2, Track 1
|
||||
Expect(tracks[3].MediaFileID).To(Equal("2001")) // Disc 2, Track 11
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@ -55,7 +55,7 @@ func (r *playlistRepository) Tracks(playlistId string, refreshSmartPlaylist bool
|
||||
"id": "playlist_tracks.id",
|
||||
"artist": "order_artist_name",
|
||||
"album_artist": "order_album_artist_name",
|
||||
"album": "order_album_name, order_album_artist_name",
|
||||
"album": "order_album_name, album_id, disc_number, track_number, order_artist_name, title",
|
||||
"title": "order_title",
|
||||
// To make sure these fields will be whitelisted
|
||||
"duration": "duration",
|
||||
|
||||
81
persistence/retry.go
Normal file
81
persistence/retry.go
Normal file
@ -0,0 +1,81 @@
|
||||
package persistence
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"math/rand"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/navidrome/navidrome/core/metrics"
|
||||
|
||||
"github.com/navidrome/navidrome/log"
|
||||
)
|
||||
|
||||
// RetryWithBackoff attempts an operation with exponential backoff
|
||||
// maxAttempts: maximum number of attempts (minimum 1)
|
||||
// initialDelay: delay before first retry
|
||||
// maxDelay: maximum delay between retries
|
||||
func RetryWithBackoff(ctx context.Context, operation string, op func() error, maxAttempts int, initialDelay, maxDelay time.Duration) error {
|
||||
var lastErr error
|
||||
delay := initialDelay
|
||||
startTime := time.Now()
|
||||
|
||||
for attempt := 1; attempt <= maxAttempts; attempt++ {
|
||||
err := op()
|
||||
if err == nil {
|
||||
if attempt > 1 {
|
||||
// Record successful retry
|
||||
metrics.IncrementSQLiteRetry(operation)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
lastErr = err
|
||||
if !isRetryableError(err) {
|
||||
// Record non-retryable error
|
||||
metrics.IncrementSQLiteLockError(operation, "non_retryable")
|
||||
return err
|
||||
}
|
||||
|
||||
metrics.IncrementSQLiteLockError(operation, "retryable")
|
||||
if attempt == maxAttempts {
|
||||
break
|
||||
}
|
||||
|
||||
// Use exponential backoff with jitter
|
||||
jitter := time.Duration(float64(delay) * (0.5 + rand.Float64())) // 50-150% of base delay
|
||||
if jitter > maxDelay {
|
||||
jitter = maxDelay
|
||||
}
|
||||
|
||||
log.Debug(ctx, "Retrying operation after error",
|
||||
"operation", operation,
|
||||
"attempt", attempt,
|
||||
"maxAttempts", maxAttempts,
|
||||
"delay", jitter,
|
||||
"error", err)
|
||||
|
||||
select {
|
||||
case <-time.After(jitter):
|
||||
metrics.ObserveSQLiteLockWait(operation, jitter.Seconds())
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
delay *= 2
|
||||
}
|
||||
|
||||
return lastErr
|
||||
}
|
||||
|
||||
func isRetryableError(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
errStr := err.Error()
|
||||
return strings.Contains(errStr, "database is locked") ||
|
||||
strings.Contains(errStr, "busy") ||
|
||||
errors.Is(err, sql.ErrConnDone)
|
||||
}
|
||||
@ -57,6 +57,7 @@ func NewUserRepository(ctx context.Context, db dbx.Builder) model.UserRepository
|
||||
r.db = db
|
||||
r.tableName = "user"
|
||||
r.registerModel(&model.User{}, map[string]filterFunc{
|
||||
"id": idFilter(r.tableName),
|
||||
"password": invalidFilter(ctx),
|
||||
"name": r.withTableName(startsWithFilter),
|
||||
})
|
||||
|
||||
@ -559,4 +559,15 @@ var _ = Describe("UserRepository", func() {
|
||||
Expect(user.Libraries[0].ID).To(Equal(1))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("filters", func() {
|
||||
It("qualifies id filter with table name", func() {
|
||||
r := repo.(*userRepository)
|
||||
qo := r.parseRestOptions(r.ctx, rest.QueryOptions{Filters: map[string]any{"id": "123"}})
|
||||
sel := r.selectUserWithLibraries(qo)
|
||||
query, _, err := r.toSQL(sel)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(query).To(ContainSubstring("user.id = {:p0}"))
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@ -95,7 +95,7 @@ export const formatFullDate = (date, locale) => {
|
||||
return new Date(date).toLocaleDateString(locale, options)
|
||||
}
|
||||
|
||||
export const formatNumber = (value) => {
|
||||
export const formatNumber = (value, locale) => {
|
||||
if (value === null || value === undefined) return '0'
|
||||
return value.toLocaleString()
|
||||
return value.toLocaleString(locale)
|
||||
}
|
||||
|
||||
@ -121,35 +121,35 @@ describe('formatDuration2', () => {
|
||||
|
||||
describe('formatNumber', () => {
|
||||
it('handles null and undefined values', () => {
|
||||
expect(formatNumber(null)).toEqual('0')
|
||||
expect(formatNumber(undefined)).toEqual('0')
|
||||
expect(formatNumber(null, 'en-CA')).toEqual('0')
|
||||
expect(formatNumber(undefined, 'en-CA')).toEqual('0')
|
||||
})
|
||||
|
||||
it('formats integers', () => {
|
||||
expect(formatNumber(0)).toEqual('0')
|
||||
expect(formatNumber(1)).toEqual('1')
|
||||
expect(formatNumber(123)).toEqual('123')
|
||||
expect(formatNumber(1000)).toEqual('1,000')
|
||||
expect(formatNumber(1234567)).toEqual('1,234,567')
|
||||
expect(formatNumber(0, 'en-CA')).toEqual('0')
|
||||
expect(formatNumber(1, 'en-CA')).toEqual('1')
|
||||
expect(formatNumber(123, 'en-CA')).toEqual('123')
|
||||
expect(formatNumber(1000, 'en-CA')).toEqual('1,000')
|
||||
expect(formatNumber(1234567, 'en-CA')).toEqual('1,234,567')
|
||||
})
|
||||
|
||||
it('formats decimal numbers', () => {
|
||||
expect(formatNumber(123.45)).toEqual('123.45')
|
||||
expect(formatNumber(1234.567)).toEqual('1,234.567')
|
||||
expect(formatNumber(123.45, 'en-CA')).toEqual('123.45')
|
||||
expect(formatNumber(1234.567, 'en-CA')).toEqual('1,234.567')
|
||||
})
|
||||
|
||||
it('formats negative numbers', () => {
|
||||
expect(formatNumber(-123)).toEqual('-123')
|
||||
expect(formatNumber(-1234)).toEqual('-1,234')
|
||||
expect(formatNumber(-123.45)).toEqual('-123.45')
|
||||
expect(formatNumber(-123, 'en-CA')).toEqual('-123')
|
||||
expect(formatNumber(-1234, 'en-CA')).toEqual('-1,234')
|
||||
expect(formatNumber(-123.45, 'en-CA')).toEqual('-123.45')
|
||||
})
|
||||
})
|
||||
|
||||
describe('formatFullDate', () => {
|
||||
it('format dates', () => {
|
||||
expect(formatFullDate('2011', 'en-US')).toEqual('2011')
|
||||
expect(formatFullDate('2011-06', 'en-US')).toEqual('Jun 2011')
|
||||
expect(formatFullDate('1985-01-01', 'en-US')).toEqual('Jan 1, 1985')
|
||||
expect(formatFullDate('2011', 'en-CA')).toEqual('2011')
|
||||
expect(formatFullDate('2011-06', 'en-CA')).toEqual('Jun 2011')
|
||||
expect(formatFullDate('1985-01-01', 'en-CA')).toEqual('Jan 1, 1985')
|
||||
expect(formatFullDate('199704')).toEqual('')
|
||||
})
|
||||
})
|
||||
|
||||
@ -2,6 +2,7 @@ package str
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
var utf8ToAscii = func() *strings.Replacer {
|
||||
@ -39,3 +40,25 @@ func LongestCommonPrefix(list []string) string {
|
||||
}
|
||||
return list[0]
|
||||
}
|
||||
|
||||
// TruncateRunes truncates a string to a maximum number of runes, adding a suffix if truncated.
|
||||
// The suffix is included in the rune count, so if maxRunes is 30 and suffix is "...", the actual
|
||||
// string content will be truncated to fit within the maxRunes limit including the suffix.
|
||||
func TruncateRunes(s string, maxRunes int, suffix string) string {
|
||||
if utf8.RuneCountInString(s) <= maxRunes {
|
||||
return s
|
||||
}
|
||||
|
||||
suffixRunes := utf8.RuneCountInString(suffix)
|
||||
truncateAt := maxRunes - suffixRunes
|
||||
if truncateAt < 0 {
|
||||
truncateAt = 0
|
||||
}
|
||||
|
||||
runes := []rune(s)
|
||||
if truncateAt >= len(runes) {
|
||||
return s + suffix
|
||||
}
|
||||
|
||||
return string(runes[:truncateAt]) + suffix
|
||||
}
|
||||
|
||||
@ -31,6 +31,72 @@ var _ = Describe("String Utils", func() {
|
||||
Expect(str.LongestCommonPrefix(albums)).To(Equal("/artist/album"))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("TruncateRunes", func() {
|
||||
It("returns string unchanged if under max runes", func() {
|
||||
Expect(str.TruncateRunes("hello", 10, "...")).To(Equal("hello"))
|
||||
})
|
||||
|
||||
It("returns string unchanged if exactly at max runes", func() {
|
||||
Expect(str.TruncateRunes("hello", 5, "...")).To(Equal("hello"))
|
||||
})
|
||||
|
||||
It("truncates and adds suffix when over max runes", func() {
|
||||
Expect(str.TruncateRunes("hello world", 8, "...")).To(Equal("hello..."))
|
||||
})
|
||||
|
||||
It("handles unicode characters correctly", func() {
|
||||
// 6 emoji characters, maxRunes=5, suffix="..." (3 runes)
|
||||
// So content gets 5-3=2 runes
|
||||
Expect(str.TruncateRunes("😀😁😂😃😄😅", 5, "...")).To(Equal("😀😁..."))
|
||||
})
|
||||
|
||||
It("handles multi-byte UTF-8 characters", func() {
|
||||
// Characters like é are single runes
|
||||
Expect(str.TruncateRunes("Café au Lait", 5, "...")).To(Equal("Ca..."))
|
||||
})
|
||||
|
||||
It("works with empty suffix", func() {
|
||||
Expect(str.TruncateRunes("hello world", 5, "")).To(Equal("hello"))
|
||||
})
|
||||
|
||||
It("accounts for suffix length in truncation", func() {
|
||||
// maxRunes=10, suffix="..." (3 runes) -> leaves 7 runes for content
|
||||
result := str.TruncateRunes("hello world this is long", 10, "...")
|
||||
Expect(result).To(Equal("hello w..."))
|
||||
// Verify total rune count is <= maxRunes
|
||||
runeCount := len([]rune(result))
|
||||
Expect(runeCount).To(BeNumerically("<=", 10))
|
||||
})
|
||||
|
||||
It("handles very long suffix gracefully", func() {
|
||||
// If suffix is longer than maxRunes, we still add it
|
||||
// but the content will be truncated to 0
|
||||
result := str.TruncateRunes("hello world", 5, "... (truncated)")
|
||||
// Result will be just the suffix (since truncateAt=0)
|
||||
Expect(result).To(Equal("... (truncated)"))
|
||||
})
|
||||
|
||||
It("handles empty string", func() {
|
||||
Expect(str.TruncateRunes("", 10, "...")).To(Equal(""))
|
||||
})
|
||||
|
||||
It("uses custom suffix", func() {
|
||||
// maxRunes=11, suffix=" [...]" (6 runes) -> content gets 5 runes
|
||||
// "hello world" is 11 runes exactly, so we need a longer string
|
||||
Expect(str.TruncateRunes("hello world extra", 11, " [...]")).To(Equal("hello [...]"))
|
||||
})
|
||||
|
||||
DescribeTable("truncates at rune boundaries (not byte boundaries)",
|
||||
func(input string, maxRunes int, suffix string, expected string) {
|
||||
Expect(str.TruncateRunes(input, maxRunes, suffix)).To(Equal(expected))
|
||||
},
|
||||
Entry("ASCII", "abcdefghij", 5, "...", "ab..."),
|
||||
Entry("Mixed ASCII and Unicode", "ab😀cd", 4, ".", "ab😀."),
|
||||
Entry("All emoji", "😀😁😂😃😄", 3, "…", "😀😁…"),
|
||||
Entry("Japanese", "こんにちは世界", 3, "…", "こん…"),
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
var testPaths = []string{
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user