Merge branch 'navidrome:master' into fix-taglib-issues-in-vscodedevcontainer/4749

This commit is contained in:
floatlesss 2025-11-29 20:15:22 +00:00 committed by GitHub
commit 382b80ccb4
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
133 changed files with 6586 additions and 597 deletions

View File

@ -25,7 +25,7 @@ jobs:
git_tag: ${{ steps.git-version.outputs.GIT_TAG }}
git_sha: ${{ steps.git-version.outputs.GIT_SHA }}
steps:
- uses: actions/checkout@v5
- uses: actions/checkout@v6
with:
fetch-depth: 0
fetch-tags: true
@ -63,7 +63,7 @@ jobs:
name: Lint Go code
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v5
- uses: actions/checkout@v6
- name: Download TagLib
uses: ./.github/actions/download-taglib
@ -71,7 +71,7 @@ jobs:
version: ${{ env.CROSS_TAGLIB_VERSION }}
- name: golangci-lint
uses: golangci/golangci-lint-action@v8
uses: golangci/golangci-lint-action@v9
with:
version: latest
problem-matchers: true
@ -93,7 +93,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v5
uses: actions/checkout@v6
- name: Download TagLib
uses: ./.github/actions/download-taglib
@ -114,7 +114,7 @@ jobs:
env:
NODE_OPTIONS: "--max_old_space_size=4096"
steps:
- uses: actions/checkout@v5
- uses: actions/checkout@v6
- uses: actions/setup-node@v6
with:
node-version: 24
@ -145,7 +145,7 @@ jobs:
name: Lint i18n files
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v5
- uses: actions/checkout@v6
- run: |
set -e
for file in resources/i18n/*.json; do
@ -191,7 +191,7 @@ jobs:
PLATFORM=$(echo ${{ matrix.platform }} | tr '/' '_')
echo "PLATFORM=$PLATFORM" >> $GITHUB_ENV
- uses: actions/checkout@v5
- uses: actions/checkout@v6
- name: Prepare Docker Buildx
uses: ./.github/actions/prepare-docker
@ -264,7 +264,7 @@ jobs:
env:
REGISTRY_IMAGE: ghcr.io/${{ github.repository }}
steps:
- uses: actions/checkout@v5
- uses: actions/checkout@v6
- name: Download digests
uses: actions/download-artifact@v6
@ -318,7 +318,7 @@ jobs:
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@v5
- uses: actions/checkout@v6
- uses: actions/download-artifact@v6
with:
@ -352,7 +352,7 @@ jobs:
outputs:
package_list: ${{ steps.set-package-list.outputs.package_list }}
steps:
- uses: actions/checkout@v5
- uses: actions/checkout@v6
with:
fetch-depth: 0
fetch-tags: true

View File

@ -8,7 +8,7 @@ jobs:
runs-on: ubuntu-latest
if: ${{ github.repository_owner == 'navidrome' }}
steps:
- uses: actions/checkout@v5
- uses: actions/checkout@v6
- name: Get updated translations
id: poeditor
env:

View File

@ -137,7 +137,6 @@ ENV ND_MUSICFOLDER=/music
ENV ND_DATAFOLDER=/data
ENV ND_CONFIGFILE=/data/navidrome.toml
ENV ND_PORT=4533
ENV GODEBUG="asyncpreemptoff=1"
RUN touch /.nddockerenv
EXPOSE ${ND_PORT}

View File

@ -16,7 +16,7 @@ DOCKER_TAG ?= deluan/navidrome:develop
# Taglib version to use in cross-compilation, from https://github.com/navidrome/cross-taglib
CROSS_TAGLIB_VERSION ?= 2.1.1-1
GOLANGCI_LINT_VERSION ?= v2.5.0
GOLANGCI_LINT_VERSION ?= v2.6.2
UI_SRC_FILES := $(shell find ui -type f -not -path "ui/build/*" -not -path "ui/node_modules/*")
@ -54,7 +54,7 @@ testall: test-race test-i18n test-js ##@Development Run Go and JS tests
.PHONY: testall
test-race: ##@Development Run Go tests with race detector
go test -tags netgo -race -shuffle=on ./...
go test -tags netgo -race -shuffle=on $(PKG)
.PHONY: test-race
test-js: ##@Development Run JS tests

View File

@ -346,7 +346,7 @@ func startPluginManager(ctx context.Context) func() error {
// TODO: Implement some struct tags to map flags to viper
func init() {
cobra.OnInitialize(func() {
conf.InitConfig(cfgFile)
conf.InitConfig(cfgFile, true)
})
rootCmd.PersistentFlags().StringVarP(&cfgFile, "configfile", "c", "", `config file (default "./navidrome.toml")`)

View File

@ -8,6 +8,7 @@ import (
"github.com/navidrome/navidrome/core"
"github.com/navidrome/navidrome/db"
"github.com/navidrome/navidrome/log"
"github.com/navidrome/navidrome/model"
"github.com/navidrome/navidrome/persistence"
"github.com/navidrome/navidrome/scanner"
"github.com/navidrome/navidrome/utils/pl"
@ -17,11 +18,13 @@ import (
var (
fullScan bool
subprocess bool
targets []string
)
func init() {
scanCmd.Flags().BoolVarP(&fullScan, "full", "f", false, "check all subfolders, ignoring timestamps")
scanCmd.Flags().BoolVarP(&subprocess, "subprocess", "", false, "run as subprocess (internal use)")
scanCmd.Flags().StringArrayVarP(&targets, "target", "t", []string{}, "list of libraryID:folderPath pairs, can be repeated (e.g., \"-t 1:Music/Rock -t 1:Music/Jazz -t 2:Classical\")")
rootCmd.AddCommand(scanCmd)
}
@ -68,7 +71,18 @@ func runScanner(ctx context.Context) {
ds := persistence.New(sqlDB)
pls := core.NewPlaylists(ds)
progress, err := scanner.CallScan(ctx, ds, pls, fullScan)
// Parse targets if provided
var scanTargets []model.ScanTarget
if len(targets) > 0 {
var err error
scanTargets, err = model.ParseTargets(targets)
if err != nil {
log.Fatal(ctx, "Failed to parse targets", err)
}
log.Info(ctx, "Scanning specific folders", "numTargets", len(scanTargets))
}
progress, err := scanner.CallScan(ctx, ds, pls, fullScan, scanTargets)
if err != nil {
log.Fatal(ctx, "Failed to scan", err)
}

View File

@ -69,9 +69,9 @@ func CreateNativeAPIRouter(ctx context.Context) *nativeapi.Router {
artworkArtwork := artwork.NewArtwork(dataStore, fileCache, fFmpeg, provider)
cacheWarmer := artwork.NewCacheWarmer(artworkArtwork, fileCache)
broker := events.GetBroker()
scannerScanner := scanner.New(ctx, dataStore, cacheWarmer, broker, playlists, metricsMetrics)
watcher := scanner.GetWatcher(dataStore, scannerScanner)
library := core.NewLibrary(dataStore, scannerScanner, watcher, broker)
modelScanner := scanner.New(ctx, dataStore, cacheWarmer, broker, playlists, metricsMetrics)
watcher := scanner.GetWatcher(dataStore, modelScanner)
library := core.NewLibrary(dataStore, modelScanner, watcher, broker)
maintenance := core.NewMaintenance(dataStore)
router := nativeapi.New(dataStore, share, playlists, insights, library, maintenance)
return router
@ -95,10 +95,10 @@ func CreateSubsonicAPIRouter(ctx context.Context) *subsonic.Router {
cacheWarmer := artwork.NewCacheWarmer(artworkArtwork, fileCache)
broker := events.GetBroker()
playlists := core.NewPlaylists(dataStore)
scannerScanner := scanner.New(ctx, dataStore, cacheWarmer, broker, playlists, metricsMetrics)
modelScanner := scanner.New(ctx, dataStore, cacheWarmer, broker, playlists, metricsMetrics)
playTracker := scrobbler.GetPlayTracker(dataStore, broker, manager)
playbackServer := playback.GetInstance(dataStore)
router := subsonic.New(dataStore, artworkArtwork, mediaStreamer, archiver, players, provider, scannerScanner, broker, playlists, playTracker, share, playbackServer, metricsMetrics)
router := subsonic.New(dataStore, artworkArtwork, mediaStreamer, archiver, players, provider, modelScanner, broker, playlists, playTracker, share, playbackServer, metricsMetrics)
return router
}
@ -150,7 +150,7 @@ func CreatePrometheus() metrics.Metrics {
return metricsMetrics
}
func CreateScanner(ctx context.Context) scanner.Scanner {
func CreateScanner(ctx context.Context) model.Scanner {
sqlDB := db.Db()
dataStore := persistence.New(sqlDB)
fileCache := artwork.GetImageCache()
@ -163,8 +163,8 @@ func CreateScanner(ctx context.Context) scanner.Scanner {
cacheWarmer := artwork.NewCacheWarmer(artworkArtwork, fileCache)
broker := events.GetBroker()
playlists := core.NewPlaylists(dataStore)
scannerScanner := scanner.New(ctx, dataStore, cacheWarmer, broker, playlists, metricsMetrics)
return scannerScanner
modelScanner := scanner.New(ctx, dataStore, cacheWarmer, broker, playlists, metricsMetrics)
return modelScanner
}
func CreateScanWatcher(ctx context.Context) scanner.Watcher {
@ -180,8 +180,8 @@ func CreateScanWatcher(ctx context.Context) scanner.Watcher {
cacheWarmer := artwork.NewCacheWarmer(artworkArtwork, fileCache)
broker := events.GetBroker()
playlists := core.NewPlaylists(dataStore)
scannerScanner := scanner.New(ctx, dataStore, cacheWarmer, broker, playlists, metricsMetrics)
watcher := scanner.GetWatcher(dataStore, scannerScanner)
modelScanner := scanner.New(ctx, dataStore, cacheWarmer, broker, playlists, metricsMetrics)
watcher := scanner.GetWatcher(dataStore, modelScanner)
return watcher
}
@ -202,7 +202,7 @@ func getPluginManager() plugins.Manager {
// wire_injectors.go:
var allProviders = wire.NewSet(core.Set, artwork.Set, server.New, subsonic.New, nativeapi.New, public.New, persistence.New, lastfm.NewRouter, listenbrainz.NewRouter, events.GetBroker, scanner.New, scanner.GetWatcher, plugins.GetManager, metrics.GetPrometheusInstance, db.Db, wire.Bind(new(agents.PluginLoader), new(plugins.Manager)), wire.Bind(new(scrobbler.PluginLoader), new(plugins.Manager)), wire.Bind(new(metrics.PluginLoader), new(plugins.Manager)), wire.Bind(new(core.Scanner), new(scanner.Scanner)), wire.Bind(new(core.Watcher), new(scanner.Watcher)))
var allProviders = wire.NewSet(core.Set, artwork.Set, server.New, subsonic.New, nativeapi.New, public.New, persistence.New, lastfm.NewRouter, listenbrainz.NewRouter, events.GetBroker, scanner.New, scanner.GetWatcher, plugins.GetManager, metrics.GetPrometheusInstance, db.Db, wire.Bind(new(agents.PluginLoader), new(plugins.Manager)), wire.Bind(new(scrobbler.PluginLoader), new(plugins.Manager)), wire.Bind(new(metrics.PluginLoader), new(plugins.Manager)), wire.Bind(new(core.Watcher), new(scanner.Watcher)))
func GetPluginManager(ctx context.Context) plugins.Manager {
manager := getPluginManager()

View File

@ -45,7 +45,6 @@ var allProviders = wire.NewSet(
wire.Bind(new(agents.PluginLoader), new(plugins.Manager)),
wire.Bind(new(scrobbler.PluginLoader), new(plugins.Manager)),
wire.Bind(new(metrics.PluginLoader), new(plugins.Manager)),
wire.Bind(new(core.Scanner), new(scanner.Scanner)),
wire.Bind(new(core.Watcher), new(scanner.Watcher)),
)
@ -103,7 +102,7 @@ func CreatePrometheus() metrics.Metrics {
))
}
func CreateScanner(ctx context.Context) scanner.Scanner {
func CreateScanner(ctx context.Context) model.Scanner {
panic(wire.Build(
allProviders,
))

View File

@ -125,11 +125,13 @@ type configOptions struct {
DevAlbumInfoTimeToLive time.Duration
DevExternalScanner bool
DevScannerThreads uint
DevSelectiveWatcher bool
DevInsightsInitialDelay time.Duration
DevEnablePlayerInsights bool
DevEnablePluginsInsights bool
DevPluginCompilationTimeout time.Duration
DevExternalArtistFetchMultiplier float64
DevOptimizeDB bool
}
type scannerOptions struct {
@ -175,7 +177,8 @@ type spotifyOptions struct {
}
type deezerOptions struct {
Enabled bool
Enabled bool
Language string
}
type listenBrainzOptions struct {
@ -425,7 +428,7 @@ func validatePurgeMissingOption() error {
}
}
if !valid {
err := fmt.Errorf("Invalid Scanner.PurgeMissing value: '%s'. Must be one of: %v", Server.Scanner.PurgeMissing, allowedValues)
err := fmt.Errorf("invalid Scanner.PurgeMissing value: '%s'. Must be one of: %v", Server.Scanner.PurgeMissing, allowedValues)
log.Error(err.Error())
Server.Scanner.PurgeMissing = consts.PurgeMissingNever
return err
@ -565,6 +568,7 @@ func setViperDefaults() {
viper.SetDefault("spotify.id", "")
viper.SetDefault("spotify.secret", "")
viper.SetDefault("deezer.enabled", true)
viper.SetDefault("deezer.language", "en")
viper.SetDefault("listenbrainz.enabled", true)
viper.SetDefault("listenbrainz.baseurl", "https://api.listenbrainz.org/1/")
viper.SetDefault("httpsecurityheaders.customframeoptionsvalue", "DENY")
@ -600,20 +604,27 @@ func setViperDefaults() {
viper.SetDefault("devalbuminfotimetolive", consts.AlbumInfoTimeToLive)
viper.SetDefault("devexternalscanner", true)
viper.SetDefault("devscannerthreads", 5)
viper.SetDefault("devselectivewatcher", true)
viper.SetDefault("devinsightsinitialdelay", consts.InsightsInitialDelay)
viper.SetDefault("devenableplayerinsights", true)
viper.SetDefault("devenablepluginsinsights", true)
viper.SetDefault("devplugincompilationtimeout", time.Minute)
viper.SetDefault("devexternalartistfetchmultiplier", 1.5)
viper.SetDefault("devoptimizedb", true)
}
func init() {
setViperDefaults()
}
func InitConfig(cfgFile string) {
func InitConfig(cfgFile string, loadEnvVars bool) {
codecRegistry := viper.NewCodecRegistry()
_ = codecRegistry.RegisterCodec("ini", ini.Codec{})
_ = codecRegistry.RegisterCodec("ini", ini.Codec{
LoadOptions: ini.LoadOptions{
UnescapeValueDoubleQuotes: true,
UnescapeValueCommentSymbols: true,
},
})
viper.SetOptions(viper.WithCodecRegistry(codecRegistry))
cfgFile = getConfigFile(cfgFile)
@ -627,10 +638,12 @@ func InitConfig(cfgFile string) {
}
_ = viper.BindEnv("port")
viper.SetEnvPrefix("ND")
replacer := strings.NewReplacer(".", "_")
viper.SetEnvKeyReplacer(replacer)
viper.AutomaticEnv()
if loadEnvVars {
viper.SetEnvPrefix("ND")
replacer := strings.NewReplacer(".", "_")
viper.SetEnvKeyReplacer(replacer)
viper.AutomaticEnv()
}
err := viper.ReadInConfig()
if viper.ConfigFileUsed() != "" && err != nil {

View File

@ -31,7 +31,7 @@ var _ = Describe("Configuration", func() {
filename := filepath.Join("testdata", "cfg."+format)
// Initialize config with the test file
conf.InitConfig(filename)
conf.InitConfig(filename, false)
// Load the configuration (with noConfigDump=true)
conf.Load(true)
@ -39,6 +39,7 @@ var _ = Describe("Configuration", func() {
Expect(conf.Server.MusicFolder).To(Equal(fmt.Sprintf("/%s/music", format)))
Expect(conf.Server.UIWelcomeMessage).To(Equal("Welcome " + format))
Expect(conf.Server.Tags["custom"].Aliases).To(Equal([]string{format, "test"}))
Expect(conf.Server.Tags["artist"].Split).To(Equal([]string{";"}))
// The config file used should be the one we created
Expect(conf.Server.ConfigFile).To(Equal(filename))

View File

@ -1,6 +1,7 @@
[default]
MusicFolder = /ini/music
UIWelcomeMessage = Welcome ini
UIWelcomeMessage = 'Welcome ini' ; Just a comment to test the LoadOptions
[Tags]
Custom.Aliases = ini,test
Custom.Aliases = ini,test
artist.Split = ";" # Should be able to read ; as a separator

View File

@ -2,6 +2,9 @@
"musicFolder": "/json/music",
"uiWelcomeMessage": "Welcome json",
"Tags": {
"artist": {
"split": ";"
},
"custom": {
"aliases": [
"json",

View File

@ -1,5 +1,7 @@
musicFolder = "/toml/music"
uiWelcomeMessage = "Welcome toml"
Tags.artist.Split = ';'
[Tags.custom]
aliases = ["toml", "test"]

View File

@ -1,6 +1,8 @@
musicFolder: "/yaml/music"
uiWelcomeMessage: "Welcome yaml"
Tags:
artist:
split: [";"]
custom:
aliases:
- yaml

View File

@ -87,7 +87,7 @@ func (a *Agents) getEnabledAgentNames() []enabledAgent {
} else if isPlugin {
validAgents = append(validAgents, enabledAgent{name: name, isPlugin: true})
} else {
log.Warn("Unknown agent ignored", "name", name)
log.Debug("Unknown agent ignored", "name", name)
}
}
return validAgents

View File

@ -1,6 +1,7 @@
package deezer
import (
bytes "bytes"
"context"
"encoding/json"
"errors"
@ -9,11 +10,14 @@ import (
"net/http"
"net/url"
"strconv"
"strings"
"github.com/microcosm-cc/bluemonday"
"github.com/navidrome/navidrome/log"
)
const apiBaseURL = "https://api.deezer.com"
const authBaseURL = "https://auth.deezer.com"
var (
ErrNotFound = errors.New("deezer: not found")
@ -25,10 +29,15 @@ type httpDoer interface {
type client struct {
httpDoer httpDoer
language string
jwt jwtToken
}
func newClient(hc httpDoer) *client {
return &client{hc}
func newClient(hc httpDoer, language string) *client {
return &client{
httpDoer: hc,
language: language,
}
}
func (c *client) searchArtists(ctx context.Context, name string, limit int) ([]Artist, error) {
@ -53,7 +62,7 @@ func (c *client) searchArtists(ctx context.Context, name string, limit int) ([]A
return results.Data, nil
}
func (c *client) makeRequest(req *http.Request, response interface{}) error {
func (c *client) makeRequest(req *http.Request, response any) error {
log.Trace(req.Context(), fmt.Sprintf("Sending Deezer %s request", req.Method), "url", req.URL)
resp, err := c.httpDoer.Do(req)
if err != nil {
@ -81,3 +90,129 @@ func (c *client) parseError(data []byte) error {
}
return fmt.Errorf("deezer error(%d): %s", deezerError.Error.Code, deezerError.Error.Message)
}
func (c *client) getRelatedArtists(ctx context.Context, artistID int) ([]Artist, error) {
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("%s/artist/%d/related", apiBaseURL, artistID), nil)
if err != nil {
return nil, err
}
var results RelatedArtists
err = c.makeRequest(req, &results)
if err != nil {
return nil, err
}
return results.Data, nil
}
func (c *client) getTopTracks(ctx context.Context, artistID int, limit int) ([]Track, error) {
params := url.Values{}
params.Add("limit", strconv.Itoa(limit))
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("%s/artist/%d/top", apiBaseURL, artistID), nil)
if err != nil {
return nil, err
}
req.URL.RawQuery = params.Encode()
var results TopTracks
err = c.makeRequest(req, &results)
if err != nil {
return nil, err
}
return results.Data, nil
}
const pipeAPIURL = "https://pipe.deezer.com/api"
var strictPolicy = bluemonday.StrictPolicy()
func (c *client) getArtistBio(ctx context.Context, artistID int) (string, error) {
jwt, err := c.getJWT(ctx)
if err != nil {
return "", fmt.Errorf("deezer: failed to get JWT: %w", err)
}
query := map[string]any{
"operationName": "ArtistBio",
"variables": map[string]any{
"artistId": strconv.Itoa(artistID),
},
"query": `query ArtistBio($artistId: String!) {
artist(artistId: $artistId) {
bio {
full
}
}
}`,
}
body, err := json.Marshal(query)
if err != nil {
return "", err
}
req, err := http.NewRequestWithContext(ctx, "POST", pipeAPIURL, bytes.NewReader(body))
if err != nil {
return "", err
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Accept-Language", c.language)
req.Header.Set("Authorization", "Bearer "+jwt)
log.Trace(ctx, "Fetching Deezer artist biography via GraphQL", "artistId", artistID, "language", c.language)
resp, err := c.httpDoer.Do(req)
if err != nil {
return "", err
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
return "", fmt.Errorf("deezer: failed to fetch biography: %s", resp.Status)
}
data, err := io.ReadAll(resp.Body)
if err != nil {
return "", err
}
type graphQLResponse struct {
Data struct {
Artist struct {
Bio struct {
Full string `json:"full"`
} `json:"bio"`
} `json:"artist"`
} `json:"data"`
Errors []struct {
Message string `json:"message"`
}
}
var result graphQLResponse
if err := json.Unmarshal(data, &result); err != nil {
return "", fmt.Errorf("deezer: failed to parse GraphQL response: %w", err)
}
if len(result.Errors) > 0 {
var errs []error
for m := range result.Errors {
errs = append(errs, errors.New(result.Errors[m].Message))
}
err := errors.Join(errs...)
return "", fmt.Errorf("deezer: GraphQL error: %w", err)
}
if result.Data.Artist.Bio.Full == "" {
return "", errors.New("deezer: biography not found")
}
return cleanBio(result.Data.Artist.Bio.Full), nil
}
func cleanBio(bio string) string {
bio = strings.ReplaceAll(bio, "</p>", "\n")
return strictPolicy.Sanitize(bio)
}

View File

@ -0,0 +1,101 @@
package deezer
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"sync"
"time"
"github.com/lestrrat-go/jwx/v2/jwt"
"github.com/navidrome/navidrome/log"
)
type jwtToken struct {
token string
expiresAt time.Time
mu sync.RWMutex
}
func (j *jwtToken) get() (string, bool) {
j.mu.RLock()
defer j.mu.RUnlock()
if time.Now().Before(j.expiresAt) {
return j.token, true
}
return "", false
}
func (j *jwtToken) set(token string, expiresIn time.Duration) {
j.mu.Lock()
defer j.mu.Unlock()
j.token = token
j.expiresAt = time.Now().Add(expiresIn)
}
func (c *client) getJWT(ctx context.Context) (string, error) {
// Check if we have a valid cached token
if token, valid := c.jwt.get(); valid {
return token, nil
}
// Fetch a new anonymous token
req, err := http.NewRequestWithContext(ctx, "GET", authBaseURL+"/login/anonymous?jo=p&rto=c", nil)
if err != nil {
return "", err
}
req.Header.Set("Accept", "application/json")
resp, err := c.httpDoer.Do(req)
if err != nil {
return "", err
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
return "", fmt.Errorf("deezer: failed to get JWT token: %s", resp.Status)
}
data, err := io.ReadAll(resp.Body)
if err != nil {
return "", err
}
type authResponse struct {
JWT string `json:"jwt"`
}
var result authResponse
if err := json.Unmarshal(data, &result); err != nil {
return "", fmt.Errorf("deezer: failed to parse auth response: %w", err)
}
if result.JWT == "" {
return "", errors.New("deezer: no JWT token in response")
}
// Parse JWT to get actual expiration time
token, err := jwt.ParseString(result.JWT, jwt.WithVerify(false), jwt.WithValidate(false))
if err != nil {
return "", fmt.Errorf("deezer: failed to parse JWT token: %w", err)
}
// Calculate TTL with a 1-minute buffer for clock skew and network delays
expiresAt := token.Expiration()
if expiresAt.IsZero() {
return "", errors.New("deezer: JWT token has no expiration time")
}
ttl := time.Until(expiresAt) - 1*time.Minute
if ttl <= 0 {
return "", errors.New("deezer: JWT token already expired or expires too soon")
}
c.jwt.set(result.JWT, ttl)
log.Trace(ctx, "Fetched new Deezer JWT token", "expiresAt", expiresAt, "ttl", ttl)
return result.JWT, nil
}

View File

@ -0,0 +1,293 @@
package deezer
import (
"bytes"
"context"
"fmt"
"io"
"net/http"
"sync"
"time"
"github.com/lestrrat-go/jwx/v2/jwt"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
var _ = Describe("JWT Authentication", func() {
var httpClient *fakeHttpClient
var client *client
var ctx context.Context
BeforeEach(func() {
httpClient = &fakeHttpClient{}
client = newClient(httpClient, "en")
ctx = context.Background()
})
Describe("getJWT", func() {
Context("with a valid JWT response", func() {
It("successfully fetches and caches a JWT token", func() {
testJWT := createTestJWT(5 * time.Minute)
httpClient.mock("https://auth.deezer.com/login/anonymous", http.Response{
StatusCode: 200,
Body: io.NopCloser(bytes.NewBufferString(fmt.Sprintf(`{"jwt":"%s"}`, testJWT))),
})
token, err := client.getJWT(ctx)
Expect(err).To(BeNil())
Expect(token).To(Equal(testJWT))
})
It("returns the cached token on subsequent calls", func() {
testJWT := createTestJWT(5 * time.Minute)
httpClient.mock("https://auth.deezer.com/login/anonymous", http.Response{
StatusCode: 200,
Body: io.NopCloser(bytes.NewBufferString(fmt.Sprintf(`{"jwt":"%s"}`, testJWT))),
})
// First call should fetch from API
token1, err := client.getJWT(ctx)
Expect(err).To(BeNil())
Expect(token1).To(Equal(testJWT))
Expect(httpClient.lastRequest.URL.Path).To(Equal("/login/anonymous"))
// Second call should return cached token without hitting API
httpClient.lastRequest = nil // Clear last request to verify no new request is made
token2, err := client.getJWT(ctx)
Expect(err).To(BeNil())
Expect(token2).To(Equal(testJWT))
Expect(httpClient.lastRequest).To(BeNil()) // No new request made
})
It("parses the JWT expiration time correctly", func() {
expectedExpiration := time.Now().Add(5 * time.Minute)
testToken, err := jwt.NewBuilder().
Expiration(expectedExpiration).
Build()
Expect(err).To(BeNil())
testJWT, err := jwt.Sign(testToken, jwt.WithInsecureNoSignature())
Expect(err).To(BeNil())
httpClient.mock("https://auth.deezer.com/login/anonymous", http.Response{
StatusCode: 200,
Body: io.NopCloser(bytes.NewBufferString(fmt.Sprintf(`{"jwt":"%s"}`, string(testJWT)))),
})
token, err := client.getJWT(ctx)
Expect(err).To(BeNil())
Expect(token).ToNot(BeEmpty())
// Verify the token is cached until close to expiration
// The cache should expire 1 minute before the JWT expires
expectedCacheExpiry := expectedExpiration.Add(-1 * time.Minute)
Expect(client.jwt.expiresAt).To(BeTemporally("~", expectedCacheExpiry, 2*time.Second))
})
})
Context("with JWT tokens that expire soon", func() {
It("rejects tokens that expire in less than 1 minute", func() {
// Create a token that expires in 30 seconds (less than 1-minute buffer)
testJWT := createTestJWT(30 * time.Second)
httpClient.mock("https://auth.deezer.com/login/anonymous", http.Response{
StatusCode: 200,
Body: io.NopCloser(bytes.NewBufferString(fmt.Sprintf(`{"jwt":"%s"}`, testJWT))),
})
_, err := client.getJWT(ctx)
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("JWT token already expired or expires too soon"))
})
It("rejects already expired tokens", func() {
// Create a token that expired 1 minute ago
testJWT := createTestJWT(-1 * time.Minute)
httpClient.mock("https://auth.deezer.com/login/anonymous", http.Response{
StatusCode: 200,
Body: io.NopCloser(bytes.NewBufferString(fmt.Sprintf(`{"jwt":"%s"}`, testJWT))),
})
_, err := client.getJWT(ctx)
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("JWT token already expired or expires too soon"))
})
It("accepts tokens that expire in more than 1 minute", func() {
// Create a token that expires in 2 minutes (just over the 1-minute buffer)
testJWT := createTestJWT(2 * time.Minute)
httpClient.mock("https://auth.deezer.com/login/anonymous", http.Response{
StatusCode: 200,
Body: io.NopCloser(bytes.NewBufferString(fmt.Sprintf(`{"jwt":"%s"}`, testJWT))),
})
token, err := client.getJWT(ctx)
Expect(err).To(BeNil())
Expect(token).ToNot(BeEmpty())
})
})
Context("with invalid responses", func() {
It("handles HTTP error responses", func() {
httpClient.mock("https://auth.deezer.com/login/anonymous", http.Response{
StatusCode: 500,
Body: io.NopCloser(bytes.NewBufferString(`{"error":"Internal server error"}`)),
})
_, err := client.getJWT(ctx)
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("failed to get JWT token"))
})
It("handles malformed JSON responses", func() {
httpClient.mock("https://auth.deezer.com/login/anonymous", http.Response{
StatusCode: 200,
Body: io.NopCloser(bytes.NewBufferString(`{invalid json}`)),
})
_, err := client.getJWT(ctx)
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("failed to parse auth response"))
})
It("handles responses with empty JWT field", func() {
httpClient.mock("https://auth.deezer.com/login/anonymous", http.Response{
StatusCode: 200,
Body: io.NopCloser(bytes.NewBufferString(`{"jwt":""}`)),
})
_, err := client.getJWT(ctx)
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(Equal("deezer: no JWT token in response"))
})
It("handles invalid JWT tokens", func() {
httpClient.mock("https://auth.deezer.com/login/anonymous", http.Response{
StatusCode: 200,
Body: io.NopCloser(bytes.NewBufferString(`{"jwt":"not-a-valid-jwt"}`)),
})
_, err := client.getJWT(ctx)
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("failed to parse JWT token"))
})
It("rejects JWT tokens without expiration", func() {
// Create a JWT without expiration claim
testToken, err := jwt.NewBuilder().
Claim("custom", "value").
Build()
Expect(err).To(BeNil())
// Verify token has no expiration
Expect(testToken.Expiration().IsZero()).To(BeTrue())
testJWT, err := jwt.Sign(testToken, jwt.WithInsecureNoSignature())
Expect(err).To(BeNil())
httpClient.mock("https://auth.deezer.com/login/anonymous", http.Response{
StatusCode: 200,
Body: io.NopCloser(bytes.NewBufferString(fmt.Sprintf(`{"jwt":"%s"}`, string(testJWT)))),
})
_, err = client.getJWT(ctx)
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(Equal("deezer: JWT token has no expiration time"))
})
})
Context("token caching behavior", func() {
It("fetches a new token when the cached token expires", func() {
// First token expires in 5 minutes
firstJWT := createTestJWT(5 * time.Minute)
httpClient.mock("https://auth.deezer.com/login/anonymous", http.Response{
StatusCode: 200,
Body: io.NopCloser(bytes.NewBufferString(fmt.Sprintf(`{"jwt":"%s"}`, firstJWT))),
})
token1, err := client.getJWT(ctx)
Expect(err).To(BeNil())
Expect(token1).To(Equal(firstJWT))
// Manually expire the cached token
client.jwt.expiresAt = time.Now().Add(-1 * time.Second)
// Second token with different expiration (10 minutes)
secondJWT := createTestJWT(10 * time.Minute)
httpClient.mock("https://auth.deezer.com/login/anonymous", http.Response{
StatusCode: 200,
Body: io.NopCloser(bytes.NewBufferString(fmt.Sprintf(`{"jwt":"%s"}`, secondJWT))),
})
token2, err := client.getJWT(ctx)
Expect(err).To(BeNil())
Expect(token2).To(Equal(secondJWT))
Expect(token2).ToNot(Equal(token1))
})
})
})
Describe("jwtToken cache", func() {
var cache *jwtToken
BeforeEach(func() {
cache = &jwtToken{}
})
It("returns false for expired tokens", func() {
cache.set("test-token", -1*time.Second) // Already expired
token, valid := cache.get()
Expect(valid).To(BeFalse())
Expect(token).To(BeEmpty())
})
It("returns true for valid tokens", func() {
cache.set("test-token", 4*time.Minute)
token, valid := cache.get()
Expect(valid).To(BeTrue())
Expect(token).To(Equal("test-token"))
})
It("is thread-safe for concurrent access", func() {
wg := sync.WaitGroup{}
// Writer goroutine
wg.Go(func() {
for i := 0; i < 100; i++ {
cache.set(fmt.Sprintf("token-%d", i), 1*time.Hour)
time.Sleep(1 * time.Millisecond)
}
})
// Reader goroutine
wg.Go(func() {
for i := 0; i < 100; i++ {
cache.get()
time.Sleep(1 * time.Millisecond)
}
})
// Wait for both goroutines to complete
wg.Wait()
// Verify final state is valid
token, valid := cache.get()
Expect(valid).To(BeTrue())
Expect(token).To(HavePrefix("token-"))
})
})
})
// createTestJWT creates a valid JWT token for testing purposes
func createTestJWT(expiresIn time.Duration) string {
token, err := jwt.NewBuilder().
Expiration(time.Now().Add(expiresIn)).
Build()
if err != nil {
panic(fmt.Sprintf("failed to create test JWT: %v", err))
}
signed, err := jwt.Sign(token, jwt.WithInsecureNoSignature())
if err != nil {
panic(fmt.Sprintf("failed to sign test JWT: %v", err))
}
return string(signed)
}

View File

@ -2,10 +2,11 @@ package deezer
import (
"bytes"
"context"
"fmt"
"io"
"net/http"
"os"
"time"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
@ -17,7 +18,7 @@ var _ = Describe("client", func() {
BeforeEach(func() {
httpClient = &fakeHttpClient{}
client = newClient(httpClient)
client = newClient(httpClient, "en")
})
Describe("ArtistImages", func() {
@ -26,7 +27,7 @@ var _ = Describe("client", func() {
Expect(err).To(BeNil())
httpClient.mock("https://api.deezer.com/search/artist", http.Response{Body: f, StatusCode: 200})
artists, err := client.searchArtists(context.TODO(), "Michael Jackson", 20)
artists, err := client.searchArtists(GinkgoT().Context(), "Michael Jackson", 20)
Expect(err).To(BeNil())
Expect(artists).To(HaveLen(17))
Expect(artists[0].Name).To(Equal("Michael Jackson"))
@ -39,10 +40,136 @@ var _ = Describe("client", func() {
Body: io.NopCloser(bytes.NewBufferString(`{"data":[],"total":0}`)),
})
_, err := client.searchArtists(context.TODO(), "Michael Jackson", 20)
_, err := client.searchArtists(GinkgoT().Context(), "Michael Jackson", 20)
Expect(err).To(MatchError(ErrNotFound))
})
})
Describe("ArtistBio", func() {
BeforeEach(func() {
// Mock the JWT token endpoint with a valid JWT that expires in 5 minutes
testJWT := createTestJWT(5 * time.Minute)
httpClient.mock("https://auth.deezer.com/login/anonymous", http.Response{
StatusCode: 200,
Body: io.NopCloser(bytes.NewBufferString(fmt.Sprintf(`{"jwt":"%s","refresh_token":""}`, testJWT))),
})
})
It("returns artist bio from a successful request", func() {
f, err := os.Open("tests/fixtures/deezer.artist.bio.json")
Expect(err).To(BeNil())
httpClient.mock("https://pipe.deezer.com/api", http.Response{Body: f, StatusCode: 200})
bio, err := client.getArtistBio(GinkgoT().Context(), 27)
Expect(err).To(BeNil())
Expect(bio).To(ContainSubstring("Schoolmates Thomas and Guy-Manuel"))
Expect(bio).ToNot(ContainSubstring("<p>"))
Expect(bio).ToNot(ContainSubstring("</p>"))
})
It("uses the configured language", func() {
client = newClient(httpClient, "fr")
// Mock JWT token for the new client instance with a valid JWT
testJWT := createTestJWT(5 * time.Minute)
httpClient.mock("https://auth.deezer.com/login/anonymous", http.Response{
StatusCode: 200,
Body: io.NopCloser(bytes.NewBufferString(fmt.Sprintf(`{"jwt":"%s","refresh_token":""}`, testJWT))),
})
f, err := os.Open("tests/fixtures/deezer.artist.bio.json")
Expect(err).To(BeNil())
httpClient.mock("https://pipe.deezer.com/api", http.Response{Body: f, StatusCode: 200})
_, err = client.getArtistBio(GinkgoT().Context(), 27)
Expect(err).To(BeNil())
Expect(httpClient.lastRequest.Header.Get("Accept-Language")).To(Equal("fr"))
})
It("includes the JWT token in the request", func() {
f, err := os.Open("tests/fixtures/deezer.artist.bio.json")
Expect(err).To(BeNil())
httpClient.mock("https://pipe.deezer.com/api", http.Response{Body: f, StatusCode: 200})
_, err = client.getArtistBio(GinkgoT().Context(), 27)
Expect(err).To(BeNil())
// Verify that the Authorization header has the Bearer token format
authHeader := httpClient.lastRequest.Header.Get("Authorization")
Expect(authHeader).To(HavePrefix("Bearer "))
Expect(len(authHeader)).To(BeNumerically(">", 20)) // JWT tokens are longer than 20 chars
})
It("handles GraphQL errors", func() {
errorResponse := `{
"data": {
"artist": {
"bio": {
"full": ""
}
}
},
"errors": [
{
"message": "Artist not found"
},
{
"message": "Invalid artist ID"
}
]
}`
httpClient.mock("https://pipe.deezer.com/api", http.Response{
StatusCode: 200,
Body: io.NopCloser(bytes.NewBufferString(errorResponse)),
})
_, err := client.getArtistBio(GinkgoT().Context(), 999)
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("GraphQL error"))
Expect(err.Error()).To(ContainSubstring("Artist not found"))
Expect(err.Error()).To(ContainSubstring("Invalid artist ID"))
})
It("handles empty biography", func() {
emptyBioResponse := `{
"data": {
"artist": {
"bio": {
"full": ""
}
}
}
}`
httpClient.mock("https://pipe.deezer.com/api", http.Response{
StatusCode: 200,
Body: io.NopCloser(bytes.NewBufferString(emptyBioResponse)),
})
_, err := client.getArtistBio(GinkgoT().Context(), 27)
Expect(err).To(MatchError("deezer: biography not found"))
})
It("handles JWT token fetch failure", func() {
httpClient.mock("https://auth.deezer.com/login/anonymous", http.Response{
StatusCode: 500,
Body: io.NopCloser(bytes.NewBufferString(`{"error":"Internal server error"}`)),
})
_, err := client.getArtistBio(GinkgoT().Context(), 27)
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("failed to get JWT"))
})
It("handles JWT token that expires too soon", func() {
// Create a JWT that expires in 30 seconds (less than the 1-minute buffer)
expiredJWT := createTestJWT(30 * time.Second)
httpClient.mock("https://auth.deezer.com/login/anonymous", http.Response{
StatusCode: 200,
Body: io.NopCloser(bytes.NewBufferString(fmt.Sprintf(`{"jwt":"%s","refresh_token":""}`, expiredJWT))),
})
_, err := client.getArtistBio(GinkgoT().Context(), 27)
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("JWT token already expired or expires too soon"))
})
})
})
type fakeHttpClient struct {

View File

@ -12,6 +12,7 @@ import (
"github.com/navidrome/navidrome/log"
"github.com/navidrome/navidrome/model"
"github.com/navidrome/navidrome/utils/cache"
"github.com/navidrome/navidrome/utils/slice"
)
const deezerAgentName = "deezer"
@ -32,7 +33,7 @@ func deezerConstructor(dataStore model.DataStore) agents.Interface {
Timeout: consts.DefaultHttpClientTimeOut,
}
cachedHttpClient := cache.NewHTTPClient(httpClient, consts.DefaultHttpClientTimeOut)
agent.client = newClient(cachedHttpClient)
agent.client = newClient(cachedHttpClient, conf.Server.Deezer.Language)
return agent
}
@ -88,6 +89,56 @@ func (s *deezerAgent) searchArtist(ctx context.Context, name string) (*Artist, e
return &artists[0], err
}
func (s *deezerAgent) GetSimilarArtists(ctx context.Context, _, name, _ string, limit int) ([]agents.Artist, error) {
artist, err := s.searchArtist(ctx, name)
if err != nil {
return nil, err
}
related, err := s.client.getRelatedArtists(ctx, artist.ID)
if err != nil {
return nil, err
}
res := slice.Map(related, func(r Artist) agents.Artist {
return agents.Artist{
Name: r.Name,
}
})
if len(res) > limit {
res = res[:limit]
}
return res, nil
}
func (s *deezerAgent) GetArtistTopSongs(ctx context.Context, _, artistName, _ string, count int) ([]agents.Song, error) {
artist, err := s.searchArtist(ctx, artistName)
if err != nil {
return nil, err
}
tracks, err := s.client.getTopTracks(ctx, artist.ID, count)
if err != nil {
return nil, err
}
res := slice.Map(tracks, func(r Track) agents.Song {
return agents.Song{
Name: r.Title,
}
})
return res, nil
}
func (s *deezerAgent) GetArtistBiography(ctx context.Context, _, name, _ string) (string, error) {
artist, err := s.searchArtist(ctx, name)
if err != nil {
return "", err
}
return s.client.getArtistBio(ctx, artist.ID)
}
func init() {
conf.AddHook(func() {
if conf.Server.Deezer.Enabled {

View File

@ -29,3 +29,38 @@ type Error struct {
Code int `json:"code"`
} `json:"error"`
}
type RelatedArtists struct {
Data []Artist `json:"data"`
Total int `json:"total"`
}
type TopTracks struct {
Data []Track `json:"data"`
Total int `json:"total"`
Next string `json:"next"`
}
type Track struct {
ID int `json:"id"`
Title string `json:"title"`
Link string `json:"link"`
Duration int `json:"duration"`
Rank int `json:"rank"`
Preview string `json:"preview"`
Artist Artist `json:"artist"`
Album Album `json:"album"`
Contributors []Artist `json:"contributors"`
}
type Album struct {
ID int `json:"id"`
Title string `json:"title"`
Cover string `json:"cover"`
CoverSmall string `json:"cover_small"`
CoverMedium string `json:"cover_medium"`
CoverBig string `json:"cover_big"`
CoverXl string `json:"cover_xl"`
Tracklist string `json:"tracklist"`
Type string `json:"type"`
}

View File

@ -35,4 +35,35 @@ var _ = Describe("Responses", func() {
Expect(errorResp.Error.Message).To(Equal("Missing parameters: q"))
})
})
Describe("Related Artists", func() {
It("parses the related artists response correctly", func() {
var resp RelatedArtists
body, err := os.ReadFile("tests/fixtures/deezer.artist.related.json")
Expect(err).To(BeNil())
err = json.Unmarshal(body, &resp)
Expect(err).To(BeNil())
Expect(resp.Data).To(HaveLen(20))
justice := resp.Data[0]
Expect(justice.Name).To(Equal("Justice"))
Expect(justice.ID).To(Equal(6404))
})
})
Describe("Top Tracks", func() {
It("parses the top tracks response correctly", func() {
var resp TopTracks
body, err := os.ReadFile("tests/fixtures/deezer.artist.top.json")
Expect(err).To(BeNil())
err = json.Unmarshal(body, &resp)
Expect(err).To(BeNil())
Expect(resp.Data).To(HaveLen(5))
track := resp.Data[0]
Expect(track.Title).To(Equal("Instant Crush (feat. Julian Casablancas)"))
Expect(track.ID).To(Equal(67238732))
Expect(track.Album.Title).To(Equal("Random Access Memories"))
})
})
})

View File

@ -38,6 +38,7 @@ type lastfmAgent struct {
secret string
lang string
client *client
httpClient httpDoer
getInfoMutex sync.Mutex
}
@ -56,6 +57,7 @@ func lastFMConstructor(ds model.DataStore) *lastfmAgent {
Timeout: consts.DefaultHttpClientTimeOut,
}
chc := cache.NewHTTPClient(hc, consts.DefaultHttpClientTimeOut)
l.httpClient = chc
l.client = newClient(l.apiKey, l.secret, l.lang, chc)
return l
}
@ -190,13 +192,13 @@ func (l *lastfmAgent) GetArtistTopSongs(ctx context.Context, id, artistName, mbi
return res, nil
}
var artistOpenGraphQuery = cascadia.MustCompile(`html > head > meta[property="og:image"]`)
var (
artistOpenGraphQuery = cascadia.MustCompile(`html > head > meta[property="og:image"]`)
artistIgnoredImage = "2a96cbd8b46e442fc41c2b86b821562f" // Last.fm artist placeholder image name
)
func (l *lastfmAgent) GetArtistImages(ctx context.Context, _, name, mbid string) ([]agents.ExternalImage, error) {
log.Debug(ctx, "Getting artist images from Last.fm", "name", name)
hc := http.Client{
Timeout: consts.DefaultHttpClientTimeOut,
}
a, err := l.callArtistGetInfo(ctx, name)
if err != nil {
return nil, fmt.Errorf("get artist info: %w", err)
@ -205,7 +207,7 @@ func (l *lastfmAgent) GetArtistImages(ctx context.Context, _, name, mbid string)
if err != nil {
return nil, fmt.Errorf("create artist image request: %w", err)
}
resp, err := hc.Do(req)
resp, err := l.httpClient.Do(req)
if err != nil {
return nil, fmt.Errorf("get artist url: %w", err)
}
@ -222,11 +224,16 @@ func (l *lastfmAgent) GetArtistImages(ctx context.Context, _, name, mbid string)
return res, nil
}
for _, attr := range n.Attr {
if attr.Key == "content" {
res = []agents.ExternalImage{
{URL: attr.Val},
}
break
if attr.Key != "content" {
continue
}
if strings.Contains(attr.Val, artistIgnoredImage) {
log.Debug(ctx, "Artist image is ignored default image", "name", name, "url", attr.Val)
return res, nil
}
res = []agents.ExternalImage{
{URL: attr.Val},
}
}
return res, nil

View File

@ -393,4 +393,73 @@ var _ = Describe("lastfmAgent", func() {
})
})
})
Describe("GetArtistImages", func() {
var agent *lastfmAgent
var apiClient *tests.FakeHttpClient
var httpClient *tests.FakeHttpClient
BeforeEach(func() {
apiClient = &tests.FakeHttpClient{}
httpClient = &tests.FakeHttpClient{}
client := newClient("API_KEY", "SECRET", "pt", apiClient)
agent = lastFMConstructor(ds)
agent.client = client
agent.httpClient = httpClient
})
It("returns the artist image from the page", func() {
fApi, _ := os.Open("tests/fixtures/lastfm.artist.getinfo.json")
apiClient.Res = http.Response{Body: fApi, StatusCode: 200}
fScraper, _ := os.Open("tests/fixtures/lastfm.artist.page.html")
httpClient.Res = http.Response{Body: fScraper, StatusCode: 200}
images, err := agent.GetArtistImages(ctx, "123", "U2", "")
Expect(err).ToNot(HaveOccurred())
Expect(images).To(HaveLen(1))
Expect(images[0].URL).To(Equal("https://lastfm.freetls.fastly.net/i/u/ar0/818148bf682d429dc21b59a73ef6f68e.png"))
})
It("returns empty list if image is the ignored default image", func() {
fApi, _ := os.Open("tests/fixtures/lastfm.artist.getinfo.json")
apiClient.Res = http.Response{Body: fApi, StatusCode: 200}
fScraper, _ := os.Open("tests/fixtures/lastfm.artist.page.ignored.html")
httpClient.Res = http.Response{Body: fScraper, StatusCode: 200}
images, err := agent.GetArtistImages(ctx, "123", "U2", "")
Expect(err).ToNot(HaveOccurred())
Expect(images).To(BeEmpty())
})
It("returns empty list if page has no meta tags", func() {
fApi, _ := os.Open("tests/fixtures/lastfm.artist.getinfo.json")
apiClient.Res = http.Response{Body: fApi, StatusCode: 200}
fScraper, _ := os.Open("tests/fixtures/lastfm.artist.page.no_meta.html")
httpClient.Res = http.Response{Body: fScraper, StatusCode: 200}
images, err := agent.GetArtistImages(ctx, "123", "U2", "")
Expect(err).ToNot(HaveOccurred())
Expect(images).To(BeEmpty())
})
It("returns error if API call fails", func() {
apiClient.Err = errors.New("api error")
_, err := agent.GetArtistImages(ctx, "123", "U2", "")
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("get artist info"))
})
It("returns error if scraper call fails", func() {
fApi, _ := os.Open("tests/fixtures/lastfm.artist.getinfo.json")
apiClient.Res = http.Response{Body: fApi, StatusCode: 200}
httpClient.Err = errors.New("scraper error")
_, err := agent.GetArtistImages(ctx, "123", "U2", "")
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("get artist url"))
})
})
})

View File

@ -1,6 +1,7 @@
package artwork
import (
"cmp"
"context"
"crypto/md5"
"fmt"
@ -11,6 +12,7 @@ import (
"time"
"github.com/Masterminds/squirrel"
"github.com/maruel/natural"
"github.com/navidrome/navidrome/conf"
"github.com/navidrome/navidrome/core"
"github.com/navidrome/navidrome/core/external"
@ -116,8 +118,30 @@ func loadAlbumFoldersPaths(ctx context.Context, ds model.DataStore, albums ...mo
}
// Sort image files to ensure consistent selection of cover art
// This prioritizes files from lower-numbered disc folders by sorting the paths
slices.Sort(imgFiles)
// This prioritizes files without numeric suffixes (e.g., cover.jpg over cover.1.jpg)
// by comparing base filenames without extensions
slices.SortFunc(imgFiles, compareImageFiles)
return paths, imgFiles, &updatedAt, nil
}
// compareImageFiles compares two image file paths for sorting.
// It extracts the base filename (without extension) and compares case-insensitively.
// This ensures that "cover.jpg" sorts before "cover.1.jpg" since "cover" < "cover.1".
// Note: This function is called O(n log n) times during sorting, but in practice albums
// typically have only 1-20 image files, making the repeated string operations negligible.
func compareImageFiles(a, b string) int {
// Case-insensitive comparison
a = strings.ToLower(a)
b = strings.ToLower(b)
// Extract base filenames without extensions
baseA := strings.TrimSuffix(filepath.Base(a), filepath.Ext(a))
baseB := strings.TrimSuffix(filepath.Base(b), filepath.Ext(b))
// Compare base names first, then full paths if equal
return cmp.Or(
natural.Compare(baseA, baseB),
natural.Compare(a, b),
)
}

View File

@ -27,26 +27,7 @@ var _ = Describe("Album Artwork Reader", func() {
expectedAt = now.Add(5 * time.Minute)
// Set up the test folders with image files
repo = &fakeFolderRepo{
result: []model.Folder{
{
Path: "Artist/Album/Disc1",
ImagesUpdatedAt: expectedAt,
ImageFiles: []string{"cover.jpg", "back.jpg"},
},
{
Path: "Artist/Album/Disc2",
ImagesUpdatedAt: now,
ImageFiles: []string{"cover.jpg"},
},
{
Path: "Artist/Album/Disc10",
ImagesUpdatedAt: now,
ImageFiles: []string{"cover.jpg"},
},
},
err: nil,
}
repo = &fakeFolderRepo{}
ds = &fakeDataStore{
folderRepo: repo,
}
@ -58,19 +39,82 @@ var _ = Describe("Album Artwork Reader", func() {
})
It("returns sorted image files", func() {
repo.result = []model.Folder{
{
Path: "Artist/Album/Disc1",
ImagesUpdatedAt: expectedAt,
ImageFiles: []string{"cover.jpg", "back.jpg", "cover.1.jpg"},
},
{
Path: "Artist/Album/Disc2",
ImagesUpdatedAt: now,
ImageFiles: []string{"cover.jpg"},
},
{
Path: "Artist/Album/Disc10",
ImagesUpdatedAt: now,
ImageFiles: []string{"cover.jpg"},
},
}
_, imgFiles, imagesUpdatedAt, err := loadAlbumFoldersPaths(ctx, ds, album)
Expect(err).ToNot(HaveOccurred())
Expect(*imagesUpdatedAt).To(Equal(expectedAt))
// Check that image files are sorted alphabetically
Expect(imgFiles).To(HaveLen(4))
// Check that image files are sorted by base name (without extension)
Expect(imgFiles).To(HaveLen(5))
// The files should be sorted by full path
// Files should be sorted by base filename without extension, then by full path
// "back" < "cover", so back.jpg comes first
// Then all cover.jpg files, sorted by path
Expect(imgFiles[0]).To(Equal(filepath.FromSlash("Artist/Album/Disc1/back.jpg")))
Expect(imgFiles[1]).To(Equal(filepath.FromSlash("Artist/Album/Disc1/cover.jpg")))
Expect(imgFiles[2]).To(Equal(filepath.FromSlash("Artist/Album/Disc10/cover.jpg")))
Expect(imgFiles[3]).To(Equal(filepath.FromSlash("Artist/Album/Disc2/cover.jpg")))
Expect(imgFiles[2]).To(Equal(filepath.FromSlash("Artist/Album/Disc2/cover.jpg")))
Expect(imgFiles[3]).To(Equal(filepath.FromSlash("Artist/Album/Disc10/cover.jpg")))
Expect(imgFiles[4]).To(Equal(filepath.FromSlash("Artist/Album/Disc1/cover.1.jpg")))
})
It("prioritizes files without numeric suffixes", func() {
// Test case for issue #4683: cover.jpg should come before cover.1.jpg
repo.result = []model.Folder{
{
Path: "Artist/Album",
ImagesUpdatedAt: now,
ImageFiles: []string{"cover.1.jpg", "cover.jpg", "cover.2.jpg"},
},
}
_, imgFiles, _, err := loadAlbumFoldersPaths(ctx, ds, album)
Expect(err).ToNot(HaveOccurred())
Expect(imgFiles).To(HaveLen(3))
// cover.jpg should come first because "cover" < "cover.1" < "cover.2"
Expect(imgFiles[0]).To(Equal(filepath.FromSlash("Artist/Album/cover.jpg")))
Expect(imgFiles[1]).To(Equal(filepath.FromSlash("Artist/Album/cover.1.jpg")))
Expect(imgFiles[2]).To(Equal(filepath.FromSlash("Artist/Album/cover.2.jpg")))
})
It("handles case-insensitive sorting", func() {
// Test that Cover.jpg and cover.jpg are treated as equivalent
repo.result = []model.Folder{
{
Path: "Artist/Album",
ImagesUpdatedAt: now,
ImageFiles: []string{"Folder.jpg", "cover.jpg", "BACK.jpg"},
},
}
_, imgFiles, _, err := loadAlbumFoldersPaths(ctx, ds, album)
Expect(err).ToNot(HaveOccurred())
Expect(imgFiles).To(HaveLen(3))
// Files should be sorted case-insensitively: BACK, cover, Folder
Expect(imgFiles[0]).To(Equal(filepath.FromSlash("Artist/Album/BACK.jpg")))
Expect(imgFiles[1]).To(Equal(filepath.FromSlash("Artist/Album/cover.jpg")))
Expect(imgFiles[2]).To(Equal(filepath.FromSlash("Artist/Album/Folder.jpg")))
})
})
})

View File

@ -8,6 +8,7 @@ import (
"io/fs"
"os"
"path/filepath"
"slices"
"strings"
"time"
@ -139,11 +140,22 @@ func findImageInFolder(ctx context.Context, folder, pattern string) (io.ReadClos
return nil, "", err
}
// Filter to valid image files
var imagePaths []string
for _, m := range matches {
if !model.IsImageFile(m) {
continue
}
filePath := filepath.Join(folder, m)
imagePaths = append(imagePaths, m)
}
// Sort image files by prioritizing base filenames without numeric
// suffixes (e.g., artist.jpg before artist.1.jpg)
slices.SortFunc(imagePaths, compareImageFiles)
// Try to open files in sorted order
for _, p := range imagePaths {
filePath := filepath.Join(folder, p)
f, err := os.Open(filePath)
if err != nil {
log.Warn(ctx, "Could not open cover art file", "file", filePath, err)

View File

@ -240,24 +240,79 @@ var _ = Describe("artistArtworkReader", func() {
Expect(os.MkdirAll(artistDir, 0755)).To(Succeed())
// Create multiple matching files
Expect(os.WriteFile(filepath.Join(artistDir, "artist.jpg"), []byte("jpg image"), 0600)).To(Succeed())
Expect(os.WriteFile(filepath.Join(artistDir, "artist.abc"), []byte("text file"), 0600)).To(Succeed())
Expect(os.WriteFile(filepath.Join(artistDir, "artist.png"), []byte("png image"), 0600)).To(Succeed())
Expect(os.WriteFile(filepath.Join(artistDir, "artist.txt"), []byte("text file"), 0600)).To(Succeed())
Expect(os.WriteFile(filepath.Join(artistDir, "artist.jpg"), []byte("jpg image"), 0600)).To(Succeed())
testFunc = fromArtistFolder(ctx, artistDir, "artist.*")
})
It("returns the first valid image file", func() {
It("returns the first valid image file in sorted order", func() {
reader, path, err := testFunc()
Expect(err).ToNot(HaveOccurred())
Expect(reader).ToNot(BeNil())
// Should return an image file, not the text file
Expect(path).To(SatisfyAny(
ContainSubstring("artist.jpg"),
ContainSubstring("artist.png"),
))
Expect(path).ToNot(ContainSubstring("artist.txt"))
// Should return an image file,
// Files are sorted: jpg comes before png alphabetically.
// .abc comes first, but it's not an image.
Expect(path).To(ContainSubstring("artist.jpg"))
reader.Close()
})
})
When("prioritizing files without numeric suffixes", func() {
BeforeEach(func() {
// Test case for issue #4683: artist.jpg should come before artist.1.jpg
artistDir := filepath.Join(tempDir, "artist")
Expect(os.MkdirAll(artistDir, 0755)).To(Succeed())
// Create multiple matches with and without numeric suffixes
Expect(os.WriteFile(filepath.Join(artistDir, "artist.1.jpg"), []byte("artist 1"), 0600)).To(Succeed())
Expect(os.WriteFile(filepath.Join(artistDir, "artist.jpg"), []byte("artist main"), 0600)).To(Succeed())
Expect(os.WriteFile(filepath.Join(artistDir, "artist.2.jpg"), []byte("artist 2"), 0600)).To(Succeed())
testFunc = fromArtistFolder(ctx, artistDir, "artist.*")
})
It("returns artist.jpg before artist.1.jpg and artist.2.jpg", func() {
reader, path, err := testFunc()
Expect(err).ToNot(HaveOccurred())
Expect(reader).ToNot(BeNil())
Expect(path).To(ContainSubstring("artist.jpg"))
// Verify it's the main file, not a numbered variant
data, err := io.ReadAll(reader)
Expect(err).ToNot(HaveOccurred())
Expect(string(data)).To(Equal("artist main"))
reader.Close()
})
})
When("handling case-insensitive sorting", func() {
BeforeEach(func() {
// Test case to ensure case-insensitive natural sorting
artistDir := filepath.Join(tempDir, "artist")
Expect(os.MkdirAll(artistDir, 0755)).To(Succeed())
// Create files with mixed case names
Expect(os.WriteFile(filepath.Join(artistDir, "Folder.jpg"), []byte("folder"), 0600)).To(Succeed())
Expect(os.WriteFile(filepath.Join(artistDir, "artist.jpg"), []byte("artist"), 0600)).To(Succeed())
Expect(os.WriteFile(filepath.Join(artistDir, "BACK.jpg"), []byte("back"), 0600)).To(Succeed())
testFunc = fromArtistFolder(ctx, artistDir, "*.*")
})
It("sorts case-insensitively", func() {
reader, path, err := testFunc()
Expect(err).ToNot(HaveOccurred())
Expect(reader).ToNot(BeNil())
// Should return artist.jpg first (case-insensitive: "artist" < "back" < "folder")
Expect(path).To(ContainSubstring("artist.jpg"))
data, err := io.ReadAll(reader)
Expect(err).ToNot(HaveOccurred())
Expect(string(data)).To(Equal("artist"))
reader.Close()
})
})

View File

@ -113,9 +113,9 @@ func WithAdminUser(ctx context.Context, ds model.DataStore) context.Context {
if err != nil {
c, err := ds.User(ctx).CountAll()
if c == 0 && err == nil {
log.Debug(ctx, "Scanner: No admin user yet!", err)
log.Debug(ctx, "No admin user yet!", err)
} else {
log.Error(ctx, "Scanner: No admin user found!", err)
log.Error(ctx, "No admin user found!", err)
}
u = &model.User{}
}

View File

@ -21,11 +21,6 @@ import (
"github.com/navidrome/navidrome/utils/slice"
)
// Scanner interface for triggering scans
type Scanner interface {
ScanAll(ctx context.Context, fullScan bool) (warnings []string, err error)
}
// Watcher interface for managing file system watchers
type Watcher interface {
Watch(ctx context.Context, lib *model.Library) error
@ -43,13 +38,13 @@ type Library interface {
type libraryService struct {
ds model.DataStore
scanner Scanner
scanner model.Scanner
watcher Watcher
broker events.Broker
}
// NewLibrary creates a new Library service
func NewLibrary(ds model.DataStore, scanner Scanner, watcher Watcher, broker events.Broker) Library {
func NewLibrary(ds model.DataStore, scanner model.Scanner, watcher Watcher, broker events.Broker) Library {
return &libraryService{
ds: ds,
scanner: scanner,
@ -155,7 +150,7 @@ type libraryRepositoryWrapper struct {
model.LibraryRepository
ctx context.Context
ds model.DataStore
scanner Scanner
scanner model.Scanner
watcher Watcher
broker events.Broker
}
@ -192,7 +187,7 @@ func (r *libraryRepositoryWrapper) Save(entity interface{}) (string, error) {
return strconv.Itoa(lib.ID), nil
}
func (r *libraryRepositoryWrapper) Update(id string, entity interface{}, cols ...string) error {
func (r *libraryRepositoryWrapper) Update(id string, entity interface{}, _ ...string) error {
lib := entity.(*model.Library)
libID, err := strconv.Atoi(id)
if err != nil {

View File

@ -29,7 +29,7 @@ var _ = Describe("Library Service", func() {
var userRepo *tests.MockedUserRepo
var ctx context.Context
var tempDir string
var scanner *mockScanner
var scanner *tests.MockScanner
var watcherManager *mockWatcherManager
var broker *mockEventBroker
@ -43,7 +43,7 @@ var _ = Describe("Library Service", func() {
ds.MockedUser = userRepo
// Create a mock scanner that tracks calls
scanner = &mockScanner{}
scanner = tests.NewMockScanner()
// Create a mock watcher manager
watcherManager = &mockWatcherManager{
libraryStates: make(map[int]model.Library),
@ -616,11 +616,12 @@ var _ = Describe("Library Service", func() {
// Wait briefly for the goroutine to complete
Eventually(func() int {
return scanner.len()
return scanner.GetScanAllCallCount()
}, "1s", "10ms").Should(Equal(1))
// Verify scan was called with correct parameters
Expect(scanner.ScanCalls[0].FullScan).To(BeFalse()) // Should be quick scan
calls := scanner.GetScanAllCalls()
Expect(calls[0].FullScan).To(BeFalse()) // Should be quick scan
})
It("triggers scan when updating library path", func() {
@ -641,11 +642,12 @@ var _ = Describe("Library Service", func() {
// Wait briefly for the goroutine to complete
Eventually(func() int {
return scanner.len()
return scanner.GetScanAllCallCount()
}, "1s", "10ms").Should(Equal(1))
// Verify scan was called with correct parameters
Expect(scanner.ScanCalls[0].FullScan).To(BeFalse()) // Should be quick scan
calls := scanner.GetScanAllCalls()
Expect(calls[0].FullScan).To(BeFalse()) // Should be quick scan
})
It("does not trigger scan when updating library without path change", func() {
@ -661,7 +663,7 @@ var _ = Describe("Library Service", func() {
// Wait a bit to ensure no scan was triggered
Consistently(func() int {
return scanner.len()
return scanner.GetScanAllCallCount()
}, "100ms", "10ms").Should(Equal(0))
})
@ -674,7 +676,7 @@ var _ = Describe("Library Service", func() {
// Ensure no scan was triggered since creation failed
Consistently(func() int {
return scanner.len()
return scanner.GetScanAllCallCount()
}, "100ms", "10ms").Should(Equal(0))
})
@ -691,7 +693,7 @@ var _ = Describe("Library Service", func() {
// Ensure no scan was triggered since update failed
Consistently(func() int {
return scanner.len()
return scanner.GetScanAllCallCount()
}, "100ms", "10ms").Should(Equal(0))
})
@ -707,11 +709,12 @@ var _ = Describe("Library Service", func() {
// Wait briefly for the goroutine to complete
Eventually(func() int {
return scanner.len()
return scanner.GetScanAllCallCount()
}, "1s", "10ms").Should(Equal(1))
// Verify scan was called with correct parameters
Expect(scanner.ScanCalls[0].FullScan).To(BeFalse()) // Should be quick scan
calls := scanner.GetScanAllCalls()
Expect(calls[0].FullScan).To(BeFalse()) // Should be quick scan
})
It("does not trigger scan when library deletion fails", func() {
@ -721,7 +724,7 @@ var _ = Describe("Library Service", func() {
// Ensure no scan was triggered since deletion failed
Consistently(func() int {
return scanner.len()
return scanner.GetScanAllCallCount()
}, "100ms", "10ms").Should(Equal(0))
})
@ -868,31 +871,6 @@ var _ = Describe("Library Service", func() {
})
})
// mockScanner provides a simple mock implementation of core.Scanner for testing
type mockScanner struct {
ScanCalls []ScanCall
mu sync.RWMutex
}
type ScanCall struct {
FullScan bool
}
func (m *mockScanner) ScanAll(ctx context.Context, fullScan bool) (warnings []string, err error) {
m.mu.Lock()
defer m.mu.Unlock()
m.ScanCalls = append(m.ScanCalls, ScanCall{
FullScan: fullScan,
})
return []string{}, nil
}
func (m *mockScanner) len() int {
m.mu.RLock()
defer m.mu.RUnlock()
return len(m.ScanCalls)
}
// mockWatcherManager provides a simple mock implementation of core.Watcher for testing
type mockWatcherManager struct {
StartedWatchers []model.Library

View File

@ -166,7 +166,7 @@ func (s *maintenanceService) getAffectedAlbumIDs(ctx context.Context, ids []stri
if len(ids) > 0 {
filters = squirrel.And{
squirrel.Eq{"missing": true},
squirrel.Eq{"id": ids},
squirrel.Eq{"media_file.id": ids},
}
}

View File

@ -14,7 +14,7 @@ import (
)
var _ = Describe("Maintenance", func() {
var ds *extendedDataStore
var ds *tests.MockDataStore
var mfRepo *extendedMediaFileRepo
var service Maintenance
var ctx context.Context
@ -42,7 +42,7 @@ var _ = Describe("Maintenance", func() {
Expect(err).ToNot(HaveOccurred())
Expect(mfRepo.deleteMissingCalled).To(BeTrue())
Expect(mfRepo.deletedIDs).To(Equal([]string{"mf1", "mf2"}))
Expect(ds.gcCalled).To(BeTrue(), "GC should be called after deletion")
Expect(ds.GCCalled).To(BeTrue(), "GC should be called after deletion")
})
It("triggers artist stats refresh and album refresh after deletion", func() {
@ -97,7 +97,7 @@ var _ = Describe("Maintenance", func() {
})
// Set GC to return error
ds.gcError = errors.New("gc failed")
ds.GCError = errors.New("gc failed")
err := service.DeleteMissingFiles(ctx, []string{"mf1"})
@ -143,7 +143,7 @@ var _ = Describe("Maintenance", func() {
err := service.DeleteAllMissingFiles(ctx)
Expect(err).ToNot(HaveOccurred())
Expect(ds.gcCalled).To(BeTrue(), "GC should be called after deletion")
Expect(ds.GCCalled).To(BeTrue(), "GC should be called after deletion")
})
It("returns error if deletion fails", func() {
@ -253,11 +253,8 @@ var _ = Describe("Maintenance", func() {
})
// Test helper to create a mock DataStore with controllable behavior
func createTestDataStore() *extendedDataStore {
// Create extended datastore with GC tracking
ds := &extendedDataStore{
MockDataStore: &tests.MockDataStore{},
}
func createTestDataStore() *tests.MockDataStore {
ds := &tests.MockDataStore{}
// Create extended album repo with Put tracking
albumRepo := &extendedAlbumRepo{
@ -365,18 +362,3 @@ func (m *extendedArtistRepo) IsRefreshStatsCalled() bool {
defer m.mu.RUnlock()
return m.refreshStatsCalled
}
// Extension of MockDataStore to track GC calls
type extendedDataStore struct {
*tests.MockDataStore
gcCalled bool
gcError error
}
func (ds *extendedDataStore) GC(ctx context.Context) error {
ds.gcCalled = true
if ds.gcError != nil {
return ds.gcError
}
return ds.MockDataStore.GC(ctx)
}

View File

@ -22,6 +22,7 @@ import (
"github.com/navidrome/navidrome/core/metrics/insights"
"github.com/navidrome/navidrome/log"
"github.com/navidrome/navidrome/model"
"github.com/navidrome/navidrome/model/request"
"github.com/navidrome/navidrome/plugins/schema"
"github.com/navidrome/navidrome/utils/singleton"
)
@ -64,9 +65,16 @@ func GetInstance(ds model.DataStore, pluginLoader PluginLoader) Insights {
}
func (c *insightsCollector) Run(ctx context.Context) {
ctx = auth.WithAdminUser(ctx, c.ds)
for {
c.sendInsights(ctx)
// Refresh admin context on each iteration to handle cases where
// admin user wasn't available on previous runs
insightsCtx := auth.WithAdminUser(ctx, c.ds)
u, _ := request.UserFrom(insightsCtx)
if !u.IsAdmin {
log.Trace(insightsCtx, "No admin user available, skipping insights collection")
} else {
c.sendInsights(insightsCtx)
}
select {
case <-time.After(consts.InsightsUpdateInterval):
continue

View File

@ -45,10 +45,12 @@ func Db() *sql.DB {
if err != nil {
log.Fatal("Error opening database", err)
}
_, err = db.Exec("PRAGMA optimize=0x10002")
if err != nil {
log.Error("Error applying PRAGMA optimize", err)
return nil
if conf.Server.DevOptimizeDB {
_, err = db.Exec("PRAGMA optimize=0x10002")
if err != nil {
log.Error("Error applying PRAGMA optimize", err)
return nil
}
}
return db
})
@ -99,7 +101,7 @@ func Init(ctx context.Context) func() {
log.Fatal(ctx, "Failed to apply new migrations", err)
}
if hasSchemaChanges {
if hasSchemaChanges && conf.Server.DevOptimizeDB {
log.Debug(ctx, "Applying PRAGMA optimize after schema changes")
_, err = db.ExecContext(ctx, "PRAGMA optimize")
if err != nil {
@ -114,6 +116,9 @@ func Init(ctx context.Context) func() {
// Optimize runs PRAGMA optimize on each connection in the pool
func Optimize(ctx context.Context) {
if !conf.Server.DevOptimizeDB {
return
}
numConns := Db().Stats().OpenConnections
if numConns == 0 {
log.Debug(ctx, "No open connections to optimize")

View File

@ -0,0 +1,7 @@
-- +goose Up
-- +goose StatementBegin
ALTER TABLE annotation ADD COLUMN rated_at datetime;
-- +goose StatementEnd
-- +goose Down

View File

@ -7,6 +7,7 @@ import (
"strings"
"sync"
"github.com/navidrome/navidrome/conf"
"github.com/navidrome/navidrome/consts"
)
@ -21,11 +22,13 @@ func notice(tx *sql.Tx, msg string) {
// Call this in migrations that requires a full rescan
func forceFullRescan(tx *sql.Tx) error {
// If a full scan is required, most probably the query optimizer is outdated, so we run `analyze`.
_, err := tx.Exec(`ANALYZE;`)
if err != nil {
return err
if conf.Server.DevOptimizeDB {
_, err := tx.Exec(`ANALYZE;`)
if err != nil {
return err
}
}
_, err = tx.Exec(fmt.Sprintf(`
_, err := tx.Exec(fmt.Sprintf(`
INSERT OR REPLACE into property (id, value) values ('%s', '1');
`, consts.FullScanAfterMigrationFlagKey))
return err

23
go.mod
View File

@ -1,6 +1,6 @@
module github.com/navidrome/navidrome
go 1.25.4
go 1.25
// Fork to fix https://github.com/navidrome/navidrome/issues/3254
replace github.com/dhowden/tag v0.0.0-20240417053706-3d75831295e8 => github.com/deluan/tag v0.0.0-20241002021117-dfe5e6ea396d
@ -39,6 +39,7 @@ require (
github.com/knqyf263/go-plugin v0.9.0
github.com/kr/pretty v0.3.1
github.com/lestrrat-go/jwx/v2 v2.1.6
github.com/maruel/natural v1.2.1
github.com/matoous/go-nanoid/v2 v2.1.0
github.com/mattn/go-sqlite3 v1.14.32
github.com/microcosm-cc/bluemonday v1.0.27
@ -56,16 +57,16 @@ require (
github.com/spf13/cobra v1.10.1
github.com/spf13/viper v1.21.0
github.com/stretchr/testify v1.11.1
github.com/tetratelabs/wazero v1.10.0
github.com/tetratelabs/wazero v1.10.1
github.com/unrolled/secure v1.17.0
github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342
go.uber.org/goleak v1.3.0
golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546
golang.org/x/image v0.32.0
golang.org/x/net v0.46.0
golang.org/x/exp v0.0.0-20251113190631-e25ba8c21ef6
golang.org/x/image v0.33.0
golang.org/x/net v0.47.0
golang.org/x/sync v0.18.0
golang.org/x/sys v0.38.0
golang.org/x/text v0.30.0
golang.org/x/text v0.31.0
golang.org/x/time v0.14.0
google.golang.org/protobuf v1.36.10
gopkg.in/yaml.v3 v3.0.1
@ -89,7 +90,7 @@ require (
github.com/goccy/go-json v0.10.5 // indirect
github.com/goccy/go-yaml v1.18.0 // indirect
github.com/google/go-cmp v0.7.0 // indirect
github.com/google/pprof v0.0.0-20251007162407-5df77e3f7d1d // indirect
github.com/google/pprof v0.0.0-20251114195745-4902fdda35c8 // indirect
github.com/google/subcommands v1.2.0 // indirect
github.com/gorilla/css v1.0.1 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
@ -127,10 +128,10 @@ require (
go.uber.org/multierr v1.11.0 // indirect
go.yaml.in/yaml/v2 v2.4.2 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
golang.org/x/crypto v0.43.0 // indirect
golang.org/x/mod v0.29.0 // indirect
golang.org/x/telemetry v0.0.0-20251008203120-078029d740a8 // indirect
golang.org/x/tools v0.38.0 // indirect
golang.org/x/crypto v0.45.0 // indirect
golang.org/x/mod v0.30.0 // indirect
golang.org/x/telemetry v0.0.0-20251111182119-bc8e575c7b54 // indirect
golang.org/x/tools v0.39.0 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect
)

44
go.sum
View File

@ -99,8 +99,8 @@ github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/go-pipeline v0.0.0-20230411140531-6cbedfc1d3fc h1:hd+uUVsB1vdxohPneMrhGH2YfQuH5hRIK9u4/XCeUtw=
github.com/google/go-pipeline v0.0.0-20230411140531-6cbedfc1d3fc/go.mod h1:SL66SJVysrh7YbDCP9tH30b8a9o/N2HeiQNUm85EKhc=
github.com/google/pprof v0.0.0-20251007162407-5df77e3f7d1d h1:KJIErDwbSHjnp/SGzE5ed8Aol7JsKiI5X7yWKAtzhM0=
github.com/google/pprof v0.0.0-20251007162407-5df77e3f7d1d/go.mod h1:I6V7YzU0XDpsHqbsyrghnFZLO1gwK6NPTNvmetQIk9U=
github.com/google/pprof v0.0.0-20251114195745-4902fdda35c8 h1:3DsUAV+VNEQa2CUVLxCY3f87278uWfIDhJnbdvDjvmE=
github.com/google/pprof v0.0.0-20251114195745-4902fdda35c8/go.mod h1:I6V7YzU0XDpsHqbsyrghnFZLO1gwK6NPTNvmetQIk9U=
github.com/google/subcommands v1.2.0 h1:vWQspBTo2nEqTUFita5/KeEWlUL8kQObDFbub/EN9oE=
github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
@ -162,8 +162,8 @@ github.com/lestrrat-go/jwx/v2 v2.1.6 h1:hxM1gfDILk/l5ylers6BX/Eq1m/pnxe9NBwW6lVf
github.com/lestrrat-go/jwx/v2 v2.1.6/go.mod h1:Y722kU5r/8mV7fYDifjug0r8FK8mZdw0K0GpJw/l8pU=
github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNBEYU=
github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I=
github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo=
github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg=
github.com/maruel/natural v1.2.1 h1:G/y4pwtTA07lbQsMefvsmEO0VN0NfqpxprxXDM4R/4o=
github.com/maruel/natural v1.2.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg=
github.com/matoous/go-nanoid/v2 v2.1.0 h1:P64+dmq21hhWdtvZfEAofnvJULaRR1Yib0+PnU669bE=
github.com/matoous/go-nanoid/v2 v2.1.0/go.mod h1:KlbGNQ+FhrUNIHUxZdL63t7tl4LaPkZNpUULS8H4uVM=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
@ -265,8 +265,8 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
github.com/tetratelabs/wazero v1.10.0 h1:CXP3zneLDl6J4Zy8N/J+d5JsWKfrjE6GtvVK1fpnDlk=
github.com/tetratelabs/wazero v1.10.0/go.mod h1:DRm5twOQ5Gr1AoEdSi0CLjDQF1J9ZAuyqFIjl1KKfQU=
github.com/tetratelabs/wazero v1.10.1 h1:2DugeJf6VVk58KTPszlNfeeN8AhhpwcZqkJj2wwFuH8=
github.com/tetratelabs/wazero v1.10.1/go.mod h1:DRm5twOQ5Gr1AoEdSi0CLjDQF1J9ZAuyqFIjl1KKfQU=
github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY=
github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
@ -298,20 +298,20 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04=
golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0=
golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 h1:mgKeJMpvi0yx/sU5GsxQ7p6s2wtOnGAHZWCHUM4KGzY=
golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546/go.mod h1:j/pmGrbnkbPtQfxEe5D0VQhZC6qKbfKifgD0oM7sR70=
golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q=
golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4=
golang.org/x/exp v0.0.0-20251113190631-e25ba8c21ef6 h1:zfMcR1Cs4KNuomFFgGefv5N0czO2XZpUbxGUy8i8ug0=
golang.org/x/exp v0.0.0-20251113190631-e25ba8c21ef6/go.mod h1:46edojNIoXTNOhySWIWdix628clX9ODXwPsQuG6hsK0=
golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/image v0.32.0 h1:6lZQWq75h7L5IWNk0r+SCpUJ6tUVd3v4ZHnbRKLkUDQ=
golang.org/x/image v0.32.0/go.mod h1:/R37rrQmKXtO6tYXAjtDLwQgFLHmhW+V6ayXlxzP2Pc=
golang.org/x/image v0.33.0 h1:LXRZRnv1+zGd5XBUVRFmYEphyyKJjQjCRiOuAP3sZfQ=
golang.org/x/image v0.33.0/go.mod h1:DD3OsTYT9chzuzTQt+zMcOlBHgfoKQb1gry8p76Y1sc=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA=
golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w=
golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk=
golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@ -323,8 +323,8 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
golang.org/x/net v0.46.0 h1:giFlY12I07fugqwPuWJi68oOnpfqFnJIJzaIIm2JVV4=
golang.org/x/net v0.46.0/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210=
golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@ -353,8 +353,8 @@ golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
golang.org/x/telemetry v0.0.0-20251008203120-078029d740a8 h1:LvzTn0GQhWuvKH/kVRS3R3bVAsdQWI7hvfLHGgh9+lU=
golang.org/x/telemetry v0.0.0-20251008203120-078029d740a8/go.mod h1:Pi4ztBfryZoJEkyFTI5/Ocsu2jXyDr6iSdgJiYE/uwE=
golang.org/x/telemetry v0.0.0-20251111182119-bc8e575c7b54 h1:E2/AqCUMZGgd73TQkxUMcMla25GB9i/5HOdLr+uH7Vo=
golang.org/x/telemetry v0.0.0-20251111182119-bc8e575c7b54/go.mod h1:hKdjCMrbv9skySur+Nek8Hd0uJ0GuxJIoIX2payrIdQ=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
@ -373,8 +373,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k=
golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM=
golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@ -384,8 +384,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ=
golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs=
golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ=
golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE=

View File

@ -80,8 +80,8 @@ var (
// SetLevel sets the global log level used by the simple logger.
func SetLevel(l Level) {
currentLevel = l
loggerMu.Lock()
currentLevel = l
defaultLogger.Level = logrus.TraceLevel
loggerMu.Unlock()
logrus.SetLevel(logrus.Level(l))
@ -114,6 +114,8 @@ func levelFromString(l string) Level {
// SetLogLevels sets the log levels for specific paths in the codebase.
func SetLogLevels(levels map[string]string) {
loggerMu.Lock()
defer loggerMu.Unlock()
logLevels = nil
for k, v := range levels {
logLevels = append(logLevels, levelPath{path: k, level: levelFromString(v)})
@ -172,6 +174,8 @@ func SetDefaultLogger(l *logrus.Logger) {
}
func CurrentLevel() Level {
loggerMu.RLock()
defer loggerMu.RUnlock()
return currentLevel
}
@ -220,10 +224,15 @@ func Writer() io.Writer {
}
func shouldLog(requiredLevel Level, skip int) bool {
if currentLevel >= requiredLevel {
loggerMu.RLock()
level := currentLevel
levels := logLevels
loggerMu.RUnlock()
if level >= requiredLevel {
return true
}
if len(logLevels) == 0 {
if len(levels) == 0 {
return false
}
@ -233,7 +242,7 @@ func shouldLog(requiredLevel Level, skip int) bool {
}
file = strings.TrimPrefix(file, rootPath)
for _, lp := range logLevels {
for _, lp := range levels {
if strings.HasPrefix(file, lp.path) {
return lp.level >= requiredLevel
}

View File

@ -6,6 +6,7 @@ type Annotations struct {
PlayCount int64 `structs:"play_count" json:"playCount,omitempty"`
PlayDate *time.Time `structs:"play_date" json:"playDate,omitempty" `
Rating int `structs:"rating" json:"rating,omitempty" `
RatedAt *time.Time `structs:"rated_at" json:"ratedAt,omitempty" `
Starred bool `structs:"starred" json:"starred,omitempty" `
StarredAt *time.Time `structs:"starred_at" json:"starredAt,omitempty"`
}

View File

@ -44,6 +44,7 @@ var fieldMap = map[string]*mappedField{
"loved": {field: "COALESCE(annotation.starred, false)"},
"dateloved": {field: "annotation.starred_at"},
"lastplayed": {field: "annotation.play_date"},
"daterated": {field: "annotation.rated_at"},
"playcount": {field: "COALESCE(annotation.play_count, 0)"},
"rating": {field: "COALESCE(annotation.rating, 0)"},
"mbz_album_id": {field: "media_file.mbz_album_id"},

View File

@ -43,5 +43,5 @@ type DataStore interface {
WithTx(block func(tx DataStore) error, scope ...string) error
WithTxImmediate(block func(tx DataStore) error, scope ...string) error
GC(ctx context.Context) error
GC(ctx context.Context, libraryIDs ...int) error
}

View File

@ -85,7 +85,7 @@ type FolderRepository interface {
GetByPath(lib Library, path string) (*Folder, error)
GetAll(...QueryOptions) ([]Folder, error)
CountAll(...QueryOptions) (int64, error)
GetLastUpdates(lib Library) (map[string]FolderUpdateInfo, error)
GetFolderUpdateInfo(lib Library, targetPaths ...string) (map[string]FolderUpdateInfo, error)
Put(*Folder) error
MarkMissing(missing bool, ids ...string) error
GetTouchedWithPlaylists() (FolderCursor, error)

View File

@ -23,7 +23,7 @@ func legacyTrackID(mf model.MediaFile, prependLibId bool) string {
}
func legacyAlbumID(mf model.MediaFile, md Metadata, prependLibId bool) string {
releaseDate := legacyReleaseDate(md)
_, _, releaseDate := md.mapDates()
albumPath := strings.ToLower(fmt.Sprintf("%s\\%s", legacyMapAlbumArtistName(md), legacyMapAlbumName(md)))
if !conf.Server.Scanner.GroupAlbumReleases {
if len(releaseDate) != 0 {
@ -55,9 +55,3 @@ func legacyMapAlbumName(md Metadata) string {
consts.UnknownAlbum,
)
}
// Keep the TaggedLikePicard logic for backwards compatibility
func legacyReleaseDate(md Metadata) string {
_, _, releaseDate := md.mapDates()
return string(releaseDate)
}

View File

@ -1,30 +0,0 @@
package metadata
import (
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
var _ = Describe("legacyReleaseDate", func() {
DescribeTable("legacyReleaseDate",
func(recordingDate, originalDate, releaseDate, expected string) {
md := New("", Info{
Tags: map[string][]string{
"DATE": {recordingDate},
"ORIGINALDATE": {originalDate},
"RELEASEDATE": {releaseDate},
},
})
result := legacyReleaseDate(md)
Expect(result).To(Equal(expected))
},
Entry("regular mapping", "2020-05-15", "2019-02-10", "2021-01-01", "2021-01-01"),
Entry("legacy mapping", "2020-05-15", "2019-02-10", "", "2020-05-15"),
Entry("legacy mapping, originalYear < year", "2018-05-15", "2019-02-10", "2021-01-01", "2021-01-01"),
Entry("legacy mapping, originalYear empty", "2020-05-15", "", "2021-01-01", "2021-01-01"),
Entry("legacy mapping, releaseYear", "2020-05-15", "2019-02-10", "2021-01-01", "2021-01-01"),
Entry("legacy mapping, same dates", "2020-05-15", "2020-05-15", "", "2020-05-15"),
)
})

View File

@ -75,6 +75,23 @@ var _ = Describe("ToMediaFile", func() {
Expect(mf.OriginalYear).To(Equal(1966))
Expect(mf.ReleaseYear).To(Equal(2014))
})
DescribeTable("legacyReleaseDate (TaggedLikePicard old behavior)",
func(recordingDate, originalDate, releaseDate, expected string) {
mf := toMediaFile(model.RawTags{
"DATE": {recordingDate},
"ORIGINALDATE": {originalDate},
"RELEASEDATE": {releaseDate},
})
Expect(mf.ReleaseDate).To(Equal(expected))
},
Entry("regular mapping", "2020-05-15", "2019-02-10", "2021-01-01", "2021-01-01"),
Entry("legacy mapping", "2020-05-15", "2019-02-10", "", "2020-05-15"),
Entry("legacy mapping, originalYear < year", "2018-05-15", "2019-02-10", "2021-01-01", "2021-01-01"),
Entry("legacy mapping, originalYear empty", "2020-05-15", "", "2021-01-01", "2021-01-01"),
Entry("legacy mapping, releaseYear", "2020-05-15", "2019-02-10", "2021-01-01", "2021-01-01"),
Entry("legacy mapping, same dates", "2020-05-15", "2020-05-15", "", "2020-05-15"),
)
})
Describe("Lyrics", func() {

81
model/scanner.go Normal file
View File

@ -0,0 +1,81 @@
package model
import (
"context"
"fmt"
"strconv"
"strings"
"time"
)
// ScanTarget represents a specific folder within a library to be scanned.
// NOTE: This struct is used as a map key, so it should only contain comparable types.
type ScanTarget struct {
LibraryID int
FolderPath string // Relative path within the library, or "" for entire library
}
func (st ScanTarget) String() string {
return fmt.Sprintf("%d:%s", st.LibraryID, st.FolderPath)
}
// ScannerStatus holds information about the current scan status
type ScannerStatus struct {
Scanning bool
LastScan time.Time
Count uint32
FolderCount uint32
LastError string
ScanType string
ElapsedTime time.Duration
}
type Scanner interface {
// ScanAll starts a scan of all libraries. This is a blocking operation.
ScanAll(ctx context.Context, fullScan bool) (warnings []string, err error)
// ScanFolders scans specific library/folder pairs, recursing into subdirectories.
// If targets is nil, it scans all libraries. This is a blocking operation.
ScanFolders(ctx context.Context, fullScan bool, targets []ScanTarget) (warnings []string, err error)
Status(context.Context) (*ScannerStatus, error)
}
// ParseTargets parses scan targets strings into ScanTarget structs.
// Example: []string{"1:Music/Rock", "2:Classical"}
func ParseTargets(libFolders []string) ([]ScanTarget, error) {
targets := make([]ScanTarget, 0, len(libFolders))
for _, part := range libFolders {
part = strings.TrimSpace(part)
if part == "" {
continue
}
// Split by the first colon
colonIdx := strings.Index(part, ":")
if colonIdx == -1 {
return nil, fmt.Errorf("invalid target format: %q (expected libraryID:folderPath)", part)
}
libIDStr := part[:colonIdx]
folderPath := part[colonIdx+1:]
libID, err := strconv.Atoi(libIDStr)
if err != nil {
return nil, fmt.Errorf("invalid library ID %q: %w", libIDStr, err)
}
if libID <= 0 {
return nil, fmt.Errorf("invalid library ID %q", libIDStr)
}
targets = append(targets, ScanTarget{
LibraryID: libID,
FolderPath: folderPath,
})
}
if len(targets) == 0 {
return nil, fmt.Errorf("no valid targets found")
}
return targets, nil
}

89
model/scanner_test.go Normal file
View File

@ -0,0 +1,89 @@
package model_test
import (
"github.com/navidrome/navidrome/model"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
var _ = Describe("ParseTargets", func() {
It("parses multiple entries in slice", func() {
targets, err := model.ParseTargets([]string{"1:Music/Rock", "1:Music/Jazz", "2:Classical"})
Expect(err).ToNot(HaveOccurred())
Expect(targets).To(HaveLen(3))
Expect(targets[0].LibraryID).To(Equal(1))
Expect(targets[0].FolderPath).To(Equal("Music/Rock"))
Expect(targets[1].LibraryID).To(Equal(1))
Expect(targets[1].FolderPath).To(Equal("Music/Jazz"))
Expect(targets[2].LibraryID).To(Equal(2))
Expect(targets[2].FolderPath).To(Equal("Classical"))
})
It("handles empty folder paths", func() {
targets, err := model.ParseTargets([]string{"1:", "2:"})
Expect(err).ToNot(HaveOccurred())
Expect(targets).To(HaveLen(2))
Expect(targets[0].FolderPath).To(Equal(""))
Expect(targets[1].FolderPath).To(Equal(""))
})
It("trims whitespace from entries", func() {
targets, err := model.ParseTargets([]string{" 1:Music/Rock", " 2:Classical "})
Expect(err).ToNot(HaveOccurred())
Expect(targets).To(HaveLen(2))
Expect(targets[0].LibraryID).To(Equal(1))
Expect(targets[0].FolderPath).To(Equal("Music/Rock"))
Expect(targets[1].LibraryID).To(Equal(2))
Expect(targets[1].FolderPath).To(Equal("Classical"))
})
It("skips empty strings", func() {
targets, err := model.ParseTargets([]string{"1:Music/Rock", "", "2:Classical"})
Expect(err).ToNot(HaveOccurred())
Expect(targets).To(HaveLen(2))
})
It("handles paths with colons", func() {
targets, err := model.ParseTargets([]string{"1:C:/Music/Rock", "2:/path:with:colons"})
Expect(err).ToNot(HaveOccurred())
Expect(targets).To(HaveLen(2))
Expect(targets[0].FolderPath).To(Equal("C:/Music/Rock"))
Expect(targets[1].FolderPath).To(Equal("/path:with:colons"))
})
It("returns error for invalid format without colon", func() {
_, err := model.ParseTargets([]string{"1Music/Rock"})
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("invalid target format"))
})
It("returns error for non-numeric library ID", func() {
_, err := model.ParseTargets([]string{"abc:Music/Rock"})
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("invalid library ID"))
})
It("returns error for negative library ID", func() {
_, err := model.ParseTargets([]string{"-1:Music/Rock"})
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("invalid library ID"))
})
It("returns error for zero library ID", func() {
_, err := model.ParseTargets([]string{"0:Music/Rock"})
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("invalid library ID"))
})
It("returns error for empty input", func() {
_, err := model.ParseTargets([]string{})
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("no valid targets found"))
})
It("returns error for all empty strings", func() {
_, err := model.ParseTargets([]string{"", " ", ""})
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("no valid targets found"))
})
})

View File

@ -106,6 +106,7 @@ func NewAlbumRepository(ctx context.Context, db dbx.Builder) model.AlbumReposito
"random": "random",
"recently_added": recentlyAddedSort(),
"starred_at": "starred, starred_at",
"rated_at": "rating, rated_at",
})
return r
}
@ -337,8 +338,12 @@ on conflict (user_id, item_id, item_type) do update
return r.executeSQL(query)
}
func (r *albumRepository) purgeEmpty() error {
func (r *albumRepository) purgeEmpty(libraryIDs ...int) error {
del := Delete(r.tableName).Where("id not in (select distinct(album_id) from media_file)")
// If libraryIDs are specified, only purge albums from those libraries
if len(libraryIDs) > 0 {
del = del.Where(Eq{"library_id": libraryIDs})
}
c, err := r.executeSQL(del)
if err != nil {
return fmt.Errorf("purging empty albums: %w", err)

View File

@ -141,6 +141,7 @@ func NewArtistRepository(ctx context.Context, db dbx.Builder) model.ArtistReposi
r.setSortMappings(map[string]string{
"name": "order_artist_name",
"starred_at": "starred, starred_at",
"rated_at": "rating, rated_at",
"song_count": "stats->>'total'->>'m'",
"album_count": "stats->>'total'->>'a'",
"size": "stats->>'total'->>'s'",

View File

@ -4,7 +4,10 @@ import (
"context"
"encoding/json"
"fmt"
"os"
"path/filepath"
"slices"
"strings"
"time"
. "github.com/Masterminds/squirrel"
@ -91,8 +94,47 @@ func (r folderRepository) CountAll(opt ...model.QueryOptions) (int64, error) {
return r.count(query)
}
func (r folderRepository) GetLastUpdates(lib model.Library) (map[string]model.FolderUpdateInfo, error) {
sq := r.newSelect().Columns("id", "updated_at", "hash").Where(Eq{"library_id": lib.ID, "missing": false})
func (r folderRepository) GetFolderUpdateInfo(lib model.Library, targetPaths ...string) (map[string]model.FolderUpdateInfo, error) {
where := And{
Eq{"library_id": lib.ID},
Eq{"missing": false},
}
// If specific paths are requested, include those folders and all their descendants
if len(targetPaths) > 0 {
// Collect folder IDs for exact target folders and path conditions for descendants
folderIDs := make([]string, 0, len(targetPaths))
pathConditions := make(Or, 0, len(targetPaths)*2)
for _, targetPath := range targetPaths {
if targetPath == "" || targetPath == "." {
// Root path - include everything in this library
pathConditions = Or{}
folderIDs = nil
break
}
// Clean the path to normalize it. Paths stored in the folder table do not have leading/trailing slashes.
cleanPath := strings.TrimPrefix(targetPath, string(os.PathSeparator))
cleanPath = filepath.Clean(cleanPath)
// Include the target folder itself by ID
folderIDs = append(folderIDs, model.FolderID(lib, cleanPath))
// Include all descendants: folders whose path field equals or starts with the target path
// Note: Folder.Path is the directory path, so children have path = targetPath
pathConditions = append(pathConditions, Eq{"path": cleanPath})
pathConditions = append(pathConditions, Like{"path": cleanPath + "/%"})
}
// Combine conditions: exact folder IDs OR descendant path patterns
if len(folderIDs) > 0 {
where = append(where, Or{Eq{"id": folderIDs}, pathConditions})
} else if len(pathConditions) > 0 {
where = append(where, pathConditions)
}
}
sq := r.newSelect().Columns("id", "updated_at", "hash").Where(where)
var res []struct {
ID string
UpdatedAt time.Time
@ -149,7 +191,7 @@ func (r folderRepository) GetTouchedWithPlaylists() (model.FolderCursor, error)
}, nil
}
func (r folderRepository) purgeEmpty() error {
func (r folderRepository) purgeEmpty(libraryIDs ...int) error {
sq := Delete(r.tableName).Where(And{
Eq{"num_audio_files": 0},
Eq{"num_playlists": 0},
@ -157,6 +199,10 @@ func (r folderRepository) purgeEmpty() error {
ConcatExpr("id not in (select parent_id from folder)"),
ConcatExpr("id not in (select folder_id from media_file)"),
})
// If libraryIDs are specified, only purge folders from those libraries
if len(libraryIDs) > 0 {
sq = sq.Where(Eq{"library_id": libraryIDs})
}
c, err := r.executeSQL(sq)
if err != nil {
return fmt.Errorf("purging empty folders: %w", err)

View File

@ -0,0 +1,213 @@
package persistence
import (
"context"
"fmt"
"github.com/navidrome/navidrome/log"
"github.com/navidrome/navidrome/model"
"github.com/navidrome/navidrome/model/request"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/pocketbase/dbx"
)
var _ = Describe("FolderRepository", func() {
var repo model.FolderRepository
var ctx context.Context
var conn *dbx.DB
var testLib, otherLib model.Library
BeforeEach(func() {
ctx = request.WithUser(log.NewContext(context.TODO()), model.User{ID: "userid"})
conn = GetDBXBuilder()
repo = newFolderRepository(ctx, conn)
// Use existing library ID 1 from test fixtures
libRepo := NewLibraryRepository(ctx, conn)
lib, err := libRepo.Get(1)
Expect(err).ToNot(HaveOccurred())
testLib = *lib
// Create a second library with its own folder to verify isolation
otherLib = model.Library{Name: "Other Library", Path: "/other/path"}
Expect(libRepo.Put(&otherLib)).To(Succeed())
})
AfterEach(func() {
// Clean up only test folders created by our tests (paths starting with "Test")
// This prevents interference with fixture data needed by other tests
_, _ = conn.NewQuery("DELETE FROM folder WHERE library_id = 1 AND path LIKE 'Test%'").Execute()
_, _ = conn.NewQuery(fmt.Sprintf("DELETE FROM library WHERE id = %d", otherLib.ID)).Execute()
})
Describe("GetFolderUpdateInfo", func() {
Context("with no target paths", func() {
It("returns all folders in the library", func() {
// Create test folders with unique names to avoid conflicts
folder1 := model.NewFolder(testLib, "TestGetLastUpdates/Folder1")
folder2 := model.NewFolder(testLib, "TestGetLastUpdates/Folder2")
err := repo.Put(folder1)
Expect(err).ToNot(HaveOccurred())
err = repo.Put(folder2)
Expect(err).ToNot(HaveOccurred())
otherFolder := model.NewFolder(otherLib, "TestOtherLib/Folder")
err = repo.Put(otherFolder)
Expect(err).ToNot(HaveOccurred())
// Query all folders (no target paths) - should only return folders from testLib
results, err := repo.GetFolderUpdateInfo(testLib)
Expect(err).ToNot(HaveOccurred())
// Should include folders from testLib
Expect(results).To(HaveKey(folder1.ID))
Expect(results).To(HaveKey(folder2.ID))
// Should NOT include folders from other library
Expect(results).ToNot(HaveKey(otherFolder.ID))
})
})
Context("with specific target paths", func() {
It("returns folder info for existing folders", func() {
// Create test folders with unique names
folder1 := model.NewFolder(testLib, "TestSpecific/Rock")
folder2 := model.NewFolder(testLib, "TestSpecific/Jazz")
folder3 := model.NewFolder(testLib, "TestSpecific/Classical")
err := repo.Put(folder1)
Expect(err).ToNot(HaveOccurred())
err = repo.Put(folder2)
Expect(err).ToNot(HaveOccurred())
err = repo.Put(folder3)
Expect(err).ToNot(HaveOccurred())
// Query specific paths
results, err := repo.GetFolderUpdateInfo(testLib, "TestSpecific/Rock", "TestSpecific/Classical")
Expect(err).ToNot(HaveOccurred())
Expect(results).To(HaveLen(2))
// Verify folder IDs are in results
Expect(results).To(HaveKey(folder1.ID))
Expect(results).To(HaveKey(folder3.ID))
Expect(results).ToNot(HaveKey(folder2.ID))
// Verify update info is populated
Expect(results[folder1.ID].UpdatedAt).ToNot(BeZero())
Expect(results[folder1.ID].Hash).To(Equal(folder1.Hash))
})
It("includes all child folders when querying parent", func() {
// Create a parent folder with multiple children
parent := model.NewFolder(testLib, "TestParent/Music")
child1 := model.NewFolder(testLib, "TestParent/Music/Rock/Queen")
child2 := model.NewFolder(testLib, "TestParent/Music/Jazz")
otherParent := model.NewFolder(testLib, "TestParent2/Music/Jazz")
Expect(repo.Put(parent)).To(Succeed())
Expect(repo.Put(child1)).To(Succeed())
Expect(repo.Put(child2)).To(Succeed())
// Query the parent folder - should return parent and all children
results, err := repo.GetFolderUpdateInfo(testLib, "TestParent/Music")
Expect(err).ToNot(HaveOccurred())
Expect(results).To(HaveLen(3))
Expect(results).To(HaveKey(parent.ID))
Expect(results).To(HaveKey(child1.ID))
Expect(results).To(HaveKey(child2.ID))
Expect(results).ToNot(HaveKey(otherParent.ID))
})
It("excludes children from other libraries", func() {
// Create parent in testLib
parent := model.NewFolder(testLib, "TestIsolation/Parent")
child := model.NewFolder(testLib, "TestIsolation/Parent/Child")
Expect(repo.Put(parent)).To(Succeed())
Expect(repo.Put(child)).To(Succeed())
// Create similar path in other library
otherParent := model.NewFolder(otherLib, "TestIsolation/Parent")
otherChild := model.NewFolder(otherLib, "TestIsolation/Parent/Child")
Expect(repo.Put(otherParent)).To(Succeed())
Expect(repo.Put(otherChild)).To(Succeed())
// Query should only return folders from testLib
results, err := repo.GetFolderUpdateInfo(testLib, "TestIsolation/Parent")
Expect(err).ToNot(HaveOccurred())
Expect(results).To(HaveLen(2))
Expect(results).To(HaveKey(parent.ID))
Expect(results).To(HaveKey(child.ID))
Expect(results).ToNot(HaveKey(otherParent.ID))
Expect(results).ToNot(HaveKey(otherChild.ID))
})
It("excludes missing children when querying parent", func() {
// Create parent and children, mark one as missing
parent := model.NewFolder(testLib, "TestMissingChild/Parent")
child1 := model.NewFolder(testLib, "TestMissingChild/Parent/Child1")
child2 := model.NewFolder(testLib, "TestMissingChild/Parent/Child2")
child2.Missing = true
Expect(repo.Put(parent)).To(Succeed())
Expect(repo.Put(child1)).To(Succeed())
Expect(repo.Put(child2)).To(Succeed())
// Query parent - should only return parent and non-missing child
results, err := repo.GetFolderUpdateInfo(testLib, "TestMissingChild/Parent")
Expect(err).ToNot(HaveOccurred())
Expect(results).To(HaveLen(2))
Expect(results).To(HaveKey(parent.ID))
Expect(results).To(HaveKey(child1.ID))
Expect(results).ToNot(HaveKey(child2.ID))
})
It("handles mix of existing and non-existing target paths", func() {
// Create folders for one path but not the other
existingParent := model.NewFolder(testLib, "TestMixed/Exists")
existingChild := model.NewFolder(testLib, "TestMixed/Exists/Child")
Expect(repo.Put(existingParent)).To(Succeed())
Expect(repo.Put(existingChild)).To(Succeed())
// Query both existing and non-existing paths
results, err := repo.GetFolderUpdateInfo(testLib, "TestMixed/Exists", "TestMixed/DoesNotExist")
Expect(err).ToNot(HaveOccurred())
Expect(results).To(HaveLen(2))
Expect(results).To(HaveKey(existingParent.ID))
Expect(results).To(HaveKey(existingChild.ID))
})
It("handles empty folder path as root", func() {
// Test querying for root folder without creating it (fixtures should have one)
rootFolderID := model.FolderID(testLib, ".")
results, err := repo.GetFolderUpdateInfo(testLib, "")
Expect(err).ToNot(HaveOccurred())
// Should return the root folder if it exists
if len(results) > 0 {
Expect(results).To(HaveKey(rootFolderID))
}
})
It("returns empty map for non-existent folders", func() {
results, err := repo.GetFolderUpdateInfo(testLib, "NonExistent/Path")
Expect(err).ToNot(HaveOccurred())
Expect(results).To(BeEmpty())
})
It("skips missing folders", func() {
// Create a folder and mark it as missing
folder := model.NewFolder(testLib, "TestMissing/Folder")
folder.Missing = true
err := repo.Put(folder)
Expect(err).ToNot(HaveOccurred())
results, err := repo.GetFolderUpdateInfo(testLib, "TestMissing/Folder")
Expect(err).ToNot(HaveOccurred())
Expect(results).To(BeEmpty())
})
})
})
})

View File

@ -177,7 +177,11 @@ func (r *libraryRepository) ScanEnd(id int) error {
return err
}
// https://www.sqlite.org/pragma.html#pragma_optimize
_, err = r.executeSQL(Expr("PRAGMA optimize=0x10012;"))
// Use mask 0x10000 to check table sizes without running ANALYZE
// Running ANALYZE can cause query planner issues with expression-based collation indexes
if conf.Server.DevOptimizeDB {
_, err = r.executeSQL(Expr("PRAGMA optimize=0x10000;"))
}
return err
}

View File

@ -142,4 +142,62 @@ var _ = Describe("LibraryRepository", func() {
Expect(libAfter.TotalSize).To(Equal(sizeRes.Sum))
Expect(libAfter.TotalDuration).To(Equal(durationRes.Sum))
})
Describe("ScanBegin and ScanEnd", func() {
var lib *model.Library
BeforeEach(func() {
lib = &model.Library{
ID: 0,
Name: "Test Scan Library",
Path: "/music/test-scan",
}
err := repo.Put(lib)
Expect(err).ToNot(HaveOccurred())
})
DescribeTable("ScanBegin",
func(fullScan bool, expectedFullScanInProgress bool) {
err := repo.ScanBegin(lib.ID, fullScan)
Expect(err).ToNot(HaveOccurred())
updatedLib, err := repo.Get(lib.ID)
Expect(err).ToNot(HaveOccurred())
Expect(updatedLib.LastScanStartedAt).ToNot(BeZero())
Expect(updatedLib.FullScanInProgress).To(Equal(expectedFullScanInProgress))
},
Entry("sets FullScanInProgress to true for full scan", true, true),
Entry("sets FullScanInProgress to false for quick scan", false, false),
)
Context("ScanEnd", func() {
BeforeEach(func() {
err := repo.ScanBegin(lib.ID, true)
Expect(err).ToNot(HaveOccurred())
})
It("sets LastScanAt and clears FullScanInProgress and LastScanStartedAt", func() {
err := repo.ScanEnd(lib.ID)
Expect(err).ToNot(HaveOccurred())
updatedLib, err := repo.Get(lib.ID)
Expect(err).ToNot(HaveOccurred())
Expect(updatedLib.LastScanAt).ToNot(BeZero())
Expect(updatedLib.FullScanInProgress).To(BeFalse())
Expect(updatedLib.LastScanStartedAt).To(BeZero())
})
It("sets LastScanAt to be after LastScanStartedAt", func() {
libBefore, err := repo.Get(lib.ID)
Expect(err).ToNot(HaveOccurred())
err = repo.ScanEnd(lib.ID)
Expect(err).ToNot(HaveOccurred())
libAfter, err := repo.Get(lib.ID)
Expect(err).ToNot(HaveOccurred())
Expect(libAfter.LastScanAt).To(BeTemporally(">=", libBefore.LastScanStartedAt))
})
})
})
})

View File

@ -84,6 +84,7 @@ func NewMediaFileRepository(ctx context.Context, db dbx.Builder) model.MediaFile
"created_at": "media_file.created_at",
"recently_added": mediaFileRecentlyAddedSort(),
"starred_at": "starred, starred_at",
"rated_at": "rating, rated_at",
})
return r
}

View File

@ -157,7 +157,7 @@ func (s *SQLStore) WithTxImmediate(block func(tx model.DataStore) error, scope .
}, scope...)
}
func (s *SQLStore) GC(ctx context.Context) error {
func (s *SQLStore) GC(ctx context.Context, libraryIDs ...int) error {
trace := func(ctx context.Context, msg string, f func() error) func() error {
return func() error {
start := time.Now()
@ -167,11 +167,17 @@ func (s *SQLStore) GC(ctx context.Context) error {
}
}
// If libraryIDs are provided, scope operations to those libraries where possible
scoped := len(libraryIDs) > 0
if scoped {
log.Debug(ctx, "GC: Running selective garbage collection", "libraryIDs", libraryIDs)
}
err := run.Sequentially(
trace(ctx, "purge empty albums", func() error { return s.Album(ctx).(*albumRepository).purgeEmpty() }),
trace(ctx, "purge empty albums", func() error { return s.Album(ctx).(*albumRepository).purgeEmpty(libraryIDs...) }),
trace(ctx, "purge empty artists", func() error { return s.Artist(ctx).(*artistRepository).purgeEmpty() }),
trace(ctx, "mark missing artists", func() error { return s.Artist(ctx).(*artistRepository).markMissing() }),
trace(ctx, "purge empty folders", func() error { return s.Folder(ctx).(*folderRepository).purgeEmpty() }),
trace(ctx, "purge empty folders", func() error { return s.Folder(ctx).(*folderRepository).purgeEmpty(libraryIDs...) }),
trace(ctx, "clean album annotations", func() error { return s.Album(ctx).(*albumRepository).cleanAnnotations() }),
trace(ctx, "clean artist annotations", func() error { return s.Artist(ctx).(*artistRepository).cleanAnnotations() }),
trace(ctx, "clean media file annotations", func() error { return s.MediaFile(ctx).(*mediaFileRepository).cleanAnnotations() }),

View File

@ -264,6 +264,11 @@ func (r *playlistRepository) refreshSmartPlaylist(pls *model.Playlist) bool {
"annotation.item_id = media_file.id" +
" AND annotation.item_type = 'media_file'" +
" AND annotation.user_id = '" + usr.ID + "')")
// Only include media files from libraries the user has access to
sq = r.applyLibraryFilter(sq, "media_file")
// Apply the criteria rules
sq = r.addCriteria(sq, rules)
insSql := Insert("playlist_tracks").Columns("id", "playlist_id", "media_file_id").Select(sq)
_, err = r.executeSQL(insSql)
@ -388,6 +393,7 @@ func (r *playlistRepository) loadTracks(sel SelectBuilder, id string) (model.Pla
"coalesce(play_count, 0) as play_count",
"play_date",
"coalesce(rating, 0) as rating",
"rated_at",
"f.*",
"playlist_tracks.*",
"library.path as library_path",

View File

@ -1,7 +1,6 @@
package persistence
import (
"context"
"time"
"github.com/navidrome/navidrome/conf"
@ -11,13 +10,14 @@ import (
"github.com/navidrome/navidrome/model/request"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/pocketbase/dbx"
)
var _ = Describe("PlaylistRepository", func() {
var repo model.PlaylistRepository
BeforeEach(func() {
ctx := log.NewContext(context.TODO())
ctx := log.NewContext(GinkgoT().Context())
ctx = request.WithUser(ctx, model.User{ID: "userid", UserName: "userid", IsAdmin: true})
repo = NewPlaylistRepository(ctx, GetDBXBuilder())
})
@ -252,4 +252,250 @@ var _ = Describe("PlaylistRepository", func() {
Expect(tracks[3].MediaFileID).To(Equal("2001")) // Disc 2, Track 11
})
})
Describe("Smart Playlists with Tag Criteria", func() {
var mfRepo model.MediaFileRepository
var testPlaylistID string
var songWithGrouping, songWithoutGrouping model.MediaFile
BeforeEach(func() {
ctx := log.NewContext(GinkgoT().Context())
ctx = request.WithUser(ctx, model.User{ID: "userid", UserName: "userid", IsAdmin: true})
mfRepo = NewMediaFileRepository(ctx, GetDBXBuilder())
// Register 'grouping' as a valid tag for smart playlists
criteria.AddTagNames([]string{"grouping"})
// Create a song with the grouping tag
songWithGrouping = model.MediaFile{
ID: "test-grouping-1",
Title: "Song With Grouping",
Artist: "Test Artist",
ArtistID: "1",
Album: "Test Album",
AlbumID: "101",
Path: "/test/grouping/song1.mp3",
Tags: model.Tags{
"grouping": []string{"My Crate"},
},
Participants: model.Participants{},
LibraryID: 1,
Lyrics: "[]",
}
Expect(mfRepo.Put(&songWithGrouping)).To(Succeed())
// Create a song without the grouping tag
songWithoutGrouping = model.MediaFile{
ID: "test-grouping-2",
Title: "Song Without Grouping",
Artist: "Test Artist",
ArtistID: "1",
Album: "Test Album",
AlbumID: "101",
Path: "/test/grouping/song2.mp3",
Tags: model.Tags{},
Participants: model.Participants{},
LibraryID: 1,
Lyrics: "[]",
}
Expect(mfRepo.Put(&songWithoutGrouping)).To(Succeed())
})
AfterEach(func() {
if testPlaylistID != "" {
_ = repo.Delete(testPlaylistID)
testPlaylistID = ""
}
// Clean up test media files
_, _ = GetDBXBuilder().Delete("media_file", dbx.HashExp{"id": "test-grouping-1"}).Execute()
_, _ = GetDBXBuilder().Delete("media_file", dbx.HashExp{"id": "test-grouping-2"}).Execute()
})
It("matches tracks with a tag value using 'contains' with empty string (issue #4728 workaround)", func() {
By("creating a smart playlist that checks if grouping tag has any value")
// This is the workaround for issue #4728: using 'contains' with empty string
// generates SQL: value LIKE '%%' which matches any non-empty string
rules := &criteria.Criteria{
Expression: criteria.All{
criteria.Contains{"grouping": ""},
},
}
newPls := model.Playlist{Name: "Tracks with Grouping", OwnerID: "userid", Rules: rules}
Expect(repo.Put(&newPls)).To(Succeed())
testPlaylistID = newPls.ID
By("refreshing the smart playlist")
conf.Server.SmartPlaylistRefreshDelay = -1 * time.Second // Force refresh
pls, err := repo.GetWithTracks(newPls.ID, true, false)
Expect(err).ToNot(HaveOccurred())
By("verifying only the track with grouping tag is matched")
Expect(pls.Tracks).To(HaveLen(1))
Expect(pls.Tracks[0].MediaFileID).To(Equal(songWithGrouping.ID))
})
It("excludes tracks with a tag value using 'notContains' with empty string", func() {
By("creating a smart playlist that checks if grouping tag is NOT set")
rules := &criteria.Criteria{
Expression: criteria.All{
criteria.NotContains{"grouping": ""},
},
}
newPls := model.Playlist{Name: "Tracks without Grouping", OwnerID: "userid", Rules: rules}
Expect(repo.Put(&newPls)).To(Succeed())
testPlaylistID = newPls.ID
By("refreshing the smart playlist")
conf.Server.SmartPlaylistRefreshDelay = -1 * time.Second // Force refresh
pls, err := repo.GetWithTracks(newPls.ID, true, false)
Expect(err).ToNot(HaveOccurred())
By("verifying the track with grouping is NOT in the playlist")
for _, track := range pls.Tracks {
Expect(track.MediaFileID).ToNot(Equal(songWithGrouping.ID))
}
By("verifying the track without grouping IS in the playlist")
var foundWithoutGrouping bool
for _, track := range pls.Tracks {
if track.MediaFileID == songWithoutGrouping.ID {
foundWithoutGrouping = true
break
}
}
Expect(foundWithoutGrouping).To(BeTrue())
})
})
Describe("Smart Playlists Library Filtering", func() {
var mfRepo model.MediaFileRepository
var testPlaylistID string
var lib2ID int
var restrictedUserID string
var uniqueLibPath string
BeforeEach(func() {
db := GetDBXBuilder()
// Generate unique IDs for this test run
uniqueSuffix := time.Now().Format("20060102150405.000")
restrictedUserID = "restricted-user-" + uniqueSuffix
uniqueLibPath = "/music/lib2-" + uniqueSuffix
// Create a second library with unique name and path to avoid conflicts with other tests
_, err := db.DB().Exec("INSERT INTO library (name, path, created_at, updated_at) VALUES (?, ?, datetime('now'), datetime('now'))", "Library 2-"+uniqueSuffix, uniqueLibPath)
Expect(err).ToNot(HaveOccurred())
err = db.DB().QueryRow("SELECT last_insert_rowid()").Scan(&lib2ID)
Expect(err).ToNot(HaveOccurred())
// Create a restricted user with access only to library 1
_, err = db.DB().Exec("INSERT INTO user (id, user_name, name, is_admin, password, created_at, updated_at) VALUES (?, ?, 'Restricted User', false, 'pass', datetime('now'), datetime('now'))", restrictedUserID, restrictedUserID)
Expect(err).ToNot(HaveOccurred())
_, err = db.DB().Exec("INSERT INTO user_library (user_id, library_id) VALUES (?, 1)", restrictedUserID)
Expect(err).ToNot(HaveOccurred())
// Create test media files in each library
ctx := log.NewContext(GinkgoT().Context())
ctx = request.WithUser(ctx, model.User{ID: "userid", UserName: "userid", IsAdmin: true})
mfRepo = NewMediaFileRepository(ctx, db)
// Song in library 1 (accessible by restricted user)
songLib1 := model.MediaFile{
ID: "lib1-song",
Title: "Song in Lib1",
Artist: "Test Artist",
ArtistID: "1",
Album: "Test Album",
AlbumID: "101",
Path: "/music/lib1/song.mp3",
LibraryID: 1,
Participants: model.Participants{},
Tags: model.Tags{},
Lyrics: "[]",
}
Expect(mfRepo.Put(&songLib1)).To(Succeed())
// Song in library 2 (NOT accessible by restricted user)
songLib2 := model.MediaFile{
ID: "lib2-song",
Title: "Song in Lib2",
Artist: "Test Artist",
ArtistID: "1",
Album: "Test Album",
AlbumID: "101",
Path: uniqueLibPath + "/song.mp3",
LibraryID: lib2ID,
Participants: model.Participants{},
Tags: model.Tags{},
Lyrics: "[]",
}
Expect(mfRepo.Put(&songLib2)).To(Succeed())
})
AfterEach(func() {
db := GetDBXBuilder()
if testPlaylistID != "" {
_ = repo.Delete(testPlaylistID)
testPlaylistID = ""
}
// Clean up test data
_, _ = db.Delete("media_file", dbx.HashExp{"id": "lib1-song"}).Execute()
_, _ = db.Delete("media_file", dbx.HashExp{"id": "lib2-song"}).Execute()
_, _ = db.Delete("user_library", dbx.HashExp{"user_id": restrictedUserID}).Execute()
_, _ = db.Delete("user", dbx.HashExp{"id": restrictedUserID}).Execute()
_, _ = db.DB().Exec("DELETE FROM library WHERE id = ?", lib2ID)
})
It("should only include tracks from libraries the user has access to (issue #4738)", func() {
db := GetDBXBuilder()
ctx := log.NewContext(GinkgoT().Context())
// Create the smart playlist as the restricted user
restrictedUser := model.User{ID: restrictedUserID, UserName: restrictedUserID, IsAdmin: false}
ctx = request.WithUser(ctx, restrictedUser)
restrictedRepo := NewPlaylistRepository(ctx, db)
// Create a smart playlist that matches all songs
rules := &criteria.Criteria{
Expression: criteria.All{
criteria.Gt{"playCount": -1}, // Matches everything
},
}
newPls := model.Playlist{Name: "All Songs", OwnerID: restrictedUserID, Rules: rules}
Expect(restrictedRepo.Put(&newPls)).To(Succeed())
testPlaylistID = newPls.ID
By("refreshing the smart playlist")
conf.Server.SmartPlaylistRefreshDelay = -1 * time.Second // Force refresh
pls, err := restrictedRepo.GetWithTracks(newPls.ID, true, false)
Expect(err).ToNot(HaveOccurred())
By("verifying only the track from library 1 is in the playlist")
var foundLib1Song, foundLib2Song bool
for _, track := range pls.Tracks {
if track.MediaFileID == "lib1-song" {
foundLib1Song = true
}
if track.MediaFileID == "lib2-song" {
foundLib2Song = true
}
}
Expect(foundLib1Song).To(BeTrue(), "Song from library 1 should be in the playlist")
Expect(foundLib2Song).To(BeFalse(), "Song from library 2 should NOT be in the playlist")
By("verifying playlist_tracks table only contains the accessible track")
var playlistTracksCount int
err = db.DB().QueryRow("SELECT count(*) FROM playlist_tracks WHERE playlist_id = ?", newPls.ID).Scan(&playlistTracksCount)
Expect(err).ToNot(HaveOccurred())
// Count should only include tracks visible to the user (lib1-song)
// The count may include other test songs from library 1, but NOT lib2-song
var lib2TrackCount int
err = db.DB().QueryRow("SELECT count(*) FROM playlist_tracks WHERE playlist_id = ? AND media_file_id = 'lib2-song'", newPls.ID).Scan(&lib2TrackCount)
Expect(err).ToNot(HaveOccurred())
Expect(lib2TrackCount).To(Equal(0), "lib2-song should not be in playlist_tracks")
By("verifying SongCount matches visible tracks")
Expect(pls.SongCount).To(Equal(len(pls.Tracks)), "SongCount should match the number of visible tracks")
})
})
})

View File

@ -97,6 +97,7 @@ func (r *playlistTrackRepository) Read(id string) (interface{}, error) {
"coalesce(rating, 0) as rating",
"starred_at",
"play_date",
"rated_at",
"f.*",
"playlist_tracks.*",
).

View File

@ -28,6 +28,7 @@ func (r sqlRepository) withAnnotation(query SelectBuilder, idField string) Selec
"coalesce(rating, 0) as rating",
"starred_at",
"play_date",
"rated_at",
)
if conf.Server.AlbumPlayCountMode == consts.AlbumPlayCountModeNormalized && r.tableName == "album" {
query = query.Columns(
@ -77,7 +78,8 @@ func (r sqlRepository) SetStar(starred bool, ids ...string) error {
}
func (r sqlRepository) SetRating(rating int, itemID string) error {
return r.annUpsert(map[string]interface{}{"rating": rating}, itemID)
ratedAt := time.Now()
return r.annUpsert(map[string]interface{}{"rating": rating, "rated_at": ratedAt}, itemID)
}
func (r sqlRepository) IncPlayCount(itemID string, ts time.Time) error {
@ -119,7 +121,7 @@ func (r sqlRepository) cleanAnnotations() error {
del := Delete(annotationTable).Where(Eq{"item_type": r.tableName}).Where("item_id not in (select id from " + r.tableName + ")")
c, err := r.executeSQL(del)
if err != nil {
return fmt.Errorf("error cleaning up annotations: %w", err)
return fmt.Errorf("error cleaning up %s annotations: %w", r.tableName, err)
}
if c > 0 {
log.Debug(r.ctx, "Clean-up annotations", "table", r.tableName, "totalDeleted", c)

View File

@ -148,10 +148,10 @@ func (r sqlRepository) cleanBookmarks() error {
del := Delete(bookmarkTable).Where(Eq{"item_type": r.tableName}).Where("item_id not in (select id from " + r.tableName + ")")
c, err := r.executeSQL(del)
if err != nil {
return fmt.Errorf("error cleaning up bookmarks: %w", err)
return fmt.Errorf("error cleaning up %s bookmarks: %w", r.tableName, err)
}
if c > 0 {
log.Debug(r.ctx, "Clean-up bookmarks", "totalDeleted", c)
log.Debug(r.ctx, "Clean-up bookmarks", "totalDeleted", c, "itemType", r.tableName)
}
return nil
}

View File

@ -2,6 +2,7 @@ package persistence
import (
"context"
"time"
"github.com/deluan/rest"
"github.com/navidrome/navidrome/conf/configtest"
@ -45,6 +46,9 @@ var _ = Describe("Tag Library Filtering", func() {
BeforeEach(func() {
DeferCleanup(configtest.SetupConfig())
// Generate unique path suffix to avoid conflicts with other tests
uniqueSuffix := time.Now().Format("20060102150405.000")
// Clean up database
db := GetDBXBuilder()
_, err := db.NewQuery("DELETE FROM library_tag").Execute()
@ -57,12 +61,12 @@ var _ = Describe("Tag Library Filtering", func() {
_, err = db.NewQuery("DELETE FROM library WHERE id > 1").Execute()
Expect(err).ToNot(HaveOccurred())
// Create test libraries
// Create test libraries with unique names and paths to avoid conflicts with other tests
_, err = db.NewQuery("INSERT INTO library (id, name, path) VALUES ({:id}, {:name}, {:path})").
Bind(dbx.Params{"id": libraryID2, "name": "Library 2", "path": "/music/lib2"}).Execute()
Bind(dbx.Params{"id": libraryID2, "name": "Library 2-" + uniqueSuffix, "path": "/music/lib2-" + uniqueSuffix}).Execute()
Expect(err).ToNot(HaveOccurred())
_, err = db.NewQuery("INSERT INTO library (id, name, path) VALUES ({:id}, {:name}, {:path})").
Bind(dbx.Params{"id": libraryID3, "name": "Library 3", "path": "/music/lib3"}).Execute()
Bind(dbx.Params{"id": libraryID3, "name": "Library 3-" + uniqueSuffix, "path": "/music/lib3-" + uniqueSuffix}).Execute()
Expect(err).ToNot(HaveOccurred())
// Give admin access to all libraries

View File

@ -88,10 +88,10 @@ func (r *tagRepository) purgeUnused() error {
`)
c, err := r.executeSQL(del)
if err != nil {
return fmt.Errorf("error purging unused tags: %w", err)
return fmt.Errorf("error purging %s unused tags: %w", r.tableName, err)
}
if c > 0 {
log.Debug(r.ctx, "Purged unused tags", "totalDeleted", c)
log.Debug(r.ctx, "Purged unused tags", "totalDeleted", c, "table", r.tableName)
}
return err
}

View File

@ -93,8 +93,12 @@ func (s *subsonicAPIServiceImpl) Call(ctx context.Context, req *subsonicapi.Call
RawQuery: query.Encode(),
}
// Create HTTP request with internal authentication
httpReq, err := http.NewRequestWithContext(ctx, "GET", finalURL.String(), nil)
// Create HTTP request with a fresh context to avoid Chi RouteContext pollution.
// Using http.NewRequest (instead of http.NewRequestWithContext) ensures the internal
// SubsonicAPI call doesn't inherit routing information from the parent handler,
// which would cause Chi to invoke the wrong handler. Authentication context is
// explicitly added in the next step via request.WithInternalAuth.
httpReq, err := http.NewRequest("GET", finalURL.String(), nil)
if err != nil {
return &subsonicapi.CallResponse{
Error: fmt.Sprintf("failed to create HTTP request: %v", err),

View File

@ -4,6 +4,7 @@ import (
"context"
"os"
"path/filepath"
"time"
"github.com/navidrome/navidrome/conf"
"github.com/navidrome/navidrome/core/agents"
@ -22,8 +23,11 @@ var _ = Describe("Plugin Manager", func() {
// but, as this is an integration test, we can't use configtest.SetupConfig() as it causes
// data races.
originalPluginsFolder := conf.Server.Plugins.Folder
originalTimeout := conf.Server.DevPluginCompilationTimeout
conf.Server.DevPluginCompilationTimeout = 2 * time.Minute
DeferCleanup(func() {
conf.Server.Plugins.Folder = originalPluginsFolder
conf.Server.DevPluginCompilationTimeout = originalTimeout
})
conf.Server.Plugins.Enabled = true
conf.Server.Plugins.Folder = testDataDir

View File

@ -300,6 +300,8 @@
},
"actions": {
"scan": "Scanear Biblioteca",
"quickScan": "Scan Rápido",
"fullScan": "Scan Completo",
"manageUsers": "Gerenciar Acesso do Usuário",
"viewDetails": "Ver Detalhes"
},
@ -308,6 +310,9 @@
"updated": "Biblioteca atualizada com sucesso",
"deleted": "Biblioteca excluída com sucesso",
"scanStarted": "Scan da biblioteca iniciada",
"quickScanStarted": "Scan rápido iniciado",
"fullScanStarted": "Scan completo iniciado",
"scanError": "Erro ao iniciar o scan. Verifique os logs",
"scanCompleted": "Scan da biblioteca concluída"
},
"validation": {
@ -598,11 +603,12 @@
"activity": {
"title": "Atividade",
"totalScanned": "Total de pastas scaneadas",
"quickScan": "Scan rápido",
"fullScan": "Scan completo",
"quickScan": "Rápido",
"fullScan": "Completo",
"selectiveScan": "Seletivo",
"serverUptime": "Uptime do servidor",
"serverDown": "DESCONECTADO",
"scanType": "Tipo",
"scanType": "Último Scan",
"status": "Erro",
"elapsedTime": "Duração"
},

View File

@ -26,24 +26,8 @@ var (
ErrAlreadyScanning = errors.New("already scanning")
)
type Scanner interface {
// ScanAll starts a full scan of the music library. This is a blocking operation.
ScanAll(ctx context.Context, fullScan bool) (warnings []string, err error)
Status(context.Context) (*StatusInfo, error)
}
type StatusInfo struct {
Scanning bool
LastScan time.Time
Count uint32
FolderCount uint32
LastError string
ScanType string
ElapsedTime time.Duration
}
func New(rootCtx context.Context, ds model.DataStore, cw artwork.CacheWarmer, broker events.Broker,
pls core.Playlists, m metrics.Metrics) Scanner {
pls core.Playlists, m metrics.Metrics) model.Scanner {
c := &controller{
rootCtx: rootCtx,
ds: ds,
@ -65,9 +49,10 @@ func (s *controller) getScanner() scanner {
return &scannerImpl{ds: s.ds, cw: s.cw, pls: s.pls}
}
// CallScan starts an in-process scan of the music library.
// CallScan starts an in-process scan of specific library/folder pairs.
// If targets is empty, it scans all libraries.
// This is meant to be called from the command line (see cmd/scan.go).
func CallScan(ctx context.Context, ds model.DataStore, pls core.Playlists, fullScan bool) (<-chan *ProgressInfo, error) {
func CallScan(ctx context.Context, ds model.DataStore, pls core.Playlists, fullScan bool, targets []model.ScanTarget) (<-chan *ProgressInfo, error) {
release, err := lockScan(ctx)
if err != nil {
return nil, err
@ -79,7 +64,7 @@ func CallScan(ctx context.Context, ds model.DataStore, pls core.Playlists, fullS
go func() {
defer close(progress)
scanner := &scannerImpl{ds: ds, cw: artwork.NoopCacheWarmer(), pls: pls}
scanner.scanAll(ctx, fullScan, progress)
scanner.scanFolders(ctx, fullScan, targets, progress)
}()
return progress, nil
}
@ -99,8 +84,11 @@ type ProgressInfo struct {
ForceUpdate bool
}
// scanner defines the interface for different scanner implementations.
// This allows for swapping between in-process and external scanners.
type scanner interface {
scanAll(ctx context.Context, fullScan bool, progress chan<- *ProgressInfo)
// scanFolders performs the actual scanning of folders. If targets is nil, it scans all libraries.
scanFolders(ctx context.Context, fullScan bool, targets []model.ScanTarget, progress chan<- *ProgressInfo)
}
type controller struct {
@ -158,7 +146,7 @@ func (s *controller) getScanInfo(ctx context.Context) (scanType string, elapsed
return scanType, elapsed, lastErr
}
func (s *controller) Status(ctx context.Context) (*StatusInfo, error) {
func (s *controller) Status(ctx context.Context) (*model.ScannerStatus, error) {
lastScanTime, err := s.getLastScanTime(ctx)
if err != nil {
return nil, fmt.Errorf("getting last scan time: %w", err)
@ -167,7 +155,7 @@ func (s *controller) Status(ctx context.Context) (*StatusInfo, error) {
scanType, elapsed, lastErr := s.getScanInfo(ctx)
if running.Load() {
status := &StatusInfo{
status := &model.ScannerStatus{
Scanning: true,
LastScan: lastScanTime,
Count: s.count.Load(),
@ -183,7 +171,7 @@ func (s *controller) Status(ctx context.Context) (*StatusInfo, error) {
if err != nil {
return nil, fmt.Errorf("getting library stats: %w", err)
}
return &StatusInfo{
return &model.ScannerStatus{
Scanning: false,
LastScan: lastScanTime,
Count: uint32(count),
@ -208,6 +196,10 @@ func (s *controller) getCounters(ctx context.Context) (int64, int64, error) {
}
func (s *controller) ScanAll(requestCtx context.Context, fullScan bool) ([]string, error) {
return s.ScanFolders(requestCtx, fullScan, nil)
}
func (s *controller) ScanFolders(requestCtx context.Context, fullScan bool, targets []model.ScanTarget) ([]string, error) {
release, err := lockScan(requestCtx)
if err != nil {
return nil, err
@ -224,7 +216,7 @@ func (s *controller) ScanAll(requestCtx context.Context, fullScan bool) ([]strin
go func() {
defer close(progress)
scanner := s.getScanner()
scanner.scanAll(ctx, fullScan, progress)
scanner.scanFolders(ctx, fullScan, targets, progress)
}()
// Wait for the scan to finish, sending progress events to all connected clients

View File

@ -9,6 +9,7 @@ import (
"github.com/navidrome/navidrome/core/artwork"
"github.com/navidrome/navidrome/core/metrics"
"github.com/navidrome/navidrome/db"
"github.com/navidrome/navidrome/model"
"github.com/navidrome/navidrome/persistence"
"github.com/navidrome/navidrome/scanner"
"github.com/navidrome/navidrome/server/events"
@ -20,7 +21,7 @@ import (
var _ = Describe("Controller", func() {
var ctx context.Context
var ds *tests.MockDataStore
var ctrl scanner.Scanner
var ctrl model.Scanner
Describe("Status", func() {
BeforeEach(func() {

View File

@ -11,7 +11,7 @@ import (
"github.com/navidrome/navidrome/conf"
"github.com/navidrome/navidrome/log"
. "github.com/navidrome/navidrome/utils/gg"
"github.com/navidrome/navidrome/model"
)
// scannerExternal is a scanner that runs an external process to do the scanning. It is used to avoid
@ -23,19 +23,42 @@ import (
// process will forward them to the caller.
type scannerExternal struct{}
func (s *scannerExternal) scanAll(ctx context.Context, fullScan bool, progress chan<- *ProgressInfo) {
func (s *scannerExternal) scanFolders(ctx context.Context, fullScan bool, targets []model.ScanTarget, progress chan<- *ProgressInfo) {
s.scan(ctx, fullScan, targets, progress)
}
func (s *scannerExternal) scan(ctx context.Context, fullScan bool, targets []model.ScanTarget, progress chan<- *ProgressInfo) {
exe, err := os.Executable()
if err != nil {
progress <- &ProgressInfo{Error: fmt.Sprintf("failed to get executable path: %s", err)}
return
}
log.Debug(ctx, "Spawning external scanner process", "fullScan", fullScan, "path", exe)
cmd := exec.CommandContext(ctx, exe, "scan",
// Build command arguments
args := []string{
"scan",
"--nobanner", "--subprocess",
"--configfile", conf.Server.ConfigFile,
"--datafolder", conf.Server.DataFolder,
"--cachefolder", conf.Server.CacheFolder,
If(fullScan, "--full", ""))
}
// Add targets if provided
if len(targets) > 0 {
for _, target := range targets {
args = append(args, "-t", target.String())
}
log.Debug(ctx, "Spawning external scanner process with targets", "fullScan", fullScan, "path", exe, "targets", targets)
} else {
log.Debug(ctx, "Spawning external scanner process", "fullScan", fullScan, "path", exe)
}
// Add full scan flag if needed
if fullScan {
args = append(args, "--full")
}
cmd := exec.CommandContext(ctx, exe, args...)
in, out := io.Pipe()
defer in.Close()

View File

@ -15,9 +15,7 @@ import (
"github.com/navidrome/navidrome/utils/chrono"
)
func newFolderEntry(job *scanJob, path string) *folderEntry {
id := model.FolderID(job.lib, path)
info := job.popLastUpdate(id)
func newFolderEntry(job *scanJob, id, path string, updTime time.Time, hash string) *folderEntry {
f := &folderEntry{
id: id,
job: job,
@ -25,8 +23,8 @@ func newFolderEntry(job *scanJob, path string) *folderEntry {
audioFiles: make(map[string]fs.DirEntry),
imageFiles: make(map[string]fs.DirEntry),
albumIDMap: make(map[string]string),
updTime: info.UpdatedAt,
prevHash: info.Hash,
updTime: updTime,
prevHash: hash,
}
return f
}

View File

@ -40,9 +40,8 @@ var _ = Describe("folder_entry", func() {
UpdatedAt: time.Now().Add(-30 * time.Minute),
Hash: "previous-hash",
}
job.lastUpdates[folderID] = updateInfo
entry := newFolderEntry(job, path)
entry := newFolderEntry(job, folderID, path, updateInfo.UpdatedAt, updateInfo.Hash)
Expect(entry.id).To(Equal(folderID))
Expect(entry.job).To(Equal(job))
@ -53,15 +52,10 @@ var _ = Describe("folder_entry", func() {
Expect(entry.updTime).To(Equal(updateInfo.UpdatedAt))
Expect(entry.prevHash).To(Equal(updateInfo.Hash))
})
})
It("creates a new folder entry with zero time when no previous update exists", func() {
entry := newFolderEntry(job, path)
Expect(entry.updTime).To(BeZero())
Expect(entry.prevHash).To(BeEmpty())
})
It("removes the lastUpdate from the job after popping", func() {
Describe("createFolderEntry", func() {
It("removes the lastUpdate from the job after creation", func() {
folderID := model.FolderID(lib, path)
updateInfo := model.FolderUpdateInfo{
UpdatedAt: time.Now().Add(-30 * time.Minute),
@ -69,8 +63,10 @@ var _ = Describe("folder_entry", func() {
}
job.lastUpdates[folderID] = updateInfo
newFolderEntry(job, path)
entry := job.createFolderEntry(path)
Expect(entry.updTime).To(Equal(updateInfo.UpdatedAt))
Expect(entry.prevHash).To(Equal(updateInfo.Hash))
Expect(job.lastUpdates).ToNot(HaveKey(folderID))
})
})
@ -79,7 +75,8 @@ var _ = Describe("folder_entry", func() {
var entry *folderEntry
BeforeEach(func() {
entry = newFolderEntry(job, path)
folderID := model.FolderID(lib, path)
entry = newFolderEntry(job, folderID, path, time.Time{}, "")
})
Describe("hasNoFiles", func() {
@ -458,7 +455,9 @@ var _ = Describe("folder_entry", func() {
Describe("integration scenarios", func() {
It("handles complete folder lifecycle", func() {
// Create new folder entry
entry := newFolderEntry(job, "music/rock/album")
folderPath := "music/rock/album"
folderID := model.FolderID(lib, folderPath)
entry := newFolderEntry(job, folderID, folderPath, time.Time{}, "")
// Initially new and has no files
Expect(entry.isNew()).To(BeTrue())

163
scanner/ignore_checker.go Normal file
View File

@ -0,0 +1,163 @@
package scanner
import (
"bufio"
"context"
"io/fs"
"path"
"strings"
"github.com/navidrome/navidrome/consts"
"github.com/navidrome/navidrome/log"
ignore "github.com/sabhiram/go-gitignore"
)
// IgnoreChecker manages .ndignore patterns using a stack-based approach.
// Use Push() to add patterns when entering a folder, Pop() when leaving,
// and ShouldIgnore() to check if a path should be ignored.
type IgnoreChecker struct {
fsys fs.FS
patternStack [][]string // Stack of patterns for each folder level
currentPatterns []string // Flattened current patterns
matcher *ignore.GitIgnore // Compiled matcher for current patterns
}
// newIgnoreChecker creates a new IgnoreChecker for the given filesystem.
func newIgnoreChecker(fsys fs.FS) *IgnoreChecker {
return &IgnoreChecker{
fsys: fsys,
patternStack: make([][]string, 0),
}
}
// Push loads .ndignore patterns from the specified folder and adds them to the pattern stack.
// Use this when entering a folder during directory tree traversal.
func (ic *IgnoreChecker) Push(ctx context.Context, folder string) error {
patterns := ic.loadPatternsFromFolder(ctx, folder)
ic.patternStack = append(ic.patternStack, patterns)
ic.rebuildCurrentPatterns()
return nil
}
// Pop removes the most recent patterns from the stack.
// Use this when leaving a folder during directory tree traversal.
func (ic *IgnoreChecker) Pop() {
if len(ic.patternStack) > 0 {
ic.patternStack = ic.patternStack[:len(ic.patternStack)-1]
ic.rebuildCurrentPatterns()
}
}
// PushAllParents pushes patterns from root down to the target path.
// This is a convenience method for when you need to check a specific path
// without recursively walking the tree. It handles the common pattern of
// pushing all parent directories from root to the target.
// This method is optimized to compile patterns only once at the end.
func (ic *IgnoreChecker) PushAllParents(ctx context.Context, targetPath string) error {
if targetPath == "." || targetPath == "" {
// Simple case: just push root
return ic.Push(ctx, ".")
}
// Load patterns for root
patterns := ic.loadPatternsFromFolder(ctx, ".")
ic.patternStack = append(ic.patternStack, patterns)
// Load patterns for each parent directory
currentPath := "."
parts := strings.Split(path.Clean(targetPath), "/")
for _, part := range parts {
if part == "." || part == "" {
continue
}
currentPath = path.Join(currentPath, part)
patterns = ic.loadPatternsFromFolder(ctx, currentPath)
ic.patternStack = append(ic.patternStack, patterns)
}
// Rebuild and compile patterns only once at the end
ic.rebuildCurrentPatterns()
return nil
}
// ShouldIgnore checks if the given path should be ignored based on the current patterns.
// Returns true if the path matches any ignore pattern, false otherwise.
func (ic *IgnoreChecker) ShouldIgnore(ctx context.Context, relPath string) bool {
// Handle root/empty path - never ignore
if relPath == "" || relPath == "." {
return false
}
// If no patterns loaded, nothing to ignore
if ic.matcher == nil {
return false
}
matches := ic.matcher.MatchesPath(relPath)
if matches {
log.Trace(ctx, "Scanner: Ignoring entry matching .ndignore", "path", relPath)
}
return matches
}
// loadPatternsFromFolder reads the .ndignore file in the specified folder and returns the patterns.
// If the file doesn't exist, returns an empty slice.
// If the file exists but is empty, returns a pattern to ignore everything ("**/*").
func (ic *IgnoreChecker) loadPatternsFromFolder(ctx context.Context, folder string) []string {
ignoreFilePath := path.Join(folder, consts.ScanIgnoreFile)
var patterns []string
// Check if .ndignore file exists
if _, err := fs.Stat(ic.fsys, ignoreFilePath); err != nil {
// No .ndignore file in this folder
return patterns
}
// Read and parse the .ndignore file
ignoreFile, err := ic.fsys.Open(ignoreFilePath)
if err != nil {
log.Warn(ctx, "Scanner: Error opening .ndignore file", "path", ignoreFilePath, err)
return patterns
}
defer ignoreFile.Close()
lineScanner := bufio.NewScanner(ignoreFile)
for lineScanner.Scan() {
line := strings.TrimSpace(lineScanner.Text())
if line == "" || strings.HasPrefix(line, "#") {
continue // Skip empty lines, whitespace-only lines, and comments
}
patterns = append(patterns, line)
}
if err := lineScanner.Err(); err != nil {
log.Warn(ctx, "Scanner: Error reading .ndignore file", "path", ignoreFilePath, err)
return patterns
}
// If the .ndignore file is empty, ignore everything
if len(patterns) == 0 {
log.Trace(ctx, "Scanner: .ndignore file is empty, ignoring everything", "path", folder)
patterns = []string{"**/*"}
}
return patterns
}
// rebuildCurrentPatterns flattens the pattern stack into currentPatterns and recompiles the matcher.
func (ic *IgnoreChecker) rebuildCurrentPatterns() {
ic.currentPatterns = make([]string, 0)
for _, patterns := range ic.patternStack {
ic.currentPatterns = append(ic.currentPatterns, patterns...)
}
ic.compilePatterns()
}
// compilePatterns compiles the current patterns into a GitIgnore matcher.
func (ic *IgnoreChecker) compilePatterns() {
if len(ic.currentPatterns) == 0 {
ic.matcher = nil
return
}
ic.matcher = ignore.CompileIgnoreLines(ic.currentPatterns...)
}

View File

@ -0,0 +1,313 @@
package scanner
import (
"context"
"testing/fstest"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
var _ = Describe("IgnoreChecker", func() {
Describe("loadPatternsFromFolder", func() {
var ic *IgnoreChecker
var ctx context.Context
BeforeEach(func() {
ctx = context.Background()
})
Context("when .ndignore file does not exist", func() {
It("should return empty patterns", func() {
fsys := fstest.MapFS{}
ic = newIgnoreChecker(fsys)
patterns := ic.loadPatternsFromFolder(ctx, ".")
Expect(patterns).To(BeEmpty())
})
})
Context("when .ndignore file is empty", func() {
It("should return wildcard to ignore everything", func() {
fsys := fstest.MapFS{
".ndignore": &fstest.MapFile{Data: []byte("")},
}
ic = newIgnoreChecker(fsys)
patterns := ic.loadPatternsFromFolder(ctx, ".")
Expect(patterns).To(Equal([]string{"**/*"}))
})
})
DescribeTable("parsing .ndignore content",
func(content string, expectedPatterns []string) {
fsys := fstest.MapFS{
".ndignore": &fstest.MapFile{Data: []byte(content)},
}
ic = newIgnoreChecker(fsys)
patterns := ic.loadPatternsFromFolder(ctx, ".")
Expect(patterns).To(Equal(expectedPatterns))
},
Entry("single pattern", "*.txt", []string{"*.txt"}),
Entry("multiple patterns", "*.txt\n*.log", []string{"*.txt", "*.log"}),
Entry("with comments", "# comment\n*.txt\n# another\n*.log", []string{"*.txt", "*.log"}),
Entry("with empty lines", "*.txt\n\n*.log\n\n", []string{"*.txt", "*.log"}),
Entry("mixed content", "# header\n\n*.txt\n# middle\n*.log\n\n", []string{"*.txt", "*.log"}),
Entry("only comments and empty lines", "# comment\n\n# another\n", []string{"**/*"}),
Entry("trailing newline", "*.txt\n*.log\n", []string{"*.txt", "*.log"}),
Entry("directory pattern", "temp/", []string{"temp/"}),
Entry("wildcard pattern", "**/*.mp3", []string{"**/*.mp3"}),
Entry("multiple wildcards", "**/*.mp3\n**/*.flac\n*.log", []string{"**/*.mp3", "**/*.flac", "*.log"}),
Entry("negation pattern", "!important.txt", []string{"!important.txt"}),
Entry("comment with hash not at start is pattern", "not#comment", []string{"not#comment"}),
Entry("whitespace-only lines skipped", "*.txt\n \n*.log\n\t\n", []string{"*.txt", "*.log"}),
Entry("patterns with whitespace trimmed", " *.txt \n\t*.log\t", []string{"*.txt", "*.log"}),
)
})
Describe("Push and Pop", func() {
var ic *IgnoreChecker
var fsys fstest.MapFS
var ctx context.Context
BeforeEach(func() {
ctx = context.Background()
fsys = fstest.MapFS{
".ndignore": &fstest.MapFile{Data: []byte("*.txt")},
"folder1/.ndignore": &fstest.MapFile{Data: []byte("*.mp3")},
"folder2/.ndignore": &fstest.MapFile{Data: []byte("*.flac")},
}
ic = newIgnoreChecker(fsys)
})
Context("Push", func() {
It("should add patterns to stack", func() {
err := ic.Push(ctx, ".")
Expect(err).ToNot(HaveOccurred())
Expect(len(ic.patternStack)).To(Equal(1))
Expect(ic.currentPatterns).To(ContainElement("*.txt"))
})
It("should compile matcher after push", func() {
err := ic.Push(ctx, ".")
Expect(err).ToNot(HaveOccurred())
Expect(ic.matcher).ToNot(BeNil())
})
It("should accumulate patterns from multiple levels", func() {
err := ic.Push(ctx, ".")
Expect(err).ToNot(HaveOccurred())
err = ic.Push(ctx, "folder1")
Expect(err).ToNot(HaveOccurred())
Expect(len(ic.patternStack)).To(Equal(2))
Expect(ic.currentPatterns).To(ConsistOf("*.txt", "*.mp3"))
})
It("should handle push when no .ndignore exists", func() {
err := ic.Push(ctx, "nonexistent")
Expect(err).ToNot(HaveOccurred())
Expect(len(ic.patternStack)).To(Equal(1))
Expect(ic.currentPatterns).To(BeEmpty())
})
})
Context("Pop", func() {
It("should remove most recent patterns", func() {
err := ic.Push(ctx, ".")
Expect(err).ToNot(HaveOccurred())
err = ic.Push(ctx, "folder1")
Expect(err).ToNot(HaveOccurred())
ic.Pop()
Expect(len(ic.patternStack)).To(Equal(1))
Expect(ic.currentPatterns).To(Equal([]string{"*.txt"}))
})
It("should handle Pop on empty stack gracefully", func() {
Expect(func() { ic.Pop() }).ToNot(Panic())
Expect(ic.patternStack).To(BeEmpty())
})
It("should set matcher to nil when all patterns popped", func() {
err := ic.Push(ctx, ".")
Expect(err).ToNot(HaveOccurred())
Expect(ic.matcher).ToNot(BeNil())
ic.Pop()
Expect(ic.matcher).To(BeNil())
})
It("should update matcher after pop", func() {
err := ic.Push(ctx, ".")
Expect(err).ToNot(HaveOccurred())
err = ic.Push(ctx, "folder1")
Expect(err).ToNot(HaveOccurred())
matcher1 := ic.matcher
ic.Pop()
matcher2 := ic.matcher
Expect(matcher1).ToNot(Equal(matcher2))
})
})
Context("multiple Push/Pop cycles", func() {
It("should maintain correct state through cycles", func() {
err := ic.Push(ctx, ".")
Expect(err).ToNot(HaveOccurred())
Expect(ic.currentPatterns).To(Equal([]string{"*.txt"}))
err = ic.Push(ctx, "folder1")
Expect(err).ToNot(HaveOccurred())
Expect(ic.currentPatterns).To(ConsistOf("*.txt", "*.mp3"))
ic.Pop()
Expect(ic.currentPatterns).To(Equal([]string{"*.txt"}))
err = ic.Push(ctx, "folder2")
Expect(err).ToNot(HaveOccurred())
Expect(ic.currentPatterns).To(ConsistOf("*.txt", "*.flac"))
ic.Pop()
Expect(ic.currentPatterns).To(Equal([]string{"*.txt"}))
ic.Pop()
Expect(ic.currentPatterns).To(BeEmpty())
})
})
})
Describe("PushAllParents", func() {
var ic *IgnoreChecker
var ctx context.Context
BeforeEach(func() {
ctx = context.Background()
fsys := fstest.MapFS{
".ndignore": &fstest.MapFile{Data: []byte("root.txt")},
"folder1/.ndignore": &fstest.MapFile{Data: []byte("level1.txt")},
"folder1/folder2/.ndignore": &fstest.MapFile{Data: []byte("level2.txt")},
"folder1/folder2/folder3/.ndignore": &fstest.MapFile{Data: []byte("level3.txt")},
}
ic = newIgnoreChecker(fsys)
})
DescribeTable("loading parent patterns",
func(targetPath string, expectedStackDepth int, expectedPatterns []string) {
err := ic.PushAllParents(ctx, targetPath)
Expect(err).ToNot(HaveOccurred())
Expect(len(ic.patternStack)).To(Equal(expectedStackDepth))
Expect(ic.currentPatterns).To(ConsistOf(expectedPatterns))
},
Entry("root path", ".", 1, []string{"root.txt"}),
Entry("empty path", "", 1, []string{"root.txt"}),
Entry("single level", "folder1", 2, []string{"root.txt", "level1.txt"}),
Entry("two levels", "folder1/folder2", 3, []string{"root.txt", "level1.txt", "level2.txt"}),
Entry("three levels", "folder1/folder2/folder3", 4, []string{"root.txt", "level1.txt", "level2.txt", "level3.txt"}),
)
It("should only compile patterns once at the end", func() {
// This is more of a behavioral test - we verify the matcher is not nil after PushAllParents
err := ic.PushAllParents(ctx, "folder1/folder2")
Expect(err).ToNot(HaveOccurred())
Expect(ic.matcher).ToNot(BeNil())
})
It("should handle paths with dot", func() {
err := ic.PushAllParents(ctx, "./folder1")
Expect(err).ToNot(HaveOccurred())
Expect(len(ic.patternStack)).To(Equal(2))
})
Context("when some parent folders have no .ndignore", func() {
BeforeEach(func() {
fsys := fstest.MapFS{
".ndignore": &fstest.MapFile{Data: []byte("root.txt")},
"folder1/folder2/.ndignore": &fstest.MapFile{Data: []byte("level2.txt")},
}
ic = newIgnoreChecker(fsys)
})
It("should still push all parent levels", func() {
err := ic.PushAllParents(ctx, "folder1/folder2")
Expect(err).ToNot(HaveOccurred())
Expect(len(ic.patternStack)).To(Equal(3)) // root, folder1 (empty), folder2
Expect(ic.currentPatterns).To(ConsistOf("root.txt", "level2.txt"))
})
})
})
Describe("ShouldIgnore", func() {
var ic *IgnoreChecker
var ctx context.Context
BeforeEach(func() {
ctx = context.Background()
})
Context("with no patterns loaded", func() {
It("should not ignore any path", func() {
fsys := fstest.MapFS{}
ic = newIgnoreChecker(fsys)
Expect(ic.ShouldIgnore(ctx, "anything.txt")).To(BeFalse())
Expect(ic.ShouldIgnore(ctx, "folder/file.mp3")).To(BeFalse())
})
})
Context("special paths", func() {
BeforeEach(func() {
fsys := fstest.MapFS{
".ndignore": &fstest.MapFile{Data: []byte("**/*")},
}
ic = newIgnoreChecker(fsys)
err := ic.Push(ctx, ".")
Expect(err).ToNot(HaveOccurred())
})
It("should never ignore root or empty paths", func() {
Expect(ic.ShouldIgnore(ctx, "")).To(BeFalse())
Expect(ic.ShouldIgnore(ctx, ".")).To(BeFalse())
})
It("should ignore all other paths with wildcard", func() {
Expect(ic.ShouldIgnore(ctx, "file.txt")).To(BeTrue())
Expect(ic.ShouldIgnore(ctx, "folder/file.mp3")).To(BeTrue())
})
})
DescribeTable("pattern matching",
func(pattern string, path string, shouldMatch bool) {
fsys := fstest.MapFS{
".ndignore": &fstest.MapFile{Data: []byte(pattern)},
}
ic = newIgnoreChecker(fsys)
err := ic.Push(ctx, ".")
Expect(err).ToNot(HaveOccurred())
Expect(ic.ShouldIgnore(ctx, path)).To(Equal(shouldMatch))
},
Entry("glob match", "*.txt", "file.txt", true),
Entry("glob no match", "*.txt", "file.mp3", false),
Entry("directory pattern match", "tmp/", "tmp/file.txt", true),
Entry("directory pattern no match", "tmp/", "temporary/file.txt", false),
Entry("nested glob match", "**/*.log", "deep/nested/file.log", true),
Entry("nested glob no match", "**/*.log", "deep/nested/file.txt", false),
Entry("specific file match", "ignore.me", "ignore.me", true),
Entry("specific file no match", "ignore.me", "keep.me", false),
Entry("wildcard all", "**/*", "any/path/file.txt", true),
Entry("nested specific match", "temp/*", "temp/cache.db", true),
Entry("nested specific no match", "temp/*", "temporary/cache.db", false),
)
Context("with multiple patterns", func() {
BeforeEach(func() {
fsys := fstest.MapFS{
".ndignore": &fstest.MapFile{Data: []byte("*.txt\n*.log\ntemp/")},
}
ic = newIgnoreChecker(fsys)
err := ic.Push(ctx, ".")
Expect(err).ToNot(HaveOccurred())
})
It("should match any of the patterns", func() {
Expect(ic.ShouldIgnore(ctx, "file.txt")).To(BeTrue())
Expect(ic.ShouldIgnore(ctx, "debug.log")).To(BeTrue())
Expect(ic.ShouldIgnore(ctx, "temp/cache")).To(BeTrue())
Expect(ic.ShouldIgnore(ctx, "music.mp3")).To(BeFalse())
})
})
})
})

View File

@ -26,58 +26,46 @@ import (
"github.com/navidrome/navidrome/utils/slice"
)
func createPhaseFolders(ctx context.Context, state *scanState, ds model.DataStore, cw artwork.CacheWarmer, libs []model.Library) *phaseFolders {
func createPhaseFolders(ctx context.Context, state *scanState, ds model.DataStore, cw artwork.CacheWarmer) *phaseFolders {
var jobs []*scanJob
var updatedLibs []model.Library
for _, lib := range libs {
if lib.LastScanStartedAt.IsZero() {
err := ds.Library(ctx).ScanBegin(lib.ID, state.fullScan)
if err != nil {
log.Error(ctx, "Scanner: Error updating last scan started at", "lib", lib.Name, err)
state.sendWarning(err.Error())
continue
}
// Reload library to get updated state
l, err := ds.Library(ctx).Get(lib.ID)
if err != nil {
log.Error(ctx, "Scanner: Error reloading library", "lib", lib.Name, err)
state.sendWarning(err.Error())
continue
}
lib = *l
} else {
log.Debug(ctx, "Scanner: Resuming previous scan", "lib", lib.Name, "lastScanStartedAt", lib.LastScanStartedAt, "fullScan", lib.FullScanInProgress)
// Create scan jobs for all libraries
for _, lib := range state.libraries {
// Get target folders for this library if selective scan
var targetFolders []string
if state.isSelectiveScan() {
targetFolders = state.targets[lib.ID]
}
job, err := newScanJob(ctx, ds, cw, lib, state.fullScan)
job, err := newScanJob(ctx, ds, cw, lib, state.fullScan, targetFolders)
if err != nil {
log.Error(ctx, "Scanner: Error creating scan context", "lib", lib.Name, err)
state.sendWarning(err.Error())
continue
}
jobs = append(jobs, job)
updatedLibs = append(updatedLibs, lib)
}
// Update the state with the libraries that have been processed and have their scan timestamps set
state.libraries = updatedLibs
return &phaseFolders{jobs: jobs, ctx: ctx, ds: ds, state: state}
}
type scanJob struct {
lib model.Library
fs storage.MusicFS
cw artwork.CacheWarmer
lastUpdates map[string]model.FolderUpdateInfo
lock sync.Mutex
numFolders atomic.Int64
lib model.Library
fs storage.MusicFS
cw artwork.CacheWarmer
lastUpdates map[string]model.FolderUpdateInfo // Holds last update info for all (DB) folders in this library
targetFolders []string // Specific folders to scan (including all descendants)
lock sync.Mutex
numFolders atomic.Int64
}
func newScanJob(ctx context.Context, ds model.DataStore, cw artwork.CacheWarmer, lib model.Library, fullScan bool) (*scanJob, error) {
lastUpdates, err := ds.Folder(ctx).GetLastUpdates(lib)
func newScanJob(ctx context.Context, ds model.DataStore, cw artwork.CacheWarmer, lib model.Library, fullScan bool, targetFolders []string) (*scanJob, error) {
// Get folder updates, optionally filtered to specific target folders
lastUpdates, err := ds.Folder(ctx).GetFolderUpdateInfo(lib, targetFolders...)
if err != nil {
return nil, fmt.Errorf("getting last updates: %w", err)
}
fileStore, err := storage.For(lib.Path)
if err != nil {
log.Error(ctx, "Error getting storage for library", "library", lib.Name, "path", lib.Path, err)
@ -88,15 +76,17 @@ func newScanJob(ctx context.Context, ds model.DataStore, cw artwork.CacheWarmer,
log.Error(ctx, "Error getting fs for library", "library", lib.Name, "path", lib.Path, err)
return nil, fmt.Errorf("getting fs for library: %w", err)
}
lib.FullScanInProgress = lib.FullScanInProgress || fullScan
return &scanJob{
lib: lib,
fs: fsys,
cw: cw,
lastUpdates: lastUpdates,
lib: lib,
fs: fsys,
cw: cw,
lastUpdates: lastUpdates,
targetFolders: targetFolders,
}, nil
}
// popLastUpdate retrieves and removes the last update info for the given folder ID
// This is used to track which folders have been found during the walk_dir_tree
func (j *scanJob) popLastUpdate(folderID string) model.FolderUpdateInfo {
j.lock.Lock()
defer j.lock.Unlock()
@ -106,6 +96,15 @@ func (j *scanJob) popLastUpdate(folderID string) model.FolderUpdateInfo {
return lastUpdate
}
// createFolderEntry creates a new folderEntry for the given path, using the last update info from the job
// to populate the previous update time and hash. It also removes the folder from the job's lastUpdates map.
// This is used to track which folders have been found during the walk_dir_tree.
func (j *scanJob) createFolderEntry(path string) *folderEntry {
id := model.FolderID(j.lib, path)
info := j.popLastUpdate(id)
return newFolderEntry(j, id, path, info.UpdatedAt, info.Hash)
}
// phaseFolders represents the first phase of the scanning process, which is responsible
// for scanning all libraries and importing new or updated files. This phase involves
// traversing the directory tree of each library, identifying new or modified media files,
@ -144,7 +143,8 @@ func (p *phaseFolders) producer() ppl.Producer[*folderEntry] {
if utils.IsCtxDone(p.ctx) {
break
}
outputChan, err := walkDirTree(p.ctx, job)
outputChan, err := walkDirTree(p.ctx, job, job.targetFolders...)
if err != nil {
log.Warn(p.ctx, "Scanner: Error scanning library", "lib", job.lib.Name, err)
}
@ -324,6 +324,9 @@ func (p *phaseFolders) persistChanges(entry *folderEntry) (*folderEntry, error)
defer p.measure(entry)()
p.state.changesDetected.Store(true)
// Collect artwork IDs to pre-cache after the transaction commits
var artworkIDs []model.ArtworkID
err := p.ds.WithTx(func(tx model.DataStore) error {
// Instantiate all repositories just once per folder
folderRepo := tx.Folder(p.ctx)
@ -362,7 +365,7 @@ func (p *phaseFolders) persistChanges(entry *folderEntry) (*folderEntry, error)
return err
}
if entry.artists[i].Name != consts.UnknownArtist && entry.artists[i].Name != consts.VariousArtists {
entry.job.cw.PreCache(entry.artists[i].CoverArtID())
artworkIDs = append(artworkIDs, entry.artists[i].CoverArtID())
}
}
@ -374,7 +377,7 @@ func (p *phaseFolders) persistChanges(entry *folderEntry) (*folderEntry, error)
return err
}
if entry.albums[i].Name != consts.UnknownAlbum {
entry.job.cw.PreCache(entry.albums[i].CoverArtID())
artworkIDs = append(artworkIDs, entry.albums[i].CoverArtID())
}
}
@ -411,6 +414,14 @@ func (p *phaseFolders) persistChanges(entry *folderEntry) (*folderEntry, error)
if err != nil {
log.Error(p.ctx, "Scanner: Error persisting changes to DB", "folder", entry.path, err)
}
// Pre-cache artwork after the transaction commits successfully
if err == nil {
for _, artID := range artworkIDs {
entry.job.cw.PreCache(artID)
}
}
return entry, err
}

View File

@ -69,9 +69,6 @@ func (p *phaseMissingTracks) produce(put func(tracks *missingTracks)) error {
}
}
for _, lib := range p.state.libraries {
if lib.LastScanStartedAt.IsZero() {
continue
}
log.Debug(p.ctx, "Scanner: Checking missing tracks", "libraryId", lib.ID, "libraryName", lib.Name)
cursor, err := p.ds.MediaFile(p.ctx).GetMissingAndMatching(lib.ID)
if err != nil {

View File

@ -27,14 +27,13 @@ import (
type phaseRefreshAlbums struct {
ds model.DataStore
ctx context.Context
libs model.Libraries
refreshed atomic.Uint32
skipped atomic.Uint32
state *scanState
}
func createPhaseRefreshAlbums(ctx context.Context, state *scanState, ds model.DataStore, libs model.Libraries) *phaseRefreshAlbums {
return &phaseRefreshAlbums{ctx: ctx, ds: ds, libs: libs, state: state}
func createPhaseRefreshAlbums(ctx context.Context, state *scanState, ds model.DataStore) *phaseRefreshAlbums {
return &phaseRefreshAlbums{ctx: ctx, ds: ds, state: state}
}
func (p *phaseRefreshAlbums) description() string {
@ -47,7 +46,7 @@ func (p *phaseRefreshAlbums) producer() ppl.Producer[*model.Album] {
func (p *phaseRefreshAlbums) produce(put func(album *model.Album)) error {
count := 0
for _, lib := range p.libs {
for _, lib := range p.state.libraries {
cursor, err := p.ds.Album(p.ctx).GetTouchedAlbums(lib.ID)
if err != nil {
return fmt.Errorf("loading touched albums: %w", err)

View File

@ -32,8 +32,8 @@ var _ = Describe("phaseRefreshAlbums", func() {
{ID: 1, Name: "Library 1"},
{ID: 2, Name: "Library 2"},
}
state = &scanState{}
phase = createPhaseRefreshAlbums(ctx, state, ds, libs)
state = &scanState{libraries: libs}
phase = createPhaseRefreshAlbums(ctx, state, ds)
})
Describe("description", func() {

View File

@ -3,6 +3,8 @@ package scanner
import (
"context"
"fmt"
"maps"
"slices"
"sync/atomic"
"time"
@ -15,6 +17,7 @@ import (
"github.com/navidrome/navidrome/log"
"github.com/navidrome/navidrome/model"
"github.com/navidrome/navidrome/utils/run"
"github.com/navidrome/navidrome/utils/slice"
)
type scannerImpl struct {
@ -28,7 +31,8 @@ type scanState struct {
progress chan<- *ProgressInfo
fullScan bool
changesDetected atomic.Bool
libraries model.Libraries // Store libraries list for consistency across phases
libraries model.Libraries // Store libraries list for consistency across phases
targets map[int][]string // Optional: map[libraryID][]folderPaths for selective scans
}
func (s *scanState) sendProgress(info *ProgressInfo) {
@ -37,6 +41,10 @@ func (s *scanState) sendProgress(info *ProgressInfo) {
}
}
func (s *scanState) isSelectiveScan() bool {
return len(s.targets) > 0
}
func (s *scanState) sendWarning(msg string) {
s.sendProgress(&ProgressInfo{Warning: msg})
}
@ -45,7 +53,7 @@ func (s *scanState) sendError(err error) {
s.sendProgress(&ProgressInfo{Error: err.Error()})
}
func (s *scannerImpl) scanAll(ctx context.Context, fullScan bool, progress chan<- *ProgressInfo) {
func (s *scannerImpl) scanFolders(ctx context.Context, fullScan bool, targets []model.ScanTarget, progress chan<- *ProgressInfo) {
startTime := time.Now()
state := scanState{
@ -59,38 +67,75 @@ func (s *scannerImpl) scanAll(ctx context.Context, fullScan bool, progress chan<
state.changesDetected.Store(true)
}
libs, err := s.ds.Library(ctx).GetAll()
// Get libraries and optionally filter by targets
allLibs, err := s.ds.Library(ctx).GetAll()
if err != nil {
state.sendWarning(fmt.Sprintf("getting libraries: %s", err))
return
}
state.libraries = libs
log.Info(ctx, "Scanner: Starting scan", "fullScan", state.fullScan, "numLibraries", len(libs))
if len(targets) > 0 {
// Selective scan: filter libraries and build targets map
state.targets = make(map[int][]string)
for _, target := range targets {
folderPath := target.FolderPath
if folderPath == "" {
folderPath = "."
}
state.targets[target.LibraryID] = append(state.targets[target.LibraryID], folderPath)
}
// Filter libraries to only those in targets
state.libraries = slice.Filter(allLibs, func(lib model.Library) bool {
return len(state.targets[lib.ID]) > 0
})
log.Info(ctx, "Scanner: Starting selective scan", "fullScan", state.fullScan, "numLibraries", len(state.libraries), "numTargets", len(targets))
} else {
// Full library scan
state.libraries = allLibs
log.Info(ctx, "Scanner: Starting scan", "fullScan", state.fullScan, "numLibraries", len(state.libraries))
}
// Store scan type and start time
scanType := "quick"
if state.fullScan {
scanType = "full"
}
if state.isSelectiveScan() {
scanType += "-selective"
}
_ = s.ds.Property(ctx).Put(consts.LastScanTypeKey, scanType)
_ = s.ds.Property(ctx).Put(consts.LastScanStartTimeKey, startTime.Format(time.RFC3339))
// if there was a full scan in progress, force a full scan
if !state.fullScan {
for _, lib := range libs {
for _, lib := range state.libraries {
if lib.FullScanInProgress {
log.Info(ctx, "Scanner: Interrupted full scan detected", "lib", lib.Name)
state.fullScan = true
_ = s.ds.Property(ctx).Put(consts.LastScanTypeKey, "full")
if state.isSelectiveScan() {
_ = s.ds.Property(ctx).Put(consts.LastScanTypeKey, "full-selective")
} else {
_ = s.ds.Property(ctx).Put(consts.LastScanTypeKey, "full")
}
break
}
}
}
// Prepare libraries for scanning (initialize LastScanStartedAt if needed)
err = s.prepareLibrariesForScan(ctx, &state)
if err != nil {
log.Error(ctx, "Scanner: Error preparing libraries for scan", err)
state.sendError(err)
return
}
err = run.Sequentially(
// Phase 1: Scan all libraries and import new/updated files
runPhase[*folderEntry](ctx, 1, createPhaseFolders(ctx, &state, s.ds, s.cw, libs)),
runPhase[*folderEntry](ctx, 1, createPhaseFolders(ctx, &state, s.ds, s.cw)),
// Phase 2: Process missing files, checking for moves
runPhase[*missingTracks](ctx, 2, createPhaseMissingTracks(ctx, &state, s.ds)),
@ -98,7 +143,7 @@ func (s *scannerImpl) scanAll(ctx context.Context, fullScan bool, progress chan<
// Phases 3 and 4 can be run in parallel
run.Parallel(
// Phase 3: Refresh all new/changed albums and update artists
runPhase[*model.Album](ctx, 3, createPhaseRefreshAlbums(ctx, &state, s.ds, libs)),
runPhase[*model.Album](ctx, 3, createPhaseRefreshAlbums(ctx, &state, s.ds)),
// Phase 4: Import/update playlists
runPhase[*model.Folder](ctx, 4, createPhasePlaylists(ctx, &state, s.ds, s.pls, s.cw)),
@ -131,7 +176,53 @@ func (s *scannerImpl) scanAll(ctx context.Context, fullScan bool, progress chan<
state.sendProgress(&ProgressInfo{ChangesDetected: true})
}
log.Info(ctx, "Scanner: Finished scanning all libraries", "duration", time.Since(startTime))
if state.isSelectiveScan() {
log.Info(ctx, "Scanner: Finished scanning selected folders", "duration", time.Since(startTime), "numTargets", len(targets))
} else {
log.Info(ctx, "Scanner: Finished scanning all libraries", "duration", time.Since(startTime))
}
}
// prepareLibrariesForScan initializes the scan for all libraries in the state.
// It calls ScanBegin for libraries that haven't started scanning yet (LastScanStartedAt is zero),
// reloads them to get the updated state, and filters out any libraries that fail to initialize.
func (s *scannerImpl) prepareLibrariesForScan(ctx context.Context, state *scanState) error {
var successfulLibs []model.Library
for _, lib := range state.libraries {
if lib.LastScanStartedAt.IsZero() {
// This is a new scan - mark it as started
err := s.ds.Library(ctx).ScanBegin(lib.ID, state.fullScan)
if err != nil {
log.Error(ctx, "Scanner: Error marking scan start", "lib", lib.Name, err)
state.sendWarning(err.Error())
continue
}
// Reload library to get updated state (timestamps, etc.)
reloadedLib, err := s.ds.Library(ctx).Get(lib.ID)
if err != nil {
log.Error(ctx, "Scanner: Error reloading library", "lib", lib.Name, err)
state.sendWarning(err.Error())
continue
}
lib = *reloadedLib
} else {
// This is a resumed scan
log.Debug(ctx, "Scanner: Resuming previous scan", "lib", lib.Name,
"lastScanStartedAt", lib.LastScanStartedAt, "fullScan", lib.FullScanInProgress)
}
successfulLibs = append(successfulLibs, lib)
}
if len(successfulLibs) == 0 {
return fmt.Errorf("no libraries available for scanning")
}
// Update state with only successfully initialized libraries
state.libraries = successfulLibs
return nil
}
func (s *scannerImpl) runGC(ctx context.Context, state *scanState) func() error {
@ -140,7 +231,15 @@ func (s *scannerImpl) runGC(ctx context.Context, state *scanState) func() error
return s.ds.WithTx(func(tx model.DataStore) error {
if state.changesDetected.Load() {
start := time.Now()
err := tx.GC(ctx)
// For selective scans, extract library IDs to scope GC operations
var libraryIDs []int
if state.isSelectiveScan() {
libraryIDs = slices.Collect(maps.Keys(state.targets))
log.Debug(ctx, "Scanner: Running selective GC", "libraryIDs", libraryIDs)
}
err := tx.GC(ctx, libraryIDs...)
if err != nil {
log.Error(ctx, "Scanner: Error running GC", err)
return fmt.Errorf("running GC: %w", err)

View File

@ -32,7 +32,7 @@ var _ = Describe("Scanner - Multi-Library", Ordered, func() {
var ctx context.Context
var lib1, lib2 model.Library
var ds *tests.MockDataStore
var s scanner.Scanner
var s model.Scanner
createFS := func(path string, files fstest.MapFS) storagetest.FakeFS {
fs := storagetest.FakeFS{}

View File

@ -0,0 +1,293 @@
package scanner_test
import (
"context"
"path/filepath"
"testing/fstest"
"github.com/Masterminds/squirrel"
"github.com/navidrome/navidrome/conf"
"github.com/navidrome/navidrome/conf/configtest"
"github.com/navidrome/navidrome/core"
"github.com/navidrome/navidrome/core/artwork"
"github.com/navidrome/navidrome/core/metrics"
"github.com/navidrome/navidrome/core/storage/storagetest"
"github.com/navidrome/navidrome/db"
"github.com/navidrome/navidrome/log"
"github.com/navidrome/navidrome/model"
"github.com/navidrome/navidrome/model/request"
"github.com/navidrome/navidrome/persistence"
"github.com/navidrome/navidrome/scanner"
"github.com/navidrome/navidrome/server/events"
"github.com/navidrome/navidrome/tests"
"github.com/navidrome/navidrome/utils/slice"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
var _ = Describe("ScanFolders", Ordered, func() {
var ctx context.Context
var lib model.Library
var ds model.DataStore
var s model.Scanner
var fsys storagetest.FakeFS
BeforeAll(func() {
ctx = request.WithUser(GinkgoT().Context(), model.User{ID: "123", IsAdmin: true})
tmpDir := GinkgoT().TempDir()
conf.Server.DbPath = filepath.Join(tmpDir, "test-selective-scan.db?_journal_mode=WAL")
log.Warn("Using DB at " + conf.Server.DbPath)
db.Db().SetMaxOpenConns(1)
})
BeforeEach(func() {
DeferCleanup(configtest.SetupConfig())
conf.Server.MusicFolder = "fake:///music"
conf.Server.DevExternalScanner = false
db.Init(ctx)
DeferCleanup(func() {
Expect(tests.ClearDB()).To(Succeed())
})
ds = persistence.New(db.Db())
// Create the admin user in the database to match the context
adminUser := model.User{
ID: "123",
UserName: "admin",
Name: "Admin User",
IsAdmin: true,
NewPassword: "password",
}
Expect(ds.User(ctx).Put(&adminUser)).To(Succeed())
s = scanner.New(ctx, ds, artwork.NoopCacheWarmer(), events.NoopBroker(),
core.NewPlaylists(ds), metrics.NewNoopInstance())
lib = model.Library{ID: 1, Name: "Fake Library", Path: "fake:///music"}
Expect(ds.Library(ctx).Put(&lib)).To(Succeed())
// Initialize fake filesystem
fsys = storagetest.FakeFS{}
storagetest.Register("fake", &fsys)
})
Describe("Adding tracks to the library", func() {
It("scans specified folders recursively including all subdirectories", func() {
rock := template(_t{"albumartist": "Rock Artist", "album": "Rock Album"})
jazz := template(_t{"albumartist": "Jazz Artist", "album": "Jazz Album"})
pop := template(_t{"albumartist": "Pop Artist", "album": "Pop Album"})
createFS(fstest.MapFS{
"rock/track1.mp3": rock(track(1, "Rock Track 1")),
"rock/track2.mp3": rock(track(2, "Rock Track 2")),
"rock/subdir/track3.mp3": rock(track(3, "Rock Track 3")),
"jazz/track4.mp3": jazz(track(1, "Jazz Track 1")),
"jazz/subdir/track5.mp3": jazz(track(2, "Jazz Track 2")),
"pop/track6.mp3": pop(track(1, "Pop Track 1")),
})
// Scan only the "rock" and "jazz" folders (including their subdirectories)
targets := []model.ScanTarget{
{LibraryID: lib.ID, FolderPath: "rock"},
{LibraryID: lib.ID, FolderPath: "jazz"},
}
warnings, err := s.ScanFolders(ctx, false, targets)
Expect(err).ToNot(HaveOccurred())
Expect(warnings).To(BeEmpty())
// Verify all tracks in rock and jazz folders (including subdirectories) were imported
allFiles, err := ds.MediaFile(ctx).GetAll()
Expect(err).ToNot(HaveOccurred())
// Should have 5 tracks (all rock and jazz tracks including subdirectories)
Expect(allFiles).To(HaveLen(5))
// Get the file paths
paths := slice.Map(allFiles, func(mf model.MediaFile) string {
return filepath.ToSlash(mf.Path)
})
// Verify the correct files were scanned (including subdirectories)
Expect(paths).To(ContainElements(
"rock/track1.mp3",
"rock/track2.mp3",
"rock/subdir/track3.mp3",
"jazz/track4.mp3",
"jazz/subdir/track5.mp3",
))
// Verify files in the pop folder were NOT scanned
Expect(paths).ToNot(ContainElement("pop/track6.mp3"))
})
})
Describe("Deleting folders", func() {
Context("when a child folder is deleted", func() {
var (
revolver, help func(...map[string]any) *fstest.MapFile
artistFolderID string
album1FolderID string
album2FolderID string
album1TrackIDs []string
album2TrackIDs []string
)
BeforeEach(func() {
// Setup template functions for creating test files
revolver = storagetest.Template(_t{"albumartist": "The Beatles", "album": "Revolver", "year": 1966})
help = storagetest.Template(_t{"albumartist": "The Beatles", "album": "Help!", "year": 1965})
// Initial filesystem with nested folders
fsys.SetFiles(fstest.MapFS{
"The Beatles/Revolver/01 - Taxman.mp3": revolver(storagetest.Track(1, "Taxman")),
"The Beatles/Revolver/02 - Eleanor Rigby.mp3": revolver(storagetest.Track(2, "Eleanor Rigby")),
"The Beatles/Help!/01 - Help!.mp3": help(storagetest.Track(1, "Help!")),
"The Beatles/Help!/02 - The Night Before.mp3": help(storagetest.Track(2, "The Night Before")),
})
// First scan - import everything
_, err := s.ScanAll(ctx, true)
Expect(err).ToNot(HaveOccurred())
// Verify initial state - all folders exist
folders, err := ds.Folder(ctx).GetAll(model.QueryOptions{Filters: squirrel.Eq{"library_id": lib.ID}})
Expect(err).ToNot(HaveOccurred())
Expect(folders).To(HaveLen(4)) // root, Artist, Album1, Album2
// Store folder IDs for later verification
for _, f := range folders {
switch f.Name {
case "The Beatles":
artistFolderID = f.ID
case "Revolver":
album1FolderID = f.ID
case "Help!":
album2FolderID = f.ID
}
}
// Verify all tracks exist
allTracks, err := ds.MediaFile(ctx).GetAll()
Expect(err).ToNot(HaveOccurred())
Expect(allTracks).To(HaveLen(4))
// Store track IDs for later verification
for _, t := range allTracks {
if t.Album == "Revolver" {
album1TrackIDs = append(album1TrackIDs, t.ID)
} else if t.Album == "Help!" {
album2TrackIDs = append(album2TrackIDs, t.ID)
}
}
// Verify no tracks are missing initially
for _, t := range allTracks {
Expect(t.Missing).To(BeFalse())
}
})
It("should mark child folder and its tracks as missing when parent is scanned", func() {
// Delete the child folder (Help!) from the filesystem
fsys.SetFiles(fstest.MapFS{
"The Beatles/Revolver/01 - Taxman.mp3": revolver(storagetest.Track(1, "Taxman")),
"The Beatles/Revolver/02 - Eleanor Rigby.mp3": revolver(storagetest.Track(2, "Eleanor Rigby")),
// "The Beatles/Help!" folder and its contents are DELETED
})
// Run selective scan on the parent folder (Artist)
// This simulates what the watcher does when a child folder is deleted
_, err := s.ScanFolders(ctx, false, []model.ScanTarget{
{LibraryID: lib.ID, FolderPath: "The Beatles"},
})
Expect(err).ToNot(HaveOccurred())
// Verify the deleted child folder is now marked as missing
deletedFolder, err := ds.Folder(ctx).Get(album2FolderID)
Expect(err).ToNot(HaveOccurred())
Expect(deletedFolder.Missing).To(BeTrue(), "Deleted child folder should be marked as missing")
// Verify the deleted folder's tracks are marked as missing
for _, trackID := range album2TrackIDs {
track, err := ds.MediaFile(ctx).Get(trackID)
Expect(err).ToNot(HaveOccurred())
Expect(track.Missing).To(BeTrue(), "Track in deleted folder should be marked as missing")
}
// Verify the parent folder is still present and not marked as missing
parentFolder, err := ds.Folder(ctx).Get(artistFolderID)
Expect(err).ToNot(HaveOccurred())
Expect(parentFolder.Missing).To(BeFalse(), "Parent folder should not be marked as missing")
// Verify the sibling folder and its tracks are still present and not missing
siblingFolder, err := ds.Folder(ctx).Get(album1FolderID)
Expect(err).ToNot(HaveOccurred())
Expect(siblingFolder.Missing).To(BeFalse(), "Sibling folder should not be marked as missing")
for _, trackID := range album1TrackIDs {
track, err := ds.MediaFile(ctx).Get(trackID)
Expect(err).ToNot(HaveOccurred())
Expect(track.Missing).To(BeFalse(), "Track in sibling folder should not be marked as missing")
}
})
It("should mark deeply nested child folders as missing", func() {
// Add a deeply nested folder structure
fsys.SetFiles(fstest.MapFS{
"The Beatles/Revolver/01 - Taxman.mp3": revolver(storagetest.Track(1, "Taxman")),
"The Beatles/Revolver/02 - Eleanor Rigby.mp3": revolver(storagetest.Track(2, "Eleanor Rigby")),
"The Beatles/Help!/01 - Help!.mp3": help(storagetest.Track(1, "Help!")),
"The Beatles/Help!/02 - The Night Before.mp3": help(storagetest.Track(2, "The Night Before")),
"The Beatles/Help!/Bonus/01 - Bonus Track.mp3": help(storagetest.Track(99, "Bonus Track")),
"The Beatles/Help!/Bonus/Nested/01 - Deep Track.mp3": help(storagetest.Track(100, "Deep Track")),
})
// Rescan to import the new nested structure
_, err := s.ScanAll(ctx, true)
Expect(err).ToNot(HaveOccurred())
// Verify nested folders were created
allFolders, err := ds.Folder(ctx).GetAll(model.QueryOptions{Filters: squirrel.Eq{"library_id": lib.ID}})
Expect(err).ToNot(HaveOccurred())
Expect(len(allFolders)).To(BeNumerically(">", 4), "Should have more folders with nested structure")
// Now delete the entire Help! folder including nested children
fsys.SetFiles(fstest.MapFS{
"The Beatles/Revolver/01 - Taxman.mp3": revolver(storagetest.Track(1, "Taxman")),
"The Beatles/Revolver/02 - Eleanor Rigby.mp3": revolver(storagetest.Track(2, "Eleanor Rigby")),
// All Help! subfolders are deleted
})
// Run selective scan on parent
_, err = s.ScanFolders(ctx, false, []model.ScanTarget{
{LibraryID: lib.ID, FolderPath: "The Beatles"},
})
Expect(err).ToNot(HaveOccurred())
// Verify all Help! folders (including nested ones) are marked as missing
missingFolders, err := ds.Folder(ctx).GetAll(model.QueryOptions{
Filters: squirrel.And{
squirrel.Eq{"library_id": lib.ID},
squirrel.Eq{"missing": true},
},
})
Expect(err).ToNot(HaveOccurred())
Expect(len(missingFolders)).To(BeNumerically(">", 0), "At least one folder should be marked as missing")
// Verify all tracks in deleted folders are marked as missing
allTracks, err := ds.MediaFile(ctx).GetAll()
Expect(err).ToNot(HaveOccurred())
Expect(allTracks).To(HaveLen(6))
for _, track := range allTracks {
if track.Album == "Help!" {
Expect(track.Missing).To(BeTrue(), "All tracks in deleted Help! folder should be marked as missing")
} else if track.Album == "Revolver" {
Expect(track.Missing).To(BeFalse(), "Tracks in Revolver folder should not be marked as missing")
}
}
})
})
})
})

View File

@ -34,19 +34,19 @@ type _t = map[string]any
var template = storagetest.Template
var track = storagetest.Track
func createFS(files fstest.MapFS) storagetest.FakeFS {
fs := storagetest.FakeFS{}
fs.SetFiles(files)
storagetest.Register("fake", &fs)
return fs
}
var _ = Describe("Scanner", Ordered, func() {
var ctx context.Context
var lib model.Library
var ds *tests.MockDataStore
var mfRepo *mockMediaFileRepo
var s scanner.Scanner
createFS := func(files fstest.MapFS) storagetest.FakeFS {
fs := storagetest.FakeFS{}
fs.SetFiles(files)
storagetest.Register("fake", &fs)
return fs
}
var s model.Scanner
BeforeAll(func() {
ctx = request.WithUser(GinkgoT().Context(), model.User{ID: "123", IsAdmin: true})
@ -478,6 +478,56 @@ var _ = Describe("Scanner", Ordered, func() {
Expect(mf.Missing).To(BeFalse())
})
It("marks tracks as missing when scanning a deleted folder with ScanFolders", func() {
By("Adding a third track to Revolver to have more test data")
fsys.Add("The Beatles/Revolver/03 - I'm Only Sleeping.mp3", revolver(track(3, "I'm Only Sleeping")))
Expect(runScanner(ctx, false)).To(Succeed())
By("Verifying initial state has 5 tracks")
Expect(ds.MediaFile(ctx).CountAll(model.QueryOptions{
Filters: squirrel.Eq{"missing": false},
})).To(Equal(int64(5)))
By("Removing the entire Revolver folder from filesystem")
fsys.Remove("The Beatles/Revolver/01 - Taxman.mp3")
fsys.Remove("The Beatles/Revolver/02 - Eleanor Rigby.mp3")
fsys.Remove("The Beatles/Revolver/03 - I'm Only Sleeping.mp3")
By("Scanning the parent folder (simulating watcher behavior)")
targets := []model.ScanTarget{
{LibraryID: lib.ID, FolderPath: "The Beatles"},
}
_, err := s.ScanFolders(ctx, false, targets)
Expect(err).To(Succeed())
By("Checking all Revolver tracks are marked as missing")
mf, err := findByPath("The Beatles/Revolver/01 - Taxman.mp3")
Expect(err).ToNot(HaveOccurred())
Expect(mf.Missing).To(BeTrue())
mf, err = findByPath("The Beatles/Revolver/02 - Eleanor Rigby.mp3")
Expect(err).ToNot(HaveOccurred())
Expect(mf.Missing).To(BeTrue())
mf, err = findByPath("The Beatles/Revolver/03 - I'm Only Sleeping.mp3")
Expect(err).ToNot(HaveOccurred())
Expect(mf.Missing).To(BeTrue())
By("Checking the Help! tracks are not affected")
mf, err = findByPath("The Beatles/Help!/01 - Help!.mp3")
Expect(err).ToNot(HaveOccurred())
Expect(mf.Missing).To(BeFalse())
mf, err = findByPath("The Beatles/Help!/02 - The Night Before.mp3")
Expect(err).ToNot(HaveOccurred())
Expect(mf.Missing).To(BeFalse())
By("Verifying only 2 non-missing tracks remain (Help! tracks)")
Expect(ds.MediaFile(ctx).CountAll(model.QueryOptions{
Filters: squirrel.Eq{"missing": false},
})).To(Equal(int64(2)))
})
It("does not override artist fields when importing an undertagged file", func() {
By("Making sure artist in the DB contains MBID and sort name")
aa, err := ds.Artist(ctx).GetAll(model.QueryOptions{

View File

@ -1,7 +1,6 @@
package scanner
import (
"bufio"
"context"
"io/fs"
"maps"
@ -11,37 +10,69 @@ import (
"strings"
"github.com/navidrome/navidrome/conf"
"github.com/navidrome/navidrome/consts"
"github.com/navidrome/navidrome/log"
"github.com/navidrome/navidrome/model"
"github.com/navidrome/navidrome/utils"
ignore "github.com/sabhiram/go-gitignore"
)
func walkDirTree(ctx context.Context, job *scanJob) (<-chan *folderEntry, error) {
// walkDirTree recursively walks the directory tree starting from the given targetFolders.
// If no targetFolders are provided, it starts from the root folder (".").
// It returns a channel of folderEntry pointers representing each folder found.
func walkDirTree(ctx context.Context, job *scanJob, targetFolders ...string) (<-chan *folderEntry, error) {
results := make(chan *folderEntry)
folders := targetFolders
if len(targetFolders) == 0 {
// No specific folders provided, scan the root folder
folders = []string{"."}
}
go func() {
defer close(results)
err := walkFolder(ctx, job, ".", nil, results)
if err != nil {
log.Error(ctx, "Scanner: There were errors reading directories from filesystem", "path", job.lib.Path, err)
return
for _, folderPath := range folders {
if utils.IsCtxDone(ctx) {
return
}
// Check if target folder exists before walking it
// If it doesn't exist (e.g., deleted between watcher detection and scan execution),
// skip it so it remains in job.lastUpdates and gets handled in following steps
_, err := fs.Stat(job.fs, folderPath)
if err != nil {
log.Warn(ctx, "Scanner: Target folder does not exist.", "path", folderPath, err)
continue
}
// Create checker and push patterns from root to this folder
checker := newIgnoreChecker(job.fs)
err = checker.PushAllParents(ctx, folderPath)
if err != nil {
log.Error(ctx, "Scanner: Error pushing ignore patterns for target folder", "path", folderPath, err)
continue
}
// Recursively walk this folder and all its children
err = walkFolder(ctx, job, folderPath, checker, results)
if err != nil {
log.Error(ctx, "Scanner: Error walking target folder", "path", folderPath, err)
continue
}
}
log.Debug(ctx, "Scanner: Finished reading folders", "lib", job.lib.Name, "path", job.lib.Path, "numFolders", job.numFolders.Load())
log.Debug(ctx, "Scanner: Finished reading target folders", "lib", job.lib.Name, "path", job.lib.Path, "numFolders", job.numFolders.Load())
}()
return results, nil
}
func walkFolder(ctx context.Context, job *scanJob, currentFolder string, ignorePatterns []string, results chan<- *folderEntry) error {
ignorePatterns = loadIgnoredPatterns(ctx, job.fs, currentFolder, ignorePatterns)
func walkFolder(ctx context.Context, job *scanJob, currentFolder string, checker *IgnoreChecker, results chan<- *folderEntry) error {
// Push patterns for this folder onto the stack
_ = checker.Push(ctx, currentFolder)
defer checker.Pop() // Pop patterns when leaving this folder
folder, children, err := loadDir(ctx, job, currentFolder, ignorePatterns)
folder, children, err := loadDir(ctx, job, currentFolder, checker)
if err != nil {
log.Warn(ctx, "Scanner: Error loading dir. Skipping", "path", currentFolder, err)
return nil
}
for _, c := range children {
err := walkFolder(ctx, job, c, ignorePatterns, results)
err := walkFolder(ctx, job, c, checker, results)
if err != nil {
return err
}
@ -59,50 +90,17 @@ func walkFolder(ctx context.Context, job *scanJob, currentFolder string, ignoreP
return nil
}
func loadIgnoredPatterns(ctx context.Context, fsys fs.FS, currentFolder string, currentPatterns []string) []string {
ignoreFilePath := path.Join(currentFolder, consts.ScanIgnoreFile)
var newPatterns []string
if _, err := fs.Stat(fsys, ignoreFilePath); err == nil {
// Read and parse the .ndignore file
ignoreFile, err := fsys.Open(ignoreFilePath)
if err != nil {
log.Warn(ctx, "Scanner: Error opening .ndignore file", "path", ignoreFilePath, err)
// Continue with previous patterns
} else {
defer ignoreFile.Close()
scanner := bufio.NewScanner(ignoreFile)
for scanner.Scan() {
line := scanner.Text()
if line == "" || strings.HasPrefix(line, "#") {
continue // Skip empty lines and comments
}
newPatterns = append(newPatterns, line)
}
if err := scanner.Err(); err != nil {
log.Warn(ctx, "Scanner: Error reading .ignore file", "path", ignoreFilePath, err)
}
}
// If the .ndignore file is empty, mimic the current behavior and ignore everything
if len(newPatterns) == 0 {
log.Trace(ctx, "Scanner: .ndignore file is empty, ignoring everything", "path", currentFolder)
newPatterns = []string{"**/*"}
} else {
log.Trace(ctx, "Scanner: .ndignore file found ", "path", ignoreFilePath, "patterns", newPatterns)
}
}
// Combine the patterns from the .ndignore file with the ones passed as argument
combinedPatterns := append([]string{}, currentPatterns...)
return append(combinedPatterns, newPatterns...)
}
func loadDir(ctx context.Context, job *scanJob, dirPath string, ignorePatterns []string) (folder *folderEntry, children []string, err error) {
folder = newFolderEntry(job, dirPath)
func loadDir(ctx context.Context, job *scanJob, dirPath string, checker *IgnoreChecker) (folder *folderEntry, children []string, err error) {
// Check if directory exists before creating the folder entry
// This is important to avoid removing the folder from lastUpdates if it doesn't exist
dirInfo, err := fs.Stat(job.fs, dirPath)
if err != nil {
log.Warn(ctx, "Scanner: Error stating dir", "path", dirPath, err)
return nil, nil, err
}
// Now that we know the folder exists, create the entry (which removes it from lastUpdates)
folder = job.createFolderEntry(dirPath)
folder.modTime = dirInfo.ModTime()
dir, err := job.fs.Open(dirPath)
@ -117,12 +115,11 @@ func loadDir(ctx context.Context, job *scanJob, dirPath string, ignorePatterns [
return folder, children, err
}
ignoreMatcher := ignore.CompileIgnoreLines(ignorePatterns...)
entries := fullReadDir(ctx, dirFile)
children = make([]string, 0, len(entries))
for _, entry := range entries {
entryPath := path.Join(dirPath, entry.Name())
if len(ignorePatterns) > 0 && isScanIgnored(ctx, ignoreMatcher, entryPath) {
if checker.ShouldIgnore(ctx, entryPath) {
log.Trace(ctx, "Scanner: Ignoring entry", "path", entryPath)
continue
}
@ -234,6 +231,7 @@ func isDirReadable(ctx context.Context, fsys fs.FS, dirPath string) bool {
var ignoredDirs = []string{
"$RECYCLE.BIN",
"#snapshot",
"@Recycle",
"@Recently-Snapshot",
".streams",
"lost+found",
@ -254,11 +252,3 @@ func isDirIgnored(name string) bool {
func isEntryIgnored(name string) bool {
return strings.HasPrefix(name, ".") && !strings.HasPrefix(name, "..")
}
func isScanIgnored(ctx context.Context, matcher *ignore.GitIgnore, entryPath string) bool {
matches := matcher.MatchesPath(entryPath)
if matches {
log.Trace(ctx, "Scanner: Ignoring entry matching .ndignore: ", "path", entryPath)
}
return matches
}

View File

@ -25,82 +25,196 @@ var _ = Describe("walk_dir_tree", func() {
ctx context.Context
)
BeforeEach(func() {
DeferCleanup(configtest.SetupConfig())
ctx = GinkgoT().Context()
fsys = &mockMusicFS{
FS: fstest.MapFS{
"root/a/.ndignore": {Data: []byte("ignored/*")},
"root/a/f1.mp3": {},
"root/a/f2.mp3": {},
"root/a/ignored/bad.mp3": {},
"root/b/cover.jpg": {},
"root/c/f3": {},
"root/d": {},
"root/d/.ndignore": {},
"root/d/f1.mp3": {},
"root/d/f2.mp3": {},
"root/d/f3.mp3": {},
"root/e/original/f1.mp3": {},
"root/e/symlink": {Mode: fs.ModeSymlink, Data: []byte("original")},
Context("full library", func() {
BeforeEach(func() {
DeferCleanup(configtest.SetupConfig())
ctx = GinkgoT().Context()
fsys = &mockMusicFS{
FS: fstest.MapFS{
"root/a/.ndignore": {Data: []byte("ignored/*")},
"root/a/f1.mp3": {},
"root/a/f2.mp3": {},
"root/a/ignored/bad.mp3": {},
"root/b/cover.jpg": {},
"root/c/f3": {},
"root/d": {},
"root/d/.ndignore": {},
"root/d/f1.mp3": {},
"root/d/f2.mp3": {},
"root/d/f3.mp3": {},
"root/e/original/f1.mp3": {},
"root/e/symlink": {Mode: fs.ModeSymlink, Data: []byte("original")},
},
}
job = &scanJob{
fs: fsys,
lib: model.Library{Path: "/music"},
}
})
// Helper function to call walkDirTree and collect folders from the results channel
getFolders := func() map[string]*folderEntry {
results, err := walkDirTree(ctx, job)
Expect(err).ToNot(HaveOccurred())
folders := map[string]*folderEntry{}
g := errgroup.Group{}
g.Go(func() error {
for folder := range results {
folders[folder.path] = folder
}
return nil
})
_ = g.Wait()
return folders
}
DescribeTable("symlink handling",
func(followSymlinks bool, expectedFolderCount int) {
conf.Server.Scanner.FollowSymlinks = followSymlinks
folders := getFolders()
Expect(folders).To(HaveLen(expectedFolderCount + 2)) // +2 for `.` and `root`
// Basic folder structure checks
Expect(folders["root/a"].audioFiles).To(SatisfyAll(
HaveLen(2),
HaveKey("f1.mp3"),
HaveKey("f2.mp3"),
))
Expect(folders["root/a"].imageFiles).To(BeEmpty())
Expect(folders["root/b"].audioFiles).To(BeEmpty())
Expect(folders["root/b"].imageFiles).To(SatisfyAll(
HaveLen(1),
HaveKey("cover.jpg"),
))
Expect(folders["root/c"].audioFiles).To(BeEmpty())
Expect(folders["root/c"].imageFiles).To(BeEmpty())
Expect(folders).ToNot(HaveKey("root/d"))
// Symlink specific checks
if followSymlinks {
Expect(folders["root/e/symlink"].audioFiles).To(HaveLen(1))
} else {
Expect(folders).ToNot(HaveKey("root/e/symlink"))
}
},
}
job = &scanJob{
fs: fsys,
lib: model.Library{Path: "/music"},
}
Entry("with symlinks enabled", true, 7),
Entry("with symlinks disabled", false, 6),
)
})
// Helper function to call walkDirTree and collect folders from the results channel
getFolders := func() map[string]*folderEntry {
results, err := walkDirTree(ctx, job)
Expect(err).ToNot(HaveOccurred())
folders := map[string]*folderEntry{}
g := errgroup.Group{}
g.Go(func() error {
for folder := range results {
folders[folder.path] = folder
Context("with target folders", func() {
BeforeEach(func() {
DeferCleanup(configtest.SetupConfig())
ctx = GinkgoT().Context()
fsys = &mockMusicFS{
FS: fstest.MapFS{
"Artist/Album1/track1.mp3": {},
"Artist/Album1/track2.mp3": {},
"Artist/Album2/track1.mp3": {},
"Artist/Album2/track2.mp3": {},
"Artist/Album2/Sub/track3.mp3": {},
"OtherArtist/Album3/track1.mp3": {},
},
}
job = &scanJob{
fs: fsys,
lib: model.Library{Path: "/music"},
}
return nil
})
_ = g.Wait()
return folders
}
DescribeTable("symlink handling",
func(followSymlinks bool, expectedFolderCount int) {
conf.Server.Scanner.FollowSymlinks = followSymlinks
folders := getFolders()
It("should recursively walk all subdirectories of target folders", func() {
results, err := walkDirTree(ctx, job, "Artist")
Expect(err).ToNot(HaveOccurred())
Expect(folders).To(HaveLen(expectedFolderCount + 2)) // +2 for `.` and `root`
folders := map[string]*folderEntry{}
g := errgroup.Group{}
g.Go(func() error {
for folder := range results {
folders[folder.path] = folder
}
return nil
})
_ = g.Wait()
// Basic folder structure checks
Expect(folders["root/a"].audioFiles).To(SatisfyAll(
HaveLen(2),
HaveKey("f1.mp3"),
HaveKey("f2.mp3"),
// Should include the target folder and all its descendants
Expect(folders).To(SatisfyAll(
HaveKey("Artist"),
HaveKey("Artist/Album1"),
HaveKey("Artist/Album2"),
HaveKey("Artist/Album2/Sub"),
))
Expect(folders["root/a"].imageFiles).To(BeEmpty())
Expect(folders["root/b"].audioFiles).To(BeEmpty())
Expect(folders["root/b"].imageFiles).To(SatisfyAll(
HaveLen(1),
HaveKey("cover.jpg"),
))
Expect(folders["root/c"].audioFiles).To(BeEmpty())
Expect(folders["root/c"].imageFiles).To(BeEmpty())
Expect(folders).ToNot(HaveKey("root/d"))
// Symlink specific checks
if followSymlinks {
Expect(folders["root/e/symlink"].audioFiles).To(HaveLen(1))
} else {
Expect(folders).ToNot(HaveKey("root/e/symlink"))
// Should not include folders outside the target
Expect(folders).ToNot(HaveKey("OtherArtist"))
Expect(folders).ToNot(HaveKey("OtherArtist/Album3"))
// Verify audio files are present
Expect(folders["Artist/Album1"].audioFiles).To(HaveLen(2))
Expect(folders["Artist/Album2"].audioFiles).To(HaveLen(2))
Expect(folders["Artist/Album2/Sub"].audioFiles).To(HaveLen(1))
})
It("should handle multiple target folders", func() {
results, err := walkDirTree(ctx, job, "Artist/Album1", "OtherArtist")
Expect(err).ToNot(HaveOccurred())
folders := map[string]*folderEntry{}
g := errgroup.Group{}
g.Go(func() error {
for folder := range results {
folders[folder.path] = folder
}
return nil
})
_ = g.Wait()
// Should include both target folders and their descendants
Expect(folders).To(SatisfyAll(
HaveKey("Artist/Album1"),
HaveKey("OtherArtist"),
HaveKey("OtherArtist/Album3"),
))
// Should not include other folders
Expect(folders).ToNot(HaveKey("Artist"))
Expect(folders).ToNot(HaveKey("Artist/Album2"))
Expect(folders).ToNot(HaveKey("Artist/Album2/Sub"))
})
It("should skip non-existent target folders and preserve them in lastUpdates", func() {
// Setup job with lastUpdates for both existing and non-existing folders
job.lastUpdates = map[string]model.FolderUpdateInfo{
model.FolderID(job.lib, "Artist/Album1"): {},
model.FolderID(job.lib, "NonExistent/DeletedFolder"): {},
model.FolderID(job.lib, "OtherArtist/Album3"): {},
}
},
Entry("with symlinks enabled", true, 7),
Entry("with symlinks disabled", false, 6),
)
// Try to scan existing folder and non-existing folder
results, err := walkDirTree(ctx, job, "Artist/Album1", "NonExistent/DeletedFolder")
Expect(err).ToNot(HaveOccurred())
// Collect results
folders := map[string]struct{}{}
for folder := range results {
folders[folder.path] = struct{}{}
}
// Should only include the existing folder
Expect(folders).To(HaveKey("Artist/Album1"))
Expect(folders).ToNot(HaveKey("NonExistent/DeletedFolder"))
// The non-existent folder should still be in lastUpdates (not removed by popLastUpdate)
Expect(job.lastUpdates).To(HaveKey(model.FolderID(job.lib, "NonExistent/DeletedFolder")))
// The existing folder should have been removed from lastUpdates
Expect(job.lastUpdates).ToNot(HaveKey(model.FolderID(job.lib, "Artist/Album1")))
// Folders not in targets should remain in lastUpdates
Expect(job.lastUpdates).To(HaveKey(model.FolderID(job.lib, "OtherArtist/Album3")))
})
})
})
Describe("helper functions", func() {

View File

@ -24,9 +24,9 @@ type Watcher interface {
type watcher struct {
mainCtx context.Context
ds model.DataStore
scanner Scanner
scanner model.Scanner
triggerWait time.Duration
watcherNotify chan model.Library
watcherNotify chan scanNotification
libraryWatchers map[int]*libraryWatcherInstance
mu sync.RWMutex
}
@ -36,14 +36,19 @@ type libraryWatcherInstance struct {
cancel context.CancelFunc
}
type scanNotification struct {
Library *model.Library
FolderPath string
}
// GetWatcher returns the watcher singleton
func GetWatcher(ds model.DataStore, s Scanner) Watcher {
func GetWatcher(ds model.DataStore, s model.Scanner) Watcher {
return singleton.GetInstance(func() *watcher {
return &watcher{
ds: ds,
scanner: s,
triggerWait: conf.Server.Scanner.WatcherWait,
watcherNotify: make(chan model.Library, 1),
watcherNotify: make(chan scanNotification, 1),
libraryWatchers: make(map[int]*libraryWatcherInstance),
}
})
@ -68,11 +73,11 @@ func (w *watcher) Run(ctx context.Context) error {
// Main scan triggering loop
trigger := time.NewTimer(w.triggerWait)
trigger.Stop()
waiting := false
targets := make(map[model.ScanTarget]struct{})
for {
select {
case <-trigger.C:
log.Info("Watcher: Triggering scan")
log.Info("Watcher: Triggering scan for changed folders", "numTargets", len(targets))
status, err := w.scanner.Status(ctx)
if err != nil {
log.Error(ctx, "Watcher: Error retrieving Scanner status", err)
@ -83,9 +88,23 @@ func (w *watcher) Run(ctx context.Context) error {
trigger.Reset(w.triggerWait * 3)
continue
}
waiting = false
// Convert targets map to slice
targetSlice := make([]model.ScanTarget, 0, len(targets))
for target := range targets {
targetSlice = append(targetSlice, target)
}
// Clear targets for next batch
targets = make(map[model.ScanTarget]struct{})
go func() {
_, err := w.scanner.ScanAll(ctx, false)
var err error
if conf.Server.DevSelectiveWatcher {
_, err = w.scanner.ScanFolders(ctx, false, targetSlice)
} else {
_, err = w.scanner.ScanAll(ctx, false)
}
if err != nil {
log.Error(ctx, "Watcher: Error scanning", err)
} else {
@ -102,13 +121,22 @@ func (w *watcher) Run(ctx context.Context) error {
w.libraryWatchers = make(map[int]*libraryWatcherInstance)
w.mu.Unlock()
return nil
case lib := <-w.watcherNotify:
if !waiting {
log.Debug(ctx, "Watcher: Detected changes. Waiting for more changes before triggering scan",
"libraryID", lib.ID, "name", lib.Name, "path", lib.Path)
waiting = true
}
case notification := <-w.watcherNotify:
// Reset the trigger timer for debounce
trigger.Reset(w.triggerWait)
lib := notification.Library
folderPath := notification.FolderPath
// If already scheduled for scan, skip
target := model.ScanTarget{LibraryID: lib.ID, FolderPath: folderPath}
if _, exists := targets[target]; exists {
continue
}
targets[target] = struct{}{}
log.Debug(ctx, "Watcher: Detected changes. Waiting for more changes before triggering scan",
"libraryID", lib.ID, "name", lib.Name, "path", lib.Path, "folderPath", folderPath)
}
}
}
@ -199,13 +227,18 @@ func (w *watcher) watchLibrary(ctx context.Context, lib *model.Library) error {
log.Info(ctx, "Watcher started for library", "libraryID", lib.ID, "name", lib.Name, "path", lib.Path, "absoluteLibPath", absLibPath)
return w.processLibraryEvents(ctx, lib, fsys, c, absLibPath)
}
// processLibraryEvents processes filesystem events for a library.
func (w *watcher) processLibraryEvents(ctx context.Context, lib *model.Library, fsys storage.MusicFS, events <-chan string, absLibPath string) error {
for {
select {
case <-ctx.Done():
log.Debug(ctx, "Watcher stopped due to context cancellation", "libraryID", lib.ID, "name", lib.Name)
return nil
case path := <-c:
path, err = filepath.Rel(absLibPath, path)
case path := <-events:
path, err := filepath.Rel(absLibPath, path)
if err != nil {
log.Error(ctx, "Error getting relative path", "libraryID", lib.ID, "absolutePath", absLibPath, "path", path, err)
continue
@ -215,12 +248,27 @@ func (w *watcher) watchLibrary(ctx context.Context, lib *model.Library) error {
log.Trace(ctx, "Ignoring change", "libraryID", lib.ID, "path", path)
continue
}
log.Trace(ctx, "Detected change", "libraryID", lib.ID, "path", path, "absoluteLibPath", absLibPath)
// Check if the original path (before resolution) matches .ndignore patterns
// This is crucial for deleted folders - if a deleted folder matches .ndignore,
// we should ignore it BEFORE resolveFolderPath walks up to the parent
if w.shouldIgnoreFolderPath(ctx, fsys, path) {
log.Debug(ctx, "Ignoring change matching .ndignore pattern", "libraryID", lib.ID, "path", path)
continue
}
// Find the folder to scan - validate path exists as directory, walk up if needed
folderPath := resolveFolderPath(fsys, path)
// Double-check after resolution in case the resolved path is different and also matches patterns
if folderPath != path && w.shouldIgnoreFolderPath(ctx, fsys, folderPath) {
log.Trace(ctx, "Ignoring change in folder matching .ndignore pattern", "libraryID", lib.ID, "folderPath", folderPath)
continue
}
// Notify the main watcher of changes
select {
case w.watcherNotify <- *lib:
case w.watcherNotify <- scanNotification{Library: lib, FolderPath: folderPath}:
default:
// Channel is full, notification already pending
}
@ -228,6 +276,47 @@ func (w *watcher) watchLibrary(ctx context.Context, lib *model.Library) error {
}
}
// resolveFolderPath takes a path (which may be a file or directory) and returns
// the folder path to scan. If the path is a file, it walks up to find the parent
// directory. Returns empty string if the path should scan the library root.
func resolveFolderPath(fsys fs.FS, path string) string {
// Handle root paths immediately
if path == "." || path == "" {
return ""
}
folderPath := path
for {
info, err := fs.Stat(fsys, folderPath)
if err == nil && info.IsDir() {
// Found a valid directory
return folderPath
}
if folderPath == "." || folderPath == "" {
// Reached root, scan entire library
return ""
}
// Walk up the tree
dir, _ := filepath.Split(folderPath)
if dir == "" || dir == "." {
return ""
}
// Remove trailing slash
folderPath = filepath.Clean(dir)
}
}
// shouldIgnoreFolderPath checks if the given folderPath should be ignored based on .ndignore patterns
// in the library. It pushes all parent folders onto the IgnoreChecker stack before checking.
func (w *watcher) shouldIgnoreFolderPath(ctx context.Context, fsys storage.MusicFS, folderPath string) bool {
checker := newIgnoreChecker(fsys)
err := checker.PushAllParents(ctx, folderPath)
if err != nil {
log.Warn(ctx, "Watcher: Error pushing ignore patterns for folder", "path", folderPath, err)
}
return checker.ShouldIgnore(ctx, folderPath)
}
func isIgnoredPath(_ context.Context, _ fs.FS, path string) bool {
baseDir, name := filepath.Split(path)
switch {

491
scanner/watcher_test.go Normal file
View File

@ -0,0 +1,491 @@
package scanner
import (
"context"
"io/fs"
"path/filepath"
"testing/fstest"
"time"
"github.com/navidrome/navidrome/conf"
"github.com/navidrome/navidrome/conf/configtest"
"github.com/navidrome/navidrome/model"
"github.com/navidrome/navidrome/tests"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
var _ = Describe("Watcher", func() {
var ctx context.Context
var cancel context.CancelFunc
var mockScanner *tests.MockScanner
var mockDS *tests.MockDataStore
var w *watcher
var lib *model.Library
BeforeEach(func() {
DeferCleanup(configtest.SetupConfig())
conf.Server.Scanner.WatcherWait = 50 * time.Millisecond // Short wait for tests
ctx, cancel = context.WithCancel(context.Background())
DeferCleanup(cancel)
lib = &model.Library{
ID: 1,
Name: "Test Library",
Path: "/test/library",
}
// Set up mocks
mockScanner = tests.NewMockScanner()
mockDS = &tests.MockDataStore{}
mockLibRepo := &tests.MockLibraryRepo{}
mockLibRepo.SetData(model.Libraries{*lib})
mockDS.MockedLibrary = mockLibRepo
// Create a new watcher instance (not singleton) for testing
w = &watcher{
ds: mockDS,
scanner: mockScanner,
triggerWait: conf.Server.Scanner.WatcherWait,
watcherNotify: make(chan scanNotification, 10),
libraryWatchers: make(map[int]*libraryWatcherInstance),
mainCtx: ctx,
}
})
Describe("Target Collection and Deduplication", func() {
BeforeEach(func() {
// Start watcher in background
go func() {
_ = w.Run(ctx)
}()
// Give watcher time to initialize
time.Sleep(10 * time.Millisecond)
})
It("creates separate targets for different folders", func() {
// Send notifications for different folders
w.watcherNotify <- scanNotification{Library: lib, FolderPath: "artist1"}
time.Sleep(10 * time.Millisecond)
w.watcherNotify <- scanNotification{Library: lib, FolderPath: "artist2"}
// Wait for watcher to process and trigger scan
Eventually(func() int {
return mockScanner.GetScanFoldersCallCount()
}, 200*time.Millisecond, 10*time.Millisecond).Should(Equal(1))
// Verify two targets
calls := mockScanner.GetScanFoldersCalls()
Expect(calls).To(HaveLen(1))
Expect(calls[0].Targets).To(HaveLen(2))
// Extract folder paths
folderPaths := make(map[string]bool)
for _, target := range calls[0].Targets {
Expect(target.LibraryID).To(Equal(1))
folderPaths[target.FolderPath] = true
}
Expect(folderPaths).To(HaveKey("artist1"))
Expect(folderPaths).To(HaveKey("artist2"))
})
It("handles different folder paths correctly", func() {
// Send notification for nested folder
w.watcherNotify <- scanNotification{Library: lib, FolderPath: "artist1/album1"}
// Wait for watcher to process and trigger scan
Eventually(func() int {
return mockScanner.GetScanFoldersCallCount()
}, 200*time.Millisecond, 10*time.Millisecond).Should(Equal(1))
// Verify the target
calls := mockScanner.GetScanFoldersCalls()
Expect(calls).To(HaveLen(1))
Expect(calls[0].Targets).To(HaveLen(1))
Expect(calls[0].Targets[0].FolderPath).To(Equal("artist1/album1"))
})
It("deduplicates folder and file within same folder", func() {
// Send notification for a folder
w.watcherNotify <- scanNotification{Library: lib, FolderPath: "artist1/album1"}
time.Sleep(10 * time.Millisecond)
// Send notification for same folder (as if file change was detected there)
// In practice, watchLibrary() would walk up from file path to folder
w.watcherNotify <- scanNotification{Library: lib, FolderPath: "artist1/album1"}
time.Sleep(10 * time.Millisecond)
// Send another for same folder
w.watcherNotify <- scanNotification{Library: lib, FolderPath: "artist1/album1"}
// Wait for watcher to process and trigger scan
Eventually(func() int {
return mockScanner.GetScanFoldersCallCount()
}, 200*time.Millisecond, 10*time.Millisecond).Should(Equal(1))
// Verify only one target despite multiple file/folder changes
calls := mockScanner.GetScanFoldersCalls()
Expect(calls).To(HaveLen(1))
Expect(calls[0].Targets).To(HaveLen(1))
Expect(calls[0].Targets[0].FolderPath).To(Equal("artist1/album1"))
})
})
Describe("Timer Behavior", func() {
BeforeEach(func() {
// Start watcher in background
go func() {
_ = w.Run(ctx)
}()
// Give watcher time to initialize
time.Sleep(10 * time.Millisecond)
})
It("resets timer on each change (debouncing)", func() {
// Send first notification
w.watcherNotify <- scanNotification{Library: lib, FolderPath: "artist1"}
// Wait a bit less than half the watcher wait time to ensure timer doesn't fire
time.Sleep(20 * time.Millisecond)
// No scan should have been triggered yet
Expect(mockScanner.GetScanFoldersCallCount()).To(Equal(0))
// Send another notification (resets timer)
w.watcherNotify <- scanNotification{Library: lib, FolderPath: "artist1"}
// Wait a bit less than half the watcher wait time again
time.Sleep(20 * time.Millisecond)
// Still no scan
Expect(mockScanner.GetScanFoldersCallCount()).To(Equal(0))
// Wait for full timer to expire after last notification (plus margin)
time.Sleep(60 * time.Millisecond)
// Now scan should have been triggered
Eventually(func() int {
return mockScanner.GetScanFoldersCallCount()
}, 100*time.Millisecond, 10*time.Millisecond).Should(Equal(1))
})
It("triggers scan after quiet period", func() {
// Send notification
w.watcherNotify <- scanNotification{Library: lib, FolderPath: "artist1"}
// No scan immediately
Expect(mockScanner.GetScanFoldersCallCount()).To(Equal(0))
// Wait for quiet period
Eventually(func() int {
return mockScanner.GetScanFoldersCallCount()
}, 200*time.Millisecond, 10*time.Millisecond).Should(Equal(1))
})
})
Describe("Empty and Root Paths", func() {
BeforeEach(func() {
// Start watcher in background
go func() {
_ = w.Run(ctx)
}()
// Give watcher time to initialize
time.Sleep(10 * time.Millisecond)
})
It("handles empty folder path (library root)", func() {
// Send notification with empty folder path
w.watcherNotify <- scanNotification{Library: lib, FolderPath: ""}
// Wait for scan
Eventually(func() int {
return mockScanner.GetScanFoldersCallCount()
}, 200*time.Millisecond, 10*time.Millisecond).Should(Equal(1))
// Should scan the library root
calls := mockScanner.GetScanFoldersCalls()
Expect(calls).To(HaveLen(1))
Expect(calls[0].Targets).To(HaveLen(1))
Expect(calls[0].Targets[0].FolderPath).To(Equal(""))
})
It("deduplicates empty and dot paths", func() {
// Send notifications with empty and dot paths
w.watcherNotify <- scanNotification{Library: lib, FolderPath: ""}
time.Sleep(10 * time.Millisecond)
w.watcherNotify <- scanNotification{Library: lib, FolderPath: ""}
// Wait for scan
Eventually(func() int {
return mockScanner.GetScanFoldersCallCount()
}, 200*time.Millisecond, 10*time.Millisecond).Should(Equal(1))
// Should have only one target
calls := mockScanner.GetScanFoldersCalls()
Expect(calls).To(HaveLen(1))
Expect(calls[0].Targets).To(HaveLen(1))
})
})
Describe("Multiple Libraries", func() {
var lib2 *model.Library
BeforeEach(func() {
// Create second library
lib2 = &model.Library{
ID: 2,
Name: "Test Library 2",
Path: "/test/library2",
}
mockLibRepo := mockDS.MockedLibrary.(*tests.MockLibraryRepo)
mockLibRepo.SetData(model.Libraries{*lib, *lib2})
// Start watcher in background
go func() {
_ = w.Run(ctx)
}()
// Give watcher time to initialize
time.Sleep(10 * time.Millisecond)
})
It("creates separate targets for different libraries", func() {
// Send notifications for both libraries
w.watcherNotify <- scanNotification{Library: lib, FolderPath: "artist1"}
time.Sleep(10 * time.Millisecond)
w.watcherNotify <- scanNotification{Library: lib2, FolderPath: "artist2"}
// Wait for scan
Eventually(func() int {
return mockScanner.GetScanFoldersCallCount()
}, 200*time.Millisecond, 10*time.Millisecond).Should(Equal(1))
// Verify two targets for different libraries
calls := mockScanner.GetScanFoldersCalls()
Expect(calls).To(HaveLen(1))
Expect(calls[0].Targets).To(HaveLen(2))
// Verify library IDs are different
libraryIDs := make(map[int]bool)
for _, target := range calls[0].Targets {
libraryIDs[target.LibraryID] = true
}
Expect(libraryIDs).To(HaveKey(1))
Expect(libraryIDs).To(HaveKey(2))
})
})
Describe(".ndignore handling", func() {
var ctx context.Context
var cancel context.CancelFunc
var w *watcher
var mockFS *mockMusicFS
var lib *model.Library
var eventChan chan string
var absLibPath string
BeforeEach(func() {
ctx, cancel = context.WithCancel(GinkgoT().Context())
DeferCleanup(cancel)
// Set up library
var err error
absLibPath, err = filepath.Abs(".")
Expect(err).NotTo(HaveOccurred())
lib = &model.Library{
ID: 1,
Name: "Test Library",
Path: absLibPath,
}
// Create watcher with notification channel
w = &watcher{
watcherNotify: make(chan scanNotification, 10),
}
eventChan = make(chan string, 10)
})
// Helper to send an event - converts relative path to absolute
sendEvent := func(relativePath string) {
path := filepath.Join(absLibPath, relativePath)
eventChan <- path
}
// Helper to start the real event processing loop
startEventProcessing := func() {
go func() {
defer GinkgoRecover()
// Call the actual processLibraryEvents method - testing the real implementation!
_ = w.processLibraryEvents(ctx, lib, mockFS, eventChan, absLibPath)
}()
}
Context("when a folder matching .ndignore is deleted", func() {
BeforeEach(func() {
// Create filesystem with .ndignore containing _TEMP pattern
// The deleted folder (_TEMP) will NOT exist in the filesystem
mockFS = &mockMusicFS{
FS: fstest.MapFS{
"rock": &fstest.MapFile{Mode: fs.ModeDir},
"rock/.ndignore": &fstest.MapFile{Data: []byte("_TEMP\n")},
"rock/valid_album": &fstest.MapFile{Mode: fs.ModeDir},
"rock/valid_album/track.mp3": &fstest.MapFile{Data: []byte("audio")},
},
}
})
It("should NOT send scan notification when deleted folder matches .ndignore", func() {
startEventProcessing()
// Simulate deletion event for rock/_TEMP
sendEvent("rock/_TEMP")
// Wait a bit to ensure event is processed
time.Sleep(50 * time.Millisecond)
// No notification should have been sent
Consistently(eventChan, 100*time.Millisecond).Should(BeEmpty())
})
It("should send scan notification for valid folder deletion", func() {
startEventProcessing()
// Simulate deletion event for rock/other_folder (not in .ndignore and doesn't exist)
// Since it doesn't exist in mockFS, resolveFolderPath will walk up to "rock"
sendEvent("rock/other_folder")
// Should receive notification for parent folder
Eventually(w.watcherNotify, 200*time.Millisecond).Should(Receive(Equal(scanNotification{
Library: lib,
FolderPath: "rock",
})))
})
})
Context("with nested folder patterns", func() {
BeforeEach(func() {
mockFS = &mockMusicFS{
FS: fstest.MapFS{
"music": &fstest.MapFile{Mode: fs.ModeDir},
"music/.ndignore": &fstest.MapFile{Data: []byte("**/temp\n**/cache\n")},
"music/rock": &fstest.MapFile{Mode: fs.ModeDir},
"music/rock/artist": &fstest.MapFile{Mode: fs.ModeDir},
},
}
})
It("should NOT send notification when nested ignored folder is deleted", func() {
startEventProcessing()
// Simulate deletion of music/rock/artist/temp (matches **/temp)
sendEvent("music/rock/artist/temp")
// Wait to ensure event is processed
time.Sleep(50 * time.Millisecond)
// No notification should be sent
Expect(w.watcherNotify).To(BeEmpty(), "Expected no scan notification for nested ignored folder")
})
It("should send notification for non-ignored nested folder", func() {
startEventProcessing()
// Simulate change in music/rock/artist (doesn't match any pattern)
sendEvent("music/rock/artist")
// Should receive notification
Eventually(w.watcherNotify, 200*time.Millisecond).Should(Receive(Equal(scanNotification{
Library: lib,
FolderPath: "music/rock/artist",
})))
})
})
Context("with file events in ignored folders", func() {
BeforeEach(func() {
mockFS = &mockMusicFS{
FS: fstest.MapFS{
"rock": &fstest.MapFile{Mode: fs.ModeDir},
"rock/.ndignore": &fstest.MapFile{Data: []byte("_TEMP\n")},
},
}
})
It("should NOT send notification for file changes in ignored folders", func() {
startEventProcessing()
// Simulate file change in rock/_TEMP/file.mp3
sendEvent("rock/_TEMP/file.mp3")
// Wait to ensure event is processed
time.Sleep(50 * time.Millisecond)
// No notification should be sent
Expect(w.watcherNotify).To(BeEmpty(), "Expected no scan notification for file in ignored folder")
})
})
})
})
var _ = Describe("resolveFolderPath", func() {
var mockFS fs.FS
BeforeEach(func() {
// Create a mock filesystem with some directories and files
mockFS = fstest.MapFS{
"artist1": &fstest.MapFile{Mode: fs.ModeDir},
"artist1/album1": &fstest.MapFile{Mode: fs.ModeDir},
"artist1/album1/track1.mp3": &fstest.MapFile{Data: []byte("audio")},
"artist1/album1/track2.mp3": &fstest.MapFile{Data: []byte("audio")},
"artist1/album2": &fstest.MapFile{Mode: fs.ModeDir},
"artist1/album2/song.flac": &fstest.MapFile{Data: []byte("audio")},
"artist2": &fstest.MapFile{Mode: fs.ModeDir},
"artist2/cover.jpg": &fstest.MapFile{Data: []byte("image")},
}
})
It("returns directory path when given a directory", func() {
result := resolveFolderPath(mockFS, "artist1/album1")
Expect(result).To(Equal("artist1/album1"))
})
It("walks up to parent directory when given a file path", func() {
result := resolveFolderPath(mockFS, "artist1/album1/track1.mp3")
Expect(result).To(Equal("artist1/album1"))
})
It("walks up multiple levels if needed", func() {
result := resolveFolderPath(mockFS, "artist1/album1/nonexistent/file.mp3")
Expect(result).To(Equal("artist1/album1"))
})
It("returns empty string for non-existent paths at root", func() {
result := resolveFolderPath(mockFS, "nonexistent/path/file.mp3")
Expect(result).To(Equal(""))
})
It("returns empty string for dot path", func() {
result := resolveFolderPath(mockFS, ".")
Expect(result).To(Equal(""))
})
It("returns empty string for empty path", func() {
result := resolveFolderPath(mockFS, "")
Expect(result).To(Equal(""))
})
It("handles nested file paths correctly", func() {
result := resolveFolderPath(mockFS, "artist1/album2/song.flac")
Expect(result).To(Equal("artist1/album2"))
})
It("resolves to top-level directory", func() {
result := resolveFolderPath(mockFS, "artist2/cover.jpg")
Expect(result).To(Equal("artist2"))
})
})

View File

@ -1,8 +1,11 @@
package server
import (
"bytes"
"cmp"
"context"
"crypto/tls"
"encoding/pem"
"errors"
"fmt"
"net"
@ -69,6 +72,13 @@ func (s *Server) Run(ctx context.Context, addr string, port int, tlsCert string,
// Determine if TLS is enabled
tlsEnabled := tlsCert != "" && tlsKey != ""
// Validate TLS certificates before starting the server
if tlsEnabled {
if err := validateTLSCertificates(tlsCert, tlsKey); err != nil {
return err
}
}
// Create a listener based on the address type (either Unix socket or TCP)
var listener net.Listener
var err error
@ -89,17 +99,17 @@ func (s *Server) Run(ctx context.Context, addr string, port int, tlsCert string,
// Start the server in a new goroutine and send an error signal to errC if there's an error
errC := make(chan error)
go func() {
var err error
if tlsEnabled {
// Start the HTTPS server
log.Info("Starting server with TLS (HTTPS) enabled", "tlsCert", tlsCert, "tlsKey", tlsKey)
if err := server.ServeTLS(listener, tlsCert, tlsKey); !errors.Is(err, http.ErrServerClosed) {
errC <- err
}
err = server.ServeTLS(listener, tlsCert, tlsKey)
} else {
// Start the HTTP server
if err := server.Serve(listener); !errors.Is(err, http.ErrServerClosed) {
errC <- err
}
err = server.Serve(listener)
}
if !errors.Is(err, http.ErrServerClosed) {
errC <- err
}
}()
@ -249,3 +259,56 @@ func AbsoluteURL(r *http.Request, u string, params url.Values) string {
}
return buildUrl.String()
}
// validateTLSCertificates validates the TLS certificate and key files before starting the server.
// It provides detailed error messages for common issues like encrypted private keys.
func validateTLSCertificates(certFile, keyFile string) error {
// Read the key file to check for encryption
keyData, err := os.ReadFile(keyFile)
if err != nil {
return fmt.Errorf("reading TLS key file: %w", err)
}
// Parse PEM blocks and check for encryption
block, _ := pem.Decode(keyData)
if block == nil {
return errors.New("TLS key file does not contain a valid PEM block")
}
// Check for encrypted private key indicators
if isEncryptedPEM(block, keyData) {
return errors.New("TLS private key is encrypted (password-protected). " +
"Navidrome does not support encrypted private keys. " +
"Please decrypt your key using: openssl pkey -in <encrypted-key> -out <decrypted-key>")
}
// Try to load the certificate pair to validate it
_, err = tls.LoadX509KeyPair(certFile, keyFile)
if err != nil {
return fmt.Errorf("loading TLS certificate/key pair: %w", err)
}
return nil
}
// isEncryptedPEM checks if a PEM block represents an encrypted private key.
func isEncryptedPEM(block *pem.Block, rawData []byte) bool {
// Check for PKCS#8 encrypted format (BEGIN ENCRYPTED PRIVATE KEY)
if block.Type == "ENCRYPTED PRIVATE KEY" {
return true
}
// Check for legacy encrypted format with Proc-Type header
if block.Headers != nil {
if procType, ok := block.Headers["Proc-Type"]; ok && strings.Contains(procType, "ENCRYPTED") {
return true
}
}
// Also check raw data for DEK-Info header (in case pem.Decode doesn't parse headers correctly)
if bytes.Contains(rawData, []byte("DEK-Info:")) || bytes.Contains(rawData, []byte("Proc-Type: 4,ENCRYPTED")) {
return true
}
return false
}

View File

@ -1,13 +1,20 @@
package server
import (
"context"
"crypto/tls"
"crypto/x509"
"fmt"
"io/fs"
"net/http"
"net/url"
"os"
"path/filepath"
"time"
"github.com/navidrome/navidrome/conf"
"github.com/navidrome/navidrome/conf/configtest"
"github.com/navidrome/navidrome/tests"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
@ -107,3 +114,146 @@ var _ = Describe("createUnixSocketFile", func() {
})
})
})
var _ = Describe("TLS support", func() {
Describe("validateTLSCertificates", func() {
const testDataDir = "server/testdata"
When("certificate and key are valid and unencrypted", func() {
It("returns nil", func() {
certFile := filepath.Join(testDataDir, "test_cert.pem")
keyFile := filepath.Join(testDataDir, "test_key.pem")
err := validateTLSCertificates(certFile, keyFile)
Expect(err).ToNot(HaveOccurred())
})
})
When("private key is encrypted with PKCS#8 format", func() {
It("returns an error with helpful message", func() {
certFile := filepath.Join(testDataDir, "test_cert_encrypted.pem")
keyFile := filepath.Join(testDataDir, "test_key_encrypted.pem")
err := validateTLSCertificates(certFile, keyFile)
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("encrypted"))
Expect(err.Error()).To(ContainSubstring("openssl"))
})
})
When("private key is encrypted with legacy format (Proc-Type header)", func() {
It("returns an error with helpful message", func() {
certFile := filepath.Join(testDataDir, "test_cert.pem")
keyFile := filepath.Join(testDataDir, "test_key_encrypted_legacy.pem")
err := validateTLSCertificates(certFile, keyFile)
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("encrypted"))
Expect(err.Error()).To(ContainSubstring("openssl"))
})
})
When("key file does not exist", func() {
It("returns an error", func() {
certFile := filepath.Join(testDataDir, "test_cert.pem")
keyFile := filepath.Join(testDataDir, "nonexistent.pem")
err := validateTLSCertificates(certFile, keyFile)
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("reading TLS key file"))
})
})
When("key file does not contain valid PEM", func() {
It("returns an error", func() {
// Create a temp file with invalid PEM content
tmpFile, err := os.CreateTemp("", "invalid_key*.pem")
Expect(err).ToNot(HaveOccurred())
DeferCleanup(func() {
_ = os.Remove(tmpFile.Name())
})
_, err = tmpFile.WriteString("not a valid PEM file")
Expect(err).ToNot(HaveOccurred())
_ = tmpFile.Close()
certFile := filepath.Join(testDataDir, "test_cert.pem")
err = validateTLSCertificates(certFile, tmpFile.Name())
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("valid PEM block"))
})
})
When("certificate file does not exist", func() {
It("returns an error from tls.LoadX509KeyPair", func() {
certFile := filepath.Join(testDataDir, "nonexistent_cert.pem")
keyFile := filepath.Join(testDataDir, "test_key.pem")
err := validateTLSCertificates(certFile, keyFile)
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("loading TLS certificate/key pair"))
})
})
})
Describe("Server TLS", func() {
const testDataDir = "server/testdata"
When("server is started with valid TLS certificates", func() {
It("accepts HTTPS connections", func() {
DeferCleanup(configtest.SetupConfig())
// Create server with mock dependencies
ds := &tests.MockDataStore{}
server := New(ds, nil, nil)
// Load the test certificate to create a trusted CA pool
certFile := filepath.Join(testDataDir, "test_cert.pem")
keyFile := filepath.Join(testDataDir, "test_key.pem")
caCert, err := os.ReadFile(certFile)
Expect(err).ToNot(HaveOccurred())
caCertPool := x509.NewCertPool()
caCertPool.AppendCertsFromPEM(caCert)
// Create an HTTPS client that trusts our test certificate
httpClient := &http.Client{
Timeout: 5 * time.Second,
Transport: &http.Transport{
TLSClientConfig: &tls.Config{
RootCAs: caCertPool,
MinVersion: tls.VersionTLS12,
},
},
}
// Start the server in a goroutine
ctx, cancel := context.WithCancel(GinkgoT().Context())
defer cancel()
errChan := make(chan error, 1)
go func() {
errChan <- server.Run(ctx, "127.0.0.1", 14534, certFile, keyFile)
}()
Eventually(func() error {
// Make an HTTPS request to the server
resp, err := httpClient.Get("https://127.0.0.1:14534/ping")
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("unexpected status code: %d", resp.StatusCode)
}
return nil
}, 2*time.Second, 100*time.Millisecond).Should(Succeed())
// Stop the server
cancel()
// Wait for server to stop (with timeout)
select {
case <-errChan:
// Server stopped
case <-time.After(2 * time.Second):
Fail("Server did not stop in time")
}
})
})
})
})

View File

@ -18,7 +18,6 @@ import (
"github.com/navidrome/navidrome/core/scrobbler"
"github.com/navidrome/navidrome/log"
"github.com/navidrome/navidrome/model"
"github.com/navidrome/navidrome/scanner"
"github.com/navidrome/navidrome/server"
"github.com/navidrome/navidrome/server/events"
"github.com/navidrome/navidrome/server/subsonic/responses"
@ -39,7 +38,7 @@ type Router struct {
players core.Players
provider external.Provider
playlists core.Playlists
scanner scanner.Scanner
scanner model.Scanner
broker events.Broker
scrobbler scrobbler.PlayTracker
share core.Share
@ -48,7 +47,7 @@ type Router struct {
}
func New(ds model.DataStore, artwork artwork.Artwork, streamer core.MediaStreamer, archiver core.Archiver,
players core.Players, provider external.Provider, scanner scanner.Scanner, broker events.Broker,
players core.Players, provider external.Provider, scanner model.Scanner, broker events.Broker,
playlists core.Playlists, scrobbler scrobbler.PlayTracker, share core.Share, playback playback.PlaybackServer,
metrics metrics.Metrics,
) *Router {

View File

@ -1,10 +1,13 @@
package subsonic
import (
"fmt"
"net/http"
"slices"
"time"
"github.com/navidrome/navidrome/log"
"github.com/navidrome/navidrome/model"
"github.com/navidrome/navidrome/model/request"
"github.com/navidrome/navidrome/server/subsonic/responses"
"github.com/navidrome/navidrome/utils/req"
@ -44,15 +47,56 @@ func (api *Router) StartScan(r *http.Request) (*responses.Subsonic, error) {
p := req.Params(r)
fullScan := p.BoolOr("fullScan", false)
// Parse optional target parameters for selective scanning
var targets []model.ScanTarget
if targetParams, err := p.Strings("target"); err == nil && len(targetParams) > 0 {
targets, err = model.ParseTargets(targetParams)
if err != nil {
return nil, newError(responses.ErrorGeneric, fmt.Sprintf("Invalid target parameter: %v", err))
}
// Validate all libraries in targets exist and user has access to them
userLibraries, err := api.ds.User(ctx).GetUserLibraries(loggedUser.ID)
if err != nil {
return nil, newError(responses.ErrorGeneric, "Internal error")
}
// Check each target library
for _, target := range targets {
if !slices.ContainsFunc(userLibraries, func(lib model.Library) bool { return lib.ID == target.LibraryID }) {
return nil, newError(responses.ErrorDataNotFound, fmt.Sprintf("Library with ID %d not found", target.LibraryID))
}
}
// Special case: if single library with empty path and it's the only library in DB, call ScanAll
if len(targets) == 1 && targets[0].FolderPath == "" {
allLibs, err := api.ds.Library(ctx).GetAll()
if err != nil {
return nil, newError(responses.ErrorGeneric, "Internal error")
}
if len(allLibs) == 1 {
targets = nil // This will trigger ScanAll below
}
}
}
go func() {
start := time.Now()
log.Info(ctx, "Triggering manual scan", "fullScan", fullScan, "user", loggedUser.UserName)
_, err := api.scanner.ScanAll(ctx, fullScan)
var err error
if len(targets) > 0 {
log.Info(ctx, "Triggering on-demand scan", "fullScan", fullScan, "targets", len(targets), "user", loggedUser.UserName)
_, err = api.scanner.ScanFolders(ctx, fullScan, targets)
} else {
log.Info(ctx, "Triggering on-demand scan", "fullScan", fullScan, "user", loggedUser.UserName)
_, err = api.scanner.ScanAll(ctx, fullScan)
}
if err != nil {
log.Error(ctx, "Error scanning", err)
return
}
log.Info(ctx, "Manual scan complete", "user", loggedUser.UserName, "elapsed", time.Since(start))
log.Info(ctx, "On-demand scan complete", "user", loggedUser.UserName, "elapsed", time.Since(start))
}()
return api.GetScanStatus(r)

View File

@ -0,0 +1,396 @@
package subsonic
import (
"context"
"errors"
"net/http/httptest"
"github.com/navidrome/navidrome/model"
"github.com/navidrome/navidrome/model/request"
"github.com/navidrome/navidrome/server/subsonic/responses"
"github.com/navidrome/navidrome/tests"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
var _ = Describe("LibraryScanning", func() {
var api *Router
var ms *tests.MockScanner
BeforeEach(func() {
ms = tests.NewMockScanner()
api = &Router{scanner: ms}
})
Describe("StartScan", func() {
It("requires admin authentication", func() {
// Create non-admin user
ctx := request.WithUser(context.Background(), model.User{
ID: "user-id",
IsAdmin: false,
})
// Create request
r := httptest.NewRequest("GET", "/rest/startScan", nil)
r = r.WithContext(ctx)
// Call endpoint
response, err := api.StartScan(r)
// Should return authorization error
Expect(err).To(HaveOccurred())
Expect(response).To(BeNil())
var subErr subError
ok := errors.As(err, &subErr)
Expect(ok).To(BeTrue())
Expect(subErr.code).To(Equal(responses.ErrorAuthorizationFail))
})
It("triggers a full scan with no parameters", func() {
// Create admin user
ctx := request.WithUser(context.Background(), model.User{
ID: "admin-id",
IsAdmin: true,
})
// Create request with no parameters
r := httptest.NewRequest("GET", "/rest/startScan", nil)
r = r.WithContext(ctx)
// Call endpoint
response, err := api.StartScan(r)
// Should succeed
Expect(err).ToNot(HaveOccurred())
Expect(response).ToNot(BeNil())
// Verify ScanAll was called (eventually, since it's in a goroutine)
Eventually(func() int {
return ms.GetScanAllCallCount()
}).Should(BeNumerically(">", 0))
calls := ms.GetScanAllCalls()
Expect(calls).To(HaveLen(1))
Expect(calls[0].FullScan).To(BeFalse())
})
It("triggers a full scan with fullScan=true", func() {
// Create admin user
ctx := request.WithUser(context.Background(), model.User{
ID: "admin-id",
IsAdmin: true,
})
// Create request with fullScan parameter
r := httptest.NewRequest("GET", "/rest/startScan?fullScan=true", nil)
r = r.WithContext(ctx)
// Call endpoint
response, err := api.StartScan(r)
// Should succeed
Expect(err).ToNot(HaveOccurred())
Expect(response).ToNot(BeNil())
// Verify ScanAll was called with fullScan=true
Eventually(func() int {
return ms.GetScanAllCallCount()
}).Should(BeNumerically(">", 0))
calls := ms.GetScanAllCalls()
Expect(calls).To(HaveLen(1))
Expect(calls[0].FullScan).To(BeTrue())
})
It("triggers a selective scan with single target parameter", func() {
// Setup mocks
mockUserRepo := tests.CreateMockUserRepo()
_ = mockUserRepo.SetUserLibraries("admin-id", []int{1, 2})
mockDS := &tests.MockDataStore{MockedUser: mockUserRepo}
api.ds = mockDS
// Create admin user
ctx := request.WithUser(context.Background(), model.User{
ID: "admin-id",
IsAdmin: true,
})
// Create request with single target parameter
r := httptest.NewRequest("GET", "/rest/startScan?target=1:Music/Rock", nil)
r = r.WithContext(ctx)
// Call endpoint
response, err := api.StartScan(r)
// Should succeed
Expect(err).ToNot(HaveOccurred())
Expect(response).ToNot(BeNil())
// Verify ScanFolders was called with correct targets
Eventually(func() int {
return ms.GetScanFoldersCallCount()
}).Should(BeNumerically(">", 0))
calls := ms.GetScanFoldersCalls()
Expect(calls).To(HaveLen(1))
targets := calls[0].Targets
Expect(targets).To(HaveLen(1))
Expect(targets[0].LibraryID).To(Equal(1))
Expect(targets[0].FolderPath).To(Equal("Music/Rock"))
})
It("triggers a selective scan with multiple target parameters", func() {
// Setup mocks
mockUserRepo := tests.CreateMockUserRepo()
_ = mockUserRepo.SetUserLibraries("admin-id", []int{1, 2})
mockDS := &tests.MockDataStore{MockedUser: mockUserRepo}
api.ds = mockDS
// Create admin user
ctx := request.WithUser(context.Background(), model.User{
ID: "admin-id",
IsAdmin: true,
})
// Create request with multiple target parameters
r := httptest.NewRequest("GET", "/rest/startScan?target=1:Music/Reggae&target=2:Classical/Bach", nil)
r = r.WithContext(ctx)
// Call endpoint
response, err := api.StartScan(r)
// Should succeed
Expect(err).ToNot(HaveOccurred())
Expect(response).ToNot(BeNil())
// Verify ScanFolders was called with correct targets
Eventually(func() int {
return ms.GetScanFoldersCallCount()
}).Should(BeNumerically(">", 0))
calls := ms.GetScanFoldersCalls()
Expect(calls).To(HaveLen(1))
targets := calls[0].Targets
Expect(targets).To(HaveLen(2))
Expect(targets[0].LibraryID).To(Equal(1))
Expect(targets[0].FolderPath).To(Equal("Music/Reggae"))
Expect(targets[1].LibraryID).To(Equal(2))
Expect(targets[1].FolderPath).To(Equal("Classical/Bach"))
})
It("triggers a selective full scan with target and fullScan parameters", func() {
// Setup mocks
mockUserRepo := tests.CreateMockUserRepo()
_ = mockUserRepo.SetUserLibraries("admin-id", []int{1})
mockDS := &tests.MockDataStore{MockedUser: mockUserRepo}
api.ds = mockDS
// Create admin user
ctx := request.WithUser(context.Background(), model.User{
ID: "admin-id",
IsAdmin: true,
})
// Create request with target and fullScan parameters
r := httptest.NewRequest("GET", "/rest/startScan?target=1:Music/Jazz&fullScan=true", nil)
r = r.WithContext(ctx)
// Call endpoint
response, err := api.StartScan(r)
// Should succeed
Expect(err).ToNot(HaveOccurred())
Expect(response).ToNot(BeNil())
// Verify ScanFolders was called with fullScan=true
Eventually(func() int {
return ms.GetScanFoldersCallCount()
}).Should(BeNumerically(">", 0))
calls := ms.GetScanFoldersCalls()
Expect(calls).To(HaveLen(1))
Expect(calls[0].FullScan).To(BeTrue())
targets := calls[0].Targets
Expect(targets).To(HaveLen(1))
})
It("returns error for invalid target format", func() {
// Create admin user
ctx := request.WithUser(context.Background(), model.User{
ID: "admin-id",
IsAdmin: true,
})
// Create request with invalid target format (missing colon)
r := httptest.NewRequest("GET", "/rest/startScan?target=1MusicRock", nil)
r = r.WithContext(ctx)
// Call endpoint
response, err := api.StartScan(r)
// Should return error
Expect(err).To(HaveOccurred())
Expect(response).To(BeNil())
var subErr subError
ok := errors.As(err, &subErr)
Expect(ok).To(BeTrue())
Expect(subErr.code).To(Equal(responses.ErrorGeneric))
})
It("returns error for invalid library ID in target", func() {
// Create admin user
ctx := request.WithUser(context.Background(), model.User{
ID: "admin-id",
IsAdmin: true,
})
// Create request with invalid library ID
r := httptest.NewRequest("GET", "/rest/startScan?target=0:Music/Rock", nil)
r = r.WithContext(ctx)
// Call endpoint
response, err := api.StartScan(r)
// Should return error
Expect(err).To(HaveOccurred())
Expect(response).To(BeNil())
var subErr subError
ok := errors.As(err, &subErr)
Expect(ok).To(BeTrue())
Expect(subErr.code).To(Equal(responses.ErrorGeneric))
})
It("returns error when library does not exist", func() {
// Setup mocks - user has access to library 1 and 2 only
mockUserRepo := tests.CreateMockUserRepo()
_ = mockUserRepo.SetUserLibraries("admin-id", []int{1, 2})
mockDS := &tests.MockDataStore{MockedUser: mockUserRepo}
api.ds = mockDS
// Create admin user
ctx := request.WithUser(context.Background(), model.User{
ID: "admin-id",
IsAdmin: true,
})
// Create request with library ID that doesn't exist
r := httptest.NewRequest("GET", "/rest/startScan?target=999:Music/Rock", nil)
r = r.WithContext(ctx)
// Call endpoint
response, err := api.StartScan(r)
// Should return ErrorDataNotFound
Expect(err).To(HaveOccurred())
Expect(response).To(BeNil())
var subErr subError
ok := errors.As(err, &subErr)
Expect(ok).To(BeTrue())
Expect(subErr.code).To(Equal(responses.ErrorDataNotFound))
})
It("calls ScanAll when single library with empty path and only one library exists", func() {
// Setup mocks - single library in DB
mockUserRepo := tests.CreateMockUserRepo()
_ = mockUserRepo.SetUserLibraries("admin-id", []int{1})
mockLibraryRepo := &tests.MockLibraryRepo{}
mockLibraryRepo.SetData(model.Libraries{
{ID: 1, Name: "Music Library", Path: "/music"},
})
mockDS := &tests.MockDataStore{
MockedUser: mockUserRepo,
MockedLibrary: mockLibraryRepo,
}
api.ds = mockDS
// Create admin user
ctx := request.WithUser(context.Background(), model.User{
ID: "admin-id",
IsAdmin: true,
})
// Create request with single library and empty path
r := httptest.NewRequest("GET", "/rest/startScan?target=1:", nil)
r = r.WithContext(ctx)
// Call endpoint
response, err := api.StartScan(r)
// Should succeed
Expect(err).ToNot(HaveOccurred())
Expect(response).ToNot(BeNil())
// Verify ScanAll was called instead of ScanFolders
Eventually(func() int {
return ms.GetScanAllCallCount()
}).Should(BeNumerically(">", 0))
Expect(ms.GetScanFoldersCallCount()).To(Equal(0))
})
It("calls ScanFolders when single library with empty path but multiple libraries exist", func() {
// Setup mocks - multiple libraries in DB
mockUserRepo := tests.CreateMockUserRepo()
_ = mockUserRepo.SetUserLibraries("admin-id", []int{1, 2})
mockLibraryRepo := &tests.MockLibraryRepo{}
mockLibraryRepo.SetData(model.Libraries{
{ID: 1, Name: "Music Library", Path: "/music"},
{ID: 2, Name: "Audiobooks", Path: "/audiobooks"},
})
mockDS := &tests.MockDataStore{
MockedUser: mockUserRepo,
MockedLibrary: mockLibraryRepo,
}
api.ds = mockDS
// Create admin user
ctx := request.WithUser(context.Background(), model.User{
ID: "admin-id",
IsAdmin: true,
})
// Create request with single library and empty path
r := httptest.NewRequest("GET", "/rest/startScan?target=1:", nil)
r = r.WithContext(ctx)
// Call endpoint
response, err := api.StartScan(r)
// Should succeed
Expect(err).ToNot(HaveOccurred())
Expect(response).ToNot(BeNil())
// Verify ScanFolders was called (not ScanAll)
Eventually(func() int {
return ms.GetScanFoldersCallCount()
}).Should(BeNumerically(">", 0))
calls := ms.GetScanFoldersCalls()
Expect(calls).To(HaveLen(1))
targets := calls[0].Targets
Expect(targets).To(HaveLen(1))
Expect(targets[0].LibraryID).To(Equal(1))
Expect(targets[0].FolderPath).To(Equal(""))
})
})
Describe("GetScanStatus", func() {
It("returns scan status", func() {
// Setup mock scanner status
ms.SetStatusResponse(&model.ScannerStatus{
Scanning: false,
Count: 100,
FolderCount: 10,
})
// Create request
ctx := context.Background()
r := httptest.NewRequest("GET", "/rest/getScanStatus", nil)
r = r.WithContext(ctx)
// Call endpoint
response, err := api.GetScanStatus(r)
// Should succeed
Expect(err).ToNot(HaveOccurred())
Expect(response).ToNot(BeNil())
Expect(response.ScanStatus).ToNot(BeNil())
Expect(response.ScanStatus.Scanning).To(BeFalse())
Expect(response.ScanStatus.Count).To(Equal(int64(100)))
Expect(response.ScanStatus.FolderCount).To(Equal(int64(10)))
})
})
})

23
server/testdata/test_cert.pem vendored Normal file
View File

@ -0,0 +1,23 @@
-----BEGIN CERTIFICATE-----
MIIDwzCCAqugAwIBAgIUXqdUxUOo8kmsDe71iTR+Vr7btP8wDQYJKoZIhvcNAQEL
BQAwYjELMAkGA1UEBhMCVVMxDTALBgNVBAgMBFRlc3QxDTALBgNVBAcMBFRlc3Qx
EjAQBgNVBAoMCU5hdmlkcm9tZTENMAsGA1UECwwEVGVzdDESMBAGA1UEAwwJbG9j
YWxob3N0MCAXDTI1MTEyODE5NTkxNVoYDzIxMjUxMTA0MTk1OTE1WjBiMQswCQYD
VQQGEwJVUzENMAsGA1UECAwEVGVzdDENMAsGA1UEBwwEVGVzdDESMBAGA1UECgwJ
TmF2aWRyb21lMQ0wCwYDVQQLDARUZXN0MRIwEAYDVQQDDAlsb2NhbGhvc3QwggEi
MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCkB/TQgl5ei5KRSHt5OJim8rKS
MzRlkK4BjSEM4D9ESbebdpEVjX48QuBYACrCvgvVp7mQGF5anl8Hm89trvd8ooVQ
x9IPQQ6gRKM+4gLrt9FHvFGGzZQS8UTQXN5oBi11E+8/Vs47HLUNXC2TRtRLCMyK
LYXQIXbhdp9anImlt+IHUxIQUchK6Zkld/gCm56X1bbzN/Zq91PQLpx2FZ0eZTjN
KaNgztLa+K/BDnTuk3iTTs9GEp6VCvqQE/6fk/UN/tkk2dLwKIFvPVR/YeAhVdz/
OHC4L3B36QN3+VQ2yDjsp1PVAPX07UnzXO3Oj7uGYnMQxwprGMEubm3nADDxAgMB
AAGjbzBtMB0GA1UdDgQWBBRAZHUVuLyzc0CfuZR9ApqMbawIqzAfBgNVHSMEGDAW
gBRAZHUVuLyzc0CfuZR9ApqMbawIqzAPBgNVHRMBAf8EBTADAQH/MBoGA1UdEQQT
MBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsFAAOCAQEAmDLXcPx9LNHs
GxQIE6Q5BXbVO7c8qrWmJf5FK5VWaifNZ9U+IBi+VlB4jCLK/OkwsviN/jOnwRYx
owjq0QG0YdRT4uD9fEMrAj+EwbnrQYZQvT0yGEWA+KW5TW08wt+/qnGJDwEgbjYJ
HTdICVMhs/e8Ex48fAgO8WSsdTDekOrhuwzIfeJ1LU4ZptLsD2ePFxuzutdIuW51
/mspQGsjXqZ1qnLsavLXh/lds2g602rTpYBNZVjV9WiOvaQS8vviOxBN6f+9vgRz
a8SEbHqBG6jeyVqVZ7MjxcYxaIkxeBwMyMwgb+wwDfVXo2FZzX2TVeB7ZppI+IKv
TXYurWPYsQ==
-----END CERTIFICATE-----

22
server/testdata/test_cert_encrypted.pem vendored Normal file
View File

@ -0,0 +1,22 @@
-----BEGIN CERTIFICATE-----
MIIDpzCCAo+gAwIBAgIUEa7gEJYwJqYEJjTY7otQ+oUyELwwDQYJKoZIhvcNAQEL
BQAwYjELMAkGA1UEBhMCVVMxDTALBgNVBAgMBFRlc3QxDTALBgNVBAcMBFRlc3Qx
EjAQBgNVBAoMCU5hdmlkcm9tZTENMAsGA1UECwwEVGVzdDESMBAGA1UEAwwJbG9j
YWxob3N0MCAXDTI1MTEyODE5NTI0OVoYDzIxMjUxMTA0MTk1MjQ5WjBiMQswCQYD
VQQGEwJVUzENMAsGA1UECAwEVGVzdDENMAsGA1UEBwwEVGVzdDESMBAGA1UECgwJ
TmF2aWRyb21lMQ0wCwYDVQQLDARUZXN0MRIwEAYDVQQDDAlsb2NhbGhvc3QwggEi
MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDBHgqJ1d9EnNxqoSZ6xXrIz/mV
Y0nWJW16/qIAvCdovSeTZhG9iqG8dUqcuu2BdD9MMHndJ2oFn3iD8EJR92dH8KBA
8xOmtZ0BEEWgXPBivywZVd1ChIflEWj6m5wwLNjb57SPpUiwaLxBQB8ByEaAAZE/
bLqvHI3vW/4s5apky17SPIqmkmqEYlRcg97tlRXsPuwoAVM9cvLMMEqtIR1CB/72
gboY2Gi2r/plLF/Rg3Dom6QljMWi57XXWJFwGYSXaZuM0gvn04e3oLu+1E+WMoq/
9rExWij2DlsmXd/RiScliFp6R4H84wQUyqrAUNytvgRO+oVnRjEA0l3oCYdRAgMB
AAGjUzBRMB0GA1UdDgQWBBQQKpB1UaKm98FnBdl8uKdRscrVTzAfBgNVHSMEGDAW
gBQQKpB1UaKm98FnBdl8uKdRscrVTzAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3
DQEBCwUAA4IBAQBP07l+2LmpFtcxqMGmsiNYwFuHpQCxJd4YRZHjLX7O+oJExMgR
2yP4mpMKurgKOv7unTDLwvjQRa6ZTYJCsYtvC6hbyqlGc7AfNTu6DKz8r35/2/V5
hPsG5lNb91HhvHE839mLAvpi02LoFH2Sr8BR7s6qxfNKYcP8PUOJQXltJ6yAa8YJ
syeXQQ3RIyGsJANeaC06S3UdkBM5H5BLfIHnHu3GybJjwL51va4WCdHe8QV6GI0g
RDiThDVkBSXAr136vnMdlrYCxMoxY56itJ0zbYg2ELQKU9o1w/ZJQo9uvmy9jCoZ
Hy1L5a2vUDbsdONdvRkYZRHqMpG4bdD8D3j2
-----END CERTIFICATE-----

28
server/testdata/test_key.pem vendored Normal file
View File

@ -0,0 +1,28 @@
-----BEGIN PRIVATE KEY-----
MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCkB/TQgl5ei5KR
SHt5OJim8rKSMzRlkK4BjSEM4D9ESbebdpEVjX48QuBYACrCvgvVp7mQGF5anl8H
m89trvd8ooVQx9IPQQ6gRKM+4gLrt9FHvFGGzZQS8UTQXN5oBi11E+8/Vs47HLUN
XC2TRtRLCMyKLYXQIXbhdp9anImlt+IHUxIQUchK6Zkld/gCm56X1bbzN/Zq91PQ
Lpx2FZ0eZTjNKaNgztLa+K/BDnTuk3iTTs9GEp6VCvqQE/6fk/UN/tkk2dLwKIFv
PVR/YeAhVdz/OHC4L3B36QN3+VQ2yDjsp1PVAPX07UnzXO3Oj7uGYnMQxwprGMEu
bm3nADDxAgMBAAECggEABqJFvesP2v4FEvgd+kSWM+ZL34rPmy3zQ5/MDuPA20ep
89EjQ/5hdRl1TknPcOnTu7PZVuENa9fM2xdrl7GEU9eU0bQLJE/KwiOUgJYObS8V
eTO+DlghHXUBhfXDjux1CS+htOuTUqOyFNS+CR9Lta8o6ou1xjmcP7kW78i17mxF
TuH5SZlS8W9PFLXHCInbMtqGFaT2ss09kvoPk2FDvHfxEdy6M9tKkguz02g+4bqI
aAMp2N7AOfmRpC0HvVa1ZfZo5Z8/KMoNcIm3pV9DEVM369J9EzhnMNpkGben90aT
FqO2JNsy52wmXFZUc9xe8uPdfDahALCkBGncLyLNmQKBgQDZREjocjdzOoPSlCdx
mRNe9suHz2FpUpsHCPOCotG63hFVKpah/ZvpHSsQx5rXs/mawDTmzGY9GQiBrSvg
OhfHIyT3NOhVaNcMxTqJX7rs7OG8D0MBacD9ASSeZ89MUn8q1EHZr5qxLtXl5Ikw
mHtiGRdiKGFFrG9H0zncbGhy7QKBgQDBRhQ9RAasTdmUiNQly9GVFkXto4T/9UHx
rVU44htCI2IVZUMTGlNfclfxpByDrzyA56rMzN9SAkiIp4nPpMDs5hayXaaPoojs
CPzV7r2OjemZ6CTeQ1ODImRL8L/E3jJSgWd6YYoHSQ5hjEX4yT6ft0u0tZUfdMKd
VENWIJ/hlQKBgQCo2hXjeOi5R8+tN3EUKwhP9HOnX7dv+D/9jqpZa5qdpPpJeyjI
SmYCHKYci1Q+sWOaLiiu+km20B65UVFZGSzjmd+fs+GghzMifKGKo/iNK2ggFKhZ
j8vplRrVdQ45XZ/xNDbdLEmHzEN2QE+Skd7KFYADzCgU0vdFFdbRBPuD3QKBgGIq
fQctMRJ9LCE0akSURGwr9vKflmMHKCpfdqTAu0WZgS0K1Mm0GlqlUiPKzizYaauz
f14sRNV7kWnPZsDPlqn8p9SKmpnj3RW97uWeMCtiyx6/+VHm8ljts/GaY1zT2s1r
KqrPNfNDWQmU3MljNeqbh9lOTWK/xEVy0gzB31MNAoGAQNWrZvVdAbL95XW6STUu
JmQlqJTlluuqS0Rrd/uVEQwW0Vd1dZjRQcFAFiSiCQWTbtId5gFZd6hiIQl53Xz0
5cd+9mcyA/TaoCJYbMOFYsKbZMCBhefsovJlVQXedqJrIY6BdeGlet4GTAH5Qyl0
ytEIUnvn5YmmbI7PDz80XpU=
-----END PRIVATE KEY-----

30
server/testdata/test_key_encrypted.pem vendored Normal file
View File

@ -0,0 +1,30 @@
-----BEGIN ENCRYPTED PRIVATE KEY-----
MIIFNTBfBgkqhkiG9w0BBQ0wUjAxBgkqhkiG9w0BBQwwJAQQPH9PYzryCI3smm81
J8rm+QICCAAwDAYIKoZIhvcNAgkFADAdBglghkgBZQMEASoEEI+9XxNfKSiMYIVB
UfcGfncEggTQVw7tPslGy3mlofCNnhBSnMViv9kj6M11smD6Y8vHG0k9Kq+6g+Dx
mQE9ILrSZBzM0uS3y484u+vkdqlT4KehhjIx0IiezurOcM45UdTAwLFLPzeEDlHI
lOWQ3gOTB3J5AxiUQOa6QsDIM7AZilidQG0BxQYWyRBA5B8evJwJoAvdzzA9wGSm
2YdNm3tA6rU5U8cVG+qTJP9pjbtRx0medC/CBZdxGkrWBQH+aySfahJdU8X1JI2e
SY4WJRw1rLCow+DnHjZS/IVHFJivJSRYvnvw8fwjOMVtkf+dAVctKlb1Fj9X+RdG
T1sq3i6zwFLE/RRz4qM4DKZ6UaD9wRFLow8FmNWVuJJiPgCLx2rrNMe32quS/kQP
iOsXAUeA/Yg1fdMCJORxl0nWDmLYcNtBghCmS1lyk+t+AKWwJudrds5tQQe8ha2t
Q41is+tDKwGDC1wt4WXJvBhgAJzuqFtr30H0M1eBhwwDdaDd9v0Zr3r8V49WZM2c
i3qkwPPYkQD+pOcR12xBV8ptvDxaUl7RGlVqnEWHagT51BaIaXQ9teUrG6UPt8o2
LELJXF6CiwkbN6Y9sYx5XiKrIGxVhlQSZ1nB3XSFRHbu6e7VHPjnVwUeeg87J2Am
MEwqDzPU5sjKRn84+M91Y4uFAIeinaOJAQ0/tZVrf1iSeCMQyMUhW/8m7JPfG19F
NbJSPRXQuKmYKbWfXcMW2UFbp0zDs7s7p4zzbfde9IbVdq/o2nv3ZrNbrLak6O7y
FVt9q/xG4Tty6hSK6xtqtNZWcmfiMcTlk1Qcz2STvScbXtqgcgR6WUZfkLuzi09I
EDYFnzU5JNSY3U3VTv2hAPeU4xjTNM6kjF7L9JFGvdjH8Ko9UdxG9RZMd8xhBM/n
hxdzdVba4bDDz2z+0A2blSObrPrNsKr/3ZbnfuUiSs5NmqmUOifZ1t1PqGGO2Y5S
/cDKtrPk226hGomsUBfHtiIJPG1VRl4UaZiduqK3GGhtF491KU1mAfYzueok3TPq
JhLtLDIvEaFgmOmitFzROI/ifm6s4ssUvcvtbjwJumbjkU38OxYZFwbhwbe268G2
vgspJamlEGJNdGDzrCFQlA2+A9kazCttztikfh5QGV6WFfkc3Bt1XTPL51vtliQy
MS2gUnJUY2fuYCfz8rxLH1kQmyYsHQz5rUYyBkeDffrG9MzarmzSJXR63FRzVMf1
LQ7BSzei7dF6+J4KVCxjbGWF3GUGmGeOP5g5vJ3xb3YPJNJLT4Vai103pay59TGP
tESM2Vn0gJEvYApi707noFH5uFTW1cp7lloF41ddIUkL/QO7j+sjvBww+4DqBB7J
BmvLMnswa23yw9egYRG5jOXyCgIr+1rnNcph1HGJsvxvgJ2gwwo5NKCG8SC6LcZQ
fbDjX+ssmobLE3ktN03FZPMp32/ciexzuZoamfyiPXh7xE++ckifNEKJlNhx+kCG
mSR2wh+UGigQkgp/JxOzl6C4fhUbrEZr17oBqGim2p8h+GE0zD5JSHcn1rP86gGU
8JG/ilG4I8uMxUwhGj7amrWXUlJBd1by7e1EAL+utCo14/Tx3otB9/JtqY+lm9Ey
1ptPhMRQxvDNWrCmYM2kyrGghdNfEMir6GKDWI6PY9cwAFv/PLOxr1c=
-----END ENCRYPTED PRIVATE KEY-----

View File

@ -0,0 +1,30 @@
-----BEGIN RSA PRIVATE KEY-----
Proc-Type: 4,ENCRYPTED
DEK-Info: AES-256-CBC,3C969050EAB73F121B7F0E6B75C42525
V6pSaAsrn9CQNo4p88QshJLbg8zkQJEom81dPbYSVqQSZa9YlPtpLZ9YtuLj/Ay0
TScEKIj/gzQ32wNl6nhcSNIL9yy+X11r5gNv1kIHkecf+EbDW20VOiJsfD+6LUyW
hA96AIbPOwc76iCuvsKHPKU9MlEmjGipmk/C2RQLHCZJ3WkiDRgCM8KQ7vKhfACT
w908yj4cB1e/P0JPq8t/3F7kPJ+6SVM1vMEffHl0otQR3rAyrK8QikwJ0K9qX62d
cqchTVlEyyZBYovR8DrRRUDbsXS5j1ZmX3NQpvTSTFowr+33fMrY+4Oz8sdR4yx1
CQc0A0sHHxSEIr2xu4KzczwOYVJN8PVdU0pgvFj9KEm66N6EY5CSFIBHyO/ycOt9
U+wpkRjf3zS6ZaUU0NKdOcop4YX33i99/tZF2RNR1i7ETLYph+/LCf09286Bi3u/
UCCuWedyECPdz0c6j0s27Fdfc/HEK90OEzeWh/fc+H2gJZhqJYK9V47HPTQNNMnB
U1a6FsJlrKE3E6nfSnTLxrSx9m/XTV7HV+HkgX+q8VhN7Q2VHUqkPzE7ZOPYpZ+A
dQzsm1TmEMxym6osYqFzQScXR1NZasrV2MTQ2J16dUgCdGAM2YMUD9JaoJR+u77M
WAjYzDiRg84rLr/KbJPAwHbsfo2KpiapJGSBBEDhz4W1/LOrFhsjaqIMSy4yZDGm
1KqXGHIlqmuHI7v4fD8vuzhj7GUujRx85HSZWakE/uc6s5WrhkSeVKYJWPfpsxTv
dT3oLOGJ+nRzWxM3aFtuJghX0nIGdKxT4EAUNXz0/vLT3OP1QCZR+oELrriFzmtj
+O30bGH2SAFZEQJ/uTQg6celoNh89IzH4DJkcn67hqpX6mUiU9CrIr/eR9C/en8Q
smTbbC1C1pDUaCwR26Z+zgM90amh4yfOFKK2geO2Kj+TmwFHUvi6ZnSzMzCvty3t
+wdIrUtf55Lw51JCpLGl70mg4b/zBj5hqBkU2YvAAnz/htjfH/wrD6ZAF1TCdlRO
gyODrJjGRnLd/v0XLk0wp+RkAjBcSlRlkUvZY5BtugL7dIdwiNGGQPcOni9IVeG0
6vDUEQnDOLYDj4d/JcckTLuHdrP+SW+0RQl2HK5+/w1hScGXN4O48gccu7yR/MN8
DmpCg5rD/nq8sxJosmSt07GrN36KppYt8LCXQbSg3NG2Ad715caS2C+0Qtdm5MPD
rM1UyTXQYSJXgUN9yZS/pmzlguCywnnvsBPU6j3ljZwcoD41QJ/1OU09/W6sIMQR
IAiM35JHiLJiccFgxSE1qx5F1UZqX4P47jF0Wzi/sE/DYXg5qw2DoauqXNzqnumH
71UDGK1V6wQIV7UCZDa0WUfFzu470XpuFb8VmMOuHSQxkZESc9cz8k/ueAuO438Q
jnlkF1Ge2EEPuaK2zeaTj/lGyYA1AUfHRRgt/EMUQSBntmhlpnwVPYTVvYtHO2N5
wp7/y39KirnlTl99i3XiOJ4WF4gIU2IaSlqMo4+e/A32h2JFi9QfNyfItXe6Fm1X
d0j2XGHzwMfHEFKdWyrgtVZwc38/1d6xWYAhs02b2basV/0AQhFTaKf5Z268eBNJ
-----END RSA PRIVATE KEY-----

9
tests/fixtures/deezer.artist.bio.json vendored Normal file
View File

@ -0,0 +1,9 @@
{
"data": {
"artist": {
"bio": {
"full": "<p>Schoolmates Thomas and Guy-Manuel began their career in 1992 with the indie rock trio Darlin' (named after The Beach Boys song) but were scathingly dismissed by Melody Maker magazine as \"daft punk.\" Turning to house-inspired electronica, they used the put down as a name for their DJ-ing partnership and became a hugely successful and influential dance act.</p>"
}
}
}
}

File diff suppressed because one or more lines are too long

1
tests/fixtures/deezer.artist.top.json vendored Normal file

File diff suppressed because one or more lines are too long

View File

@ -0,0 +1,7 @@
<html>
<head>
<meta property="og:image" content="https://lastfm.freetls.fastly.net/i/u/ar0/818148bf682d429dc21b59a73ef6f68e.png" />
</head>
<body>
</body>
</html>

Some files were not shown because too many files have changed in this diff Show More