mirror of
https://github.com/navidrome/navidrome.git
synced 2026-08-31 07:30:32 +00:00
Merge branch 'navidrome:master' into fix-taglib-issues-in-vscodedevcontainer/4749
This commit is contained in:
commit
382b80ccb4
20
.github/workflows/pipeline.yml
vendored
20
.github/workflows/pipeline.yml
vendored
@ -25,7 +25,7 @@ jobs:
|
||||
git_tag: ${{ steps.git-version.outputs.GIT_TAG }}
|
||||
git_sha: ${{ steps.git-version.outputs.GIT_SHA }}
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
fetch-tags: true
|
||||
@ -63,7 +63,7 @@ jobs:
|
||||
name: Lint Go code
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Download TagLib
|
||||
uses: ./.github/actions/download-taglib
|
||||
@ -71,7 +71,7 @@ jobs:
|
||||
version: ${{ env.CROSS_TAGLIB_VERSION }}
|
||||
|
||||
- name: golangci-lint
|
||||
uses: golangci/golangci-lint-action@v8
|
||||
uses: golangci/golangci-lint-action@v9
|
||||
with:
|
||||
version: latest
|
||||
problem-matchers: true
|
||||
@ -93,7 +93,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v5
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Download TagLib
|
||||
uses: ./.github/actions/download-taglib
|
||||
@ -114,7 +114,7 @@ jobs:
|
||||
env:
|
||||
NODE_OPTIONS: "--max_old_space_size=4096"
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: 24
|
||||
@ -145,7 +145,7 @@ jobs:
|
||||
name: Lint i18n files
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@v6
|
||||
- run: |
|
||||
set -e
|
||||
for file in resources/i18n/*.json; do
|
||||
@ -191,7 +191,7 @@ jobs:
|
||||
PLATFORM=$(echo ${{ matrix.platform }} | tr '/' '_')
|
||||
echo "PLATFORM=$PLATFORM" >> $GITHUB_ENV
|
||||
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Prepare Docker Buildx
|
||||
uses: ./.github/actions/prepare-docker
|
||||
@ -264,7 +264,7 @@ jobs:
|
||||
env:
|
||||
REGISTRY_IMAGE: ghcr.io/${{ github.repository }}
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Download digests
|
||||
uses: actions/download-artifact@v6
|
||||
@ -318,7 +318,7 @@ jobs:
|
||||
runs-on: ubuntu-24.04
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- uses: actions/download-artifact@v6
|
||||
with:
|
||||
@ -352,7 +352,7 @@ jobs:
|
||||
outputs:
|
||||
package_list: ${{ steps.set-package-list.outputs.package_list }}
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
fetch-tags: true
|
||||
|
||||
2
.github/workflows/update-translations.yml
vendored
2
.github/workflows/update-translations.yml
vendored
@ -8,7 +8,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
if: ${{ github.repository_owner == 'navidrome' }}
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/checkout@v6
|
||||
- name: Get updated translations
|
||||
id: poeditor
|
||||
env:
|
||||
|
||||
@ -137,7 +137,6 @@ ENV ND_MUSICFOLDER=/music
|
||||
ENV ND_DATAFOLDER=/data
|
||||
ENV ND_CONFIGFILE=/data/navidrome.toml
|
||||
ENV ND_PORT=4533
|
||||
ENV GODEBUG="asyncpreemptoff=1"
|
||||
RUN touch /.nddockerenv
|
||||
|
||||
EXPOSE ${ND_PORT}
|
||||
|
||||
4
Makefile
4
Makefile
@ -16,7 +16,7 @@ DOCKER_TAG ?= deluan/navidrome:develop
|
||||
|
||||
# Taglib version to use in cross-compilation, from https://github.com/navidrome/cross-taglib
|
||||
CROSS_TAGLIB_VERSION ?= 2.1.1-1
|
||||
GOLANGCI_LINT_VERSION ?= v2.5.0
|
||||
GOLANGCI_LINT_VERSION ?= v2.6.2
|
||||
|
||||
UI_SRC_FILES := $(shell find ui -type f -not -path "ui/build/*" -not -path "ui/node_modules/*")
|
||||
|
||||
@ -54,7 +54,7 @@ testall: test-race test-i18n test-js ##@Development Run Go and JS tests
|
||||
.PHONY: testall
|
||||
|
||||
test-race: ##@Development Run Go tests with race detector
|
||||
go test -tags netgo -race -shuffle=on ./...
|
||||
go test -tags netgo -race -shuffle=on $(PKG)
|
||||
.PHONY: test-race
|
||||
|
||||
test-js: ##@Development Run JS tests
|
||||
|
||||
@ -346,7 +346,7 @@ func startPluginManager(ctx context.Context) func() error {
|
||||
// TODO: Implement some struct tags to map flags to viper
|
||||
func init() {
|
||||
cobra.OnInitialize(func() {
|
||||
conf.InitConfig(cfgFile)
|
||||
conf.InitConfig(cfgFile, true)
|
||||
})
|
||||
|
||||
rootCmd.PersistentFlags().StringVarP(&cfgFile, "configfile", "c", "", `config file (default "./navidrome.toml")`)
|
||||
|
||||
16
cmd/scan.go
16
cmd/scan.go
@ -8,6 +8,7 @@ import (
|
||||
"github.com/navidrome/navidrome/core"
|
||||
"github.com/navidrome/navidrome/db"
|
||||
"github.com/navidrome/navidrome/log"
|
||||
"github.com/navidrome/navidrome/model"
|
||||
"github.com/navidrome/navidrome/persistence"
|
||||
"github.com/navidrome/navidrome/scanner"
|
||||
"github.com/navidrome/navidrome/utils/pl"
|
||||
@ -17,11 +18,13 @@ import (
|
||||
var (
|
||||
fullScan bool
|
||||
subprocess bool
|
||||
targets []string
|
||||
)
|
||||
|
||||
func init() {
|
||||
scanCmd.Flags().BoolVarP(&fullScan, "full", "f", false, "check all subfolders, ignoring timestamps")
|
||||
scanCmd.Flags().BoolVarP(&subprocess, "subprocess", "", false, "run as subprocess (internal use)")
|
||||
scanCmd.Flags().StringArrayVarP(&targets, "target", "t", []string{}, "list of libraryID:folderPath pairs, can be repeated (e.g., \"-t 1:Music/Rock -t 1:Music/Jazz -t 2:Classical\")")
|
||||
rootCmd.AddCommand(scanCmd)
|
||||
}
|
||||
|
||||
@ -68,7 +71,18 @@ func runScanner(ctx context.Context) {
|
||||
ds := persistence.New(sqlDB)
|
||||
pls := core.NewPlaylists(ds)
|
||||
|
||||
progress, err := scanner.CallScan(ctx, ds, pls, fullScan)
|
||||
// Parse targets if provided
|
||||
var scanTargets []model.ScanTarget
|
||||
if len(targets) > 0 {
|
||||
var err error
|
||||
scanTargets, err = model.ParseTargets(targets)
|
||||
if err != nil {
|
||||
log.Fatal(ctx, "Failed to parse targets", err)
|
||||
}
|
||||
log.Info(ctx, "Scanning specific folders", "numTargets", len(scanTargets))
|
||||
}
|
||||
|
||||
progress, err := scanner.CallScan(ctx, ds, pls, fullScan, scanTargets)
|
||||
if err != nil {
|
||||
log.Fatal(ctx, "Failed to scan", err)
|
||||
}
|
||||
|
||||
@ -69,9 +69,9 @@ func CreateNativeAPIRouter(ctx context.Context) *nativeapi.Router {
|
||||
artworkArtwork := artwork.NewArtwork(dataStore, fileCache, fFmpeg, provider)
|
||||
cacheWarmer := artwork.NewCacheWarmer(artworkArtwork, fileCache)
|
||||
broker := events.GetBroker()
|
||||
scannerScanner := scanner.New(ctx, dataStore, cacheWarmer, broker, playlists, metricsMetrics)
|
||||
watcher := scanner.GetWatcher(dataStore, scannerScanner)
|
||||
library := core.NewLibrary(dataStore, scannerScanner, watcher, broker)
|
||||
modelScanner := scanner.New(ctx, dataStore, cacheWarmer, broker, playlists, metricsMetrics)
|
||||
watcher := scanner.GetWatcher(dataStore, modelScanner)
|
||||
library := core.NewLibrary(dataStore, modelScanner, watcher, broker)
|
||||
maintenance := core.NewMaintenance(dataStore)
|
||||
router := nativeapi.New(dataStore, share, playlists, insights, library, maintenance)
|
||||
return router
|
||||
@ -95,10 +95,10 @@ func CreateSubsonicAPIRouter(ctx context.Context) *subsonic.Router {
|
||||
cacheWarmer := artwork.NewCacheWarmer(artworkArtwork, fileCache)
|
||||
broker := events.GetBroker()
|
||||
playlists := core.NewPlaylists(dataStore)
|
||||
scannerScanner := scanner.New(ctx, dataStore, cacheWarmer, broker, playlists, metricsMetrics)
|
||||
modelScanner := scanner.New(ctx, dataStore, cacheWarmer, broker, playlists, metricsMetrics)
|
||||
playTracker := scrobbler.GetPlayTracker(dataStore, broker, manager)
|
||||
playbackServer := playback.GetInstance(dataStore)
|
||||
router := subsonic.New(dataStore, artworkArtwork, mediaStreamer, archiver, players, provider, scannerScanner, broker, playlists, playTracker, share, playbackServer, metricsMetrics)
|
||||
router := subsonic.New(dataStore, artworkArtwork, mediaStreamer, archiver, players, provider, modelScanner, broker, playlists, playTracker, share, playbackServer, metricsMetrics)
|
||||
return router
|
||||
}
|
||||
|
||||
@ -150,7 +150,7 @@ func CreatePrometheus() metrics.Metrics {
|
||||
return metricsMetrics
|
||||
}
|
||||
|
||||
func CreateScanner(ctx context.Context) scanner.Scanner {
|
||||
func CreateScanner(ctx context.Context) model.Scanner {
|
||||
sqlDB := db.Db()
|
||||
dataStore := persistence.New(sqlDB)
|
||||
fileCache := artwork.GetImageCache()
|
||||
@ -163,8 +163,8 @@ func CreateScanner(ctx context.Context) scanner.Scanner {
|
||||
cacheWarmer := artwork.NewCacheWarmer(artworkArtwork, fileCache)
|
||||
broker := events.GetBroker()
|
||||
playlists := core.NewPlaylists(dataStore)
|
||||
scannerScanner := scanner.New(ctx, dataStore, cacheWarmer, broker, playlists, metricsMetrics)
|
||||
return scannerScanner
|
||||
modelScanner := scanner.New(ctx, dataStore, cacheWarmer, broker, playlists, metricsMetrics)
|
||||
return modelScanner
|
||||
}
|
||||
|
||||
func CreateScanWatcher(ctx context.Context) scanner.Watcher {
|
||||
@ -180,8 +180,8 @@ func CreateScanWatcher(ctx context.Context) scanner.Watcher {
|
||||
cacheWarmer := artwork.NewCacheWarmer(artworkArtwork, fileCache)
|
||||
broker := events.GetBroker()
|
||||
playlists := core.NewPlaylists(dataStore)
|
||||
scannerScanner := scanner.New(ctx, dataStore, cacheWarmer, broker, playlists, metricsMetrics)
|
||||
watcher := scanner.GetWatcher(dataStore, scannerScanner)
|
||||
modelScanner := scanner.New(ctx, dataStore, cacheWarmer, broker, playlists, metricsMetrics)
|
||||
watcher := scanner.GetWatcher(dataStore, modelScanner)
|
||||
return watcher
|
||||
}
|
||||
|
||||
@ -202,7 +202,7 @@ func getPluginManager() plugins.Manager {
|
||||
|
||||
// wire_injectors.go:
|
||||
|
||||
var allProviders = wire.NewSet(core.Set, artwork.Set, server.New, subsonic.New, nativeapi.New, public.New, persistence.New, lastfm.NewRouter, listenbrainz.NewRouter, events.GetBroker, scanner.New, scanner.GetWatcher, plugins.GetManager, metrics.GetPrometheusInstance, db.Db, wire.Bind(new(agents.PluginLoader), new(plugins.Manager)), wire.Bind(new(scrobbler.PluginLoader), new(plugins.Manager)), wire.Bind(new(metrics.PluginLoader), new(plugins.Manager)), wire.Bind(new(core.Scanner), new(scanner.Scanner)), wire.Bind(new(core.Watcher), new(scanner.Watcher)))
|
||||
var allProviders = wire.NewSet(core.Set, artwork.Set, server.New, subsonic.New, nativeapi.New, public.New, persistence.New, lastfm.NewRouter, listenbrainz.NewRouter, events.GetBroker, scanner.New, scanner.GetWatcher, plugins.GetManager, metrics.GetPrometheusInstance, db.Db, wire.Bind(new(agents.PluginLoader), new(plugins.Manager)), wire.Bind(new(scrobbler.PluginLoader), new(plugins.Manager)), wire.Bind(new(metrics.PluginLoader), new(plugins.Manager)), wire.Bind(new(core.Watcher), new(scanner.Watcher)))
|
||||
|
||||
func GetPluginManager(ctx context.Context) plugins.Manager {
|
||||
manager := getPluginManager()
|
||||
|
||||
@ -45,7 +45,6 @@ var allProviders = wire.NewSet(
|
||||
wire.Bind(new(agents.PluginLoader), new(plugins.Manager)),
|
||||
wire.Bind(new(scrobbler.PluginLoader), new(plugins.Manager)),
|
||||
wire.Bind(new(metrics.PluginLoader), new(plugins.Manager)),
|
||||
wire.Bind(new(core.Scanner), new(scanner.Scanner)),
|
||||
wire.Bind(new(core.Watcher), new(scanner.Watcher)),
|
||||
)
|
||||
|
||||
@ -103,7 +102,7 @@ func CreatePrometheus() metrics.Metrics {
|
||||
))
|
||||
}
|
||||
|
||||
func CreateScanner(ctx context.Context) scanner.Scanner {
|
||||
func CreateScanner(ctx context.Context) model.Scanner {
|
||||
panic(wire.Build(
|
||||
allProviders,
|
||||
))
|
||||
|
||||
@ -125,11 +125,13 @@ type configOptions struct {
|
||||
DevAlbumInfoTimeToLive time.Duration
|
||||
DevExternalScanner bool
|
||||
DevScannerThreads uint
|
||||
DevSelectiveWatcher bool
|
||||
DevInsightsInitialDelay time.Duration
|
||||
DevEnablePlayerInsights bool
|
||||
DevEnablePluginsInsights bool
|
||||
DevPluginCompilationTimeout time.Duration
|
||||
DevExternalArtistFetchMultiplier float64
|
||||
DevOptimizeDB bool
|
||||
}
|
||||
|
||||
type scannerOptions struct {
|
||||
@ -175,7 +177,8 @@ type spotifyOptions struct {
|
||||
}
|
||||
|
||||
type deezerOptions struct {
|
||||
Enabled bool
|
||||
Enabled bool
|
||||
Language string
|
||||
}
|
||||
|
||||
type listenBrainzOptions struct {
|
||||
@ -425,7 +428,7 @@ func validatePurgeMissingOption() error {
|
||||
}
|
||||
}
|
||||
if !valid {
|
||||
err := fmt.Errorf("Invalid Scanner.PurgeMissing value: '%s'. Must be one of: %v", Server.Scanner.PurgeMissing, allowedValues)
|
||||
err := fmt.Errorf("invalid Scanner.PurgeMissing value: '%s'. Must be one of: %v", Server.Scanner.PurgeMissing, allowedValues)
|
||||
log.Error(err.Error())
|
||||
Server.Scanner.PurgeMissing = consts.PurgeMissingNever
|
||||
return err
|
||||
@ -565,6 +568,7 @@ func setViperDefaults() {
|
||||
viper.SetDefault("spotify.id", "")
|
||||
viper.SetDefault("spotify.secret", "")
|
||||
viper.SetDefault("deezer.enabled", true)
|
||||
viper.SetDefault("deezer.language", "en")
|
||||
viper.SetDefault("listenbrainz.enabled", true)
|
||||
viper.SetDefault("listenbrainz.baseurl", "https://api.listenbrainz.org/1/")
|
||||
viper.SetDefault("httpsecurityheaders.customframeoptionsvalue", "DENY")
|
||||
@ -600,20 +604,27 @@ func setViperDefaults() {
|
||||
viper.SetDefault("devalbuminfotimetolive", consts.AlbumInfoTimeToLive)
|
||||
viper.SetDefault("devexternalscanner", true)
|
||||
viper.SetDefault("devscannerthreads", 5)
|
||||
viper.SetDefault("devselectivewatcher", true)
|
||||
viper.SetDefault("devinsightsinitialdelay", consts.InsightsInitialDelay)
|
||||
viper.SetDefault("devenableplayerinsights", true)
|
||||
viper.SetDefault("devenablepluginsinsights", true)
|
||||
viper.SetDefault("devplugincompilationtimeout", time.Minute)
|
||||
viper.SetDefault("devexternalartistfetchmultiplier", 1.5)
|
||||
viper.SetDefault("devoptimizedb", true)
|
||||
}
|
||||
|
||||
func init() {
|
||||
setViperDefaults()
|
||||
}
|
||||
|
||||
func InitConfig(cfgFile string) {
|
||||
func InitConfig(cfgFile string, loadEnvVars bool) {
|
||||
codecRegistry := viper.NewCodecRegistry()
|
||||
_ = codecRegistry.RegisterCodec("ini", ini.Codec{})
|
||||
_ = codecRegistry.RegisterCodec("ini", ini.Codec{
|
||||
LoadOptions: ini.LoadOptions{
|
||||
UnescapeValueDoubleQuotes: true,
|
||||
UnescapeValueCommentSymbols: true,
|
||||
},
|
||||
})
|
||||
viper.SetOptions(viper.WithCodecRegistry(codecRegistry))
|
||||
|
||||
cfgFile = getConfigFile(cfgFile)
|
||||
@ -627,10 +638,12 @@ func InitConfig(cfgFile string) {
|
||||
}
|
||||
|
||||
_ = viper.BindEnv("port")
|
||||
viper.SetEnvPrefix("ND")
|
||||
replacer := strings.NewReplacer(".", "_")
|
||||
viper.SetEnvKeyReplacer(replacer)
|
||||
viper.AutomaticEnv()
|
||||
if loadEnvVars {
|
||||
viper.SetEnvPrefix("ND")
|
||||
replacer := strings.NewReplacer(".", "_")
|
||||
viper.SetEnvKeyReplacer(replacer)
|
||||
viper.AutomaticEnv()
|
||||
}
|
||||
|
||||
err := viper.ReadInConfig()
|
||||
if viper.ConfigFileUsed() != "" && err != nil {
|
||||
|
||||
@ -31,7 +31,7 @@ var _ = Describe("Configuration", func() {
|
||||
filename := filepath.Join("testdata", "cfg."+format)
|
||||
|
||||
// Initialize config with the test file
|
||||
conf.InitConfig(filename)
|
||||
conf.InitConfig(filename, false)
|
||||
// Load the configuration (with noConfigDump=true)
|
||||
conf.Load(true)
|
||||
|
||||
@ -39,6 +39,7 @@ var _ = Describe("Configuration", func() {
|
||||
Expect(conf.Server.MusicFolder).To(Equal(fmt.Sprintf("/%s/music", format)))
|
||||
Expect(conf.Server.UIWelcomeMessage).To(Equal("Welcome " + format))
|
||||
Expect(conf.Server.Tags["custom"].Aliases).To(Equal([]string{format, "test"}))
|
||||
Expect(conf.Server.Tags["artist"].Split).To(Equal([]string{";"}))
|
||||
|
||||
// The config file used should be the one we created
|
||||
Expect(conf.Server.ConfigFile).To(Equal(filename))
|
||||
|
||||
5
conf/testdata/cfg.ini
vendored
5
conf/testdata/cfg.ini
vendored
@ -1,6 +1,7 @@
|
||||
[default]
|
||||
MusicFolder = /ini/music
|
||||
UIWelcomeMessage = Welcome ini
|
||||
UIWelcomeMessage = 'Welcome ini' ; Just a comment to test the LoadOptions
|
||||
|
||||
[Tags]
|
||||
Custom.Aliases = ini,test
|
||||
Custom.Aliases = ini,test
|
||||
artist.Split = ";" # Should be able to read ; as a separator
|
||||
3
conf/testdata/cfg.json
vendored
3
conf/testdata/cfg.json
vendored
@ -2,6 +2,9 @@
|
||||
"musicFolder": "/json/music",
|
||||
"uiWelcomeMessage": "Welcome json",
|
||||
"Tags": {
|
||||
"artist": {
|
||||
"split": ";"
|
||||
},
|
||||
"custom": {
|
||||
"aliases": [
|
||||
"json",
|
||||
|
||||
2
conf/testdata/cfg.toml
vendored
2
conf/testdata/cfg.toml
vendored
@ -1,5 +1,7 @@
|
||||
musicFolder = "/toml/music"
|
||||
uiWelcomeMessage = "Welcome toml"
|
||||
|
||||
Tags.artist.Split = ';'
|
||||
|
||||
[Tags.custom]
|
||||
aliases = ["toml", "test"]
|
||||
|
||||
2
conf/testdata/cfg.yaml
vendored
2
conf/testdata/cfg.yaml
vendored
@ -1,6 +1,8 @@
|
||||
musicFolder: "/yaml/music"
|
||||
uiWelcomeMessage: "Welcome yaml"
|
||||
Tags:
|
||||
artist:
|
||||
split: [";"]
|
||||
custom:
|
||||
aliases:
|
||||
- yaml
|
||||
|
||||
@ -87,7 +87,7 @@ func (a *Agents) getEnabledAgentNames() []enabledAgent {
|
||||
} else if isPlugin {
|
||||
validAgents = append(validAgents, enabledAgent{name: name, isPlugin: true})
|
||||
} else {
|
||||
log.Warn("Unknown agent ignored", "name", name)
|
||||
log.Debug("Unknown agent ignored", "name", name)
|
||||
}
|
||||
}
|
||||
return validAgents
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
package deezer
|
||||
|
||||
import (
|
||||
bytes "bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
@ -9,11 +10,14 @@ import (
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/microcosm-cc/bluemonday"
|
||||
"github.com/navidrome/navidrome/log"
|
||||
)
|
||||
|
||||
const apiBaseURL = "https://api.deezer.com"
|
||||
const authBaseURL = "https://auth.deezer.com"
|
||||
|
||||
var (
|
||||
ErrNotFound = errors.New("deezer: not found")
|
||||
@ -25,10 +29,15 @@ type httpDoer interface {
|
||||
|
||||
type client struct {
|
||||
httpDoer httpDoer
|
||||
language string
|
||||
jwt jwtToken
|
||||
}
|
||||
|
||||
func newClient(hc httpDoer) *client {
|
||||
return &client{hc}
|
||||
func newClient(hc httpDoer, language string) *client {
|
||||
return &client{
|
||||
httpDoer: hc,
|
||||
language: language,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *client) searchArtists(ctx context.Context, name string, limit int) ([]Artist, error) {
|
||||
@ -53,7 +62,7 @@ func (c *client) searchArtists(ctx context.Context, name string, limit int) ([]A
|
||||
return results.Data, nil
|
||||
}
|
||||
|
||||
func (c *client) makeRequest(req *http.Request, response interface{}) error {
|
||||
func (c *client) makeRequest(req *http.Request, response any) error {
|
||||
log.Trace(req.Context(), fmt.Sprintf("Sending Deezer %s request", req.Method), "url", req.URL)
|
||||
resp, err := c.httpDoer.Do(req)
|
||||
if err != nil {
|
||||
@ -81,3 +90,129 @@ func (c *client) parseError(data []byte) error {
|
||||
}
|
||||
return fmt.Errorf("deezer error(%d): %s", deezerError.Error.Code, deezerError.Error.Message)
|
||||
}
|
||||
|
||||
func (c *client) getRelatedArtists(ctx context.Context, artistID int) ([]Artist, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("%s/artist/%d/related", apiBaseURL, artistID), nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var results RelatedArtists
|
||||
err = c.makeRequest(req, &results)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return results.Data, nil
|
||||
}
|
||||
|
||||
func (c *client) getTopTracks(ctx context.Context, artistID int, limit int) ([]Track, error) {
|
||||
params := url.Values{}
|
||||
params.Add("limit", strconv.Itoa(limit))
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("%s/artist/%d/top", apiBaseURL, artistID), nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.URL.RawQuery = params.Encode()
|
||||
|
||||
var results TopTracks
|
||||
err = c.makeRequest(req, &results)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return results.Data, nil
|
||||
}
|
||||
|
||||
const pipeAPIURL = "https://pipe.deezer.com/api"
|
||||
|
||||
var strictPolicy = bluemonday.StrictPolicy()
|
||||
|
||||
func (c *client) getArtistBio(ctx context.Context, artistID int) (string, error) {
|
||||
jwt, err := c.getJWT(ctx)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("deezer: failed to get JWT: %w", err)
|
||||
}
|
||||
|
||||
query := map[string]any{
|
||||
"operationName": "ArtistBio",
|
||||
"variables": map[string]any{
|
||||
"artistId": strconv.Itoa(artistID),
|
||||
},
|
||||
"query": `query ArtistBio($artistId: String!) {
|
||||
artist(artistId: $artistId) {
|
||||
bio {
|
||||
full
|
||||
}
|
||||
}
|
||||
}`,
|
||||
}
|
||||
|
||||
body, err := json.Marshal(query)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", pipeAPIURL, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Accept-Language", c.language)
|
||||
req.Header.Set("Authorization", "Bearer "+jwt)
|
||||
|
||||
log.Trace(ctx, "Fetching Deezer artist biography via GraphQL", "artistId", artistID, "language", c.language)
|
||||
resp, err := c.httpDoer.Do(req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != 200 {
|
||||
return "", fmt.Errorf("deezer: failed to fetch biography: %s", resp.Status)
|
||||
}
|
||||
|
||||
data, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
type graphQLResponse struct {
|
||||
Data struct {
|
||||
Artist struct {
|
||||
Bio struct {
|
||||
Full string `json:"full"`
|
||||
} `json:"bio"`
|
||||
} `json:"artist"`
|
||||
} `json:"data"`
|
||||
Errors []struct {
|
||||
Message string `json:"message"`
|
||||
}
|
||||
}
|
||||
|
||||
var result graphQLResponse
|
||||
if err := json.Unmarshal(data, &result); err != nil {
|
||||
return "", fmt.Errorf("deezer: failed to parse GraphQL response: %w", err)
|
||||
}
|
||||
|
||||
if len(result.Errors) > 0 {
|
||||
var errs []error
|
||||
for m := range result.Errors {
|
||||
errs = append(errs, errors.New(result.Errors[m].Message))
|
||||
}
|
||||
err := errors.Join(errs...)
|
||||
return "", fmt.Errorf("deezer: GraphQL error: %w", err)
|
||||
}
|
||||
|
||||
if result.Data.Artist.Bio.Full == "" {
|
||||
return "", errors.New("deezer: biography not found")
|
||||
}
|
||||
|
||||
return cleanBio(result.Data.Artist.Bio.Full), nil
|
||||
}
|
||||
|
||||
func cleanBio(bio string) string {
|
||||
bio = strings.ReplaceAll(bio, "</p>", "\n")
|
||||
return strictPolicy.Sanitize(bio)
|
||||
}
|
||||
|
||||
101
core/agents/deezer/client_auth.go
Normal file
101
core/agents/deezer/client_auth.go
Normal file
@ -0,0 +1,101 @@
|
||||
package deezer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lestrrat-go/jwx/v2/jwt"
|
||||
"github.com/navidrome/navidrome/log"
|
||||
)
|
||||
|
||||
type jwtToken struct {
|
||||
token string
|
||||
expiresAt time.Time
|
||||
mu sync.RWMutex
|
||||
}
|
||||
|
||||
func (j *jwtToken) get() (string, bool) {
|
||||
j.mu.RLock()
|
||||
defer j.mu.RUnlock()
|
||||
if time.Now().Before(j.expiresAt) {
|
||||
return j.token, true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
func (j *jwtToken) set(token string, expiresIn time.Duration) {
|
||||
j.mu.Lock()
|
||||
defer j.mu.Unlock()
|
||||
j.token = token
|
||||
j.expiresAt = time.Now().Add(expiresIn)
|
||||
}
|
||||
|
||||
func (c *client) getJWT(ctx context.Context) (string, error) {
|
||||
// Check if we have a valid cached token
|
||||
if token, valid := c.jwt.get(); valid {
|
||||
return token, nil
|
||||
}
|
||||
|
||||
// Fetch a new anonymous token
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", authBaseURL+"/login/anonymous?jo=p&rto=c", nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
req.Header.Set("Accept", "application/json")
|
||||
|
||||
resp, err := c.httpDoer.Do(req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != 200 {
|
||||
return "", fmt.Errorf("deezer: failed to get JWT token: %s", resp.Status)
|
||||
}
|
||||
|
||||
data, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
type authResponse struct {
|
||||
JWT string `json:"jwt"`
|
||||
}
|
||||
|
||||
var result authResponse
|
||||
if err := json.Unmarshal(data, &result); err != nil {
|
||||
return "", fmt.Errorf("deezer: failed to parse auth response: %w", err)
|
||||
}
|
||||
|
||||
if result.JWT == "" {
|
||||
return "", errors.New("deezer: no JWT token in response")
|
||||
}
|
||||
|
||||
// Parse JWT to get actual expiration time
|
||||
token, err := jwt.ParseString(result.JWT, jwt.WithVerify(false), jwt.WithValidate(false))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("deezer: failed to parse JWT token: %w", err)
|
||||
}
|
||||
|
||||
// Calculate TTL with a 1-minute buffer for clock skew and network delays
|
||||
expiresAt := token.Expiration()
|
||||
if expiresAt.IsZero() {
|
||||
return "", errors.New("deezer: JWT token has no expiration time")
|
||||
}
|
||||
|
||||
ttl := time.Until(expiresAt) - 1*time.Minute
|
||||
if ttl <= 0 {
|
||||
return "", errors.New("deezer: JWT token already expired or expires too soon")
|
||||
}
|
||||
|
||||
c.jwt.set(result.JWT, ttl)
|
||||
log.Trace(ctx, "Fetched new Deezer JWT token", "expiresAt", expiresAt, "ttl", ttl)
|
||||
|
||||
return result.JWT, nil
|
||||
}
|
||||
293
core/agents/deezer/client_auth_test.go
Normal file
293
core/agents/deezer/client_auth_test.go
Normal file
@ -0,0 +1,293 @@
|
||||
package deezer
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lestrrat-go/jwx/v2/jwt"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
var _ = Describe("JWT Authentication", func() {
|
||||
var httpClient *fakeHttpClient
|
||||
var client *client
|
||||
var ctx context.Context
|
||||
|
||||
BeforeEach(func() {
|
||||
httpClient = &fakeHttpClient{}
|
||||
client = newClient(httpClient, "en")
|
||||
ctx = context.Background()
|
||||
})
|
||||
|
||||
Describe("getJWT", func() {
|
||||
Context("with a valid JWT response", func() {
|
||||
It("successfully fetches and caches a JWT token", func() {
|
||||
testJWT := createTestJWT(5 * time.Minute)
|
||||
httpClient.mock("https://auth.deezer.com/login/anonymous", http.Response{
|
||||
StatusCode: 200,
|
||||
Body: io.NopCloser(bytes.NewBufferString(fmt.Sprintf(`{"jwt":"%s"}`, testJWT))),
|
||||
})
|
||||
|
||||
token, err := client.getJWT(ctx)
|
||||
Expect(err).To(BeNil())
|
||||
Expect(token).To(Equal(testJWT))
|
||||
})
|
||||
|
||||
It("returns the cached token on subsequent calls", func() {
|
||||
testJWT := createTestJWT(5 * time.Minute)
|
||||
httpClient.mock("https://auth.deezer.com/login/anonymous", http.Response{
|
||||
StatusCode: 200,
|
||||
Body: io.NopCloser(bytes.NewBufferString(fmt.Sprintf(`{"jwt":"%s"}`, testJWT))),
|
||||
})
|
||||
|
||||
// First call should fetch from API
|
||||
token1, err := client.getJWT(ctx)
|
||||
Expect(err).To(BeNil())
|
||||
Expect(token1).To(Equal(testJWT))
|
||||
Expect(httpClient.lastRequest.URL.Path).To(Equal("/login/anonymous"))
|
||||
|
||||
// Second call should return cached token without hitting API
|
||||
httpClient.lastRequest = nil // Clear last request to verify no new request is made
|
||||
token2, err := client.getJWT(ctx)
|
||||
Expect(err).To(BeNil())
|
||||
Expect(token2).To(Equal(testJWT))
|
||||
Expect(httpClient.lastRequest).To(BeNil()) // No new request made
|
||||
})
|
||||
|
||||
It("parses the JWT expiration time correctly", func() {
|
||||
expectedExpiration := time.Now().Add(5 * time.Minute)
|
||||
testToken, err := jwt.NewBuilder().
|
||||
Expiration(expectedExpiration).
|
||||
Build()
|
||||
Expect(err).To(BeNil())
|
||||
testJWT, err := jwt.Sign(testToken, jwt.WithInsecureNoSignature())
|
||||
Expect(err).To(BeNil())
|
||||
|
||||
httpClient.mock("https://auth.deezer.com/login/anonymous", http.Response{
|
||||
StatusCode: 200,
|
||||
Body: io.NopCloser(bytes.NewBufferString(fmt.Sprintf(`{"jwt":"%s"}`, string(testJWT)))),
|
||||
})
|
||||
|
||||
token, err := client.getJWT(ctx)
|
||||
Expect(err).To(BeNil())
|
||||
Expect(token).ToNot(BeEmpty())
|
||||
|
||||
// Verify the token is cached until close to expiration
|
||||
// The cache should expire 1 minute before the JWT expires
|
||||
expectedCacheExpiry := expectedExpiration.Add(-1 * time.Minute)
|
||||
Expect(client.jwt.expiresAt).To(BeTemporally("~", expectedCacheExpiry, 2*time.Second))
|
||||
})
|
||||
})
|
||||
|
||||
Context("with JWT tokens that expire soon", func() {
|
||||
It("rejects tokens that expire in less than 1 minute", func() {
|
||||
// Create a token that expires in 30 seconds (less than 1-minute buffer)
|
||||
testJWT := createTestJWT(30 * time.Second)
|
||||
httpClient.mock("https://auth.deezer.com/login/anonymous", http.Response{
|
||||
StatusCode: 200,
|
||||
Body: io.NopCloser(bytes.NewBufferString(fmt.Sprintf(`{"jwt":"%s"}`, testJWT))),
|
||||
})
|
||||
|
||||
_, err := client.getJWT(ctx)
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring("JWT token already expired or expires too soon"))
|
||||
})
|
||||
|
||||
It("rejects already expired tokens", func() {
|
||||
// Create a token that expired 1 minute ago
|
||||
testJWT := createTestJWT(-1 * time.Minute)
|
||||
httpClient.mock("https://auth.deezer.com/login/anonymous", http.Response{
|
||||
StatusCode: 200,
|
||||
Body: io.NopCloser(bytes.NewBufferString(fmt.Sprintf(`{"jwt":"%s"}`, testJWT))),
|
||||
})
|
||||
|
||||
_, err := client.getJWT(ctx)
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring("JWT token already expired or expires too soon"))
|
||||
})
|
||||
|
||||
It("accepts tokens that expire in more than 1 minute", func() {
|
||||
// Create a token that expires in 2 minutes (just over the 1-minute buffer)
|
||||
testJWT := createTestJWT(2 * time.Minute)
|
||||
httpClient.mock("https://auth.deezer.com/login/anonymous", http.Response{
|
||||
StatusCode: 200,
|
||||
Body: io.NopCloser(bytes.NewBufferString(fmt.Sprintf(`{"jwt":"%s"}`, testJWT))),
|
||||
})
|
||||
|
||||
token, err := client.getJWT(ctx)
|
||||
Expect(err).To(BeNil())
|
||||
Expect(token).ToNot(BeEmpty())
|
||||
})
|
||||
})
|
||||
|
||||
Context("with invalid responses", func() {
|
||||
It("handles HTTP error responses", func() {
|
||||
httpClient.mock("https://auth.deezer.com/login/anonymous", http.Response{
|
||||
StatusCode: 500,
|
||||
Body: io.NopCloser(bytes.NewBufferString(`{"error":"Internal server error"}`)),
|
||||
})
|
||||
|
||||
_, err := client.getJWT(ctx)
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring("failed to get JWT token"))
|
||||
})
|
||||
|
||||
It("handles malformed JSON responses", func() {
|
||||
httpClient.mock("https://auth.deezer.com/login/anonymous", http.Response{
|
||||
StatusCode: 200,
|
||||
Body: io.NopCloser(bytes.NewBufferString(`{invalid json}`)),
|
||||
})
|
||||
|
||||
_, err := client.getJWT(ctx)
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring("failed to parse auth response"))
|
||||
})
|
||||
|
||||
It("handles responses with empty JWT field", func() {
|
||||
httpClient.mock("https://auth.deezer.com/login/anonymous", http.Response{
|
||||
StatusCode: 200,
|
||||
Body: io.NopCloser(bytes.NewBufferString(`{"jwt":""}`)),
|
||||
})
|
||||
|
||||
_, err := client.getJWT(ctx)
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(Equal("deezer: no JWT token in response"))
|
||||
})
|
||||
|
||||
It("handles invalid JWT tokens", func() {
|
||||
httpClient.mock("https://auth.deezer.com/login/anonymous", http.Response{
|
||||
StatusCode: 200,
|
||||
Body: io.NopCloser(bytes.NewBufferString(`{"jwt":"not-a-valid-jwt"}`)),
|
||||
})
|
||||
|
||||
_, err := client.getJWT(ctx)
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring("failed to parse JWT token"))
|
||||
})
|
||||
|
||||
It("rejects JWT tokens without expiration", func() {
|
||||
// Create a JWT without expiration claim
|
||||
testToken, err := jwt.NewBuilder().
|
||||
Claim("custom", "value").
|
||||
Build()
|
||||
Expect(err).To(BeNil())
|
||||
|
||||
// Verify token has no expiration
|
||||
Expect(testToken.Expiration().IsZero()).To(BeTrue())
|
||||
|
||||
testJWT, err := jwt.Sign(testToken, jwt.WithInsecureNoSignature())
|
||||
Expect(err).To(BeNil())
|
||||
|
||||
httpClient.mock("https://auth.deezer.com/login/anonymous", http.Response{
|
||||
StatusCode: 200,
|
||||
Body: io.NopCloser(bytes.NewBufferString(fmt.Sprintf(`{"jwt":"%s"}`, string(testJWT)))),
|
||||
})
|
||||
|
||||
_, err = client.getJWT(ctx)
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(Equal("deezer: JWT token has no expiration time"))
|
||||
})
|
||||
})
|
||||
|
||||
Context("token caching behavior", func() {
|
||||
It("fetches a new token when the cached token expires", func() {
|
||||
// First token expires in 5 minutes
|
||||
firstJWT := createTestJWT(5 * time.Minute)
|
||||
httpClient.mock("https://auth.deezer.com/login/anonymous", http.Response{
|
||||
StatusCode: 200,
|
||||
Body: io.NopCloser(bytes.NewBufferString(fmt.Sprintf(`{"jwt":"%s"}`, firstJWT))),
|
||||
})
|
||||
|
||||
token1, err := client.getJWT(ctx)
|
||||
Expect(err).To(BeNil())
|
||||
Expect(token1).To(Equal(firstJWT))
|
||||
|
||||
// Manually expire the cached token
|
||||
client.jwt.expiresAt = time.Now().Add(-1 * time.Second)
|
||||
|
||||
// Second token with different expiration (10 minutes)
|
||||
secondJWT := createTestJWT(10 * time.Minute)
|
||||
httpClient.mock("https://auth.deezer.com/login/anonymous", http.Response{
|
||||
StatusCode: 200,
|
||||
Body: io.NopCloser(bytes.NewBufferString(fmt.Sprintf(`{"jwt":"%s"}`, secondJWT))),
|
||||
})
|
||||
|
||||
token2, err := client.getJWT(ctx)
|
||||
Expect(err).To(BeNil())
|
||||
Expect(token2).To(Equal(secondJWT))
|
||||
Expect(token2).ToNot(Equal(token1))
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
Describe("jwtToken cache", func() {
|
||||
var cache *jwtToken
|
||||
|
||||
BeforeEach(func() {
|
||||
cache = &jwtToken{}
|
||||
})
|
||||
|
||||
It("returns false for expired tokens", func() {
|
||||
cache.set("test-token", -1*time.Second) // Already expired
|
||||
token, valid := cache.get()
|
||||
Expect(valid).To(BeFalse())
|
||||
Expect(token).To(BeEmpty())
|
||||
})
|
||||
|
||||
It("returns true for valid tokens", func() {
|
||||
cache.set("test-token", 4*time.Minute)
|
||||
token, valid := cache.get()
|
||||
Expect(valid).To(BeTrue())
|
||||
Expect(token).To(Equal("test-token"))
|
||||
})
|
||||
|
||||
It("is thread-safe for concurrent access", func() {
|
||||
wg := sync.WaitGroup{}
|
||||
|
||||
// Writer goroutine
|
||||
wg.Go(func() {
|
||||
for i := 0; i < 100; i++ {
|
||||
cache.set(fmt.Sprintf("token-%d", i), 1*time.Hour)
|
||||
time.Sleep(1 * time.Millisecond)
|
||||
}
|
||||
})
|
||||
|
||||
// Reader goroutine
|
||||
wg.Go(func() {
|
||||
for i := 0; i < 100; i++ {
|
||||
cache.get()
|
||||
time.Sleep(1 * time.Millisecond)
|
||||
}
|
||||
})
|
||||
|
||||
// Wait for both goroutines to complete
|
||||
wg.Wait()
|
||||
|
||||
// Verify final state is valid
|
||||
token, valid := cache.get()
|
||||
Expect(valid).To(BeTrue())
|
||||
Expect(token).To(HavePrefix("token-"))
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
// createTestJWT creates a valid JWT token for testing purposes
|
||||
func createTestJWT(expiresIn time.Duration) string {
|
||||
token, err := jwt.NewBuilder().
|
||||
Expiration(time.Now().Add(expiresIn)).
|
||||
Build()
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("failed to create test JWT: %v", err))
|
||||
}
|
||||
signed, err := jwt.Sign(token, jwt.WithInsecureNoSignature())
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("failed to sign test JWT: %v", err))
|
||||
}
|
||||
return string(signed)
|
||||
}
|
||||
@ -2,10 +2,11 @@ package deezer
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
@ -17,7 +18,7 @@ var _ = Describe("client", func() {
|
||||
|
||||
BeforeEach(func() {
|
||||
httpClient = &fakeHttpClient{}
|
||||
client = newClient(httpClient)
|
||||
client = newClient(httpClient, "en")
|
||||
})
|
||||
|
||||
Describe("ArtistImages", func() {
|
||||
@ -26,7 +27,7 @@ var _ = Describe("client", func() {
|
||||
Expect(err).To(BeNil())
|
||||
httpClient.mock("https://api.deezer.com/search/artist", http.Response{Body: f, StatusCode: 200})
|
||||
|
||||
artists, err := client.searchArtists(context.TODO(), "Michael Jackson", 20)
|
||||
artists, err := client.searchArtists(GinkgoT().Context(), "Michael Jackson", 20)
|
||||
Expect(err).To(BeNil())
|
||||
Expect(artists).To(HaveLen(17))
|
||||
Expect(artists[0].Name).To(Equal("Michael Jackson"))
|
||||
@ -39,10 +40,136 @@ var _ = Describe("client", func() {
|
||||
Body: io.NopCloser(bytes.NewBufferString(`{"data":[],"total":0}`)),
|
||||
})
|
||||
|
||||
_, err := client.searchArtists(context.TODO(), "Michael Jackson", 20)
|
||||
_, err := client.searchArtists(GinkgoT().Context(), "Michael Jackson", 20)
|
||||
Expect(err).To(MatchError(ErrNotFound))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("ArtistBio", func() {
|
||||
BeforeEach(func() {
|
||||
// Mock the JWT token endpoint with a valid JWT that expires in 5 minutes
|
||||
testJWT := createTestJWT(5 * time.Minute)
|
||||
httpClient.mock("https://auth.deezer.com/login/anonymous", http.Response{
|
||||
StatusCode: 200,
|
||||
Body: io.NopCloser(bytes.NewBufferString(fmt.Sprintf(`{"jwt":"%s","refresh_token":""}`, testJWT))),
|
||||
})
|
||||
})
|
||||
|
||||
It("returns artist bio from a successful request", func() {
|
||||
f, err := os.Open("tests/fixtures/deezer.artist.bio.json")
|
||||
Expect(err).To(BeNil())
|
||||
httpClient.mock("https://pipe.deezer.com/api", http.Response{Body: f, StatusCode: 200})
|
||||
|
||||
bio, err := client.getArtistBio(GinkgoT().Context(), 27)
|
||||
Expect(err).To(BeNil())
|
||||
Expect(bio).To(ContainSubstring("Schoolmates Thomas and Guy-Manuel"))
|
||||
Expect(bio).ToNot(ContainSubstring("<p>"))
|
||||
Expect(bio).ToNot(ContainSubstring("</p>"))
|
||||
})
|
||||
|
||||
It("uses the configured language", func() {
|
||||
client = newClient(httpClient, "fr")
|
||||
// Mock JWT token for the new client instance with a valid JWT
|
||||
testJWT := createTestJWT(5 * time.Minute)
|
||||
httpClient.mock("https://auth.deezer.com/login/anonymous", http.Response{
|
||||
StatusCode: 200,
|
||||
Body: io.NopCloser(bytes.NewBufferString(fmt.Sprintf(`{"jwt":"%s","refresh_token":""}`, testJWT))),
|
||||
})
|
||||
f, err := os.Open("tests/fixtures/deezer.artist.bio.json")
|
||||
Expect(err).To(BeNil())
|
||||
httpClient.mock("https://pipe.deezer.com/api", http.Response{Body: f, StatusCode: 200})
|
||||
|
||||
_, err = client.getArtistBio(GinkgoT().Context(), 27)
|
||||
Expect(err).To(BeNil())
|
||||
Expect(httpClient.lastRequest.Header.Get("Accept-Language")).To(Equal("fr"))
|
||||
})
|
||||
|
||||
It("includes the JWT token in the request", func() {
|
||||
f, err := os.Open("tests/fixtures/deezer.artist.bio.json")
|
||||
Expect(err).To(BeNil())
|
||||
httpClient.mock("https://pipe.deezer.com/api", http.Response{Body: f, StatusCode: 200})
|
||||
|
||||
_, err = client.getArtistBio(GinkgoT().Context(), 27)
|
||||
Expect(err).To(BeNil())
|
||||
// Verify that the Authorization header has the Bearer token format
|
||||
authHeader := httpClient.lastRequest.Header.Get("Authorization")
|
||||
Expect(authHeader).To(HavePrefix("Bearer "))
|
||||
Expect(len(authHeader)).To(BeNumerically(">", 20)) // JWT tokens are longer than 20 chars
|
||||
})
|
||||
|
||||
It("handles GraphQL errors", func() {
|
||||
errorResponse := `{
|
||||
"data": {
|
||||
"artist": {
|
||||
"bio": {
|
||||
"full": ""
|
||||
}
|
||||
}
|
||||
},
|
||||
"errors": [
|
||||
{
|
||||
"message": "Artist not found"
|
||||
},
|
||||
{
|
||||
"message": "Invalid artist ID"
|
||||
}
|
||||
]
|
||||
}`
|
||||
httpClient.mock("https://pipe.deezer.com/api", http.Response{
|
||||
StatusCode: 200,
|
||||
Body: io.NopCloser(bytes.NewBufferString(errorResponse)),
|
||||
})
|
||||
|
||||
_, err := client.getArtistBio(GinkgoT().Context(), 999)
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring("GraphQL error"))
|
||||
Expect(err.Error()).To(ContainSubstring("Artist not found"))
|
||||
Expect(err.Error()).To(ContainSubstring("Invalid artist ID"))
|
||||
})
|
||||
|
||||
It("handles empty biography", func() {
|
||||
emptyBioResponse := `{
|
||||
"data": {
|
||||
"artist": {
|
||||
"bio": {
|
||||
"full": ""
|
||||
}
|
||||
}
|
||||
}
|
||||
}`
|
||||
httpClient.mock("https://pipe.deezer.com/api", http.Response{
|
||||
StatusCode: 200,
|
||||
Body: io.NopCloser(bytes.NewBufferString(emptyBioResponse)),
|
||||
})
|
||||
|
||||
_, err := client.getArtistBio(GinkgoT().Context(), 27)
|
||||
Expect(err).To(MatchError("deezer: biography not found"))
|
||||
})
|
||||
|
||||
It("handles JWT token fetch failure", func() {
|
||||
httpClient.mock("https://auth.deezer.com/login/anonymous", http.Response{
|
||||
StatusCode: 500,
|
||||
Body: io.NopCloser(bytes.NewBufferString(`{"error":"Internal server error"}`)),
|
||||
})
|
||||
|
||||
_, err := client.getArtistBio(GinkgoT().Context(), 27)
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring("failed to get JWT"))
|
||||
})
|
||||
|
||||
It("handles JWT token that expires too soon", func() {
|
||||
// Create a JWT that expires in 30 seconds (less than the 1-minute buffer)
|
||||
expiredJWT := createTestJWT(30 * time.Second)
|
||||
httpClient.mock("https://auth.deezer.com/login/anonymous", http.Response{
|
||||
StatusCode: 200,
|
||||
Body: io.NopCloser(bytes.NewBufferString(fmt.Sprintf(`{"jwt":"%s","refresh_token":""}`, expiredJWT))),
|
||||
})
|
||||
|
||||
_, err := client.getArtistBio(GinkgoT().Context(), 27)
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring("JWT token already expired or expires too soon"))
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
type fakeHttpClient struct {
|
||||
|
||||
@ -12,6 +12,7 @@ import (
|
||||
"github.com/navidrome/navidrome/log"
|
||||
"github.com/navidrome/navidrome/model"
|
||||
"github.com/navidrome/navidrome/utils/cache"
|
||||
"github.com/navidrome/navidrome/utils/slice"
|
||||
)
|
||||
|
||||
const deezerAgentName = "deezer"
|
||||
@ -32,7 +33,7 @@ func deezerConstructor(dataStore model.DataStore) agents.Interface {
|
||||
Timeout: consts.DefaultHttpClientTimeOut,
|
||||
}
|
||||
cachedHttpClient := cache.NewHTTPClient(httpClient, consts.DefaultHttpClientTimeOut)
|
||||
agent.client = newClient(cachedHttpClient)
|
||||
agent.client = newClient(cachedHttpClient, conf.Server.Deezer.Language)
|
||||
return agent
|
||||
}
|
||||
|
||||
@ -88,6 +89,56 @@ func (s *deezerAgent) searchArtist(ctx context.Context, name string) (*Artist, e
|
||||
return &artists[0], err
|
||||
}
|
||||
|
||||
func (s *deezerAgent) GetSimilarArtists(ctx context.Context, _, name, _ string, limit int) ([]agents.Artist, error) {
|
||||
artist, err := s.searchArtist(ctx, name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
related, err := s.client.getRelatedArtists(ctx, artist.ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
res := slice.Map(related, func(r Artist) agents.Artist {
|
||||
return agents.Artist{
|
||||
Name: r.Name,
|
||||
}
|
||||
})
|
||||
if len(res) > limit {
|
||||
res = res[:limit]
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (s *deezerAgent) GetArtistTopSongs(ctx context.Context, _, artistName, _ string, count int) ([]agents.Song, error) {
|
||||
artist, err := s.searchArtist(ctx, artistName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
tracks, err := s.client.getTopTracks(ctx, artist.ID, count)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
res := slice.Map(tracks, func(r Track) agents.Song {
|
||||
return agents.Song{
|
||||
Name: r.Title,
|
||||
}
|
||||
})
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (s *deezerAgent) GetArtistBiography(ctx context.Context, _, name, _ string) (string, error) {
|
||||
artist, err := s.searchArtist(ctx, name)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return s.client.getArtistBio(ctx, artist.ID)
|
||||
}
|
||||
|
||||
func init() {
|
||||
conf.AddHook(func() {
|
||||
if conf.Server.Deezer.Enabled {
|
||||
|
||||
@ -29,3 +29,38 @@ type Error struct {
|
||||
Code int `json:"code"`
|
||||
} `json:"error"`
|
||||
}
|
||||
|
||||
type RelatedArtists struct {
|
||||
Data []Artist `json:"data"`
|
||||
Total int `json:"total"`
|
||||
}
|
||||
|
||||
type TopTracks struct {
|
||||
Data []Track `json:"data"`
|
||||
Total int `json:"total"`
|
||||
Next string `json:"next"`
|
||||
}
|
||||
|
||||
type Track struct {
|
||||
ID int `json:"id"`
|
||||
Title string `json:"title"`
|
||||
Link string `json:"link"`
|
||||
Duration int `json:"duration"`
|
||||
Rank int `json:"rank"`
|
||||
Preview string `json:"preview"`
|
||||
Artist Artist `json:"artist"`
|
||||
Album Album `json:"album"`
|
||||
Contributors []Artist `json:"contributors"`
|
||||
}
|
||||
|
||||
type Album struct {
|
||||
ID int `json:"id"`
|
||||
Title string `json:"title"`
|
||||
Cover string `json:"cover"`
|
||||
CoverSmall string `json:"cover_small"`
|
||||
CoverMedium string `json:"cover_medium"`
|
||||
CoverBig string `json:"cover_big"`
|
||||
CoverXl string `json:"cover_xl"`
|
||||
Tracklist string `json:"tracklist"`
|
||||
Type string `json:"type"`
|
||||
}
|
||||
|
||||
@ -35,4 +35,35 @@ var _ = Describe("Responses", func() {
|
||||
Expect(errorResp.Error.Message).To(Equal("Missing parameters: q"))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Related Artists", func() {
|
||||
It("parses the related artists response correctly", func() {
|
||||
var resp RelatedArtists
|
||||
body, err := os.ReadFile("tests/fixtures/deezer.artist.related.json")
|
||||
Expect(err).To(BeNil())
|
||||
err = json.Unmarshal(body, &resp)
|
||||
Expect(err).To(BeNil())
|
||||
|
||||
Expect(resp.Data).To(HaveLen(20))
|
||||
justice := resp.Data[0]
|
||||
Expect(justice.Name).To(Equal("Justice"))
|
||||
Expect(justice.ID).To(Equal(6404))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Top Tracks", func() {
|
||||
It("parses the top tracks response correctly", func() {
|
||||
var resp TopTracks
|
||||
body, err := os.ReadFile("tests/fixtures/deezer.artist.top.json")
|
||||
Expect(err).To(BeNil())
|
||||
err = json.Unmarshal(body, &resp)
|
||||
Expect(err).To(BeNil())
|
||||
|
||||
Expect(resp.Data).To(HaveLen(5))
|
||||
track := resp.Data[0]
|
||||
Expect(track.Title).To(Equal("Instant Crush (feat. Julian Casablancas)"))
|
||||
Expect(track.ID).To(Equal(67238732))
|
||||
Expect(track.Album.Title).To(Equal("Random Access Memories"))
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@ -38,6 +38,7 @@ type lastfmAgent struct {
|
||||
secret string
|
||||
lang string
|
||||
client *client
|
||||
httpClient httpDoer
|
||||
getInfoMutex sync.Mutex
|
||||
}
|
||||
|
||||
@ -56,6 +57,7 @@ func lastFMConstructor(ds model.DataStore) *lastfmAgent {
|
||||
Timeout: consts.DefaultHttpClientTimeOut,
|
||||
}
|
||||
chc := cache.NewHTTPClient(hc, consts.DefaultHttpClientTimeOut)
|
||||
l.httpClient = chc
|
||||
l.client = newClient(l.apiKey, l.secret, l.lang, chc)
|
||||
return l
|
||||
}
|
||||
@ -190,13 +192,13 @@ func (l *lastfmAgent) GetArtistTopSongs(ctx context.Context, id, artistName, mbi
|
||||
return res, nil
|
||||
}
|
||||
|
||||
var artistOpenGraphQuery = cascadia.MustCompile(`html > head > meta[property="og:image"]`)
|
||||
var (
|
||||
artistOpenGraphQuery = cascadia.MustCompile(`html > head > meta[property="og:image"]`)
|
||||
artistIgnoredImage = "2a96cbd8b46e442fc41c2b86b821562f" // Last.fm artist placeholder image name
|
||||
)
|
||||
|
||||
func (l *lastfmAgent) GetArtistImages(ctx context.Context, _, name, mbid string) ([]agents.ExternalImage, error) {
|
||||
log.Debug(ctx, "Getting artist images from Last.fm", "name", name)
|
||||
hc := http.Client{
|
||||
Timeout: consts.DefaultHttpClientTimeOut,
|
||||
}
|
||||
a, err := l.callArtistGetInfo(ctx, name)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get artist info: %w", err)
|
||||
@ -205,7 +207,7 @@ func (l *lastfmAgent) GetArtistImages(ctx context.Context, _, name, mbid string)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create artist image request: %w", err)
|
||||
}
|
||||
resp, err := hc.Do(req)
|
||||
resp, err := l.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get artist url: %w", err)
|
||||
}
|
||||
@ -222,11 +224,16 @@ func (l *lastfmAgent) GetArtistImages(ctx context.Context, _, name, mbid string)
|
||||
return res, nil
|
||||
}
|
||||
for _, attr := range n.Attr {
|
||||
if attr.Key == "content" {
|
||||
res = []agents.ExternalImage{
|
||||
{URL: attr.Val},
|
||||
}
|
||||
break
|
||||
if attr.Key != "content" {
|
||||
continue
|
||||
}
|
||||
if strings.Contains(attr.Val, artistIgnoredImage) {
|
||||
log.Debug(ctx, "Artist image is ignored default image", "name", name, "url", attr.Val)
|
||||
return res, nil
|
||||
}
|
||||
|
||||
res = []agents.ExternalImage{
|
||||
{URL: attr.Val},
|
||||
}
|
||||
}
|
||||
return res, nil
|
||||
|
||||
@ -393,4 +393,73 @@ var _ = Describe("lastfmAgent", func() {
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
Describe("GetArtistImages", func() {
|
||||
var agent *lastfmAgent
|
||||
var apiClient *tests.FakeHttpClient
|
||||
var httpClient *tests.FakeHttpClient
|
||||
|
||||
BeforeEach(func() {
|
||||
apiClient = &tests.FakeHttpClient{}
|
||||
httpClient = &tests.FakeHttpClient{}
|
||||
client := newClient("API_KEY", "SECRET", "pt", apiClient)
|
||||
agent = lastFMConstructor(ds)
|
||||
agent.client = client
|
||||
agent.httpClient = httpClient
|
||||
})
|
||||
|
||||
It("returns the artist image from the page", func() {
|
||||
fApi, _ := os.Open("tests/fixtures/lastfm.artist.getinfo.json")
|
||||
apiClient.Res = http.Response{Body: fApi, StatusCode: 200}
|
||||
|
||||
fScraper, _ := os.Open("tests/fixtures/lastfm.artist.page.html")
|
||||
httpClient.Res = http.Response{Body: fScraper, StatusCode: 200}
|
||||
|
||||
images, err := agent.GetArtistImages(ctx, "123", "U2", "")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(images).To(HaveLen(1))
|
||||
Expect(images[0].URL).To(Equal("https://lastfm.freetls.fastly.net/i/u/ar0/818148bf682d429dc21b59a73ef6f68e.png"))
|
||||
})
|
||||
|
||||
It("returns empty list if image is the ignored default image", func() {
|
||||
fApi, _ := os.Open("tests/fixtures/lastfm.artist.getinfo.json")
|
||||
apiClient.Res = http.Response{Body: fApi, StatusCode: 200}
|
||||
|
||||
fScraper, _ := os.Open("tests/fixtures/lastfm.artist.page.ignored.html")
|
||||
httpClient.Res = http.Response{Body: fScraper, StatusCode: 200}
|
||||
|
||||
images, err := agent.GetArtistImages(ctx, "123", "U2", "")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(images).To(BeEmpty())
|
||||
})
|
||||
|
||||
It("returns empty list if page has no meta tags", func() {
|
||||
fApi, _ := os.Open("tests/fixtures/lastfm.artist.getinfo.json")
|
||||
apiClient.Res = http.Response{Body: fApi, StatusCode: 200}
|
||||
|
||||
fScraper, _ := os.Open("tests/fixtures/lastfm.artist.page.no_meta.html")
|
||||
httpClient.Res = http.Response{Body: fScraper, StatusCode: 200}
|
||||
|
||||
images, err := agent.GetArtistImages(ctx, "123", "U2", "")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(images).To(BeEmpty())
|
||||
})
|
||||
|
||||
It("returns error if API call fails", func() {
|
||||
apiClient.Err = errors.New("api error")
|
||||
_, err := agent.GetArtistImages(ctx, "123", "U2", "")
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring("get artist info"))
|
||||
})
|
||||
|
||||
It("returns error if scraper call fails", func() {
|
||||
fApi, _ := os.Open("tests/fixtures/lastfm.artist.getinfo.json")
|
||||
apiClient.Res = http.Response{Body: fApi, StatusCode: 200}
|
||||
|
||||
httpClient.Err = errors.New("scraper error")
|
||||
_, err := agent.GetArtistImages(ctx, "123", "U2", "")
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring("get artist url"))
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
package artwork
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"fmt"
|
||||
@ -11,6 +12,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/Masterminds/squirrel"
|
||||
"github.com/maruel/natural"
|
||||
"github.com/navidrome/navidrome/conf"
|
||||
"github.com/navidrome/navidrome/core"
|
||||
"github.com/navidrome/navidrome/core/external"
|
||||
@ -116,8 +118,30 @@ func loadAlbumFoldersPaths(ctx context.Context, ds model.DataStore, albums ...mo
|
||||
}
|
||||
|
||||
// Sort image files to ensure consistent selection of cover art
|
||||
// This prioritizes files from lower-numbered disc folders by sorting the paths
|
||||
slices.Sort(imgFiles)
|
||||
// This prioritizes files without numeric suffixes (e.g., cover.jpg over cover.1.jpg)
|
||||
// by comparing base filenames without extensions
|
||||
slices.SortFunc(imgFiles, compareImageFiles)
|
||||
|
||||
return paths, imgFiles, &updatedAt, nil
|
||||
}
|
||||
|
||||
// compareImageFiles compares two image file paths for sorting.
|
||||
// It extracts the base filename (without extension) and compares case-insensitively.
|
||||
// This ensures that "cover.jpg" sorts before "cover.1.jpg" since "cover" < "cover.1".
|
||||
// Note: This function is called O(n log n) times during sorting, but in practice albums
|
||||
// typically have only 1-20 image files, making the repeated string operations negligible.
|
||||
func compareImageFiles(a, b string) int {
|
||||
// Case-insensitive comparison
|
||||
a = strings.ToLower(a)
|
||||
b = strings.ToLower(b)
|
||||
|
||||
// Extract base filenames without extensions
|
||||
baseA := strings.TrimSuffix(filepath.Base(a), filepath.Ext(a))
|
||||
baseB := strings.TrimSuffix(filepath.Base(b), filepath.Ext(b))
|
||||
|
||||
// Compare base names first, then full paths if equal
|
||||
return cmp.Or(
|
||||
natural.Compare(baseA, baseB),
|
||||
natural.Compare(a, b),
|
||||
)
|
||||
}
|
||||
|
||||
@ -27,26 +27,7 @@ var _ = Describe("Album Artwork Reader", func() {
|
||||
expectedAt = now.Add(5 * time.Minute)
|
||||
|
||||
// Set up the test folders with image files
|
||||
repo = &fakeFolderRepo{
|
||||
result: []model.Folder{
|
||||
{
|
||||
Path: "Artist/Album/Disc1",
|
||||
ImagesUpdatedAt: expectedAt,
|
||||
ImageFiles: []string{"cover.jpg", "back.jpg"},
|
||||
},
|
||||
{
|
||||
Path: "Artist/Album/Disc2",
|
||||
ImagesUpdatedAt: now,
|
||||
ImageFiles: []string{"cover.jpg"},
|
||||
},
|
||||
{
|
||||
Path: "Artist/Album/Disc10",
|
||||
ImagesUpdatedAt: now,
|
||||
ImageFiles: []string{"cover.jpg"},
|
||||
},
|
||||
},
|
||||
err: nil,
|
||||
}
|
||||
repo = &fakeFolderRepo{}
|
||||
ds = &fakeDataStore{
|
||||
folderRepo: repo,
|
||||
}
|
||||
@ -58,19 +39,82 @@ var _ = Describe("Album Artwork Reader", func() {
|
||||
})
|
||||
|
||||
It("returns sorted image files", func() {
|
||||
repo.result = []model.Folder{
|
||||
{
|
||||
Path: "Artist/Album/Disc1",
|
||||
ImagesUpdatedAt: expectedAt,
|
||||
ImageFiles: []string{"cover.jpg", "back.jpg", "cover.1.jpg"},
|
||||
},
|
||||
{
|
||||
Path: "Artist/Album/Disc2",
|
||||
ImagesUpdatedAt: now,
|
||||
ImageFiles: []string{"cover.jpg"},
|
||||
},
|
||||
{
|
||||
Path: "Artist/Album/Disc10",
|
||||
ImagesUpdatedAt: now,
|
||||
ImageFiles: []string{"cover.jpg"},
|
||||
},
|
||||
}
|
||||
|
||||
_, imgFiles, imagesUpdatedAt, err := loadAlbumFoldersPaths(ctx, ds, album)
|
||||
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(*imagesUpdatedAt).To(Equal(expectedAt))
|
||||
|
||||
// Check that image files are sorted alphabetically
|
||||
Expect(imgFiles).To(HaveLen(4))
|
||||
// Check that image files are sorted by base name (without extension)
|
||||
Expect(imgFiles).To(HaveLen(5))
|
||||
|
||||
// The files should be sorted by full path
|
||||
// Files should be sorted by base filename without extension, then by full path
|
||||
// "back" < "cover", so back.jpg comes first
|
||||
// Then all cover.jpg files, sorted by path
|
||||
Expect(imgFiles[0]).To(Equal(filepath.FromSlash("Artist/Album/Disc1/back.jpg")))
|
||||
Expect(imgFiles[1]).To(Equal(filepath.FromSlash("Artist/Album/Disc1/cover.jpg")))
|
||||
Expect(imgFiles[2]).To(Equal(filepath.FromSlash("Artist/Album/Disc10/cover.jpg")))
|
||||
Expect(imgFiles[3]).To(Equal(filepath.FromSlash("Artist/Album/Disc2/cover.jpg")))
|
||||
Expect(imgFiles[2]).To(Equal(filepath.FromSlash("Artist/Album/Disc2/cover.jpg")))
|
||||
Expect(imgFiles[3]).To(Equal(filepath.FromSlash("Artist/Album/Disc10/cover.jpg")))
|
||||
Expect(imgFiles[4]).To(Equal(filepath.FromSlash("Artist/Album/Disc1/cover.1.jpg")))
|
||||
})
|
||||
|
||||
It("prioritizes files without numeric suffixes", func() {
|
||||
// Test case for issue #4683: cover.jpg should come before cover.1.jpg
|
||||
repo.result = []model.Folder{
|
||||
{
|
||||
Path: "Artist/Album",
|
||||
ImagesUpdatedAt: now,
|
||||
ImageFiles: []string{"cover.1.jpg", "cover.jpg", "cover.2.jpg"},
|
||||
},
|
||||
}
|
||||
|
||||
_, imgFiles, _, err := loadAlbumFoldersPaths(ctx, ds, album)
|
||||
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(imgFiles).To(HaveLen(3))
|
||||
|
||||
// cover.jpg should come first because "cover" < "cover.1" < "cover.2"
|
||||
Expect(imgFiles[0]).To(Equal(filepath.FromSlash("Artist/Album/cover.jpg")))
|
||||
Expect(imgFiles[1]).To(Equal(filepath.FromSlash("Artist/Album/cover.1.jpg")))
|
||||
Expect(imgFiles[2]).To(Equal(filepath.FromSlash("Artist/Album/cover.2.jpg")))
|
||||
})
|
||||
|
||||
It("handles case-insensitive sorting", func() {
|
||||
// Test that Cover.jpg and cover.jpg are treated as equivalent
|
||||
repo.result = []model.Folder{
|
||||
{
|
||||
Path: "Artist/Album",
|
||||
ImagesUpdatedAt: now,
|
||||
ImageFiles: []string{"Folder.jpg", "cover.jpg", "BACK.jpg"},
|
||||
},
|
||||
}
|
||||
|
||||
_, imgFiles, _, err := loadAlbumFoldersPaths(ctx, ds, album)
|
||||
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(imgFiles).To(HaveLen(3))
|
||||
|
||||
// Files should be sorted case-insensitively: BACK, cover, Folder
|
||||
Expect(imgFiles[0]).To(Equal(filepath.FromSlash("Artist/Album/BACK.jpg")))
|
||||
Expect(imgFiles[1]).To(Equal(filepath.FromSlash("Artist/Album/cover.jpg")))
|
||||
Expect(imgFiles[2]).To(Equal(filepath.FromSlash("Artist/Album/Folder.jpg")))
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@ -8,6 +8,7 @@ import (
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@ -139,11 +140,22 @@ func findImageInFolder(ctx context.Context, folder, pattern string) (io.ReadClos
|
||||
return nil, "", err
|
||||
}
|
||||
|
||||
// Filter to valid image files
|
||||
var imagePaths []string
|
||||
for _, m := range matches {
|
||||
if !model.IsImageFile(m) {
|
||||
continue
|
||||
}
|
||||
filePath := filepath.Join(folder, m)
|
||||
imagePaths = append(imagePaths, m)
|
||||
}
|
||||
|
||||
// Sort image files by prioritizing base filenames without numeric
|
||||
// suffixes (e.g., artist.jpg before artist.1.jpg)
|
||||
slices.SortFunc(imagePaths, compareImageFiles)
|
||||
|
||||
// Try to open files in sorted order
|
||||
for _, p := range imagePaths {
|
||||
filePath := filepath.Join(folder, p)
|
||||
f, err := os.Open(filePath)
|
||||
if err != nil {
|
||||
log.Warn(ctx, "Could not open cover art file", "file", filePath, err)
|
||||
|
||||
@ -240,24 +240,79 @@ var _ = Describe("artistArtworkReader", func() {
|
||||
Expect(os.MkdirAll(artistDir, 0755)).To(Succeed())
|
||||
|
||||
// Create multiple matching files
|
||||
Expect(os.WriteFile(filepath.Join(artistDir, "artist.jpg"), []byte("jpg image"), 0600)).To(Succeed())
|
||||
Expect(os.WriteFile(filepath.Join(artistDir, "artist.abc"), []byte("text file"), 0600)).To(Succeed())
|
||||
Expect(os.WriteFile(filepath.Join(artistDir, "artist.png"), []byte("png image"), 0600)).To(Succeed())
|
||||
Expect(os.WriteFile(filepath.Join(artistDir, "artist.txt"), []byte("text file"), 0600)).To(Succeed())
|
||||
Expect(os.WriteFile(filepath.Join(artistDir, "artist.jpg"), []byte("jpg image"), 0600)).To(Succeed())
|
||||
|
||||
testFunc = fromArtistFolder(ctx, artistDir, "artist.*")
|
||||
})
|
||||
|
||||
It("returns the first valid image file", func() {
|
||||
It("returns the first valid image file in sorted order", func() {
|
||||
reader, path, err := testFunc()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(reader).ToNot(BeNil())
|
||||
|
||||
// Should return an image file, not the text file
|
||||
Expect(path).To(SatisfyAny(
|
||||
ContainSubstring("artist.jpg"),
|
||||
ContainSubstring("artist.png"),
|
||||
))
|
||||
Expect(path).ToNot(ContainSubstring("artist.txt"))
|
||||
// Should return an image file,
|
||||
// Files are sorted: jpg comes before png alphabetically.
|
||||
// .abc comes first, but it's not an image.
|
||||
Expect(path).To(ContainSubstring("artist.jpg"))
|
||||
reader.Close()
|
||||
})
|
||||
})
|
||||
|
||||
When("prioritizing files without numeric suffixes", func() {
|
||||
BeforeEach(func() {
|
||||
// Test case for issue #4683: artist.jpg should come before artist.1.jpg
|
||||
artistDir := filepath.Join(tempDir, "artist")
|
||||
Expect(os.MkdirAll(artistDir, 0755)).To(Succeed())
|
||||
|
||||
// Create multiple matches with and without numeric suffixes
|
||||
Expect(os.WriteFile(filepath.Join(artistDir, "artist.1.jpg"), []byte("artist 1"), 0600)).To(Succeed())
|
||||
Expect(os.WriteFile(filepath.Join(artistDir, "artist.jpg"), []byte("artist main"), 0600)).To(Succeed())
|
||||
Expect(os.WriteFile(filepath.Join(artistDir, "artist.2.jpg"), []byte("artist 2"), 0600)).To(Succeed())
|
||||
|
||||
testFunc = fromArtistFolder(ctx, artistDir, "artist.*")
|
||||
})
|
||||
|
||||
It("returns artist.jpg before artist.1.jpg and artist.2.jpg", func() {
|
||||
reader, path, err := testFunc()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(reader).ToNot(BeNil())
|
||||
Expect(path).To(ContainSubstring("artist.jpg"))
|
||||
|
||||
// Verify it's the main file, not a numbered variant
|
||||
data, err := io.ReadAll(reader)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(string(data)).To(Equal("artist main"))
|
||||
reader.Close()
|
||||
})
|
||||
})
|
||||
|
||||
When("handling case-insensitive sorting", func() {
|
||||
BeforeEach(func() {
|
||||
// Test case to ensure case-insensitive natural sorting
|
||||
artistDir := filepath.Join(tempDir, "artist")
|
||||
Expect(os.MkdirAll(artistDir, 0755)).To(Succeed())
|
||||
|
||||
// Create files with mixed case names
|
||||
Expect(os.WriteFile(filepath.Join(artistDir, "Folder.jpg"), []byte("folder"), 0600)).To(Succeed())
|
||||
Expect(os.WriteFile(filepath.Join(artistDir, "artist.jpg"), []byte("artist"), 0600)).To(Succeed())
|
||||
Expect(os.WriteFile(filepath.Join(artistDir, "BACK.jpg"), []byte("back"), 0600)).To(Succeed())
|
||||
|
||||
testFunc = fromArtistFolder(ctx, artistDir, "*.*")
|
||||
})
|
||||
|
||||
It("sorts case-insensitively", func() {
|
||||
reader, path, err := testFunc()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(reader).ToNot(BeNil())
|
||||
|
||||
// Should return artist.jpg first (case-insensitive: "artist" < "back" < "folder")
|
||||
Expect(path).To(ContainSubstring("artist.jpg"))
|
||||
|
||||
data, err := io.ReadAll(reader)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(string(data)).To(Equal("artist"))
|
||||
reader.Close()
|
||||
})
|
||||
})
|
||||
|
||||
@ -113,9 +113,9 @@ func WithAdminUser(ctx context.Context, ds model.DataStore) context.Context {
|
||||
if err != nil {
|
||||
c, err := ds.User(ctx).CountAll()
|
||||
if c == 0 && err == nil {
|
||||
log.Debug(ctx, "Scanner: No admin user yet!", err)
|
||||
log.Debug(ctx, "No admin user yet!", err)
|
||||
} else {
|
||||
log.Error(ctx, "Scanner: No admin user found!", err)
|
||||
log.Error(ctx, "No admin user found!", err)
|
||||
}
|
||||
u = &model.User{}
|
||||
}
|
||||
|
||||
@ -21,11 +21,6 @@ import (
|
||||
"github.com/navidrome/navidrome/utils/slice"
|
||||
)
|
||||
|
||||
// Scanner interface for triggering scans
|
||||
type Scanner interface {
|
||||
ScanAll(ctx context.Context, fullScan bool) (warnings []string, err error)
|
||||
}
|
||||
|
||||
// Watcher interface for managing file system watchers
|
||||
type Watcher interface {
|
||||
Watch(ctx context.Context, lib *model.Library) error
|
||||
@ -43,13 +38,13 @@ type Library interface {
|
||||
|
||||
type libraryService struct {
|
||||
ds model.DataStore
|
||||
scanner Scanner
|
||||
scanner model.Scanner
|
||||
watcher Watcher
|
||||
broker events.Broker
|
||||
}
|
||||
|
||||
// NewLibrary creates a new Library service
|
||||
func NewLibrary(ds model.DataStore, scanner Scanner, watcher Watcher, broker events.Broker) Library {
|
||||
func NewLibrary(ds model.DataStore, scanner model.Scanner, watcher Watcher, broker events.Broker) Library {
|
||||
return &libraryService{
|
||||
ds: ds,
|
||||
scanner: scanner,
|
||||
@ -155,7 +150,7 @@ type libraryRepositoryWrapper struct {
|
||||
model.LibraryRepository
|
||||
ctx context.Context
|
||||
ds model.DataStore
|
||||
scanner Scanner
|
||||
scanner model.Scanner
|
||||
watcher Watcher
|
||||
broker events.Broker
|
||||
}
|
||||
@ -192,7 +187,7 @@ func (r *libraryRepositoryWrapper) Save(entity interface{}) (string, error) {
|
||||
return strconv.Itoa(lib.ID), nil
|
||||
}
|
||||
|
||||
func (r *libraryRepositoryWrapper) Update(id string, entity interface{}, cols ...string) error {
|
||||
func (r *libraryRepositoryWrapper) Update(id string, entity interface{}, _ ...string) error {
|
||||
lib := entity.(*model.Library)
|
||||
libID, err := strconv.Atoi(id)
|
||||
if err != nil {
|
||||
|
||||
@ -29,7 +29,7 @@ var _ = Describe("Library Service", func() {
|
||||
var userRepo *tests.MockedUserRepo
|
||||
var ctx context.Context
|
||||
var tempDir string
|
||||
var scanner *mockScanner
|
||||
var scanner *tests.MockScanner
|
||||
var watcherManager *mockWatcherManager
|
||||
var broker *mockEventBroker
|
||||
|
||||
@ -43,7 +43,7 @@ var _ = Describe("Library Service", func() {
|
||||
ds.MockedUser = userRepo
|
||||
|
||||
// Create a mock scanner that tracks calls
|
||||
scanner = &mockScanner{}
|
||||
scanner = tests.NewMockScanner()
|
||||
// Create a mock watcher manager
|
||||
watcherManager = &mockWatcherManager{
|
||||
libraryStates: make(map[int]model.Library),
|
||||
@ -616,11 +616,12 @@ var _ = Describe("Library Service", func() {
|
||||
|
||||
// Wait briefly for the goroutine to complete
|
||||
Eventually(func() int {
|
||||
return scanner.len()
|
||||
return scanner.GetScanAllCallCount()
|
||||
}, "1s", "10ms").Should(Equal(1))
|
||||
|
||||
// Verify scan was called with correct parameters
|
||||
Expect(scanner.ScanCalls[0].FullScan).To(BeFalse()) // Should be quick scan
|
||||
calls := scanner.GetScanAllCalls()
|
||||
Expect(calls[0].FullScan).To(BeFalse()) // Should be quick scan
|
||||
})
|
||||
|
||||
It("triggers scan when updating library path", func() {
|
||||
@ -641,11 +642,12 @@ var _ = Describe("Library Service", func() {
|
||||
|
||||
// Wait briefly for the goroutine to complete
|
||||
Eventually(func() int {
|
||||
return scanner.len()
|
||||
return scanner.GetScanAllCallCount()
|
||||
}, "1s", "10ms").Should(Equal(1))
|
||||
|
||||
// Verify scan was called with correct parameters
|
||||
Expect(scanner.ScanCalls[0].FullScan).To(BeFalse()) // Should be quick scan
|
||||
calls := scanner.GetScanAllCalls()
|
||||
Expect(calls[0].FullScan).To(BeFalse()) // Should be quick scan
|
||||
})
|
||||
|
||||
It("does not trigger scan when updating library without path change", func() {
|
||||
@ -661,7 +663,7 @@ var _ = Describe("Library Service", func() {
|
||||
|
||||
// Wait a bit to ensure no scan was triggered
|
||||
Consistently(func() int {
|
||||
return scanner.len()
|
||||
return scanner.GetScanAllCallCount()
|
||||
}, "100ms", "10ms").Should(Equal(0))
|
||||
})
|
||||
|
||||
@ -674,7 +676,7 @@ var _ = Describe("Library Service", func() {
|
||||
|
||||
// Ensure no scan was triggered since creation failed
|
||||
Consistently(func() int {
|
||||
return scanner.len()
|
||||
return scanner.GetScanAllCallCount()
|
||||
}, "100ms", "10ms").Should(Equal(0))
|
||||
})
|
||||
|
||||
@ -691,7 +693,7 @@ var _ = Describe("Library Service", func() {
|
||||
|
||||
// Ensure no scan was triggered since update failed
|
||||
Consistently(func() int {
|
||||
return scanner.len()
|
||||
return scanner.GetScanAllCallCount()
|
||||
}, "100ms", "10ms").Should(Equal(0))
|
||||
})
|
||||
|
||||
@ -707,11 +709,12 @@ var _ = Describe("Library Service", func() {
|
||||
|
||||
// Wait briefly for the goroutine to complete
|
||||
Eventually(func() int {
|
||||
return scanner.len()
|
||||
return scanner.GetScanAllCallCount()
|
||||
}, "1s", "10ms").Should(Equal(1))
|
||||
|
||||
// Verify scan was called with correct parameters
|
||||
Expect(scanner.ScanCalls[0].FullScan).To(BeFalse()) // Should be quick scan
|
||||
calls := scanner.GetScanAllCalls()
|
||||
Expect(calls[0].FullScan).To(BeFalse()) // Should be quick scan
|
||||
})
|
||||
|
||||
It("does not trigger scan when library deletion fails", func() {
|
||||
@ -721,7 +724,7 @@ var _ = Describe("Library Service", func() {
|
||||
|
||||
// Ensure no scan was triggered since deletion failed
|
||||
Consistently(func() int {
|
||||
return scanner.len()
|
||||
return scanner.GetScanAllCallCount()
|
||||
}, "100ms", "10ms").Should(Equal(0))
|
||||
})
|
||||
|
||||
@ -868,31 +871,6 @@ var _ = Describe("Library Service", func() {
|
||||
})
|
||||
})
|
||||
|
||||
// mockScanner provides a simple mock implementation of core.Scanner for testing
|
||||
type mockScanner struct {
|
||||
ScanCalls []ScanCall
|
||||
mu sync.RWMutex
|
||||
}
|
||||
|
||||
type ScanCall struct {
|
||||
FullScan bool
|
||||
}
|
||||
|
||||
func (m *mockScanner) ScanAll(ctx context.Context, fullScan bool) (warnings []string, err error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.ScanCalls = append(m.ScanCalls, ScanCall{
|
||||
FullScan: fullScan,
|
||||
})
|
||||
return []string{}, nil
|
||||
}
|
||||
|
||||
func (m *mockScanner) len() int {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
return len(m.ScanCalls)
|
||||
}
|
||||
|
||||
// mockWatcherManager provides a simple mock implementation of core.Watcher for testing
|
||||
type mockWatcherManager struct {
|
||||
StartedWatchers []model.Library
|
||||
|
||||
@ -166,7 +166,7 @@ func (s *maintenanceService) getAffectedAlbumIDs(ctx context.Context, ids []stri
|
||||
if len(ids) > 0 {
|
||||
filters = squirrel.And{
|
||||
squirrel.Eq{"missing": true},
|
||||
squirrel.Eq{"id": ids},
|
||||
squirrel.Eq{"media_file.id": ids},
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -14,7 +14,7 @@ import (
|
||||
)
|
||||
|
||||
var _ = Describe("Maintenance", func() {
|
||||
var ds *extendedDataStore
|
||||
var ds *tests.MockDataStore
|
||||
var mfRepo *extendedMediaFileRepo
|
||||
var service Maintenance
|
||||
var ctx context.Context
|
||||
@ -42,7 +42,7 @@ var _ = Describe("Maintenance", func() {
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(mfRepo.deleteMissingCalled).To(BeTrue())
|
||||
Expect(mfRepo.deletedIDs).To(Equal([]string{"mf1", "mf2"}))
|
||||
Expect(ds.gcCalled).To(BeTrue(), "GC should be called after deletion")
|
||||
Expect(ds.GCCalled).To(BeTrue(), "GC should be called after deletion")
|
||||
})
|
||||
|
||||
It("triggers artist stats refresh and album refresh after deletion", func() {
|
||||
@ -97,7 +97,7 @@ var _ = Describe("Maintenance", func() {
|
||||
})
|
||||
|
||||
// Set GC to return error
|
||||
ds.gcError = errors.New("gc failed")
|
||||
ds.GCError = errors.New("gc failed")
|
||||
|
||||
err := service.DeleteMissingFiles(ctx, []string{"mf1"})
|
||||
|
||||
@ -143,7 +143,7 @@ var _ = Describe("Maintenance", func() {
|
||||
err := service.DeleteAllMissingFiles(ctx)
|
||||
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(ds.gcCalled).To(BeTrue(), "GC should be called after deletion")
|
||||
Expect(ds.GCCalled).To(BeTrue(), "GC should be called after deletion")
|
||||
})
|
||||
|
||||
It("returns error if deletion fails", func() {
|
||||
@ -253,11 +253,8 @@ var _ = Describe("Maintenance", func() {
|
||||
})
|
||||
|
||||
// Test helper to create a mock DataStore with controllable behavior
|
||||
func createTestDataStore() *extendedDataStore {
|
||||
// Create extended datastore with GC tracking
|
||||
ds := &extendedDataStore{
|
||||
MockDataStore: &tests.MockDataStore{},
|
||||
}
|
||||
func createTestDataStore() *tests.MockDataStore {
|
||||
ds := &tests.MockDataStore{}
|
||||
|
||||
// Create extended album repo with Put tracking
|
||||
albumRepo := &extendedAlbumRepo{
|
||||
@ -365,18 +362,3 @@ func (m *extendedArtistRepo) IsRefreshStatsCalled() bool {
|
||||
defer m.mu.RUnlock()
|
||||
return m.refreshStatsCalled
|
||||
}
|
||||
|
||||
// Extension of MockDataStore to track GC calls
|
||||
type extendedDataStore struct {
|
||||
*tests.MockDataStore
|
||||
gcCalled bool
|
||||
gcError error
|
||||
}
|
||||
|
||||
func (ds *extendedDataStore) GC(ctx context.Context) error {
|
||||
ds.gcCalled = true
|
||||
if ds.gcError != nil {
|
||||
return ds.gcError
|
||||
}
|
||||
return ds.MockDataStore.GC(ctx)
|
||||
}
|
||||
|
||||
@ -22,6 +22,7 @@ import (
|
||||
"github.com/navidrome/navidrome/core/metrics/insights"
|
||||
"github.com/navidrome/navidrome/log"
|
||||
"github.com/navidrome/navidrome/model"
|
||||
"github.com/navidrome/navidrome/model/request"
|
||||
"github.com/navidrome/navidrome/plugins/schema"
|
||||
"github.com/navidrome/navidrome/utils/singleton"
|
||||
)
|
||||
@ -64,9 +65,16 @@ func GetInstance(ds model.DataStore, pluginLoader PluginLoader) Insights {
|
||||
}
|
||||
|
||||
func (c *insightsCollector) Run(ctx context.Context) {
|
||||
ctx = auth.WithAdminUser(ctx, c.ds)
|
||||
for {
|
||||
c.sendInsights(ctx)
|
||||
// Refresh admin context on each iteration to handle cases where
|
||||
// admin user wasn't available on previous runs
|
||||
insightsCtx := auth.WithAdminUser(ctx, c.ds)
|
||||
u, _ := request.UserFrom(insightsCtx)
|
||||
if !u.IsAdmin {
|
||||
log.Trace(insightsCtx, "No admin user available, skipping insights collection")
|
||||
} else {
|
||||
c.sendInsights(insightsCtx)
|
||||
}
|
||||
select {
|
||||
case <-time.After(consts.InsightsUpdateInterval):
|
||||
continue
|
||||
|
||||
15
db/db.go
15
db/db.go
@ -45,10 +45,12 @@ func Db() *sql.DB {
|
||||
if err != nil {
|
||||
log.Fatal("Error opening database", err)
|
||||
}
|
||||
_, err = db.Exec("PRAGMA optimize=0x10002")
|
||||
if err != nil {
|
||||
log.Error("Error applying PRAGMA optimize", err)
|
||||
return nil
|
||||
if conf.Server.DevOptimizeDB {
|
||||
_, err = db.Exec("PRAGMA optimize=0x10002")
|
||||
if err != nil {
|
||||
log.Error("Error applying PRAGMA optimize", err)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return db
|
||||
})
|
||||
@ -99,7 +101,7 @@ func Init(ctx context.Context) func() {
|
||||
log.Fatal(ctx, "Failed to apply new migrations", err)
|
||||
}
|
||||
|
||||
if hasSchemaChanges {
|
||||
if hasSchemaChanges && conf.Server.DevOptimizeDB {
|
||||
log.Debug(ctx, "Applying PRAGMA optimize after schema changes")
|
||||
_, err = db.ExecContext(ctx, "PRAGMA optimize")
|
||||
if err != nil {
|
||||
@ -114,6 +116,9 @@ func Init(ctx context.Context) func() {
|
||||
|
||||
// Optimize runs PRAGMA optimize on each connection in the pool
|
||||
func Optimize(ctx context.Context) {
|
||||
if !conf.Server.DevOptimizeDB {
|
||||
return
|
||||
}
|
||||
numConns := Db().Stats().OpenConnections
|
||||
if numConns == 0 {
|
||||
log.Debug(ctx, "No open connections to optimize")
|
||||
|
||||
@ -0,0 +1,7 @@
|
||||
-- +goose Up
|
||||
-- +goose StatementBegin
|
||||
ALTER TABLE annotation ADD COLUMN rated_at datetime;
|
||||
-- +goose StatementEnd
|
||||
|
||||
-- +goose Down
|
||||
|
||||
@ -7,6 +7,7 @@ import (
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/navidrome/navidrome/conf"
|
||||
"github.com/navidrome/navidrome/consts"
|
||||
)
|
||||
|
||||
@ -21,11 +22,13 @@ func notice(tx *sql.Tx, msg string) {
|
||||
// Call this in migrations that requires a full rescan
|
||||
func forceFullRescan(tx *sql.Tx) error {
|
||||
// If a full scan is required, most probably the query optimizer is outdated, so we run `analyze`.
|
||||
_, err := tx.Exec(`ANALYZE;`)
|
||||
if err != nil {
|
||||
return err
|
||||
if conf.Server.DevOptimizeDB {
|
||||
_, err := tx.Exec(`ANALYZE;`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
_, err = tx.Exec(fmt.Sprintf(`
|
||||
_, err := tx.Exec(fmt.Sprintf(`
|
||||
INSERT OR REPLACE into property (id, value) values ('%s', '1');
|
||||
`, consts.FullScanAfterMigrationFlagKey))
|
||||
return err
|
||||
|
||||
23
go.mod
23
go.mod
@ -1,6 +1,6 @@
|
||||
module github.com/navidrome/navidrome
|
||||
|
||||
go 1.25.4
|
||||
go 1.25
|
||||
|
||||
// Fork to fix https://github.com/navidrome/navidrome/issues/3254
|
||||
replace github.com/dhowden/tag v0.0.0-20240417053706-3d75831295e8 => github.com/deluan/tag v0.0.0-20241002021117-dfe5e6ea396d
|
||||
@ -39,6 +39,7 @@ require (
|
||||
github.com/knqyf263/go-plugin v0.9.0
|
||||
github.com/kr/pretty v0.3.1
|
||||
github.com/lestrrat-go/jwx/v2 v2.1.6
|
||||
github.com/maruel/natural v1.2.1
|
||||
github.com/matoous/go-nanoid/v2 v2.1.0
|
||||
github.com/mattn/go-sqlite3 v1.14.32
|
||||
github.com/microcosm-cc/bluemonday v1.0.27
|
||||
@ -56,16 +57,16 @@ require (
|
||||
github.com/spf13/cobra v1.10.1
|
||||
github.com/spf13/viper v1.21.0
|
||||
github.com/stretchr/testify v1.11.1
|
||||
github.com/tetratelabs/wazero v1.10.0
|
||||
github.com/tetratelabs/wazero v1.10.1
|
||||
github.com/unrolled/secure v1.17.0
|
||||
github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342
|
||||
go.uber.org/goleak v1.3.0
|
||||
golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546
|
||||
golang.org/x/image v0.32.0
|
||||
golang.org/x/net v0.46.0
|
||||
golang.org/x/exp v0.0.0-20251113190631-e25ba8c21ef6
|
||||
golang.org/x/image v0.33.0
|
||||
golang.org/x/net v0.47.0
|
||||
golang.org/x/sync v0.18.0
|
||||
golang.org/x/sys v0.38.0
|
||||
golang.org/x/text v0.30.0
|
||||
golang.org/x/text v0.31.0
|
||||
golang.org/x/time v0.14.0
|
||||
google.golang.org/protobuf v1.36.10
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
@ -89,7 +90,7 @@ require (
|
||||
github.com/goccy/go-json v0.10.5 // indirect
|
||||
github.com/goccy/go-yaml v1.18.0 // indirect
|
||||
github.com/google/go-cmp v0.7.0 // indirect
|
||||
github.com/google/pprof v0.0.0-20251007162407-5df77e3f7d1d // indirect
|
||||
github.com/google/pprof v0.0.0-20251114195745-4902fdda35c8 // indirect
|
||||
github.com/google/subcommands v1.2.0 // indirect
|
||||
github.com/gorilla/css v1.0.1 // indirect
|
||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||
@ -127,10 +128,10 @@ require (
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.2 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/crypto v0.43.0 // indirect
|
||||
golang.org/x/mod v0.29.0 // indirect
|
||||
golang.org/x/telemetry v0.0.0-20251008203120-078029d740a8 // indirect
|
||||
golang.org/x/tools v0.38.0 // indirect
|
||||
golang.org/x/crypto v0.45.0 // indirect
|
||||
golang.org/x/mod v0.30.0 // indirect
|
||||
golang.org/x/telemetry v0.0.0-20251111182119-bc8e575c7b54 // indirect
|
||||
golang.org/x/tools v0.39.0 // indirect
|
||||
gopkg.in/ini.v1 v1.67.0 // indirect
|
||||
gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect
|
||||
)
|
||||
|
||||
44
go.sum
44
go.sum
@ -99,8 +99,8 @@ github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/google/go-pipeline v0.0.0-20230411140531-6cbedfc1d3fc h1:hd+uUVsB1vdxohPneMrhGH2YfQuH5hRIK9u4/XCeUtw=
|
||||
github.com/google/go-pipeline v0.0.0-20230411140531-6cbedfc1d3fc/go.mod h1:SL66SJVysrh7YbDCP9tH30b8a9o/N2HeiQNUm85EKhc=
|
||||
github.com/google/pprof v0.0.0-20251007162407-5df77e3f7d1d h1:KJIErDwbSHjnp/SGzE5ed8Aol7JsKiI5X7yWKAtzhM0=
|
||||
github.com/google/pprof v0.0.0-20251007162407-5df77e3f7d1d/go.mod h1:I6V7YzU0XDpsHqbsyrghnFZLO1gwK6NPTNvmetQIk9U=
|
||||
github.com/google/pprof v0.0.0-20251114195745-4902fdda35c8 h1:3DsUAV+VNEQa2CUVLxCY3f87278uWfIDhJnbdvDjvmE=
|
||||
github.com/google/pprof v0.0.0-20251114195745-4902fdda35c8/go.mod h1:I6V7YzU0XDpsHqbsyrghnFZLO1gwK6NPTNvmetQIk9U=
|
||||
github.com/google/subcommands v1.2.0 h1:vWQspBTo2nEqTUFita5/KeEWlUL8kQObDFbub/EN9oE=
|
||||
github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
@ -162,8 +162,8 @@ github.com/lestrrat-go/jwx/v2 v2.1.6 h1:hxM1gfDILk/l5ylers6BX/Eq1m/pnxe9NBwW6lVf
|
||||
github.com/lestrrat-go/jwx/v2 v2.1.6/go.mod h1:Y722kU5r/8mV7fYDifjug0r8FK8mZdw0K0GpJw/l8pU=
|
||||
github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNBEYU=
|
||||
github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I=
|
||||
github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo=
|
||||
github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg=
|
||||
github.com/maruel/natural v1.2.1 h1:G/y4pwtTA07lbQsMefvsmEO0VN0NfqpxprxXDM4R/4o=
|
||||
github.com/maruel/natural v1.2.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg=
|
||||
github.com/matoous/go-nanoid/v2 v2.1.0 h1:P64+dmq21hhWdtvZfEAofnvJULaRR1Yib0+PnU669bE=
|
||||
github.com/matoous/go-nanoid/v2 v2.1.0/go.mod h1:KlbGNQ+FhrUNIHUxZdL63t7tl4LaPkZNpUULS8H4uVM=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
@ -265,8 +265,8 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
|
||||
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
|
||||
github.com/tetratelabs/wazero v1.10.0 h1:CXP3zneLDl6J4Zy8N/J+d5JsWKfrjE6GtvVK1fpnDlk=
|
||||
github.com/tetratelabs/wazero v1.10.0/go.mod h1:DRm5twOQ5Gr1AoEdSi0CLjDQF1J9ZAuyqFIjl1KKfQU=
|
||||
github.com/tetratelabs/wazero v1.10.1 h1:2DugeJf6VVk58KTPszlNfeeN8AhhpwcZqkJj2wwFuH8=
|
||||
github.com/tetratelabs/wazero v1.10.1/go.mod h1:DRm5twOQ5Gr1AoEdSi0CLjDQF1J9ZAuyqFIjl1KKfQU=
|
||||
github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY=
|
||||
github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
|
||||
github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
|
||||
@ -298,20 +298,20 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY
|
||||
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
|
||||
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
|
||||
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
|
||||
golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04=
|
||||
golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0=
|
||||
golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 h1:mgKeJMpvi0yx/sU5GsxQ7p6s2wtOnGAHZWCHUM4KGzY=
|
||||
golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546/go.mod h1:j/pmGrbnkbPtQfxEe5D0VQhZC6qKbfKifgD0oM7sR70=
|
||||
golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q=
|
||||
golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4=
|
||||
golang.org/x/exp v0.0.0-20251113190631-e25ba8c21ef6 h1:zfMcR1Cs4KNuomFFgGefv5N0czO2XZpUbxGUy8i8ug0=
|
||||
golang.org/x/exp v0.0.0-20251113190631-e25ba8c21ef6/go.mod h1:46edojNIoXTNOhySWIWdix628clX9ODXwPsQuG6hsK0=
|
||||
golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
|
||||
golang.org/x/image v0.32.0 h1:6lZQWq75h7L5IWNk0r+SCpUJ6tUVd3v4ZHnbRKLkUDQ=
|
||||
golang.org/x/image v0.32.0/go.mod h1:/R37rrQmKXtO6tYXAjtDLwQgFLHmhW+V6ayXlxzP2Pc=
|
||||
golang.org/x/image v0.33.0 h1:LXRZRnv1+zGd5XBUVRFmYEphyyKJjQjCRiOuAP3sZfQ=
|
||||
golang.org/x/image v0.33.0/go.mod h1:DD3OsTYT9chzuzTQt+zMcOlBHgfoKQb1gry8p76Y1sc=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA=
|
||||
golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w=
|
||||
golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk=
|
||||
golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc=
|
||||
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
@ -323,8 +323,8 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
|
||||
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
|
||||
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
|
||||
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
|
||||
golang.org/x/net v0.46.0 h1:giFlY12I07fugqwPuWJi68oOnpfqFnJIJzaIIm2JVV4=
|
||||
golang.org/x/net v0.46.0/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210=
|
||||
golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
|
||||
golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
@ -353,8 +353,8 @@ golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
|
||||
golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
|
||||
golang.org/x/telemetry v0.0.0-20251008203120-078029d740a8 h1:LvzTn0GQhWuvKH/kVRS3R3bVAsdQWI7hvfLHGgh9+lU=
|
||||
golang.org/x/telemetry v0.0.0-20251008203120-078029d740a8/go.mod h1:Pi4ztBfryZoJEkyFTI5/Ocsu2jXyDr6iSdgJiYE/uwE=
|
||||
golang.org/x/telemetry v0.0.0-20251111182119-bc8e575c7b54 h1:E2/AqCUMZGgd73TQkxUMcMla25GB9i/5HOdLr+uH7Vo=
|
||||
golang.org/x/telemetry v0.0.0-20251111182119-bc8e575c7b54/go.mod h1:hKdjCMrbv9skySur+Nek8Hd0uJ0GuxJIoIX2payrIdQ=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
||||
@ -373,8 +373,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
|
||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
|
||||
golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k=
|
||||
golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM=
|
||||
golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
|
||||
golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
|
||||
golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
|
||||
golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
@ -384,8 +384,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
|
||||
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
|
||||
golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ=
|
||||
golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs=
|
||||
golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ=
|
||||
golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
|
||||
google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE=
|
||||
|
||||
17
log/log.go
17
log/log.go
@ -80,8 +80,8 @@ var (
|
||||
|
||||
// SetLevel sets the global log level used by the simple logger.
|
||||
func SetLevel(l Level) {
|
||||
currentLevel = l
|
||||
loggerMu.Lock()
|
||||
currentLevel = l
|
||||
defaultLogger.Level = logrus.TraceLevel
|
||||
loggerMu.Unlock()
|
||||
logrus.SetLevel(logrus.Level(l))
|
||||
@ -114,6 +114,8 @@ func levelFromString(l string) Level {
|
||||
|
||||
// SetLogLevels sets the log levels for specific paths in the codebase.
|
||||
func SetLogLevels(levels map[string]string) {
|
||||
loggerMu.Lock()
|
||||
defer loggerMu.Unlock()
|
||||
logLevels = nil
|
||||
for k, v := range levels {
|
||||
logLevels = append(logLevels, levelPath{path: k, level: levelFromString(v)})
|
||||
@ -172,6 +174,8 @@ func SetDefaultLogger(l *logrus.Logger) {
|
||||
}
|
||||
|
||||
func CurrentLevel() Level {
|
||||
loggerMu.RLock()
|
||||
defer loggerMu.RUnlock()
|
||||
return currentLevel
|
||||
}
|
||||
|
||||
@ -220,10 +224,15 @@ func Writer() io.Writer {
|
||||
}
|
||||
|
||||
func shouldLog(requiredLevel Level, skip int) bool {
|
||||
if currentLevel >= requiredLevel {
|
||||
loggerMu.RLock()
|
||||
level := currentLevel
|
||||
levels := logLevels
|
||||
loggerMu.RUnlock()
|
||||
|
||||
if level >= requiredLevel {
|
||||
return true
|
||||
}
|
||||
if len(logLevels) == 0 {
|
||||
if len(levels) == 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
@ -233,7 +242,7 @@ func shouldLog(requiredLevel Level, skip int) bool {
|
||||
}
|
||||
|
||||
file = strings.TrimPrefix(file, rootPath)
|
||||
for _, lp := range logLevels {
|
||||
for _, lp := range levels {
|
||||
if strings.HasPrefix(file, lp.path) {
|
||||
return lp.level >= requiredLevel
|
||||
}
|
||||
|
||||
@ -6,6 +6,7 @@ type Annotations struct {
|
||||
PlayCount int64 `structs:"play_count" json:"playCount,omitempty"`
|
||||
PlayDate *time.Time `structs:"play_date" json:"playDate,omitempty" `
|
||||
Rating int `structs:"rating" json:"rating,omitempty" `
|
||||
RatedAt *time.Time `structs:"rated_at" json:"ratedAt,omitempty" `
|
||||
Starred bool `structs:"starred" json:"starred,omitempty" `
|
||||
StarredAt *time.Time `structs:"starred_at" json:"starredAt,omitempty"`
|
||||
}
|
||||
|
||||
@ -44,6 +44,7 @@ var fieldMap = map[string]*mappedField{
|
||||
"loved": {field: "COALESCE(annotation.starred, false)"},
|
||||
"dateloved": {field: "annotation.starred_at"},
|
||||
"lastplayed": {field: "annotation.play_date"},
|
||||
"daterated": {field: "annotation.rated_at"},
|
||||
"playcount": {field: "COALESCE(annotation.play_count, 0)"},
|
||||
"rating": {field: "COALESCE(annotation.rating, 0)"},
|
||||
"mbz_album_id": {field: "media_file.mbz_album_id"},
|
||||
|
||||
@ -43,5 +43,5 @@ type DataStore interface {
|
||||
|
||||
WithTx(block func(tx DataStore) error, scope ...string) error
|
||||
WithTxImmediate(block func(tx DataStore) error, scope ...string) error
|
||||
GC(ctx context.Context) error
|
||||
GC(ctx context.Context, libraryIDs ...int) error
|
||||
}
|
||||
|
||||
@ -85,7 +85,7 @@ type FolderRepository interface {
|
||||
GetByPath(lib Library, path string) (*Folder, error)
|
||||
GetAll(...QueryOptions) ([]Folder, error)
|
||||
CountAll(...QueryOptions) (int64, error)
|
||||
GetLastUpdates(lib Library) (map[string]FolderUpdateInfo, error)
|
||||
GetFolderUpdateInfo(lib Library, targetPaths ...string) (map[string]FolderUpdateInfo, error)
|
||||
Put(*Folder) error
|
||||
MarkMissing(missing bool, ids ...string) error
|
||||
GetTouchedWithPlaylists() (FolderCursor, error)
|
||||
|
||||
@ -23,7 +23,7 @@ func legacyTrackID(mf model.MediaFile, prependLibId bool) string {
|
||||
}
|
||||
|
||||
func legacyAlbumID(mf model.MediaFile, md Metadata, prependLibId bool) string {
|
||||
releaseDate := legacyReleaseDate(md)
|
||||
_, _, releaseDate := md.mapDates()
|
||||
albumPath := strings.ToLower(fmt.Sprintf("%s\\%s", legacyMapAlbumArtistName(md), legacyMapAlbumName(md)))
|
||||
if !conf.Server.Scanner.GroupAlbumReleases {
|
||||
if len(releaseDate) != 0 {
|
||||
@ -55,9 +55,3 @@ func legacyMapAlbumName(md Metadata) string {
|
||||
consts.UnknownAlbum,
|
||||
)
|
||||
}
|
||||
|
||||
// Keep the TaggedLikePicard logic for backwards compatibility
|
||||
func legacyReleaseDate(md Metadata) string {
|
||||
_, _, releaseDate := md.mapDates()
|
||||
return string(releaseDate)
|
||||
}
|
||||
|
||||
@ -1,30 +0,0 @@
|
||||
package metadata
|
||||
|
||||
import (
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
var _ = Describe("legacyReleaseDate", func() {
|
||||
|
||||
DescribeTable("legacyReleaseDate",
|
||||
func(recordingDate, originalDate, releaseDate, expected string) {
|
||||
md := New("", Info{
|
||||
Tags: map[string][]string{
|
||||
"DATE": {recordingDate},
|
||||
"ORIGINALDATE": {originalDate},
|
||||
"RELEASEDATE": {releaseDate},
|
||||
},
|
||||
})
|
||||
|
||||
result := legacyReleaseDate(md)
|
||||
Expect(result).To(Equal(expected))
|
||||
},
|
||||
Entry("regular mapping", "2020-05-15", "2019-02-10", "2021-01-01", "2021-01-01"),
|
||||
Entry("legacy mapping", "2020-05-15", "2019-02-10", "", "2020-05-15"),
|
||||
Entry("legacy mapping, originalYear < year", "2018-05-15", "2019-02-10", "2021-01-01", "2021-01-01"),
|
||||
Entry("legacy mapping, originalYear empty", "2020-05-15", "", "2021-01-01", "2021-01-01"),
|
||||
Entry("legacy mapping, releaseYear", "2020-05-15", "2019-02-10", "2021-01-01", "2021-01-01"),
|
||||
Entry("legacy mapping, same dates", "2020-05-15", "2020-05-15", "", "2020-05-15"),
|
||||
)
|
||||
})
|
||||
@ -75,6 +75,23 @@ var _ = Describe("ToMediaFile", func() {
|
||||
Expect(mf.OriginalYear).To(Equal(1966))
|
||||
Expect(mf.ReleaseYear).To(Equal(2014))
|
||||
})
|
||||
DescribeTable("legacyReleaseDate (TaggedLikePicard old behavior)",
|
||||
func(recordingDate, originalDate, releaseDate, expected string) {
|
||||
mf := toMediaFile(model.RawTags{
|
||||
"DATE": {recordingDate},
|
||||
"ORIGINALDATE": {originalDate},
|
||||
"RELEASEDATE": {releaseDate},
|
||||
})
|
||||
|
||||
Expect(mf.ReleaseDate).To(Equal(expected))
|
||||
},
|
||||
Entry("regular mapping", "2020-05-15", "2019-02-10", "2021-01-01", "2021-01-01"),
|
||||
Entry("legacy mapping", "2020-05-15", "2019-02-10", "", "2020-05-15"),
|
||||
Entry("legacy mapping, originalYear < year", "2018-05-15", "2019-02-10", "2021-01-01", "2021-01-01"),
|
||||
Entry("legacy mapping, originalYear empty", "2020-05-15", "", "2021-01-01", "2021-01-01"),
|
||||
Entry("legacy mapping, releaseYear", "2020-05-15", "2019-02-10", "2021-01-01", "2021-01-01"),
|
||||
Entry("legacy mapping, same dates", "2020-05-15", "2020-05-15", "", "2020-05-15"),
|
||||
)
|
||||
})
|
||||
|
||||
Describe("Lyrics", func() {
|
||||
|
||||
81
model/scanner.go
Normal file
81
model/scanner.go
Normal file
@ -0,0 +1,81 @@
|
||||
package model
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ScanTarget represents a specific folder within a library to be scanned.
|
||||
// NOTE: This struct is used as a map key, so it should only contain comparable types.
|
||||
type ScanTarget struct {
|
||||
LibraryID int
|
||||
FolderPath string // Relative path within the library, or "" for entire library
|
||||
}
|
||||
|
||||
func (st ScanTarget) String() string {
|
||||
return fmt.Sprintf("%d:%s", st.LibraryID, st.FolderPath)
|
||||
}
|
||||
|
||||
// ScannerStatus holds information about the current scan status
|
||||
type ScannerStatus struct {
|
||||
Scanning bool
|
||||
LastScan time.Time
|
||||
Count uint32
|
||||
FolderCount uint32
|
||||
LastError string
|
||||
ScanType string
|
||||
ElapsedTime time.Duration
|
||||
}
|
||||
|
||||
type Scanner interface {
|
||||
// ScanAll starts a scan of all libraries. This is a blocking operation.
|
||||
ScanAll(ctx context.Context, fullScan bool) (warnings []string, err error)
|
||||
// ScanFolders scans specific library/folder pairs, recursing into subdirectories.
|
||||
// If targets is nil, it scans all libraries. This is a blocking operation.
|
||||
ScanFolders(ctx context.Context, fullScan bool, targets []ScanTarget) (warnings []string, err error)
|
||||
Status(context.Context) (*ScannerStatus, error)
|
||||
}
|
||||
|
||||
// ParseTargets parses scan targets strings into ScanTarget structs.
|
||||
// Example: []string{"1:Music/Rock", "2:Classical"}
|
||||
func ParseTargets(libFolders []string) ([]ScanTarget, error) {
|
||||
targets := make([]ScanTarget, 0, len(libFolders))
|
||||
|
||||
for _, part := range libFolders {
|
||||
part = strings.TrimSpace(part)
|
||||
if part == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
// Split by the first colon
|
||||
colonIdx := strings.Index(part, ":")
|
||||
if colonIdx == -1 {
|
||||
return nil, fmt.Errorf("invalid target format: %q (expected libraryID:folderPath)", part)
|
||||
}
|
||||
|
||||
libIDStr := part[:colonIdx]
|
||||
folderPath := part[colonIdx+1:]
|
||||
|
||||
libID, err := strconv.Atoi(libIDStr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid library ID %q: %w", libIDStr, err)
|
||||
}
|
||||
if libID <= 0 {
|
||||
return nil, fmt.Errorf("invalid library ID %q", libIDStr)
|
||||
}
|
||||
|
||||
targets = append(targets, ScanTarget{
|
||||
LibraryID: libID,
|
||||
FolderPath: folderPath,
|
||||
})
|
||||
}
|
||||
|
||||
if len(targets) == 0 {
|
||||
return nil, fmt.Errorf("no valid targets found")
|
||||
}
|
||||
|
||||
return targets, nil
|
||||
}
|
||||
89
model/scanner_test.go
Normal file
89
model/scanner_test.go
Normal file
@ -0,0 +1,89 @@
|
||||
package model_test
|
||||
|
||||
import (
|
||||
"github.com/navidrome/navidrome/model"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
var _ = Describe("ParseTargets", func() {
|
||||
It("parses multiple entries in slice", func() {
|
||||
targets, err := model.ParseTargets([]string{"1:Music/Rock", "1:Music/Jazz", "2:Classical"})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(targets).To(HaveLen(3))
|
||||
Expect(targets[0].LibraryID).To(Equal(1))
|
||||
Expect(targets[0].FolderPath).To(Equal("Music/Rock"))
|
||||
Expect(targets[1].LibraryID).To(Equal(1))
|
||||
Expect(targets[1].FolderPath).To(Equal("Music/Jazz"))
|
||||
Expect(targets[2].LibraryID).To(Equal(2))
|
||||
Expect(targets[2].FolderPath).To(Equal("Classical"))
|
||||
})
|
||||
|
||||
It("handles empty folder paths", func() {
|
||||
targets, err := model.ParseTargets([]string{"1:", "2:"})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(targets).To(HaveLen(2))
|
||||
Expect(targets[0].FolderPath).To(Equal(""))
|
||||
Expect(targets[1].FolderPath).To(Equal(""))
|
||||
})
|
||||
|
||||
It("trims whitespace from entries", func() {
|
||||
targets, err := model.ParseTargets([]string{" 1:Music/Rock", " 2:Classical "})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(targets).To(HaveLen(2))
|
||||
Expect(targets[0].LibraryID).To(Equal(1))
|
||||
Expect(targets[0].FolderPath).To(Equal("Music/Rock"))
|
||||
Expect(targets[1].LibraryID).To(Equal(2))
|
||||
Expect(targets[1].FolderPath).To(Equal("Classical"))
|
||||
})
|
||||
|
||||
It("skips empty strings", func() {
|
||||
targets, err := model.ParseTargets([]string{"1:Music/Rock", "", "2:Classical"})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(targets).To(HaveLen(2))
|
||||
})
|
||||
|
||||
It("handles paths with colons", func() {
|
||||
targets, err := model.ParseTargets([]string{"1:C:/Music/Rock", "2:/path:with:colons"})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(targets).To(HaveLen(2))
|
||||
Expect(targets[0].FolderPath).To(Equal("C:/Music/Rock"))
|
||||
Expect(targets[1].FolderPath).To(Equal("/path:with:colons"))
|
||||
})
|
||||
|
||||
It("returns error for invalid format without colon", func() {
|
||||
_, err := model.ParseTargets([]string{"1Music/Rock"})
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring("invalid target format"))
|
||||
})
|
||||
|
||||
It("returns error for non-numeric library ID", func() {
|
||||
_, err := model.ParseTargets([]string{"abc:Music/Rock"})
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring("invalid library ID"))
|
||||
})
|
||||
|
||||
It("returns error for negative library ID", func() {
|
||||
_, err := model.ParseTargets([]string{"-1:Music/Rock"})
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring("invalid library ID"))
|
||||
})
|
||||
|
||||
It("returns error for zero library ID", func() {
|
||||
_, err := model.ParseTargets([]string{"0:Music/Rock"})
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring("invalid library ID"))
|
||||
})
|
||||
|
||||
It("returns error for empty input", func() {
|
||||
_, err := model.ParseTargets([]string{})
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring("no valid targets found"))
|
||||
})
|
||||
|
||||
It("returns error for all empty strings", func() {
|
||||
_, err := model.ParseTargets([]string{"", " ", ""})
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring("no valid targets found"))
|
||||
})
|
||||
})
|
||||
@ -106,6 +106,7 @@ func NewAlbumRepository(ctx context.Context, db dbx.Builder) model.AlbumReposito
|
||||
"random": "random",
|
||||
"recently_added": recentlyAddedSort(),
|
||||
"starred_at": "starred, starred_at",
|
||||
"rated_at": "rating, rated_at",
|
||||
})
|
||||
return r
|
||||
}
|
||||
@ -337,8 +338,12 @@ on conflict (user_id, item_id, item_type) do update
|
||||
return r.executeSQL(query)
|
||||
}
|
||||
|
||||
func (r *albumRepository) purgeEmpty() error {
|
||||
func (r *albumRepository) purgeEmpty(libraryIDs ...int) error {
|
||||
del := Delete(r.tableName).Where("id not in (select distinct(album_id) from media_file)")
|
||||
// If libraryIDs are specified, only purge albums from those libraries
|
||||
if len(libraryIDs) > 0 {
|
||||
del = del.Where(Eq{"library_id": libraryIDs})
|
||||
}
|
||||
c, err := r.executeSQL(del)
|
||||
if err != nil {
|
||||
return fmt.Errorf("purging empty albums: %w", err)
|
||||
|
||||
@ -141,6 +141,7 @@ func NewArtistRepository(ctx context.Context, db dbx.Builder) model.ArtistReposi
|
||||
r.setSortMappings(map[string]string{
|
||||
"name": "order_artist_name",
|
||||
"starred_at": "starred, starred_at",
|
||||
"rated_at": "rating, rated_at",
|
||||
"song_count": "stats->>'total'->>'m'",
|
||||
"album_count": "stats->>'total'->>'a'",
|
||||
"size": "stats->>'total'->>'s'",
|
||||
|
||||
@ -4,7 +4,10 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
. "github.com/Masterminds/squirrel"
|
||||
@ -91,8 +94,47 @@ func (r folderRepository) CountAll(opt ...model.QueryOptions) (int64, error) {
|
||||
return r.count(query)
|
||||
}
|
||||
|
||||
func (r folderRepository) GetLastUpdates(lib model.Library) (map[string]model.FolderUpdateInfo, error) {
|
||||
sq := r.newSelect().Columns("id", "updated_at", "hash").Where(Eq{"library_id": lib.ID, "missing": false})
|
||||
func (r folderRepository) GetFolderUpdateInfo(lib model.Library, targetPaths ...string) (map[string]model.FolderUpdateInfo, error) {
|
||||
where := And{
|
||||
Eq{"library_id": lib.ID},
|
||||
Eq{"missing": false},
|
||||
}
|
||||
|
||||
// If specific paths are requested, include those folders and all their descendants
|
||||
if len(targetPaths) > 0 {
|
||||
// Collect folder IDs for exact target folders and path conditions for descendants
|
||||
folderIDs := make([]string, 0, len(targetPaths))
|
||||
pathConditions := make(Or, 0, len(targetPaths)*2)
|
||||
|
||||
for _, targetPath := range targetPaths {
|
||||
if targetPath == "" || targetPath == "." {
|
||||
// Root path - include everything in this library
|
||||
pathConditions = Or{}
|
||||
folderIDs = nil
|
||||
break
|
||||
}
|
||||
// Clean the path to normalize it. Paths stored in the folder table do not have leading/trailing slashes.
|
||||
cleanPath := strings.TrimPrefix(targetPath, string(os.PathSeparator))
|
||||
cleanPath = filepath.Clean(cleanPath)
|
||||
|
||||
// Include the target folder itself by ID
|
||||
folderIDs = append(folderIDs, model.FolderID(lib, cleanPath))
|
||||
|
||||
// Include all descendants: folders whose path field equals or starts with the target path
|
||||
// Note: Folder.Path is the directory path, so children have path = targetPath
|
||||
pathConditions = append(pathConditions, Eq{"path": cleanPath})
|
||||
pathConditions = append(pathConditions, Like{"path": cleanPath + "/%"})
|
||||
}
|
||||
|
||||
// Combine conditions: exact folder IDs OR descendant path patterns
|
||||
if len(folderIDs) > 0 {
|
||||
where = append(where, Or{Eq{"id": folderIDs}, pathConditions})
|
||||
} else if len(pathConditions) > 0 {
|
||||
where = append(where, pathConditions)
|
||||
}
|
||||
}
|
||||
|
||||
sq := r.newSelect().Columns("id", "updated_at", "hash").Where(where)
|
||||
var res []struct {
|
||||
ID string
|
||||
UpdatedAt time.Time
|
||||
@ -149,7 +191,7 @@ func (r folderRepository) GetTouchedWithPlaylists() (model.FolderCursor, error)
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (r folderRepository) purgeEmpty() error {
|
||||
func (r folderRepository) purgeEmpty(libraryIDs ...int) error {
|
||||
sq := Delete(r.tableName).Where(And{
|
||||
Eq{"num_audio_files": 0},
|
||||
Eq{"num_playlists": 0},
|
||||
@ -157,6 +199,10 @@ func (r folderRepository) purgeEmpty() error {
|
||||
ConcatExpr("id not in (select parent_id from folder)"),
|
||||
ConcatExpr("id not in (select folder_id from media_file)"),
|
||||
})
|
||||
// If libraryIDs are specified, only purge folders from those libraries
|
||||
if len(libraryIDs) > 0 {
|
||||
sq = sq.Where(Eq{"library_id": libraryIDs})
|
||||
}
|
||||
c, err := r.executeSQL(sq)
|
||||
if err != nil {
|
||||
return fmt.Errorf("purging empty folders: %w", err)
|
||||
|
||||
213
persistence/folder_repository_test.go
Normal file
213
persistence/folder_repository_test.go
Normal file
@ -0,0 +1,213 @@
|
||||
package persistence
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/navidrome/navidrome/log"
|
||||
"github.com/navidrome/navidrome/model"
|
||||
"github.com/navidrome/navidrome/model/request"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
"github.com/pocketbase/dbx"
|
||||
)
|
||||
|
||||
var _ = Describe("FolderRepository", func() {
|
||||
var repo model.FolderRepository
|
||||
var ctx context.Context
|
||||
var conn *dbx.DB
|
||||
var testLib, otherLib model.Library
|
||||
|
||||
BeforeEach(func() {
|
||||
ctx = request.WithUser(log.NewContext(context.TODO()), model.User{ID: "userid"})
|
||||
conn = GetDBXBuilder()
|
||||
repo = newFolderRepository(ctx, conn)
|
||||
|
||||
// Use existing library ID 1 from test fixtures
|
||||
libRepo := NewLibraryRepository(ctx, conn)
|
||||
lib, err := libRepo.Get(1)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
testLib = *lib
|
||||
|
||||
// Create a second library with its own folder to verify isolation
|
||||
otherLib = model.Library{Name: "Other Library", Path: "/other/path"}
|
||||
Expect(libRepo.Put(&otherLib)).To(Succeed())
|
||||
})
|
||||
|
||||
AfterEach(func() {
|
||||
// Clean up only test folders created by our tests (paths starting with "Test")
|
||||
// This prevents interference with fixture data needed by other tests
|
||||
_, _ = conn.NewQuery("DELETE FROM folder WHERE library_id = 1 AND path LIKE 'Test%'").Execute()
|
||||
_, _ = conn.NewQuery(fmt.Sprintf("DELETE FROM library WHERE id = %d", otherLib.ID)).Execute()
|
||||
})
|
||||
|
||||
Describe("GetFolderUpdateInfo", func() {
|
||||
Context("with no target paths", func() {
|
||||
It("returns all folders in the library", func() {
|
||||
// Create test folders with unique names to avoid conflicts
|
||||
folder1 := model.NewFolder(testLib, "TestGetLastUpdates/Folder1")
|
||||
folder2 := model.NewFolder(testLib, "TestGetLastUpdates/Folder2")
|
||||
|
||||
err := repo.Put(folder1)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
err = repo.Put(folder2)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
otherFolder := model.NewFolder(otherLib, "TestOtherLib/Folder")
|
||||
err = repo.Put(otherFolder)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
// Query all folders (no target paths) - should only return folders from testLib
|
||||
results, err := repo.GetFolderUpdateInfo(testLib)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
// Should include folders from testLib
|
||||
Expect(results).To(HaveKey(folder1.ID))
|
||||
Expect(results).To(HaveKey(folder2.ID))
|
||||
// Should NOT include folders from other library
|
||||
Expect(results).ToNot(HaveKey(otherFolder.ID))
|
||||
})
|
||||
})
|
||||
|
||||
Context("with specific target paths", func() {
|
||||
It("returns folder info for existing folders", func() {
|
||||
// Create test folders with unique names
|
||||
folder1 := model.NewFolder(testLib, "TestSpecific/Rock")
|
||||
folder2 := model.NewFolder(testLib, "TestSpecific/Jazz")
|
||||
folder3 := model.NewFolder(testLib, "TestSpecific/Classical")
|
||||
|
||||
err := repo.Put(folder1)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
err = repo.Put(folder2)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
err = repo.Put(folder3)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
// Query specific paths
|
||||
results, err := repo.GetFolderUpdateInfo(testLib, "TestSpecific/Rock", "TestSpecific/Classical")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(results).To(HaveLen(2))
|
||||
|
||||
// Verify folder IDs are in results
|
||||
Expect(results).To(HaveKey(folder1.ID))
|
||||
Expect(results).To(HaveKey(folder3.ID))
|
||||
Expect(results).ToNot(HaveKey(folder2.ID))
|
||||
|
||||
// Verify update info is populated
|
||||
Expect(results[folder1.ID].UpdatedAt).ToNot(BeZero())
|
||||
Expect(results[folder1.ID].Hash).To(Equal(folder1.Hash))
|
||||
})
|
||||
|
||||
It("includes all child folders when querying parent", func() {
|
||||
// Create a parent folder with multiple children
|
||||
parent := model.NewFolder(testLib, "TestParent/Music")
|
||||
child1 := model.NewFolder(testLib, "TestParent/Music/Rock/Queen")
|
||||
child2 := model.NewFolder(testLib, "TestParent/Music/Jazz")
|
||||
otherParent := model.NewFolder(testLib, "TestParent2/Music/Jazz")
|
||||
|
||||
Expect(repo.Put(parent)).To(Succeed())
|
||||
Expect(repo.Put(child1)).To(Succeed())
|
||||
Expect(repo.Put(child2)).To(Succeed())
|
||||
|
||||
// Query the parent folder - should return parent and all children
|
||||
results, err := repo.GetFolderUpdateInfo(testLib, "TestParent/Music")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(results).To(HaveLen(3))
|
||||
Expect(results).To(HaveKey(parent.ID))
|
||||
Expect(results).To(HaveKey(child1.ID))
|
||||
Expect(results).To(HaveKey(child2.ID))
|
||||
Expect(results).ToNot(HaveKey(otherParent.ID))
|
||||
})
|
||||
|
||||
It("excludes children from other libraries", func() {
|
||||
// Create parent in testLib
|
||||
parent := model.NewFolder(testLib, "TestIsolation/Parent")
|
||||
child := model.NewFolder(testLib, "TestIsolation/Parent/Child")
|
||||
|
||||
Expect(repo.Put(parent)).To(Succeed())
|
||||
Expect(repo.Put(child)).To(Succeed())
|
||||
|
||||
// Create similar path in other library
|
||||
otherParent := model.NewFolder(otherLib, "TestIsolation/Parent")
|
||||
otherChild := model.NewFolder(otherLib, "TestIsolation/Parent/Child")
|
||||
|
||||
Expect(repo.Put(otherParent)).To(Succeed())
|
||||
Expect(repo.Put(otherChild)).To(Succeed())
|
||||
|
||||
// Query should only return folders from testLib
|
||||
results, err := repo.GetFolderUpdateInfo(testLib, "TestIsolation/Parent")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(results).To(HaveLen(2))
|
||||
Expect(results).To(HaveKey(parent.ID))
|
||||
Expect(results).To(HaveKey(child.ID))
|
||||
Expect(results).ToNot(HaveKey(otherParent.ID))
|
||||
Expect(results).ToNot(HaveKey(otherChild.ID))
|
||||
})
|
||||
|
||||
It("excludes missing children when querying parent", func() {
|
||||
// Create parent and children, mark one as missing
|
||||
parent := model.NewFolder(testLib, "TestMissingChild/Parent")
|
||||
child1 := model.NewFolder(testLib, "TestMissingChild/Parent/Child1")
|
||||
child2 := model.NewFolder(testLib, "TestMissingChild/Parent/Child2")
|
||||
child2.Missing = true
|
||||
|
||||
Expect(repo.Put(parent)).To(Succeed())
|
||||
Expect(repo.Put(child1)).To(Succeed())
|
||||
Expect(repo.Put(child2)).To(Succeed())
|
||||
|
||||
// Query parent - should only return parent and non-missing child
|
||||
results, err := repo.GetFolderUpdateInfo(testLib, "TestMissingChild/Parent")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(results).To(HaveLen(2))
|
||||
Expect(results).To(HaveKey(parent.ID))
|
||||
Expect(results).To(HaveKey(child1.ID))
|
||||
Expect(results).ToNot(HaveKey(child2.ID))
|
||||
})
|
||||
|
||||
It("handles mix of existing and non-existing target paths", func() {
|
||||
// Create folders for one path but not the other
|
||||
existingParent := model.NewFolder(testLib, "TestMixed/Exists")
|
||||
existingChild := model.NewFolder(testLib, "TestMixed/Exists/Child")
|
||||
|
||||
Expect(repo.Put(existingParent)).To(Succeed())
|
||||
Expect(repo.Put(existingChild)).To(Succeed())
|
||||
|
||||
// Query both existing and non-existing paths
|
||||
results, err := repo.GetFolderUpdateInfo(testLib, "TestMixed/Exists", "TestMixed/DoesNotExist")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(results).To(HaveLen(2))
|
||||
Expect(results).To(HaveKey(existingParent.ID))
|
||||
Expect(results).To(HaveKey(existingChild.ID))
|
||||
})
|
||||
|
||||
It("handles empty folder path as root", func() {
|
||||
// Test querying for root folder without creating it (fixtures should have one)
|
||||
rootFolderID := model.FolderID(testLib, ".")
|
||||
|
||||
results, err := repo.GetFolderUpdateInfo(testLib, "")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
// Should return the root folder if it exists
|
||||
if len(results) > 0 {
|
||||
Expect(results).To(HaveKey(rootFolderID))
|
||||
}
|
||||
})
|
||||
|
||||
It("returns empty map for non-existent folders", func() {
|
||||
results, err := repo.GetFolderUpdateInfo(testLib, "NonExistent/Path")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(results).To(BeEmpty())
|
||||
})
|
||||
|
||||
It("skips missing folders", func() {
|
||||
// Create a folder and mark it as missing
|
||||
folder := model.NewFolder(testLib, "TestMissing/Folder")
|
||||
folder.Missing = true
|
||||
err := repo.Put(folder)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
results, err := repo.GetFolderUpdateInfo(testLib, "TestMissing/Folder")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(results).To(BeEmpty())
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
@ -177,7 +177,11 @@ func (r *libraryRepository) ScanEnd(id int) error {
|
||||
return err
|
||||
}
|
||||
// https://www.sqlite.org/pragma.html#pragma_optimize
|
||||
_, err = r.executeSQL(Expr("PRAGMA optimize=0x10012;"))
|
||||
// Use mask 0x10000 to check table sizes without running ANALYZE
|
||||
// Running ANALYZE can cause query planner issues with expression-based collation indexes
|
||||
if conf.Server.DevOptimizeDB {
|
||||
_, err = r.executeSQL(Expr("PRAGMA optimize=0x10000;"))
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
|
||||
@ -142,4 +142,62 @@ var _ = Describe("LibraryRepository", func() {
|
||||
Expect(libAfter.TotalSize).To(Equal(sizeRes.Sum))
|
||||
Expect(libAfter.TotalDuration).To(Equal(durationRes.Sum))
|
||||
})
|
||||
|
||||
Describe("ScanBegin and ScanEnd", func() {
|
||||
var lib *model.Library
|
||||
|
||||
BeforeEach(func() {
|
||||
lib = &model.Library{
|
||||
ID: 0,
|
||||
Name: "Test Scan Library",
|
||||
Path: "/music/test-scan",
|
||||
}
|
||||
err := repo.Put(lib)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
|
||||
DescribeTable("ScanBegin",
|
||||
func(fullScan bool, expectedFullScanInProgress bool) {
|
||||
err := repo.ScanBegin(lib.ID, fullScan)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
updatedLib, err := repo.Get(lib.ID)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(updatedLib.LastScanStartedAt).ToNot(BeZero())
|
||||
Expect(updatedLib.FullScanInProgress).To(Equal(expectedFullScanInProgress))
|
||||
},
|
||||
Entry("sets FullScanInProgress to true for full scan", true, true),
|
||||
Entry("sets FullScanInProgress to false for quick scan", false, false),
|
||||
)
|
||||
|
||||
Context("ScanEnd", func() {
|
||||
BeforeEach(func() {
|
||||
err := repo.ScanBegin(lib.ID, true)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
|
||||
It("sets LastScanAt and clears FullScanInProgress and LastScanStartedAt", func() {
|
||||
err := repo.ScanEnd(lib.ID)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
updatedLib, err := repo.Get(lib.ID)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(updatedLib.LastScanAt).ToNot(BeZero())
|
||||
Expect(updatedLib.FullScanInProgress).To(BeFalse())
|
||||
Expect(updatedLib.LastScanStartedAt).To(BeZero())
|
||||
})
|
||||
|
||||
It("sets LastScanAt to be after LastScanStartedAt", func() {
|
||||
libBefore, err := repo.Get(lib.ID)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
err = repo.ScanEnd(lib.ID)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
libAfter, err := repo.Get(lib.ID)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(libAfter.LastScanAt).To(BeTemporally(">=", libBefore.LastScanStartedAt))
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@ -84,6 +84,7 @@ func NewMediaFileRepository(ctx context.Context, db dbx.Builder) model.MediaFile
|
||||
"created_at": "media_file.created_at",
|
||||
"recently_added": mediaFileRecentlyAddedSort(),
|
||||
"starred_at": "starred, starred_at",
|
||||
"rated_at": "rating, rated_at",
|
||||
})
|
||||
return r
|
||||
}
|
||||
|
||||
@ -157,7 +157,7 @@ func (s *SQLStore) WithTxImmediate(block func(tx model.DataStore) error, scope .
|
||||
}, scope...)
|
||||
}
|
||||
|
||||
func (s *SQLStore) GC(ctx context.Context) error {
|
||||
func (s *SQLStore) GC(ctx context.Context, libraryIDs ...int) error {
|
||||
trace := func(ctx context.Context, msg string, f func() error) func() error {
|
||||
return func() error {
|
||||
start := time.Now()
|
||||
@ -167,11 +167,17 @@ func (s *SQLStore) GC(ctx context.Context) error {
|
||||
}
|
||||
}
|
||||
|
||||
// If libraryIDs are provided, scope operations to those libraries where possible
|
||||
scoped := len(libraryIDs) > 0
|
||||
if scoped {
|
||||
log.Debug(ctx, "GC: Running selective garbage collection", "libraryIDs", libraryIDs)
|
||||
}
|
||||
|
||||
err := run.Sequentially(
|
||||
trace(ctx, "purge empty albums", func() error { return s.Album(ctx).(*albumRepository).purgeEmpty() }),
|
||||
trace(ctx, "purge empty albums", func() error { return s.Album(ctx).(*albumRepository).purgeEmpty(libraryIDs...) }),
|
||||
trace(ctx, "purge empty artists", func() error { return s.Artist(ctx).(*artistRepository).purgeEmpty() }),
|
||||
trace(ctx, "mark missing artists", func() error { return s.Artist(ctx).(*artistRepository).markMissing() }),
|
||||
trace(ctx, "purge empty folders", func() error { return s.Folder(ctx).(*folderRepository).purgeEmpty() }),
|
||||
trace(ctx, "purge empty folders", func() error { return s.Folder(ctx).(*folderRepository).purgeEmpty(libraryIDs...) }),
|
||||
trace(ctx, "clean album annotations", func() error { return s.Album(ctx).(*albumRepository).cleanAnnotations() }),
|
||||
trace(ctx, "clean artist annotations", func() error { return s.Artist(ctx).(*artistRepository).cleanAnnotations() }),
|
||||
trace(ctx, "clean media file annotations", func() error { return s.MediaFile(ctx).(*mediaFileRepository).cleanAnnotations() }),
|
||||
|
||||
@ -264,6 +264,11 @@ func (r *playlistRepository) refreshSmartPlaylist(pls *model.Playlist) bool {
|
||||
"annotation.item_id = media_file.id" +
|
||||
" AND annotation.item_type = 'media_file'" +
|
||||
" AND annotation.user_id = '" + usr.ID + "')")
|
||||
|
||||
// Only include media files from libraries the user has access to
|
||||
sq = r.applyLibraryFilter(sq, "media_file")
|
||||
|
||||
// Apply the criteria rules
|
||||
sq = r.addCriteria(sq, rules)
|
||||
insSql := Insert("playlist_tracks").Columns("id", "playlist_id", "media_file_id").Select(sq)
|
||||
_, err = r.executeSQL(insSql)
|
||||
@ -388,6 +393,7 @@ func (r *playlistRepository) loadTracks(sel SelectBuilder, id string) (model.Pla
|
||||
"coalesce(play_count, 0) as play_count",
|
||||
"play_date",
|
||||
"coalesce(rating, 0) as rating",
|
||||
"rated_at",
|
||||
"f.*",
|
||||
"playlist_tracks.*",
|
||||
"library.path as library_path",
|
||||
|
||||
@ -1,7 +1,6 @@
|
||||
package persistence
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/navidrome/navidrome/conf"
|
||||
@ -11,13 +10,14 @@ import (
|
||||
"github.com/navidrome/navidrome/model/request"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
"github.com/pocketbase/dbx"
|
||||
)
|
||||
|
||||
var _ = Describe("PlaylistRepository", func() {
|
||||
var repo model.PlaylistRepository
|
||||
|
||||
BeforeEach(func() {
|
||||
ctx := log.NewContext(context.TODO())
|
||||
ctx := log.NewContext(GinkgoT().Context())
|
||||
ctx = request.WithUser(ctx, model.User{ID: "userid", UserName: "userid", IsAdmin: true})
|
||||
repo = NewPlaylistRepository(ctx, GetDBXBuilder())
|
||||
})
|
||||
@ -252,4 +252,250 @@ var _ = Describe("PlaylistRepository", func() {
|
||||
Expect(tracks[3].MediaFileID).To(Equal("2001")) // Disc 2, Track 11
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Smart Playlists with Tag Criteria", func() {
|
||||
var mfRepo model.MediaFileRepository
|
||||
var testPlaylistID string
|
||||
var songWithGrouping, songWithoutGrouping model.MediaFile
|
||||
|
||||
BeforeEach(func() {
|
||||
ctx := log.NewContext(GinkgoT().Context())
|
||||
ctx = request.WithUser(ctx, model.User{ID: "userid", UserName: "userid", IsAdmin: true})
|
||||
mfRepo = NewMediaFileRepository(ctx, GetDBXBuilder())
|
||||
|
||||
// Register 'grouping' as a valid tag for smart playlists
|
||||
criteria.AddTagNames([]string{"grouping"})
|
||||
|
||||
// Create a song with the grouping tag
|
||||
songWithGrouping = model.MediaFile{
|
||||
ID: "test-grouping-1",
|
||||
Title: "Song With Grouping",
|
||||
Artist: "Test Artist",
|
||||
ArtistID: "1",
|
||||
Album: "Test Album",
|
||||
AlbumID: "101",
|
||||
Path: "/test/grouping/song1.mp3",
|
||||
Tags: model.Tags{
|
||||
"grouping": []string{"My Crate"},
|
||||
},
|
||||
Participants: model.Participants{},
|
||||
LibraryID: 1,
|
||||
Lyrics: "[]",
|
||||
}
|
||||
Expect(mfRepo.Put(&songWithGrouping)).To(Succeed())
|
||||
|
||||
// Create a song without the grouping tag
|
||||
songWithoutGrouping = model.MediaFile{
|
||||
ID: "test-grouping-2",
|
||||
Title: "Song Without Grouping",
|
||||
Artist: "Test Artist",
|
||||
ArtistID: "1",
|
||||
Album: "Test Album",
|
||||
AlbumID: "101",
|
||||
Path: "/test/grouping/song2.mp3",
|
||||
Tags: model.Tags{},
|
||||
Participants: model.Participants{},
|
||||
LibraryID: 1,
|
||||
Lyrics: "[]",
|
||||
}
|
||||
Expect(mfRepo.Put(&songWithoutGrouping)).To(Succeed())
|
||||
})
|
||||
|
||||
AfterEach(func() {
|
||||
if testPlaylistID != "" {
|
||||
_ = repo.Delete(testPlaylistID)
|
||||
testPlaylistID = ""
|
||||
}
|
||||
// Clean up test media files
|
||||
_, _ = GetDBXBuilder().Delete("media_file", dbx.HashExp{"id": "test-grouping-1"}).Execute()
|
||||
_, _ = GetDBXBuilder().Delete("media_file", dbx.HashExp{"id": "test-grouping-2"}).Execute()
|
||||
})
|
||||
|
||||
It("matches tracks with a tag value using 'contains' with empty string (issue #4728 workaround)", func() {
|
||||
By("creating a smart playlist that checks if grouping tag has any value")
|
||||
// This is the workaround for issue #4728: using 'contains' with empty string
|
||||
// generates SQL: value LIKE '%%' which matches any non-empty string
|
||||
rules := &criteria.Criteria{
|
||||
Expression: criteria.All{
|
||||
criteria.Contains{"grouping": ""},
|
||||
},
|
||||
}
|
||||
newPls := model.Playlist{Name: "Tracks with Grouping", OwnerID: "userid", Rules: rules}
|
||||
Expect(repo.Put(&newPls)).To(Succeed())
|
||||
testPlaylistID = newPls.ID
|
||||
|
||||
By("refreshing the smart playlist")
|
||||
conf.Server.SmartPlaylistRefreshDelay = -1 * time.Second // Force refresh
|
||||
pls, err := repo.GetWithTracks(newPls.ID, true, false)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
By("verifying only the track with grouping tag is matched")
|
||||
Expect(pls.Tracks).To(HaveLen(1))
|
||||
Expect(pls.Tracks[0].MediaFileID).To(Equal(songWithGrouping.ID))
|
||||
})
|
||||
|
||||
It("excludes tracks with a tag value using 'notContains' with empty string", func() {
|
||||
By("creating a smart playlist that checks if grouping tag is NOT set")
|
||||
rules := &criteria.Criteria{
|
||||
Expression: criteria.All{
|
||||
criteria.NotContains{"grouping": ""},
|
||||
},
|
||||
}
|
||||
newPls := model.Playlist{Name: "Tracks without Grouping", OwnerID: "userid", Rules: rules}
|
||||
Expect(repo.Put(&newPls)).To(Succeed())
|
||||
testPlaylistID = newPls.ID
|
||||
|
||||
By("refreshing the smart playlist")
|
||||
conf.Server.SmartPlaylistRefreshDelay = -1 * time.Second // Force refresh
|
||||
pls, err := repo.GetWithTracks(newPls.ID, true, false)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
By("verifying the track with grouping is NOT in the playlist")
|
||||
for _, track := range pls.Tracks {
|
||||
Expect(track.MediaFileID).ToNot(Equal(songWithGrouping.ID))
|
||||
}
|
||||
|
||||
By("verifying the track without grouping IS in the playlist")
|
||||
var foundWithoutGrouping bool
|
||||
for _, track := range pls.Tracks {
|
||||
if track.MediaFileID == songWithoutGrouping.ID {
|
||||
foundWithoutGrouping = true
|
||||
break
|
||||
}
|
||||
}
|
||||
Expect(foundWithoutGrouping).To(BeTrue())
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Smart Playlists Library Filtering", func() {
|
||||
var mfRepo model.MediaFileRepository
|
||||
var testPlaylistID string
|
||||
var lib2ID int
|
||||
var restrictedUserID string
|
||||
var uniqueLibPath string
|
||||
|
||||
BeforeEach(func() {
|
||||
db := GetDBXBuilder()
|
||||
|
||||
// Generate unique IDs for this test run
|
||||
uniqueSuffix := time.Now().Format("20060102150405.000")
|
||||
restrictedUserID = "restricted-user-" + uniqueSuffix
|
||||
uniqueLibPath = "/music/lib2-" + uniqueSuffix
|
||||
|
||||
// Create a second library with unique name and path to avoid conflicts with other tests
|
||||
_, err := db.DB().Exec("INSERT INTO library (name, path, created_at, updated_at) VALUES (?, ?, datetime('now'), datetime('now'))", "Library 2-"+uniqueSuffix, uniqueLibPath)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
err = db.DB().QueryRow("SELECT last_insert_rowid()").Scan(&lib2ID)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
// Create a restricted user with access only to library 1
|
||||
_, err = db.DB().Exec("INSERT INTO user (id, user_name, name, is_admin, password, created_at, updated_at) VALUES (?, ?, 'Restricted User', false, 'pass', datetime('now'), datetime('now'))", restrictedUserID, restrictedUserID)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
_, err = db.DB().Exec("INSERT INTO user_library (user_id, library_id) VALUES (?, 1)", restrictedUserID)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
// Create test media files in each library
|
||||
ctx := log.NewContext(GinkgoT().Context())
|
||||
ctx = request.WithUser(ctx, model.User{ID: "userid", UserName: "userid", IsAdmin: true})
|
||||
mfRepo = NewMediaFileRepository(ctx, db)
|
||||
|
||||
// Song in library 1 (accessible by restricted user)
|
||||
songLib1 := model.MediaFile{
|
||||
ID: "lib1-song",
|
||||
Title: "Song in Lib1",
|
||||
Artist: "Test Artist",
|
||||
ArtistID: "1",
|
||||
Album: "Test Album",
|
||||
AlbumID: "101",
|
||||
Path: "/music/lib1/song.mp3",
|
||||
LibraryID: 1,
|
||||
Participants: model.Participants{},
|
||||
Tags: model.Tags{},
|
||||
Lyrics: "[]",
|
||||
}
|
||||
Expect(mfRepo.Put(&songLib1)).To(Succeed())
|
||||
|
||||
// Song in library 2 (NOT accessible by restricted user)
|
||||
songLib2 := model.MediaFile{
|
||||
ID: "lib2-song",
|
||||
Title: "Song in Lib2",
|
||||
Artist: "Test Artist",
|
||||
ArtistID: "1",
|
||||
Album: "Test Album",
|
||||
AlbumID: "101",
|
||||
Path: uniqueLibPath + "/song.mp3",
|
||||
LibraryID: lib2ID,
|
||||
Participants: model.Participants{},
|
||||
Tags: model.Tags{},
|
||||
Lyrics: "[]",
|
||||
}
|
||||
Expect(mfRepo.Put(&songLib2)).To(Succeed())
|
||||
})
|
||||
|
||||
AfterEach(func() {
|
||||
db := GetDBXBuilder()
|
||||
if testPlaylistID != "" {
|
||||
_ = repo.Delete(testPlaylistID)
|
||||
testPlaylistID = ""
|
||||
}
|
||||
// Clean up test data
|
||||
_, _ = db.Delete("media_file", dbx.HashExp{"id": "lib1-song"}).Execute()
|
||||
_, _ = db.Delete("media_file", dbx.HashExp{"id": "lib2-song"}).Execute()
|
||||
_, _ = db.Delete("user_library", dbx.HashExp{"user_id": restrictedUserID}).Execute()
|
||||
_, _ = db.Delete("user", dbx.HashExp{"id": restrictedUserID}).Execute()
|
||||
_, _ = db.DB().Exec("DELETE FROM library WHERE id = ?", lib2ID)
|
||||
})
|
||||
|
||||
It("should only include tracks from libraries the user has access to (issue #4738)", func() {
|
||||
db := GetDBXBuilder()
|
||||
ctx := log.NewContext(GinkgoT().Context())
|
||||
|
||||
// Create the smart playlist as the restricted user
|
||||
restrictedUser := model.User{ID: restrictedUserID, UserName: restrictedUserID, IsAdmin: false}
|
||||
ctx = request.WithUser(ctx, restrictedUser)
|
||||
restrictedRepo := NewPlaylistRepository(ctx, db)
|
||||
|
||||
// Create a smart playlist that matches all songs
|
||||
rules := &criteria.Criteria{
|
||||
Expression: criteria.All{
|
||||
criteria.Gt{"playCount": -1}, // Matches everything
|
||||
},
|
||||
}
|
||||
newPls := model.Playlist{Name: "All Songs", OwnerID: restrictedUserID, Rules: rules}
|
||||
Expect(restrictedRepo.Put(&newPls)).To(Succeed())
|
||||
testPlaylistID = newPls.ID
|
||||
|
||||
By("refreshing the smart playlist")
|
||||
conf.Server.SmartPlaylistRefreshDelay = -1 * time.Second // Force refresh
|
||||
pls, err := restrictedRepo.GetWithTracks(newPls.ID, true, false)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
By("verifying only the track from library 1 is in the playlist")
|
||||
var foundLib1Song, foundLib2Song bool
|
||||
for _, track := range pls.Tracks {
|
||||
if track.MediaFileID == "lib1-song" {
|
||||
foundLib1Song = true
|
||||
}
|
||||
if track.MediaFileID == "lib2-song" {
|
||||
foundLib2Song = true
|
||||
}
|
||||
}
|
||||
Expect(foundLib1Song).To(BeTrue(), "Song from library 1 should be in the playlist")
|
||||
Expect(foundLib2Song).To(BeFalse(), "Song from library 2 should NOT be in the playlist")
|
||||
|
||||
By("verifying playlist_tracks table only contains the accessible track")
|
||||
var playlistTracksCount int
|
||||
err = db.DB().QueryRow("SELECT count(*) FROM playlist_tracks WHERE playlist_id = ?", newPls.ID).Scan(&playlistTracksCount)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
// Count should only include tracks visible to the user (lib1-song)
|
||||
// The count may include other test songs from library 1, but NOT lib2-song
|
||||
var lib2TrackCount int
|
||||
err = db.DB().QueryRow("SELECT count(*) FROM playlist_tracks WHERE playlist_id = ? AND media_file_id = 'lib2-song'", newPls.ID).Scan(&lib2TrackCount)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(lib2TrackCount).To(Equal(0), "lib2-song should not be in playlist_tracks")
|
||||
|
||||
By("verifying SongCount matches visible tracks")
|
||||
Expect(pls.SongCount).To(Equal(len(pls.Tracks)), "SongCount should match the number of visible tracks")
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@ -97,6 +97,7 @@ func (r *playlistTrackRepository) Read(id string) (interface{}, error) {
|
||||
"coalesce(rating, 0) as rating",
|
||||
"starred_at",
|
||||
"play_date",
|
||||
"rated_at",
|
||||
"f.*",
|
||||
"playlist_tracks.*",
|
||||
).
|
||||
|
||||
@ -28,6 +28,7 @@ func (r sqlRepository) withAnnotation(query SelectBuilder, idField string) Selec
|
||||
"coalesce(rating, 0) as rating",
|
||||
"starred_at",
|
||||
"play_date",
|
||||
"rated_at",
|
||||
)
|
||||
if conf.Server.AlbumPlayCountMode == consts.AlbumPlayCountModeNormalized && r.tableName == "album" {
|
||||
query = query.Columns(
|
||||
@ -77,7 +78,8 @@ func (r sqlRepository) SetStar(starred bool, ids ...string) error {
|
||||
}
|
||||
|
||||
func (r sqlRepository) SetRating(rating int, itemID string) error {
|
||||
return r.annUpsert(map[string]interface{}{"rating": rating}, itemID)
|
||||
ratedAt := time.Now()
|
||||
return r.annUpsert(map[string]interface{}{"rating": rating, "rated_at": ratedAt}, itemID)
|
||||
}
|
||||
|
||||
func (r sqlRepository) IncPlayCount(itemID string, ts time.Time) error {
|
||||
@ -119,7 +121,7 @@ func (r sqlRepository) cleanAnnotations() error {
|
||||
del := Delete(annotationTable).Where(Eq{"item_type": r.tableName}).Where("item_id not in (select id from " + r.tableName + ")")
|
||||
c, err := r.executeSQL(del)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error cleaning up annotations: %w", err)
|
||||
return fmt.Errorf("error cleaning up %s annotations: %w", r.tableName, err)
|
||||
}
|
||||
if c > 0 {
|
||||
log.Debug(r.ctx, "Clean-up annotations", "table", r.tableName, "totalDeleted", c)
|
||||
|
||||
@ -148,10 +148,10 @@ func (r sqlRepository) cleanBookmarks() error {
|
||||
del := Delete(bookmarkTable).Where(Eq{"item_type": r.tableName}).Where("item_id not in (select id from " + r.tableName + ")")
|
||||
c, err := r.executeSQL(del)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error cleaning up bookmarks: %w", err)
|
||||
return fmt.Errorf("error cleaning up %s bookmarks: %w", r.tableName, err)
|
||||
}
|
||||
if c > 0 {
|
||||
log.Debug(r.ctx, "Clean-up bookmarks", "totalDeleted", c)
|
||||
log.Debug(r.ctx, "Clean-up bookmarks", "totalDeleted", c, "itemType", r.tableName)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -2,6 +2,7 @@ package persistence
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/deluan/rest"
|
||||
"github.com/navidrome/navidrome/conf/configtest"
|
||||
@ -45,6 +46,9 @@ var _ = Describe("Tag Library Filtering", func() {
|
||||
BeforeEach(func() {
|
||||
DeferCleanup(configtest.SetupConfig())
|
||||
|
||||
// Generate unique path suffix to avoid conflicts with other tests
|
||||
uniqueSuffix := time.Now().Format("20060102150405.000")
|
||||
|
||||
// Clean up database
|
||||
db := GetDBXBuilder()
|
||||
_, err := db.NewQuery("DELETE FROM library_tag").Execute()
|
||||
@ -57,12 +61,12 @@ var _ = Describe("Tag Library Filtering", func() {
|
||||
_, err = db.NewQuery("DELETE FROM library WHERE id > 1").Execute()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
// Create test libraries
|
||||
// Create test libraries with unique names and paths to avoid conflicts with other tests
|
||||
_, err = db.NewQuery("INSERT INTO library (id, name, path) VALUES ({:id}, {:name}, {:path})").
|
||||
Bind(dbx.Params{"id": libraryID2, "name": "Library 2", "path": "/music/lib2"}).Execute()
|
||||
Bind(dbx.Params{"id": libraryID2, "name": "Library 2-" + uniqueSuffix, "path": "/music/lib2-" + uniqueSuffix}).Execute()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
_, err = db.NewQuery("INSERT INTO library (id, name, path) VALUES ({:id}, {:name}, {:path})").
|
||||
Bind(dbx.Params{"id": libraryID3, "name": "Library 3", "path": "/music/lib3"}).Execute()
|
||||
Bind(dbx.Params{"id": libraryID3, "name": "Library 3-" + uniqueSuffix, "path": "/music/lib3-" + uniqueSuffix}).Execute()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
// Give admin access to all libraries
|
||||
|
||||
@ -88,10 +88,10 @@ func (r *tagRepository) purgeUnused() error {
|
||||
`)
|
||||
c, err := r.executeSQL(del)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error purging unused tags: %w", err)
|
||||
return fmt.Errorf("error purging %s unused tags: %w", r.tableName, err)
|
||||
}
|
||||
if c > 0 {
|
||||
log.Debug(r.ctx, "Purged unused tags", "totalDeleted", c)
|
||||
log.Debug(r.ctx, "Purged unused tags", "totalDeleted", c, "table", r.tableName)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
@ -93,8 +93,12 @@ func (s *subsonicAPIServiceImpl) Call(ctx context.Context, req *subsonicapi.Call
|
||||
RawQuery: query.Encode(),
|
||||
}
|
||||
|
||||
// Create HTTP request with internal authentication
|
||||
httpReq, err := http.NewRequestWithContext(ctx, "GET", finalURL.String(), nil)
|
||||
// Create HTTP request with a fresh context to avoid Chi RouteContext pollution.
|
||||
// Using http.NewRequest (instead of http.NewRequestWithContext) ensures the internal
|
||||
// SubsonicAPI call doesn't inherit routing information from the parent handler,
|
||||
// which would cause Chi to invoke the wrong handler. Authentication context is
|
||||
// explicitly added in the next step via request.WithInternalAuth.
|
||||
httpReq, err := http.NewRequest("GET", finalURL.String(), nil)
|
||||
if err != nil {
|
||||
return &subsonicapi.CallResponse{
|
||||
Error: fmt.Sprintf("failed to create HTTP request: %v", err),
|
||||
|
||||
@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/navidrome/navidrome/conf"
|
||||
"github.com/navidrome/navidrome/core/agents"
|
||||
@ -22,8 +23,11 @@ var _ = Describe("Plugin Manager", func() {
|
||||
// but, as this is an integration test, we can't use configtest.SetupConfig() as it causes
|
||||
// data races.
|
||||
originalPluginsFolder := conf.Server.Plugins.Folder
|
||||
originalTimeout := conf.Server.DevPluginCompilationTimeout
|
||||
conf.Server.DevPluginCompilationTimeout = 2 * time.Minute
|
||||
DeferCleanup(func() {
|
||||
conf.Server.Plugins.Folder = originalPluginsFolder
|
||||
conf.Server.DevPluginCompilationTimeout = originalTimeout
|
||||
})
|
||||
conf.Server.Plugins.Enabled = true
|
||||
conf.Server.Plugins.Folder = testDataDir
|
||||
|
||||
@ -300,6 +300,8 @@
|
||||
},
|
||||
"actions": {
|
||||
"scan": "Scanear Biblioteca",
|
||||
"quickScan": "Scan Rápido",
|
||||
"fullScan": "Scan Completo",
|
||||
"manageUsers": "Gerenciar Acesso do Usuário",
|
||||
"viewDetails": "Ver Detalhes"
|
||||
},
|
||||
@ -308,6 +310,9 @@
|
||||
"updated": "Biblioteca atualizada com sucesso",
|
||||
"deleted": "Biblioteca excluída com sucesso",
|
||||
"scanStarted": "Scan da biblioteca iniciada",
|
||||
"quickScanStarted": "Scan rápido iniciado",
|
||||
"fullScanStarted": "Scan completo iniciado",
|
||||
"scanError": "Erro ao iniciar o scan. Verifique os logs",
|
||||
"scanCompleted": "Scan da biblioteca concluída"
|
||||
},
|
||||
"validation": {
|
||||
@ -598,11 +603,12 @@
|
||||
"activity": {
|
||||
"title": "Atividade",
|
||||
"totalScanned": "Total de pastas scaneadas",
|
||||
"quickScan": "Scan rápido",
|
||||
"fullScan": "Scan completo",
|
||||
"quickScan": "Rápido",
|
||||
"fullScan": "Completo",
|
||||
"selectiveScan": "Seletivo",
|
||||
"serverUptime": "Uptime do servidor",
|
||||
"serverDown": "DESCONECTADO",
|
||||
"scanType": "Tipo",
|
||||
"scanType": "Último Scan",
|
||||
"status": "Erro",
|
||||
"elapsedTime": "Duração"
|
||||
},
|
||||
|
||||
@ -26,24 +26,8 @@ var (
|
||||
ErrAlreadyScanning = errors.New("already scanning")
|
||||
)
|
||||
|
||||
type Scanner interface {
|
||||
// ScanAll starts a full scan of the music library. This is a blocking operation.
|
||||
ScanAll(ctx context.Context, fullScan bool) (warnings []string, err error)
|
||||
Status(context.Context) (*StatusInfo, error)
|
||||
}
|
||||
|
||||
type StatusInfo struct {
|
||||
Scanning bool
|
||||
LastScan time.Time
|
||||
Count uint32
|
||||
FolderCount uint32
|
||||
LastError string
|
||||
ScanType string
|
||||
ElapsedTime time.Duration
|
||||
}
|
||||
|
||||
func New(rootCtx context.Context, ds model.DataStore, cw artwork.CacheWarmer, broker events.Broker,
|
||||
pls core.Playlists, m metrics.Metrics) Scanner {
|
||||
pls core.Playlists, m metrics.Metrics) model.Scanner {
|
||||
c := &controller{
|
||||
rootCtx: rootCtx,
|
||||
ds: ds,
|
||||
@ -65,9 +49,10 @@ func (s *controller) getScanner() scanner {
|
||||
return &scannerImpl{ds: s.ds, cw: s.cw, pls: s.pls}
|
||||
}
|
||||
|
||||
// CallScan starts an in-process scan of the music library.
|
||||
// CallScan starts an in-process scan of specific library/folder pairs.
|
||||
// If targets is empty, it scans all libraries.
|
||||
// This is meant to be called from the command line (see cmd/scan.go).
|
||||
func CallScan(ctx context.Context, ds model.DataStore, pls core.Playlists, fullScan bool) (<-chan *ProgressInfo, error) {
|
||||
func CallScan(ctx context.Context, ds model.DataStore, pls core.Playlists, fullScan bool, targets []model.ScanTarget) (<-chan *ProgressInfo, error) {
|
||||
release, err := lockScan(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@ -79,7 +64,7 @@ func CallScan(ctx context.Context, ds model.DataStore, pls core.Playlists, fullS
|
||||
go func() {
|
||||
defer close(progress)
|
||||
scanner := &scannerImpl{ds: ds, cw: artwork.NoopCacheWarmer(), pls: pls}
|
||||
scanner.scanAll(ctx, fullScan, progress)
|
||||
scanner.scanFolders(ctx, fullScan, targets, progress)
|
||||
}()
|
||||
return progress, nil
|
||||
}
|
||||
@ -99,8 +84,11 @@ type ProgressInfo struct {
|
||||
ForceUpdate bool
|
||||
}
|
||||
|
||||
// scanner defines the interface for different scanner implementations.
|
||||
// This allows for swapping between in-process and external scanners.
|
||||
type scanner interface {
|
||||
scanAll(ctx context.Context, fullScan bool, progress chan<- *ProgressInfo)
|
||||
// scanFolders performs the actual scanning of folders. If targets is nil, it scans all libraries.
|
||||
scanFolders(ctx context.Context, fullScan bool, targets []model.ScanTarget, progress chan<- *ProgressInfo)
|
||||
}
|
||||
|
||||
type controller struct {
|
||||
@ -158,7 +146,7 @@ func (s *controller) getScanInfo(ctx context.Context) (scanType string, elapsed
|
||||
return scanType, elapsed, lastErr
|
||||
}
|
||||
|
||||
func (s *controller) Status(ctx context.Context) (*StatusInfo, error) {
|
||||
func (s *controller) Status(ctx context.Context) (*model.ScannerStatus, error) {
|
||||
lastScanTime, err := s.getLastScanTime(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("getting last scan time: %w", err)
|
||||
@ -167,7 +155,7 @@ func (s *controller) Status(ctx context.Context) (*StatusInfo, error) {
|
||||
scanType, elapsed, lastErr := s.getScanInfo(ctx)
|
||||
|
||||
if running.Load() {
|
||||
status := &StatusInfo{
|
||||
status := &model.ScannerStatus{
|
||||
Scanning: true,
|
||||
LastScan: lastScanTime,
|
||||
Count: s.count.Load(),
|
||||
@ -183,7 +171,7 @@ func (s *controller) Status(ctx context.Context) (*StatusInfo, error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("getting library stats: %w", err)
|
||||
}
|
||||
return &StatusInfo{
|
||||
return &model.ScannerStatus{
|
||||
Scanning: false,
|
||||
LastScan: lastScanTime,
|
||||
Count: uint32(count),
|
||||
@ -208,6 +196,10 @@ func (s *controller) getCounters(ctx context.Context) (int64, int64, error) {
|
||||
}
|
||||
|
||||
func (s *controller) ScanAll(requestCtx context.Context, fullScan bool) ([]string, error) {
|
||||
return s.ScanFolders(requestCtx, fullScan, nil)
|
||||
}
|
||||
|
||||
func (s *controller) ScanFolders(requestCtx context.Context, fullScan bool, targets []model.ScanTarget) ([]string, error) {
|
||||
release, err := lockScan(requestCtx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@ -224,7 +216,7 @@ func (s *controller) ScanAll(requestCtx context.Context, fullScan bool) ([]strin
|
||||
go func() {
|
||||
defer close(progress)
|
||||
scanner := s.getScanner()
|
||||
scanner.scanAll(ctx, fullScan, progress)
|
||||
scanner.scanFolders(ctx, fullScan, targets, progress)
|
||||
}()
|
||||
|
||||
// Wait for the scan to finish, sending progress events to all connected clients
|
||||
|
||||
@ -9,6 +9,7 @@ import (
|
||||
"github.com/navidrome/navidrome/core/artwork"
|
||||
"github.com/navidrome/navidrome/core/metrics"
|
||||
"github.com/navidrome/navidrome/db"
|
||||
"github.com/navidrome/navidrome/model"
|
||||
"github.com/navidrome/navidrome/persistence"
|
||||
"github.com/navidrome/navidrome/scanner"
|
||||
"github.com/navidrome/navidrome/server/events"
|
||||
@ -20,7 +21,7 @@ import (
|
||||
var _ = Describe("Controller", func() {
|
||||
var ctx context.Context
|
||||
var ds *tests.MockDataStore
|
||||
var ctrl scanner.Scanner
|
||||
var ctrl model.Scanner
|
||||
|
||||
Describe("Status", func() {
|
||||
BeforeEach(func() {
|
||||
|
||||
@ -11,7 +11,7 @@ import (
|
||||
|
||||
"github.com/navidrome/navidrome/conf"
|
||||
"github.com/navidrome/navidrome/log"
|
||||
. "github.com/navidrome/navidrome/utils/gg"
|
||||
"github.com/navidrome/navidrome/model"
|
||||
)
|
||||
|
||||
// scannerExternal is a scanner that runs an external process to do the scanning. It is used to avoid
|
||||
@ -23,19 +23,42 @@ import (
|
||||
// process will forward them to the caller.
|
||||
type scannerExternal struct{}
|
||||
|
||||
func (s *scannerExternal) scanAll(ctx context.Context, fullScan bool, progress chan<- *ProgressInfo) {
|
||||
func (s *scannerExternal) scanFolders(ctx context.Context, fullScan bool, targets []model.ScanTarget, progress chan<- *ProgressInfo) {
|
||||
s.scan(ctx, fullScan, targets, progress)
|
||||
}
|
||||
|
||||
func (s *scannerExternal) scan(ctx context.Context, fullScan bool, targets []model.ScanTarget, progress chan<- *ProgressInfo) {
|
||||
exe, err := os.Executable()
|
||||
if err != nil {
|
||||
progress <- &ProgressInfo{Error: fmt.Sprintf("failed to get executable path: %s", err)}
|
||||
return
|
||||
}
|
||||
log.Debug(ctx, "Spawning external scanner process", "fullScan", fullScan, "path", exe)
|
||||
cmd := exec.CommandContext(ctx, exe, "scan",
|
||||
|
||||
// Build command arguments
|
||||
args := []string{
|
||||
"scan",
|
||||
"--nobanner", "--subprocess",
|
||||
"--configfile", conf.Server.ConfigFile,
|
||||
"--datafolder", conf.Server.DataFolder,
|
||||
"--cachefolder", conf.Server.CacheFolder,
|
||||
If(fullScan, "--full", ""))
|
||||
}
|
||||
|
||||
// Add targets if provided
|
||||
if len(targets) > 0 {
|
||||
for _, target := range targets {
|
||||
args = append(args, "-t", target.String())
|
||||
}
|
||||
log.Debug(ctx, "Spawning external scanner process with targets", "fullScan", fullScan, "path", exe, "targets", targets)
|
||||
} else {
|
||||
log.Debug(ctx, "Spawning external scanner process", "fullScan", fullScan, "path", exe)
|
||||
}
|
||||
|
||||
// Add full scan flag if needed
|
||||
if fullScan {
|
||||
args = append(args, "--full")
|
||||
}
|
||||
|
||||
cmd := exec.CommandContext(ctx, exe, args...)
|
||||
|
||||
in, out := io.Pipe()
|
||||
defer in.Close()
|
||||
|
||||
@ -15,9 +15,7 @@ import (
|
||||
"github.com/navidrome/navidrome/utils/chrono"
|
||||
)
|
||||
|
||||
func newFolderEntry(job *scanJob, path string) *folderEntry {
|
||||
id := model.FolderID(job.lib, path)
|
||||
info := job.popLastUpdate(id)
|
||||
func newFolderEntry(job *scanJob, id, path string, updTime time.Time, hash string) *folderEntry {
|
||||
f := &folderEntry{
|
||||
id: id,
|
||||
job: job,
|
||||
@ -25,8 +23,8 @@ func newFolderEntry(job *scanJob, path string) *folderEntry {
|
||||
audioFiles: make(map[string]fs.DirEntry),
|
||||
imageFiles: make(map[string]fs.DirEntry),
|
||||
albumIDMap: make(map[string]string),
|
||||
updTime: info.UpdatedAt,
|
||||
prevHash: info.Hash,
|
||||
updTime: updTime,
|
||||
prevHash: hash,
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
@ -40,9 +40,8 @@ var _ = Describe("folder_entry", func() {
|
||||
UpdatedAt: time.Now().Add(-30 * time.Minute),
|
||||
Hash: "previous-hash",
|
||||
}
|
||||
job.lastUpdates[folderID] = updateInfo
|
||||
|
||||
entry := newFolderEntry(job, path)
|
||||
entry := newFolderEntry(job, folderID, path, updateInfo.UpdatedAt, updateInfo.Hash)
|
||||
|
||||
Expect(entry.id).To(Equal(folderID))
|
||||
Expect(entry.job).To(Equal(job))
|
||||
@ -53,15 +52,10 @@ var _ = Describe("folder_entry", func() {
|
||||
Expect(entry.updTime).To(Equal(updateInfo.UpdatedAt))
|
||||
Expect(entry.prevHash).To(Equal(updateInfo.Hash))
|
||||
})
|
||||
})
|
||||
|
||||
It("creates a new folder entry with zero time when no previous update exists", func() {
|
||||
entry := newFolderEntry(job, path)
|
||||
|
||||
Expect(entry.updTime).To(BeZero())
|
||||
Expect(entry.prevHash).To(BeEmpty())
|
||||
})
|
||||
|
||||
It("removes the lastUpdate from the job after popping", func() {
|
||||
Describe("createFolderEntry", func() {
|
||||
It("removes the lastUpdate from the job after creation", func() {
|
||||
folderID := model.FolderID(lib, path)
|
||||
updateInfo := model.FolderUpdateInfo{
|
||||
UpdatedAt: time.Now().Add(-30 * time.Minute),
|
||||
@ -69,8 +63,10 @@ var _ = Describe("folder_entry", func() {
|
||||
}
|
||||
job.lastUpdates[folderID] = updateInfo
|
||||
|
||||
newFolderEntry(job, path)
|
||||
entry := job.createFolderEntry(path)
|
||||
|
||||
Expect(entry.updTime).To(Equal(updateInfo.UpdatedAt))
|
||||
Expect(entry.prevHash).To(Equal(updateInfo.Hash))
|
||||
Expect(job.lastUpdates).ToNot(HaveKey(folderID))
|
||||
})
|
||||
})
|
||||
@ -79,7 +75,8 @@ var _ = Describe("folder_entry", func() {
|
||||
var entry *folderEntry
|
||||
|
||||
BeforeEach(func() {
|
||||
entry = newFolderEntry(job, path)
|
||||
folderID := model.FolderID(lib, path)
|
||||
entry = newFolderEntry(job, folderID, path, time.Time{}, "")
|
||||
})
|
||||
|
||||
Describe("hasNoFiles", func() {
|
||||
@ -458,7 +455,9 @@ var _ = Describe("folder_entry", func() {
|
||||
Describe("integration scenarios", func() {
|
||||
It("handles complete folder lifecycle", func() {
|
||||
// Create new folder entry
|
||||
entry := newFolderEntry(job, "music/rock/album")
|
||||
folderPath := "music/rock/album"
|
||||
folderID := model.FolderID(lib, folderPath)
|
||||
entry := newFolderEntry(job, folderID, folderPath, time.Time{}, "")
|
||||
|
||||
// Initially new and has no files
|
||||
Expect(entry.isNew()).To(BeTrue())
|
||||
|
||||
163
scanner/ignore_checker.go
Normal file
163
scanner/ignore_checker.go
Normal file
@ -0,0 +1,163 @@
|
||||
package scanner
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"io/fs"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"github.com/navidrome/navidrome/consts"
|
||||
"github.com/navidrome/navidrome/log"
|
||||
ignore "github.com/sabhiram/go-gitignore"
|
||||
)
|
||||
|
||||
// IgnoreChecker manages .ndignore patterns using a stack-based approach.
|
||||
// Use Push() to add patterns when entering a folder, Pop() when leaving,
|
||||
// and ShouldIgnore() to check if a path should be ignored.
|
||||
type IgnoreChecker struct {
|
||||
fsys fs.FS
|
||||
patternStack [][]string // Stack of patterns for each folder level
|
||||
currentPatterns []string // Flattened current patterns
|
||||
matcher *ignore.GitIgnore // Compiled matcher for current patterns
|
||||
}
|
||||
|
||||
// newIgnoreChecker creates a new IgnoreChecker for the given filesystem.
|
||||
func newIgnoreChecker(fsys fs.FS) *IgnoreChecker {
|
||||
return &IgnoreChecker{
|
||||
fsys: fsys,
|
||||
patternStack: make([][]string, 0),
|
||||
}
|
||||
}
|
||||
|
||||
// Push loads .ndignore patterns from the specified folder and adds them to the pattern stack.
|
||||
// Use this when entering a folder during directory tree traversal.
|
||||
func (ic *IgnoreChecker) Push(ctx context.Context, folder string) error {
|
||||
patterns := ic.loadPatternsFromFolder(ctx, folder)
|
||||
ic.patternStack = append(ic.patternStack, patterns)
|
||||
ic.rebuildCurrentPatterns()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Pop removes the most recent patterns from the stack.
|
||||
// Use this when leaving a folder during directory tree traversal.
|
||||
func (ic *IgnoreChecker) Pop() {
|
||||
if len(ic.patternStack) > 0 {
|
||||
ic.patternStack = ic.patternStack[:len(ic.patternStack)-1]
|
||||
ic.rebuildCurrentPatterns()
|
||||
}
|
||||
}
|
||||
|
||||
// PushAllParents pushes patterns from root down to the target path.
|
||||
// This is a convenience method for when you need to check a specific path
|
||||
// without recursively walking the tree. It handles the common pattern of
|
||||
// pushing all parent directories from root to the target.
|
||||
// This method is optimized to compile patterns only once at the end.
|
||||
func (ic *IgnoreChecker) PushAllParents(ctx context.Context, targetPath string) error {
|
||||
if targetPath == "." || targetPath == "" {
|
||||
// Simple case: just push root
|
||||
return ic.Push(ctx, ".")
|
||||
}
|
||||
|
||||
// Load patterns for root
|
||||
patterns := ic.loadPatternsFromFolder(ctx, ".")
|
||||
ic.patternStack = append(ic.patternStack, patterns)
|
||||
|
||||
// Load patterns for each parent directory
|
||||
currentPath := "."
|
||||
parts := strings.Split(path.Clean(targetPath), "/")
|
||||
for _, part := range parts {
|
||||
if part == "." || part == "" {
|
||||
continue
|
||||
}
|
||||
currentPath = path.Join(currentPath, part)
|
||||
patterns = ic.loadPatternsFromFolder(ctx, currentPath)
|
||||
ic.patternStack = append(ic.patternStack, patterns)
|
||||
}
|
||||
|
||||
// Rebuild and compile patterns only once at the end
|
||||
ic.rebuildCurrentPatterns()
|
||||
return nil
|
||||
}
|
||||
|
||||
// ShouldIgnore checks if the given path should be ignored based on the current patterns.
|
||||
// Returns true if the path matches any ignore pattern, false otherwise.
|
||||
func (ic *IgnoreChecker) ShouldIgnore(ctx context.Context, relPath string) bool {
|
||||
// Handle root/empty path - never ignore
|
||||
if relPath == "" || relPath == "." {
|
||||
return false
|
||||
}
|
||||
|
||||
// If no patterns loaded, nothing to ignore
|
||||
if ic.matcher == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
matches := ic.matcher.MatchesPath(relPath)
|
||||
if matches {
|
||||
log.Trace(ctx, "Scanner: Ignoring entry matching .ndignore", "path", relPath)
|
||||
}
|
||||
return matches
|
||||
}
|
||||
|
||||
// loadPatternsFromFolder reads the .ndignore file in the specified folder and returns the patterns.
|
||||
// If the file doesn't exist, returns an empty slice.
|
||||
// If the file exists but is empty, returns a pattern to ignore everything ("**/*").
|
||||
func (ic *IgnoreChecker) loadPatternsFromFolder(ctx context.Context, folder string) []string {
|
||||
ignoreFilePath := path.Join(folder, consts.ScanIgnoreFile)
|
||||
var patterns []string
|
||||
|
||||
// Check if .ndignore file exists
|
||||
if _, err := fs.Stat(ic.fsys, ignoreFilePath); err != nil {
|
||||
// No .ndignore file in this folder
|
||||
return patterns
|
||||
}
|
||||
|
||||
// Read and parse the .ndignore file
|
||||
ignoreFile, err := ic.fsys.Open(ignoreFilePath)
|
||||
if err != nil {
|
||||
log.Warn(ctx, "Scanner: Error opening .ndignore file", "path", ignoreFilePath, err)
|
||||
return patterns
|
||||
}
|
||||
defer ignoreFile.Close()
|
||||
|
||||
lineScanner := bufio.NewScanner(ignoreFile)
|
||||
for lineScanner.Scan() {
|
||||
line := strings.TrimSpace(lineScanner.Text())
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue // Skip empty lines, whitespace-only lines, and comments
|
||||
}
|
||||
patterns = append(patterns, line)
|
||||
}
|
||||
|
||||
if err := lineScanner.Err(); err != nil {
|
||||
log.Warn(ctx, "Scanner: Error reading .ndignore file", "path", ignoreFilePath, err)
|
||||
return patterns
|
||||
}
|
||||
|
||||
// If the .ndignore file is empty, ignore everything
|
||||
if len(patterns) == 0 {
|
||||
log.Trace(ctx, "Scanner: .ndignore file is empty, ignoring everything", "path", folder)
|
||||
patterns = []string{"**/*"}
|
||||
}
|
||||
|
||||
return patterns
|
||||
}
|
||||
|
||||
// rebuildCurrentPatterns flattens the pattern stack into currentPatterns and recompiles the matcher.
|
||||
func (ic *IgnoreChecker) rebuildCurrentPatterns() {
|
||||
ic.currentPatterns = make([]string, 0)
|
||||
for _, patterns := range ic.patternStack {
|
||||
ic.currentPatterns = append(ic.currentPatterns, patterns...)
|
||||
}
|
||||
ic.compilePatterns()
|
||||
}
|
||||
|
||||
// compilePatterns compiles the current patterns into a GitIgnore matcher.
|
||||
func (ic *IgnoreChecker) compilePatterns() {
|
||||
if len(ic.currentPatterns) == 0 {
|
||||
ic.matcher = nil
|
||||
return
|
||||
}
|
||||
ic.matcher = ignore.CompileIgnoreLines(ic.currentPatterns...)
|
||||
}
|
||||
313
scanner/ignore_checker_test.go
Normal file
313
scanner/ignore_checker_test.go
Normal file
@ -0,0 +1,313 @@
|
||||
package scanner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing/fstest"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
var _ = Describe("IgnoreChecker", func() {
|
||||
Describe("loadPatternsFromFolder", func() {
|
||||
var ic *IgnoreChecker
|
||||
var ctx context.Context
|
||||
|
||||
BeforeEach(func() {
|
||||
ctx = context.Background()
|
||||
})
|
||||
|
||||
Context("when .ndignore file does not exist", func() {
|
||||
It("should return empty patterns", func() {
|
||||
fsys := fstest.MapFS{}
|
||||
ic = newIgnoreChecker(fsys)
|
||||
patterns := ic.loadPatternsFromFolder(ctx, ".")
|
||||
Expect(patterns).To(BeEmpty())
|
||||
})
|
||||
})
|
||||
|
||||
Context("when .ndignore file is empty", func() {
|
||||
It("should return wildcard to ignore everything", func() {
|
||||
fsys := fstest.MapFS{
|
||||
".ndignore": &fstest.MapFile{Data: []byte("")},
|
||||
}
|
||||
ic = newIgnoreChecker(fsys)
|
||||
patterns := ic.loadPatternsFromFolder(ctx, ".")
|
||||
Expect(patterns).To(Equal([]string{"**/*"}))
|
||||
})
|
||||
})
|
||||
|
||||
DescribeTable("parsing .ndignore content",
|
||||
func(content string, expectedPatterns []string) {
|
||||
fsys := fstest.MapFS{
|
||||
".ndignore": &fstest.MapFile{Data: []byte(content)},
|
||||
}
|
||||
ic = newIgnoreChecker(fsys)
|
||||
patterns := ic.loadPatternsFromFolder(ctx, ".")
|
||||
Expect(patterns).To(Equal(expectedPatterns))
|
||||
},
|
||||
Entry("single pattern", "*.txt", []string{"*.txt"}),
|
||||
Entry("multiple patterns", "*.txt\n*.log", []string{"*.txt", "*.log"}),
|
||||
Entry("with comments", "# comment\n*.txt\n# another\n*.log", []string{"*.txt", "*.log"}),
|
||||
Entry("with empty lines", "*.txt\n\n*.log\n\n", []string{"*.txt", "*.log"}),
|
||||
Entry("mixed content", "# header\n\n*.txt\n# middle\n*.log\n\n", []string{"*.txt", "*.log"}),
|
||||
Entry("only comments and empty lines", "# comment\n\n# another\n", []string{"**/*"}),
|
||||
Entry("trailing newline", "*.txt\n*.log\n", []string{"*.txt", "*.log"}),
|
||||
Entry("directory pattern", "temp/", []string{"temp/"}),
|
||||
Entry("wildcard pattern", "**/*.mp3", []string{"**/*.mp3"}),
|
||||
Entry("multiple wildcards", "**/*.mp3\n**/*.flac\n*.log", []string{"**/*.mp3", "**/*.flac", "*.log"}),
|
||||
Entry("negation pattern", "!important.txt", []string{"!important.txt"}),
|
||||
Entry("comment with hash not at start is pattern", "not#comment", []string{"not#comment"}),
|
||||
Entry("whitespace-only lines skipped", "*.txt\n \n*.log\n\t\n", []string{"*.txt", "*.log"}),
|
||||
Entry("patterns with whitespace trimmed", " *.txt \n\t*.log\t", []string{"*.txt", "*.log"}),
|
||||
)
|
||||
})
|
||||
|
||||
Describe("Push and Pop", func() {
|
||||
var ic *IgnoreChecker
|
||||
var fsys fstest.MapFS
|
||||
var ctx context.Context
|
||||
|
||||
BeforeEach(func() {
|
||||
ctx = context.Background()
|
||||
fsys = fstest.MapFS{
|
||||
".ndignore": &fstest.MapFile{Data: []byte("*.txt")},
|
||||
"folder1/.ndignore": &fstest.MapFile{Data: []byte("*.mp3")},
|
||||
"folder2/.ndignore": &fstest.MapFile{Data: []byte("*.flac")},
|
||||
}
|
||||
ic = newIgnoreChecker(fsys)
|
||||
})
|
||||
|
||||
Context("Push", func() {
|
||||
It("should add patterns to stack", func() {
|
||||
err := ic.Push(ctx, ".")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(len(ic.patternStack)).To(Equal(1))
|
||||
Expect(ic.currentPatterns).To(ContainElement("*.txt"))
|
||||
})
|
||||
|
||||
It("should compile matcher after push", func() {
|
||||
err := ic.Push(ctx, ".")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(ic.matcher).ToNot(BeNil())
|
||||
})
|
||||
|
||||
It("should accumulate patterns from multiple levels", func() {
|
||||
err := ic.Push(ctx, ".")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
err = ic.Push(ctx, "folder1")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(len(ic.patternStack)).To(Equal(2))
|
||||
Expect(ic.currentPatterns).To(ConsistOf("*.txt", "*.mp3"))
|
||||
})
|
||||
|
||||
It("should handle push when no .ndignore exists", func() {
|
||||
err := ic.Push(ctx, "nonexistent")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(len(ic.patternStack)).To(Equal(1))
|
||||
Expect(ic.currentPatterns).To(BeEmpty())
|
||||
})
|
||||
})
|
||||
|
||||
Context("Pop", func() {
|
||||
It("should remove most recent patterns", func() {
|
||||
err := ic.Push(ctx, ".")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
err = ic.Push(ctx, "folder1")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
ic.Pop()
|
||||
Expect(len(ic.patternStack)).To(Equal(1))
|
||||
Expect(ic.currentPatterns).To(Equal([]string{"*.txt"}))
|
||||
})
|
||||
|
||||
It("should handle Pop on empty stack gracefully", func() {
|
||||
Expect(func() { ic.Pop() }).ToNot(Panic())
|
||||
Expect(ic.patternStack).To(BeEmpty())
|
||||
})
|
||||
|
||||
It("should set matcher to nil when all patterns popped", func() {
|
||||
err := ic.Push(ctx, ".")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(ic.matcher).ToNot(BeNil())
|
||||
ic.Pop()
|
||||
Expect(ic.matcher).To(BeNil())
|
||||
})
|
||||
|
||||
It("should update matcher after pop", func() {
|
||||
err := ic.Push(ctx, ".")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
err = ic.Push(ctx, "folder1")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
matcher1 := ic.matcher
|
||||
ic.Pop()
|
||||
matcher2 := ic.matcher
|
||||
Expect(matcher1).ToNot(Equal(matcher2))
|
||||
})
|
||||
})
|
||||
|
||||
Context("multiple Push/Pop cycles", func() {
|
||||
It("should maintain correct state through cycles", func() {
|
||||
err := ic.Push(ctx, ".")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(ic.currentPatterns).To(Equal([]string{"*.txt"}))
|
||||
|
||||
err = ic.Push(ctx, "folder1")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(ic.currentPatterns).To(ConsistOf("*.txt", "*.mp3"))
|
||||
|
||||
ic.Pop()
|
||||
Expect(ic.currentPatterns).To(Equal([]string{"*.txt"}))
|
||||
|
||||
err = ic.Push(ctx, "folder2")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(ic.currentPatterns).To(ConsistOf("*.txt", "*.flac"))
|
||||
|
||||
ic.Pop()
|
||||
Expect(ic.currentPatterns).To(Equal([]string{"*.txt"}))
|
||||
|
||||
ic.Pop()
|
||||
Expect(ic.currentPatterns).To(BeEmpty())
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
Describe("PushAllParents", func() {
|
||||
var ic *IgnoreChecker
|
||||
var ctx context.Context
|
||||
|
||||
BeforeEach(func() {
|
||||
ctx = context.Background()
|
||||
fsys := fstest.MapFS{
|
||||
".ndignore": &fstest.MapFile{Data: []byte("root.txt")},
|
||||
"folder1/.ndignore": &fstest.MapFile{Data: []byte("level1.txt")},
|
||||
"folder1/folder2/.ndignore": &fstest.MapFile{Data: []byte("level2.txt")},
|
||||
"folder1/folder2/folder3/.ndignore": &fstest.MapFile{Data: []byte("level3.txt")},
|
||||
}
|
||||
ic = newIgnoreChecker(fsys)
|
||||
})
|
||||
|
||||
DescribeTable("loading parent patterns",
|
||||
func(targetPath string, expectedStackDepth int, expectedPatterns []string) {
|
||||
err := ic.PushAllParents(ctx, targetPath)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(len(ic.patternStack)).To(Equal(expectedStackDepth))
|
||||
Expect(ic.currentPatterns).To(ConsistOf(expectedPatterns))
|
||||
},
|
||||
Entry("root path", ".", 1, []string{"root.txt"}),
|
||||
Entry("empty path", "", 1, []string{"root.txt"}),
|
||||
Entry("single level", "folder1", 2, []string{"root.txt", "level1.txt"}),
|
||||
Entry("two levels", "folder1/folder2", 3, []string{"root.txt", "level1.txt", "level2.txt"}),
|
||||
Entry("three levels", "folder1/folder2/folder3", 4, []string{"root.txt", "level1.txt", "level2.txt", "level3.txt"}),
|
||||
)
|
||||
|
||||
It("should only compile patterns once at the end", func() {
|
||||
// This is more of a behavioral test - we verify the matcher is not nil after PushAllParents
|
||||
err := ic.PushAllParents(ctx, "folder1/folder2")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(ic.matcher).ToNot(BeNil())
|
||||
})
|
||||
|
||||
It("should handle paths with dot", func() {
|
||||
err := ic.PushAllParents(ctx, "./folder1")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(len(ic.patternStack)).To(Equal(2))
|
||||
})
|
||||
|
||||
Context("when some parent folders have no .ndignore", func() {
|
||||
BeforeEach(func() {
|
||||
fsys := fstest.MapFS{
|
||||
".ndignore": &fstest.MapFile{Data: []byte("root.txt")},
|
||||
"folder1/folder2/.ndignore": &fstest.MapFile{Data: []byte("level2.txt")},
|
||||
}
|
||||
ic = newIgnoreChecker(fsys)
|
||||
})
|
||||
|
||||
It("should still push all parent levels", func() {
|
||||
err := ic.PushAllParents(ctx, "folder1/folder2")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(len(ic.patternStack)).To(Equal(3)) // root, folder1 (empty), folder2
|
||||
Expect(ic.currentPatterns).To(ConsistOf("root.txt", "level2.txt"))
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
Describe("ShouldIgnore", func() {
|
||||
var ic *IgnoreChecker
|
||||
var ctx context.Context
|
||||
|
||||
BeforeEach(func() {
|
||||
ctx = context.Background()
|
||||
})
|
||||
|
||||
Context("with no patterns loaded", func() {
|
||||
It("should not ignore any path", func() {
|
||||
fsys := fstest.MapFS{}
|
||||
ic = newIgnoreChecker(fsys)
|
||||
Expect(ic.ShouldIgnore(ctx, "anything.txt")).To(BeFalse())
|
||||
Expect(ic.ShouldIgnore(ctx, "folder/file.mp3")).To(BeFalse())
|
||||
})
|
||||
})
|
||||
|
||||
Context("special paths", func() {
|
||||
BeforeEach(func() {
|
||||
fsys := fstest.MapFS{
|
||||
".ndignore": &fstest.MapFile{Data: []byte("**/*")},
|
||||
}
|
||||
ic = newIgnoreChecker(fsys)
|
||||
err := ic.Push(ctx, ".")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
|
||||
It("should never ignore root or empty paths", func() {
|
||||
Expect(ic.ShouldIgnore(ctx, "")).To(BeFalse())
|
||||
Expect(ic.ShouldIgnore(ctx, ".")).To(BeFalse())
|
||||
})
|
||||
|
||||
It("should ignore all other paths with wildcard", func() {
|
||||
Expect(ic.ShouldIgnore(ctx, "file.txt")).To(BeTrue())
|
||||
Expect(ic.ShouldIgnore(ctx, "folder/file.mp3")).To(BeTrue())
|
||||
})
|
||||
})
|
||||
|
||||
DescribeTable("pattern matching",
|
||||
func(pattern string, path string, shouldMatch bool) {
|
||||
fsys := fstest.MapFS{
|
||||
".ndignore": &fstest.MapFile{Data: []byte(pattern)},
|
||||
}
|
||||
ic = newIgnoreChecker(fsys)
|
||||
err := ic.Push(ctx, ".")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(ic.ShouldIgnore(ctx, path)).To(Equal(shouldMatch))
|
||||
},
|
||||
Entry("glob match", "*.txt", "file.txt", true),
|
||||
Entry("glob no match", "*.txt", "file.mp3", false),
|
||||
Entry("directory pattern match", "tmp/", "tmp/file.txt", true),
|
||||
Entry("directory pattern no match", "tmp/", "temporary/file.txt", false),
|
||||
Entry("nested glob match", "**/*.log", "deep/nested/file.log", true),
|
||||
Entry("nested glob no match", "**/*.log", "deep/nested/file.txt", false),
|
||||
Entry("specific file match", "ignore.me", "ignore.me", true),
|
||||
Entry("specific file no match", "ignore.me", "keep.me", false),
|
||||
Entry("wildcard all", "**/*", "any/path/file.txt", true),
|
||||
Entry("nested specific match", "temp/*", "temp/cache.db", true),
|
||||
Entry("nested specific no match", "temp/*", "temporary/cache.db", false),
|
||||
)
|
||||
|
||||
Context("with multiple patterns", func() {
|
||||
BeforeEach(func() {
|
||||
fsys := fstest.MapFS{
|
||||
".ndignore": &fstest.MapFile{Data: []byte("*.txt\n*.log\ntemp/")},
|
||||
}
|
||||
ic = newIgnoreChecker(fsys)
|
||||
err := ic.Push(ctx, ".")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
|
||||
It("should match any of the patterns", func() {
|
||||
Expect(ic.ShouldIgnore(ctx, "file.txt")).To(BeTrue())
|
||||
Expect(ic.ShouldIgnore(ctx, "debug.log")).To(BeTrue())
|
||||
Expect(ic.ShouldIgnore(ctx, "temp/cache")).To(BeTrue())
|
||||
Expect(ic.ShouldIgnore(ctx, "music.mp3")).To(BeFalse())
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
@ -26,58 +26,46 @@ import (
|
||||
"github.com/navidrome/navidrome/utils/slice"
|
||||
)
|
||||
|
||||
func createPhaseFolders(ctx context.Context, state *scanState, ds model.DataStore, cw artwork.CacheWarmer, libs []model.Library) *phaseFolders {
|
||||
func createPhaseFolders(ctx context.Context, state *scanState, ds model.DataStore, cw artwork.CacheWarmer) *phaseFolders {
|
||||
var jobs []*scanJob
|
||||
var updatedLibs []model.Library
|
||||
for _, lib := range libs {
|
||||
if lib.LastScanStartedAt.IsZero() {
|
||||
err := ds.Library(ctx).ScanBegin(lib.ID, state.fullScan)
|
||||
if err != nil {
|
||||
log.Error(ctx, "Scanner: Error updating last scan started at", "lib", lib.Name, err)
|
||||
state.sendWarning(err.Error())
|
||||
continue
|
||||
}
|
||||
// Reload library to get updated state
|
||||
l, err := ds.Library(ctx).Get(lib.ID)
|
||||
if err != nil {
|
||||
log.Error(ctx, "Scanner: Error reloading library", "lib", lib.Name, err)
|
||||
state.sendWarning(err.Error())
|
||||
continue
|
||||
}
|
||||
lib = *l
|
||||
} else {
|
||||
log.Debug(ctx, "Scanner: Resuming previous scan", "lib", lib.Name, "lastScanStartedAt", lib.LastScanStartedAt, "fullScan", lib.FullScanInProgress)
|
||||
|
||||
// Create scan jobs for all libraries
|
||||
for _, lib := range state.libraries {
|
||||
// Get target folders for this library if selective scan
|
||||
var targetFolders []string
|
||||
if state.isSelectiveScan() {
|
||||
targetFolders = state.targets[lib.ID]
|
||||
}
|
||||
job, err := newScanJob(ctx, ds, cw, lib, state.fullScan)
|
||||
|
||||
job, err := newScanJob(ctx, ds, cw, lib, state.fullScan, targetFolders)
|
||||
if err != nil {
|
||||
log.Error(ctx, "Scanner: Error creating scan context", "lib", lib.Name, err)
|
||||
state.sendWarning(err.Error())
|
||||
continue
|
||||
}
|
||||
jobs = append(jobs, job)
|
||||
updatedLibs = append(updatedLibs, lib)
|
||||
}
|
||||
|
||||
// Update the state with the libraries that have been processed and have their scan timestamps set
|
||||
state.libraries = updatedLibs
|
||||
|
||||
return &phaseFolders{jobs: jobs, ctx: ctx, ds: ds, state: state}
|
||||
}
|
||||
|
||||
type scanJob struct {
|
||||
lib model.Library
|
||||
fs storage.MusicFS
|
||||
cw artwork.CacheWarmer
|
||||
lastUpdates map[string]model.FolderUpdateInfo
|
||||
lock sync.Mutex
|
||||
numFolders atomic.Int64
|
||||
lib model.Library
|
||||
fs storage.MusicFS
|
||||
cw artwork.CacheWarmer
|
||||
lastUpdates map[string]model.FolderUpdateInfo // Holds last update info for all (DB) folders in this library
|
||||
targetFolders []string // Specific folders to scan (including all descendants)
|
||||
lock sync.Mutex
|
||||
numFolders atomic.Int64
|
||||
}
|
||||
|
||||
func newScanJob(ctx context.Context, ds model.DataStore, cw artwork.CacheWarmer, lib model.Library, fullScan bool) (*scanJob, error) {
|
||||
lastUpdates, err := ds.Folder(ctx).GetLastUpdates(lib)
|
||||
func newScanJob(ctx context.Context, ds model.DataStore, cw artwork.CacheWarmer, lib model.Library, fullScan bool, targetFolders []string) (*scanJob, error) {
|
||||
// Get folder updates, optionally filtered to specific target folders
|
||||
lastUpdates, err := ds.Folder(ctx).GetFolderUpdateInfo(lib, targetFolders...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("getting last updates: %w", err)
|
||||
}
|
||||
|
||||
fileStore, err := storage.For(lib.Path)
|
||||
if err != nil {
|
||||
log.Error(ctx, "Error getting storage for library", "library", lib.Name, "path", lib.Path, err)
|
||||
@ -88,15 +76,17 @@ func newScanJob(ctx context.Context, ds model.DataStore, cw artwork.CacheWarmer,
|
||||
log.Error(ctx, "Error getting fs for library", "library", lib.Name, "path", lib.Path, err)
|
||||
return nil, fmt.Errorf("getting fs for library: %w", err)
|
||||
}
|
||||
lib.FullScanInProgress = lib.FullScanInProgress || fullScan
|
||||
return &scanJob{
|
||||
lib: lib,
|
||||
fs: fsys,
|
||||
cw: cw,
|
||||
lastUpdates: lastUpdates,
|
||||
lib: lib,
|
||||
fs: fsys,
|
||||
cw: cw,
|
||||
lastUpdates: lastUpdates,
|
||||
targetFolders: targetFolders,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// popLastUpdate retrieves and removes the last update info for the given folder ID
|
||||
// This is used to track which folders have been found during the walk_dir_tree
|
||||
func (j *scanJob) popLastUpdate(folderID string) model.FolderUpdateInfo {
|
||||
j.lock.Lock()
|
||||
defer j.lock.Unlock()
|
||||
@ -106,6 +96,15 @@ func (j *scanJob) popLastUpdate(folderID string) model.FolderUpdateInfo {
|
||||
return lastUpdate
|
||||
}
|
||||
|
||||
// createFolderEntry creates a new folderEntry for the given path, using the last update info from the job
|
||||
// to populate the previous update time and hash. It also removes the folder from the job's lastUpdates map.
|
||||
// This is used to track which folders have been found during the walk_dir_tree.
|
||||
func (j *scanJob) createFolderEntry(path string) *folderEntry {
|
||||
id := model.FolderID(j.lib, path)
|
||||
info := j.popLastUpdate(id)
|
||||
return newFolderEntry(j, id, path, info.UpdatedAt, info.Hash)
|
||||
}
|
||||
|
||||
// phaseFolders represents the first phase of the scanning process, which is responsible
|
||||
// for scanning all libraries and importing new or updated files. This phase involves
|
||||
// traversing the directory tree of each library, identifying new or modified media files,
|
||||
@ -144,7 +143,8 @@ func (p *phaseFolders) producer() ppl.Producer[*folderEntry] {
|
||||
if utils.IsCtxDone(p.ctx) {
|
||||
break
|
||||
}
|
||||
outputChan, err := walkDirTree(p.ctx, job)
|
||||
|
||||
outputChan, err := walkDirTree(p.ctx, job, job.targetFolders...)
|
||||
if err != nil {
|
||||
log.Warn(p.ctx, "Scanner: Error scanning library", "lib", job.lib.Name, err)
|
||||
}
|
||||
@ -324,6 +324,9 @@ func (p *phaseFolders) persistChanges(entry *folderEntry) (*folderEntry, error)
|
||||
defer p.measure(entry)()
|
||||
p.state.changesDetected.Store(true)
|
||||
|
||||
// Collect artwork IDs to pre-cache after the transaction commits
|
||||
var artworkIDs []model.ArtworkID
|
||||
|
||||
err := p.ds.WithTx(func(tx model.DataStore) error {
|
||||
// Instantiate all repositories just once per folder
|
||||
folderRepo := tx.Folder(p.ctx)
|
||||
@ -362,7 +365,7 @@ func (p *phaseFolders) persistChanges(entry *folderEntry) (*folderEntry, error)
|
||||
return err
|
||||
}
|
||||
if entry.artists[i].Name != consts.UnknownArtist && entry.artists[i].Name != consts.VariousArtists {
|
||||
entry.job.cw.PreCache(entry.artists[i].CoverArtID())
|
||||
artworkIDs = append(artworkIDs, entry.artists[i].CoverArtID())
|
||||
}
|
||||
}
|
||||
|
||||
@ -374,7 +377,7 @@ func (p *phaseFolders) persistChanges(entry *folderEntry) (*folderEntry, error)
|
||||
return err
|
||||
}
|
||||
if entry.albums[i].Name != consts.UnknownAlbum {
|
||||
entry.job.cw.PreCache(entry.albums[i].CoverArtID())
|
||||
artworkIDs = append(artworkIDs, entry.albums[i].CoverArtID())
|
||||
}
|
||||
}
|
||||
|
||||
@ -411,6 +414,14 @@ func (p *phaseFolders) persistChanges(entry *folderEntry) (*folderEntry, error)
|
||||
if err != nil {
|
||||
log.Error(p.ctx, "Scanner: Error persisting changes to DB", "folder", entry.path, err)
|
||||
}
|
||||
|
||||
// Pre-cache artwork after the transaction commits successfully
|
||||
if err == nil {
|
||||
for _, artID := range artworkIDs {
|
||||
entry.job.cw.PreCache(artID)
|
||||
}
|
||||
}
|
||||
|
||||
return entry, err
|
||||
}
|
||||
|
||||
|
||||
@ -69,9 +69,6 @@ func (p *phaseMissingTracks) produce(put func(tracks *missingTracks)) error {
|
||||
}
|
||||
}
|
||||
for _, lib := range p.state.libraries {
|
||||
if lib.LastScanStartedAt.IsZero() {
|
||||
continue
|
||||
}
|
||||
log.Debug(p.ctx, "Scanner: Checking missing tracks", "libraryId", lib.ID, "libraryName", lib.Name)
|
||||
cursor, err := p.ds.MediaFile(p.ctx).GetMissingAndMatching(lib.ID)
|
||||
if err != nil {
|
||||
|
||||
@ -27,14 +27,13 @@ import (
|
||||
type phaseRefreshAlbums struct {
|
||||
ds model.DataStore
|
||||
ctx context.Context
|
||||
libs model.Libraries
|
||||
refreshed atomic.Uint32
|
||||
skipped atomic.Uint32
|
||||
state *scanState
|
||||
}
|
||||
|
||||
func createPhaseRefreshAlbums(ctx context.Context, state *scanState, ds model.DataStore, libs model.Libraries) *phaseRefreshAlbums {
|
||||
return &phaseRefreshAlbums{ctx: ctx, ds: ds, libs: libs, state: state}
|
||||
func createPhaseRefreshAlbums(ctx context.Context, state *scanState, ds model.DataStore) *phaseRefreshAlbums {
|
||||
return &phaseRefreshAlbums{ctx: ctx, ds: ds, state: state}
|
||||
}
|
||||
|
||||
func (p *phaseRefreshAlbums) description() string {
|
||||
@ -47,7 +46,7 @@ func (p *phaseRefreshAlbums) producer() ppl.Producer[*model.Album] {
|
||||
|
||||
func (p *phaseRefreshAlbums) produce(put func(album *model.Album)) error {
|
||||
count := 0
|
||||
for _, lib := range p.libs {
|
||||
for _, lib := range p.state.libraries {
|
||||
cursor, err := p.ds.Album(p.ctx).GetTouchedAlbums(lib.ID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("loading touched albums: %w", err)
|
||||
|
||||
@ -32,8 +32,8 @@ var _ = Describe("phaseRefreshAlbums", func() {
|
||||
{ID: 1, Name: "Library 1"},
|
||||
{ID: 2, Name: "Library 2"},
|
||||
}
|
||||
state = &scanState{}
|
||||
phase = createPhaseRefreshAlbums(ctx, state, ds, libs)
|
||||
state = &scanState{libraries: libs}
|
||||
phase = createPhaseRefreshAlbums(ctx, state, ds)
|
||||
})
|
||||
|
||||
Describe("description", func() {
|
||||
|
||||
@ -3,6 +3,8 @@ package scanner
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
@ -15,6 +17,7 @@ import (
|
||||
"github.com/navidrome/navidrome/log"
|
||||
"github.com/navidrome/navidrome/model"
|
||||
"github.com/navidrome/navidrome/utils/run"
|
||||
"github.com/navidrome/navidrome/utils/slice"
|
||||
)
|
||||
|
||||
type scannerImpl struct {
|
||||
@ -28,7 +31,8 @@ type scanState struct {
|
||||
progress chan<- *ProgressInfo
|
||||
fullScan bool
|
||||
changesDetected atomic.Bool
|
||||
libraries model.Libraries // Store libraries list for consistency across phases
|
||||
libraries model.Libraries // Store libraries list for consistency across phases
|
||||
targets map[int][]string // Optional: map[libraryID][]folderPaths for selective scans
|
||||
}
|
||||
|
||||
func (s *scanState) sendProgress(info *ProgressInfo) {
|
||||
@ -37,6 +41,10 @@ func (s *scanState) sendProgress(info *ProgressInfo) {
|
||||
}
|
||||
}
|
||||
|
||||
func (s *scanState) isSelectiveScan() bool {
|
||||
return len(s.targets) > 0
|
||||
}
|
||||
|
||||
func (s *scanState) sendWarning(msg string) {
|
||||
s.sendProgress(&ProgressInfo{Warning: msg})
|
||||
}
|
||||
@ -45,7 +53,7 @@ func (s *scanState) sendError(err error) {
|
||||
s.sendProgress(&ProgressInfo{Error: err.Error()})
|
||||
}
|
||||
|
||||
func (s *scannerImpl) scanAll(ctx context.Context, fullScan bool, progress chan<- *ProgressInfo) {
|
||||
func (s *scannerImpl) scanFolders(ctx context.Context, fullScan bool, targets []model.ScanTarget, progress chan<- *ProgressInfo) {
|
||||
startTime := time.Now()
|
||||
|
||||
state := scanState{
|
||||
@ -59,38 +67,75 @@ func (s *scannerImpl) scanAll(ctx context.Context, fullScan bool, progress chan<
|
||||
state.changesDetected.Store(true)
|
||||
}
|
||||
|
||||
libs, err := s.ds.Library(ctx).GetAll()
|
||||
// Get libraries and optionally filter by targets
|
||||
allLibs, err := s.ds.Library(ctx).GetAll()
|
||||
if err != nil {
|
||||
state.sendWarning(fmt.Sprintf("getting libraries: %s", err))
|
||||
return
|
||||
}
|
||||
state.libraries = libs
|
||||
|
||||
log.Info(ctx, "Scanner: Starting scan", "fullScan", state.fullScan, "numLibraries", len(libs))
|
||||
if len(targets) > 0 {
|
||||
// Selective scan: filter libraries and build targets map
|
||||
state.targets = make(map[int][]string)
|
||||
|
||||
for _, target := range targets {
|
||||
folderPath := target.FolderPath
|
||||
if folderPath == "" {
|
||||
folderPath = "."
|
||||
}
|
||||
state.targets[target.LibraryID] = append(state.targets[target.LibraryID], folderPath)
|
||||
}
|
||||
|
||||
// Filter libraries to only those in targets
|
||||
state.libraries = slice.Filter(allLibs, func(lib model.Library) bool {
|
||||
return len(state.targets[lib.ID]) > 0
|
||||
})
|
||||
|
||||
log.Info(ctx, "Scanner: Starting selective scan", "fullScan", state.fullScan, "numLibraries", len(state.libraries), "numTargets", len(targets))
|
||||
} else {
|
||||
// Full library scan
|
||||
state.libraries = allLibs
|
||||
log.Info(ctx, "Scanner: Starting scan", "fullScan", state.fullScan, "numLibraries", len(state.libraries))
|
||||
}
|
||||
|
||||
// Store scan type and start time
|
||||
scanType := "quick"
|
||||
if state.fullScan {
|
||||
scanType = "full"
|
||||
}
|
||||
if state.isSelectiveScan() {
|
||||
scanType += "-selective"
|
||||
}
|
||||
_ = s.ds.Property(ctx).Put(consts.LastScanTypeKey, scanType)
|
||||
_ = s.ds.Property(ctx).Put(consts.LastScanStartTimeKey, startTime.Format(time.RFC3339))
|
||||
|
||||
// if there was a full scan in progress, force a full scan
|
||||
if !state.fullScan {
|
||||
for _, lib := range libs {
|
||||
for _, lib := range state.libraries {
|
||||
if lib.FullScanInProgress {
|
||||
log.Info(ctx, "Scanner: Interrupted full scan detected", "lib", lib.Name)
|
||||
state.fullScan = true
|
||||
_ = s.ds.Property(ctx).Put(consts.LastScanTypeKey, "full")
|
||||
if state.isSelectiveScan() {
|
||||
_ = s.ds.Property(ctx).Put(consts.LastScanTypeKey, "full-selective")
|
||||
} else {
|
||||
_ = s.ds.Property(ctx).Put(consts.LastScanTypeKey, "full")
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Prepare libraries for scanning (initialize LastScanStartedAt if needed)
|
||||
err = s.prepareLibrariesForScan(ctx, &state)
|
||||
if err != nil {
|
||||
log.Error(ctx, "Scanner: Error preparing libraries for scan", err)
|
||||
state.sendError(err)
|
||||
return
|
||||
}
|
||||
|
||||
err = run.Sequentially(
|
||||
// Phase 1: Scan all libraries and import new/updated files
|
||||
runPhase[*folderEntry](ctx, 1, createPhaseFolders(ctx, &state, s.ds, s.cw, libs)),
|
||||
runPhase[*folderEntry](ctx, 1, createPhaseFolders(ctx, &state, s.ds, s.cw)),
|
||||
|
||||
// Phase 2: Process missing files, checking for moves
|
||||
runPhase[*missingTracks](ctx, 2, createPhaseMissingTracks(ctx, &state, s.ds)),
|
||||
@ -98,7 +143,7 @@ func (s *scannerImpl) scanAll(ctx context.Context, fullScan bool, progress chan<
|
||||
// Phases 3 and 4 can be run in parallel
|
||||
run.Parallel(
|
||||
// Phase 3: Refresh all new/changed albums and update artists
|
||||
runPhase[*model.Album](ctx, 3, createPhaseRefreshAlbums(ctx, &state, s.ds, libs)),
|
||||
runPhase[*model.Album](ctx, 3, createPhaseRefreshAlbums(ctx, &state, s.ds)),
|
||||
|
||||
// Phase 4: Import/update playlists
|
||||
runPhase[*model.Folder](ctx, 4, createPhasePlaylists(ctx, &state, s.ds, s.pls, s.cw)),
|
||||
@ -131,7 +176,53 @@ func (s *scannerImpl) scanAll(ctx context.Context, fullScan bool, progress chan<
|
||||
state.sendProgress(&ProgressInfo{ChangesDetected: true})
|
||||
}
|
||||
|
||||
log.Info(ctx, "Scanner: Finished scanning all libraries", "duration", time.Since(startTime))
|
||||
if state.isSelectiveScan() {
|
||||
log.Info(ctx, "Scanner: Finished scanning selected folders", "duration", time.Since(startTime), "numTargets", len(targets))
|
||||
} else {
|
||||
log.Info(ctx, "Scanner: Finished scanning all libraries", "duration", time.Since(startTime))
|
||||
}
|
||||
}
|
||||
|
||||
// prepareLibrariesForScan initializes the scan for all libraries in the state.
|
||||
// It calls ScanBegin for libraries that haven't started scanning yet (LastScanStartedAt is zero),
|
||||
// reloads them to get the updated state, and filters out any libraries that fail to initialize.
|
||||
func (s *scannerImpl) prepareLibrariesForScan(ctx context.Context, state *scanState) error {
|
||||
var successfulLibs []model.Library
|
||||
|
||||
for _, lib := range state.libraries {
|
||||
if lib.LastScanStartedAt.IsZero() {
|
||||
// This is a new scan - mark it as started
|
||||
err := s.ds.Library(ctx).ScanBegin(lib.ID, state.fullScan)
|
||||
if err != nil {
|
||||
log.Error(ctx, "Scanner: Error marking scan start", "lib", lib.Name, err)
|
||||
state.sendWarning(err.Error())
|
||||
continue
|
||||
}
|
||||
|
||||
// Reload library to get updated state (timestamps, etc.)
|
||||
reloadedLib, err := s.ds.Library(ctx).Get(lib.ID)
|
||||
if err != nil {
|
||||
log.Error(ctx, "Scanner: Error reloading library", "lib", lib.Name, err)
|
||||
state.sendWarning(err.Error())
|
||||
continue
|
||||
}
|
||||
lib = *reloadedLib
|
||||
} else {
|
||||
// This is a resumed scan
|
||||
log.Debug(ctx, "Scanner: Resuming previous scan", "lib", lib.Name,
|
||||
"lastScanStartedAt", lib.LastScanStartedAt, "fullScan", lib.FullScanInProgress)
|
||||
}
|
||||
|
||||
successfulLibs = append(successfulLibs, lib)
|
||||
}
|
||||
|
||||
if len(successfulLibs) == 0 {
|
||||
return fmt.Errorf("no libraries available for scanning")
|
||||
}
|
||||
|
||||
// Update state with only successfully initialized libraries
|
||||
state.libraries = successfulLibs
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *scannerImpl) runGC(ctx context.Context, state *scanState) func() error {
|
||||
@ -140,7 +231,15 @@ func (s *scannerImpl) runGC(ctx context.Context, state *scanState) func() error
|
||||
return s.ds.WithTx(func(tx model.DataStore) error {
|
||||
if state.changesDetected.Load() {
|
||||
start := time.Now()
|
||||
err := tx.GC(ctx)
|
||||
|
||||
// For selective scans, extract library IDs to scope GC operations
|
||||
var libraryIDs []int
|
||||
if state.isSelectiveScan() {
|
||||
libraryIDs = slices.Collect(maps.Keys(state.targets))
|
||||
log.Debug(ctx, "Scanner: Running selective GC", "libraryIDs", libraryIDs)
|
||||
}
|
||||
|
||||
err := tx.GC(ctx, libraryIDs...)
|
||||
if err != nil {
|
||||
log.Error(ctx, "Scanner: Error running GC", err)
|
||||
return fmt.Errorf("running GC: %w", err)
|
||||
|
||||
@ -32,7 +32,7 @@ var _ = Describe("Scanner - Multi-Library", Ordered, func() {
|
||||
var ctx context.Context
|
||||
var lib1, lib2 model.Library
|
||||
var ds *tests.MockDataStore
|
||||
var s scanner.Scanner
|
||||
var s model.Scanner
|
||||
|
||||
createFS := func(path string, files fstest.MapFS) storagetest.FakeFS {
|
||||
fs := storagetest.FakeFS{}
|
||||
|
||||
293
scanner/scanner_selective_test.go
Normal file
293
scanner/scanner_selective_test.go
Normal file
@ -0,0 +1,293 @@
|
||||
package scanner_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"testing/fstest"
|
||||
|
||||
"github.com/Masterminds/squirrel"
|
||||
"github.com/navidrome/navidrome/conf"
|
||||
"github.com/navidrome/navidrome/conf/configtest"
|
||||
"github.com/navidrome/navidrome/core"
|
||||
"github.com/navidrome/navidrome/core/artwork"
|
||||
"github.com/navidrome/navidrome/core/metrics"
|
||||
"github.com/navidrome/navidrome/core/storage/storagetest"
|
||||
"github.com/navidrome/navidrome/db"
|
||||
"github.com/navidrome/navidrome/log"
|
||||
"github.com/navidrome/navidrome/model"
|
||||
"github.com/navidrome/navidrome/model/request"
|
||||
"github.com/navidrome/navidrome/persistence"
|
||||
"github.com/navidrome/navidrome/scanner"
|
||||
"github.com/navidrome/navidrome/server/events"
|
||||
"github.com/navidrome/navidrome/tests"
|
||||
"github.com/navidrome/navidrome/utils/slice"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
var _ = Describe("ScanFolders", Ordered, func() {
|
||||
var ctx context.Context
|
||||
var lib model.Library
|
||||
var ds model.DataStore
|
||||
var s model.Scanner
|
||||
var fsys storagetest.FakeFS
|
||||
|
||||
BeforeAll(func() {
|
||||
ctx = request.WithUser(GinkgoT().Context(), model.User{ID: "123", IsAdmin: true})
|
||||
tmpDir := GinkgoT().TempDir()
|
||||
conf.Server.DbPath = filepath.Join(tmpDir, "test-selective-scan.db?_journal_mode=WAL")
|
||||
log.Warn("Using DB at " + conf.Server.DbPath)
|
||||
db.Db().SetMaxOpenConns(1)
|
||||
})
|
||||
|
||||
BeforeEach(func() {
|
||||
DeferCleanup(configtest.SetupConfig())
|
||||
conf.Server.MusicFolder = "fake:///music"
|
||||
conf.Server.DevExternalScanner = false
|
||||
|
||||
db.Init(ctx)
|
||||
DeferCleanup(func() {
|
||||
Expect(tests.ClearDB()).To(Succeed())
|
||||
})
|
||||
|
||||
ds = persistence.New(db.Db())
|
||||
|
||||
// Create the admin user in the database to match the context
|
||||
adminUser := model.User{
|
||||
ID: "123",
|
||||
UserName: "admin",
|
||||
Name: "Admin User",
|
||||
IsAdmin: true,
|
||||
NewPassword: "password",
|
||||
}
|
||||
Expect(ds.User(ctx).Put(&adminUser)).To(Succeed())
|
||||
|
||||
s = scanner.New(ctx, ds, artwork.NoopCacheWarmer(), events.NoopBroker(),
|
||||
core.NewPlaylists(ds), metrics.NewNoopInstance())
|
||||
|
||||
lib = model.Library{ID: 1, Name: "Fake Library", Path: "fake:///music"}
|
||||
Expect(ds.Library(ctx).Put(&lib)).To(Succeed())
|
||||
|
||||
// Initialize fake filesystem
|
||||
fsys = storagetest.FakeFS{}
|
||||
storagetest.Register("fake", &fsys)
|
||||
})
|
||||
|
||||
Describe("Adding tracks to the library", func() {
|
||||
It("scans specified folders recursively including all subdirectories", func() {
|
||||
rock := template(_t{"albumartist": "Rock Artist", "album": "Rock Album"})
|
||||
jazz := template(_t{"albumartist": "Jazz Artist", "album": "Jazz Album"})
|
||||
pop := template(_t{"albumartist": "Pop Artist", "album": "Pop Album"})
|
||||
createFS(fstest.MapFS{
|
||||
"rock/track1.mp3": rock(track(1, "Rock Track 1")),
|
||||
"rock/track2.mp3": rock(track(2, "Rock Track 2")),
|
||||
"rock/subdir/track3.mp3": rock(track(3, "Rock Track 3")),
|
||||
"jazz/track4.mp3": jazz(track(1, "Jazz Track 1")),
|
||||
"jazz/subdir/track5.mp3": jazz(track(2, "Jazz Track 2")),
|
||||
"pop/track6.mp3": pop(track(1, "Pop Track 1")),
|
||||
})
|
||||
|
||||
// Scan only the "rock" and "jazz" folders (including their subdirectories)
|
||||
targets := []model.ScanTarget{
|
||||
{LibraryID: lib.ID, FolderPath: "rock"},
|
||||
{LibraryID: lib.ID, FolderPath: "jazz"},
|
||||
}
|
||||
|
||||
warnings, err := s.ScanFolders(ctx, false, targets)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(warnings).To(BeEmpty())
|
||||
|
||||
// Verify all tracks in rock and jazz folders (including subdirectories) were imported
|
||||
allFiles, err := ds.MediaFile(ctx).GetAll()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
// Should have 5 tracks (all rock and jazz tracks including subdirectories)
|
||||
Expect(allFiles).To(HaveLen(5))
|
||||
|
||||
// Get the file paths
|
||||
paths := slice.Map(allFiles, func(mf model.MediaFile) string {
|
||||
return filepath.ToSlash(mf.Path)
|
||||
})
|
||||
|
||||
// Verify the correct files were scanned (including subdirectories)
|
||||
Expect(paths).To(ContainElements(
|
||||
"rock/track1.mp3",
|
||||
"rock/track2.mp3",
|
||||
"rock/subdir/track3.mp3",
|
||||
"jazz/track4.mp3",
|
||||
"jazz/subdir/track5.mp3",
|
||||
))
|
||||
|
||||
// Verify files in the pop folder were NOT scanned
|
||||
Expect(paths).ToNot(ContainElement("pop/track6.mp3"))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Deleting folders", func() {
|
||||
Context("when a child folder is deleted", func() {
|
||||
var (
|
||||
revolver, help func(...map[string]any) *fstest.MapFile
|
||||
artistFolderID string
|
||||
album1FolderID string
|
||||
album2FolderID string
|
||||
album1TrackIDs []string
|
||||
album2TrackIDs []string
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
// Setup template functions for creating test files
|
||||
revolver = storagetest.Template(_t{"albumartist": "The Beatles", "album": "Revolver", "year": 1966})
|
||||
help = storagetest.Template(_t{"albumartist": "The Beatles", "album": "Help!", "year": 1965})
|
||||
|
||||
// Initial filesystem with nested folders
|
||||
fsys.SetFiles(fstest.MapFS{
|
||||
"The Beatles/Revolver/01 - Taxman.mp3": revolver(storagetest.Track(1, "Taxman")),
|
||||
"The Beatles/Revolver/02 - Eleanor Rigby.mp3": revolver(storagetest.Track(2, "Eleanor Rigby")),
|
||||
"The Beatles/Help!/01 - Help!.mp3": help(storagetest.Track(1, "Help!")),
|
||||
"The Beatles/Help!/02 - The Night Before.mp3": help(storagetest.Track(2, "The Night Before")),
|
||||
})
|
||||
|
||||
// First scan - import everything
|
||||
_, err := s.ScanAll(ctx, true)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
// Verify initial state - all folders exist
|
||||
folders, err := ds.Folder(ctx).GetAll(model.QueryOptions{Filters: squirrel.Eq{"library_id": lib.ID}})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(folders).To(HaveLen(4)) // root, Artist, Album1, Album2
|
||||
|
||||
// Store folder IDs for later verification
|
||||
for _, f := range folders {
|
||||
switch f.Name {
|
||||
case "The Beatles":
|
||||
artistFolderID = f.ID
|
||||
case "Revolver":
|
||||
album1FolderID = f.ID
|
||||
case "Help!":
|
||||
album2FolderID = f.ID
|
||||
}
|
||||
}
|
||||
|
||||
// Verify all tracks exist
|
||||
allTracks, err := ds.MediaFile(ctx).GetAll()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(allTracks).To(HaveLen(4))
|
||||
|
||||
// Store track IDs for later verification
|
||||
for _, t := range allTracks {
|
||||
if t.Album == "Revolver" {
|
||||
album1TrackIDs = append(album1TrackIDs, t.ID)
|
||||
} else if t.Album == "Help!" {
|
||||
album2TrackIDs = append(album2TrackIDs, t.ID)
|
||||
}
|
||||
}
|
||||
|
||||
// Verify no tracks are missing initially
|
||||
for _, t := range allTracks {
|
||||
Expect(t.Missing).To(BeFalse())
|
||||
}
|
||||
})
|
||||
|
||||
It("should mark child folder and its tracks as missing when parent is scanned", func() {
|
||||
// Delete the child folder (Help!) from the filesystem
|
||||
fsys.SetFiles(fstest.MapFS{
|
||||
"The Beatles/Revolver/01 - Taxman.mp3": revolver(storagetest.Track(1, "Taxman")),
|
||||
"The Beatles/Revolver/02 - Eleanor Rigby.mp3": revolver(storagetest.Track(2, "Eleanor Rigby")),
|
||||
// "The Beatles/Help!" folder and its contents are DELETED
|
||||
})
|
||||
|
||||
// Run selective scan on the parent folder (Artist)
|
||||
// This simulates what the watcher does when a child folder is deleted
|
||||
_, err := s.ScanFolders(ctx, false, []model.ScanTarget{
|
||||
{LibraryID: lib.ID, FolderPath: "The Beatles"},
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
// Verify the deleted child folder is now marked as missing
|
||||
deletedFolder, err := ds.Folder(ctx).Get(album2FolderID)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(deletedFolder.Missing).To(BeTrue(), "Deleted child folder should be marked as missing")
|
||||
|
||||
// Verify the deleted folder's tracks are marked as missing
|
||||
for _, trackID := range album2TrackIDs {
|
||||
track, err := ds.MediaFile(ctx).Get(trackID)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(track.Missing).To(BeTrue(), "Track in deleted folder should be marked as missing")
|
||||
}
|
||||
|
||||
// Verify the parent folder is still present and not marked as missing
|
||||
parentFolder, err := ds.Folder(ctx).Get(artistFolderID)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(parentFolder.Missing).To(BeFalse(), "Parent folder should not be marked as missing")
|
||||
|
||||
// Verify the sibling folder and its tracks are still present and not missing
|
||||
siblingFolder, err := ds.Folder(ctx).Get(album1FolderID)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(siblingFolder.Missing).To(BeFalse(), "Sibling folder should not be marked as missing")
|
||||
|
||||
for _, trackID := range album1TrackIDs {
|
||||
track, err := ds.MediaFile(ctx).Get(trackID)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(track.Missing).To(BeFalse(), "Track in sibling folder should not be marked as missing")
|
||||
}
|
||||
})
|
||||
|
||||
It("should mark deeply nested child folders as missing", func() {
|
||||
// Add a deeply nested folder structure
|
||||
fsys.SetFiles(fstest.MapFS{
|
||||
"The Beatles/Revolver/01 - Taxman.mp3": revolver(storagetest.Track(1, "Taxman")),
|
||||
"The Beatles/Revolver/02 - Eleanor Rigby.mp3": revolver(storagetest.Track(2, "Eleanor Rigby")),
|
||||
"The Beatles/Help!/01 - Help!.mp3": help(storagetest.Track(1, "Help!")),
|
||||
"The Beatles/Help!/02 - The Night Before.mp3": help(storagetest.Track(2, "The Night Before")),
|
||||
"The Beatles/Help!/Bonus/01 - Bonus Track.mp3": help(storagetest.Track(99, "Bonus Track")),
|
||||
"The Beatles/Help!/Bonus/Nested/01 - Deep Track.mp3": help(storagetest.Track(100, "Deep Track")),
|
||||
})
|
||||
|
||||
// Rescan to import the new nested structure
|
||||
_, err := s.ScanAll(ctx, true)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
// Verify nested folders were created
|
||||
allFolders, err := ds.Folder(ctx).GetAll(model.QueryOptions{Filters: squirrel.Eq{"library_id": lib.ID}})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(len(allFolders)).To(BeNumerically(">", 4), "Should have more folders with nested structure")
|
||||
|
||||
// Now delete the entire Help! folder including nested children
|
||||
fsys.SetFiles(fstest.MapFS{
|
||||
"The Beatles/Revolver/01 - Taxman.mp3": revolver(storagetest.Track(1, "Taxman")),
|
||||
"The Beatles/Revolver/02 - Eleanor Rigby.mp3": revolver(storagetest.Track(2, "Eleanor Rigby")),
|
||||
// All Help! subfolders are deleted
|
||||
})
|
||||
|
||||
// Run selective scan on parent
|
||||
_, err = s.ScanFolders(ctx, false, []model.ScanTarget{
|
||||
{LibraryID: lib.ID, FolderPath: "The Beatles"},
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
// Verify all Help! folders (including nested ones) are marked as missing
|
||||
missingFolders, err := ds.Folder(ctx).GetAll(model.QueryOptions{
|
||||
Filters: squirrel.And{
|
||||
squirrel.Eq{"library_id": lib.ID},
|
||||
squirrel.Eq{"missing": true},
|
||||
},
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(len(missingFolders)).To(BeNumerically(">", 0), "At least one folder should be marked as missing")
|
||||
|
||||
// Verify all tracks in deleted folders are marked as missing
|
||||
allTracks, err := ds.MediaFile(ctx).GetAll()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(allTracks).To(HaveLen(6))
|
||||
|
||||
for _, track := range allTracks {
|
||||
if track.Album == "Help!" {
|
||||
Expect(track.Missing).To(BeTrue(), "All tracks in deleted Help! folder should be marked as missing")
|
||||
} else if track.Album == "Revolver" {
|
||||
Expect(track.Missing).To(BeFalse(), "Tracks in Revolver folder should not be marked as missing")
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
@ -34,19 +34,19 @@ type _t = map[string]any
|
||||
var template = storagetest.Template
|
||||
var track = storagetest.Track
|
||||
|
||||
func createFS(files fstest.MapFS) storagetest.FakeFS {
|
||||
fs := storagetest.FakeFS{}
|
||||
fs.SetFiles(files)
|
||||
storagetest.Register("fake", &fs)
|
||||
return fs
|
||||
}
|
||||
|
||||
var _ = Describe("Scanner", Ordered, func() {
|
||||
var ctx context.Context
|
||||
var lib model.Library
|
||||
var ds *tests.MockDataStore
|
||||
var mfRepo *mockMediaFileRepo
|
||||
var s scanner.Scanner
|
||||
|
||||
createFS := func(files fstest.MapFS) storagetest.FakeFS {
|
||||
fs := storagetest.FakeFS{}
|
||||
fs.SetFiles(files)
|
||||
storagetest.Register("fake", &fs)
|
||||
return fs
|
||||
}
|
||||
var s model.Scanner
|
||||
|
||||
BeforeAll(func() {
|
||||
ctx = request.WithUser(GinkgoT().Context(), model.User{ID: "123", IsAdmin: true})
|
||||
@ -478,6 +478,56 @@ var _ = Describe("Scanner", Ordered, func() {
|
||||
Expect(mf.Missing).To(BeFalse())
|
||||
})
|
||||
|
||||
It("marks tracks as missing when scanning a deleted folder with ScanFolders", func() {
|
||||
By("Adding a third track to Revolver to have more test data")
|
||||
fsys.Add("The Beatles/Revolver/03 - I'm Only Sleeping.mp3", revolver(track(3, "I'm Only Sleeping")))
|
||||
Expect(runScanner(ctx, false)).To(Succeed())
|
||||
|
||||
By("Verifying initial state has 5 tracks")
|
||||
Expect(ds.MediaFile(ctx).CountAll(model.QueryOptions{
|
||||
Filters: squirrel.Eq{"missing": false},
|
||||
})).To(Equal(int64(5)))
|
||||
|
||||
By("Removing the entire Revolver folder from filesystem")
|
||||
fsys.Remove("The Beatles/Revolver/01 - Taxman.mp3")
|
||||
fsys.Remove("The Beatles/Revolver/02 - Eleanor Rigby.mp3")
|
||||
fsys.Remove("The Beatles/Revolver/03 - I'm Only Sleeping.mp3")
|
||||
|
||||
By("Scanning the parent folder (simulating watcher behavior)")
|
||||
targets := []model.ScanTarget{
|
||||
{LibraryID: lib.ID, FolderPath: "The Beatles"},
|
||||
}
|
||||
_, err := s.ScanFolders(ctx, false, targets)
|
||||
Expect(err).To(Succeed())
|
||||
|
||||
By("Checking all Revolver tracks are marked as missing")
|
||||
mf, err := findByPath("The Beatles/Revolver/01 - Taxman.mp3")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(mf.Missing).To(BeTrue())
|
||||
|
||||
mf, err = findByPath("The Beatles/Revolver/02 - Eleanor Rigby.mp3")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(mf.Missing).To(BeTrue())
|
||||
|
||||
mf, err = findByPath("The Beatles/Revolver/03 - I'm Only Sleeping.mp3")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(mf.Missing).To(BeTrue())
|
||||
|
||||
By("Checking the Help! tracks are not affected")
|
||||
mf, err = findByPath("The Beatles/Help!/01 - Help!.mp3")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(mf.Missing).To(BeFalse())
|
||||
|
||||
mf, err = findByPath("The Beatles/Help!/02 - The Night Before.mp3")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(mf.Missing).To(BeFalse())
|
||||
|
||||
By("Verifying only 2 non-missing tracks remain (Help! tracks)")
|
||||
Expect(ds.MediaFile(ctx).CountAll(model.QueryOptions{
|
||||
Filters: squirrel.Eq{"missing": false},
|
||||
})).To(Equal(int64(2)))
|
||||
})
|
||||
|
||||
It("does not override artist fields when importing an undertagged file", func() {
|
||||
By("Making sure artist in the DB contains MBID and sort name")
|
||||
aa, err := ds.Artist(ctx).GetAll(model.QueryOptions{
|
||||
|
||||
@ -1,7 +1,6 @@
|
||||
package scanner
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"io/fs"
|
||||
"maps"
|
||||
@ -11,37 +10,69 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/navidrome/navidrome/conf"
|
||||
"github.com/navidrome/navidrome/consts"
|
||||
"github.com/navidrome/navidrome/log"
|
||||
"github.com/navidrome/navidrome/model"
|
||||
"github.com/navidrome/navidrome/utils"
|
||||
ignore "github.com/sabhiram/go-gitignore"
|
||||
)
|
||||
|
||||
func walkDirTree(ctx context.Context, job *scanJob) (<-chan *folderEntry, error) {
|
||||
// walkDirTree recursively walks the directory tree starting from the given targetFolders.
|
||||
// If no targetFolders are provided, it starts from the root folder (".").
|
||||
// It returns a channel of folderEntry pointers representing each folder found.
|
||||
func walkDirTree(ctx context.Context, job *scanJob, targetFolders ...string) (<-chan *folderEntry, error) {
|
||||
results := make(chan *folderEntry)
|
||||
folders := targetFolders
|
||||
if len(targetFolders) == 0 {
|
||||
// No specific folders provided, scan the root folder
|
||||
folders = []string{"."}
|
||||
}
|
||||
go func() {
|
||||
defer close(results)
|
||||
err := walkFolder(ctx, job, ".", nil, results)
|
||||
if err != nil {
|
||||
log.Error(ctx, "Scanner: There were errors reading directories from filesystem", "path", job.lib.Path, err)
|
||||
return
|
||||
for _, folderPath := range folders {
|
||||
if utils.IsCtxDone(ctx) {
|
||||
return
|
||||
}
|
||||
|
||||
// Check if target folder exists before walking it
|
||||
// If it doesn't exist (e.g., deleted between watcher detection and scan execution),
|
||||
// skip it so it remains in job.lastUpdates and gets handled in following steps
|
||||
_, err := fs.Stat(job.fs, folderPath)
|
||||
if err != nil {
|
||||
log.Warn(ctx, "Scanner: Target folder does not exist.", "path", folderPath, err)
|
||||
continue
|
||||
}
|
||||
|
||||
// Create checker and push patterns from root to this folder
|
||||
checker := newIgnoreChecker(job.fs)
|
||||
err = checker.PushAllParents(ctx, folderPath)
|
||||
if err != nil {
|
||||
log.Error(ctx, "Scanner: Error pushing ignore patterns for target folder", "path", folderPath, err)
|
||||
continue
|
||||
}
|
||||
|
||||
// Recursively walk this folder and all its children
|
||||
err = walkFolder(ctx, job, folderPath, checker, results)
|
||||
if err != nil {
|
||||
log.Error(ctx, "Scanner: Error walking target folder", "path", folderPath, err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
log.Debug(ctx, "Scanner: Finished reading folders", "lib", job.lib.Name, "path", job.lib.Path, "numFolders", job.numFolders.Load())
|
||||
log.Debug(ctx, "Scanner: Finished reading target folders", "lib", job.lib.Name, "path", job.lib.Path, "numFolders", job.numFolders.Load())
|
||||
}()
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func walkFolder(ctx context.Context, job *scanJob, currentFolder string, ignorePatterns []string, results chan<- *folderEntry) error {
|
||||
ignorePatterns = loadIgnoredPatterns(ctx, job.fs, currentFolder, ignorePatterns)
|
||||
func walkFolder(ctx context.Context, job *scanJob, currentFolder string, checker *IgnoreChecker, results chan<- *folderEntry) error {
|
||||
// Push patterns for this folder onto the stack
|
||||
_ = checker.Push(ctx, currentFolder)
|
||||
defer checker.Pop() // Pop patterns when leaving this folder
|
||||
|
||||
folder, children, err := loadDir(ctx, job, currentFolder, ignorePatterns)
|
||||
folder, children, err := loadDir(ctx, job, currentFolder, checker)
|
||||
if err != nil {
|
||||
log.Warn(ctx, "Scanner: Error loading dir. Skipping", "path", currentFolder, err)
|
||||
return nil
|
||||
}
|
||||
for _, c := range children {
|
||||
err := walkFolder(ctx, job, c, ignorePatterns, results)
|
||||
err := walkFolder(ctx, job, c, checker, results)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -59,50 +90,17 @@ func walkFolder(ctx context.Context, job *scanJob, currentFolder string, ignoreP
|
||||
return nil
|
||||
}
|
||||
|
||||
func loadIgnoredPatterns(ctx context.Context, fsys fs.FS, currentFolder string, currentPatterns []string) []string {
|
||||
ignoreFilePath := path.Join(currentFolder, consts.ScanIgnoreFile)
|
||||
var newPatterns []string
|
||||
if _, err := fs.Stat(fsys, ignoreFilePath); err == nil {
|
||||
// Read and parse the .ndignore file
|
||||
ignoreFile, err := fsys.Open(ignoreFilePath)
|
||||
if err != nil {
|
||||
log.Warn(ctx, "Scanner: Error opening .ndignore file", "path", ignoreFilePath, err)
|
||||
// Continue with previous patterns
|
||||
} else {
|
||||
defer ignoreFile.Close()
|
||||
scanner := bufio.NewScanner(ignoreFile)
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue // Skip empty lines and comments
|
||||
}
|
||||
newPatterns = append(newPatterns, line)
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
log.Warn(ctx, "Scanner: Error reading .ignore file", "path", ignoreFilePath, err)
|
||||
}
|
||||
}
|
||||
// If the .ndignore file is empty, mimic the current behavior and ignore everything
|
||||
if len(newPatterns) == 0 {
|
||||
log.Trace(ctx, "Scanner: .ndignore file is empty, ignoring everything", "path", currentFolder)
|
||||
newPatterns = []string{"**/*"}
|
||||
} else {
|
||||
log.Trace(ctx, "Scanner: .ndignore file found ", "path", ignoreFilePath, "patterns", newPatterns)
|
||||
}
|
||||
}
|
||||
// Combine the patterns from the .ndignore file with the ones passed as argument
|
||||
combinedPatterns := append([]string{}, currentPatterns...)
|
||||
return append(combinedPatterns, newPatterns...)
|
||||
}
|
||||
|
||||
func loadDir(ctx context.Context, job *scanJob, dirPath string, ignorePatterns []string) (folder *folderEntry, children []string, err error) {
|
||||
folder = newFolderEntry(job, dirPath)
|
||||
|
||||
func loadDir(ctx context.Context, job *scanJob, dirPath string, checker *IgnoreChecker) (folder *folderEntry, children []string, err error) {
|
||||
// Check if directory exists before creating the folder entry
|
||||
// This is important to avoid removing the folder from lastUpdates if it doesn't exist
|
||||
dirInfo, err := fs.Stat(job.fs, dirPath)
|
||||
if err != nil {
|
||||
log.Warn(ctx, "Scanner: Error stating dir", "path", dirPath, err)
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
// Now that we know the folder exists, create the entry (which removes it from lastUpdates)
|
||||
folder = job.createFolderEntry(dirPath)
|
||||
folder.modTime = dirInfo.ModTime()
|
||||
|
||||
dir, err := job.fs.Open(dirPath)
|
||||
@ -117,12 +115,11 @@ func loadDir(ctx context.Context, job *scanJob, dirPath string, ignorePatterns [
|
||||
return folder, children, err
|
||||
}
|
||||
|
||||
ignoreMatcher := ignore.CompileIgnoreLines(ignorePatterns...)
|
||||
entries := fullReadDir(ctx, dirFile)
|
||||
children = make([]string, 0, len(entries))
|
||||
for _, entry := range entries {
|
||||
entryPath := path.Join(dirPath, entry.Name())
|
||||
if len(ignorePatterns) > 0 && isScanIgnored(ctx, ignoreMatcher, entryPath) {
|
||||
if checker.ShouldIgnore(ctx, entryPath) {
|
||||
log.Trace(ctx, "Scanner: Ignoring entry", "path", entryPath)
|
||||
continue
|
||||
}
|
||||
@ -234,6 +231,7 @@ func isDirReadable(ctx context.Context, fsys fs.FS, dirPath string) bool {
|
||||
var ignoredDirs = []string{
|
||||
"$RECYCLE.BIN",
|
||||
"#snapshot",
|
||||
"@Recycle",
|
||||
"@Recently-Snapshot",
|
||||
".streams",
|
||||
"lost+found",
|
||||
@ -254,11 +252,3 @@ func isDirIgnored(name string) bool {
|
||||
func isEntryIgnored(name string) bool {
|
||||
return strings.HasPrefix(name, ".") && !strings.HasPrefix(name, "..")
|
||||
}
|
||||
|
||||
func isScanIgnored(ctx context.Context, matcher *ignore.GitIgnore, entryPath string) bool {
|
||||
matches := matcher.MatchesPath(entryPath)
|
||||
if matches {
|
||||
log.Trace(ctx, "Scanner: Ignoring entry matching .ndignore: ", "path", entryPath)
|
||||
}
|
||||
return matches
|
||||
}
|
||||
|
||||
@ -25,82 +25,196 @@ var _ = Describe("walk_dir_tree", func() {
|
||||
ctx context.Context
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
DeferCleanup(configtest.SetupConfig())
|
||||
ctx = GinkgoT().Context()
|
||||
fsys = &mockMusicFS{
|
||||
FS: fstest.MapFS{
|
||||
"root/a/.ndignore": {Data: []byte("ignored/*")},
|
||||
"root/a/f1.mp3": {},
|
||||
"root/a/f2.mp3": {},
|
||||
"root/a/ignored/bad.mp3": {},
|
||||
"root/b/cover.jpg": {},
|
||||
"root/c/f3": {},
|
||||
"root/d": {},
|
||||
"root/d/.ndignore": {},
|
||||
"root/d/f1.mp3": {},
|
||||
"root/d/f2.mp3": {},
|
||||
"root/d/f3.mp3": {},
|
||||
"root/e/original/f1.mp3": {},
|
||||
"root/e/symlink": {Mode: fs.ModeSymlink, Data: []byte("original")},
|
||||
Context("full library", func() {
|
||||
BeforeEach(func() {
|
||||
DeferCleanup(configtest.SetupConfig())
|
||||
ctx = GinkgoT().Context()
|
||||
fsys = &mockMusicFS{
|
||||
FS: fstest.MapFS{
|
||||
"root/a/.ndignore": {Data: []byte("ignored/*")},
|
||||
"root/a/f1.mp3": {},
|
||||
"root/a/f2.mp3": {},
|
||||
"root/a/ignored/bad.mp3": {},
|
||||
"root/b/cover.jpg": {},
|
||||
"root/c/f3": {},
|
||||
"root/d": {},
|
||||
"root/d/.ndignore": {},
|
||||
"root/d/f1.mp3": {},
|
||||
"root/d/f2.mp3": {},
|
||||
"root/d/f3.mp3": {},
|
||||
"root/e/original/f1.mp3": {},
|
||||
"root/e/symlink": {Mode: fs.ModeSymlink, Data: []byte("original")},
|
||||
},
|
||||
}
|
||||
job = &scanJob{
|
||||
fs: fsys,
|
||||
lib: model.Library{Path: "/music"},
|
||||
}
|
||||
})
|
||||
|
||||
// Helper function to call walkDirTree and collect folders from the results channel
|
||||
getFolders := func() map[string]*folderEntry {
|
||||
results, err := walkDirTree(ctx, job)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
folders := map[string]*folderEntry{}
|
||||
g := errgroup.Group{}
|
||||
g.Go(func() error {
|
||||
for folder := range results {
|
||||
folders[folder.path] = folder
|
||||
}
|
||||
return nil
|
||||
})
|
||||
_ = g.Wait()
|
||||
return folders
|
||||
}
|
||||
|
||||
DescribeTable("symlink handling",
|
||||
func(followSymlinks bool, expectedFolderCount int) {
|
||||
conf.Server.Scanner.FollowSymlinks = followSymlinks
|
||||
folders := getFolders()
|
||||
|
||||
Expect(folders).To(HaveLen(expectedFolderCount + 2)) // +2 for `.` and `root`
|
||||
|
||||
// Basic folder structure checks
|
||||
Expect(folders["root/a"].audioFiles).To(SatisfyAll(
|
||||
HaveLen(2),
|
||||
HaveKey("f1.mp3"),
|
||||
HaveKey("f2.mp3"),
|
||||
))
|
||||
Expect(folders["root/a"].imageFiles).To(BeEmpty())
|
||||
Expect(folders["root/b"].audioFiles).To(BeEmpty())
|
||||
Expect(folders["root/b"].imageFiles).To(SatisfyAll(
|
||||
HaveLen(1),
|
||||
HaveKey("cover.jpg"),
|
||||
))
|
||||
Expect(folders["root/c"].audioFiles).To(BeEmpty())
|
||||
Expect(folders["root/c"].imageFiles).To(BeEmpty())
|
||||
Expect(folders).ToNot(HaveKey("root/d"))
|
||||
|
||||
// Symlink specific checks
|
||||
if followSymlinks {
|
||||
Expect(folders["root/e/symlink"].audioFiles).To(HaveLen(1))
|
||||
} else {
|
||||
Expect(folders).ToNot(HaveKey("root/e/symlink"))
|
||||
}
|
||||
},
|
||||
}
|
||||
job = &scanJob{
|
||||
fs: fsys,
|
||||
lib: model.Library{Path: "/music"},
|
||||
}
|
||||
Entry("with symlinks enabled", true, 7),
|
||||
Entry("with symlinks disabled", false, 6),
|
||||
)
|
||||
})
|
||||
|
||||
// Helper function to call walkDirTree and collect folders from the results channel
|
||||
getFolders := func() map[string]*folderEntry {
|
||||
results, err := walkDirTree(ctx, job)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
folders := map[string]*folderEntry{}
|
||||
g := errgroup.Group{}
|
||||
g.Go(func() error {
|
||||
for folder := range results {
|
||||
folders[folder.path] = folder
|
||||
Context("with target folders", func() {
|
||||
BeforeEach(func() {
|
||||
DeferCleanup(configtest.SetupConfig())
|
||||
ctx = GinkgoT().Context()
|
||||
fsys = &mockMusicFS{
|
||||
FS: fstest.MapFS{
|
||||
"Artist/Album1/track1.mp3": {},
|
||||
"Artist/Album1/track2.mp3": {},
|
||||
"Artist/Album2/track1.mp3": {},
|
||||
"Artist/Album2/track2.mp3": {},
|
||||
"Artist/Album2/Sub/track3.mp3": {},
|
||||
"OtherArtist/Album3/track1.mp3": {},
|
||||
},
|
||||
}
|
||||
job = &scanJob{
|
||||
fs: fsys,
|
||||
lib: model.Library{Path: "/music"},
|
||||
}
|
||||
return nil
|
||||
})
|
||||
_ = g.Wait()
|
||||
return folders
|
||||
}
|
||||
|
||||
DescribeTable("symlink handling",
|
||||
func(followSymlinks bool, expectedFolderCount int) {
|
||||
conf.Server.Scanner.FollowSymlinks = followSymlinks
|
||||
folders := getFolders()
|
||||
It("should recursively walk all subdirectories of target folders", func() {
|
||||
results, err := walkDirTree(ctx, job, "Artist")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
Expect(folders).To(HaveLen(expectedFolderCount + 2)) // +2 for `.` and `root`
|
||||
folders := map[string]*folderEntry{}
|
||||
g := errgroup.Group{}
|
||||
g.Go(func() error {
|
||||
for folder := range results {
|
||||
folders[folder.path] = folder
|
||||
}
|
||||
return nil
|
||||
})
|
||||
_ = g.Wait()
|
||||
|
||||
// Basic folder structure checks
|
||||
Expect(folders["root/a"].audioFiles).To(SatisfyAll(
|
||||
HaveLen(2),
|
||||
HaveKey("f1.mp3"),
|
||||
HaveKey("f2.mp3"),
|
||||
// Should include the target folder and all its descendants
|
||||
Expect(folders).To(SatisfyAll(
|
||||
HaveKey("Artist"),
|
||||
HaveKey("Artist/Album1"),
|
||||
HaveKey("Artist/Album2"),
|
||||
HaveKey("Artist/Album2/Sub"),
|
||||
))
|
||||
Expect(folders["root/a"].imageFiles).To(BeEmpty())
|
||||
Expect(folders["root/b"].audioFiles).To(BeEmpty())
|
||||
Expect(folders["root/b"].imageFiles).To(SatisfyAll(
|
||||
HaveLen(1),
|
||||
HaveKey("cover.jpg"),
|
||||
))
|
||||
Expect(folders["root/c"].audioFiles).To(BeEmpty())
|
||||
Expect(folders["root/c"].imageFiles).To(BeEmpty())
|
||||
Expect(folders).ToNot(HaveKey("root/d"))
|
||||
|
||||
// Symlink specific checks
|
||||
if followSymlinks {
|
||||
Expect(folders["root/e/symlink"].audioFiles).To(HaveLen(1))
|
||||
} else {
|
||||
Expect(folders).ToNot(HaveKey("root/e/symlink"))
|
||||
// Should not include folders outside the target
|
||||
Expect(folders).ToNot(HaveKey("OtherArtist"))
|
||||
Expect(folders).ToNot(HaveKey("OtherArtist/Album3"))
|
||||
|
||||
// Verify audio files are present
|
||||
Expect(folders["Artist/Album1"].audioFiles).To(HaveLen(2))
|
||||
Expect(folders["Artist/Album2"].audioFiles).To(HaveLen(2))
|
||||
Expect(folders["Artist/Album2/Sub"].audioFiles).To(HaveLen(1))
|
||||
})
|
||||
|
||||
It("should handle multiple target folders", func() {
|
||||
results, err := walkDirTree(ctx, job, "Artist/Album1", "OtherArtist")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
folders := map[string]*folderEntry{}
|
||||
g := errgroup.Group{}
|
||||
g.Go(func() error {
|
||||
for folder := range results {
|
||||
folders[folder.path] = folder
|
||||
}
|
||||
return nil
|
||||
})
|
||||
_ = g.Wait()
|
||||
|
||||
// Should include both target folders and their descendants
|
||||
Expect(folders).To(SatisfyAll(
|
||||
HaveKey("Artist/Album1"),
|
||||
HaveKey("OtherArtist"),
|
||||
HaveKey("OtherArtist/Album3"),
|
||||
))
|
||||
|
||||
// Should not include other folders
|
||||
Expect(folders).ToNot(HaveKey("Artist"))
|
||||
Expect(folders).ToNot(HaveKey("Artist/Album2"))
|
||||
Expect(folders).ToNot(HaveKey("Artist/Album2/Sub"))
|
||||
})
|
||||
|
||||
It("should skip non-existent target folders and preserve them in lastUpdates", func() {
|
||||
// Setup job with lastUpdates for both existing and non-existing folders
|
||||
job.lastUpdates = map[string]model.FolderUpdateInfo{
|
||||
model.FolderID(job.lib, "Artist/Album1"): {},
|
||||
model.FolderID(job.lib, "NonExistent/DeletedFolder"): {},
|
||||
model.FolderID(job.lib, "OtherArtist/Album3"): {},
|
||||
}
|
||||
},
|
||||
Entry("with symlinks enabled", true, 7),
|
||||
Entry("with symlinks disabled", false, 6),
|
||||
)
|
||||
|
||||
// Try to scan existing folder and non-existing folder
|
||||
results, err := walkDirTree(ctx, job, "Artist/Album1", "NonExistent/DeletedFolder")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
// Collect results
|
||||
folders := map[string]struct{}{}
|
||||
for folder := range results {
|
||||
folders[folder.path] = struct{}{}
|
||||
}
|
||||
|
||||
// Should only include the existing folder
|
||||
Expect(folders).To(HaveKey("Artist/Album1"))
|
||||
Expect(folders).ToNot(HaveKey("NonExistent/DeletedFolder"))
|
||||
|
||||
// The non-existent folder should still be in lastUpdates (not removed by popLastUpdate)
|
||||
Expect(job.lastUpdates).To(HaveKey(model.FolderID(job.lib, "NonExistent/DeletedFolder")))
|
||||
|
||||
// The existing folder should have been removed from lastUpdates
|
||||
Expect(job.lastUpdates).ToNot(HaveKey(model.FolderID(job.lib, "Artist/Album1")))
|
||||
|
||||
// Folders not in targets should remain in lastUpdates
|
||||
Expect(job.lastUpdates).To(HaveKey(model.FolderID(job.lib, "OtherArtist/Album3")))
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
Describe("helper functions", func() {
|
||||
|
||||
@ -24,9 +24,9 @@ type Watcher interface {
|
||||
type watcher struct {
|
||||
mainCtx context.Context
|
||||
ds model.DataStore
|
||||
scanner Scanner
|
||||
scanner model.Scanner
|
||||
triggerWait time.Duration
|
||||
watcherNotify chan model.Library
|
||||
watcherNotify chan scanNotification
|
||||
libraryWatchers map[int]*libraryWatcherInstance
|
||||
mu sync.RWMutex
|
||||
}
|
||||
@ -36,14 +36,19 @@ type libraryWatcherInstance struct {
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
|
||||
type scanNotification struct {
|
||||
Library *model.Library
|
||||
FolderPath string
|
||||
}
|
||||
|
||||
// GetWatcher returns the watcher singleton
|
||||
func GetWatcher(ds model.DataStore, s Scanner) Watcher {
|
||||
func GetWatcher(ds model.DataStore, s model.Scanner) Watcher {
|
||||
return singleton.GetInstance(func() *watcher {
|
||||
return &watcher{
|
||||
ds: ds,
|
||||
scanner: s,
|
||||
triggerWait: conf.Server.Scanner.WatcherWait,
|
||||
watcherNotify: make(chan model.Library, 1),
|
||||
watcherNotify: make(chan scanNotification, 1),
|
||||
libraryWatchers: make(map[int]*libraryWatcherInstance),
|
||||
}
|
||||
})
|
||||
@ -68,11 +73,11 @@ func (w *watcher) Run(ctx context.Context) error {
|
||||
// Main scan triggering loop
|
||||
trigger := time.NewTimer(w.triggerWait)
|
||||
trigger.Stop()
|
||||
waiting := false
|
||||
targets := make(map[model.ScanTarget]struct{})
|
||||
for {
|
||||
select {
|
||||
case <-trigger.C:
|
||||
log.Info("Watcher: Triggering scan")
|
||||
log.Info("Watcher: Triggering scan for changed folders", "numTargets", len(targets))
|
||||
status, err := w.scanner.Status(ctx)
|
||||
if err != nil {
|
||||
log.Error(ctx, "Watcher: Error retrieving Scanner status", err)
|
||||
@ -83,9 +88,23 @@ func (w *watcher) Run(ctx context.Context) error {
|
||||
trigger.Reset(w.triggerWait * 3)
|
||||
continue
|
||||
}
|
||||
waiting = false
|
||||
|
||||
// Convert targets map to slice
|
||||
targetSlice := make([]model.ScanTarget, 0, len(targets))
|
||||
for target := range targets {
|
||||
targetSlice = append(targetSlice, target)
|
||||
}
|
||||
|
||||
// Clear targets for next batch
|
||||
targets = make(map[model.ScanTarget]struct{})
|
||||
|
||||
go func() {
|
||||
_, err := w.scanner.ScanAll(ctx, false)
|
||||
var err error
|
||||
if conf.Server.DevSelectiveWatcher {
|
||||
_, err = w.scanner.ScanFolders(ctx, false, targetSlice)
|
||||
} else {
|
||||
_, err = w.scanner.ScanAll(ctx, false)
|
||||
}
|
||||
if err != nil {
|
||||
log.Error(ctx, "Watcher: Error scanning", err)
|
||||
} else {
|
||||
@ -102,13 +121,22 @@ func (w *watcher) Run(ctx context.Context) error {
|
||||
w.libraryWatchers = make(map[int]*libraryWatcherInstance)
|
||||
w.mu.Unlock()
|
||||
return nil
|
||||
case lib := <-w.watcherNotify:
|
||||
if !waiting {
|
||||
log.Debug(ctx, "Watcher: Detected changes. Waiting for more changes before triggering scan",
|
||||
"libraryID", lib.ID, "name", lib.Name, "path", lib.Path)
|
||||
waiting = true
|
||||
}
|
||||
case notification := <-w.watcherNotify:
|
||||
// Reset the trigger timer for debounce
|
||||
trigger.Reset(w.triggerWait)
|
||||
|
||||
lib := notification.Library
|
||||
folderPath := notification.FolderPath
|
||||
|
||||
// If already scheduled for scan, skip
|
||||
target := model.ScanTarget{LibraryID: lib.ID, FolderPath: folderPath}
|
||||
if _, exists := targets[target]; exists {
|
||||
continue
|
||||
}
|
||||
targets[target] = struct{}{}
|
||||
|
||||
log.Debug(ctx, "Watcher: Detected changes. Waiting for more changes before triggering scan",
|
||||
"libraryID", lib.ID, "name", lib.Name, "path", lib.Path, "folderPath", folderPath)
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -199,13 +227,18 @@ func (w *watcher) watchLibrary(ctx context.Context, lib *model.Library) error {
|
||||
|
||||
log.Info(ctx, "Watcher started for library", "libraryID", lib.ID, "name", lib.Name, "path", lib.Path, "absoluteLibPath", absLibPath)
|
||||
|
||||
return w.processLibraryEvents(ctx, lib, fsys, c, absLibPath)
|
||||
}
|
||||
|
||||
// processLibraryEvents processes filesystem events for a library.
|
||||
func (w *watcher) processLibraryEvents(ctx context.Context, lib *model.Library, fsys storage.MusicFS, events <-chan string, absLibPath string) error {
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
log.Debug(ctx, "Watcher stopped due to context cancellation", "libraryID", lib.ID, "name", lib.Name)
|
||||
return nil
|
||||
case path := <-c:
|
||||
path, err = filepath.Rel(absLibPath, path)
|
||||
case path := <-events:
|
||||
path, err := filepath.Rel(absLibPath, path)
|
||||
if err != nil {
|
||||
log.Error(ctx, "Error getting relative path", "libraryID", lib.ID, "absolutePath", absLibPath, "path", path, err)
|
||||
continue
|
||||
@ -215,12 +248,27 @@ func (w *watcher) watchLibrary(ctx context.Context, lib *model.Library) error {
|
||||
log.Trace(ctx, "Ignoring change", "libraryID", lib.ID, "path", path)
|
||||
continue
|
||||
}
|
||||
|
||||
log.Trace(ctx, "Detected change", "libraryID", lib.ID, "path", path, "absoluteLibPath", absLibPath)
|
||||
|
||||
// Check if the original path (before resolution) matches .ndignore patterns
|
||||
// This is crucial for deleted folders - if a deleted folder matches .ndignore,
|
||||
// we should ignore it BEFORE resolveFolderPath walks up to the parent
|
||||
if w.shouldIgnoreFolderPath(ctx, fsys, path) {
|
||||
log.Debug(ctx, "Ignoring change matching .ndignore pattern", "libraryID", lib.ID, "path", path)
|
||||
continue
|
||||
}
|
||||
|
||||
// Find the folder to scan - validate path exists as directory, walk up if needed
|
||||
folderPath := resolveFolderPath(fsys, path)
|
||||
// Double-check after resolution in case the resolved path is different and also matches patterns
|
||||
if folderPath != path && w.shouldIgnoreFolderPath(ctx, fsys, folderPath) {
|
||||
log.Trace(ctx, "Ignoring change in folder matching .ndignore pattern", "libraryID", lib.ID, "folderPath", folderPath)
|
||||
continue
|
||||
}
|
||||
|
||||
// Notify the main watcher of changes
|
||||
select {
|
||||
case w.watcherNotify <- *lib:
|
||||
case w.watcherNotify <- scanNotification{Library: lib, FolderPath: folderPath}:
|
||||
default:
|
||||
// Channel is full, notification already pending
|
||||
}
|
||||
@ -228,6 +276,47 @@ func (w *watcher) watchLibrary(ctx context.Context, lib *model.Library) error {
|
||||
}
|
||||
}
|
||||
|
||||
// resolveFolderPath takes a path (which may be a file or directory) and returns
|
||||
// the folder path to scan. If the path is a file, it walks up to find the parent
|
||||
// directory. Returns empty string if the path should scan the library root.
|
||||
func resolveFolderPath(fsys fs.FS, path string) string {
|
||||
// Handle root paths immediately
|
||||
if path == "." || path == "" {
|
||||
return ""
|
||||
}
|
||||
|
||||
folderPath := path
|
||||
for {
|
||||
info, err := fs.Stat(fsys, folderPath)
|
||||
if err == nil && info.IsDir() {
|
||||
// Found a valid directory
|
||||
return folderPath
|
||||
}
|
||||
if folderPath == "." || folderPath == "" {
|
||||
// Reached root, scan entire library
|
||||
return ""
|
||||
}
|
||||
// Walk up the tree
|
||||
dir, _ := filepath.Split(folderPath)
|
||||
if dir == "" || dir == "." {
|
||||
return ""
|
||||
}
|
||||
// Remove trailing slash
|
||||
folderPath = filepath.Clean(dir)
|
||||
}
|
||||
}
|
||||
|
||||
// shouldIgnoreFolderPath checks if the given folderPath should be ignored based on .ndignore patterns
|
||||
// in the library. It pushes all parent folders onto the IgnoreChecker stack before checking.
|
||||
func (w *watcher) shouldIgnoreFolderPath(ctx context.Context, fsys storage.MusicFS, folderPath string) bool {
|
||||
checker := newIgnoreChecker(fsys)
|
||||
err := checker.PushAllParents(ctx, folderPath)
|
||||
if err != nil {
|
||||
log.Warn(ctx, "Watcher: Error pushing ignore patterns for folder", "path", folderPath, err)
|
||||
}
|
||||
return checker.ShouldIgnore(ctx, folderPath)
|
||||
}
|
||||
|
||||
func isIgnoredPath(_ context.Context, _ fs.FS, path string) bool {
|
||||
baseDir, name := filepath.Split(path)
|
||||
switch {
|
||||
|
||||
491
scanner/watcher_test.go
Normal file
491
scanner/watcher_test.go
Normal file
@ -0,0 +1,491 @@
|
||||
package scanner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io/fs"
|
||||
"path/filepath"
|
||||
"testing/fstest"
|
||||
"time"
|
||||
|
||||
"github.com/navidrome/navidrome/conf"
|
||||
"github.com/navidrome/navidrome/conf/configtest"
|
||||
"github.com/navidrome/navidrome/model"
|
||||
"github.com/navidrome/navidrome/tests"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
var _ = Describe("Watcher", func() {
|
||||
var ctx context.Context
|
||||
var cancel context.CancelFunc
|
||||
var mockScanner *tests.MockScanner
|
||||
var mockDS *tests.MockDataStore
|
||||
var w *watcher
|
||||
var lib *model.Library
|
||||
|
||||
BeforeEach(func() {
|
||||
DeferCleanup(configtest.SetupConfig())
|
||||
conf.Server.Scanner.WatcherWait = 50 * time.Millisecond // Short wait for tests
|
||||
|
||||
ctx, cancel = context.WithCancel(context.Background())
|
||||
DeferCleanup(cancel)
|
||||
|
||||
lib = &model.Library{
|
||||
ID: 1,
|
||||
Name: "Test Library",
|
||||
Path: "/test/library",
|
||||
}
|
||||
|
||||
// Set up mocks
|
||||
mockScanner = tests.NewMockScanner()
|
||||
mockDS = &tests.MockDataStore{}
|
||||
mockLibRepo := &tests.MockLibraryRepo{}
|
||||
mockLibRepo.SetData(model.Libraries{*lib})
|
||||
mockDS.MockedLibrary = mockLibRepo
|
||||
|
||||
// Create a new watcher instance (not singleton) for testing
|
||||
w = &watcher{
|
||||
ds: mockDS,
|
||||
scanner: mockScanner,
|
||||
triggerWait: conf.Server.Scanner.WatcherWait,
|
||||
watcherNotify: make(chan scanNotification, 10),
|
||||
libraryWatchers: make(map[int]*libraryWatcherInstance),
|
||||
mainCtx: ctx,
|
||||
}
|
||||
})
|
||||
|
||||
Describe("Target Collection and Deduplication", func() {
|
||||
BeforeEach(func() {
|
||||
// Start watcher in background
|
||||
go func() {
|
||||
_ = w.Run(ctx)
|
||||
}()
|
||||
|
||||
// Give watcher time to initialize
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
})
|
||||
|
||||
It("creates separate targets for different folders", func() {
|
||||
// Send notifications for different folders
|
||||
w.watcherNotify <- scanNotification{Library: lib, FolderPath: "artist1"}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
w.watcherNotify <- scanNotification{Library: lib, FolderPath: "artist2"}
|
||||
|
||||
// Wait for watcher to process and trigger scan
|
||||
Eventually(func() int {
|
||||
return mockScanner.GetScanFoldersCallCount()
|
||||
}, 200*time.Millisecond, 10*time.Millisecond).Should(Equal(1))
|
||||
|
||||
// Verify two targets
|
||||
calls := mockScanner.GetScanFoldersCalls()
|
||||
Expect(calls).To(HaveLen(1))
|
||||
Expect(calls[0].Targets).To(HaveLen(2))
|
||||
|
||||
// Extract folder paths
|
||||
folderPaths := make(map[string]bool)
|
||||
for _, target := range calls[0].Targets {
|
||||
Expect(target.LibraryID).To(Equal(1))
|
||||
folderPaths[target.FolderPath] = true
|
||||
}
|
||||
Expect(folderPaths).To(HaveKey("artist1"))
|
||||
Expect(folderPaths).To(HaveKey("artist2"))
|
||||
})
|
||||
|
||||
It("handles different folder paths correctly", func() {
|
||||
// Send notification for nested folder
|
||||
w.watcherNotify <- scanNotification{Library: lib, FolderPath: "artist1/album1"}
|
||||
|
||||
// Wait for watcher to process and trigger scan
|
||||
Eventually(func() int {
|
||||
return mockScanner.GetScanFoldersCallCount()
|
||||
}, 200*time.Millisecond, 10*time.Millisecond).Should(Equal(1))
|
||||
|
||||
// Verify the target
|
||||
calls := mockScanner.GetScanFoldersCalls()
|
||||
Expect(calls).To(HaveLen(1))
|
||||
Expect(calls[0].Targets).To(HaveLen(1))
|
||||
Expect(calls[0].Targets[0].FolderPath).To(Equal("artist1/album1"))
|
||||
})
|
||||
|
||||
It("deduplicates folder and file within same folder", func() {
|
||||
// Send notification for a folder
|
||||
w.watcherNotify <- scanNotification{Library: lib, FolderPath: "artist1/album1"}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
// Send notification for same folder (as if file change was detected there)
|
||||
// In practice, watchLibrary() would walk up from file path to folder
|
||||
w.watcherNotify <- scanNotification{Library: lib, FolderPath: "artist1/album1"}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
// Send another for same folder
|
||||
w.watcherNotify <- scanNotification{Library: lib, FolderPath: "artist1/album1"}
|
||||
|
||||
// Wait for watcher to process and trigger scan
|
||||
Eventually(func() int {
|
||||
return mockScanner.GetScanFoldersCallCount()
|
||||
}, 200*time.Millisecond, 10*time.Millisecond).Should(Equal(1))
|
||||
|
||||
// Verify only one target despite multiple file/folder changes
|
||||
calls := mockScanner.GetScanFoldersCalls()
|
||||
Expect(calls).To(HaveLen(1))
|
||||
Expect(calls[0].Targets).To(HaveLen(1))
|
||||
Expect(calls[0].Targets[0].FolderPath).To(Equal("artist1/album1"))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Timer Behavior", func() {
|
||||
BeforeEach(func() {
|
||||
// Start watcher in background
|
||||
go func() {
|
||||
_ = w.Run(ctx)
|
||||
}()
|
||||
|
||||
// Give watcher time to initialize
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
})
|
||||
|
||||
It("resets timer on each change (debouncing)", func() {
|
||||
// Send first notification
|
||||
w.watcherNotify <- scanNotification{Library: lib, FolderPath: "artist1"}
|
||||
|
||||
// Wait a bit less than half the watcher wait time to ensure timer doesn't fire
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
|
||||
// No scan should have been triggered yet
|
||||
Expect(mockScanner.GetScanFoldersCallCount()).To(Equal(0))
|
||||
|
||||
// Send another notification (resets timer)
|
||||
w.watcherNotify <- scanNotification{Library: lib, FolderPath: "artist1"}
|
||||
|
||||
// Wait a bit less than half the watcher wait time again
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
|
||||
// Still no scan
|
||||
Expect(mockScanner.GetScanFoldersCallCount()).To(Equal(0))
|
||||
|
||||
// Wait for full timer to expire after last notification (plus margin)
|
||||
time.Sleep(60 * time.Millisecond)
|
||||
|
||||
// Now scan should have been triggered
|
||||
Eventually(func() int {
|
||||
return mockScanner.GetScanFoldersCallCount()
|
||||
}, 100*time.Millisecond, 10*time.Millisecond).Should(Equal(1))
|
||||
})
|
||||
|
||||
It("triggers scan after quiet period", func() {
|
||||
// Send notification
|
||||
w.watcherNotify <- scanNotification{Library: lib, FolderPath: "artist1"}
|
||||
|
||||
// No scan immediately
|
||||
Expect(mockScanner.GetScanFoldersCallCount()).To(Equal(0))
|
||||
|
||||
// Wait for quiet period
|
||||
Eventually(func() int {
|
||||
return mockScanner.GetScanFoldersCallCount()
|
||||
}, 200*time.Millisecond, 10*time.Millisecond).Should(Equal(1))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Empty and Root Paths", func() {
|
||||
BeforeEach(func() {
|
||||
// Start watcher in background
|
||||
go func() {
|
||||
_ = w.Run(ctx)
|
||||
}()
|
||||
|
||||
// Give watcher time to initialize
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
})
|
||||
|
||||
It("handles empty folder path (library root)", func() {
|
||||
// Send notification with empty folder path
|
||||
w.watcherNotify <- scanNotification{Library: lib, FolderPath: ""}
|
||||
|
||||
// Wait for scan
|
||||
Eventually(func() int {
|
||||
return mockScanner.GetScanFoldersCallCount()
|
||||
}, 200*time.Millisecond, 10*time.Millisecond).Should(Equal(1))
|
||||
|
||||
// Should scan the library root
|
||||
calls := mockScanner.GetScanFoldersCalls()
|
||||
Expect(calls).To(HaveLen(1))
|
||||
Expect(calls[0].Targets).To(HaveLen(1))
|
||||
Expect(calls[0].Targets[0].FolderPath).To(Equal(""))
|
||||
})
|
||||
|
||||
It("deduplicates empty and dot paths", func() {
|
||||
// Send notifications with empty and dot paths
|
||||
w.watcherNotify <- scanNotification{Library: lib, FolderPath: ""}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
w.watcherNotify <- scanNotification{Library: lib, FolderPath: ""}
|
||||
|
||||
// Wait for scan
|
||||
Eventually(func() int {
|
||||
return mockScanner.GetScanFoldersCallCount()
|
||||
}, 200*time.Millisecond, 10*time.Millisecond).Should(Equal(1))
|
||||
|
||||
// Should have only one target
|
||||
calls := mockScanner.GetScanFoldersCalls()
|
||||
Expect(calls).To(HaveLen(1))
|
||||
Expect(calls[0].Targets).To(HaveLen(1))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Multiple Libraries", func() {
|
||||
var lib2 *model.Library
|
||||
|
||||
BeforeEach(func() {
|
||||
// Create second library
|
||||
lib2 = &model.Library{
|
||||
ID: 2,
|
||||
Name: "Test Library 2",
|
||||
Path: "/test/library2",
|
||||
}
|
||||
|
||||
mockLibRepo := mockDS.MockedLibrary.(*tests.MockLibraryRepo)
|
||||
mockLibRepo.SetData(model.Libraries{*lib, *lib2})
|
||||
|
||||
// Start watcher in background
|
||||
go func() {
|
||||
_ = w.Run(ctx)
|
||||
}()
|
||||
|
||||
// Give watcher time to initialize
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
})
|
||||
|
||||
It("creates separate targets for different libraries", func() {
|
||||
// Send notifications for both libraries
|
||||
w.watcherNotify <- scanNotification{Library: lib, FolderPath: "artist1"}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
w.watcherNotify <- scanNotification{Library: lib2, FolderPath: "artist2"}
|
||||
|
||||
// Wait for scan
|
||||
Eventually(func() int {
|
||||
return mockScanner.GetScanFoldersCallCount()
|
||||
}, 200*time.Millisecond, 10*time.Millisecond).Should(Equal(1))
|
||||
|
||||
// Verify two targets for different libraries
|
||||
calls := mockScanner.GetScanFoldersCalls()
|
||||
Expect(calls).To(HaveLen(1))
|
||||
Expect(calls[0].Targets).To(HaveLen(2))
|
||||
|
||||
// Verify library IDs are different
|
||||
libraryIDs := make(map[int]bool)
|
||||
for _, target := range calls[0].Targets {
|
||||
libraryIDs[target.LibraryID] = true
|
||||
}
|
||||
Expect(libraryIDs).To(HaveKey(1))
|
||||
Expect(libraryIDs).To(HaveKey(2))
|
||||
})
|
||||
})
|
||||
|
||||
Describe(".ndignore handling", func() {
|
||||
var ctx context.Context
|
||||
var cancel context.CancelFunc
|
||||
var w *watcher
|
||||
var mockFS *mockMusicFS
|
||||
var lib *model.Library
|
||||
var eventChan chan string
|
||||
var absLibPath string
|
||||
|
||||
BeforeEach(func() {
|
||||
ctx, cancel = context.WithCancel(GinkgoT().Context())
|
||||
DeferCleanup(cancel)
|
||||
|
||||
// Set up library
|
||||
var err error
|
||||
absLibPath, err = filepath.Abs(".")
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
lib = &model.Library{
|
||||
ID: 1,
|
||||
Name: "Test Library",
|
||||
Path: absLibPath,
|
||||
}
|
||||
|
||||
// Create watcher with notification channel
|
||||
w = &watcher{
|
||||
watcherNotify: make(chan scanNotification, 10),
|
||||
}
|
||||
|
||||
eventChan = make(chan string, 10)
|
||||
})
|
||||
|
||||
// Helper to send an event - converts relative path to absolute
|
||||
sendEvent := func(relativePath string) {
|
||||
path := filepath.Join(absLibPath, relativePath)
|
||||
eventChan <- path
|
||||
}
|
||||
|
||||
// Helper to start the real event processing loop
|
||||
startEventProcessing := func() {
|
||||
go func() {
|
||||
defer GinkgoRecover()
|
||||
// Call the actual processLibraryEvents method - testing the real implementation!
|
||||
_ = w.processLibraryEvents(ctx, lib, mockFS, eventChan, absLibPath)
|
||||
}()
|
||||
}
|
||||
|
||||
Context("when a folder matching .ndignore is deleted", func() {
|
||||
BeforeEach(func() {
|
||||
// Create filesystem with .ndignore containing _TEMP pattern
|
||||
// The deleted folder (_TEMP) will NOT exist in the filesystem
|
||||
mockFS = &mockMusicFS{
|
||||
FS: fstest.MapFS{
|
||||
"rock": &fstest.MapFile{Mode: fs.ModeDir},
|
||||
"rock/.ndignore": &fstest.MapFile{Data: []byte("_TEMP\n")},
|
||||
"rock/valid_album": &fstest.MapFile{Mode: fs.ModeDir},
|
||||
"rock/valid_album/track.mp3": &fstest.MapFile{Data: []byte("audio")},
|
||||
},
|
||||
}
|
||||
})
|
||||
|
||||
It("should NOT send scan notification when deleted folder matches .ndignore", func() {
|
||||
startEventProcessing()
|
||||
|
||||
// Simulate deletion event for rock/_TEMP
|
||||
sendEvent("rock/_TEMP")
|
||||
|
||||
// Wait a bit to ensure event is processed
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
|
||||
// No notification should have been sent
|
||||
Consistently(eventChan, 100*time.Millisecond).Should(BeEmpty())
|
||||
})
|
||||
|
||||
It("should send scan notification for valid folder deletion", func() {
|
||||
startEventProcessing()
|
||||
|
||||
// Simulate deletion event for rock/other_folder (not in .ndignore and doesn't exist)
|
||||
// Since it doesn't exist in mockFS, resolveFolderPath will walk up to "rock"
|
||||
sendEvent("rock/other_folder")
|
||||
|
||||
// Should receive notification for parent folder
|
||||
Eventually(w.watcherNotify, 200*time.Millisecond).Should(Receive(Equal(scanNotification{
|
||||
Library: lib,
|
||||
FolderPath: "rock",
|
||||
})))
|
||||
})
|
||||
})
|
||||
|
||||
Context("with nested folder patterns", func() {
|
||||
BeforeEach(func() {
|
||||
mockFS = &mockMusicFS{
|
||||
FS: fstest.MapFS{
|
||||
"music": &fstest.MapFile{Mode: fs.ModeDir},
|
||||
"music/.ndignore": &fstest.MapFile{Data: []byte("**/temp\n**/cache\n")},
|
||||
"music/rock": &fstest.MapFile{Mode: fs.ModeDir},
|
||||
"music/rock/artist": &fstest.MapFile{Mode: fs.ModeDir},
|
||||
},
|
||||
}
|
||||
})
|
||||
|
||||
It("should NOT send notification when nested ignored folder is deleted", func() {
|
||||
startEventProcessing()
|
||||
|
||||
// Simulate deletion of music/rock/artist/temp (matches **/temp)
|
||||
sendEvent("music/rock/artist/temp")
|
||||
|
||||
// Wait to ensure event is processed
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
|
||||
// No notification should be sent
|
||||
Expect(w.watcherNotify).To(BeEmpty(), "Expected no scan notification for nested ignored folder")
|
||||
})
|
||||
|
||||
It("should send notification for non-ignored nested folder", func() {
|
||||
startEventProcessing()
|
||||
|
||||
// Simulate change in music/rock/artist (doesn't match any pattern)
|
||||
sendEvent("music/rock/artist")
|
||||
|
||||
// Should receive notification
|
||||
Eventually(w.watcherNotify, 200*time.Millisecond).Should(Receive(Equal(scanNotification{
|
||||
Library: lib,
|
||||
FolderPath: "music/rock/artist",
|
||||
})))
|
||||
})
|
||||
})
|
||||
|
||||
Context("with file events in ignored folders", func() {
|
||||
BeforeEach(func() {
|
||||
mockFS = &mockMusicFS{
|
||||
FS: fstest.MapFS{
|
||||
"rock": &fstest.MapFile{Mode: fs.ModeDir},
|
||||
"rock/.ndignore": &fstest.MapFile{Data: []byte("_TEMP\n")},
|
||||
},
|
||||
}
|
||||
})
|
||||
|
||||
It("should NOT send notification for file changes in ignored folders", func() {
|
||||
startEventProcessing()
|
||||
|
||||
// Simulate file change in rock/_TEMP/file.mp3
|
||||
sendEvent("rock/_TEMP/file.mp3")
|
||||
|
||||
// Wait to ensure event is processed
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
|
||||
// No notification should be sent
|
||||
Expect(w.watcherNotify).To(BeEmpty(), "Expected no scan notification for file in ignored folder")
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
var _ = Describe("resolveFolderPath", func() {
|
||||
var mockFS fs.FS
|
||||
|
||||
BeforeEach(func() {
|
||||
// Create a mock filesystem with some directories and files
|
||||
mockFS = fstest.MapFS{
|
||||
"artist1": &fstest.MapFile{Mode: fs.ModeDir},
|
||||
"artist1/album1": &fstest.MapFile{Mode: fs.ModeDir},
|
||||
"artist1/album1/track1.mp3": &fstest.MapFile{Data: []byte("audio")},
|
||||
"artist1/album1/track2.mp3": &fstest.MapFile{Data: []byte("audio")},
|
||||
"artist1/album2": &fstest.MapFile{Mode: fs.ModeDir},
|
||||
"artist1/album2/song.flac": &fstest.MapFile{Data: []byte("audio")},
|
||||
"artist2": &fstest.MapFile{Mode: fs.ModeDir},
|
||||
"artist2/cover.jpg": &fstest.MapFile{Data: []byte("image")},
|
||||
}
|
||||
})
|
||||
|
||||
It("returns directory path when given a directory", func() {
|
||||
result := resolveFolderPath(mockFS, "artist1/album1")
|
||||
Expect(result).To(Equal("artist1/album1"))
|
||||
})
|
||||
|
||||
It("walks up to parent directory when given a file path", func() {
|
||||
result := resolveFolderPath(mockFS, "artist1/album1/track1.mp3")
|
||||
Expect(result).To(Equal("artist1/album1"))
|
||||
})
|
||||
|
||||
It("walks up multiple levels if needed", func() {
|
||||
result := resolveFolderPath(mockFS, "artist1/album1/nonexistent/file.mp3")
|
||||
Expect(result).To(Equal("artist1/album1"))
|
||||
})
|
||||
|
||||
It("returns empty string for non-existent paths at root", func() {
|
||||
result := resolveFolderPath(mockFS, "nonexistent/path/file.mp3")
|
||||
Expect(result).To(Equal(""))
|
||||
})
|
||||
|
||||
It("returns empty string for dot path", func() {
|
||||
result := resolveFolderPath(mockFS, ".")
|
||||
Expect(result).To(Equal(""))
|
||||
})
|
||||
|
||||
It("returns empty string for empty path", func() {
|
||||
result := resolveFolderPath(mockFS, "")
|
||||
Expect(result).To(Equal(""))
|
||||
})
|
||||
|
||||
It("handles nested file paths correctly", func() {
|
||||
result := resolveFolderPath(mockFS, "artist1/album2/song.flac")
|
||||
Expect(result).To(Equal("artist1/album2"))
|
||||
})
|
||||
|
||||
It("resolves to top-level directory", func() {
|
||||
result := resolveFolderPath(mockFS, "artist2/cover.jpg")
|
||||
Expect(result).To(Equal("artist2"))
|
||||
})
|
||||
})
|
||||
@ -1,8 +1,11 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"cmp"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"encoding/pem"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
@ -69,6 +72,13 @@ func (s *Server) Run(ctx context.Context, addr string, port int, tlsCert string,
|
||||
// Determine if TLS is enabled
|
||||
tlsEnabled := tlsCert != "" && tlsKey != ""
|
||||
|
||||
// Validate TLS certificates before starting the server
|
||||
if tlsEnabled {
|
||||
if err := validateTLSCertificates(tlsCert, tlsKey); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Create a listener based on the address type (either Unix socket or TCP)
|
||||
var listener net.Listener
|
||||
var err error
|
||||
@ -89,17 +99,17 @@ func (s *Server) Run(ctx context.Context, addr string, port int, tlsCert string,
|
||||
// Start the server in a new goroutine and send an error signal to errC if there's an error
|
||||
errC := make(chan error)
|
||||
go func() {
|
||||
var err error
|
||||
if tlsEnabled {
|
||||
// Start the HTTPS server
|
||||
log.Info("Starting server with TLS (HTTPS) enabled", "tlsCert", tlsCert, "tlsKey", tlsKey)
|
||||
if err := server.ServeTLS(listener, tlsCert, tlsKey); !errors.Is(err, http.ErrServerClosed) {
|
||||
errC <- err
|
||||
}
|
||||
err = server.ServeTLS(listener, tlsCert, tlsKey)
|
||||
} else {
|
||||
// Start the HTTP server
|
||||
if err := server.Serve(listener); !errors.Is(err, http.ErrServerClosed) {
|
||||
errC <- err
|
||||
}
|
||||
err = server.Serve(listener)
|
||||
}
|
||||
if !errors.Is(err, http.ErrServerClosed) {
|
||||
errC <- err
|
||||
}
|
||||
}()
|
||||
|
||||
@ -249,3 +259,56 @@ func AbsoluteURL(r *http.Request, u string, params url.Values) string {
|
||||
}
|
||||
return buildUrl.String()
|
||||
}
|
||||
|
||||
// validateTLSCertificates validates the TLS certificate and key files before starting the server.
|
||||
// It provides detailed error messages for common issues like encrypted private keys.
|
||||
func validateTLSCertificates(certFile, keyFile string) error {
|
||||
// Read the key file to check for encryption
|
||||
keyData, err := os.ReadFile(keyFile)
|
||||
if err != nil {
|
||||
return fmt.Errorf("reading TLS key file: %w", err)
|
||||
}
|
||||
|
||||
// Parse PEM blocks and check for encryption
|
||||
block, _ := pem.Decode(keyData)
|
||||
if block == nil {
|
||||
return errors.New("TLS key file does not contain a valid PEM block")
|
||||
}
|
||||
|
||||
// Check for encrypted private key indicators
|
||||
if isEncryptedPEM(block, keyData) {
|
||||
return errors.New("TLS private key is encrypted (password-protected). " +
|
||||
"Navidrome does not support encrypted private keys. " +
|
||||
"Please decrypt your key using: openssl pkey -in <encrypted-key> -out <decrypted-key>")
|
||||
}
|
||||
|
||||
// Try to load the certificate pair to validate it
|
||||
_, err = tls.LoadX509KeyPair(certFile, keyFile)
|
||||
if err != nil {
|
||||
return fmt.Errorf("loading TLS certificate/key pair: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// isEncryptedPEM checks if a PEM block represents an encrypted private key.
|
||||
func isEncryptedPEM(block *pem.Block, rawData []byte) bool {
|
||||
// Check for PKCS#8 encrypted format (BEGIN ENCRYPTED PRIVATE KEY)
|
||||
if block.Type == "ENCRYPTED PRIVATE KEY" {
|
||||
return true
|
||||
}
|
||||
|
||||
// Check for legacy encrypted format with Proc-Type header
|
||||
if block.Headers != nil {
|
||||
if procType, ok := block.Headers["Proc-Type"]; ok && strings.Contains(procType, "ENCRYPTED") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// Also check raw data for DEK-Info header (in case pem.Decode doesn't parse headers correctly)
|
||||
if bytes.Contains(rawData, []byte("DEK-Info:")) || bytes.Contains(rawData, []byte("Proc-Type: 4,ENCRYPTED")) {
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
@ -1,13 +1,20 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/navidrome/navidrome/conf"
|
||||
"github.com/navidrome/navidrome/conf/configtest"
|
||||
"github.com/navidrome/navidrome/tests"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
@ -107,3 +114,146 @@ var _ = Describe("createUnixSocketFile", func() {
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
var _ = Describe("TLS support", func() {
|
||||
Describe("validateTLSCertificates", func() {
|
||||
const testDataDir = "server/testdata"
|
||||
|
||||
When("certificate and key are valid and unencrypted", func() {
|
||||
It("returns nil", func() {
|
||||
certFile := filepath.Join(testDataDir, "test_cert.pem")
|
||||
keyFile := filepath.Join(testDataDir, "test_key.pem")
|
||||
err := validateTLSCertificates(certFile, keyFile)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
})
|
||||
|
||||
When("private key is encrypted with PKCS#8 format", func() {
|
||||
It("returns an error with helpful message", func() {
|
||||
certFile := filepath.Join(testDataDir, "test_cert_encrypted.pem")
|
||||
keyFile := filepath.Join(testDataDir, "test_key_encrypted.pem")
|
||||
err := validateTLSCertificates(certFile, keyFile)
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring("encrypted"))
|
||||
Expect(err.Error()).To(ContainSubstring("openssl"))
|
||||
})
|
||||
})
|
||||
|
||||
When("private key is encrypted with legacy format (Proc-Type header)", func() {
|
||||
It("returns an error with helpful message", func() {
|
||||
certFile := filepath.Join(testDataDir, "test_cert.pem")
|
||||
keyFile := filepath.Join(testDataDir, "test_key_encrypted_legacy.pem")
|
||||
err := validateTLSCertificates(certFile, keyFile)
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring("encrypted"))
|
||||
Expect(err.Error()).To(ContainSubstring("openssl"))
|
||||
})
|
||||
})
|
||||
|
||||
When("key file does not exist", func() {
|
||||
It("returns an error", func() {
|
||||
certFile := filepath.Join(testDataDir, "test_cert.pem")
|
||||
keyFile := filepath.Join(testDataDir, "nonexistent.pem")
|
||||
err := validateTLSCertificates(certFile, keyFile)
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring("reading TLS key file"))
|
||||
})
|
||||
})
|
||||
|
||||
When("key file does not contain valid PEM", func() {
|
||||
It("returns an error", func() {
|
||||
// Create a temp file with invalid PEM content
|
||||
tmpFile, err := os.CreateTemp("", "invalid_key*.pem")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
DeferCleanup(func() {
|
||||
_ = os.Remove(tmpFile.Name())
|
||||
})
|
||||
_, err = tmpFile.WriteString("not a valid PEM file")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
_ = tmpFile.Close()
|
||||
|
||||
certFile := filepath.Join(testDataDir, "test_cert.pem")
|
||||
err = validateTLSCertificates(certFile, tmpFile.Name())
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring("valid PEM block"))
|
||||
})
|
||||
})
|
||||
|
||||
When("certificate file does not exist", func() {
|
||||
It("returns an error from tls.LoadX509KeyPair", func() {
|
||||
certFile := filepath.Join(testDataDir, "nonexistent_cert.pem")
|
||||
keyFile := filepath.Join(testDataDir, "test_key.pem")
|
||||
err := validateTLSCertificates(certFile, keyFile)
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring("loading TLS certificate/key pair"))
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Server TLS", func() {
|
||||
const testDataDir = "server/testdata"
|
||||
|
||||
When("server is started with valid TLS certificates", func() {
|
||||
It("accepts HTTPS connections", func() {
|
||||
DeferCleanup(configtest.SetupConfig())
|
||||
|
||||
// Create server with mock dependencies
|
||||
ds := &tests.MockDataStore{}
|
||||
server := New(ds, nil, nil)
|
||||
|
||||
// Load the test certificate to create a trusted CA pool
|
||||
certFile := filepath.Join(testDataDir, "test_cert.pem")
|
||||
keyFile := filepath.Join(testDataDir, "test_key.pem")
|
||||
caCert, err := os.ReadFile(certFile)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
caCertPool := x509.NewCertPool()
|
||||
caCertPool.AppendCertsFromPEM(caCert)
|
||||
|
||||
// Create an HTTPS client that trusts our test certificate
|
||||
httpClient := &http.Client{
|
||||
Timeout: 5 * time.Second,
|
||||
Transport: &http.Transport{
|
||||
TLSClientConfig: &tls.Config{
|
||||
RootCAs: caCertPool,
|
||||
MinVersion: tls.VersionTLS12,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
// Start the server in a goroutine
|
||||
ctx, cancel := context.WithCancel(GinkgoT().Context())
|
||||
defer cancel()
|
||||
|
||||
errChan := make(chan error, 1)
|
||||
go func() {
|
||||
errChan <- server.Run(ctx, "127.0.0.1", 14534, certFile, keyFile)
|
||||
}()
|
||||
|
||||
Eventually(func() error {
|
||||
// Make an HTTPS request to the server
|
||||
resp, err := httpClient.Get("https://127.0.0.1:14534/ping")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("unexpected status code: %d", resp.StatusCode)
|
||||
}
|
||||
return nil
|
||||
}, 2*time.Second, 100*time.Millisecond).Should(Succeed())
|
||||
|
||||
// Stop the server
|
||||
cancel()
|
||||
|
||||
// Wait for server to stop (with timeout)
|
||||
select {
|
||||
case <-errChan:
|
||||
// Server stopped
|
||||
case <-time.After(2 * time.Second):
|
||||
Fail("Server did not stop in time")
|
||||
}
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@ -18,7 +18,6 @@ import (
|
||||
"github.com/navidrome/navidrome/core/scrobbler"
|
||||
"github.com/navidrome/navidrome/log"
|
||||
"github.com/navidrome/navidrome/model"
|
||||
"github.com/navidrome/navidrome/scanner"
|
||||
"github.com/navidrome/navidrome/server"
|
||||
"github.com/navidrome/navidrome/server/events"
|
||||
"github.com/navidrome/navidrome/server/subsonic/responses"
|
||||
@ -39,7 +38,7 @@ type Router struct {
|
||||
players core.Players
|
||||
provider external.Provider
|
||||
playlists core.Playlists
|
||||
scanner scanner.Scanner
|
||||
scanner model.Scanner
|
||||
broker events.Broker
|
||||
scrobbler scrobbler.PlayTracker
|
||||
share core.Share
|
||||
@ -48,7 +47,7 @@ type Router struct {
|
||||
}
|
||||
|
||||
func New(ds model.DataStore, artwork artwork.Artwork, streamer core.MediaStreamer, archiver core.Archiver,
|
||||
players core.Players, provider external.Provider, scanner scanner.Scanner, broker events.Broker,
|
||||
players core.Players, provider external.Provider, scanner model.Scanner, broker events.Broker,
|
||||
playlists core.Playlists, scrobbler scrobbler.PlayTracker, share core.Share, playback playback.PlaybackServer,
|
||||
metrics metrics.Metrics,
|
||||
) *Router {
|
||||
|
||||
@ -1,10 +1,13 @@
|
||||
package subsonic
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
"github.com/navidrome/navidrome/log"
|
||||
"github.com/navidrome/navidrome/model"
|
||||
"github.com/navidrome/navidrome/model/request"
|
||||
"github.com/navidrome/navidrome/server/subsonic/responses"
|
||||
"github.com/navidrome/navidrome/utils/req"
|
||||
@ -44,15 +47,56 @@ func (api *Router) StartScan(r *http.Request) (*responses.Subsonic, error) {
|
||||
p := req.Params(r)
|
||||
fullScan := p.BoolOr("fullScan", false)
|
||||
|
||||
// Parse optional target parameters for selective scanning
|
||||
var targets []model.ScanTarget
|
||||
if targetParams, err := p.Strings("target"); err == nil && len(targetParams) > 0 {
|
||||
targets, err = model.ParseTargets(targetParams)
|
||||
if err != nil {
|
||||
return nil, newError(responses.ErrorGeneric, fmt.Sprintf("Invalid target parameter: %v", err))
|
||||
}
|
||||
|
||||
// Validate all libraries in targets exist and user has access to them
|
||||
userLibraries, err := api.ds.User(ctx).GetUserLibraries(loggedUser.ID)
|
||||
if err != nil {
|
||||
return nil, newError(responses.ErrorGeneric, "Internal error")
|
||||
}
|
||||
|
||||
// Check each target library
|
||||
for _, target := range targets {
|
||||
if !slices.ContainsFunc(userLibraries, func(lib model.Library) bool { return lib.ID == target.LibraryID }) {
|
||||
return nil, newError(responses.ErrorDataNotFound, fmt.Sprintf("Library with ID %d not found", target.LibraryID))
|
||||
}
|
||||
}
|
||||
|
||||
// Special case: if single library with empty path and it's the only library in DB, call ScanAll
|
||||
if len(targets) == 1 && targets[0].FolderPath == "" {
|
||||
allLibs, err := api.ds.Library(ctx).GetAll()
|
||||
if err != nil {
|
||||
return nil, newError(responses.ErrorGeneric, "Internal error")
|
||||
}
|
||||
if len(allLibs) == 1 {
|
||||
targets = nil // This will trigger ScanAll below
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
go func() {
|
||||
start := time.Now()
|
||||
log.Info(ctx, "Triggering manual scan", "fullScan", fullScan, "user", loggedUser.UserName)
|
||||
_, err := api.scanner.ScanAll(ctx, fullScan)
|
||||
var err error
|
||||
|
||||
if len(targets) > 0 {
|
||||
log.Info(ctx, "Triggering on-demand scan", "fullScan", fullScan, "targets", len(targets), "user", loggedUser.UserName)
|
||||
_, err = api.scanner.ScanFolders(ctx, fullScan, targets)
|
||||
} else {
|
||||
log.Info(ctx, "Triggering on-demand scan", "fullScan", fullScan, "user", loggedUser.UserName)
|
||||
_, err = api.scanner.ScanAll(ctx, fullScan)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
log.Error(ctx, "Error scanning", err)
|
||||
return
|
||||
}
|
||||
log.Info(ctx, "Manual scan complete", "user", loggedUser.UserName, "elapsed", time.Since(start))
|
||||
log.Info(ctx, "On-demand scan complete", "user", loggedUser.UserName, "elapsed", time.Since(start))
|
||||
}()
|
||||
|
||||
return api.GetScanStatus(r)
|
||||
|
||||
396
server/subsonic/library_scanning_test.go
Normal file
396
server/subsonic/library_scanning_test.go
Normal file
@ -0,0 +1,396 @@
|
||||
package subsonic
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http/httptest"
|
||||
|
||||
"github.com/navidrome/navidrome/model"
|
||||
"github.com/navidrome/navidrome/model/request"
|
||||
"github.com/navidrome/navidrome/server/subsonic/responses"
|
||||
"github.com/navidrome/navidrome/tests"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
var _ = Describe("LibraryScanning", func() {
|
||||
var api *Router
|
||||
var ms *tests.MockScanner
|
||||
|
||||
BeforeEach(func() {
|
||||
ms = tests.NewMockScanner()
|
||||
api = &Router{scanner: ms}
|
||||
})
|
||||
|
||||
Describe("StartScan", func() {
|
||||
It("requires admin authentication", func() {
|
||||
// Create non-admin user
|
||||
ctx := request.WithUser(context.Background(), model.User{
|
||||
ID: "user-id",
|
||||
IsAdmin: false,
|
||||
})
|
||||
|
||||
// Create request
|
||||
r := httptest.NewRequest("GET", "/rest/startScan", nil)
|
||||
r = r.WithContext(ctx)
|
||||
|
||||
// Call endpoint
|
||||
response, err := api.StartScan(r)
|
||||
|
||||
// Should return authorization error
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(response).To(BeNil())
|
||||
var subErr subError
|
||||
ok := errors.As(err, &subErr)
|
||||
Expect(ok).To(BeTrue())
|
||||
Expect(subErr.code).To(Equal(responses.ErrorAuthorizationFail))
|
||||
})
|
||||
|
||||
It("triggers a full scan with no parameters", func() {
|
||||
// Create admin user
|
||||
ctx := request.WithUser(context.Background(), model.User{
|
||||
ID: "admin-id",
|
||||
IsAdmin: true,
|
||||
})
|
||||
|
||||
// Create request with no parameters
|
||||
r := httptest.NewRequest("GET", "/rest/startScan", nil)
|
||||
r = r.WithContext(ctx)
|
||||
|
||||
// Call endpoint
|
||||
response, err := api.StartScan(r)
|
||||
|
||||
// Should succeed
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(response).ToNot(BeNil())
|
||||
|
||||
// Verify ScanAll was called (eventually, since it's in a goroutine)
|
||||
Eventually(func() int {
|
||||
return ms.GetScanAllCallCount()
|
||||
}).Should(BeNumerically(">", 0))
|
||||
calls := ms.GetScanAllCalls()
|
||||
Expect(calls).To(HaveLen(1))
|
||||
Expect(calls[0].FullScan).To(BeFalse())
|
||||
})
|
||||
|
||||
It("triggers a full scan with fullScan=true", func() {
|
||||
// Create admin user
|
||||
ctx := request.WithUser(context.Background(), model.User{
|
||||
ID: "admin-id",
|
||||
IsAdmin: true,
|
||||
})
|
||||
|
||||
// Create request with fullScan parameter
|
||||
r := httptest.NewRequest("GET", "/rest/startScan?fullScan=true", nil)
|
||||
r = r.WithContext(ctx)
|
||||
|
||||
// Call endpoint
|
||||
response, err := api.StartScan(r)
|
||||
|
||||
// Should succeed
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(response).ToNot(BeNil())
|
||||
|
||||
// Verify ScanAll was called with fullScan=true
|
||||
Eventually(func() int {
|
||||
return ms.GetScanAllCallCount()
|
||||
}).Should(BeNumerically(">", 0))
|
||||
calls := ms.GetScanAllCalls()
|
||||
Expect(calls).To(HaveLen(1))
|
||||
Expect(calls[0].FullScan).To(BeTrue())
|
||||
})
|
||||
|
||||
It("triggers a selective scan with single target parameter", func() {
|
||||
// Setup mocks
|
||||
mockUserRepo := tests.CreateMockUserRepo()
|
||||
_ = mockUserRepo.SetUserLibraries("admin-id", []int{1, 2})
|
||||
mockDS := &tests.MockDataStore{MockedUser: mockUserRepo}
|
||||
api.ds = mockDS
|
||||
|
||||
// Create admin user
|
||||
ctx := request.WithUser(context.Background(), model.User{
|
||||
ID: "admin-id",
|
||||
IsAdmin: true,
|
||||
})
|
||||
|
||||
// Create request with single target parameter
|
||||
r := httptest.NewRequest("GET", "/rest/startScan?target=1:Music/Rock", nil)
|
||||
r = r.WithContext(ctx)
|
||||
|
||||
// Call endpoint
|
||||
response, err := api.StartScan(r)
|
||||
|
||||
// Should succeed
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(response).ToNot(BeNil())
|
||||
|
||||
// Verify ScanFolders was called with correct targets
|
||||
Eventually(func() int {
|
||||
return ms.GetScanFoldersCallCount()
|
||||
}).Should(BeNumerically(">", 0))
|
||||
calls := ms.GetScanFoldersCalls()
|
||||
Expect(calls).To(HaveLen(1))
|
||||
targets := calls[0].Targets
|
||||
Expect(targets).To(HaveLen(1))
|
||||
Expect(targets[0].LibraryID).To(Equal(1))
|
||||
Expect(targets[0].FolderPath).To(Equal("Music/Rock"))
|
||||
})
|
||||
|
||||
It("triggers a selective scan with multiple target parameters", func() {
|
||||
// Setup mocks
|
||||
mockUserRepo := tests.CreateMockUserRepo()
|
||||
_ = mockUserRepo.SetUserLibraries("admin-id", []int{1, 2})
|
||||
mockDS := &tests.MockDataStore{MockedUser: mockUserRepo}
|
||||
api.ds = mockDS
|
||||
|
||||
// Create admin user
|
||||
ctx := request.WithUser(context.Background(), model.User{
|
||||
ID: "admin-id",
|
||||
IsAdmin: true,
|
||||
})
|
||||
|
||||
// Create request with multiple target parameters
|
||||
r := httptest.NewRequest("GET", "/rest/startScan?target=1:Music/Reggae&target=2:Classical/Bach", nil)
|
||||
r = r.WithContext(ctx)
|
||||
|
||||
// Call endpoint
|
||||
response, err := api.StartScan(r)
|
||||
|
||||
// Should succeed
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(response).ToNot(BeNil())
|
||||
|
||||
// Verify ScanFolders was called with correct targets
|
||||
Eventually(func() int {
|
||||
return ms.GetScanFoldersCallCount()
|
||||
}).Should(BeNumerically(">", 0))
|
||||
calls := ms.GetScanFoldersCalls()
|
||||
Expect(calls).To(HaveLen(1))
|
||||
targets := calls[0].Targets
|
||||
Expect(targets).To(HaveLen(2))
|
||||
Expect(targets[0].LibraryID).To(Equal(1))
|
||||
Expect(targets[0].FolderPath).To(Equal("Music/Reggae"))
|
||||
Expect(targets[1].LibraryID).To(Equal(2))
|
||||
Expect(targets[1].FolderPath).To(Equal("Classical/Bach"))
|
||||
})
|
||||
|
||||
It("triggers a selective full scan with target and fullScan parameters", func() {
|
||||
// Setup mocks
|
||||
mockUserRepo := tests.CreateMockUserRepo()
|
||||
_ = mockUserRepo.SetUserLibraries("admin-id", []int{1})
|
||||
mockDS := &tests.MockDataStore{MockedUser: mockUserRepo}
|
||||
api.ds = mockDS
|
||||
|
||||
// Create admin user
|
||||
ctx := request.WithUser(context.Background(), model.User{
|
||||
ID: "admin-id",
|
||||
IsAdmin: true,
|
||||
})
|
||||
|
||||
// Create request with target and fullScan parameters
|
||||
r := httptest.NewRequest("GET", "/rest/startScan?target=1:Music/Jazz&fullScan=true", nil)
|
||||
r = r.WithContext(ctx)
|
||||
|
||||
// Call endpoint
|
||||
response, err := api.StartScan(r)
|
||||
|
||||
// Should succeed
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(response).ToNot(BeNil())
|
||||
|
||||
// Verify ScanFolders was called with fullScan=true
|
||||
Eventually(func() int {
|
||||
return ms.GetScanFoldersCallCount()
|
||||
}).Should(BeNumerically(">", 0))
|
||||
calls := ms.GetScanFoldersCalls()
|
||||
Expect(calls).To(HaveLen(1))
|
||||
Expect(calls[0].FullScan).To(BeTrue())
|
||||
targets := calls[0].Targets
|
||||
Expect(targets).To(HaveLen(1))
|
||||
})
|
||||
|
||||
It("returns error for invalid target format", func() {
|
||||
// Create admin user
|
||||
ctx := request.WithUser(context.Background(), model.User{
|
||||
ID: "admin-id",
|
||||
IsAdmin: true,
|
||||
})
|
||||
|
||||
// Create request with invalid target format (missing colon)
|
||||
r := httptest.NewRequest("GET", "/rest/startScan?target=1MusicRock", nil)
|
||||
r = r.WithContext(ctx)
|
||||
|
||||
// Call endpoint
|
||||
response, err := api.StartScan(r)
|
||||
|
||||
// Should return error
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(response).To(BeNil())
|
||||
var subErr subError
|
||||
ok := errors.As(err, &subErr)
|
||||
Expect(ok).To(BeTrue())
|
||||
Expect(subErr.code).To(Equal(responses.ErrorGeneric))
|
||||
})
|
||||
|
||||
It("returns error for invalid library ID in target", func() {
|
||||
// Create admin user
|
||||
ctx := request.WithUser(context.Background(), model.User{
|
||||
ID: "admin-id",
|
||||
IsAdmin: true,
|
||||
})
|
||||
|
||||
// Create request with invalid library ID
|
||||
r := httptest.NewRequest("GET", "/rest/startScan?target=0:Music/Rock", nil)
|
||||
r = r.WithContext(ctx)
|
||||
|
||||
// Call endpoint
|
||||
response, err := api.StartScan(r)
|
||||
|
||||
// Should return error
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(response).To(BeNil())
|
||||
var subErr subError
|
||||
ok := errors.As(err, &subErr)
|
||||
Expect(ok).To(BeTrue())
|
||||
Expect(subErr.code).To(Equal(responses.ErrorGeneric))
|
||||
})
|
||||
|
||||
It("returns error when library does not exist", func() {
|
||||
// Setup mocks - user has access to library 1 and 2 only
|
||||
mockUserRepo := tests.CreateMockUserRepo()
|
||||
_ = mockUserRepo.SetUserLibraries("admin-id", []int{1, 2})
|
||||
mockDS := &tests.MockDataStore{MockedUser: mockUserRepo}
|
||||
api.ds = mockDS
|
||||
|
||||
// Create admin user
|
||||
ctx := request.WithUser(context.Background(), model.User{
|
||||
ID: "admin-id",
|
||||
IsAdmin: true,
|
||||
})
|
||||
|
||||
// Create request with library ID that doesn't exist
|
||||
r := httptest.NewRequest("GET", "/rest/startScan?target=999:Music/Rock", nil)
|
||||
r = r.WithContext(ctx)
|
||||
|
||||
// Call endpoint
|
||||
response, err := api.StartScan(r)
|
||||
|
||||
// Should return ErrorDataNotFound
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(response).To(BeNil())
|
||||
var subErr subError
|
||||
ok := errors.As(err, &subErr)
|
||||
Expect(ok).To(BeTrue())
|
||||
Expect(subErr.code).To(Equal(responses.ErrorDataNotFound))
|
||||
})
|
||||
|
||||
It("calls ScanAll when single library with empty path and only one library exists", func() {
|
||||
// Setup mocks - single library in DB
|
||||
mockUserRepo := tests.CreateMockUserRepo()
|
||||
_ = mockUserRepo.SetUserLibraries("admin-id", []int{1})
|
||||
mockLibraryRepo := &tests.MockLibraryRepo{}
|
||||
mockLibraryRepo.SetData(model.Libraries{
|
||||
{ID: 1, Name: "Music Library", Path: "/music"},
|
||||
})
|
||||
mockDS := &tests.MockDataStore{
|
||||
MockedUser: mockUserRepo,
|
||||
MockedLibrary: mockLibraryRepo,
|
||||
}
|
||||
api.ds = mockDS
|
||||
|
||||
// Create admin user
|
||||
ctx := request.WithUser(context.Background(), model.User{
|
||||
ID: "admin-id",
|
||||
IsAdmin: true,
|
||||
})
|
||||
|
||||
// Create request with single library and empty path
|
||||
r := httptest.NewRequest("GET", "/rest/startScan?target=1:", nil)
|
||||
r = r.WithContext(ctx)
|
||||
|
||||
// Call endpoint
|
||||
response, err := api.StartScan(r)
|
||||
|
||||
// Should succeed
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(response).ToNot(BeNil())
|
||||
|
||||
// Verify ScanAll was called instead of ScanFolders
|
||||
Eventually(func() int {
|
||||
return ms.GetScanAllCallCount()
|
||||
}).Should(BeNumerically(">", 0))
|
||||
Expect(ms.GetScanFoldersCallCount()).To(Equal(0))
|
||||
})
|
||||
|
||||
It("calls ScanFolders when single library with empty path but multiple libraries exist", func() {
|
||||
// Setup mocks - multiple libraries in DB
|
||||
mockUserRepo := tests.CreateMockUserRepo()
|
||||
_ = mockUserRepo.SetUserLibraries("admin-id", []int{1, 2})
|
||||
mockLibraryRepo := &tests.MockLibraryRepo{}
|
||||
mockLibraryRepo.SetData(model.Libraries{
|
||||
{ID: 1, Name: "Music Library", Path: "/music"},
|
||||
{ID: 2, Name: "Audiobooks", Path: "/audiobooks"},
|
||||
})
|
||||
mockDS := &tests.MockDataStore{
|
||||
MockedUser: mockUserRepo,
|
||||
MockedLibrary: mockLibraryRepo,
|
||||
}
|
||||
api.ds = mockDS
|
||||
|
||||
// Create admin user
|
||||
ctx := request.WithUser(context.Background(), model.User{
|
||||
ID: "admin-id",
|
||||
IsAdmin: true,
|
||||
})
|
||||
|
||||
// Create request with single library and empty path
|
||||
r := httptest.NewRequest("GET", "/rest/startScan?target=1:", nil)
|
||||
r = r.WithContext(ctx)
|
||||
|
||||
// Call endpoint
|
||||
response, err := api.StartScan(r)
|
||||
|
||||
// Should succeed
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(response).ToNot(BeNil())
|
||||
|
||||
// Verify ScanFolders was called (not ScanAll)
|
||||
Eventually(func() int {
|
||||
return ms.GetScanFoldersCallCount()
|
||||
}).Should(BeNumerically(">", 0))
|
||||
calls := ms.GetScanFoldersCalls()
|
||||
Expect(calls).To(HaveLen(1))
|
||||
targets := calls[0].Targets
|
||||
Expect(targets).To(HaveLen(1))
|
||||
Expect(targets[0].LibraryID).To(Equal(1))
|
||||
Expect(targets[0].FolderPath).To(Equal(""))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("GetScanStatus", func() {
|
||||
It("returns scan status", func() {
|
||||
// Setup mock scanner status
|
||||
ms.SetStatusResponse(&model.ScannerStatus{
|
||||
Scanning: false,
|
||||
Count: 100,
|
||||
FolderCount: 10,
|
||||
})
|
||||
|
||||
// Create request
|
||||
ctx := context.Background()
|
||||
r := httptest.NewRequest("GET", "/rest/getScanStatus", nil)
|
||||
r = r.WithContext(ctx)
|
||||
|
||||
// Call endpoint
|
||||
response, err := api.GetScanStatus(r)
|
||||
|
||||
// Should succeed
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(response).ToNot(BeNil())
|
||||
Expect(response.ScanStatus).ToNot(BeNil())
|
||||
Expect(response.ScanStatus.Scanning).To(BeFalse())
|
||||
Expect(response.ScanStatus.Count).To(Equal(int64(100)))
|
||||
Expect(response.ScanStatus.FolderCount).To(Equal(int64(10)))
|
||||
})
|
||||
})
|
||||
})
|
||||
23
server/testdata/test_cert.pem
vendored
Normal file
23
server/testdata/test_cert.pem
vendored
Normal file
@ -0,0 +1,23 @@
|
||||
-----BEGIN CERTIFICATE-----
|
||||
MIIDwzCCAqugAwIBAgIUXqdUxUOo8kmsDe71iTR+Vr7btP8wDQYJKoZIhvcNAQEL
|
||||
BQAwYjELMAkGA1UEBhMCVVMxDTALBgNVBAgMBFRlc3QxDTALBgNVBAcMBFRlc3Qx
|
||||
EjAQBgNVBAoMCU5hdmlkcm9tZTENMAsGA1UECwwEVGVzdDESMBAGA1UEAwwJbG9j
|
||||
YWxob3N0MCAXDTI1MTEyODE5NTkxNVoYDzIxMjUxMTA0MTk1OTE1WjBiMQswCQYD
|
||||
VQQGEwJVUzENMAsGA1UECAwEVGVzdDENMAsGA1UEBwwEVGVzdDESMBAGA1UECgwJ
|
||||
TmF2aWRyb21lMQ0wCwYDVQQLDARUZXN0MRIwEAYDVQQDDAlsb2NhbGhvc3QwggEi
|
||||
MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCkB/TQgl5ei5KRSHt5OJim8rKS
|
||||
MzRlkK4BjSEM4D9ESbebdpEVjX48QuBYACrCvgvVp7mQGF5anl8Hm89trvd8ooVQ
|
||||
x9IPQQ6gRKM+4gLrt9FHvFGGzZQS8UTQXN5oBi11E+8/Vs47HLUNXC2TRtRLCMyK
|
||||
LYXQIXbhdp9anImlt+IHUxIQUchK6Zkld/gCm56X1bbzN/Zq91PQLpx2FZ0eZTjN
|
||||
KaNgztLa+K/BDnTuk3iTTs9GEp6VCvqQE/6fk/UN/tkk2dLwKIFvPVR/YeAhVdz/
|
||||
OHC4L3B36QN3+VQ2yDjsp1PVAPX07UnzXO3Oj7uGYnMQxwprGMEubm3nADDxAgMB
|
||||
AAGjbzBtMB0GA1UdDgQWBBRAZHUVuLyzc0CfuZR9ApqMbawIqzAfBgNVHSMEGDAW
|
||||
gBRAZHUVuLyzc0CfuZR9ApqMbawIqzAPBgNVHRMBAf8EBTADAQH/MBoGA1UdEQQT
|
||||
MBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsFAAOCAQEAmDLXcPx9LNHs
|
||||
GxQIE6Q5BXbVO7c8qrWmJf5FK5VWaifNZ9U+IBi+VlB4jCLK/OkwsviN/jOnwRYx
|
||||
owjq0QG0YdRT4uD9fEMrAj+EwbnrQYZQvT0yGEWA+KW5TW08wt+/qnGJDwEgbjYJ
|
||||
HTdICVMhs/e8Ex48fAgO8WSsdTDekOrhuwzIfeJ1LU4ZptLsD2ePFxuzutdIuW51
|
||||
/mspQGsjXqZ1qnLsavLXh/lds2g602rTpYBNZVjV9WiOvaQS8vviOxBN6f+9vgRz
|
||||
a8SEbHqBG6jeyVqVZ7MjxcYxaIkxeBwMyMwgb+wwDfVXo2FZzX2TVeB7ZppI+IKv
|
||||
TXYurWPYsQ==
|
||||
-----END CERTIFICATE-----
|
||||
22
server/testdata/test_cert_encrypted.pem
vendored
Normal file
22
server/testdata/test_cert_encrypted.pem
vendored
Normal file
@ -0,0 +1,22 @@
|
||||
-----BEGIN CERTIFICATE-----
|
||||
MIIDpzCCAo+gAwIBAgIUEa7gEJYwJqYEJjTY7otQ+oUyELwwDQYJKoZIhvcNAQEL
|
||||
BQAwYjELMAkGA1UEBhMCVVMxDTALBgNVBAgMBFRlc3QxDTALBgNVBAcMBFRlc3Qx
|
||||
EjAQBgNVBAoMCU5hdmlkcm9tZTENMAsGA1UECwwEVGVzdDESMBAGA1UEAwwJbG9j
|
||||
YWxob3N0MCAXDTI1MTEyODE5NTI0OVoYDzIxMjUxMTA0MTk1MjQ5WjBiMQswCQYD
|
||||
VQQGEwJVUzENMAsGA1UECAwEVGVzdDENMAsGA1UEBwwEVGVzdDESMBAGA1UECgwJ
|
||||
TmF2aWRyb21lMQ0wCwYDVQQLDARUZXN0MRIwEAYDVQQDDAlsb2NhbGhvc3QwggEi
|
||||
MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDBHgqJ1d9EnNxqoSZ6xXrIz/mV
|
||||
Y0nWJW16/qIAvCdovSeTZhG9iqG8dUqcuu2BdD9MMHndJ2oFn3iD8EJR92dH8KBA
|
||||
8xOmtZ0BEEWgXPBivywZVd1ChIflEWj6m5wwLNjb57SPpUiwaLxBQB8ByEaAAZE/
|
||||
bLqvHI3vW/4s5apky17SPIqmkmqEYlRcg97tlRXsPuwoAVM9cvLMMEqtIR1CB/72
|
||||
gboY2Gi2r/plLF/Rg3Dom6QljMWi57XXWJFwGYSXaZuM0gvn04e3oLu+1E+WMoq/
|
||||
9rExWij2DlsmXd/RiScliFp6R4H84wQUyqrAUNytvgRO+oVnRjEA0l3oCYdRAgMB
|
||||
AAGjUzBRMB0GA1UdDgQWBBQQKpB1UaKm98FnBdl8uKdRscrVTzAfBgNVHSMEGDAW
|
||||
gBQQKpB1UaKm98FnBdl8uKdRscrVTzAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3
|
||||
DQEBCwUAA4IBAQBP07l+2LmpFtcxqMGmsiNYwFuHpQCxJd4YRZHjLX7O+oJExMgR
|
||||
2yP4mpMKurgKOv7unTDLwvjQRa6ZTYJCsYtvC6hbyqlGc7AfNTu6DKz8r35/2/V5
|
||||
hPsG5lNb91HhvHE839mLAvpi02LoFH2Sr8BR7s6qxfNKYcP8PUOJQXltJ6yAa8YJ
|
||||
syeXQQ3RIyGsJANeaC06S3UdkBM5H5BLfIHnHu3GybJjwL51va4WCdHe8QV6GI0g
|
||||
RDiThDVkBSXAr136vnMdlrYCxMoxY56itJ0zbYg2ELQKU9o1w/ZJQo9uvmy9jCoZ
|
||||
Hy1L5a2vUDbsdONdvRkYZRHqMpG4bdD8D3j2
|
||||
-----END CERTIFICATE-----
|
||||
28
server/testdata/test_key.pem
vendored
Normal file
28
server/testdata/test_key.pem
vendored
Normal file
@ -0,0 +1,28 @@
|
||||
-----BEGIN PRIVATE KEY-----
|
||||
MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCkB/TQgl5ei5KR
|
||||
SHt5OJim8rKSMzRlkK4BjSEM4D9ESbebdpEVjX48QuBYACrCvgvVp7mQGF5anl8H
|
||||
m89trvd8ooVQx9IPQQ6gRKM+4gLrt9FHvFGGzZQS8UTQXN5oBi11E+8/Vs47HLUN
|
||||
XC2TRtRLCMyKLYXQIXbhdp9anImlt+IHUxIQUchK6Zkld/gCm56X1bbzN/Zq91PQ
|
||||
Lpx2FZ0eZTjNKaNgztLa+K/BDnTuk3iTTs9GEp6VCvqQE/6fk/UN/tkk2dLwKIFv
|
||||
PVR/YeAhVdz/OHC4L3B36QN3+VQ2yDjsp1PVAPX07UnzXO3Oj7uGYnMQxwprGMEu
|
||||
bm3nADDxAgMBAAECggEABqJFvesP2v4FEvgd+kSWM+ZL34rPmy3zQ5/MDuPA20ep
|
||||
89EjQ/5hdRl1TknPcOnTu7PZVuENa9fM2xdrl7GEU9eU0bQLJE/KwiOUgJYObS8V
|
||||
eTO+DlghHXUBhfXDjux1CS+htOuTUqOyFNS+CR9Lta8o6ou1xjmcP7kW78i17mxF
|
||||
TuH5SZlS8W9PFLXHCInbMtqGFaT2ss09kvoPk2FDvHfxEdy6M9tKkguz02g+4bqI
|
||||
aAMp2N7AOfmRpC0HvVa1ZfZo5Z8/KMoNcIm3pV9DEVM369J9EzhnMNpkGben90aT
|
||||
FqO2JNsy52wmXFZUc9xe8uPdfDahALCkBGncLyLNmQKBgQDZREjocjdzOoPSlCdx
|
||||
mRNe9suHz2FpUpsHCPOCotG63hFVKpah/ZvpHSsQx5rXs/mawDTmzGY9GQiBrSvg
|
||||
OhfHIyT3NOhVaNcMxTqJX7rs7OG8D0MBacD9ASSeZ89MUn8q1EHZr5qxLtXl5Ikw
|
||||
mHtiGRdiKGFFrG9H0zncbGhy7QKBgQDBRhQ9RAasTdmUiNQly9GVFkXto4T/9UHx
|
||||
rVU44htCI2IVZUMTGlNfclfxpByDrzyA56rMzN9SAkiIp4nPpMDs5hayXaaPoojs
|
||||
CPzV7r2OjemZ6CTeQ1ODImRL8L/E3jJSgWd6YYoHSQ5hjEX4yT6ft0u0tZUfdMKd
|
||||
VENWIJ/hlQKBgQCo2hXjeOi5R8+tN3EUKwhP9HOnX7dv+D/9jqpZa5qdpPpJeyjI
|
||||
SmYCHKYci1Q+sWOaLiiu+km20B65UVFZGSzjmd+fs+GghzMifKGKo/iNK2ggFKhZ
|
||||
j8vplRrVdQ45XZ/xNDbdLEmHzEN2QE+Skd7KFYADzCgU0vdFFdbRBPuD3QKBgGIq
|
||||
fQctMRJ9LCE0akSURGwr9vKflmMHKCpfdqTAu0WZgS0K1Mm0GlqlUiPKzizYaauz
|
||||
f14sRNV7kWnPZsDPlqn8p9SKmpnj3RW97uWeMCtiyx6/+VHm8ljts/GaY1zT2s1r
|
||||
KqrPNfNDWQmU3MljNeqbh9lOTWK/xEVy0gzB31MNAoGAQNWrZvVdAbL95XW6STUu
|
||||
JmQlqJTlluuqS0Rrd/uVEQwW0Vd1dZjRQcFAFiSiCQWTbtId5gFZd6hiIQl53Xz0
|
||||
5cd+9mcyA/TaoCJYbMOFYsKbZMCBhefsovJlVQXedqJrIY6BdeGlet4GTAH5Qyl0
|
||||
ytEIUnvn5YmmbI7PDz80XpU=
|
||||
-----END PRIVATE KEY-----
|
||||
30
server/testdata/test_key_encrypted.pem
vendored
Normal file
30
server/testdata/test_key_encrypted.pem
vendored
Normal file
@ -0,0 +1,30 @@
|
||||
-----BEGIN ENCRYPTED PRIVATE KEY-----
|
||||
MIIFNTBfBgkqhkiG9w0BBQ0wUjAxBgkqhkiG9w0BBQwwJAQQPH9PYzryCI3smm81
|
||||
J8rm+QICCAAwDAYIKoZIhvcNAgkFADAdBglghkgBZQMEASoEEI+9XxNfKSiMYIVB
|
||||
UfcGfncEggTQVw7tPslGy3mlofCNnhBSnMViv9kj6M11smD6Y8vHG0k9Kq+6g+Dx
|
||||
mQE9ILrSZBzM0uS3y484u+vkdqlT4KehhjIx0IiezurOcM45UdTAwLFLPzeEDlHI
|
||||
lOWQ3gOTB3J5AxiUQOa6QsDIM7AZilidQG0BxQYWyRBA5B8evJwJoAvdzzA9wGSm
|
||||
2YdNm3tA6rU5U8cVG+qTJP9pjbtRx0medC/CBZdxGkrWBQH+aySfahJdU8X1JI2e
|
||||
SY4WJRw1rLCow+DnHjZS/IVHFJivJSRYvnvw8fwjOMVtkf+dAVctKlb1Fj9X+RdG
|
||||
T1sq3i6zwFLE/RRz4qM4DKZ6UaD9wRFLow8FmNWVuJJiPgCLx2rrNMe32quS/kQP
|
||||
iOsXAUeA/Yg1fdMCJORxl0nWDmLYcNtBghCmS1lyk+t+AKWwJudrds5tQQe8ha2t
|
||||
Q41is+tDKwGDC1wt4WXJvBhgAJzuqFtr30H0M1eBhwwDdaDd9v0Zr3r8V49WZM2c
|
||||
i3qkwPPYkQD+pOcR12xBV8ptvDxaUl7RGlVqnEWHagT51BaIaXQ9teUrG6UPt8o2
|
||||
LELJXF6CiwkbN6Y9sYx5XiKrIGxVhlQSZ1nB3XSFRHbu6e7VHPjnVwUeeg87J2Am
|
||||
MEwqDzPU5sjKRn84+M91Y4uFAIeinaOJAQ0/tZVrf1iSeCMQyMUhW/8m7JPfG19F
|
||||
NbJSPRXQuKmYKbWfXcMW2UFbp0zDs7s7p4zzbfde9IbVdq/o2nv3ZrNbrLak6O7y
|
||||
FVt9q/xG4Tty6hSK6xtqtNZWcmfiMcTlk1Qcz2STvScbXtqgcgR6WUZfkLuzi09I
|
||||
EDYFnzU5JNSY3U3VTv2hAPeU4xjTNM6kjF7L9JFGvdjH8Ko9UdxG9RZMd8xhBM/n
|
||||
hxdzdVba4bDDz2z+0A2blSObrPrNsKr/3ZbnfuUiSs5NmqmUOifZ1t1PqGGO2Y5S
|
||||
/cDKtrPk226hGomsUBfHtiIJPG1VRl4UaZiduqK3GGhtF491KU1mAfYzueok3TPq
|
||||
JhLtLDIvEaFgmOmitFzROI/ifm6s4ssUvcvtbjwJumbjkU38OxYZFwbhwbe268G2
|
||||
vgspJamlEGJNdGDzrCFQlA2+A9kazCttztikfh5QGV6WFfkc3Bt1XTPL51vtliQy
|
||||
MS2gUnJUY2fuYCfz8rxLH1kQmyYsHQz5rUYyBkeDffrG9MzarmzSJXR63FRzVMf1
|
||||
LQ7BSzei7dF6+J4KVCxjbGWF3GUGmGeOP5g5vJ3xb3YPJNJLT4Vai103pay59TGP
|
||||
tESM2Vn0gJEvYApi707noFH5uFTW1cp7lloF41ddIUkL/QO7j+sjvBww+4DqBB7J
|
||||
BmvLMnswa23yw9egYRG5jOXyCgIr+1rnNcph1HGJsvxvgJ2gwwo5NKCG8SC6LcZQ
|
||||
fbDjX+ssmobLE3ktN03FZPMp32/ciexzuZoamfyiPXh7xE++ckifNEKJlNhx+kCG
|
||||
mSR2wh+UGigQkgp/JxOzl6C4fhUbrEZr17oBqGim2p8h+GE0zD5JSHcn1rP86gGU
|
||||
8JG/ilG4I8uMxUwhGj7amrWXUlJBd1by7e1EAL+utCo14/Tx3otB9/JtqY+lm9Ey
|
||||
1ptPhMRQxvDNWrCmYM2kyrGghdNfEMir6GKDWI6PY9cwAFv/PLOxr1c=
|
||||
-----END ENCRYPTED PRIVATE KEY-----
|
||||
30
server/testdata/test_key_encrypted_legacy.pem
vendored
Normal file
30
server/testdata/test_key_encrypted_legacy.pem
vendored
Normal file
@ -0,0 +1,30 @@
|
||||
-----BEGIN RSA PRIVATE KEY-----
|
||||
Proc-Type: 4,ENCRYPTED
|
||||
DEK-Info: AES-256-CBC,3C969050EAB73F121B7F0E6B75C42525
|
||||
|
||||
V6pSaAsrn9CQNo4p88QshJLbg8zkQJEom81dPbYSVqQSZa9YlPtpLZ9YtuLj/Ay0
|
||||
TScEKIj/gzQ32wNl6nhcSNIL9yy+X11r5gNv1kIHkecf+EbDW20VOiJsfD+6LUyW
|
||||
hA96AIbPOwc76iCuvsKHPKU9MlEmjGipmk/C2RQLHCZJ3WkiDRgCM8KQ7vKhfACT
|
||||
w908yj4cB1e/P0JPq8t/3F7kPJ+6SVM1vMEffHl0otQR3rAyrK8QikwJ0K9qX62d
|
||||
cqchTVlEyyZBYovR8DrRRUDbsXS5j1ZmX3NQpvTSTFowr+33fMrY+4Oz8sdR4yx1
|
||||
CQc0A0sHHxSEIr2xu4KzczwOYVJN8PVdU0pgvFj9KEm66N6EY5CSFIBHyO/ycOt9
|
||||
U+wpkRjf3zS6ZaUU0NKdOcop4YX33i99/tZF2RNR1i7ETLYph+/LCf09286Bi3u/
|
||||
UCCuWedyECPdz0c6j0s27Fdfc/HEK90OEzeWh/fc+H2gJZhqJYK9V47HPTQNNMnB
|
||||
U1a6FsJlrKE3E6nfSnTLxrSx9m/XTV7HV+HkgX+q8VhN7Q2VHUqkPzE7ZOPYpZ+A
|
||||
dQzsm1TmEMxym6osYqFzQScXR1NZasrV2MTQ2J16dUgCdGAM2YMUD9JaoJR+u77M
|
||||
WAjYzDiRg84rLr/KbJPAwHbsfo2KpiapJGSBBEDhz4W1/LOrFhsjaqIMSy4yZDGm
|
||||
1KqXGHIlqmuHI7v4fD8vuzhj7GUujRx85HSZWakE/uc6s5WrhkSeVKYJWPfpsxTv
|
||||
dT3oLOGJ+nRzWxM3aFtuJghX0nIGdKxT4EAUNXz0/vLT3OP1QCZR+oELrriFzmtj
|
||||
+O30bGH2SAFZEQJ/uTQg6celoNh89IzH4DJkcn67hqpX6mUiU9CrIr/eR9C/en8Q
|
||||
smTbbC1C1pDUaCwR26Z+zgM90amh4yfOFKK2geO2Kj+TmwFHUvi6ZnSzMzCvty3t
|
||||
+wdIrUtf55Lw51JCpLGl70mg4b/zBj5hqBkU2YvAAnz/htjfH/wrD6ZAF1TCdlRO
|
||||
gyODrJjGRnLd/v0XLk0wp+RkAjBcSlRlkUvZY5BtugL7dIdwiNGGQPcOni9IVeG0
|
||||
6vDUEQnDOLYDj4d/JcckTLuHdrP+SW+0RQl2HK5+/w1hScGXN4O48gccu7yR/MN8
|
||||
DmpCg5rD/nq8sxJosmSt07GrN36KppYt8LCXQbSg3NG2Ad715caS2C+0Qtdm5MPD
|
||||
rM1UyTXQYSJXgUN9yZS/pmzlguCywnnvsBPU6j3ljZwcoD41QJ/1OU09/W6sIMQR
|
||||
IAiM35JHiLJiccFgxSE1qx5F1UZqX4P47jF0Wzi/sE/DYXg5qw2DoauqXNzqnumH
|
||||
71UDGK1V6wQIV7UCZDa0WUfFzu470XpuFb8VmMOuHSQxkZESc9cz8k/ueAuO438Q
|
||||
jnlkF1Ge2EEPuaK2zeaTj/lGyYA1AUfHRRgt/EMUQSBntmhlpnwVPYTVvYtHO2N5
|
||||
wp7/y39KirnlTl99i3XiOJ4WF4gIU2IaSlqMo4+e/A32h2JFi9QfNyfItXe6Fm1X
|
||||
d0j2XGHzwMfHEFKdWyrgtVZwc38/1d6xWYAhs02b2basV/0AQhFTaKf5Z268eBNJ
|
||||
-----END RSA PRIVATE KEY-----
|
||||
9
tests/fixtures/deezer.artist.bio.json
vendored
Normal file
9
tests/fixtures/deezer.artist.bio.json
vendored
Normal file
@ -0,0 +1,9 @@
|
||||
{
|
||||
"data": {
|
||||
"artist": {
|
||||
"bio": {
|
||||
"full": "<p>Schoolmates Thomas and Guy-Manuel began their career in 1992 with the indie rock trio Darlin' (named after The Beach Boys song) but were scathingly dismissed by Melody Maker magazine as \"daft punk.\" Turning to house-inspired electronica, they used the put down as a name for their DJ-ing partnership and became a hugely successful and influential dance act.</p>"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
1
tests/fixtures/deezer.artist.related.json
vendored
Normal file
1
tests/fixtures/deezer.artist.related.json
vendored
Normal file
File diff suppressed because one or more lines are too long
1
tests/fixtures/deezer.artist.top.json
vendored
Normal file
1
tests/fixtures/deezer.artist.top.json
vendored
Normal file
File diff suppressed because one or more lines are too long
7
tests/fixtures/lastfm.artist.page.html
vendored
Normal file
7
tests/fixtures/lastfm.artist.page.html
vendored
Normal file
@ -0,0 +1,7 @@
|
||||
<html>
|
||||
<head>
|
||||
<meta property="og:image" content="https://lastfm.freetls.fastly.net/i/u/ar0/818148bf682d429dc21b59a73ef6f68e.png" />
|
||||
</head>
|
||||
<body>
|
||||
</body>
|
||||
</html>
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
x
Reference in New Issue
Block a user