perf(cli): stream 'missing list' instead of loading every missing file into memory

GetAll materialised the whole result set before a single row was written;
on a library with 97k missing files that peaked at 1.28 GB of RSS. Iterate
the repository cursor and write rows as they arrive.

cmd/missing.go

@@ -6,6 +6,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io"
 	"os"
 	"strconv"
 
@@ -68,29 +69,45 @@ func runMissingList(ctx context.Context) {
 	}
 
 	ds, ctx := getAdminContext(ctx)
-	mfs, err := ds.MediaFile(ctx).GetAll(model.QueryOptions{
+	mfs, err := ds.MediaFile(ctx).GetCursor(model.QueryOptions{
 		Filters: squirrel.Eq{"missing": true},
 		Sort:    "path",
 	})
+	if err == nil {
+		err = writeMissingList(os.Stdout, missingListFormat, mfs)
+	}
 	if err != nil {
 		log.Fatal(ctx, "Failed to retrieve missing files", err)
 	}
+}
 
-	if missingListFormat == "json" {
-		display := make([]displayMissingFile, len(mfs))
-		for i, mf := range mfs {
-			display[i] = displayMissingFile{ID: mf.ID, LibraryID: mf.LibraryID, Path: mf.Path, Title: mf.Title, Album: mf.Album, Artist: mf.Artist}
+// writeMissingList streams the cursor so a library with many missing files doesn't get loaded into memory
+func writeMissingList(w io.Writer, format string, mfs model.MediaFileCursor) error {
+	if format == "json" {
+		_, _ = io.WriteString(w, "[")
+		sep := ""
+		for mf, err := range mfs {
+			if err != nil {
+				return err
+			}
+			j, _ := json.Marshal(displayMissingFile{ID: mf.ID, LibraryID: mf.LibraryID, Path: mf.Path, Title: mf.Title, Album: mf.Album, Artist: mf.Artist})
+			_, _ = fmt.Fprintf(w, "%s%s", sep, j)
+			sep = ","
 		}
-		j, _ := json.Marshal(display)
-		fmt.Printf("%s\n", j)
-	} else {
-		w := csv.NewWriter(os.Stdout)
-		_ = w.Write([]string{"id", "library id", "path", "title", "album", "artist"})
-		for _, mf := range mfs {
-			_ = w.Write([]string{mf.ID, strconv.Itoa(mf.LibraryID), mf.Path, mf.Title, mf.Album, mf.Artist})
-		}
-		w.Flush()
+		_, err := io.WriteString(w, "]\n")
+		return err
 	}
+
+	cw := csv.NewWriter(w)
+	_ = cw.Write([]string{"id", "library id", "path", "title", "album", "artist"})
+	for mf, err := range mfs {
+		if err != nil {
+			return err
+		}
+		_ = cw.Write([]string{mf.ID, strconv.Itoa(mf.LibraryID), mf.Path, mf.Title, mf.Album, mf.Artist})
+	}
+	cw.Flush()
+	return cw.Error()
 }
 
 func runMissingFix(ctx context.Context, missingRef, targetRef string) {

cmd/missing_test.go

@@ -0,0 +1,52 @@
+package cmd
+
+import (
+	"errors"
+	"strings"
+
+	"github.com/navidrome/navidrome/model"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("writeMissingList", func() {
+	cursor := func(err error, mfs ...model.MediaFile) model.MediaFileCursor {
+		return func(yield func(model.MediaFile, error) bool) {
+			for _, mf := range mfs {
+				if !yield(mf, nil) {
+					return
+				}
+			}
+			if err != nil {
+				yield(model.MediaFile{}, err)
+			}
+		}
+	}
+	song := model.MediaFile{ID: "1", LibraryID: 1, Path: "Bach: Goldberg/01.mp3", Title: "Aria", Album: "Goldberg", Artist: "Bach"}
+
+	It("writes csv with a header, quoting as needed", func() {
+		var out strings.Builder
+		Expect(writeMissingList(&out, "csv", cursor(nil, song))).To(Succeed())
+		Expect(out.String()).To(Equal("id,library id,path,title,album,artist\n1,1,Bach: Goldberg/01.mp3,Aria,Goldberg,Bach\n"))
+	})
+
+	It("writes a json array", func() {
+		var out strings.Builder
+		Expect(writeMissingList(&out, "json", cursor(nil, song, song))).To(Succeed())
+		Expect(out.String()).To(MatchJSON(`[
+			{"id":"1","libraryId":1,"path":"Bach: Goldberg/01.mp3","title":"Aria","album":"Goldberg","artist":"Bach"},
+			{"id":"1","libraryId":1,"path":"Bach: Goldberg/01.mp3","title":"Aria","album":"Goldberg","artist":"Bach"}
+		]`))
+	})
+
+	It("writes an empty json array when nothing is missing", func() {
+		var out strings.Builder
+		Expect(writeMissingList(&out, "json", cursor(nil))).To(Succeed())
+		Expect(out.String()).To(MatchJSON(`[]`))
+	})
+
+	It("returns the cursor's error", func() {
+		var out strings.Builder
+		Expect(writeMissingList(&out, "csv", cursor(errors.New("boom"), song))).To(MatchError("boom"))
+	})
+})