From aeae6d221706e392272b8c2697b886144e81b48e Mon Sep 17 00:00:00 2001 From: ranokay Date: Tue, 14 Apr 2026 02:19:09 +0300 Subject: [PATCH] feat(lyrics): require cue byte offsets --- core/lyrics/lyrics_test.go | 16 +- core/lyrics/sources_test.go | 14 ++ core/lyrics/ttml.go | 226 +++++++++++++++--- core/lyrics/ttml_test.go | 20 +- model/lyrics.go | 86 ++++--- model/lyrics_test.go | 34 ++- server/subsonic/helpers.go | 6 +- server/subsonic/media_retrieval_test.go | 164 +++++++++++-- server/subsonic/responses/responses.go | 10 +- ui/src/audioplayer/KaraokeLyricsOverlay.jsx | 67 ++++++ .../audioplayer/KaraokeLyricsOverlay.test.jsx | 68 ++++++ ui/src/audioplayer/lyrics.js | 80 +++++++ ui/src/audioplayer/lyrics.test.js | 56 +++++ ui/src/subsonic/index.test.js | 3 +- 14 files changed, 730 insertions(+), 120 deletions(-) diff --git a/core/lyrics/lyrics_test.go b/core/lyrics/lyrics_test.go index 917c530ac..822e975ce 100644 --- a/core/lyrics/lyrics_test.go +++ b/core/lyrics/lyrics_test.go @@ -56,14 +56,18 @@ var _ = Describe("sources", func() { Value: "Lead words", Cue: []model.Cue{ { - Start: gg.P(int64(1000)), - End: gg.P(int64(1500)), - Value: "Lead ", + Start: gg.P(int64(1000)), + End: gg.P(int64(1500)), + Value: "Lead ", + ByteStart: 0, + ByteEnd: 4, }, { - Start: gg.P(int64(1500)), - End: gg.P(int64(3000)), - Value: "words", + Start: gg.P(int64(1500)), + End: gg.P(int64(3000)), + Value: "words", + ByteStart: 5, + ByteEnd: 9, }, }, }, diff --git a/core/lyrics/sources_test.go b/core/lyrics/sources_test.go index a86c84cd0..5ba03336e 100644 --- a/core/lyrics/sources_test.go +++ b/core/lyrics/sources_test.go @@ -108,12 +108,18 @@ var _ = Describe("sources", func() { Expect(*lyrics[0].Line[0].Cue[0].Start).To(Equal(int64(1000))) Expect(lyrics[0].Line[0].Cue[0].Value).To(Equal("Some ")) Expect(lyrics[0].Line[0].Cue[0].End).To(Equal(gg.P(int64(1500)))) + Expect(lyrics[0].Line[0].Cue[0].ByteStart).To(Equal(0)) + Expect(lyrics[0].Line[0].Cue[0].ByteEnd).To(Equal(4)) Expect(*lyrics[0].Line[0].Cue[1].Start).To(Equal(int64(1500))) Expect(lyrics[0].Line[0].Cue[1].Value).To(Equal("lyrics ")) Expect(lyrics[0].Line[0].Cue[1].End).To(Equal(gg.P(int64(2000)))) + Expect(lyrics[0].Line[0].Cue[1].ByteStart).To(Equal(5)) + Expect(lyrics[0].Line[0].Cue[1].ByteEnd).To(Equal(11)) Expect(*lyrics[0].Line[0].Cue[2].Start).To(Equal(int64(2000))) Expect(lyrics[0].Line[0].Cue[2].Value).To(Equal("here")) Expect(lyrics[0].Line[0].Cue[2].End).To(Equal(gg.P(int64(3000)))) + Expect(lyrics[0].Line[0].Cue[2].ByteStart).To(Equal(12)) + Expect(lyrics[0].Line[0].Cue[2].ByteEnd).To(Equal(15)) // Line 2: has inline markers Expect(lyrics[0].Line[1].Start).To(Equal(gg.P(int64(3000)))) @@ -122,6 +128,10 @@ var _ = Describe("sources", func() { Expect(lyrics[0].Line[1].Cue).To(HaveLen(2)) Expect(lyrics[0].Line[1].Cue[0].End).To(Equal(gg.P(int64(3500)))) Expect(lyrics[0].Line[1].Cue[1].End).To(Equal(gg.P(int64(5000)))) + Expect(lyrics[0].Line[1].Cue[0].ByteStart).To(Equal(0)) + Expect(lyrics[0].Line[1].Cue[0].ByteEnd).To(Equal(4)) + Expect(lyrics[0].Line[1].Cue[1].ByteStart).To(Equal(5)) + Expect(lyrics[0].Line[1].Cue[1].ByteEnd).To(Equal(9)) // Line 3: plain line, no cues Expect(lyrics[0].Line[2].Start).To(Equal(gg.P(int64(5000)))) @@ -148,9 +158,13 @@ var _ = Describe("sources", func() { Expect(*lyrics[0].Line[0].Cue[0].Start).To(Equal(int64(1000))) Expect(lyrics[0].Line[0].Cue[0].Value).To(Equal("Lead ")) Expect(lyrics[0].Line[0].Cue[0].End).To(Equal(gg.P(int64(1500)))) + Expect(lyrics[0].Line[0].Cue[0].ByteStart).To(Equal(0)) + Expect(lyrics[0].Line[0].Cue[0].ByteEnd).To(Equal(4)) Expect(*lyrics[0].Line[0].Cue[1].Start).To(Equal(int64(1500))) Expect(lyrics[0].Line[0].Cue[1].Value).To(Equal("words")) Expect(lyrics[0].Line[0].Cue[1].End).To(Equal(gg.P(int64(3000)))) + Expect(lyrics[0].Line[0].Cue[1].ByteStart).To(Equal(5)) + Expect(lyrics[0].Line[0].Cue[1].ByteEnd).To(Equal(9)) Expect(lyrics[0].Line[1].Start).To(Equal(gg.P(int64(3000)))) Expect(lyrics[0].Line[1].Value).To(Equal("Fallback line")) diff --git a/core/lyrics/ttml.go b/core/lyrics/ttml.go index a02fa52d8..6e4ce9da3 100644 --- a/core/lyrics/ttml.go +++ b/core/lyrics/ttml.go @@ -10,6 +10,7 @@ import ( "sort" "strconv" "strings" + "unicode" "github.com/navidrome/navidrome/log" "github.com/navidrome/navidrome/model" @@ -78,6 +79,11 @@ type ttmlDefinedAgent struct { Name string } +type ttmlPiece struct { + raw string + cue *model.Cue +} + type ttmlParser struct { decoder *xml.Decoder params ttmlTimingParams @@ -294,7 +300,7 @@ func (p *ttmlParser) parseMetadataText(start xml.StartElement, parent ttmlTiming forKey, hasFor := attrValue(start.Attr, "for") forKey = strings.TrimSpace(forKey) - value, tokens, err := p.parseInlineElement(start, parent) + pieces, err := p.parseInlineElement(start, parent) if err != nil { return ttmlMetadataEntry{}, false, err } @@ -307,7 +313,8 @@ func (p *ttmlParser) parseMetadataText(start xml.StartElement, parent ttmlTiming return ttmlMetadataEntry{}, false, nil } - line := model.Line{Value: sanitizeTTMLText(value)} + value, tokens := buildTTMLLineFromPieces(pieces) + line := model.Line{Value: value} if ctx.hasBegin { startMs := ctx.begin line.Start = &startMs @@ -329,8 +336,7 @@ func (p *ttmlParser) parseMetadataText(start xml.StartElement, parent ttmlTiming } func (p *ttmlParser) parseParagraph(parent ttmlTimingContext) (string, []model.Cue, error) { - var text strings.Builder - var tokens []model.Cue + var pieces []ttmlPiece for { token, err := p.decoder.Token() @@ -340,26 +346,26 @@ func (p *ttmlParser) parseParagraph(parent ttmlTimingContext) (string, []model.C switch t := token.(type) { case xml.StartElement: - value, inlineTokens, err := p.parseInlineElement(t, parent) + inlinePieces, err := p.parseInlineElement(t, parent) if err != nil { return "", nil, err } - text.WriteString(value) - tokens = append(tokens, inlineTokens...) + pieces = append(pieces, inlinePieces...) case xml.EndElement: if strings.EqualFold(t.Name.Local, "p") { - return sanitizeTTMLText(text.String()), tokens, nil + value, tokens := buildTTMLLineFromPieces(pieces) + return value, tokens, nil } case xml.CharData: - text.WriteString(string(t)) + pieces = append(pieces, ttmlPiece{raw: string(t)}) } } } -func (p *ttmlParser) parseInlineElement(start xml.StartElement, parent ttmlTimingContext) (string, []model.Cue, error) { +func (p *ttmlParser) parseInlineElement(start xml.StartElement, parent ttmlTimingContext) ([]ttmlPiece, error) { local := strings.ToLower(start.Name.Local) if local == "br" { - return "\n", nil, nil + return []ttmlPiece{{raw: "\n"}}, nil } ctx := p.childContext(start.Attr, parent) @@ -368,53 +374,203 @@ func (p *ttmlParser) parseInlineElement(start xml.StartElement, parent ttmlTimin _, hasDur := attrValue(start.Attr, "dur") hasOwnTiming := hasBegin || hasEnd || hasDur - var text strings.Builder - var tokens []model.Cue + var pieces []ttmlPiece for { token, err := p.decoder.Token() if err != nil { - return "", nil, err + return nil, err } switch t := token.(type) { case xml.StartElement: - value, inlineTokens, err := p.parseInlineElement(t, ctx) + inlinePieces, err := p.parseInlineElement(t, ctx) if err != nil { - return "", nil, err + return nil, err } - text.WriteString(value) - tokens = append(tokens, inlineTokens...) + pieces = append(pieces, inlinePieces...) case xml.EndElement: if !strings.EqualFold(t.Name.Local, start.Name.Local) { continue } - value := text.String() - tokenText := sanitizeTTMLText(value) - if local == "span" && hasOwnTiming && !ctx.invalid && tokenText != "" && len(tokens) == 0 { - parsedToken := model.Cue{ - Value: tokenText, - AgentID: p.resolveCueAgentID(ctx), + if local == "span" && hasOwnTiming && !ctx.invalid && !ttmlPiecesContainCue(pieces) { + rawValue := concatTTMLPieceRaw(pieces) + tokenText := sanitizeTTMLText(rawValue) + if tokenText != "" { + parsedToken := model.Cue{ + AgentID: p.resolveCueAgentID(ctx), + } + if ctx.hasBegin { + startMs := ctx.begin + parsedToken.Start = &startMs + } + if ctx.hasEnd { + endMs := ctx.end + parsedToken.End = &endMs + } + + return []ttmlPiece{{ + raw: rawValue, + cue: &parsedToken, + }}, nil } - if ctx.hasBegin { - startMs := ctx.begin - parsedToken.Start = &startMs - } - if ctx.hasEnd { - endMs := ctx.end - parsedToken.End = &endMs - } - tokens = append(tokens, parsedToken) } - return value, tokens, nil + return pieces, nil case xml.CharData: - text.WriteString(string(t)) + pieces = append(pieces, ttmlPiece{raw: string(t)}) } } } +func buildTTMLLineFromPieces(pieces []ttmlPiece) (string, []model.Cue) { + finalized := finalizeTTMLLines(splitTTMLPiecesByNewline(pieces)) + for len(finalized) > 0 && finalized[0].text == "" && len(finalized[0].cues) == 0 { + finalized = finalized[1:] + } + for len(finalized) > 0 { + last := finalized[len(finalized)-1] + if last.text != "" || len(last.cues) > 0 { + break + } + finalized = finalized[:len(finalized)-1] + } + + var value strings.Builder + cues := make([]model.Cue, 0, 8) + byteOffset := 0 + for i, line := range finalized { + if i > 0 { + value.WriteByte('\n') + byteOffset++ + } + value.WriteString(line.text) + for _, cue := range line.cues { + cue.ByteStart += byteOffset + cue.ByteEnd += byteOffset + cues = append(cues, cue) + } + byteOffset += len(line.text) + } + + return value.String(), cues +} + +type ttmlFinalLine struct { + text string + cues []model.Cue +} + +func finalizeTTMLLines(lines [][]ttmlPiece) []ttmlFinalLine { + finalized := make([]ttmlFinalLine, 0, len(lines)) + for _, line := range lines { + text, cues := finalizeTTMLLogicalLine(line) + finalized = append(finalized, ttmlFinalLine{text: text, cues: cues}) + } + return finalized +} + +func splitTTMLPiecesByNewline(pieces []ttmlPiece) [][]ttmlPiece { + lines := [][]ttmlPiece{{}} + for _, piece := range pieces { + raw := normalizeTTMLPieceRaw(piece.raw) + if raw == "" { + continue + } + + start := 0 + for i := 0; i < len(raw); i++ { + if raw[i] != '\n' { + continue + } + if start < i { + lines[len(lines)-1] = append(lines[len(lines)-1], ttmlPiece{ + raw: raw[start:i], + cue: cloneTTMLCue(piece.cue), + }) + } + lines = append(lines, []ttmlPiece{}) + start = i + 1 + } + if start < len(raw) { + lines[len(lines)-1] = append(lines[len(lines)-1], ttmlPiece{ + raw: raw[start:], + cue: cloneTTMLCue(piece.cue), + }) + } + } + return lines +} + +func finalizeTTMLLogicalLine(line []ttmlPiece) (string, []model.Cue) { + rawLine := concatTTMLPieceRaw(line) + if rawLine == "" { + return "", nil + } + + leftTrimBytes := len(rawLine) - len(strings.TrimLeftFunc(rawLine, unicode.IsSpace)) + rightTrimBytes := len(rawLine) - len(strings.TrimRightFunc(rawLine, unicode.IsSpace)) + trimmedEnd := len(rawLine) - rightTrimBytes + if trimmedEnd < leftTrimBytes { + trimmedEnd = leftTrimBytes + } + + trimmed := strings.TrimSpace(rawLine) + cues := make([]model.Cue, 0, len(line)) + cursor := 0 + for _, piece := range line { + pieceEnd := cursor + len(piece.raw) + if piece.cue != nil { + byteStart := max(cursor, leftTrimBytes) + byteEnd := min(pieceEnd, trimmedEnd) + if byteStart < byteEnd { + cue := *piece.cue + cue.Value = rawLine[byteStart:byteEnd] + cue.ByteStart = byteStart - leftTrimBytes + cue.ByteEnd = byteEnd - leftTrimBytes - 1 + cues = append(cues, cue) + } + } + cursor = pieceEnd + } + + return trimmed, cues +} + +func normalizeTTMLPieceRaw(raw string) string { + raw = str.SanitizeText(raw) + raw = strings.ReplaceAll(raw, "\r\n", "\n") + raw = strings.ReplaceAll(raw, "\r", "\n") + return raw +} + +func concatTTMLPieceRaw(pieces []ttmlPiece) string { + var raw strings.Builder + for _, piece := range pieces { + raw.WriteString(normalizeTTMLPieceRaw(piece.raw)) + } + return raw.String() +} + +func ttmlPiecesContainCue(pieces []ttmlPiece) bool { + for _, piece := range pieces { + if piece.cue != nil { + return true + } + } + return false +} + +func cloneTTMLCue(cue *model.Cue) *model.Cue { + if cue == nil { + return nil + } + + cloned := *cue + return &cloned +} + func (p *ttmlParser) toLyricList() model.LyricList { res := make(model.LyricList, 0, len(p.mainLangOrder)+len(p.translationLangOrder)+len(p.pronunciationLangOrder)) for _, lang := range p.mainLangOrder { diff --git a/core/lyrics/ttml_test.go b/core/lyrics/ttml_test.go index 4e81197d4..5f9092e36 100644 --- a/core/lyrics/ttml_test.go +++ b/core/lyrics/ttml_test.go @@ -141,9 +141,9 @@ var _ = Describe("parseTTML", func() { Expect(line.End).To(Equal(gg.P(int64(3000)))) Expect(line.Cue).To(HaveLen(3)) - Expect(line.Cue[0]).To(Equal(model.Cue{Start: gg.P(int64(1000)), End: gg.P(int64(1400)), Value: "He", AgentID: "main"})) - Expect(line.Cue[1]).To(Equal(model.Cue{Start: gg.P(int64(1400)), End: gg.P(int64(1800)), Value: "llo", AgentID: "main"})) - Expect(line.Cue[2]).To(Equal(model.Cue{Start: gg.P(int64(2000)), End: gg.P(int64(2500)), Value: "echo", AgentID: "__nd_bg__|main"})) + Expect(line.Cue[0]).To(Equal(model.Cue{Start: gg.P(int64(1000)), End: gg.P(int64(1400)), Value: "He", ByteStart: 0, ByteEnd: 1, AgentID: "main"})) + Expect(line.Cue[1]).To(Equal(model.Cue{Start: gg.P(int64(1400)), End: gg.P(int64(1800)), Value: "llo", ByteStart: 2, ByteEnd: 4, AgentID: "main"})) + Expect(line.Cue[2]).To(Equal(model.Cue{Start: gg.P(int64(2000)), End: gg.P(int64(2500)), Value: "echo", ByteStart: 6, ByteEnd: 9, AgentID: "__nd_bg__|main"})) }) It("should parse named TTML agents into main, voice, and group roles", func() { @@ -241,8 +241,8 @@ var _ = Describe("parseTTML", func() { Expect(line.Value).To(Equal("go\ngo")) Expect(line.End).To(Equal(gg.P(int64(45570)))) Expect(line.Cue).To(HaveLen(2)) - Expect(line.Cue[0]).To(Equal(model.Cue{Start: gg.P(int64(43444)), End: gg.P(int64(43716)), Value: "go"})) - Expect(line.Cue[1]).To(Equal(model.Cue{Start: gg.P(int64(43716)), End: gg.P(int64(43887)), Value: "go"})) + Expect(line.Cue[0]).To(Equal(model.Cue{Start: gg.P(int64(43444)), End: gg.P(int64(43716)), Value: "go", ByteStart: 0, ByteEnd: 1})) + Expect(line.Cue[1]).To(Equal(model.Cue{Start: gg.P(int64(43716)), End: gg.P(int64(43887)), Value: "go", ByteStart: 3, ByteEnd: 4})) }) }) @@ -325,8 +325,8 @@ var _ = Describe("parseTTML", func() { Expect(pronunciation.Line[0].Value).To(Equal("konni")) Expect(pronunciation.Line[0].End).To(Equal(gg.P(int64(2600)))) Expect(pronunciation.Line[0].Cue).To(HaveLen(2)) - Expect(pronunciation.Line[0].Cue[0]).To(Equal(model.Cue{Start: gg.P(int64(2000)), End: gg.P(int64(2300)), Value: "ko"})) - Expect(pronunciation.Line[0].Cue[1]).To(Equal(model.Cue{Start: gg.P(int64(2300)), End: gg.P(int64(2600)), Value: "nni"})) + Expect(pronunciation.Line[0].Cue[0]).To(Equal(model.Cue{Start: gg.P(int64(2000)), End: gg.P(int64(2300)), Value: "ko", ByteStart: 0, ByteEnd: 1})) + Expect(pronunciation.Line[0].Cue[1]).To(Equal(model.Cue{Start: gg.P(int64(2300)), End: gg.P(int64(2600)), Value: "nni", ByteStart: 2, ByteEnd: 4})) }) }) @@ -369,9 +369,9 @@ var _ = Describe("parseTTML", func() { Expect(line.Start).To(Equal(gg.P(int64(2747)))) Expect(line.Value).To(Equal("I woke up")) Expect(line.Cue).To(HaveLen(3)) - Expect(line.Cue[0]).To(Equal(model.Cue{Start: gg.P(int64(2747)), End: gg.P(int64(3018)), Value: "I"})) - Expect(line.Cue[1]).To(Equal(model.Cue{Start: gg.P(int64(3018)), End: gg.P(int64(3179)), Value: "woke"})) - Expect(line.Cue[2]).To(Equal(model.Cue{Start: gg.P(int64(3179)), End: gg.P(int64(3582)), Value: "up"})) + Expect(line.Cue[0]).To(Equal(model.Cue{Start: gg.P(int64(2747)), End: gg.P(int64(3018)), Value: "I", ByteStart: 0, ByteEnd: 0})) + Expect(line.Cue[1]).To(Equal(model.Cue{Start: gg.P(int64(3018)), End: gg.P(int64(3179)), Value: "woke", ByteStart: 2, ByteEnd: 5})) + Expect(line.Cue[2]).To(Equal(model.Cue{Start: gg.P(int64(3179)), End: gg.P(int64(3582)), Value: "up", ByteStart: 7, ByteEnd: 8})) }) }) }) diff --git a/model/lyrics.go b/model/lyrics.go index ec0df9f34..9a57ebaad 100644 --- a/model/lyrics.go +++ b/model/lyrics.go @@ -6,16 +6,19 @@ import ( "slices" "strconv" "strings" + "unicode" "github.com/navidrome/navidrome/log" "github.com/navidrome/navidrome/utils/str" ) type Cue struct { - Start *int64 `structs:"start,omitempty" json:"start,omitempty"` - End *int64 `structs:"end,omitempty" json:"end,omitempty"` - Value string `structs:"value" json:"value"` - AgentID string `structs:"agentId,omitempty" json:"agentId,omitempty"` + Start *int64 `structs:"start,omitempty" json:"start,omitempty"` + End *int64 `structs:"end,omitempty" json:"end,omitempty"` + Value string `structs:"value" json:"value"` + ByteStart int `structs:"byteStart" json:"byteStart"` + ByteEnd int `structs:"byteEnd" json:"byteEnd"` + AgentID string `structs:"agentId,omitempty" json:"agentId,omitempty"` } type Agent struct { @@ -127,14 +130,10 @@ func ToLyrics(language, text string) (*Lyrics, error) { if validLine { for idx := range timestamps { - cues := parseEnhancedCues(priorLine) - value := priorLine - if cues != nil { - value = stripEnhancedMarkers(value) - } + value, cues := parseEnhancedLine(priorLine) structuredLines = append(structuredLines, Line{ Start: ×tamps[idx], - Value: strings.TrimSpace(value), + Value: value, Cue: cues, }) } @@ -181,14 +180,10 @@ func ToLyrics(language, text string) (*Lyrics, error) { if validLine { for idx := range timestamps { - cues := parseEnhancedCues(priorLine) - value := priorLine - if cues != nil { - value = stripEnhancedMarkers(value) - } + value, cues := parseEnhancedLine(priorLine) structuredLines = append(structuredLines, Line{ Start: ×tamps[idx], - Value: strings.TrimSpace(value), + Value: value, Cue: cues, }) } @@ -213,21 +208,22 @@ func ToLyrics(language, text string) (*Lyrics, error) { return &lyrics, nil } -// parseEnhancedCues extracts word-level timing cues from Enhanced LRC inline markers. -// Format: word word ... -// Returns nil if no inline markers are found. -func parseEnhancedCues(text string) []Cue { +// parseEnhancedLine extracts word-level timing cues from Enhanced LRC inline markers +// and computes UTF-8 byte offsets against the final stripped line value. +func parseEnhancedLine(text string) (string, []Cue) { matches := enhancedLRCRegex.FindAllStringSubmatchIndex(text, -1) if len(matches) == 0 { - return nil + return strings.TrimSpace(text), nil } type segment struct { - start int64 - text string + start int64 + rawStart int + rawEnd int } segments := make([]segment, 0, len(matches)) + var rawValue strings.Builder for i, match := range matches { timeMs, err := parseTime( // Rewrite <...> as [...] so parseTime can handle it with the same logic @@ -258,22 +254,46 @@ func parseEnhancedCues(text string) []Cue { if word == "" { continue } - segments = append(segments, segment{start: timeMs, text: word}) + + rawStart := rawValue.Len() + rawValue.WriteString(word) + segments = append(segments, segment{ + start: timeMs, + rawStart: rawStart, + rawEnd: rawValue.Len(), + }) } if len(segments) == 0 { - return nil + return strings.TrimSpace(stripEnhancedMarkers(text)), nil } - cues := make([]Cue, len(segments)) - for i, seg := range segments { - start := seg.start - cues[i] = Cue{ - Start: &start, - Value: seg.text, - } + finalRaw := rawValue.String() + leftTrimBytes := len(finalRaw) - len(strings.TrimLeftFunc(finalRaw, unicode.IsSpace)) + rightTrimBytes := len(finalRaw) - len(strings.TrimRightFunc(finalRaw, unicode.IsSpace)) + trimmedEnd := len(finalRaw) - rightTrimBytes + if trimmedEnd < leftTrimBytes { + trimmedEnd = leftTrimBytes } - return cues + + cues := make([]Cue, 0, len(segments)) + for _, seg := range segments { + start := seg.start + byteStart := max(seg.rawStart, leftTrimBytes) + byteEnd := min(seg.rawEnd, trimmedEnd) + if byteStart >= byteEnd { + continue + } + + cues = append(cues, Cue{ + Start: &start, + Value: finalRaw[byteStart:byteEnd], + ByteStart: byteStart - leftTrimBytes, + ByteEnd: byteEnd - leftTrimBytes - 1, + }) + } + + return strings.TrimSpace(finalRaw), cues } // adjustGroup remaps a capture group index from the original match to our rewritten "[...]" string. diff --git a/model/lyrics_test.go b/model/lyrics_test.go index 6f189f024..1fa82f258 100644 --- a/model/lyrics_test.go +++ b/model/lyrics_test.go @@ -130,9 +130,9 @@ var _ = Describe("ToLyrics", func() { Expect(line0.End).To(Equal(&t3000)) Expect(line0.Value).To(Equal("Some lyrics here")) Expect(line0.Cue).To(Equal([]Cue{ - {Start: &t1000, End: &t1500, Value: "Some "}, - {Start: &t1500, End: &t2000, Value: "lyrics "}, - {Start: &t2000, End: &t3000, Value: "here"}, + {Start: &t1000, End: &t1500, Value: "Some ", ByteStart: 0, ByteEnd: 4}, + {Start: &t1500, End: &t2000, Value: "lyrics ", ByteStart: 5, ByteEnd: 11}, + {Start: &t2000, End: &t3000, Value: "here", ByteStart: 12, ByteEnd: 15}, })) line1 := lyrics.Line[1] @@ -140,8 +140,8 @@ var _ = Describe("ToLyrics", func() { Expect(line1.End).To(Equal(&t3500)) Expect(line1.Value).To(Equal("More words")) Expect(line1.Cue).To(Equal([]Cue{ - {Start: &t3000, Value: "More "}, - {Start: &t3500, Value: "words"}, + {Start: &t3000, Value: "More ", ByteStart: 0, ByteEnd: 4}, + {Start: &t3500, Value: "words", ByteStart: 5, ByteEnd: 9}, })) Expect(line1.Cue[1].End).To(BeNil()) @@ -166,8 +166,8 @@ var _ = Describe("ToLyrics", func() { t3000 := int64(3000) Expect(lyrics.Line[0].Cue).To(Equal([]Cue{ - {Start: &t1000, End: &t1500, Value: "Some "}, - {Start: &t1500, End: &t3000, Value: "lyrics"}, + {Start: &t1000, End: &t1500, Value: "Some ", ByteStart: 0, ByteEnd: 4}, + {Start: &t1500, End: &t3000, Value: "lyrics", ByteStart: 5, ByteEnd: 10}, })) Expect(lyrics.Line[0].Value).To(Equal("Some lyrics")) Expect(lyrics.Line[0].End).To(Equal(&t3000)) @@ -176,9 +176,25 @@ var _ = Describe("ToLyrics", func() { Expect(lyrics.Line[1].Value).To(Equal("Plain line")) Expect(lyrics.Line[2].Cue).To(Equal([]Cue{ - {Start: &t5000, Value: "More "}, - {Start: &t5500, Value: "words"}, + {Start: &t5000, Value: "More ", ByteStart: 0, ByteEnd: 4}, + {Start: &t5500, Value: "words", ByteStart: 5, ByteEnd: 9}, })) Expect(lyrics.Line[2].Value).To(Equal("More words")) }) + + It("should preserve byte offsets for Enhanced LRC cues", func() { + lyrics, err := ToLyrics("xxx", "[00:00.00]<00:00.00>Oh <00:00.90>love<00:01.30> me <00:01.60>tonight") + Expect(err).ToNot(HaveOccurred()) + Expect(lyrics.Line).To(HaveLen(1)) + + t0, t900, t1300, t1600 := int64(0), int64(900), int64(1300), int64(1600) + line := lyrics.Line[0] + Expect(line.Value).To(Equal("Oh love me tonight")) + Expect(line.Cue).To(Equal([]Cue{ + {Start: &t0, Value: "Oh ", ByteStart: 0, ByteEnd: 2}, + {Start: &t900, Value: "love", ByteStart: 3, ByteEnd: 6}, + {Start: &t1300, Value: " me ", ByteStart: 7, ByteEnd: 10}, + {Start: &t1600, Value: "tonight", ByteStart: 11, ByteEnd: 17}, + })) + }) }) diff --git a/server/subsonic/helpers.go b/server/subsonic/helpers.go index ad769ee94..6a14aa4aa 100644 --- a/server/subsonic/helpers.go +++ b/server/subsonic/helpers.go @@ -619,8 +619,10 @@ func buildLyricCues(cues []model.Cue, lineEnd *int64) []responses.LyricCue { } cue := responses.LyricCue{ - Start: *cues[i].Start, - Value: cues[i].Value, + Start: *cues[i].Start, + Value: cues[i].Value, + ByteStart: cues[i].ByteStart, + ByteEnd: cues[i].ByteEnd, } if hasAnyEnd { end := cues[i].End diff --git a/server/subsonic/media_retrieval_test.go b/server/subsonic/media_retrieval_test.go index e4f6a21d4..faa90e375 100644 --- a/server/subsonic/media_retrieval_test.go +++ b/server/subsonic/media_retrieval_test.go @@ -277,6 +277,8 @@ var _ = Describe("MediaRetrievalController", func() { expectedCue := expectedCueLine.Cue[k] Expect(realCue.Value).To(Equal(expectedCue.Value)) Expect(realCue.Start).To(Equal(expectedCue.Start)) + Expect(realCue.ByteStart).To(Equal(expectedCue.ByteStart)) + Expect(realCue.ByteEnd).To(Equal(expectedCue.ByteEnd)) if expectedCue.End == nil { Expect(realCue.End).To(BeNil()) } else { @@ -514,14 +516,18 @@ var _ = Describe("MediaRetrievalController", func() { Value: "konni", Cue: []responses.LyricCue{ { - Start: tokenStartA, - End: &tokenEndA, - Value: "ko", + Start: tokenStartA, + End: &tokenEndA, + ByteStart: 0, + ByteEnd: 1, + Value: "ko", }, { - Start: tokenStartB, - End: &tokenEndB, - Value: "nni", + Start: tokenStartB, + End: &tokenEndB, + ByteStart: 2, + ByteEnd: 4, + Value: "nni", }, }, }, @@ -552,16 +558,20 @@ var _ = Describe("MediaRetrievalController", func() { Value: "Hello echo", Cue: []model.Cue{ { - Start: &tokenStartA, - End: &tokenEndA, - Value: "Hello", - AgentID: "lead", + Start: &tokenStartA, + End: &tokenEndA, + Value: "Hello", + ByteStart: 0, + ByteEnd: 4, + AgentID: "lead", }, { - Start: &tokenStartB, - End: &tokenEndB, - Value: "echo", - AgentID: "__nd_bg__|lead", + Start: &tokenStartB, + End: &tokenEndB, + Value: "echo", + ByteStart: 6, + ByteEnd: 9, + AgentID: "__nd_bg__|lead", }, }, }, @@ -608,9 +618,11 @@ var _ = Describe("MediaRetrievalController", func() { AgentID: "lead", Cue: []responses.LyricCue{ { - Start: tokenStartA, - End: &tokenEndA, - Value: "Hello", + Start: tokenStartA, + End: &tokenEndA, + ByteStart: 0, + ByteEnd: 4, + Value: "Hello", }, }, }, @@ -622,9 +634,11 @@ var _ = Describe("MediaRetrievalController", func() { AgentID: "__nd_bg__|lead", Cue: []responses.LyricCue{ { - Start: tokenStartB, - End: &tokenEndB, - Value: "echo", + Start: tokenStartB, + End: &tokenEndB, + ByteStart: 6, + ByteEnd: 9, + Value: "echo", }, }, }, @@ -633,6 +647,116 @@ var _ = Describe("MediaRetrievalController", func() { }, }) }) + + It("should return required cue byte offsets for ambiguous and multibyte cue lines", func() { + r := newGetRequest("id=1&enhanced=true") + + asciiLineStart := int64(0) + asciiLineEnd := int64(2400) + asciiCueStartA := int64(0) + asciiCueEndA := int64(300) + asciiCueStartB := int64(900) + asciiCueEndB := int64(1300) + asciiCueStartC := int64(1300) + asciiCueEndC := int64(1600) + asciiCueStartD := int64(1600) + + utfLineStart := int64(2747) + utfLineEnd := int64(6214) + utfCueStartA := int64(2747) + utfCueEndA := int64(3018) + utfCueStartB := int64(3018) + utfCueEndB := int64(3179) + utfCueStartC := int64(3582) + utfCueEndC := int64(4100) + utfCueStartD := int64(4500) + utfCueEndD := int64(6214) + + lyricsJSON, err := json.Marshal(model.LyricList{ + { + Lang: "eng", + Synced: true, + Line: []model.Line{ + { + Start: &asciiLineStart, + End: &asciiLineEnd, + Value: "Oh love love me tonight", + Cue: []model.Cue{ + {Start: &asciiCueStartA, End: &asciiCueEndA, Value: "Oh", ByteStart: 0, ByteEnd: 1}, + {Start: &asciiCueStartB, End: &asciiCueEndB, Value: "love", ByteStart: 8, ByteEnd: 11}, + {Start: &asciiCueStartC, End: &asciiCueEndC, Value: "me", ByteStart: 13, ByteEnd: 14}, + {Start: &asciiCueStartD, Value: "tonight", ByteStart: 16, ByteEnd: 22}, + }, + }, + { + Start: &utfLineStart, + End: &utfLineEnd, + Value: "눈을 뜬 순간", + Cue: []model.Cue{ + {Start: &utfCueStartA, End: &utfCueEndA, Value: "눈", ByteStart: 0, ByteEnd: 2}, + {Start: &utfCueStartB, End: &utfCueEndB, Value: "을", ByteStart: 3, ByteEnd: 5}, + {Start: &utfCueStartC, End: &utfCueEndC, Value: "뜬", ByteStart: 7, ByteEnd: 9}, + {Start: &utfCueStartD, End: &utfCueEndD, Value: "순간", ByteStart: 11, ByteEnd: 16}, + }, + }, + }, + }, + }) + Expect(err).ToNot(HaveOccurred()) + + mockRepo.SetData(model.MediaFiles{ + { + ID: "1", + Artist: "Rick Astley", + Title: "Never Gonna Give You Up", + Lyrics: string(lyricsJSON), + }, + }) + + response, err := router.GetLyricsBySongId(r) + Expect(err).ToNot(HaveOccurred()) + compareResponses(response.LyricsList, responses.LyricsList{ + StructuredLyrics: responses.StructuredLyrics{ + { + DisplayArtist: "Rick Astley", + DisplayTitle: "Never Gonna Give You Up", + Kind: "main", + Lang: "eng", + Synced: true, + Line: []responses.Line{ + {Start: &asciiLineStart, Value: "Oh love love me tonight"}, + {Start: &utfLineStart, Value: "눈을 뜬 순간"}, + }, + CueLine: []responses.CueLine{ + { + Index: 0, + Start: &asciiLineStart, + End: &asciiLineEnd, + Value: "Oh love love me tonight", + Cue: []responses.LyricCue{ + {Start: asciiCueStartA, End: &asciiCueEndA, Value: "Oh", ByteStart: 0, ByteEnd: 1}, + {Start: asciiCueStartB, End: &asciiCueEndB, Value: "love", ByteStart: 8, ByteEnd: 11}, + {Start: asciiCueStartC, End: &asciiCueEndC, Value: "me", ByteStart: 13, ByteEnd: 14}, + {Start: asciiCueStartD, End: &asciiLineEnd, Value: "tonight", ByteStart: 16, ByteEnd: 22}, + }, + }, + { + Index: 1, + Start: &utfLineStart, + End: &utfLineEnd, + Value: "눈을 뜬 순간", + Cue: []responses.LyricCue{ + {Start: utfCueStartA, End: &utfCueEndA, Value: "눈", ByteStart: 0, ByteEnd: 2}, + {Start: utfCueStartB, End: &utfCueEndB, Value: "을", ByteStart: 3, ByteEnd: 5}, + {Start: utfCueStartC, End: &utfCueEndC, Value: "뜬", ByteStart: 7, ByteEnd: 9}, + {Start: utfCueStartD, End: &utfCueEndD, Value: "순간", ByteStart: 11, ByteEnd: 16}, + }, + }, + }, + }, + }, + }) + }) }) }) diff --git a/server/subsonic/responses/responses.go b/server/subsonic/responses/responses.go index 344dd9999..d1ecdb307 100644 --- a/server/subsonic/responses/responses.go +++ b/server/subsonic/responses/responses.go @@ -538,9 +538,11 @@ type Line struct { } type LyricCue struct { - Start int64 `xml:"start,attr" json:"start"` - End *int64 `xml:"end,attr,omitempty" json:"end,omitempty"` - Value string `xml:",chardata" json:"value"` + Start int64 `xml:"start,attr" json:"start"` + End *int64 `xml:"end,attr,omitempty" json:"end,omitempty"` + ByteStart int `xml:"byteStart,attr" json:"byteStart"` + ByteEnd int `xml:"byteEnd,attr" json:"byteEnd"` + Value string `xml:",chardata" json:"value"` } type Agent struct { @@ -553,7 +555,7 @@ type CueLine struct { Index int32 `xml:"index,attr" json:"index"` Start *int64 `xml:"start,attr,omitempty" json:"start,omitempty"` End *int64 `xml:"end,attr,omitempty" json:"end,omitempty"` - Value string `xml:"value,attr,omitempty" json:"value,omitempty"` + Value string `xml:"value,attr" json:"value"` AgentID string `xml:"agentId,attr,omitempty" json:"agentId,omitempty"` Cue []LyricCue `xml:"cue,omitempty" json:"cue,omitempty"` } diff --git a/ui/src/audioplayer/KaraokeLyricsOverlay.jsx b/ui/src/audioplayer/KaraokeLyricsOverlay.jsx index cd1484e41..799f8bdc2 100644 --- a/ui/src/audioplayer/KaraokeLyricsOverlay.jsx +++ b/ui/src/audioplayer/KaraokeLyricsOverlay.jsx @@ -26,6 +26,7 @@ import { hasStructuredLyricContent, resolveKaraokeTokenWindow, resolveLayerLineForMain, + utf8ByteRangeToCodeUnitRange, } from './lyrics' const KARAOKE_RENDER_LEAD_MS = 24 @@ -635,6 +636,72 @@ const buildSegmentsFromLine = (line) => { } const text = line.value || '' + const exactSegments = (() => { + if (!text) { + return null + } + + const rangedTokens = line.tokens + .map((token, tokenIndex) => ({ + token, + tokenIndex, + range: utf8ByteRangeToCodeUnitRange( + text, + token?.byteStart, + token?.byteEnd, + ), + })) + .filter((entry) => entry.range != null) + + if ( + rangedTokens.length !== line.tokens.length || + rangedTokens.length === 0 + ) { + return null + } + + rangedTokens.sort( + (a, b) => + a.range.start - b.range.start || + a.range.end - b.range.end || + a.tokenIndex - b.tokenIndex, + ) + + const segments = [] + let cursor = 0 + for (const entry of rangedTokens) { + if (entry.range.start < cursor) { + return null + } + if (entry.range.start > cursor) { + segments.push({ + text: text.slice(cursor, entry.range.start), + token: null, + tokenIndex: -1, + }) + } + segments.push({ + text: entry.range.text, + token: entry.token, + tokenIndex: entry.tokenIndex, + }) + cursor = entry.range.end + } + + if (cursor < text.length) { + segments.push({ + text: text.slice(cursor), + token: null, + tokenIndex: -1, + }) + } + + return segments + })() + if (exactSegments) { + return exactSegments + } + const matchedSegments = [] const fallbackSegments = [] let cursor = 0 diff --git a/ui/src/audioplayer/KaraokeLyricsOverlay.test.jsx b/ui/src/audioplayer/KaraokeLyricsOverlay.test.jsx index 411116eae..412bc3946 100644 --- a/ui/src/audioplayer/KaraokeLyricsOverlay.test.jsx +++ b/ui/src/audioplayer/KaraokeLyricsOverlay.test.jsx @@ -177,6 +177,74 @@ describe(' behavior', () => { expect(translationLine.querySelectorAll('span')).toHaveLength(1) }) + it('uses cue byte offsets to segment repeated words in the karaoke line', () => { + renderOverlay({ + mainLyric: { + kind: 'main', + lang: 'en', + synced: true, + line: [{ start: 0, end: 2400, value: 'Oh love love me tonight' }], + cueLine: [ + { + index: 0, + start: 0, + end: 2400, + value: 'Oh love love me tonight', + cue: [ + { start: 0, end: 300, value: 'Oh', byteStart: 0, byteEnd: 1 }, + { + start: 900, + end: 1300, + value: 'love', + byteStart: 8, + byteEnd: 11, + }, + { + start: 1300, + end: 1600, + value: 'me', + byteStart: 13, + byteEnd: 14, + }, + { + start: 1600, + end: 2400, + value: 'tonight', + byteStart: 16, + byteEnd: 22, + }, + ], + }, + ], + }, + translationLyric: null, + pronunciationLyric: null, + showTranslation: false, + showPronunciation: false, + translationEnabled: false, + pronunciationEnabled: false, + audioInstance: { + ...audioInstance, + currentTime: 1.0, + }, + }) + + const mainLine = screen.getByText('Oh').parentElement + const segments = Array.from(mainLine.querySelectorAll('span')).map( + (span) => span.textContent, + ) + + expect(segments).toEqual([ + 'Oh', + ' love ', + 'love', + ' ', + 'me', + ' ', + 'tonight', + ]) + }) + it('highlights line-timed pronunciation and translation rows with the active main line', () => { renderOverlay({ mainLyric: { diff --git a/ui/src/audioplayer/lyrics.js b/ui/src/audioplayer/lyrics.js index e9cd16d5a..6fa627ee5 100644 --- a/ui/src/audioplayer/lyrics.js +++ b/ui/src/audioplayer/lyrics.js @@ -19,6 +19,17 @@ const toTime = (value) => { return Number.isFinite(numeric) ? numeric : null } +const toByteOffset = (value) => { + if (value == null || value === '') { + return null + } + const numeric = Number(value) + if (!Number.isInteger(numeric) || numeric < 0) { + return null + } + return numeric +} + const compareNullableTime = (a, b) => { if (a == null && b == null) { return 0 @@ -78,10 +89,79 @@ const normalizeToken = (token) => { if (!value.trim()) { return null } + const byteStart = toByteOffset(token.byteStart) + const byteEnd = toByteOffset(token.byteEnd) return { start: toTime(token.start), end: toTime(token.end), value, + ...(byteStart != null ? { byteStart } : {}), + ...(byteEnd != null ? { byteEnd } : {}), + } +} + +const utf8BytesForCodePoint = (codePoint) => { + if (codePoint <= 0x7f) { + return 1 + } + if (codePoint <= 0x7ff) { + return 2 + } + if (codePoint <= 0xffff) { + return 3 + } + return 4 +} + +export const utf8ByteOffsetToCodeUnitIndex = (text, targetByteOffset) => { + if (typeof text !== 'string' || text.length === 0) { + return 0 + } + + const target = toByteOffset(targetByteOffset) + if (target == null || target <= 0) { + return 0 + } + + let byteOffset = 0 + let index = 0 + while (index < text.length) { + if (byteOffset >= target) { + return index + } + const codePoint = text.codePointAt(index) + byteOffset += utf8BytesForCodePoint(codePoint) + index += codePoint > 0xffff ? 2 : 1 + } + + return text.length +} + +export const utf8ByteRangeToCodeUnitRange = (text, byteStart, byteEnd) => { + if (typeof text !== 'string') { + return null + } + + const start = toByteOffset(byteStart) + const end = toByteOffset(byteEnd) + if (start == null || end == null || end < start) { + return null + } + + const startIndex = utf8ByteOffsetToCodeUnitIndex(text, start) + const endIndex = utf8ByteOffsetToCodeUnitIndex(text, end + 1) + if ( + startIndex >= endIndex || + startIndex > text.length || + endIndex > text.length + ) { + return null + } + + return { + start: startIndex, + end: endIndex, + text: text.slice(startIndex, endIndex), } } diff --git a/ui/src/audioplayer/lyrics.test.js b/ui/src/audioplayer/lyrics.test.js index 2fcf1df40..961fdb10b 100644 --- a/ui/src/audioplayer/lyrics.test.js +++ b/ui/src/audioplayer/lyrics.test.js @@ -13,6 +13,8 @@ import { selectLyricLayers, structuredLyricsToLrc, structuredLyricToLrc, + utf8ByteOffsetToCodeUnitIndex, + utf8ByteRangeToCodeUnitRange, } from './lyrics' describe('lyrics helpers', () => { @@ -412,6 +414,60 @@ describe('lyrics helpers', () => { ]) }) + it('preserves cue byte offsets on karaoke tokens', () => { + const lines = buildKaraokeLines({ + lang: 'eng', + synced: true, + line: [{ start: 0, end: 2400, value: 'Oh love love me tonight' }], + cueLine: [ + { + index: 0, + start: 0, + end: 2400, + value: 'Oh love love me tonight', + cue: [ + { start: 0, end: 300, value: 'Oh', byteStart: 0, byteEnd: 1 }, + { start: 900, end: 1300, value: 'love', byteStart: 8, byteEnd: 11 }, + { start: 1300, end: 1600, value: 'me', byteStart: 13, byteEnd: 14 }, + { + start: 1600, + end: 2400, + value: 'tonight', + byteStart: 16, + byteEnd: 22, + }, + ], + }, + ], + }) + + expect( + lines[0].tokens.map((token) => [ + token.value, + token.byteStart, + token.byteEnd, + ]), + ).toEqual([ + ['Oh', 0, 1], + ['love', 8, 11], + ['me', 13, 14], + ['tonight', 16, 22], + ]) + }) + + it('maps UTF-8 byte offsets to string ranges for multibyte lyrics', () => { + const text = '눈을 뜬 순간' + + expect(utf8ByteOffsetToCodeUnitIndex(text, 0)).toBe(0) + expect(utf8ByteOffsetToCodeUnitIndex(text, 3)).toBe(1) + expect(utf8ByteOffsetToCodeUnitIndex(text, 7)).toBe(3) + expect(utf8ByteRangeToCodeUnitRange(text, 11, 16)).toEqual({ + start: 5, + end: 7, + text: '순간', + }) + }) + it('falls back to legacy cueLine role values when agents are absent', () => { const lines = buildKaraokeLines({ lang: 'eng', diff --git a/ui/src/subsonic/index.test.js b/ui/src/subsonic/index.test.js index 6910fdc8d..6bd5e08ee 100644 --- a/ui/src/subsonic/index.test.js +++ b/ui/src/subsonic/index.test.js @@ -1,5 +1,4 @@ import { vi } from 'vitest' -import { COVER_ART_SIZE } from '../consts' import { httpClient } from '../dataProvider' import subsonic from './index' @@ -7,6 +6,8 @@ vi.mock('../dataProvider', () => ({ httpClient: vi.fn(() => Promise.resolve({})), })) +const COVER_ART_SIZE = 600 + describe('getCoverArtUrl', () => { beforeEach(() => { // Mock window.location