nik/navidrome
1
0
Fork 0
mirror of https://github.com/navidrome/navidrome.git synced 2026-08-31 07:30:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(lyrics): require cue byte offsets

Browse Source
This commit is contained in:
ranokay 2026-04-14 02:19:09 +03:00
parent 73d94962e0
commit aeae6d2217
No known key found for this signature in database
14 changed files with 730 additions and 120 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
core/lyrics/lyrics_test.go
View File

@ -56,14 +56,18 @@ var _ = Describe("sources", func() {
Value: "Lead words",
Cue: []model.Cue{
{
Start: gg.P(int64(1000)),
End: gg.P(int64(1500)),
Value: "Lead ",
Start: gg.P(int64(1000)),
End: gg.P(int64(1500)),
Value: "Lead ",
ByteStart: 0,
ByteEnd: 4,
},
{
Start: gg.P(int64(1500)),
End: gg.P(int64(3000)),
Value: "words",
Start: gg.P(int64(1500)),
End: gg.P(int64(3000)),
Value: "words",
ByteStart: 5,
ByteEnd: 9,
},
},
},

14
core/lyrics/sources_test.go
View File

@ -108,12 +108,18 @@ var _ = Describe("sources", func() {
Expect(*lyrics[0].Line[0].Cue[0].Start).To(Equal(int64(1000)))
Expect(lyrics[0].Line[0].Cue[0].Value).To(Equal("Some "))
Expect(lyrics[0].Line[0].Cue[0].End).To(Equal(gg.P(int64(1500))))
Expect(lyrics[0].Line[0].Cue[0].ByteStart).To(Equal(0))
Expect(lyrics[0].Line[0].Cue[0].ByteEnd).To(Equal(4))
Expect(*lyrics[0].Line[0].Cue[1].Start).To(Equal(int64(1500)))
Expect(lyrics[0].Line[0].Cue[1].Value).To(Equal("lyrics "))
Expect(lyrics[0].Line[0].Cue[1].End).To(Equal(gg.P(int64(2000))))
Expect(lyrics[0].Line[0].Cue[1].ByteStart).To(Equal(5))
Expect(lyrics[0].Line[0].Cue[1].ByteEnd).To(Equal(11))
Expect(*lyrics[0].Line[0].Cue[2].Start).To(Equal(int64(2000)))
Expect(lyrics[0].Line[0].Cue[2].Value).To(Equal("here"))
Expect(lyrics[0].Line[0].Cue[2].End).To(Equal(gg.P(int64(3000))))
Expect(lyrics[0].Line[0].Cue[2].ByteStart).To(Equal(12))
Expect(lyrics[0].Line[0].Cue[2].ByteEnd).To(Equal(15))
// Line 2: has inline markers
Expect(lyrics[0].Line[1].Start).To(Equal(gg.P(int64(3000))))
@ -122,6 +128,10 @@ var _ = Describe("sources", func() {
Expect(lyrics[0].Line[1].Cue).To(HaveLen(2))
Expect(lyrics[0].Line[1].Cue[0].End).To(Equal(gg.P(int64(3500))))
Expect(lyrics[0].Line[1].Cue[1].End).To(Equal(gg.P(int64(5000))))
Expect(lyrics[0].Line[1].Cue[0].ByteStart).To(Equal(0))
Expect(lyrics[0].Line[1].Cue[0].ByteEnd).To(Equal(4))
Expect(lyrics[0].Line[1].Cue[1].ByteStart).To(Equal(5))
Expect(lyrics[0].Line[1].Cue[1].ByteEnd).To(Equal(9))
// Line 3: plain line, no cues
Expect(lyrics[0].Line[2].Start).To(Equal(gg.P(int64(5000))))
@ -148,9 +158,13 @@ var _ = Describe("sources", func() {
Expect(*lyrics[0].Line[0].Cue[0].Start).To(Equal(int64(1000)))
Expect(lyrics[0].Line[0].Cue[0].Value).To(Equal("Lead "))
Expect(lyrics[0].Line[0].Cue[0].End).To(Equal(gg.P(int64(1500))))
Expect(lyrics[0].Line[0].Cue[0].ByteStart).To(Equal(0))
Expect(lyrics[0].Line[0].Cue[0].ByteEnd).To(Equal(4))
Expect(*lyrics[0].Line[0].Cue[1].Start).To(Equal(int64(1500)))
Expect(lyrics[0].Line[0].Cue[1].Value).To(Equal("words"))
Expect(lyrics[0].Line[0].Cue[1].End).To(Equal(gg.P(int64(3000))))
Expect(lyrics[0].Line[0].Cue[1].ByteStart).To(Equal(5))
Expect(lyrics[0].Line[0].Cue[1].ByteEnd).To(Equal(9))
Expect(lyrics[0].Line[1].Start).To(Equal(gg.P(int64(3000))))
Expect(lyrics[0].Line[1].Value).To(Equal("Fallback line"))

226
core/lyrics/ttml.go
View File

@ -10,6 +10,7 @@ import (
"sort"
"strconv"
"strings"
"unicode"
"github.com/navidrome/navidrome/log"
"github.com/navidrome/navidrome/model"
@ -78,6 +79,11 @@ type ttmlDefinedAgent struct {
Name string
}
type ttmlPiece struct {
raw string
cue *model.Cue
}
type ttmlParser struct {
decoder *xml.Decoder
params ttmlTimingParams
@ -294,7 +300,7 @@ func (p *ttmlParser) parseMetadataText(start xml.StartElement, parent ttmlTiming
forKey, hasFor := attrValue(start.Attr, "for")
forKey = strings.TrimSpace(forKey)
value, tokens, err := p.parseInlineElement(start, parent)
pieces, err := p.parseInlineElement(start, parent)
if err != nil {
return ttmlMetadataEntry{}, false, err
}
@ -307,7 +313,8 @@ func (p *ttmlParser) parseMetadataText(start xml.StartElement, parent ttmlTiming
return ttmlMetadataEntry{}, false, nil
}
line := model.Line{Value: sanitizeTTMLText(value)}
value, tokens := buildTTMLLineFromPieces(pieces)
line := model.Line{Value: value}
if ctx.hasBegin {
startMs := ctx.begin
line.Start = &startMs
@ -329,8 +336,7 @@ func (p *ttmlParser) parseMetadataText(start xml.StartElement, parent ttmlTiming
}
func (p *ttmlParser) parseParagraph(parent ttmlTimingContext) (string, []model.Cue, error) {
var text strings.Builder
var tokens []model.Cue
var pieces []ttmlPiece
for {
token, err := p.decoder.Token()
@ -340,26 +346,26 @@ func (p *ttmlParser) parseParagraph(parent ttmlTimingContext) (string, []model.C
switch t := token.(type) {
case xml.StartElement:
value, inlineTokens, err := p.parseInlineElement(t, parent)
inlinePieces, err := p.parseInlineElement(t, parent)
if err != nil {
return "", nil, err
}
text.WriteString(value)
tokens = append(tokens, inlineTokens...)
pieces = append(pieces, inlinePieces...)
case xml.EndElement:
if strings.EqualFold(t.Name.Local, "p") {
return sanitizeTTMLText(text.String()), tokens, nil
value, tokens := buildTTMLLineFromPieces(pieces)
return value, tokens, nil
}
case xml.CharData:
text.WriteString(string(t))
pieces = append(pieces, ttmlPiece{raw: string(t)})
}
}
}
func (p *ttmlParser) parseInlineElement(start xml.StartElement, parent ttmlTimingContext) (string, []model.Cue, error) {
func (p *ttmlParser) parseInlineElement(start xml.StartElement, parent ttmlTimingContext) ([]ttmlPiece, error) {
local := strings.ToLower(start.Name.Local)
if local == "br" {
return "\n", nil, nil
return []ttmlPiece{{raw: "\n"}}, nil
}
ctx := p.childContext(start.Attr, parent)
@ -368,53 +374,203 @@ func (p *ttmlParser) parseInlineElement(start xml.StartElement, parent ttmlTimin
_, hasDur := attrValue(start.Attr, "dur")
hasOwnTiming := hasBegin || hasEnd || hasDur
var text strings.Builder
var tokens []model.Cue
var pieces []ttmlPiece
for {
token, err := p.decoder.Token()
if err != nil {
return "", nil, err
return nil, err
}
switch t := token.(type) {
case xml.StartElement:
value, inlineTokens, err := p.parseInlineElement(t, ctx)
inlinePieces, err := p.parseInlineElement(t, ctx)
if err != nil {
return "", nil, err
return nil, err
}
text.WriteString(value)
tokens = append(tokens, inlineTokens...)
pieces = append(pieces, inlinePieces...)
case xml.EndElement:
if !strings.EqualFold(t.Name.Local, start.Name.Local) {
continue
}
value := text.String()
tokenText := sanitizeTTMLText(value)
if local == "span" && hasOwnTiming && !ctx.invalid && tokenText != "" && len(tokens) == 0 {
parsedToken := model.Cue{
Value: tokenText,
AgentID: p.resolveCueAgentID(ctx),
if local == "span" && hasOwnTiming && !ctx.invalid && !ttmlPiecesContainCue(pieces) {
rawValue := concatTTMLPieceRaw(pieces)
tokenText := sanitizeTTMLText(rawValue)
if tokenText != "" {
parsedToken := model.Cue{
AgentID: p.resolveCueAgentID(ctx),
}
if ctx.hasBegin {
startMs := ctx.begin
parsedToken.Start = &startMs
}
if ctx.hasEnd {
endMs := ctx.end
parsedToken.End = &endMs
}
return []ttmlPiece{{
raw: rawValue,
cue: &parsedToken,
}}, nil
}
if ctx.hasBegin {
startMs := ctx.begin
parsedToken.Start = &startMs
}
if ctx.hasEnd {
endMs := ctx.end
parsedToken.End = &endMs
}
tokens = append(tokens, parsedToken)
}
return value, tokens, nil
return pieces, nil
case xml.CharData:
text.WriteString(string(t))
pieces = append(pieces, ttmlPiece{raw: string(t)})
}
}
}
func buildTTMLLineFromPieces(pieces []ttmlPiece) (string, []model.Cue) {
finalized := finalizeTTMLLines(splitTTMLPiecesByNewline(pieces))
for len(finalized) > 0 && finalized[0].text == "" && len(finalized[0].cues) == 0 {
finalized = finalized[1:]
}
for len(finalized) > 0 {
last := finalized[len(finalized)-1]
if last.text != "" || len(last.cues) > 0 {
break
}
finalized = finalized[:len(finalized)-1]
}
var value strings.Builder
cues := make([]model.Cue, 0, 8)
byteOffset := 0
for i, line := range finalized {
if i > 0 {
value.WriteByte('\n')
byteOffset++
}
value.WriteString(line.text)
for _, cue := range line.cues {
cue.ByteStart += byteOffset
cue.ByteEnd += byteOffset
cues = append(cues, cue)
}
byteOffset += len(line.text)
}
return value.String(), cues
}
type ttmlFinalLine struct {
text string
cues []model.Cue
}
func finalizeTTMLLines(lines [][]ttmlPiece) []ttmlFinalLine {
finalized := make([]ttmlFinalLine, 0, len(lines))
for _, line := range lines {
text, cues := finalizeTTMLLogicalLine(line)
finalized = append(finalized, ttmlFinalLine{text: text, cues: cues})
}
return finalized
}
func splitTTMLPiecesByNewline(pieces []ttmlPiece) [][]ttmlPiece {
lines := [][]ttmlPiece{{}}
for _, piece := range pieces {
raw := normalizeTTMLPieceRaw(piece.raw)
if raw == "" {
continue
}
start := 0
for i := 0; i < len(raw); i++ {
if raw[i] != '\n' {
continue
}
if start < i {
lines[len(lines)-1] = append(lines[len(lines)-1], ttmlPiece{
raw: raw[start:i],
cue: cloneTTMLCue(piece.cue),
})
}
lines = append(lines, []ttmlPiece{})
start = i + 1
}
if start < len(raw) {
lines[len(lines)-1] = append(lines[len(lines)-1], ttmlPiece{
raw: raw[start:],
cue: cloneTTMLCue(piece.cue),
})
}
}
return lines
}
func finalizeTTMLLogicalLine(line []ttmlPiece) (string, []model.Cue) {
rawLine := concatTTMLPieceRaw(line)
if rawLine == "" {
return "", nil
}
leftTrimBytes := len(rawLine) - len(strings.TrimLeftFunc(rawLine, unicode.IsSpace))
rightTrimBytes := len(rawLine) - len(strings.TrimRightFunc(rawLine, unicode.IsSpace))
trimmedEnd := len(rawLine) - rightTrimBytes
if trimmedEnd < leftTrimBytes {
trimmedEnd = leftTrimBytes
}
trimmed := strings.TrimSpace(rawLine)
cues := make([]model.Cue, 0, len(line))
cursor := 0
for _, piece := range line {
pieceEnd := cursor + len(piece.raw)
if piece.cue != nil {
byteStart := max(cursor, leftTrimBytes)
byteEnd := min(pieceEnd, trimmedEnd)
if byteStart < byteEnd {
cue := *piece.cue
cue.Value = rawLine[byteStart:byteEnd]
cue.ByteStart = byteStart - leftTrimBytes
cue.ByteEnd = byteEnd - leftTrimBytes - 1
cues = append(cues, cue)
}
}
cursor = pieceEnd
}
return trimmed, cues
}
func normalizeTTMLPieceRaw(raw string) string {
raw = str.SanitizeText(raw)
raw = strings.ReplaceAll(raw, "\r\n", "\n")
raw = strings.ReplaceAll(raw, "\r", "\n")
return raw
}
func concatTTMLPieceRaw(pieces []ttmlPiece) string {
var raw strings.Builder
for _, piece := range pieces {
raw.WriteString(normalizeTTMLPieceRaw(piece.raw))
}
return raw.String()
}
func ttmlPiecesContainCue(pieces []ttmlPiece) bool {
for _, piece := range pieces {
if piece.cue != nil {
return true
}
}
return false
}
func cloneTTMLCue(cue *model.Cue) *model.Cue {
if cue == nil {
return nil
}
cloned := *cue
return &cloned
}
func (p *ttmlParser) toLyricList() model.LyricList {
res := make(model.LyricList, 0, len(p.mainLangOrder)+len(p.translationLangOrder)+len(p.pronunciationLangOrder))
for _, lang := range p.mainLangOrder {

20
core/lyrics/ttml_test.go
View File

@ -141,9 +141,9 @@ var _ = Describe("parseTTML", func() {
Expect(line.End).To(Equal(gg.P(int64(3000))))
Expect(line.Cue).To(HaveLen(3))
Expect(line.Cue[0]).To(Equal(model.Cue{Start: gg.P(int64(1000)), End: gg.P(int64(1400)), Value: "He", AgentID: "main"}))
Expect(line.Cue[1]).To(Equal(model.Cue{Start: gg.P(int64(1400)), End: gg.P(int64(1800)), Value: "llo", AgentID: "main"}))
Expect(line.Cue[2]).To(Equal(model.Cue{Start: gg.P(int64(2000)), End: gg.P(int64(2500)), Value: "echo", AgentID: "__nd_bg__|main"}))
Expect(line.Cue[0]).To(Equal(model.Cue{Start: gg.P(int64(1000)), End: gg.P(int64(1400)), Value: "He", ByteStart: 0, ByteEnd: 1, AgentID: "main"}))
Expect(line.Cue[1]).To(Equal(model.Cue{Start: gg.P(int64(1400)), End: gg.P(int64(1800)), Value: "llo", ByteStart: 2, ByteEnd: 4, AgentID: "main"}))
Expect(line.Cue[2]).To(Equal(model.Cue{Start: gg.P(int64(2000)), End: gg.P(int64(2500)), Value: "echo", ByteStart: 6, ByteEnd: 9, AgentID: "__nd_bg__|main"}))
})
It("should parse named TTML agents into main, voice, and group roles", func() {
@ -241,8 +241,8 @@ var _ = Describe("parseTTML", func() {
Expect(line.Value).To(Equal("go\ngo"))
Expect(line.End).To(Equal(gg.P(int64(45570))))
Expect(line.Cue).To(HaveLen(2))
Expect(line.Cue[0]).To(Equal(model.Cue{Start: gg.P(int64(43444)), End: gg.P(int64(43716)), Value: "go"}))
Expect(line.Cue[1]).To(Equal(model.Cue{Start: gg.P(int64(43716)), End: gg.P(int64(43887)), Value: "go"}))
Expect(line.Cue[0]).To(Equal(model.Cue{Start: gg.P(int64(43444)), End: gg.P(int64(43716)), Value: "go", ByteStart: 0, ByteEnd: 1}))
Expect(line.Cue[1]).To(Equal(model.Cue{Start: gg.P(int64(43716)), End: gg.P(int64(43887)), Value: "go", ByteStart: 3, ByteEnd: 4}))
})
})
@ -325,8 +325,8 @@ var _ = Describe("parseTTML", func() {
Expect(pronunciation.Line[0].Value).To(Equal("konni"))
Expect(pronunciation.Line[0].End).To(Equal(gg.P(int64(2600))))
Expect(pronunciation.Line[0].Cue).To(HaveLen(2))
Expect(pronunciation.Line[0].Cue[0]).To(Equal(model.Cue{Start: gg.P(int64(2000)), End: gg.P(int64(2300)), Value: "ko"}))
Expect(pronunciation.Line[0].Cue[1]).To(Equal(model.Cue{Start: gg.P(int64(2300)), End: gg.P(int64(2600)), Value: "nni"}))
Expect(pronunciation.Line[0].Cue[0]).To(Equal(model.Cue{Start: gg.P(int64(2000)), End: gg.P(int64(2300)), Value: "ko", ByteStart: 0, ByteEnd: 1}))
Expect(pronunciation.Line[0].Cue[1]).To(Equal(model.Cue{Start: gg.P(int64(2300)), End: gg.P(int64(2600)), Value: "nni", ByteStart: 2, ByteEnd: 4}))
})
})
@ -369,9 +369,9 @@ var _ = Describe("parseTTML", func() {
Expect(line.Start).To(Equal(gg.P(int64(2747))))
Expect(line.Value).To(Equal("I woke up"))
Expect(line.Cue).To(HaveLen(3))
Expect(line.Cue[0]).To(Equal(model.Cue{Start: gg.P(int64(2747)), End: gg.P(int64(3018)), Value: "I"}))
Expect(line.Cue[1]).To(Equal(model.Cue{Start: gg.P(int64(3018)), End: gg.P(int64(3179)), Value: "woke"}))
Expect(line.Cue[2]).To(Equal(model.Cue{Start: gg.P(int64(3179)), End: gg.P(int64(3582)), Value: "up"}))
Expect(line.Cue[0]).To(Equal(model.Cue{Start: gg.P(int64(2747)), End: gg.P(int64(3018)), Value: "I", ByteStart: 0, ByteEnd: 0}))
Expect(line.Cue[1]).To(Equal(model.Cue{Start: gg.P(int64(3018)), End: gg.P(int64(3179)), Value: "woke", ByteStart: 2, ByteEnd: 5}))
Expect(line.Cue[2]).To(Equal(model.Cue{Start: gg.P(int64(3179)), End: gg.P(int64(3582)), Value: "up", ByteStart: 7, ByteEnd: 8}))
})
})
})

86
model/lyrics.go
View File

@ -6,16 +6,19 @@ import (
"slices"
"strconv"
"strings"
"unicode"
"github.com/navidrome/navidrome/log"
"github.com/navidrome/navidrome/utils/str"
)
type Cue struct {
Start *int64 `structs:"start,omitempty" json:"start,omitempty"`
End *int64 `structs:"end,omitempty" json:"end,omitempty"`
Value string `structs:"value" json:"value"`
AgentID string `structs:"agentId,omitempty" json:"agentId,omitempty"`
Start *int64 `structs:"start,omitempty" json:"start,omitempty"`
End *int64 `structs:"end,omitempty" json:"end,omitempty"`
Value string `structs:"value" json:"value"`
ByteStart int `structs:"byteStart" json:"byteStart"`
ByteEnd int `structs:"byteEnd" json:"byteEnd"`
AgentID string `structs:"agentId,omitempty" json:"agentId,omitempty"`
}
type Agent struct {
@ -127,14 +130,10 @@ func ToLyrics(language, text string) (*Lyrics, error) {
if validLine {
for idx := range timestamps {
cues := parseEnhancedCues(priorLine)
value := priorLine
if cues != nil {
value = stripEnhancedMarkers(value)
}
value, cues := parseEnhancedLine(priorLine)
structuredLines = append(structuredLines, Line{
Start: &timestamps[idx],
Value: strings.TrimSpace(value),
Value: value,
Cue: cues,
})
}
@ -181,14 +180,10 @@ func ToLyrics(language, text string) (*Lyrics, error) {
if validLine {
for idx := range timestamps {
cues := parseEnhancedCues(priorLine)
value := priorLine
if cues != nil {
value = stripEnhancedMarkers(value)
}
value, cues := parseEnhancedLine(priorLine)
structuredLines = append(structuredLines, Line{
Start: &timestamps[idx],
Value: strings.TrimSpace(value),
Value: value,
Cue: cues,
})
}
@ -213,21 +208,22 @@ func ToLyrics(language, text string) (*Lyrics, error) {
return &lyrics, nil
}
// parseEnhancedCues extracts word-level timing cues from Enhanced LRC inline markers.
// Format: <mm:ss.mm>word <mm:ss.mm>word ...
// Returns nil if no inline markers are found.
func parseEnhancedCues(text string) []Cue {
// parseEnhancedLine extracts word-level timing cues from Enhanced LRC inline markers
// and computes UTF-8 byte offsets against the final stripped line value.
func parseEnhancedLine(text string) (string, []Cue) {
matches := enhancedLRCRegex.FindAllStringSubmatchIndex(text, -1)
if len(matches) == 0 {
return nil
return strings.TrimSpace(text), nil
}
type segment struct {
start int64
text string
start int64
rawStart int
rawEnd int
}
segments := make([]segment, 0, len(matches))
var rawValue strings.Builder
for i, match := range matches {
timeMs, err := parseTime(
// Rewrite <...> as [...] so parseTime can handle it with the same logic
@ -258,22 +254,46 @@ func parseEnhancedCues(text string) []Cue {
if word == "" {
continue
}
segments = append(segments, segment{start: timeMs, text: word})
rawStart := rawValue.Len()
rawValue.WriteString(word)
segments = append(segments, segment{
start: timeMs,
rawStart: rawStart,
rawEnd: rawValue.Len(),
})
}
if len(segments) == 0 {
return nil
return strings.TrimSpace(stripEnhancedMarkers(text)), nil
}
cues := make([]Cue, len(segments))
for i, seg := range segments {
start := seg.start
cues[i] = Cue{
Start: &start,
Value: seg.text,
}
finalRaw := rawValue.String()
leftTrimBytes := len(finalRaw) - len(strings.TrimLeftFunc(finalRaw, unicode.IsSpace))
rightTrimBytes := len(finalRaw) - len(strings.TrimRightFunc(finalRaw, unicode.IsSpace))
trimmedEnd := len(finalRaw) - rightTrimBytes
if trimmedEnd < leftTrimBytes {
trimmedEnd = leftTrimBytes
}
return cues
cues := make([]Cue, 0, len(segments))
for _, seg := range segments {
start := seg.start
byteStart := max(seg.rawStart, leftTrimBytes)
byteEnd := min(seg.rawEnd, trimmedEnd)
if byteStart >= byteEnd {
continue
}
cues = append(cues, Cue{
Start: &start,
Value: finalRaw[byteStart:byteEnd],
ByteStart: byteStart - leftTrimBytes,
ByteEnd: byteEnd - leftTrimBytes - 1,
})
}
return strings.TrimSpace(finalRaw), cues
}
// adjustGroup remaps a capture group index from the original match to our rewritten "[...]" string.

34
model/lyrics_test.go
View File

@ -130,9 +130,9 @@ var _ = Describe("ToLyrics", func() {
Expect(line0.End).To(Equal(&t3000))
Expect(line0.Value).To(Equal("Some lyrics here"))
Expect(line0.Cue).To(Equal([]Cue{
{Start: &t1000, End: &t1500, Value: "Some "},
{Start: &t1500, End: &t2000, Value: "lyrics "},
{Start: &t2000, End: &t3000, Value: "here"},
{Start: &t1000, End: &t1500, Value: "Some ", ByteStart: 0, ByteEnd: 4},
{Start: &t1500, End: &t2000, Value: "lyrics ", ByteStart: 5, ByteEnd: 11},
{Start: &t2000, End: &t3000, Value: "here", ByteStart: 12, ByteEnd: 15},
}))
line1 := lyrics.Line[1]
@ -140,8 +140,8 @@ var _ = Describe("ToLyrics", func() {
Expect(line1.End).To(Equal(&t3500))
Expect(line1.Value).To(Equal("More words"))
Expect(line1.Cue).To(Equal([]Cue{
{Start: &t3000, Value: "More "},
{Start: &t3500, Value: "words"},
{Start: &t3000, Value: "More ", ByteStart: 0, ByteEnd: 4},
{Start: &t3500, Value: "words", ByteStart: 5, ByteEnd: 9},
}))
Expect(line1.Cue[1].End).To(BeNil())
@ -166,8 +166,8 @@ var _ = Describe("ToLyrics", func() {
t3000 := int64(3000)
Expect(lyrics.Line[0].Cue).To(Equal([]Cue{
{Start: &t1000, End: &t1500, Value: "Some "},
{Start: &t1500, End: &t3000, Value: "lyrics"},
{Start: &t1000, End: &t1500, Value: "Some ", ByteStart: 0, ByteEnd: 4},
{Start: &t1500, End: &t3000, Value: "lyrics", ByteStart: 5, ByteEnd: 10},
}))
Expect(lyrics.Line[0].Value).To(Equal("Some lyrics"))
Expect(lyrics.Line[0].End).To(Equal(&t3000))
@ -176,9 +176,25 @@ var _ = Describe("ToLyrics", func() {
Expect(lyrics.Line[1].Value).To(Equal("Plain line"))
Expect(lyrics.Line[2].Cue).To(Equal([]Cue{
{Start: &t5000, Value: "More "},
{Start: &t5500, Value: "words"},
{Start: &t5000, Value: "More ", ByteStart: 0, ByteEnd: 4},
{Start: &t5500, Value: "words", ByteStart: 5, ByteEnd: 9},
}))
Expect(lyrics.Line[2].Value).To(Equal("More words"))
})
It("should preserve byte offsets for Enhanced LRC cues", func() {
lyrics, err := ToLyrics("xxx", "[00:00.00]<00:00.00>Oh <00:00.90>love<00:01.30> me <00:01.60>tonight")
Expect(err).ToNot(HaveOccurred())
Expect(lyrics.Line).To(HaveLen(1))
t0, t900, t1300, t1600 := int64(0), int64(900), int64(1300), int64(1600)
line := lyrics.Line[0]
Expect(line.Value).To(Equal("Oh love me tonight"))
Expect(line.Cue).To(Equal([]Cue{
{Start: &t0, Value: "Oh ", ByteStart: 0, ByteEnd: 2},
{Start: &t900, Value: "love", ByteStart: 3, ByteEnd: 6},
{Start: &t1300, Value: " me ", ByteStart: 7, ByteEnd: 10},
{Start: &t1600, Value: "tonight", ByteStart: 11, ByteEnd: 17},
}))
})
})

6
server/subsonic/helpers.go
View File

@ -619,8 +619,10 @@ func buildLyricCues(cues []model.Cue, lineEnd *int64) []responses.LyricCue {
}
cue := responses.LyricCue{
Start: *cues[i].Start,
Value: cues[i].Value,
Start: *cues[i].Start,
Value: cues[i].Value,
ByteStart: cues[i].ByteStart,
ByteEnd: cues[i].ByteEnd,
}
if hasAnyEnd {
end := cues[i].End

164
server/subsonic/media_retrieval_test.go
View File

@ -277,6 +277,8 @@ var _ = Describe("MediaRetrievalController", func() {
expectedCue := expectedCueLine.Cue[k]
Expect(realCue.Value).To(Equal(expectedCue.Value))
Expect(realCue.Start).To(Equal(expectedCue.Start))
Expect(realCue.ByteStart).To(Equal(expectedCue.ByteStart))
Expect(realCue.ByteEnd).To(Equal(expectedCue.ByteEnd))
if expectedCue.End == nil {
Expect(realCue.End).To(BeNil())
} else {
@ -514,14 +516,18 @@ var _ = Describe("MediaRetrievalController", func() {
Value: "konni",
Cue: []responses.LyricCue{
{
Start: tokenStartA,
End: &tokenEndA,
Value: "ko",
Start: tokenStartA,
End: &tokenEndA,
ByteStart: 0,
ByteEnd: 1,
Value: "ko",
},
{
Start: tokenStartB,
End: &tokenEndB,
Value: "nni",
Start: tokenStartB,
End: &tokenEndB,
ByteStart: 2,
ByteEnd: 4,
Value: "nni",
},
},
},
@ -552,16 +558,20 @@ var _ = Describe("MediaRetrievalController", func() {
Value: "Hello echo",
Cue: []model.Cue{
{
Start: &tokenStartA,
End: &tokenEndA,
Value: "Hello",
AgentID: "lead",
Start: &tokenStartA,
End: &tokenEndA,
Value: "Hello",
ByteStart: 0,
ByteEnd: 4,
AgentID: "lead",
},
{
Start: &tokenStartB,
End: &tokenEndB,
Value: "echo",
AgentID: "__nd_bg__|lead",
Start: &tokenStartB,
End: &tokenEndB,
Value: "echo",
ByteStart: 6,
ByteEnd: 9,
AgentID: "__nd_bg__|lead",
},
},
},
@ -608,9 +618,11 @@ var _ = Describe("MediaRetrievalController", func() {
AgentID: "lead",
Cue: []responses.LyricCue{
{
Start: tokenStartA,
End: &tokenEndA,
Value: "Hello",
Start: tokenStartA,
End: &tokenEndA,
ByteStart: 0,
ByteEnd: 4,
Value: "Hello",
},
},
},
@ -622,9 +634,11 @@ var _ = Describe("MediaRetrievalController", func() {
AgentID: "__nd_bg__|lead",
Cue: []responses.LyricCue{
{
Start: tokenStartB,
End: &tokenEndB,
Value: "echo",
Start: tokenStartB,
End: &tokenEndB,
ByteStart: 6,
ByteEnd: 9,
Value: "echo",
},
},
},
@ -633,6 +647,116 @@ var _ = Describe("MediaRetrievalController", func() {
},
})
})
It("should return required cue byte offsets for ambiguous and multibyte cue lines", func() {
r := newGetRequest("id=1&enhanced=true")
asciiLineStart := int64(0)
asciiLineEnd := int64(2400)
asciiCueStartA := int64(0)
asciiCueEndA := int64(300)
asciiCueStartB := int64(900)
asciiCueEndB := int64(1300)
asciiCueStartC := int64(1300)
asciiCueEndC := int64(1600)
asciiCueStartD := int64(1600)
utfLineStart := int64(2747)
utfLineEnd := int64(6214)
utfCueStartA := int64(2747)
utfCueEndA := int64(3018)
utfCueStartB := int64(3018)
utfCueEndB := int64(3179)
utfCueStartC := int64(3582)
utfCueEndC := int64(4100)
utfCueStartD := int64(4500)
utfCueEndD := int64(6214)
lyricsJSON, err := json.Marshal(model.LyricList{
{
Lang: "eng",
Synced: true,
Line: []model.Line{
{
Start: &asciiLineStart,
End: &asciiLineEnd,
Value: "Oh love love me tonight",
Cue: []model.Cue{
{Start: &asciiCueStartA, End: &asciiCueEndA, Value: "Oh", ByteStart: 0, ByteEnd: 1},
{Start: &asciiCueStartB, End: &asciiCueEndB, Value: "love", ByteStart: 8, ByteEnd: 11},
{Start: &asciiCueStartC, End: &asciiCueEndC, Value: "me", ByteStart: 13, ByteEnd: 14},
{Start: &asciiCueStartD, Value: "tonight", ByteStart: 16, ByteEnd: 22},
},
},
{
Start: &utfLineStart,
End: &utfLineEnd,
Value: "눈을 뜬 순간",
Cue: []model.Cue{
{Start: &utfCueStartA, End: &utfCueEndA, Value: "눈", ByteStart: 0, ByteEnd: 2},
{Start: &utfCueStartB, End: &utfCueEndB, Value: "을", ByteStart: 3, ByteEnd: 5},
{Start: &utfCueStartC, End: &utfCueEndC, Value: "뜬", ByteStart: 7, ByteEnd: 9},
{Start: &utfCueStartD, End: &utfCueEndD, Value: "순간", ByteStart: 11, ByteEnd: 16},
},
},
},
},
})
Expect(err).ToNot(HaveOccurred())
mockRepo.SetData(model.MediaFiles{
{
ID: "1",
Artist: "Rick Astley",
Title: "Never Gonna Give You Up",
Lyrics: string(lyricsJSON),
},
})
response, err := router.GetLyricsBySongId(r)
Expect(err).ToNot(HaveOccurred())
compareResponses(response.LyricsList, responses.LyricsList{
StructuredLyrics: responses.StructuredLyrics{
{
DisplayArtist: "Rick Astley",
DisplayTitle: "Never Gonna Give You Up",
Kind: "main",
Lang: "eng",
Synced: true,
Line: []responses.Line{
{Start: &asciiLineStart, Value: "Oh love love me tonight"},
{Start: &utfLineStart, Value: "눈을 뜬 순간"},
},
CueLine: []responses.CueLine{
{
Index: 0,
Start: &asciiLineStart,
End: &asciiLineEnd,
Value: "Oh love love me tonight",
Cue: []responses.LyricCue{
{Start: asciiCueStartA, End: &asciiCueEndA, Value: "Oh", ByteStart: 0, ByteEnd: 1},
{Start: asciiCueStartB, End: &asciiCueEndB, Value: "love", ByteStart: 8, ByteEnd: 11},
{Start: asciiCueStartC, End: &asciiCueEndC, Value: "me", ByteStart: 13, ByteEnd: 14},
{Start: asciiCueStartD, End: &asciiLineEnd, Value: "tonight", ByteStart: 16, ByteEnd: 22},
},
},
{
Index: 1,
Start: &utfLineStart,
End: &utfLineEnd,
Value: "눈을 뜬 순간",
Cue: []responses.LyricCue{
{Start: utfCueStartA, End: &utfCueEndA, Value: "눈", ByteStart: 0, ByteEnd: 2},
{Start: utfCueStartB, End: &utfCueEndB, Value: "을", ByteStart: 3, ByteEnd: 5},
{Start: utfCueStartC, End: &utfCueEndC, Value: "뜬", ByteStart: 7, ByteEnd: 9},
{Start: utfCueStartD, End: &utfCueEndD, Value: "순간", ByteStart: 11, ByteEnd: 16},
},
},
},
},
},
})
})
})
})

10
server/subsonic/responses/responses.go
View File

@ -538,9 +538,11 @@ type Line struct {
}
type LyricCue struct {
Start int64 `xml:"start,attr" json:"start"`
End *int64 `xml:"end,attr,omitempty" json:"end,omitempty"`
Value string `xml:",chardata" json:"value"`
Start int64 `xml:"start,attr" json:"start"`
End *int64 `xml:"end,attr,omitempty" json:"end,omitempty"`
ByteStart int `xml:"byteStart,attr" json:"byteStart"`
ByteEnd int `xml:"byteEnd,attr" json:"byteEnd"`
Value string `xml:",chardata" json:"value"`
}
type Agent struct {
@ -553,7 +555,7 @@ type CueLine struct {
Index int32 `xml:"index,attr" json:"index"`
Start *int64 `xml:"start,attr,omitempty" json:"start,omitempty"`
End *int64 `xml:"end,attr,omitempty" json:"end,omitempty"`
Value string `xml:"value,attr,omitempty" json:"value,omitempty"`
Value string `xml:"value,attr" json:"value"`
AgentID string `xml:"agentId,attr,omitempty" json:"agentId,omitempty"`
Cue []LyricCue `xml:"cue,omitempty" json:"cue,omitempty"`
}

67
ui/src/audioplayer/KaraokeLyricsOverlay.jsx
View File

@ -26,6 +26,7 @@ import {
hasStructuredLyricContent,
resolveKaraokeTokenWindow,
resolveLayerLineForMain,
utf8ByteRangeToCodeUnitRange,
} from './lyrics'
const KARAOKE_RENDER_LEAD_MS = 24
@ -635,6 +636,72 @@ const buildSegmentsFromLine = (line) => {
}
const text = line.value || ''
const exactSegments = (() => {
if (!text) {
return null
}
const rangedTokens = line.tokens
.map((token, tokenIndex) => ({
token,
tokenIndex,
range: utf8ByteRangeToCodeUnitRange(
text,
token?.byteStart,
token?.byteEnd,
),
}))
.filter((entry) => entry.range != null)
if (
rangedTokens.length !== line.tokens.length ||
rangedTokens.length === 0
) {
return null
}
rangedTokens.sort(
(a, b) =>
a.range.start - b.range.start ||
a.range.end - b.range.end ||
a.tokenIndex - b.tokenIndex,
)
const segments = []
let cursor = 0
for (const entry of rangedTokens) {
if (entry.range.start < cursor) {
return null
}
if (entry.range.start > cursor) {
segments.push({
text: text.slice(cursor, entry.range.start),
token: null,
tokenIndex: -1,
})
}
segments.push({
text: entry.range.text,
token: entry.token,
tokenIndex: entry.tokenIndex,
})
cursor = entry.range.end
}
if (cursor < text.length) {
segments.push({
text: text.slice(cursor),
token: null,
tokenIndex: -1,
})
}
return segments
})()
if (exactSegments) {
return exactSegments
}
const matchedSegments = []
const fallbackSegments = []
let cursor = 0

68
ui/src/audioplayer/KaraokeLyricsOverlay.test.jsx
View File

@ -177,6 +177,74 @@ describe('<KaraokeLyricsOverlay /> behavior', () => {
expect(translationLine.querySelectorAll('span')).toHaveLength(1)
})
it('uses cue byte offsets to segment repeated words in the karaoke line', () => {
renderOverlay({
mainLyric: {
kind: 'main',
lang: 'en',
synced: true,
line: [{ start: 0, end: 2400, value: 'Oh love love me tonight' }],
cueLine: [
{
index: 0,
start: 0,
end: 2400,
value: 'Oh love love me tonight',
cue: [
{ start: 0, end: 300, value: 'Oh', byteStart: 0, byteEnd: 1 },
{
start: 900,
end: 1300,
value: 'love',
byteStart: 8,
byteEnd: 11,
},
{
start: 1300,
end: 1600,
value: 'me',
byteStart: 13,
byteEnd: 14,
},
{
start: 1600,
end: 2400,
value: 'tonight',
byteStart: 16,
byteEnd: 22,
},
],
},
],
},
translationLyric: null,
pronunciationLyric: null,
showTranslation: false,
showPronunciation: false,
translationEnabled: false,
pronunciationEnabled: false,
audioInstance: {
...audioInstance,
currentTime: 1.0,
},
})
const mainLine = screen.getByText('Oh').parentElement
const segments = Array.from(mainLine.querySelectorAll('span')).map(
(span) => span.textContent,
)
expect(segments).toEqual([
'Oh',
' love ',
'love',
' ',
'me',
' ',
'tonight',
])
})
it('highlights line-timed pronunciation and translation rows with the active main line', () => {
renderOverlay({
mainLyric: {

80
ui/src/audioplayer/lyrics.js
View File

@ -19,6 +19,17 @@ const toTime = (value) => {
return Number.isFinite(numeric) ? numeric : null
}
const toByteOffset = (value) => {
if (value == null || value === '') {
return null
}
const numeric = Number(value)
if (!Number.isInteger(numeric) || numeric < 0) {
return null
}
return numeric
}
const compareNullableTime = (a, b) => {
if (a == null && b == null) {
return 0
@ -78,10 +89,79 @@ const normalizeToken = (token) => {
if (!value.trim()) {
return null
}
const byteStart = toByteOffset(token.byteStart)
const byteEnd = toByteOffset(token.byteEnd)
return {
start: toTime(token.start),
end: toTime(token.end),
value,
...(byteStart != null ? { byteStart } : {}),
...(byteEnd != null ? { byteEnd } : {}),
}
}
const utf8BytesForCodePoint = (codePoint) => {
if (codePoint <= 0x7f) {
return 1
}
if (codePoint <= 0x7ff) {
return 2
}
if (codePoint <= 0xffff) {
return 3
}
return 4
}
export const utf8ByteOffsetToCodeUnitIndex = (text, targetByteOffset) => {
if (typeof text !== 'string' || text.length === 0) {
return 0
}
const target = toByteOffset(targetByteOffset)
if (target == null || target <= 0) {
return 0
}
let byteOffset = 0
let index = 0
while (index < text.length) {
if (byteOffset >= target) {
return index
}
const codePoint = text.codePointAt(index)
byteOffset += utf8BytesForCodePoint(codePoint)
index += codePoint > 0xffff ? 2 : 1
}
return text.length
}
export const utf8ByteRangeToCodeUnitRange = (text, byteStart, byteEnd) => {
if (typeof text !== 'string') {
return null
}
const start = toByteOffset(byteStart)
const end = toByteOffset(byteEnd)
if (start == null || end == null || end < start) {
return null
}
const startIndex = utf8ByteOffsetToCodeUnitIndex(text, start)
const endIndex = utf8ByteOffsetToCodeUnitIndex(text, end + 1)
if (
startIndex >= endIndex ||
startIndex > text.length ||
endIndex > text.length
) {
return null
}
return {
start: startIndex,
end: endIndex,
text: text.slice(startIndex, endIndex),
}
}

56
ui/src/audioplayer/lyrics.test.js
View File

@ -13,6 +13,8 @@ import {
selectLyricLayers,
structuredLyricsToLrc,
structuredLyricToLrc,
utf8ByteOffsetToCodeUnitIndex,
utf8ByteRangeToCodeUnitRange,
} from './lyrics'
describe('lyrics helpers', () => {
@ -412,6 +414,60 @@ describe('lyrics helpers', () => {
])
})
it('preserves cue byte offsets on karaoke tokens', () => {
const lines = buildKaraokeLines({
lang: 'eng',
synced: true,
line: [{ start: 0, end: 2400, value: 'Oh love love me tonight' }],
cueLine: [
{
index: 0,
start: 0,
end: 2400,
value: 'Oh love love me tonight',
cue: [
{ start: 0, end: 300, value: 'Oh', byteStart: 0, byteEnd: 1 },
{ start: 900, end: 1300, value: 'love', byteStart: 8, byteEnd: 11 },
{ start: 1300, end: 1600, value: 'me', byteStart: 13, byteEnd: 14 },
{
start: 1600,
end: 2400,
value: 'tonight',
byteStart: 16,
byteEnd: 22,
},
],
},
],
})
expect(
lines[0].tokens.map((token) => [
token.value,
token.byteStart,
token.byteEnd,
]),
).toEqual([
['Oh', 0, 1],
['love', 8, 11],
['me', 13, 14],
['tonight', 16, 22],
])
})
it('maps UTF-8 byte offsets to string ranges for multibyte lyrics', () => {
const text = '눈을 뜬 순간'
expect(utf8ByteOffsetToCodeUnitIndex(text, 0)).toBe(0)
expect(utf8ByteOffsetToCodeUnitIndex(text, 3)).toBe(1)
expect(utf8ByteOffsetToCodeUnitIndex(text, 7)).toBe(3)
expect(utf8ByteRangeToCodeUnitRange(text, 11, 16)).toEqual({
start: 5,
end: 7,
text: '순간',
})
})
it('falls back to legacy cueLine role values when agents are absent', () => {
const lines = buildKaraokeLines({
lang: 'eng',

3
ui/src/subsonic/index.test.js
View File

@ -1,5 +1,4 @@
import { vi } from 'vitest'
import { COVER_ART_SIZE } from '../consts'
import { httpClient } from '../dataProvider'
import subsonic from './index'
@ -7,6 +6,8 @@ vi.mock('../dataProvider', () => ({
httpClient: vi.fn(() => Promise.resolve({})),
}))
const COVER_ART_SIZE = 600
describe('getCoverArtUrl', () => {
beforeEach(() => {
// Mock window.location