feat: light up lyrics word by word with translations

web/src/features/lyrics/lrc.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, it } from 'vitest'
-import { lineIndexAt, parseLrc } from './lrc'
+import { buildLines, lineIndexAt, parseLrc, parseYrc, withTranslations, wordProgress } from './lrc'
 
 describe('parseLrc', () => {
   it('解析分、秒和不同精度的小数部分', () => {
@@ -39,3 +39,121 @@ describe('lineIndexAt', () => {
     expect(lineIndexAt([], 1000)).toBe(-1)
   })
 })
+
+describe('parseYrc', () => {
+  it('解析行和字的绝对时间，行文本由字拼成', () => {
+    expect(parseYrc('[1000,900](1000,400,0)故(1400,500,0)事')[0]).toEqual({
+      timeMs: 1000,
+      text: '故事',
+      words: [
+        { startMs: 1000, durationMs: 400, text: '故' },
+        { startMs: 1400, durationMs: 500, text: '事' },
+      ],
+    })
+  })
+
+  it('保留字里的括号、方括号和空格，只把完整的时间标记当作分隔', () => {
+    const [line] = parseYrc('[0,1000](0,500,0)(ooh) (500,250,0)[yeah] (750,250,0)(1,2)')
+    expect(line.words?.map(word => word.text)).toEqual(['(ooh) ', '[yeah] ', '(1,2)'])
+    expect(line.text).toBe('(ooh) [yeah] (1,2)')
+  })
+
+  it('跳过不是 [ 开头的行、没有字的行和全是空白的行，兼容 CRLF', () => {
+    const lines = parseYrc('{"t":0,"c":[]}\r\n\r\n[0,500]\r\n[500,500](500,500,0)  \r\n注[700,100](700,100,0)释\r\n[1000,500](1000,500,0)词\r\n')
+    expect(lines.map(line => line.text)).toEqual(['词', ''])
+    expect(lines[0].words).toEqual([{ startMs: 1000, durationMs: 500, text: '词' }])
+  })
+
+  it('按行起始时间排序', () => {
+    const lines = parseYrc('[2000,500](2000,500,0)二\n[1000,500](1000,500,0)一')
+    expect(lines.filter(line => line.text).map(line => line.text)).toEqual(['一', '二'])
+  })
+
+  it('两行之间有间奏时在上一行结束处补空白行，换气的短停顿不补', () => {
+    const lines = parseYrc('[0,1000](0,1000,0)一\n[1500,1000](1500,1000,0)二\n[20000,1000](20000,1000,0)三')
+    expect(lines.map(({ timeMs, text }) => [timeMs, text])).toEqual([
+      [0, '一'],
+      [1500, '二'],
+      [2500, ''],
+      [20_000, '三'],
+      [21_000, ''],
+    ])
+  })
+})
+
+describe('wordProgress', () => {
+  const word = { startMs: 1000, durationMs: 400, text: '故' }
+
+  it('唱到这个字之前是 0，唱完之后是 1，中间按时间线性增长', () => {
+    expect(wordProgress(word, 999)).toBe(0)
+    expect(wordProgress(word, 1100)).toBe(0.25)
+    expect(wordProgress(word, 1400)).toBe(1)
+    expect(wordProgress(word, 5000)).toBe(1)
+  })
+
+  it('时长为 0 的字唱到时直接点亮', () => {
+    const instant = { startMs: 1000, durationMs: 0, text: '故' }
+    expect(wordProgress(instant, 999)).toBe(0)
+    expect(wordProgress(instant, 1000)).toBe(0)
+    expect(wordProgress(instant, 1001)).toBe(1)
+  })
+})
+
+describe('withTranslations', () => {
+  const line = (timeMs: number, text: string) => ({ timeMs, text })
+
+  it('翻译配到时间最近的一行，容忍取整造成的偏差', () => {
+    expect(withTranslations([line(1000, '一'), line(3000, '二')], [line(991, 'one'), line(3009, 'two')])).toEqual([
+      { timeMs: 1000, text: '一', translation: 'one' },
+      { timeMs: 3000, text: '二', translation: 'two' },
+    ])
+  })
+
+  it('翻译离后一行更近时配到后一行', () => {
+    const paired = withTranslations([line(1000, '一'), line(1400, '二')], [line(1390, 'two')])
+    expect(paired.map(({ translation }) => translation)).toEqual([undefined, 'two'])
+  })
+
+  it('偏差超过 500ms 时视为这一行没有翻译', () => {
+    expect(withTranslations([line(1000, '一')], [line(1500, 'one')])[0].translation).toBe('one')
+    expect(withTranslations([line(1000, '一')], [line(1501, 'one')])[0].translation).toBeUndefined()
+  })
+
+  it('没有翻译的行不借用相邻行的翻译', () => {
+    const paired = withTranslations([line(1000, '一'), line(1300, '二')], [line(1000, 'one')])
+    expect(paired.map(({ translation }) => translation)).toEqual(['one', undefined])
+  })
+
+  it('同一行有多条候选翻译时取最近的一条', () => {
+    expect(withTranslations([line(1000, '一')], [line(800, 'far'), line(990, 'near'), line(1200, 'far too')])[0].translation).toBe('near')
+  })
+
+  it('空白行和空翻译不参与配对', () => {
+    const paired = withTranslations([line(1000, '一'), line(1100, ''), line(5000, '二')], [line(1100, 'one'), line(5000, '')])
+    expect(paired).toEqual([
+      { timeMs: 1000, text: '一', translation: 'one' },
+      { timeMs: 1100, text: '' },
+      { timeMs: 5000, text: '二' },
+    ])
+  })
+
+  it('没有歌词行时不报错', () => {
+    expect(withTranslations([], [line(1000, 'one')])).toEqual([])
+  })
+})
+
+describe('buildLines', () => {
+  it('有逐字歌词时用逐字歌词的行，翻译配到这些行上', () => {
+    const lines = buildLines('[00:01.00]行级歌词', '[1000,500](1000,500,0)逐(1250,250,0)字', '[ti:标题]\n[00:01.00]Word by word')
+    expect(lines[0]).toMatchObject({ timeMs: 1000, text: '逐字', translation: 'Word by word' })
+    expect(lines[0].words).toHaveLength(2)
+  })
+
+  it('没有逐字歌词时保持行级歌词，不带逐字数据', () => {
+    expect(buildLines('[00:01.00]行级歌词')).toEqual([{ timeMs: 1000, text: '行级歌词' }])
+  })
+
+  it('逐字歌词解析不出内容时退回行级歌词', () => {
+    expect(buildLines('[00:01.00]行级歌词', '{"t":0}')).toEqual([{ timeMs: 1000, text: '行级歌词' }])
+  })
+})

web/src/features/lyrics/lrc.ts

@@ -1,9 +1,22 @@
-export interface LyricLine {
-  timeMs: number
+export interface LyricWord {
+  startMs: number
+  durationMs: number
   text: string
 }
 
+export interface LyricLine {
+  timeMs: number
+  text: string
+  words?: LyricWord[]
+  translation?: string
+}
+
 const timeTag = /\[(\d{1,2}):(\d{2})(?:\.(\d{1,3}))?\]/g
+const yrcLineTag = /^\[(\d+),(\d+)\]/
+const yrcWordTag = /\((\d+),(\d+),\d+\)/g
+// 逐字歌词没有空白行，要靠行尾和下一行的间隔判断间奏。间隔太短就算作换气：清除高亮再点亮会闪烁。
+const interludeGapMs = 3000
+const maxTranslationGapMs = 500
 
 export function parseLrc(lrc: string): LyricLine[] {
   const lines: LyricLine[] = []
@@ -20,6 +33,59 @@ export function parseLrc(lrc: string): LyricLine[] {
   return lines.filter((line, i) => line.text || i === 0 || lines[i - 1].text)
 }
 
+export function parseYrc(yrc: string): LyricLine[] {
+  const lines: (LyricLine & { endMs: number })[] = []
+  for (const row of yrc.split(/\r?\n/)) {
+    const head = yrcLineTag.exec(row)
+    if (!head)
+      continue
+    const tags = [...row.matchAll(yrcWordTag)]
+    const words = tags.map((tag, i) => ({
+      startMs: Number(tag[1]),
+      durationMs: Number(tag[2]),
+      text: row.slice(tag.index + tag[0].length, tags[i + 1]?.index),
+    }))
+    const text = words.map(word => word.text).join('').trim()
+    const timeMs = Number(head[1])
+    if (text)
+      lines.push({ timeMs, text, words, endMs: timeMs + Number(head[2]) })
+  }
+  lines.sort((a, b) => a.timeMs - b.timeMs)
+  return lines.flatMap(({ endMs, ...line }, i) => {
+    const next = lines[i + 1]
+    return !next || next.timeMs - endMs >= interludeGapMs ? [line, { timeMs: endMs, text: '' }] : [line]
+  })
+}
+
+export function wordProgress({ startMs, durationMs }: LyricWord, positionMs: number): number {
+  return Math.min(Math.max((positionMs - startMs) / Math.max(durationMs, 1), 0), 1)
+}
+
+// 由翻译找行，而不是由行找翻译：否则没有翻译的行会借用相邻行的翻译。
+export function withTranslations(lines: LyricLine[], translations: LyricLine[]): LyricLine[] {
+  const sung = lines.filter(line => line.text)
+  const gap = (a: LyricLine, b: LyricLine) => Math.abs(a.timeMs - b.timeMs)
+  const nearest = new Map<LyricLine, LyricLine>()
+  for (const translation of translations) {
+    const index = lineIndexAt(sung, translation.timeMs)
+    const [line] = sung.slice(Math.max(index, 0), index + 2).sort((a, b) => gap(a, translation) - gap(b, translation))
+    if (!line || !translation.text || gap(line, translation) > maxTranslationGapMs)
+      continue
+    const current = nearest.get(line)
+    if (!current || gap(line, translation) < gap(line, current))
+      nearest.set(line, translation)
+  }
+  return lines.map((line) => {
+    const translation = nearest.get(line)
+    return translation ? { ...line, translation: translation.text } : line
+  })
+}
+
+export function buildLines(lyric: string, words = '', translation = ''): LyricLine[] {
+  const wordLines = parseYrc(words)
+  return withTranslations(wordLines.length ? wordLines : parseLrc(lyric), parseLrc(translation))
+}
+
 export function lineIndexAt(lines: LyricLine[], positionMs: number): number {
   let low = 0
   let high = lines.length - 1

web/src/features/lyrics/lyrics-store.test.ts

@@ -0,0 +1,62 @@
+import type { NowPlaying } from '@/core/realtime'
+import { describe, expect, it, vi } from 'vitest'
+import { Emitter } from '@/shared/lib/emitter'
+
+function nowPlaying(id: string, overrides: Partial<NowPlaying> = {}): NowPlaying {
+  return { entry: { id }, mediaUrl: '', startedAtMs: 1000, durationMs: 60_000, gain: 1, lyric: '[00:01.00]行级歌词', ...overrides } as NowPlaying
+}
+
+async function setup() {
+  vi.resetModules()
+  const room = Object.assign(new EventTarget(), { state: { room: null as { nowPlaying: NowPlaying | null } | null } })
+  const frames = new Emitter<number>()
+  vi.doMock('@/core/room', () => ({ roomStore: room }))
+  vi.doMock('@/features/playback', () => ({ playerStore: { frames } }))
+  const { lyricsStore } = await import('./lyrics-store')
+  lyricsStore.start()
+  const play = (next: NowPlaying | null) => {
+    room.state = { room: next && { nowPlaying: next } }
+    room.dispatchEvent(new Event('change'))
+  }
+  return { lyricsStore, frames, play }
+}
+
+describe('lyricsStore', () => {
+  it('有逐字歌词和翻译时，歌词行带上逐字数据和翻译', async () => {
+    const { lyricsStore, play } = await setup()
+    play(nowPlaying('1', { lyricWords: '[1000,500](1000,500,0)逐字', lyricTrans: '[00:01.00]Words' }))
+    expect(lyricsStore.state.lines[0]).toEqual({
+      timeMs: 1000,
+      text: '逐字',
+      translation: 'Words',
+      words: [{ startMs: 1000, durationMs: 500, text: '逐字' }],
+    })
+  })
+
+  it('没有逐字歌词时使用行级歌词', async () => {
+    const { lyricsStore, play } = await setup()
+    play(nowPlaying('1'))
+    expect(lyricsStore.state.lines).toEqual([{ timeMs: 1000, text: '行级歌词' }])
+  })
+
+  it('换歌时先按歌曲开头计算当前行，之后跟随播放位置', async () => {
+    const { lyricsStore, frames, play } = await setup()
+    play(nowPlaying('1'))
+    frames.emit(30_000)
+    expect(lyricsStore.state.index).toBe(0)
+    play(nowPlaying('2', { lyric: '[00:01.00]一\n[00:02.00]二' }))
+    expect(lyricsStore.state.index).toBe(-1)
+    frames.emit(2000)
+    expect(lyricsStore.state.index).toBe(1)
+  })
+
+  it('同一首歌的新快照不重建歌词行，离开房间后清空', async () => {
+    const { lyricsStore, play } = await setup()
+    play(nowPlaying('1'))
+    const { lines } = lyricsStore.state
+    play(nowPlaying('1'))
+    expect(lyricsStore.state.lines).toBe(lines)
+    play(null)
+    expect(lyricsStore.state).toEqual({ lines: [], index: -1 })
+  })
+})

web/src/features/lyrics/lyrics-store.ts

@@ -2,7 +2,7 @@ import type { LyricLine } from './lrc'
 import { roomStore } from '@/core/room'
 import { playerStore } from '@/features/playback'
 import { Store } from '@/shared/lib/store'
-import { lineIndexAt, parseLrc } from './lrc'
+import { buildLines, lineIndexAt } from './lrc'
 
 class LyricsStore extends Store<{ lines: LyricLine[], index: number }> {
   #track = ''
@@ -22,7 +22,7 @@ class LyricsStore extends Store<{ lines: LyricLine[], index: number }> {
     if (track === this.#track)
       return
     this.#track = track
-    const lines = parseLrc(nowPlaying?.lyric ?? '')
+    const lines = nowPlaying ? buildLines(nowPlaying.lyric, nowPlaying.lyricWords, nowPlaying.lyricTrans) : []
     // 换歌时新歌尚未开始播放，播放器给出的仍是上一首的位置：先按歌曲开头计算，开始播放后再由每帧的位置更新。
     this.setState({ lines, index: lineIndexAt(lines, 0) })
   }

web/src/features/lyrics/lyrics.css

@@ -8,6 +8,8 @@
 .lyric-line { max-width: 880px; margin: 0 auto; padding: 8px 32px; line-height: 1.5; text-align: center; color: var(--color-foreground-subtle); opacity: .78; font-weight: 600; overflow-wrap: anywhere; transform: translateY(4px) scale(.7); transform-origin: center; transition: transform 420ms cubic-bezier(.2, .8, .2, 1), opacity 300ms ease, color 300ms ease; }
 .lyric-line.near-lyric { opacity: .9; transform: translateY(2px) scale(.8); }
 .lyric-line.active-lyric { color: var(--color-foreground); opacity: 1; transform: translateY(0) scale(1.18); }
+.lyric-line.active-lyric .lyric-word { color: transparent; background-image: linear-gradient(90deg, var(--color-foreground) calc(var(--lyric-fill, 0) * 100%), var(--color-foreground-subtle) 0); -webkit-background-clip: text; background-clip: text; }
+.lyric-translation { display: block; margin-top: 2px; color: var(--color-foreground-subtle); font-size: .62em; font-weight: 500; }
 .lyrics-scroll:has(.empty-state) { display: grid; place-items: center; }
 :root[data-performance='low'] .lyric-line { transition: opacity 160ms ease; }
 :root[data-performance='off'] .lyrics-scroll { mask-image: none; }

web/src/features/lyrics/lyrics.tsx

@@ -1,8 +1,11 @@
+import type { LyricWord } from './lrc'
 import { AlignLeft } from 'lucide-react'
 import { useEffect, useRef } from 'react'
+import { playerStore } from '@/features/playback'
 import { performanceStore } from '@/features/settings'
 import { useStore } from '@/shared/lib/use-store'
 import { EmptyState } from '@/shared/ui'
+import { wordProgress } from './lrc'
 import { lyricsStore } from './lyrics-store'
 import './lyrics.css'
 
@@ -39,6 +42,7 @@ export function Lyrics() {
     }
     let active = -2
     let near: Element[] = []
+    let activeWords: { node: HTMLElement, word: LyricWord, fill?: number }[] = []
     let frame: number | null = null
     const highlight = () => {
       const index = lyricsStore.state.index
@@ -51,6 +55,10 @@ export function Lyrics() {
       previous?.classList.remove('active-lyric')
       previous?.removeAttribute('aria-current')
       const current = nodes.get(index)
+      const words = lines[index]?.words
+      activeWords = current && words
+        ? Array.from(current.querySelectorAll<HTMLElement>('.lyric-word'), (node, i) => ({ node, word: words[i] }))
+        : []
       if (current) {
         current.classList.add('active-lyric')
         current.setAttribute('aria-current', 'true')
@@ -74,6 +82,18 @@ export function Lyrics() {
         node.classList.add('near-lyric')
       active = index
     }
+    const paintWords = (positionMs: number) => {
+      const smooth = performanceStore.smoothMotion()
+      for (const item of activeWords) {
+        const progress = wordProgress(item.word, positionMs)
+        // 不需要流畅动效时按字切换，每帧不用重绘。
+        const fill = smooth ? progress : Math.ceil(progress)
+        if (fill !== item.fill) {
+          item.fill = fill
+          item.node.style.setProperty('--lyric-fill', String(fill))
+        }
+      }
+    }
     const onChange = () => {
       highlight()
       frame ??= requestAnimationFrame(() => {
@@ -82,11 +102,14 @@ export function Lyrics() {
       })
     }
     highlight()
+    paintWords(playerStore.positionMs())
     centerLine(container, nodes, lyricsStore.state.index, false)
     const resizeObserver = new ResizeObserver(() => centerLine(container, nodes, lyricsStore.state.index, false))
     resizeObserver.observe(container)
     lyricsStore.addEventListener('change', onChange)
+    const stopPainting = playerStore.frames.on(paintWords)
     return () => {
+      stopPainting()
       resizeObserver.disconnect()
       lyricsStore.removeEventListener('change', onChange)
       if (frame !== null)
@@ -98,7 +121,10 @@ export function Lyrics() {
       <div className="lyrics-scroll" ref={scrollRef}>
         {lines.some(line => line.text)
           ? lines.map((line, index) => line.text && (
-              <p className="lyric-line text-ui-xl" data-lyric-index={index} key={`${line.timeMs}:${line.text}`}>{line.text}</p>
+              <p className="lyric-line text-ui-xl" data-lyric-index={index} key={`${line.timeMs}:${line.text}`}>
+                {line.words?.map(word => <span className="lyric-word" key={`${word.startMs}:${word.text}`}>{word.text}</span>) ?? line.text}
+                {line.translation && <span className="lyric-translation">{line.translation}</span>}
+              </p>
             ))
           : <EmptyState icon={AlignLeft} title="歌词随音乐浮现">播放带有歌词的歌曲时，会在这里同步显示。</EmptyState>}
       </div>