fix: cap media file sizes in the prepare pipeline
This commit is contained in:
1 parent
41bf78d110
commit
eed1b3989b
9 files changed
+246
-21
No files matched your search
@@ -9,6 +9,7 @@ import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/dihemoe/alisten/internal/app/port"
|
||||
@@ -55,7 +56,10 @@ func (t *Transcoder) ToMP3(ctx context.Context, audio port.LinkFile) (port.LinkF
|
||||
func (t *Transcoder) transcode(ctx context.Context, audio port.LinkFile, dir string) (port.LinkFile, error) {
|
||||
// 先复制成文件再交给 ffmpeg:MP4 的 moov 可能在文件末尾,从管道读不了。文件名不带扩展名,ffmpeg 只能按内容识别格式。
|
||||
input := filepath.Join(dir, "input")
|
||||
if err := copyFile(input, audio); err != nil {
|
||||
switch err := copyFile(input, audio); {
|
||||
case errors.Is(err, port.ErrAudioTooLarge):
|
||||
return nil, &port.RejectedError{Err: err}
|
||||
case err != nil:
|
||||
return nil, fmt.Errorf("复制输入的音频:%w", err)
|
||||
}
|
||||
output := filepath.Join(dir, "audio.mp3")
|
||||
@@ -83,6 +87,16 @@ func copyFile(dst string, src port.LinkFile) error {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = in.Close() }()
|
||||
size, err := in.Seek(0, io.SeekEnd)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if size > port.MaxAudioSize {
|
||||
return port.ErrAudioTooLarge
|
||||
}
|
||||
if _, err := in.Seek(0, io.SeekStart); err != nil {
|
||||
return err
|
||||
}
|
||||
out, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -105,6 +119,8 @@ func args(input, output string) []string {
|
||||
"-map_metadata", "-1", "-map_chapters", "-1",
|
||||
// 与 yt-dlp 的 --audio-format mp3 默认质量一致。
|
||||
"-c:a", "libmp3lame", "-q:a", "5",
|
||||
// ffmpeg 写到不少于 -fs 的字节数就停止并正常退出。多一个字节,截断的输出才一定超过上限,由准备流程拒绝。
|
||||
"-fs", strconv.Itoa(port.MaxAudioSize + 1),
|
||||
output,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -252,6 +252,60 @@ func TestToMP3FailsWithoutAudio(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestToMP3RejectsInputOverLimit(t *testing.T) {
|
||||
tests := []struct {
|
||||
size int
|
||||
rejected bool
|
||||
logged int
|
||||
}{
|
||||
// 内容全是零,复制完之后 ffmpeg 认不出格式:转码失败,但不是因为超过上限。
|
||||
{port.MaxAudioSize, false, 1},
|
||||
// 超过上限时不运行 ffmpeg,也就没有转码失败的日志。
|
||||
{port.MaxAudioSize + 1, true, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
root := t.TempDir()
|
||||
logger, records := recorder()
|
||||
_, release, err := New(root, logger).ToMP3(t.Context(), porttest.LinkFile(make([]byte, tt.size)))
|
||||
if err == nil {
|
||||
release()
|
||||
t.Fatalf("%d 字节的零转码成功了", tt.size)
|
||||
}
|
||||
var rejected *port.RejectedError
|
||||
if errors.As(err, &rejected) != tt.rejected || tt.rejected && err.Error() != port.ErrAudioTooLarge.Error() {
|
||||
t.Errorf("输入 %d 字节时 = %v,期望不能准备 = %v,原因是 %q", tt.size, err, tt.rejected, port.ErrAudioTooLarge)
|
||||
}
|
||||
requireEmpty(t, root)
|
||||
if got := records(t); len(got) != tt.logged {
|
||||
t.Errorf("输入 %d 字节时的日志 = %+v,期望 %d 条", tt.size, got, tt.logged)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 把 -fs 换成很小的值,看真实的 ffmpeg 怎样处理:截断时正常退出,输出不少于 -fs 的字节数。
|
||||
func TestOutputStopsAtFileSizeLimit(t *testing.T) {
|
||||
input := generate(t, "long.m4a", "-f", "lavfi", "-i", "sine=frequency=440:duration=10", "-c:a", "aac")
|
||||
output := filepath.Join(t.TempDir(), "audio.mp3")
|
||||
argv := args(input, output)
|
||||
i := slices.Index(argv, "-fs")
|
||||
if i < 0 || argv[i+1] != strconv.Itoa(port.MaxAudioSize+1) {
|
||||
t.Fatalf("参数 = %q,期望 -fs %d", argv, port.MaxAudioSize+1)
|
||||
}
|
||||
// 10 秒的 mp3 大约 40 KB。
|
||||
const limit = 10000
|
||||
argv[i+1] = strconv.Itoa(limit)
|
||||
if stderr, err := run(t.Context(), argv); err != nil {
|
||||
t.Fatalf("ffmpeg 截断输出时失败:%v\n%s", err, stderr)
|
||||
}
|
||||
info, err := os.Stat(output)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if info.Size() < limit || info.Size() > 2*limit {
|
||||
t.Errorf("截断的输出有 %d 字节,期望不少于 -fs 的 %d 字节,并且在它附近截断", info.Size(), limit)
|
||||
}
|
||||
}
|
||||
|
||||
// 握手完成的连接留在 accept 队列里,不必先 Accept 也不会漏掉。
|
||||
func listenLocal(t *testing.T) (address string, connected func() bool) {
|
||||
t.Helper()
|
||||
|
||||
@@ -17,6 +17,7 @@ import (
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
@@ -31,12 +32,14 @@ const (
|
||||
waitDelay = 5 * time.Second
|
||||
outputLimit = 64 << 10
|
||||
reasonLimit = 300
|
||||
metaFile = "meta.json"
|
||||
)
|
||||
|
||||
// 下载失败的原因,用作指标的标签。
|
||||
const (
|
||||
failureUnsupported = "unsupported"
|
||||
failureBlocked = "blocked"
|
||||
failureTooLarge = "too_large"
|
||||
failureTimeout = "timeout"
|
||||
failureError = "error"
|
||||
)
|
||||
@@ -127,17 +130,22 @@ func args(link, dir, proxy, cookies string) []string {
|
||||
"--postprocessor-args", "FixupM4a+ffmpeg_i:-protocol_whitelist file",
|
||||
// 来源文件里的标签是用户提交的内容,转码时 ffmpeg 默认把它们复制成 mp3 的标签,所以不复制。
|
||||
// 来源已经是 mp3 时 yt-dlp 不转码,文件连同标签原样保留,存储的音频仍然不可信。
|
||||
"--postprocessor-args", "ExtractAudio+ffmpeg_o:-map_metadata -1",
|
||||
// 转出的 mp3 写到上限多一个字节就停止,截断的文件由准备流程拒绝。
|
||||
"--postprocessor-args", "ExtractAudio+ffmpeg_o:-map_metadata -1 -fs " + strconv.Itoa(port.MaxAudioSize+1),
|
||||
// --no-playlist 只对同时指向视频和播放列表的链接有效;对播放列表本身,--playlist-items 1 只下载第一项。
|
||||
"--no-playlist", "--playlist-items", "1",
|
||||
"--format", "bestaudio/best",
|
||||
// 只在服务端给出 Content-Length 时生效;准备流程上传之前还会检查大小。
|
||||
"--max-filesize", strconv.Itoa(port.MaxAudioSize),
|
||||
"--extract-audio", "--audio-format", "mp3",
|
||||
"--write-thumbnail",
|
||||
"--paths", dir,
|
||||
"--output", "audio.%(ext)s",
|
||||
"--output", "thumbnail:cover.%(ext)s",
|
||||
"--no-progress",
|
||||
"--print", "after_move:%(.{title,artist,uploader,album,duration})j",
|
||||
// 屏幕输出里有页面可控的标题,元数据混在其中无法可靠地取出,所以写进单独的文件。不用 --print 也是因为它带上 --quiet,
|
||||
// --max-filesize 中止下载时的提示就不会输出。
|
||||
"--print-to-file", "after_move:%(.{title,artist,uploader,album,duration})j", metaFile,
|
||||
// 链接以 - 开头时会被当作选项。
|
||||
"--", link,
|
||||
}
|
||||
@@ -217,10 +225,18 @@ var errNoAudio = errors.New("yt-dlp 没有下载到音频")
|
||||
func collect(dir string, stdout []byte) (port.LinkDownload, error) {
|
||||
audio := filepath.Join(dir, "audio.mp3")
|
||||
if _, err := os.Stat(audio); err != nil {
|
||||
// 服务端给出的长度超过 --max-filesize 时,yt-dlp 只输出这条提示就结束,退出码为 0。
|
||||
if bytes.Contains(stdout, []byte("File is larger than max-filesize")) {
|
||||
return port.LinkDownload{}, port.ErrAudioTooLarge
|
||||
}
|
||||
return port.LinkDownload{}, errNoAudio
|
||||
}
|
||||
meta, err := os.ReadFile(filepath.Join(dir, metaFile))
|
||||
if err != nil {
|
||||
return port.LinkDownload{}, fmt.Errorf("读取 yt-dlp 输出的元数据:%w", err)
|
||||
}
|
||||
var i info
|
||||
if err := json.Unmarshal(bytes.TrimSpace(stdout), &i); err != nil {
|
||||
if err := json.Unmarshal(bytes.TrimSpace(meta), &i); err != nil {
|
||||
return port.LinkDownload{}, fmt.Errorf("解析 yt-dlp 输出的元数据:%w", err)
|
||||
}
|
||||
d := port.LinkDownload{
|
||||
@@ -254,6 +270,8 @@ func classify(ctx context.Context, err error, stderr []byte, refused bool) (stri
|
||||
return failureTimeout, fmt.Errorf("yt-dlp 下载超时:%w", ctx.Err())
|
||||
case refused:
|
||||
return failureBlocked, &port.RejectedError{Err: errors.New("链接指向的地址不是公网地址,不能下载")}
|
||||
case errors.Is(err, port.ErrAudioTooLarge):
|
||||
return failureTooLarge, &port.RejectedError{Err: port.ErrAudioTooLarge}
|
||||
case errors.Is(err, errNoAudio) || bytes.Contains(stderr, []byte("ERROR: Unsupported URL: ")):
|
||||
return failureUnsupported, &port.RejectedError{Err: errors.New("yt-dlp 不支持这个链接")}
|
||||
}
|
||||
|
||||
@@ -95,8 +95,8 @@ func TestYtdlpDownloadsAudioAndCover(t *testing.T) {
|
||||
if requests.Load() < 3 {
|
||||
t.Errorf("网站收到 %d 个请求,期望网页、音频和封面都经代理请求", requests.Load())
|
||||
}
|
||||
if entries, _ := os.ReadDir(dir); len(entries) != 2 {
|
||||
t.Errorf("目录中有 %d 个文件,期望只有音频和封面", len(entries))
|
||||
if entries, _ := os.ReadDir(dir); len(entries) != 3 {
|
||||
t.Errorf("目录中有 %d 个文件,期望只有音频、封面和元数据", len(entries))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -131,6 +131,41 @@ func TestYtdlpRejectsUnsupportedLink(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
type zeros struct{}
|
||||
|
||||
func (zeros) ReadAt(p []byte, _ int64) (int, error) {
|
||||
clear(p)
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
// 音频的内容全是零:yt-dlp 如果没有在看到长度时中止,就会下载完再转码失败。
|
||||
func TestYtdlpRejectsAudioOverLimit(t *testing.T) {
|
||||
link, _ := startPaths(t, map[string]http.Handler{
|
||||
"/page": http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
_, _ = io.WriteString(w, `<!doctype html><html><head><title>太长</title></head><body><audio src="/big.wav"></audio></body></html>`)
|
||||
}),
|
||||
"/big.wav": http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "audio/wav")
|
||||
http.ServeContent(w, r, "big.wav", time.Time{}, io.NewSectionReader(zeros{}, 0, port.MaxAudioSize+1))
|
||||
}),
|
||||
})
|
||||
p := siteProxy(t)
|
||||
dir := t.TempDir()
|
||||
stdout, stderr, err := run(t.Context(), dir, args(link("/page"), dir, p.url, ""))
|
||||
if err == nil {
|
||||
_, err = collect(dir, stdout)
|
||||
}
|
||||
failure, err := classify(t.Context(), err, stderr, p.wasRefused())
|
||||
var rejected *port.RejectedError
|
||||
if failure != failureTooLarge || !errors.As(err, &rejected) || err.Error() != port.ErrAudioTooLarge.Error() {
|
||||
t.Errorf("音频超过上限时 = %q, %v,期望不能准备,原因是 %q\nstdout:\n%s\nstderr:\n%s", failure, err, port.ErrAudioTooLarge, stdout, stderr)
|
||||
}
|
||||
if entries, _ := os.ReadDir(dir); len(entries) != 0 {
|
||||
t.Errorf("目录中有 %v,期望 yt-dlp 什么都没写", entries)
|
||||
}
|
||||
}
|
||||
|
||||
type observed struct {
|
||||
mu sync.Mutex
|
||||
failures []string
|
||||
@@ -175,7 +210,7 @@ func fakeYtdlp(t *testing.T, script string) {
|
||||
}
|
||||
|
||||
func TestDownloadKeepsFilesUntilReleased(t *testing.T) {
|
||||
fakeYtdlp(t, "#!/bin/sh\nprintf mp3 > audio.mp3\necho '{}'\n")
|
||||
fakeYtdlp(t, "#!/bin/sh\nprintf mp3 > audio.mp3\necho '{}' > meta.json\n")
|
||||
root := t.TempDir()
|
||||
d, release, err := New(root, net.DefaultResolver, noCookies{}, func(time.Duration, string) {}, slog.New(slog.DiscardHandler)).Download(t.Context(), "https://example.com/page")
|
||||
if err != nil {
|
||||
@@ -283,6 +318,14 @@ func TestArgsEndWithLink(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// 转出超限的 mp3 要几百 MB 的输入,测不起;ffmpeg 遇到 -fs 时怎样截断由 ffmpeg 适配器的测试确认。
|
||||
func TestArgsCapExtractedAudio(t *testing.T) {
|
||||
want := "ExtractAudio+ffmpeg_o:-map_metadata -1 -fs " + strconv.Itoa(port.MaxAudioSize+1)
|
||||
if got := args("http://example.com/a", "/tmp/d", "http://127.0.0.1:1", ""); !slices.Contains(got, want) {
|
||||
t.Errorf("参数 = %q,期望转码时限制 mp3 的大小:%q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassify(t *testing.T) {
|
||||
exited := errors.New("exit status 1")
|
||||
expired, cancel := context.WithDeadline(context.Background(), time.Now().Add(-time.Second))
|
||||
@@ -302,6 +345,7 @@ func TestClassify(t *testing.T) {
|
||||
{t.Context(), exited, "ERROR: [generic] a: HTTP Error 403: Forbidden\n", true, failureBlocked, true, "链接指向的地址不是公网地址,不能下载"},
|
||||
{t.Context(), exited, "WARNING: x\nERROR: Unsupported URL: http://example.com/\n", false, failureUnsupported, true, "yt-dlp 不支持这个链接"},
|
||||
{t.Context(), errNoAudio, "", false, failureUnsupported, true, "yt-dlp 不支持这个链接"},
|
||||
{t.Context(), port.ErrAudioTooLarge, "", false, failureTooLarge, true, port.ErrAudioTooLarge.Error()},
|
||||
{t.Context(), exited, "ERROR: 第一条\nWARNING: x\nERROR: [youtube] x: Video unavailable\n", false, failureError, false, "yt-dlp 下载失败:[youtube] x: Video unavailable"},
|
||||
{t.Context(), exited, "ERROR: " + long, false, failureError, false, "yt-dlp 下载失败:" + strings.Repeat("长", reasonLimit) + "…"},
|
||||
{t.Context(), exited, "", false, failureError, false, "yt-dlp 下载失败:exit status 1"},
|
||||
@@ -317,21 +361,31 @@ func TestClassify(t *testing.T) {
|
||||
|
||||
func TestCollect(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
if _, err := collect(dir, []byte(`{"title": "晴天"}`)); !errors.Is(err, errNoAudio) {
|
||||
t.Errorf("没有音频时 = %v", err)
|
||||
}
|
||||
for _, name := range []string{"audio.mp3", "cover.WEBP"} {
|
||||
if err := os.WriteFile(filepath.Join(dir, name), []byte("x"), 0o600); err != nil {
|
||||
writeFile := func(name, content string) {
|
||||
t.Helper()
|
||||
if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
d, err := collect(dir, []byte("{\"title\": \"晴天\", \"uploader\": \"周杰伦\", \"album\": \"叶惠美\", \"duration\": 269.5}\n"))
|
||||
writeFile(metaFile, `{"title": "晴天"}`)
|
||||
if _, err := collect(dir, nil); !errors.Is(err, errNoAudio) {
|
||||
t.Errorf("没有音频时 = %v", err)
|
||||
}
|
||||
if _, err := collect(dir, []byte("[info] x: Downloading 1 format(s): 0\n\r[download] File is larger than max-filesize (209715201 bytes > 209715200 bytes). Aborting.\n")); !errors.Is(err, port.ErrAudioTooLarge) {
|
||||
t.Errorf("因为音频超过上限没有下载时 = %v", err)
|
||||
}
|
||||
writeFile("audio.mp3", "x")
|
||||
writeFile("cover.WEBP", "x")
|
||||
writeFile(metaFile, "{\"title\": \"晴天\", \"uploader\": \"周杰伦\", \"album\": \"叶惠美\", \"duration\": 269.5}\n")
|
||||
// 屏幕输出里页面可控的标题可能伪造出 JSON 行,元数据只从文件读取。
|
||||
d, err := collect(dir, []byte("[download] Downloading playlist: x\n{\"title\": \"伪造\"}\n"))
|
||||
want := port.LinkDownload{Audio: file(filepath.Join(dir, "audio.mp3")), Cover: file(filepath.Join(dir, "cover.WEBP")), CoverExt: "webp"}
|
||||
want.Meta.Title, want.Meta.Artist, want.Meta.Album, want.Meta.Duration = "晴天", "周杰伦", "叶惠美", 269500*time.Millisecond
|
||||
if err != nil || d != want {
|
||||
t.Errorf("collect = %+v, %v\n期望 %+v", d, err, want)
|
||||
}
|
||||
if d, _ := collect(dir, []byte(`{"title": "晴天", "artist": "周杰伦、方文山", "uploader": "JVR Music"}`)); d.Meta.Artist != "周杰伦、方文山" {
|
||||
writeFile(metaFile, `{"title": "晴天", "artist": "周杰伦、方文山", "uploader": "JVR Music"}`)
|
||||
if d, _ := collect(dir, nil); d.Meta.Artist != "周杰伦、方文山" {
|
||||
t.Errorf("有歌手时 Artist = %q,期望取歌手而不是上传者", d.Meta.Artist)
|
||||
}
|
||||
}
|
||||
@@ -414,7 +468,7 @@ func recordedArgs(t *testing.T, root string) []string {
|
||||
}
|
||||
|
||||
func TestDownloadPassesCookiesFile(t *testing.T) {
|
||||
fakeYtdlp(t, "#!/bin/sh\nprintf '%s\\n' \"$@\" > args.txt\nprintf mp3 > audio.mp3\necho '{}'\n")
|
||||
fakeYtdlp(t, "#!/bin/sh\nprintf '%s\\n' \"$@\" > args.txt\nprintf mp3 > audio.mp3\necho '{}' > meta.json\n")
|
||||
root := t.TempDir()
|
||||
cookies := &fakeCookies{path: filepath.Join(t.TempDir(), "cookies.txt"), ok: true}
|
||||
_, release, err := New(root, net.DefaultResolver, cookies, func(time.Duration, string) {}, slog.New(slog.DiscardHandler)).Download(t.Context(), "https://www.youtube.com/watch?v=dQw4w9WgXcQ")
|
||||
@@ -435,7 +489,7 @@ func TestDownloadPassesCookiesFile(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestDownloadOmitsCookiesWhenNotOffered(t *testing.T) {
|
||||
fakeYtdlp(t, "#!/bin/sh\nprintf '%s\\n' \"$@\" > args.txt\nprintf mp3 > audio.mp3\necho '{}'\n")
|
||||
fakeYtdlp(t, "#!/bin/sh\nprintf '%s\\n' \"$@\" > args.txt\nprintf mp3 > audio.mp3\necho '{}' > meta.json\n")
|
||||
root := t.TempDir()
|
||||
cookies := &fakeCookies{}
|
||||
_, release, err := New(root, net.DefaultResolver, cookies, func(time.Duration, string) {}, slog.New(slog.DiscardHandler)).Download(t.Context(), "https://example.com/page")
|
||||
@@ -467,7 +521,7 @@ func TestDownloadReleasesCookiesOnFailure(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestDownloadFailsWhenCookiesUnavailable(t *testing.T) {
|
||||
fakeYtdlp(t, "#!/bin/sh\nprintf '%s\\n' \"$@\" > args.txt\nprintf mp3 > audio.mp3\necho '{}'\n")
|
||||
fakeYtdlp(t, "#!/bin/sh\nprintf '%s\\n' \"$@\" > args.txt\nprintf mp3 > audio.mp3\necho '{}' > meta.json\n")
|
||||
root := t.TempDir()
|
||||
cookies := &fakeCookies{err: errors.New("数据库不可用")}
|
||||
_, _, err := New(root, net.DefaultResolver, cookies, func(time.Duration, string) {}, slog.New(slog.DiscardHandler)).Download(t.Context(), "https://www.youtube.com/watch?v=dQw4w9WgXcQ")
|
||||
|
||||
@@ -9,7 +9,8 @@ import (
|
||||
type BilibiliSource interface {
|
||||
// CoverURL 和 PageURL 留空:平台的图片有防盗链,封面在准备时下载存储;页面地址由调用方按 ID 拼出。
|
||||
TrackMeta(ctx context.Context, id string) (track.Meta, error)
|
||||
// 读完文件之后调用 release 删除它们;返回错误时没有要删除的文件。视频不存在、需要付费或地区受限时返回 *RejectedError。
|
||||
// 读完文件之后调用 release 删除它们;返回错误时没有要删除的文件。视频不存在、需要付费、地区受限,
|
||||
// 或者下载时发现音频超过 MaxAudioSize 时返回 *RejectedError。
|
||||
Download(ctx context.Context, id string) (d BilibiliDownload, release func(), err error)
|
||||
}
|
||||
|
||||
@@ -35,7 +36,7 @@ type BilibiliQRLoginSource interface {
|
||||
CredentialStatus(context.Context) (CredentialStatus, error)
|
||||
}
|
||||
|
||||
// 只读本地文件,不访问网络。读完 mp3 之后调用 release 删除它;返回错误时没有要删除的文件。
|
||||
// 只读本地文件,不访问网络。读完 mp3 之后调用 release 删除它;返回错误时没有要删除的文件。输入超过 MaxAudioSize 时返回 *RejectedError。
|
||||
type Transcoder interface {
|
||||
ToMP3(ctx context.Context, audio LinkFile) (mp3 LinkFile, release func(), err error)
|
||||
}
|
||||
@@ -2,6 +2,7 @@ package port
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/dihemoe/alisten/internal/domain/track"
|
||||
@@ -22,8 +23,19 @@ type LinkFile interface {
|
||||
Open() (io.ReadSeekCloser, error)
|
||||
}
|
||||
|
||||
// 准备流程在上传之前拒绝超过 MaxAudioSize 的音频,把超过 MaxCoverSize 的封面当作没有封面。
|
||||
// 下载和转码也要在发现音频超过上限时尽早中止,免得超限的文件先占满临时目录。
|
||||
const (
|
||||
MaxAudioSize = 200 << 20
|
||||
MaxCoverSize = 10 << 20
|
||||
)
|
||||
|
||||
// 音频超过 MaxAudioSize 时用 &RejectedError{Err: ErrAudioTooLarge} 拒绝。
|
||||
var ErrAudioTooLarge = fmt.Errorf("音频文件超过 %d MB", MaxAudioSize>>20)
|
||||
|
||||
type LinkDownloader interface {
|
||||
// 读完文件之后调用 release 删除它们;返回错误时没有要删除的文件。链接不受支持或指向非公网地址时返回 *RejectedError。
|
||||
// 读完文件之后调用 release 删除它们;返回错误时没有要删除的文件。链接不受支持、指向非公网地址,
|
||||
// 或者下载时发现音频超过 MaxAudioSize 时返回 *RejectedError。
|
||||
Download(ctx context.Context, link string) (d LinkDownload, release func(), err error)
|
||||
}
|
||||
|
||||
|
||||
@@ -41,6 +41,9 @@ func (r *MediaRunner) Run(ctx context.Context, t track.Track) (err error) {
|
||||
return err
|
||||
}
|
||||
defer release()
|
||||
if d, err = limitSizes(d); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
key := media.Key(t.ID)
|
||||
if err := r.clear(ctx, key); err != nil {
|
||||
@@ -112,6 +115,36 @@ func (r *MediaRunner) downloadBilibili(ctx context.Context, id string) (port.Lin
|
||||
}, nil
|
||||
}
|
||||
|
||||
// 下载和转码只能尽量限制文件的大小,上传之前要再检查一次。封面不影响播放,超过上限就当作没有封面,不让整首歌准备失败。
|
||||
func limitSizes(d port.LinkDownload) (port.LinkDownload, error) {
|
||||
size, err := fileSize(d.Audio)
|
||||
if err != nil {
|
||||
return d, err
|
||||
}
|
||||
if size > port.MaxAudioSize {
|
||||
return d, &port.RejectedError{Err: port.ErrAudioTooLarge}
|
||||
}
|
||||
if d.Cover == nil {
|
||||
return d, nil
|
||||
}
|
||||
if size, err = fileSize(d.Cover); err != nil {
|
||||
return d, err
|
||||
}
|
||||
if size > port.MaxCoverSize {
|
||||
d.Cover, d.CoverExt = nil, ""
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
|
||||
func fileSize(file port.LinkFile) (int64, error) {
|
||||
f, err := file.Open()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer func() { _ = f.Close() }()
|
||||
return f.Seek(0, io.SeekEnd)
|
||||
}
|
||||
|
||||
func pageURL(t track.Track) string {
|
||||
if t.Source == track.Bilibili {
|
||||
return track.BilibiliPageURL(t.ID)
|
||||
|
||||
@@ -123,6 +123,43 @@ func TestLinkRunnerSkipsCoverOfOtherFormats(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// 元数据里有时长:没有检查大小时,内容全是零的音频会按元数据的时长准备成功,而不是因为无法分析而失败。
|
||||
func TestLinkRunnerRejectsAudioOverLimit(t *testing.T) {
|
||||
d := &downloader{meta: track.Meta{Title: "晴天", Duration: 3 * time.Second}, audio: make([]byte, port.MaxAudioSize+1), cover: []byte("jpeg"), coverExt: "jpg"}
|
||||
store := porttest.NewMediaStore()
|
||||
tracks, err := runLink(t, d, store, link)
|
||||
var rejected *port.RejectedError
|
||||
if !errors.As(err, &rejected) || err.Error() != port.ErrAudioTooLarge.Error() {
|
||||
t.Errorf("音频超过上限时 = %v,期望不能准备,原因是 %q", err, port.ErrAudioTooLarge)
|
||||
}
|
||||
if keys := store.Keys(); len(keys) != 0 {
|
||||
t.Errorf("上传了 %q,期望什么都不上传", keys)
|
||||
}
|
||||
if _, ok := tracks.row(link); ok {
|
||||
t.Error("不能准备的歌不应写入曲目表")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLinkRunnerDropsCoverOverLimit(t *testing.T) {
|
||||
cover := media.CoverObject(media.Key(link.ID), "jpg")
|
||||
for _, tt := range []struct {
|
||||
size int
|
||||
kept bool
|
||||
}{{port.MaxCoverSize, true}, {port.MaxCoverSize + 1, false}} {
|
||||
d := &downloader{meta: track.Meta{Title: "晴天"}, audio: porttest.SilentWAV(time.Second), cover: make([]byte, tt.size), coverExt: "jpg"}
|
||||
store := porttest.NewMediaStore()
|
||||
tracks, err := runLink(t, d, store, link)
|
||||
if err != nil {
|
||||
t.Fatalf("封面 %d 字节时 = %v,期望准备成功", tt.size, err)
|
||||
}
|
||||
_, uploaded := store.Object(cover)
|
||||
got, _ := tracks.row(link)
|
||||
if uploaded != tt.kept || (got.Meta.CoverKey != "") != tt.kept {
|
||||
t.Errorf("封面 %d 字节时上传了封面 = %v、曲目表中的封面 = %q,期望保留封面 = %v", tt.size, uploaded, got.Meta.CoverKey, tt.kept)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLinkRunnerFallsBackWhenAudioCannotBeAnalyzed(t *testing.T) {
|
||||
d := &downloader{meta: track.Meta{Title: "晴天", Duration: 3 * time.Second}, audio: []byte("不是音频")}
|
||||
tracks, err := runLink(t, d, porttest.NewMediaStore(), link)
|
||||
|
||||
@@ -101,7 +101,7 @@ func newWorkerMetrics(reg prometheus.Registerer) *workerMetrics {
|
||||
}),
|
||||
downloadFailures: prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "alisten_ytdlp_download_failures_total",
|
||||
Help: "yt-dlp 下载失败的次数,reason 为 unsupported(链接不受支持)、blocked(指向非公网地址)、timeout(超过任务的期限)或 error(其他错误)。",
|
||||
Help: "yt-dlp 下载失败的次数,reason 为 unsupported(链接不受支持)、blocked(指向非公网地址)、too_large(音频超过大小上限)、timeout(超过任务的期限)或 error(其他错误)。",
|
||||
}, []string{"reason"}),
|
||||
waitingJobs: prometheus.NewGauge(prometheus.GaugeOpts{
|
||||
Name: "alisten_ytdlp_jobs_waiting",
|
||||
|
||||
Reference in new issue
Block a user