feat: recognize NetEase and QQ Music playlist links and resolve their short links
This commit is contained in:
DiheChen committed
1 parent
1b292fa5e3
commit
92a8e63240
5 files changed
+396
No files matched your search
@@ -0,0 +1,85 @@
|
||||
package shortlink
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"github.com/dihemoe/alisten/internal/app/port"
|
||||
)
|
||||
|
||||
const (
|
||||
maxRedirects = 5
|
||||
timeout = 5 * time.Second
|
||||
)
|
||||
|
||||
type Resolver struct {
|
||||
allowed func(host string) bool
|
||||
done func(link string) bool
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
var _ port.ShortLinks = (*Resolver)(nil)
|
||||
|
||||
// allowed 检查每一跳（包括起点）的主机。跳转到 done 认得的地址时就停下，不再请求它：
|
||||
// 网易云的短链接最后一跳是很慢的歌单页面，逐跳请求到底会超过总超时。
|
||||
func New(allowed func(host string) bool, done func(link string) bool) *Resolver {
|
||||
return &Resolver{
|
||||
allowed: allowed,
|
||||
done: done,
|
||||
// 跳转由 Resolve 逐跳处理，这样每一跳的主机都能先检查再发请求。
|
||||
client: &http.Client{
|
||||
CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Resolver) Resolve(ctx context.Context, link string) (string, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
|
||||
current := link
|
||||
for hop := 0; ; hop++ {
|
||||
u, err := url.Parse(current)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("解析链接: %w", err)
|
||||
}
|
||||
if u.Scheme != "http" && u.Scheme != "https" {
|
||||
return "", fmt.Errorf("不支持的协议 %q", u.Scheme)
|
||||
}
|
||||
if !r.allowed(u.Hostname()) {
|
||||
return "", fmt.Errorf("跳转到不受信任的主机 %q", u.Hostname())
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, current, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
resp, err := r.client.Do(req)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("请求短链接: %w", err)
|
||||
}
|
||||
_ = resp.Body.Close()
|
||||
|
||||
switch resp.StatusCode {
|
||||
case http.StatusMovedPermanently, http.StatusFound, http.StatusSeeOther,
|
||||
http.StatusTemporaryRedirect, http.StatusPermanentRedirect:
|
||||
default:
|
||||
return current, nil
|
||||
}
|
||||
if hop == maxRedirects {
|
||||
return "", errors.New("跳转次数过多")
|
||||
}
|
||||
next, err := resp.Location()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("读取跳转地址: %w", err)
|
||||
}
|
||||
current = next.String()
|
||||
if r.done(current) {
|
||||
return current, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
package shortlink
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
"time"
|
||||
)
|
||||
|
||||
func allowLoopback(host string) bool { return host == "127.0.0.1" }
|
||||
|
||||
func never(string) bool { return false }
|
||||
|
||||
func TestResolve(t *testing.T) {
|
||||
var methods []string
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/short", func(w http.ResponseWriter, r *http.Request) {
|
||||
methods = append(methods, r.Method)
|
||||
http.Redirect(w, r, "/mid", http.StatusFound)
|
||||
})
|
||||
mux.HandleFunc("/mid", func(w http.ResponseWriter, r *http.Request) {
|
||||
methods = append(methods, r.Method)
|
||||
http.Redirect(w, r, "/playlist?id=1", http.StatusMovedPermanently)
|
||||
})
|
||||
mux.HandleFunc("/playlist", func(w http.ResponseWriter, r *http.Request) {
|
||||
methods = append(methods, r.Method)
|
||||
_, _ = w.Write([]byte("body"))
|
||||
})
|
||||
mux.HandleFunc("/l", func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Redirect(w, r, r.URL.Path+"x", http.StatusFound)
|
||||
})
|
||||
mux.HandleFunc("/lx", func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Redirect(w, r, "/l", http.StatusFound)
|
||||
})
|
||||
mux.HandleFunc("/five", func(w http.ResponseWriter, r *http.Request) { http.Redirect(w, r, "/f2", http.StatusFound) })
|
||||
mux.HandleFunc("/f2", func(w http.ResponseWriter, r *http.Request) { http.Redirect(w, r, "/f3", http.StatusFound) })
|
||||
mux.HandleFunc("/f3", func(w http.ResponseWriter, r *http.Request) { http.Redirect(w, r, "/f4", http.StatusFound) })
|
||||
mux.HandleFunc("/f4", func(w http.ResponseWriter, r *http.Request) { http.Redirect(w, r, "/f5", http.StatusFound) })
|
||||
mux.HandleFunc("/f5", func(w http.ResponseWriter, r *http.Request) { http.Redirect(w, r, "/playlist?id=2", http.StatusFound) })
|
||||
mux.HandleFunc("/evil", func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Redirect(w, r, "http://localhost:1/x", http.StatusFound)
|
||||
})
|
||||
srv := httptest.NewServer(mux)
|
||||
defer srv.Close()
|
||||
|
||||
r := New(allowLoopback, never)
|
||||
ctx := context.Background()
|
||||
|
||||
t.Run("正常跳转且只发 GET", func(t *testing.T) {
|
||||
methods = nil
|
||||
got, err := r.Resolve(ctx, srv.URL+"/short")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if want := srv.URL + "/playlist?id=1"; got != want {
|
||||
t.Fatalf("got %q; want %q", got, want)
|
||||
}
|
||||
if strings.Join(methods, ",") != "GET,GET,GET" {
|
||||
t.Fatalf("methods = %v", methods)
|
||||
}
|
||||
})
|
||||
t.Run("跳到认得的地址时不再请求它", func(t *testing.T) {
|
||||
methods = nil
|
||||
recognized := New(allowLoopback, func(link string) bool { return strings.Contains(link, "/playlist?") })
|
||||
got, err := recognized.Resolve(ctx, srv.URL+"/short")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if want := srv.URL + "/playlist?id=1"; got != want {
|
||||
t.Fatalf("got %q; want %q", got, want)
|
||||
}
|
||||
if strings.Join(methods, ",") != "GET,GET" {
|
||||
t.Fatalf("methods = %v，期望不请求 /playlist", methods)
|
||||
}
|
||||
})
|
||||
t.Run("五次跳转可以", func(t *testing.T) {
|
||||
got, err := r.Resolve(ctx, srv.URL+"/five")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if want := srv.URL + "/playlist?id=2"; got != want {
|
||||
t.Fatalf("got %q; want %q", got, want)
|
||||
}
|
||||
})
|
||||
t.Run("超过五次", func(t *testing.T) {
|
||||
if _, err := r.Resolve(ctx, srv.URL+"/l"); err == nil {
|
||||
t.Fatal("want error")
|
||||
}
|
||||
})
|
||||
t.Run("跳到不允许的主机", func(t *testing.T) {
|
||||
if _, err := r.Resolve(ctx, srv.URL+"/evil"); err == nil {
|
||||
t.Fatal("want error")
|
||||
}
|
||||
})
|
||||
t.Run("起点不允许", func(t *testing.T) {
|
||||
if _, err := New(func(string) bool { return false }, never).Resolve(ctx, srv.URL+"/playlist"); err == nil {
|
||||
t.Fatal("want error")
|
||||
}
|
||||
})
|
||||
t.Run("非 http 协议", func(t *testing.T) {
|
||||
if _, err := r.Resolve(ctx, "file:///etc/passwd"); err == nil {
|
||||
t.Fatal("want error")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// 不响应的上游：请求一直挂着，直到请求的 ctx 结束。
|
||||
type hangingTransport struct{}
|
||||
|
||||
func (hangingTransport) RoundTrip(r *http.Request) (*http.Response, error) {
|
||||
<-r.Context().Done()
|
||||
return nil, r.Context().Err()
|
||||
}
|
||||
|
||||
func TestResolveTimesOut(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
r := New(allowLoopback, never)
|
||||
r.client.Transport = hangingTransport{}
|
||||
start := time.Now()
|
||||
if _, err := r.Resolve(t.Context(), "http://127.0.0.1/slow"); !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Errorf("上游不响应时的错误 = %v，期望超过期限", err)
|
||||
}
|
||||
if elapsed := time.Since(start); elapsed != 5*time.Second {
|
||||
t.Errorf("等了 %v 才中止，期望 5 秒", elapsed)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
package port
|
||||
|
||||
import "context"
|
||||
|
||||
type ShortLinks interface {
|
||||
// 返回跳转之后的最终地址；跳转到不受信任的主机、跳转过多或超时都返回错误。
|
||||
Resolve(ctx context.Context, link string) (string, error)
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
package playlist
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/dihemoe/alisten/internal/domain/track"
|
||||
)
|
||||
|
||||
// URL 是从文本中取出的链接；短链接要靠它去解析。
|
||||
type Link struct {
|
||||
Source track.Source
|
||||
ID string
|
||||
Short bool
|
||||
URL string
|
||||
}
|
||||
|
||||
var (
|
||||
urlPattern = regexp.MustCompile(`(?i)https?://[^\s<>"'，。；！？、）】》]+`)
|
||||
digits = regexp.MustCompile(`^[0-9]+$`)
|
||||
qqPath = regexp.MustCompile(`^/n/(?:ryqq|yqq)/playlist/([0-9]+)(?:\.html)?$`)
|
||||
)
|
||||
|
||||
// 重定向只能落在这些主机上，按小写比较。
|
||||
var redirectHosts = map[string]bool{
|
||||
"163cn.tv": true,
|
||||
"music.163.com": true,
|
||||
"y.music.163.com": true,
|
||||
"c6.y.qq.com": true,
|
||||
"y.qq.com": true,
|
||||
"i.y.qq.com": true,
|
||||
}
|
||||
|
||||
func AllowedRedirectHost(host string) bool {
|
||||
return redirectHosts[strings.ToLower(host)]
|
||||
}
|
||||
|
||||
// 用户粘贴的分享文本里夹着别的文字，所以只取第一个 http(s) 链接来识别，后面的链接不看。
|
||||
func ParseLink(raw string) (Link, bool) {
|
||||
link := strings.TrimRight(urlPattern.FindString(raw), ".,;:!?)]}>")
|
||||
if link == "" {
|
||||
return Link{}, false
|
||||
}
|
||||
u, err := url.Parse(link)
|
||||
if err != nil || u.User != nil || u.Port() != "" {
|
||||
return Link{}, false
|
||||
}
|
||||
|
||||
switch strings.ToLower(u.Hostname()) {
|
||||
case "163cn.tv":
|
||||
return Link{Source: track.NetEase, Short: true, URL: link}, true
|
||||
case "c6.y.qq.com":
|
||||
return Link{Source: track.QQ, Short: true, URL: link}, true
|
||||
case "music.163.com", "y.music.163.com":
|
||||
path, query := u.Path, u.Query()
|
||||
// 桌面版把路由放在片段里：/#/playlist?id=<id>。
|
||||
if frag, ok := strings.CutPrefix(u.Fragment, "/"); ok && path == "/" {
|
||||
p, q, _ := strings.Cut(frag, "?")
|
||||
path = "/" + p
|
||||
if query, err = url.ParseQuery(q); err != nil {
|
||||
return Link{}, false
|
||||
}
|
||||
}
|
||||
if path != "/playlist" && path != "/m/playlist" {
|
||||
return Link{}, false
|
||||
}
|
||||
if id := query.Get("id"); digits.MatchString(id) {
|
||||
return Link{Source: track.NetEase, ID: id, URL: link}, true
|
||||
}
|
||||
case "y.qq.com":
|
||||
if m := qqPath.FindStringSubmatch(u.Path); m != nil {
|
||||
return Link{Source: track.QQ, ID: m[1], URL: link}, true
|
||||
}
|
||||
case "i.y.qq.com":
|
||||
if u.Path != "/n2/m/share/details/taoge.html" {
|
||||
return Link{}, false
|
||||
}
|
||||
q := u.Query()
|
||||
for _, key := range []string{"id", "disstid"} {
|
||||
if id := q.Get(key); digits.MatchString(id) {
|
||||
return Link{Source: track.QQ, ID: id, URL: link}, true
|
||||
}
|
||||
}
|
||||
}
|
||||
return Link{}, false
|
||||
}
|
||||
|
||||
// 链接已经能直接识别出歌单，不必再跟随跳转。
|
||||
func Recognized(link string) bool {
|
||||
l, ok := ParseLink(link)
|
||||
return ok && !l.Short
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
package playlist
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/dihemoe/alisten/internal/domain/track"
|
||||
)
|
||||
|
||||
func TestParseLink(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
raw string
|
||||
want Link
|
||||
ok bool
|
||||
}{
|
||||
{"网易云哈希路由", "https://music.163.com/#/playlist?id=123", Link{Source: track.NetEase, ID: "123", URL: "https://music.163.com/#/playlist?id=123"}, true},
|
||||
{"网易云桌面", "https://music.163.com/playlist?id=123", Link{Source: track.NetEase, ID: "123", URL: "https://music.163.com/playlist?id=123"}, true},
|
||||
{"网易云移动", "https://y.music.163.com/m/playlist?id=123&userid=9&creatorId=9", Link{Source: track.NetEase, ID: "123", URL: "https://y.music.163.com/m/playlist?id=123&userid=9&creatorId=9"}, true},
|
||||
{"网易云哈希带其他参数", "https://music.163.com/#/playlist?app_version=1&id=123", Link{Source: track.NetEase, ID: "123", URL: "https://music.163.com/#/playlist?app_version=1&id=123"}, true},
|
||||
{"网易云 http", "http://music.163.com/playlist?id=123", Link{Source: track.NetEase, ID: "123", URL: "http://music.163.com/playlist?id=123"}, true},
|
||||
{"QQ 桌面", "https://y.qq.com/n/ryqq/playlist/456", Link{Source: track.QQ, ID: "456", URL: "https://y.qq.com/n/ryqq/playlist/456"}, true},
|
||||
{"QQ 旧版", "https://y.qq.com/n/yqq/playlist/456.html", Link{Source: track.QQ, ID: "456", URL: "https://y.qq.com/n/yqq/playlist/456.html"}, true},
|
||||
{"QQ 移动 id", "https://i.y.qq.com/n2/m/share/details/taoge.html?id=456&hosteuin=x", Link{Source: track.QQ, ID: "456", URL: "https://i.y.qq.com/n2/m/share/details/taoge.html?id=456&hosteuin=x"}, true},
|
||||
{"QQ 移动 disstid", "https://i.y.qq.com/n2/m/share/details/taoge.html?disstid=456", Link{Source: track.QQ, ID: "456", URL: "https://i.y.qq.com/n2/m/share/details/taoge.html?disstid=456"}, true},
|
||||
{"网易云短链接", "https://163cn.tv/abc", Link{Source: track.NetEase, Short: true, URL: "https://163cn.tv/abc"}, true},
|
||||
{"QQ 短链接", "https://c6.y.qq.com/base/fcgi-bin/u?__=abc", Link{Source: track.QQ, Short: true, URL: "https://c6.y.qq.com/base/fcgi-bin/u?__=abc"}, true},
|
||||
{"分享文本", "分享歌单：我的歌单 https://163cn.tv/abc (来自网易云音乐)", Link{Source: track.NetEase, Short: true, URL: "https://163cn.tv/abc"}, true},
|
||||
{"分享文本带句号", "看看这个 https://music.163.com/playlist?id=123。", Link{Source: track.NetEase, ID: "123", URL: "https://music.163.com/playlist?id=123"}, true},
|
||||
{"主机大小写", "HTTPS://Music.163.COM/playlist?id=123", Link{Source: track.NetEase, ID: "123", URL: "HTTPS://Music.163.COM/playlist?id=123"}, true},
|
||||
|
||||
{"其他网站", "https://example.com/playlist?id=123", Link{}, false},
|
||||
{"主机后缀伪装", "https://music.163.com.evil.com/playlist?id=123", Link{}, false},
|
||||
{"userinfo 伪装", "https://music.163.com@evil.com/playlist?id=123", Link{}, false},
|
||||
{"网易云缺少 id", "https://music.163.com/playlist", Link{}, false},
|
||||
{"网易云 id 非数字", "https://music.163.com/playlist?id=abc", Link{}, false},
|
||||
{"网易云歌曲链接", "https://music.163.com/song?id=123", Link{}, false},
|
||||
{"QQ 缺少 id", "https://y.qq.com/n/ryqq/playlist/", Link{}, false},
|
||||
{"QQ id 非数字", "https://y.qq.com/n/ryqq/playlist/abc", Link{}, false},
|
||||
{"QQ 移动缺少 id", "https://i.y.qq.com/n2/m/share/details/taoge.html", Link{}, false},
|
||||
{"QQ 歌曲链接", "https://y.qq.com/n/ryqq/songDetail/abc", Link{}, false},
|
||||
{"没有链接", "只是一段文字", Link{}, false},
|
||||
{"空", "", Link{}, false},
|
||||
{"非 http", "ftp://music.163.com/playlist?id=123", Link{}, false},
|
||||
{"只取第一个链接", "https://example.com/a https://music.163.com/playlist?id=123", Link{}, false},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, ok := ParseLink(tt.raw)
|
||||
if ok != tt.ok || got != tt.want {
|
||||
t.Fatalf("ParseLink(%q) = %+v, %v; want %+v, %v", tt.raw, got, ok, tt.want, tt.ok)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAllowedRedirectHost(t *testing.T) {
|
||||
for host, want := range map[string]bool{
|
||||
"163cn.tv": true, "music.163.com": true, "y.music.163.com": true,
|
||||
"c6.y.qq.com": true, "y.qq.com": true, "i.y.qq.com": true, "Y.QQ.com": true,
|
||||
"evil.com": false, "music.163.com.evil.com": false, "qq.com": false, "": false, "127.0.0.1": false,
|
||||
} {
|
||||
if got := AllowedRedirectHost(host); got != want {
|
||||
t.Errorf("AllowedRedirectHost(%q) = %v; want %v", host, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecognized(t *testing.T) {
|
||||
for link, want := range map[string]bool{
|
||||
"https://music.163.com/#/playlist?id=123": true,
|
||||
"https://y.qq.com/n/ryqq/playlist/7039876": true,
|
||||
"https://163cn.tv/abc": false,
|
||||
"https://example.com/playlist?id=1": false,
|
||||
} {
|
||||
if got := Recognized(link); got != want {
|
||||
t.Errorf("Recognized(%q) = %v，期望 %v", link, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user