bilibili: fix downloads, add b23.tv and bilibili.tv support (#354)

src/modules/processing/match.js

@@ -56,9 +56,7 @@ export default async function(host, patternMatch, url, lang, obj) {
                 });
                 break;
             case "bilibili":
-                r = await bilibili({
+                r = await bilibili(patternMatch);
-                    id: patternMatch.id.slice(0, 12)
-                });
                 break;
             case "youtube":
                 let fetchInfo = {

src/modules/processing/services/bilibili.js

@@ -1,27 +1,105 @@
 import { genericUserAgent, maxVideoDuration } from "../../config.js";
 
-// TO-DO: quality picking, bilibili.tv support, and higher quality downloads (currently requires an account)
+// TO-DO: higher quality downloads (currently requires an account)
-export default async function(obj) {
+
-    let html = await fetch(`https://bilibili.com/video/${obj.id}`, {
+function com_resolveShortlink(shortId) {
+    return fetch(`https://b23.tv/${shortId}`, { redirect: 'manual' })
+            .then(r => r.status > 300 && r.status < 400 && r.headers.get('location'))
+            .then(url => {
+                if (!url) return;
+                const path = new URL(url).pathname;
+                if (path.startsWith('/video/'))
+                    return path.split('/')[2];
+            })
+            .catch(() => {})
+}
+
+function getBest(content) {
+    return content?.filter(v => v.baseUrl || v.url)
+                .map(v => (v.baseUrl = v.baseUrl || v.url, v))
+                .reduce((a, b) => a?.bandwidth > b?.bandwidth ? a : b);
+}
+
+function extractBestQuality(dashData) {
+    const bestVideo = getBest(dashData.video),
+          bestAudio = getBest(dashData.audio);
+
+    if (!bestVideo || !bestAudio) return [];
+    return [ bestVideo, bestAudio ];
+}
+
+async function com_download(id) {
+    let html = await fetch(`https://bilibili.com/video/${id}`, {
         headers: { "user-agent": genericUserAgent }
     }).then((r) => { return r.text() }).catch(() => { return false });
     if (!html) return { error: 'ErrorCouldntFetch' };
-    if (!(html.includes('<script>window.__playinfo__=') && html.includes('"video_codecid"'))) return { error: 'ErrorEmptyDownload' };
+
+    if (!(html.includes('<script>window.__playinfo__=') && html.includes('"video_codecid"'))) {
+        return { error: 'ErrorEmptyDownload' };
+    }
 
     let streamData = JSON.parse(html.split('<script>window.__playinfo__=')[1].split('</script>')[0]);
-    if (streamData.data.timelength > maxVideoDuration) return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] };
+    if (streamData.data.timelength > maxVideoDuration) {
+        return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] };
+    }
 
-    let video = streamData["data"]["dash"]["video"].filter(v => 
+    const [ video, audio ] = extractBestQuality(streamData.data.dash);
-        !v["baseUrl"].includes("https://upos-sz-mirrorcosov.bilivideo.com/")
+    if (!video || !audio) {
-    ).sort((a, b) => Number(b.bandwidth) - Number(a.bandwidth));
+        return { error: 'ErrorEmptyDownload' };
-
+    }
-    let audio = streamData["data"]["dash"]["audio"].filter(a => 
-        !a["baseUrl"].includes("https://upos-sz-mirrorcosov.bilivideo.com/")
-    ).sort((a, b) => Number(b.bandwidth) - Number(a.bandwidth));
 
     return {
-        urls: [video[0]["baseUrl"], audio[0]["baseUrl"]],
+        urls: [video.baseUrl, audio.baseUrl],
-        audioFilename: `bilibili_${obj.id}_audio`,
+        audioFilename: `bilibili_${id}_audio`,
-        filename: `bilibili_${obj.id}_${video[0]["width"]}x${video[0]["height"]}.mp4`
+        filename: `bilibili_${id}_${video.width}x${video.height}.mp4`
     };
 }
+
+async function tv_download(id) {
+    const url = new URL(
+        'https://api.bilibili.tv/intl/gateway/web/playurl'
+        + '?s_locale=en_US&platform=web&qn=64&type=0&device=wap'
+        + '&tf=0&spm_id=bstar-web.ugc-video-detail.0.0&from_spm_id='
+    );
+
+    url.searchParams.set('aid', id);
+
+    const { data } = await fetch(url).then(a => a.json());
+    if (!data?.playurl?.video) {
+        return { error: 'ErrorEmptyDownload' };
+    }
+
+    const [ video, audio ] = extractBestQuality({
+        video: data.playurl.video.map(s => s.video_resource)
+                                 .filter(s => s.codecs.includes('avc1')),
+        audio: data.playurl.audio_resource
+    });
+
+    if (!video || !audio) {
+        return { error: 'ErrorEmptyDownload' };
+    }
+
+    if (video.duration > maxVideoDuration) {
+        return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] };
+    }
+
+    return {
+        urls: [video.url, audio.url],
+        audioFilename: `bilibili_tv_${id}_audio`,
+        filename: `bilibili_tv_${id}.mp4`
+    };
+}
+
+export default async function({ comId, tvId, comShortLink }) {
+    if (comShortLink) {
+        comId = await com_resolveShortlink(comShortLink);
+    }
+
+    if (comId) {
+        return com_download(comId);
+    } else if (tvId) {
+        return tv_download(tvId);
+    }
+
+    return { error: 'ErrorCouldntFetch' };
+}

src/modules/processing/servicesConfig.json

@@ -2,8 +2,11 @@
     "audioIgnore": ["vk", "ok"],
     "config": {
         "bilibili": {
-            "alias": "bilibili.com videos",
+            "alias": "bilibili videos",
-            "patterns": ["video/:id"],
+            "patterns": [
+                "video/:comId", "_shortLink/:comShortLink",
+                "_tv/:lang/video/:tvId", "_tv/video/:tvId"
+            ],
             "enabled": true
         },
         "reddit": {

src/modules/processing/servicesPatternTesters.js

@@ -1,6 +1,7 @@
 export const testers = {
-    "bilibili": (patternMatch) =>
+    "bilibili": (patternMatch) => 
-        patternMatch.id?.length <= 12,
+        patternMatch.comId?.length <= 12 || patternMatch.comShortLink?.length <= 16
+        || patternMatch.tvId?.length <= 24,
 
     "instagram": (patternMatch) =>
         patternMatch.postId?.length <= 12

src/modules/processing/url.js

@@ -16,6 +16,7 @@ export function aliasURL(url) {
                 url.search = `?v=${encodeURIComponent(parts[2])}`
             }
             break;
+
         case "youtu":
             if (url.hostname === 'youtu.be' && parts.length >= 2) {
                 /* youtu.be urls can be weird, e.g. https://youtu.be/<id>//asdasd// still works
@@ -25,6 +26,7 @@ export function aliasURL(url) {
                 }`)
             }
             break;
+
         case "pin":
             if (url.hostname === 'pin.it' && parts.length === 2) {
                 url = new URL(`https://pinterest.com/url_shortener/${
@@ -46,6 +48,17 @@ export function aliasURL(url) {
                 url = new URL(`https://twitch.tv/_/clip/${parts[1]}`);
             }
             break;
+
+        case "bilibili":
+            if (host.tld === 'tv') {
+                url = new URL(`https://bilibili.com/_tv${url.pathname}`);
+            }
+            break;
+        case "b23":
+            if (url.hostname === 'b23.tv' && parts.length === 2) {
+                url = new URL(`https://bilibili.com/_shortLink/${parts[1]}`)
+            }
+            break;
     }
 
     return url

src/modules/stream/types.js

@@ -80,17 +80,33 @@ export async function streamLiveRender(streamInfo, res) {
         if (streamInfo.urls.length !== 2) return shutdown();
 
         const { body: audio } = await request(streamInfo.urls[1], {
-            maxRedirections: 16, signal: abortController.signal
+            maxRedirections: 16, signal: abortController.signal,
+            headers: {
+                'user-agent': genericUserAgent,
+                referer: streamInfo.service === 'bilibili'
+                            ? 'https://www.bilibili.com/'
+                            : undefined,
+            }
         });
 
-        let format = streamInfo.filename.split('.')[streamInfo.filename.split('.').length - 1],
+        const format = streamInfo.filename.split('.')[streamInfo.filename.split('.').length - 1];
-        args = [
+        let args = [
             '-loglevel', '-8',
+            '-user_agent', genericUserAgent
+        ];
+
+        if (streamInfo.service === 'bilibili') {
+            args.push(
+                '-headers', 'Referer: https://www.bilibili.com/\r\n',
+            )
+        }
+
+        args.push(
             '-i', streamInfo.urls[0],
             '-i', 'pipe:3',
             '-map', '0:v',
             '-map', '1:a',
-        ];
+        );
 
         args = args.concat(ffmpegArgs[format]);
         if (streamInfo.metadata) {
@@ -129,11 +145,16 @@ export function streamAudioOnly(streamInfo, res) {
 
     try {
         let args = [
-            '-loglevel', '-8'
+            '-loglevel', '-8',
-        ]
+            '-user_agent', genericUserAgent
+        ];
+
         if (streamInfo.service === "twitter") {
-            args.push('-seekable', '0')
+            args.push('-seekable', '0');
+        } else if (streamInfo.service === 'bilibili') {
+            args.push('-headers', 'Referer: https://www.bilibili.com/\r\n');
         }
+
         args.push(
             '-i', streamInfo.urls,
             '-vn'
@@ -178,17 +199,23 @@ export function streamVideoOnly(streamInfo, res) {
         let args = [
             '-loglevel', '-8'
         ]
+
         if (streamInfo.service === "twitter") {
             args.push('-seekable', '0')
+        } else if (streamInfo.service === 'bilibili') {
+            args.push('-headers', 'Referer: https://www.bilibili.com/\r\n')
         }
+
         args.push(
             '-i', streamInfo.urls,
             '-c', 'copy'
         )
+
         if (streamInfo.mute) {
             args.push('-an')
         }
-        if (streamInfo.service === "vimeo" || streamInfo.service === "rutube") {
+
+        if (['vimeo', 'rutube'].includes(streamInfo.service)) {
             args.push('-bsf:a', 'aac_adtstoasc')
         }
 

src/test/tests.json

@@ -746,6 +746,23 @@
             "code": 200,
             "status": "stream"
         }
+    }, {
+        "name": "b23.tv shortlink",
+        "url": "https://b23.tv/lbMyOI9",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "stream"
+        }
+    },
+    {
+        "name": "bilibili.tv link",
+        "url": "https://www.bilibili.tv/en/video/4789599404426256",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "stream"
+        }
     }],
     "tumblr": [{
         "name": "at.tumblr link",