From 332eae16b276bc8afc70c83c5783533395dbb1db Mon Sep 17 00:00:00 2001 From: wukko Date: Sat, 3 Aug 2024 16:27:20 +0600 Subject: [PATCH] api: convert service config to JS and remove it from main config --- api/src/config.js | 14 -- api/src/processing/match-action.js | 4 +- api/src/processing/service-config.js | 197 +++++++++++++++++++++++++ api/src/processing/service-config.json | 142 ------------------ api/src/processing/url.js | 2 +- api/src/stream/types.js | 3 +- api/src/util/test-ci.js | 3 +- api/src/util/test.js | 2 +- 8 files changed, 206 insertions(+), 161 deletions(-) create mode 100644 api/src/processing/service-config.js delete mode 100644 api/src/processing/service-config.json diff --git a/api/src/config.js b/api/src/config.js index 0f9bf07d..32b65d7a 100644 --- a/api/src/config.js +++ b/api/src/config.js @@ -1,16 +1,6 @@ -import UrlPattern from "url-pattern"; import { loadJSON } from "./misc/load-from-fs.js"; const packageJson = loadJSON("./package.json"); -const servicesConfigJson = loadJSON("./src/processing/service-config.json"); - -Object.values(servicesConfigJson.config).forEach(service => { - service.patterns = service.patterns.map( - pattern => new UrlPattern(pattern, { - segmentValueCharset: UrlPattern.defaultOptions.segmentValueCharset + '@\\.' - }) - ) -}) export const version = packageJson.version, @@ -26,10 +16,6 @@ export const gif: ["-vf", "scale=-1:-1:flags=lanczos,split[s0][s1];[s0]palettegen[p];[s1][p]paletteuse", "-loop", "0"] }, - services = servicesConfigJson.config, - hlsExceptions = servicesConfigJson.hlsExceptions, - audioIgnore = servicesConfigJson.audioIgnore, - env = { apiURL: process.env.API_URL || '', apiPort: process.env.API_PORT || 9000, diff --git a/api/src/processing/match-action.js b/api/src/processing/match-action.js index d6ec98ca..80f63b41 100644 --- a/api/src/processing/match-action.js +++ b/api/src/processing/match-action.js @@ -1,4 +1,6 @@ -import { audioIgnore, services, supportedAudio } from "../config.js"; +import { supportedAudio } from "../config.js"; +import { audioIgnore, services } from "./service-config.js"; + import { createResponse } from "./request.js"; import createFilename from "./create-filename.js"; import { createStream } from "../stream/manage.js"; diff --git a/api/src/processing/service-config.js b/api/src/processing/service-config.js new file mode 100644 index 00000000..68164ace --- /dev/null +++ b/api/src/processing/service-config.js @@ -0,0 +1,197 @@ +import UrlPattern from "url-pattern"; + +export const audioIgnore = ["vk", "ok", "loom"]; +export const hlsExceptions = ["dailymotion", "vimeo", "rutube"]; + +export const services = { + bilibili: { + enabled: true, + patterns: [ + "video/:comId", + "_shortLink/:comShortLink", + "_tv/:lang/video/:tvId", + "_tv/video/:tvId" + ], + subdomains: ["m"], + }, + dailymotion: { + enabled: true, + patterns: ["video/:id"], + }, + facebook: { + enabled: true, + patterns: [ + "_shortLink/:shortLink", + ":username/videos/:caption/:id", + ":username/videos/:id", + "reel/:id", + "share/:shareType/:id" + ], + subdomains: ["web"], + altDomains: ["fb.watch"], + }, + instagram: { + enabled: true, + patterns: [ + "reels/:postId", + ":username/reel/:postId", + "reel/:postId", + "p/:postId", + ":username/p/:postId", + "tv/:postId", + "stories/:username/:storyId" + ], + altDomains: ["ddinstagram.com"], + }, + loom: { + enabled: true, + patterns: ["share/:id"], + }, + ok: { + enabled: true, + patterns: [ + "video/:id", + "videoembed/:id" + ], + tld: "ru", + }, + pinterest: { + enabled: true, + patterns: [ + "pin/:id", + "pin/:id/:garbage", + "url_shortener/:shortLink" + ], + }, + reddit: { + enabled: true, + patterns: [ + "r/:sub/comments/:id/:title", + "user/:user/comments/:id/:title" + ], + subdomains: "*", + }, + rutube: { + enabled: true, + patterns: [ + "video/:id", + "play/embed/:id", + "shorts/:id", + "yappy/:yappyId", + "video/private/:id?p=:key", + "video/private/:id" + ], + tld: "ru", + }, + snapchat: { + enabled: true, + patterns: [ + ":shortLink", + "spotlight/:spotlightId", + "add/:username/:storyId", + "u/:username/:storyId", + "add/:username", + "u/:username" + ], + subdomains: ["t", "story"], + }, + soundcloud: { + enabled: true, + patterns: [ + ":author/:song/s-:accessKey", + ":author/:song", + ":shortLink" + ], + subdomains: ["on", "m"], + }, + streamable: { + enabled: true, + patterns: [ + ":id", + "o/:id", + "e/:id", + "s/:id" + ], + }, + tiktok: { + enabled: true, + patterns: [ + ":user/video/:postId", + ":id", + "t/:id", + ":user/photo/:postId", + "v/:id.html" + ], + subdomains: ["vt", "vm", "m"], + }, + tumblr: { + enabled: true, + patterns: [ + "post/:id", + "blog/view/:user/:id", + ":user/:id", + ":user/:id/:trackingId" + ], + subdomains: "*", + }, + twitch: { + enabled: true, + patterns: [":channel/clip/:clip"], + tld: "tv", + }, + twitter: { + enabled: true, + patterns: [ + ":user/status/:id", + ":user/status/:id/video/:index", + ":user/status/:id/photo/:index", + ":user/status/:id/mediaviewer", + ":user/status/:id/mediaViewer" + ], + subdomains: ["mobile"], + altDomains: ["x.com", "vxtwitter.com", "fixvx.com"], + }, + vine: { + enabled: true, + patterns: ["v/:id"], + tld: "co", + }, + vimeo: { + enabled: true, + patterns: [ + ":id", + "video/:id", + ":id/:password", + "/channels/:user/:id" + ], + subdomains: ["player"], + bestAudio: "mp3", + }, + vk: { + enabled: true, + patterns: [ + "video:userId_:videoId", + "clip:userId_:videoId", + "clips:duplicate?z=clip:userId_:videoId" + ], + subdomains: ["m"], + }, + youtube: { + enabled: true, + patterns: [ + "watch?v=:id", + "embed/:id", + "watch/:id" + ], + subdomains: ["music", "m"], + bestAudio: "opus", + } +} + +Object.values(services).forEach(service => { + service.patterns = service.patterns.map( + pattern => new UrlPattern(pattern, { + segmentValueCharset: UrlPattern.defaultOptions.segmentValueCharset + '@\\.' + }) + ) +}) diff --git a/api/src/processing/service-config.json b/api/src/processing/service-config.json deleted file mode 100644 index 6955a946..00000000 --- a/api/src/processing/service-config.json +++ /dev/null @@ -1,142 +0,0 @@ -{ - "audioIgnore": ["vk", "ok", "loom"], - "hlsExceptions": ["dailymotion", "vimeo", "rutube"], - "config": { - "bilibili": { - "alias": "bilibili.com & bilibili.tv", - "patterns": [ - "video/:comId", "_shortLink/:comShortLink", - "_tv/:lang/video/:tvId", "_tv/video/:tvId" - ], - "subdomains": ["m"], - "enabled": true - }, - "reddit": { - "alias": "reddit videos & gifs", - "patterns": ["r/:sub/comments/:id/:title", "user/:user/comments/:id/:title"], - "subdomains": "*", - "enabled": true - }, - "twitter": { - "alias": "twitter videos & voice", - "altDomains": ["x.com", "vxtwitter.com", "fixvx.com"], - "subdomains": ["mobile"], - "patterns": [ - ":user/status/:id", - ":user/status/:id/video/:index", - ":user/status/:id/photo/:index", - ":user/status/:id/mediaviewer", - ":user/status/:id/mediaViewer" - ], - "enabled": true - }, - "vk": { - "alias": "vk video & clips", - "patterns": ["video:userId_:videoId", "clip:userId_:videoId", "clips:duplicate?z=clip:userId_:videoId"], - "subdomains": ["m"], - "enabled": true - }, - "ok": { - "alias": "ok video", - "tld": "ru", - "patterns": ["video/:id", "videoembed/:id"], - "enabled": true - }, - "youtube": { - "alias": "youtube videos, shorts & music", - "patterns": ["watch?v=:id", "embed/:id", "watch/:id"], - "subdomains": ["music", "m"], - "bestAudio": "opus", - "enabled": true - }, - "tumblr": { - "alias": "tumblr video & audio", - "patterns": ["post/:id", "blog/view/:user/:id", ":user/:id", ":user/:id/:trackingId"], - "subdomains": "*", - "enabled": true - }, - "tiktok": { - "alias": "tiktok videos, photos & audio", - "patterns": [":user/video/:postId", ":id", "t/:id", ":user/photo/:postId", "v/:id.html"], - "subdomains": ["vt", "vm", "m"], - "enabled": true - }, - "vimeo": { - "patterns": [":id", "video/:id", ":id/:password", "/channels/:user/:id"], - "subdomains": ["player"], - "bestAudio": "mp3", - "enabled": true - }, - "soundcloud": { - "patterns": [":author/:song/s-:accessKey", ":author/:song", ":shortLink"], - "subdomains": ["on", "m"], - "enabled": true - }, - "instagram": { - "alias": "instagram posts & reels", - "altDomains": ["ddinstagram.com"], - "patterns": [ - "reels/:postId", ":username/reel/:postId", "reel/:postId", "p/:postId", ":username/p/:postId", - "tv/:postId", "stories/:username/:storyId" - ], - "enabled": true - }, - "vine": { - "alias": "vine archive", - "tld": "co", - "patterns": ["v/:id"], - "enabled": true - }, - "pinterest": { - "alias": "pinterest (all media)", - "patterns": ["pin/:id", "pin/:id/:garbage", "url_shortener/:shortLink"], - "enabled": true - }, - "streamable": { - "alias": "streamable.com", - "patterns": [":id", "o/:id", "e/:id", "s/:id"], - "enabled": true - }, - "twitch": { - "alias": "twitch clips", - "tld": "tv", - "patterns": [":channel/clip/:clip"], - "enabled": true - }, - "rutube": { - "alias": "rutube videos", - "tld": "ru", - "patterns": ["video/:id", "play/embed/:id", "shorts/:id", "yappy/:yappyId", "video/private/:id?p=:key", "video/private/:id"], - "enabled": true - }, - "dailymotion": { - "alias": "dailymotion videos", - "patterns": ["video/:id"], - "enabled": true - }, - "snapchat": { - "alias": "snapchat stories & spotlights", - "subdomains": ["t", "story"], - "patterns": [":shortLink", "spotlight/:spotlightId", "add/:username/:storyId", "u/:username/:storyId", "add/:username", "u/:username"], - "enabled": true - }, - "loom": { - "alias": "loom videos", - "patterns": ["share/:id"], - "enabled": true - }, - "facebook": { - "alias": "facebook videos", - "altDomains": ["fb.watch"], - "subdomains": ["web"], - "patterns": [ - "_shortLink/:shortLink", - ":username/videos/:caption/:id", - ":username/videos/:id", - "reel/:id", - "share/:shareType/:id" - ], - "enabled": true - } - } -} diff --git a/api/src/processing/url.js b/api/src/processing/url.js index a2b9cd7d..a9313e25 100644 --- a/api/src/processing/url.js +++ b/api/src/processing/url.js @@ -1,4 +1,4 @@ -import { services } from "../config.js"; +import { services } from "./service-config.js"; import { strict as assert } from "node:assert"; import psl from "psl"; diff --git a/api/src/stream/types.js b/api/src/stream/types.js index f566a196..b12e5de8 100644 --- a/api/src/stream/types.js +++ b/api/src/stream/types.js @@ -3,9 +3,10 @@ import ffmpeg from "ffmpeg-static"; import { spawn } from "child_process"; import { create as contentDisposition } from "content-disposition-header"; +import { env, ffmpegArgs } from "../config.js"; import { metadataManager } from "../misc/utils.js"; import { destroyInternalStream } from "./manage.js"; -import { env, ffmpegArgs, hlsExceptions } from "../config.js"; +import { hlsExceptions } from "../processing/service-config.js"; import { getHeaders, closeRequest, closeResponse, pipe } from "./shared.js"; function toRawHeaders(headers) { diff --git a/api/src/util/test-ci.js b/api/src/util/test-ci.js index 6559aadc..1c9a7011 100644 --- a/api/src/util/test-ci.js +++ b/api/src/util/test-ci.js @@ -3,8 +3,9 @@ import { runTest } from "../misc/run-test.js"; import { loadJSON } from "../misc/load-from-fs.js"; import { Red, Bright } from "../misc/console-text.js"; +import { services } from "../processing/service-config.js"; + const tests = loadJSON('./src/util/tests.json'); -const services = loadJSON('./src/processing/service-config.json'); // services that are known to frequently fail due to external // factors (e.g. rate limiting) diff --git a/api/src/util/test.js b/api/src/util/test.js index 2a2c084c..c0097964 100644 --- a/api/src/util/test.js +++ b/api/src/util/test.js @@ -1,7 +1,7 @@ import "dotenv/config"; import "../misc/alias-envs.js"; -import { services } from "../config.js"; +import { services } from "../processing/service-config.js"; import { extract } from "../processing/url.js"; import match from "../processing/match.js"; import { loadJSON } from "../misc/load-from-fs.js";